backends/drm: ensure hardware transforms are properly applied and tested

Testing only the pipeline that this output uses is not enough and can
cause breakage on multi-monitor setups.

src/backends/drm/drm_gpu.cpp

@@ -335,24 +335,7 @@ bool DrmGpu::checkCrtcAssignment(QVector<DrmConnector*> connectors, QVector<DrmC
             qCWarning(KWIN_DRM) << "disabling connector" << conn->modelName() << "without a crtc";
             conn->pipeline()->pending.crtc = nullptr;
         }
-        // pipelines that are enabled but not active need to be activated for the test
-        QVector<DrmPipeline*> inactivePipelines;
-        for (const auto &pipeline : qAsConst(m_pipelines)) {
-            if (!pipeline->pending.active) {
-                pipeline->pending.active = true;
-                inactivePipelines << pipeline;
-            }
-        }
-        const auto unused = unusedObjects();
-        bool test = DrmPipeline::commitPipelines(m_pipelines, DrmPipeline::CommitMode::Test, unused);
-        // disable inactive pipelines again
-        for (const auto &pipeline : qAsConst(inactivePipelines)) {
-            pipeline->pending.active = false;
-        }
-        if (!inactivePipelines.isEmpty() && test) {
-            test = DrmPipeline::commitPipelines(m_pipelines, DrmPipeline::CommitMode::Test, unused);
-        }
-        return test;
+        return testPipelines();
     }
     auto connector = connectors.takeFirst();
     auto pipeline = connector->pipeline();
@@ -388,7 +371,7 @@ bool DrmGpu::checkCrtcAssignment(QVector<DrmConnector*> connectors, QVector<DrmC
     return false;
 }
 
-bool DrmGpu::testPendingConfiguration()
+bool DrmGpu::testPendingConfiguration(TestMode mode)
 {
     QVector<DrmConnector *> connectors;
     for (const auto &conn : qAsConst(m_connectors)) {
@@ -410,7 +393,33 @@ bool DrmGpu::testPendingConfiguration()
             return c1->getProp(DrmConnector::PropertyIndex::CrtcId)->current() > c2->getProp(DrmConnector::PropertyIndex::CrtcId)->current();
         });
     }
-    return checkCrtcAssignment(connectors, crtcs);
+    if (mode == TestMode::TestWithCrtcReallocation) {
+        return checkCrtcAssignment(connectors, crtcs);
+    } else {
+        return testPipelines();
+    }
+}
+
+bool DrmGpu::testPipelines()
+{
+    // pipelines that are enabled but not active need to be activated for the test
+    QVector<DrmPipeline*> inactivePipelines;
+    for (const auto &pipeline : qAsConst(m_pipelines)) {
+        if (!pipeline->pending.active) {
+            pipeline->pending.active = true;
+            inactivePipelines << pipeline;
+        }
+    }
+    const auto unused = unusedObjects();
+    bool test = DrmPipeline::commitPipelines(m_pipelines, DrmPipeline::CommitMode::Test, unused);
+    // disable inactive pipelines again
+    for (const auto &pipeline : qAsConst(inactivePipelines)) {
+        pipeline->pending.active = false;
+    }
+    if (!inactivePipelines.isEmpty() && test) {
+        test = DrmPipeline::commitPipelines(m_pipelines, DrmPipeline::CommitMode::Test, unused);
+    }
+    return test;
 }
 
 DrmOutput *DrmGpu::findOutput(quint32 connector)

src/backends/drm/drm_gpu.h

@@ -70,7 +70,6 @@ public:
 
     QVector<DrmAbstractOutput*> outputs() const;
     const QVector<DrmPipeline*> pipelines() const;
-    bool testPendingConfiguration();
 
     void setEglDisplay(EGLDisplay display);
     void setEglBackend(EglGbmBackend *eglBackend);
@@ -81,6 +80,11 @@ public:
     DrmVirtualOutput *createVirtualOutput(const QString &name, const QSize &size, double scale, VirtualOutputMode mode);
     void removeVirtualOutput(DrmVirtualOutput *output);
 
+    enum class TestMode {
+        TestOnly,
+        TestWithCrtcReallocation
+    };
+    bool testPendingConfiguration(TestMode mode = TestMode::TestWithCrtcReallocation);
     bool needsModeset() const;
     bool maybeModeset();
 
@@ -100,6 +104,7 @@ private:
     void waitIdle();
 
     bool checkCrtcAssignment(QVector<DrmConnector*> connectors, QVector<DrmCrtc*> crtcs);
+    bool testPipelines();
     QVector<DrmObject*> unusedObjects() const;
 
     void handleLeaseRequest(KWaylandServer::DrmLeaseV1Interface *leaseRequest);

src/backends/drm/drm_output.cpp

@@ -298,15 +298,10 @@ void DrmOutput::updateTransform(Transform transform)
     static int envOnlySoftwareRotations = qEnvironmentVariableIntValue("KWIN_DRM_SW_ROTATIONS_ONLY", &valid) != 0;
     if (valid && !envOnlySoftwareRotations) {
         m_pipeline->pending.transformation = outputToPlaneTransform(transform);
-        if (!DrmPipeline::commitPipelines({m_pipeline}, DrmPipeline::CommitMode::Test)) {
-            // just in case, if we had any rotation before, clear it
-            m_pipeline->pending.transformation = DrmPlane::Transformation::Rotate0;
-            if (DrmPipeline::commitPipelines({m_pipeline}, DrmPipeline::CommitMode::Test)) {
-                m_pipeline->applyPendingChanges();
-            } else {
-                m_pipeline->revertPendingChanges();
-                qCWarning(KWIN_DRM) << "Can't switch rotation back to Rotate0!";
-            }
+        if (m_gpu->testPendingConfiguration(DrmGpu::TestMode::TestOnly)) {
+            m_pipeline->applyPendingChanges();
+        } else {
+            m_pipeline->revertPendingChanges();
         }
     }
 }

src/backends/drm/drm_pipeline.cpp

@@ -235,7 +235,7 @@ void DrmPipeline::prepareAtomicModeset()
     pending.crtc->setPending(DrmCrtc::PropertyIndex::ModeId, activePending() ? mode->blobId() : 0);
 
     pending.crtc->primaryPlane()->setPending(DrmPlane::PropertyIndex::CrtcId, activePending() ? pending.crtc->id() : 0);
-    pending.crtc->primaryPlane()->setTransformation(DrmPlane::Transformation::Rotate0);
+    pending.crtc->primaryPlane()->setTransformation(pending.transformation);
     if (pending.crtc->cursorPlane()) {
         pending.crtc->cursorPlane()->setTransformation(DrmPlane::Transformation::Rotate0);
     }