screencasting: Minimise pixel format conversions while streaming

Instead of best-guessing, at BGR (which in retrospect was a bad guess),
offer whatever resembles most the internal representation. This way the
frame gets to be least treated as it goes into the client.

src/backends/drm/drm_egl_cursor_layer.cpp

@@ -7,6 +7,7 @@
     SPDX-License-Identifier: GPL-2.0-or-later
 */
 #include "drm_egl_cursor_layer.h"
+#include "drm_buffer.h"
 #include "drm_egl_backend.h"
 #include "drm_gpu.h"
 #include "drm_pipeline.h"
@@ -57,4 +58,8 @@ void EglGbmCursorLayer::releaseBuffers()
     m_surface.destroyResources();
 }
 
+quint32 EglGbmCursorLayer::format() const
+{
+    return m_surface.currentBuffer()->buffer()->format();
+}
 }

src/backends/drm/drm_egl_cursor_layer.h

@@ -36,6 +36,7 @@ public:
     QRegion currentDamage() const override;
     bool checkTestBuffer() override;
     void releaseBuffers() override;
+    quint32 format() const override;
 
 private:
     EglGbmLayerSurface m_surface;

src/backends/drm/drm_layer.cpp

@@ -10,6 +10,7 @@
 #include "drm_pipeline.h"
 
 #include <QMatrix4x4>
+#include <drm_fourcc.h>
 
 namespace KWin
 {
@@ -21,6 +22,11 @@ QRegion DrmOutputLayer::currentDamage() const
     return {};
 }
 
+quint32 DrmOutputLayer::format() const
+{
+    return DRM_FORMAT_RGBA8888;
+}
+
 std::shared_ptr<GLTexture> DrmOutputLayer::texture() const
 {
     return nullptr;

src/backends/drm/drm_layer.h

@@ -29,6 +29,7 @@ public:
     virtual std::shared_ptr<GLTexture> texture() const;
     virtual QRegion currentDamage() const;
     virtual void releaseBuffers() = 0;
+    quint32 format() const override;
 };
 
 class DrmPipelineLayer : public DrmOutputLayer

src/backends/drm/drm_qpainter_layer.cpp

@@ -92,6 +92,11 @@ void DrmQPainterLayer::releaseBuffers()
     m_swapchain.reset();
 }
 
+quint32 DrmQPainterLayer::format() const
+{
+    return DRM_FORMAT_XRGB8888;
+}
+
 DrmCursorQPainterLayer::DrmCursorQPainterLayer(DrmPipeline *pipeline)
     : DrmOverlayLayer(pipeline)
 {

src/backends/drm/drm_qpainter_layer.h

@@ -32,6 +32,7 @@ public:
     std::shared_ptr<DrmFramebuffer> currentBuffer() const override;
     QRegion currentDamage() const override;
     void releaseBuffers() override;
+    quint32 format() const override;
 
 private:
     bool doesSwapchainFit() const;

src/backends/drm/drm_virtual_egl_layer.cpp

@@ -175,4 +175,9 @@ void VirtualEglGbmLayer::releaseBuffers()
     m_gbmSurface.reset();
     m_oldGbmSurface.reset();
 }
+
+quint32 VirtualEglGbmLayer::format() const
+{
+    return m_gbmSurface->format();
+}
 }

src/backends/drm/drm_virtual_egl_layer.h

@@ -42,6 +42,7 @@ public:
     QRegion currentDamage() const override;
     std::shared_ptr<GLTexture> texture() const override;
     void releaseBuffers() override;
+    quint32 format() const override;
 
 private:
     bool createGbmSurface();

src/backends/virtual/virtual_egl_backend.cpp

@@ -15,6 +15,7 @@
 #include "virtual_backend.h"
 #include "virtual_output.h"
 // kwin libs
+#include <drm_fourcc.h>
 #include <kwinglutils.h>
 
 #ifndef EGL_PLATFORM_SURFACELESS_MESA
@@ -61,6 +62,12 @@ bool VirtualEglLayer::endFrame(const QRegion &renderedRegion, const QRegion &dam
     return true;
 }
 
+quint32 VirtualEglLayer::format() const
+{
+    // the texture format is hardcoded in VirtualEglLayer::beginFrame
+    return DRM_FORMAT_RGB888;
+}
+
 VirtualEglBackend::VirtualEglBackend(VirtualBackend *b)
     : AbstractEglBackend()
     , m_backend(b)

src/backends/virtual/virtual_egl_backend.h

@@ -29,6 +29,7 @@ public:
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
 
     std::shared_ptr<GLTexture> texture() const;
+    quint32 format() const override;
 
 private:
     VirtualEglBackend *const m_backend;

src/backends/virtual/virtual_qpainter_backend.cpp

@@ -12,6 +12,7 @@
 #include "virtual_output.h"
 
 #include <QPainter>
+#include <drm_fourcc.h>
 
 namespace KWin
 {
@@ -41,6 +42,11 @@ QImage *VirtualQPainterLayer::image()
     return &m_image;
 }
 
+quint32 VirtualQPainterLayer::format() const
+{
+    return DRM_FORMAT_RGBX8888;
+}
+
 VirtualQPainterBackend::VirtualQPainterBackend(VirtualBackend *backend)
 {
     connect(backend, &VirtualBackend::outputAdded, this, &VirtualQPainterBackend::addOutput);

src/backends/virtual/virtual_qpainter_backend.h

@@ -29,6 +29,7 @@ public:
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
     QImage *image();
+    quint32 format() const override;
 
 private:
     Output *const m_output;

src/backends/wayland/wayland_egl_backend.cpp

@@ -125,6 +125,11 @@ int WaylandEglLayerBuffer::age() const
     return m_age;
 }
 
+gbm_bo *WaylandEglLayerBuffer::bo() const
+{
+    return m_bo;
+}
+
 WaylandEglLayerSwapchain::WaylandEglLayerSwapchain(const QSize &size, uint32_t format, const QVector<uint64_t> &modifiers, WaylandEglBackend *backend)
     : m_backend(backend)
     , m_size(size)
@@ -284,6 +289,16 @@ bool WaylandEglCursorLayer::endFrame(const QRegion &renderedRegion, const QRegio
     return true;
 }
 
+quint32 WaylandEglCursorLayer::format() const
+{
+    return gbm_bo_get_format(m_buffer->bo());
+}
+
+quint32 WaylandEglPrimaryLayer::format() const
+{
+    return gbm_bo_get_format(m_buffer->bo());
+}
+
 WaylandEglBackend::WaylandEglBackend(WaylandBackend *b)
     : AbstractEglBackend()
     , m_backend(b)

src/backends/wayland/wayland_egl_backend.h

@@ -42,6 +42,7 @@ public:
     GLFramebuffer *framebuffer() const;
     std::shared_ptr<GLTexture> texture() const;
     int age() const;
+    gbm_bo *bo() const;
 
 private:
     WaylandEglBackend *m_backend;
@@ -83,6 +84,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    quint32 format() const override;
 
 private:
     WaylandOutput *m_waylandOutput;
@@ -104,6 +106,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    quint32 format() const override;
 
 private:
     WaylandOutput *m_output;

src/backends/wayland/wayland_qpainter_backend.cpp

@@ -18,6 +18,7 @@
 #include <KWayland/Client/surface.h>
 
 #include <cmath>
+#include <drm_fourcc.h>
 
 namespace KWin
 {
@@ -129,6 +130,11 @@ bool WaylandQPainterPrimaryLayer::endFrame(const QRegion &renderedRegion, const
     return true;
 }
 
+quint32 WaylandQPainterPrimaryLayer::format() const
+{
+    return DRM_FORMAT_RGBA8888;
+}
+
 WaylandQPainterCursorLayer::WaylandQPainterCursorLayer(WaylandOutput *output)
     : m_output(output)
 {
@@ -159,6 +165,11 @@ bool WaylandQPainterCursorLayer::endFrame(const QRegion &renderedRegion, const Q
     return true;
 }
 
+quint32 WaylandQPainterCursorLayer::format() const
+{
+    return DRM_FORMAT_RGBA8888;
+}
+
 WaylandQPainterBackend::WaylandQPainterBackend(Wayland::WaylandBackend *b)
     : QPainterBackend()
     , m_backend(b)

src/backends/wayland/wayland_qpainter_backend.h

@@ -56,6 +56,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    quint32 format() const override;
 
     void remapBuffer();
 
@@ -88,6 +89,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    quint32 format() const override;
 
 private:
     WaylandOutput *m_output;

src/backends/x11/standalone/x11_standalone_egl_backend.cpp

@@ -23,6 +23,7 @@
 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
 #include <QtPlatformHeaders/QEGLNativeContext>
 #endif
+#include <drm_fourcc.h>
 
 namespace KWin
 {
@@ -43,6 +44,11 @@ bool EglLayer::endFrame(const QRegion &renderedRegion, const QRegion &damagedReg
     return true;
 }
 
+uint EglLayer::format() const
+{
+    return DRM_FORMAT_RGBA8888;
+}
+
 EglBackend::EglBackend(Display *display, X11StandaloneBackend *backend)
     : EglOnXBackend(kwinApp()->x11Connection(), display, kwinApp()->x11RootWindow())
     , m_backend(backend)
@@ -162,7 +168,6 @@ OutputLayerBeginFrameInfo EglBackend::beginFrame()
     if (supportsBufferAge()) {
         repaint = m_damageJournal.accumulate(m_bufferAge, infiniteRegion());
     }
-
     eglWaitNative(EGL_CORE_NATIVE_ENGINE);
 
     return OutputLayerBeginFrameInfo{

src/backends/x11/standalone/x11_standalone_egl_backend.h

@@ -29,6 +29,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    uint format() const override;
 
 private:
     EglBackend *const m_backend;

src/backends/x11/standalone/x11_standalone_glx_backend.cpp

@@ -74,6 +74,7 @@ typedef struct xcb_glx_buffer_swap_complete_event_t
 } xcb_glx_buffer_swap_complete_event_t;
 #endif
 
+#include <drm_fourcc.h>
 #include <tuple>
 
 namespace KWin
@@ -120,6 +121,11 @@ bool GlxLayer::endFrame(const QRegion &renderedRegion, const QRegion &damagedReg
     return true;
 }
 
+uint GlxLayer::format() const
+{
+    return DRM_FORMAT_RGBA8888;
+}
+
 GlxBackend::GlxBackend(Display *display, X11StandaloneBackend *backend)
     : OpenGLBackend()
     , m_overlayWindow(std::make_unique<OverlayWindowX11>())

src/backends/x11/standalone/x11_standalone_glx_backend.h

@@ -65,6 +65,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    uint format() const override;
 
 private:
     GlxBackend *const m_backend;

src/backends/x11/windowed/x11_windowed_egl_backend.cpp

@@ -13,6 +13,7 @@
 #include "x11_windowed_backend.h"
 #include "x11_windowed_output.h"
 // kwin libs
+#include <drm_fourcc.h>
 #include <kwinglplatform.h>
 
 namespace KWin
@@ -72,6 +73,11 @@ GLFramebuffer *X11WindowedEglPrimaryLayer::fbo() const
     return m_fbo.get();
 }
 
+quint32 X11WindowedEglPrimaryLayer::format() const
+{
+    return DRM_FORMAT_RGBA8888;
+}
+
 X11WindowedEglCursorLayer::X11WindowedEglCursorLayer(X11WindowedEglBackend *backend, X11WindowedOutput *output)
     : m_output(output)
     , m_backend(backend)
@@ -115,6 +121,11 @@ bool X11WindowedEglCursorLayer::endFrame(const QRegion &renderedRegion, const QR
     return true;
 }
 
+quint32 X11WindowedEglCursorLayer::format() const
+{
+    return DRM_FORMAT_RGBA8888;
+}
+
 X11WindowedEglBackend::X11WindowedEglBackend(X11WindowedBackend *backend)
     : EglOnXBackend(backend->connection(), backend->display(), backend->rootWindow())
     , m_backend(backend)

src/backends/x11/windowed/x11_windowed_egl_backend.h

@@ -29,6 +29,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    quint32 format() const override;
     EGLSurface surface() const;
     QRegion lastDamage() const;
     GLFramebuffer *fbo() const;
@@ -54,6 +55,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    quint32 format() const override;
 
 private:
     X11WindowedOutput *const m_output;

src/backends/x11/windowed/x11_windowed_qpainter_backend.cpp

@@ -13,6 +13,7 @@
 
 #include <cerrno>
 #include <cmath>
+#include <drm_fourcc.h>
 #include <string.h>
 #include <sys/shm.h>
 #include <xcb/present.h>
@@ -173,6 +174,17 @@ void X11WindowedQPainterPrimaryLayer::present()
     m_swapchain->release(m_buffer);
 }
 
+quint32 X11WindowedQPainterPrimaryLayer::format() const
+{
+    switch (m_buffer->view()->format()) {
+    case QImage::Format_A2RGB30_Premultiplied:
+        return DRM_FORMAT_ARGB2101010;
+    case QImage::Format_ARGB32_Premultiplied:
+    default:
+        return DRM_FORMAT_ARGB8888;
+    }
+}
+
 X11WindowedQPainterCursorLayer::X11WindowedQPainterCursorLayer(X11WindowedOutput *output)
     : m_output(output)
 {
@@ -192,6 +204,11 @@ std::optional<OutputLayerBeginFrameInfo> X11WindowedQPainterCursorLayer::beginFr
     };
 }
 
+quint32 X11WindowedQPainterCursorLayer::format() const
+{
+    return DRM_FORMAT_ARGB8888;
+}
+
 bool X11WindowedQPainterCursorLayer::endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion)
 {
     m_output->cursor()->update(m_buffer, hotspot());

src/backends/x11/windowed/x11_windowed_qpainter_backend.h

@@ -66,6 +66,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    quint32 format() const override;
 
     void present();
 
@@ -84,6 +85,7 @@ public:
 
     std::optional<OutputLayerBeginFrameInfo> beginFrame() override;
     bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
+    quint32 format() const override;
 
 private:
     QImage m_buffer;

src/composite.cpp

@@ -638,6 +638,12 @@ void Compositor::handleFrameRequested(RenderLoop *renderLoop)
     composite(renderLoop);
 }
 
+uint Compositor::outputFormat(Output *output)
+{
+    OutputLayer *primaryLayer = m_backend->primaryLayer(output);
+    return primaryLayer->format();
+}
+
 void Compositor::composite(RenderLoop *renderLoop)
 {
     if (m_backend->checkGraphicsReset()) {

src/composite.h

@@ -139,6 +139,13 @@ public:
      */
     virtual void createOpenGLSafePoint(OpenGLSafePoint safePoint);
 
+    /**
+     * @returns the format of the contents in the @p output
+     *
+     * This format is provided using the drm fourcc encoding
+     */
+    uint outputFormat(Output *output);
+
 Q_SIGNALS:
     void compositingToggled(bool active);
     void aboutToDestroy();

src/core/outputlayer.h

@@ -53,6 +53,11 @@ public:
     virtual std::optional<OutputLayerBeginFrameInfo> beginFrame() = 0;
     virtual bool endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) = 0;
 
+    /**
+     * Format in which the output data is internally stored in a drm fourcc format
+     */
+    virtual quint32 format() const = 0;
+
     /**
      * Tries to import the newest buffer of the surface for direct scanout
      * Returns @c true if scanout succeeds, @c false if rendering is necessary

src/plugins/screencast/outputscreencastsource.cpp

@@ -29,6 +29,11 @@ bool OutputScreenCastSource::hasAlphaChannel() const
     return true;
 }
 
+quint32 OutputScreenCastSource::drmFormat() const
+{
+    return Compositor::self()->outputFormat(m_output);
+}
+
 QSize OutputScreenCastSource::textureSize() const
 {
     return m_output->pixelSize();

src/plugins/screencast/outputscreencastsource.h

@@ -24,6 +24,7 @@ public:
 
     bool hasAlphaChannel() const override;
     QSize textureSize() const override;
+    quint32 drmFormat() const override;
 
     void render(GLFramebuffer *target) override;
     void render(QImage *image) override;

src/plugins/screencast/regionscreencastsource.cpp

@@ -9,6 +9,7 @@
 
 #include <composite.h>
 #include <core/output.h>
+#include <drm_fourcc.h>
 #include <kwingltexture.h>
 #include <kwinglutils.h>
 #include <scene/workspacescene.h>
@@ -38,6 +39,11 @@ bool RegionScreenCastSource::hasAlphaChannel() const
     return true;
 }
 
+quint32 RegionScreenCastSource::drmFormat() const
+{
+    return DRM_FORMAT_ARGB8888;
+}
+
 void RegionScreenCastSource::updateOutput(Output *output)
 {
     m_last = output->renderLoop()->lastPresentationTimestamp();
@@ -73,7 +79,7 @@ std::chrono::nanoseconds RegionScreenCastSource::clock() const
 void RegionScreenCastSource::ensureTexture()
 {
     if (!m_renderedTexture) {
-        m_renderedTexture.reset(new GLTexture(hasAlphaChannel() ? GL_RGBA8 : GL_RGB8, textureSize()));
+        m_renderedTexture.reset(new GLTexture(GL_RGBA8, textureSize()));
         m_target.reset(new GLFramebuffer(m_renderedTexture.get()));
         const auto allOutputs = workspace()->outputs();
         for (auto output : allOutputs) {

src/plugins/screencast/regionscreencastsource.h

@@ -23,6 +23,7 @@ class RegionScreenCastSource : public ScreenCastSource
 public:
     explicit RegionScreenCastSource(const QRect &region, qreal scale, QObject *parent = nullptr);
 
+    quint32 drmFormat() const override;
     bool hasAlphaChannel() const override;
     QSize textureSize() const override;
 

src/plugins/screencast/screencastsource.h

@@ -21,6 +21,7 @@ public:
     explicit ScreenCastSource(QObject *parent = nullptr);
 
     virtual bool hasAlphaChannel() const = 0;
+    virtual quint32 drmFormat() const = 0;
     virtual QSize textureSize() const = 0;
 
     virtual void render(GLFramebuffer *target) = 0;

src/plugins/screencast/screencaststream.cpp

@@ -41,24 +41,30 @@
 namespace KWin
 {
 
-uint32_t spaVideoFormatToDrmFormat(spa_video_format spa_format)
+static spa_video_format drmFourCCToSpaVideoFormat(quint32 format)
 {
-    switch (spa_format) {
-    case SPA_VIDEO_FORMAT_RGBA:
-        return DRM_FORMAT_ABGR8888;
-    case SPA_VIDEO_FORMAT_RGBx:
-        return DRM_FORMAT_XBGR8888;
-    case SPA_VIDEO_FORMAT_BGRA:
-        return DRM_FORMAT_ARGB8888;
-    case SPA_VIDEO_FORMAT_BGRx:
-        return DRM_FORMAT_XRGB8888;
-    case SPA_VIDEO_FORMAT_BGR:
-        return DRM_FORMAT_BGR888;
-    case SPA_VIDEO_FORMAT_RGB:
-        return DRM_FORMAT_RGB888;
+    switch (format) {
+    case DRM_FORMAT_ARGB8888:
+        return SPA_VIDEO_FORMAT_BGRA;
+    case DRM_FORMAT_XRGB8888:
+        return SPA_VIDEO_FORMAT_BGRx;
+    case DRM_FORMAT_RGBA8888:
+        return SPA_VIDEO_FORMAT_ABGR;
+    case DRM_FORMAT_RGBX8888:
+        return SPA_VIDEO_FORMAT_xBGR;
+    case DRM_FORMAT_ABGR8888:
+        return SPA_VIDEO_FORMAT_RGBA;
+    case DRM_FORMAT_XBGR8888:
+        return SPA_VIDEO_FORMAT_RGBx;
+    case DRM_FORMAT_BGRA8888:
+        return SPA_VIDEO_FORMAT_ARGB;
+    case DRM_FORMAT_BGRX8888:
+        return SPA_VIDEO_FORMAT_xRGB;
+    case DRM_FORMAT_NV12:
+        return SPA_VIDEO_FORMAT_NV12;
     default:
-        qCDebug(KWIN_SCREENCAST) << "unknown format" << spa_format;
-        return DRM_FORMAT_INVALID;
+        qCDebug(KWIN_SCREENCAST) << "unknown format" << format;
+        return SPA_VIDEO_FORMAT_xRGB;
     }
 }
 
@@ -158,9 +164,9 @@ void ScreenCastStream::onStreamParamChanged(void *data, uint32_t id, const struc
     }
     if (modifierProperty && (!pw->m_dmabufParams || !receivedModifiers.contains(pw->m_dmabufParams->modifier))) {
         if (modifierProperty->flags & SPA_POD_PROP_FLAG_DONT_FIXATE) {
-            pw->m_dmabufParams = kwinApp()->outputBackend()->testCreateDmaBuf(pw->m_resolution, spaVideoFormatToDrmFormat(pw->videoFormat.format), receivedModifiers);
+            pw->m_dmabufParams = kwinApp()->outputBackend()->testCreateDmaBuf(pw->m_resolution, pw->m_drmFormat, receivedModifiers);
         } else {
-            pw->m_dmabufParams = kwinApp()->outputBackend()->testCreateDmaBuf(pw->m_resolution, spaVideoFormatToDrmFormat(pw->videoFormat.format), {DRM_FORMAT_MOD_INVALID});
+            pw->m_dmabufParams = kwinApp()->outputBackend()->testCreateDmaBuf(pw->m_resolution, pw->m_drmFormat, {DRM_FORMAT_MOD_INVALID});
         }
 
         qCDebug(KWIN_SCREENCAST) << "Stream dmabuf modifiers received, offering our best suited modifier" << pw->m_dmabufParams.has_value();
@@ -333,11 +339,25 @@ bool ScreenCastStream::createStream()
     const QByteArray objname = "kwin-screencast-" + objectName().toUtf8();
     pwStream = pw_stream_new(pwCore->pwCore, objname, nullptr);
 
-    // it could make sense to offer the same format as the source
-    const auto format = m_source->hasAlphaChannel() ? SPA_VIDEO_FORMAT_BGRA : SPA_VIDEO_FORMAT_BGR;
-    const int drmFormat = spaVideoFormatToDrmFormat(format);
-    m_hasDmaBuf = kwinApp()->outputBackend()->testCreateDmaBuf(m_resolution, drmFormat, {DRM_FORMAT_MOD_INVALID}).has_value();
-    m_modifiers = Compositor::self()->backend()->supportedFormats().value(drmFormat);
+    const auto supported = Compositor::self()->backend()->supportedFormats();
+    auto itModifiers = supported.constFind(m_source->drmFormat());
+
+    // If the offered format is not available for dmabuf, prefer converting to another one than resorting to memfd
+    if (itModifiers == supported.constEnd() && !supported.isEmpty()) {
+        itModifiers = supported.constFind(DRM_FORMAT_ARGB8888);
+        if (itModifiers == supported.constEnd()) {
+            m_drmFormat = itModifiers.key();
+        }
+    }
+
+    if (itModifiers == supported.constEnd()) {
+        m_drmFormat = m_source->drmFormat();
+        m_modifiers = {};
+    } else {
+        m_drmFormat = itModifiers.key();
+        m_modifiers = *itModifiers;
+    }
+    m_hasDmaBuf = kwinApp()->outputBackend()->testCreateDmaBuf(m_resolution, m_drmFormat, {DRM_FORMAT_MOD_INVALID}).has_value();
 
     char buffer[2048];
     QVector<const spa_pod *> params = buildFormats(false, buffer);
@@ -633,7 +653,7 @@ void ScreenCastStream::enqueue()
 
 QVector<const spa_pod *> ScreenCastStream::buildFormats(bool fixate, char buffer[2048])
 {
-    const auto format = m_source->hasAlphaChannel() ? SPA_VIDEO_FORMAT_BGRA : SPA_VIDEO_FORMAT_BGR;
+    const auto format = drmFourCCToSpaVideoFormat(m_drmFormat);
     spa_pod_builder podBuilder = SPA_POD_BUILDER_INIT(buffer, 2048);
     spa_fraction minFramerate = SPA_FRACTION(1, 1);
     spa_fraction maxFramerate = SPA_FRACTION(25, 1);
@@ -672,6 +692,9 @@ spa_pod *ScreenCastStream::buildFormat(struct spa_pod_builder *b, enum spa_video
     if (format == SPA_VIDEO_FORMAT_BGRA) {
         /* announce equivalent format without alpha */
         spa_pod_builder_add(b, SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(3, format, format, SPA_VIDEO_FORMAT_BGRx), 0);
+    } else if (format == SPA_VIDEO_FORMAT_RGBA) {
+        /* announce equivalent format without alpha */
+        spa_pod_builder_add(b, SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(3, format, format, SPA_VIDEO_FORMAT_RGBx), 0);
     } else {
         spa_pod_builder_add(b, SPA_FORMAT_VIDEO_format, SPA_POD_Id(format), 0);
     }

src/plugins/screencast/screencaststream.h

@@ -129,6 +129,7 @@ private:
     quint64 m_sequential = 0;
     bool m_hasDmaBuf = false;
     bool m_waitForNewBuffers = false;
+    quint32 m_drmFormat = 0;
 };
 
 } // namespace KWin

src/plugins/screencast/screencastutils.h

@@ -25,6 +25,24 @@ static void mirrorVertically(uchar *data, int height, int stride)
     }
 }
 
+static GLenum closestGLType(const QImage &image)
+{
+    switch (image.format()) {
+    case QImage::Format_RGB888:
+        return GL_RGB;
+    case QImage::Format_BGR888:
+        return GL_BGR;
+    case QImage::Format_RGB32:
+    case QImage::Format_RGBX8888:
+    case QImage::Format_RGBA8888:
+    case QImage::Format_RGBA8888_Premultiplied:
+        return GL_RGBA;
+    default:
+        qDebug() << "unknown format" << image.format();
+        return GL_RGBA;
+    }
+}
+
 static void grabTexture(GLTexture *texture, QImage *image)
 {
     const bool invert = !texture->isYInverted();
@@ -40,11 +58,11 @@ static void grabTexture(GLTexture *texture, QImage *image)
 
     texture->bind();
     if (GLPlatform::instance()->isGLES()) {
-        glReadPixels(0, 0, image->width(), image->height(), image->hasAlphaChannel() ? GL_BGRA : GL_BGR, GL_UNSIGNED_BYTE, (GLvoid *)image->bits());
+        glReadPixels(0, 0, image->width(), image->height(), closestGLType(*image), GL_UNSIGNED_BYTE, (GLvoid *)image->bits());
     } else if (GLPlatform::instance()->glVersion() >= kVersionNumber(4, 5)) {
-        glGetTextureImage(texture->texture(), 0, image->hasAlphaChannel() ? GL_BGRA : GL_BGR, GL_UNSIGNED_BYTE, image->sizeInBytes(), image->bits());
+        glGetTextureImage(texture->texture(), 0, closestGLType(*image), GL_UNSIGNED_BYTE, image->sizeInBytes(), image->bits());
     } else {
-        glGetTexImage(texture->target(), 0, image->hasAlphaChannel() ? GL_BGRA : GL_BGR, GL_UNSIGNED_BYTE, image->bits());
+        glGetTexImage(texture->target(), 0, closestGLType(*image), GL_UNSIGNED_BYTE, image->bits());
     }
 
     if (invertNeededAndSupported) {

src/plugins/screencast/windowscreencastsource.cpp

@@ -18,6 +18,7 @@
 #include "scene/itemrenderer.h"
 #include "scene/windowitem.h"
 #include "scene/workspacescene.h"
+#include <drm_fourcc.h>
 
 namespace KWin
 {
@@ -30,6 +31,11 @@ WindowScreenCastSource::WindowScreenCastSource(Window *window, QObject *parent)
     connect(m_window, &Window::windowClosed, this, &ScreenCastSource::closed);
 }
 
+quint32 WindowScreenCastSource::drmFormat() const
+{
+    return DRM_FORMAT_RGBA8888;
+}
+
 bool WindowScreenCastSource::hasAlphaChannel() const
 {
     return true;

src/plugins/screencast/windowscreencastsource.h

@@ -22,6 +22,7 @@ class WindowScreenCastSource : public ScreenCastSource
 public:
     explicit WindowScreenCastSource(Window *window, QObject *parent = nullptr);
 
+    quint32 drmFormat() const override;
     bool hasAlphaChannel() const override;
     QSize textureSize() const override;