[scene-xrender] Do not render deco part if it doesn't end on screen

If the deco part doesn't exist or the target rect is empty, KWin should
not try to composite the deco part on the screen. It can be an empty
rect for maximized windows. If the render composite is tried it results
in an error (note - man page says "This request does never generate any
errors.").

To simplify the macro is turned into a lambda.

The root cause for this problem is that PaintRedirector creates an
XRenderPicture for those invalid geometries and this requests arleady
fails. Thus RasterXRenderPaintRedirector is adjusted to not create the
XRenderPicture in that case and set it to nullptr. For a null
XRenderPicture XCB_RENDER_PICTURE_NONE is returned.

BUG: 332247
REVIEW: 117373

paintredirector.cpp

@@ -439,19 +439,32 @@ RasterXRenderPaintRedirector::~RasterXRenderPaintRedirector()
 
 xcb_render_picture_t RasterXRenderPaintRedirector::picture(PaintRedirector::DecorationPixmap border) const
 {
-    return *m_pictures[border];
+    XRenderPicture *picture = m_pictures[border];
+    if (!picture) {
+        return XCB_RENDER_PICTURE_NONE;
+    }
+    return *picture;
 }
 
 void RasterXRenderPaintRedirector::resize(PaintRedirector::DecorationPixmap border, const QSize &size)
 {
     if (m_sizes[border] != size) {
+        m_sizes[border] = size;
         if (m_pixmaps[border] != XCB_PIXMAP_NONE) {
             xcb_free_pixmap(connection(), m_pixmaps[border]);
         }
-        m_pixmaps[border] = xcb_generate_id(connection());
-        xcb_create_pixmap(connection(), 32, m_pixmaps[border], rootWindow(), size.width(), size.height());
         delete m_pictures[border];
-        m_pictures[border] = new XRenderPicture(m_pixmaps[border], 32);
+        if (!size.isEmpty()) {
+            m_pixmaps[border] = xcb_generate_id(connection());
+            xcb_create_pixmap(connection(), 32, m_pixmaps[border], rootWindow(), size.width(), size.height());
+            m_pictures[border] = new XRenderPicture(m_pixmaps[border], 32);
+        } else {
+            m_pixmaps[border] = XCB_PIXMAP_NONE;
+            m_pictures[border] = nullptr;
+        }
+    }
+    if (!m_pictures[border]) {
+        return;
     }
     // fill transparent
     xcb_rectangle_t rect = {0, 0, uint16_t(size.width()), uint16_t(size.height())};

scene_xrender.cpp

@@ -845,23 +845,25 @@ xcb_render_composite(connection(), XCB_RENDER_PICT_OP_OVER, m_xrenderShadow->pic
                 transformed_shape = QRegion();
         }
 
-#define RENDER_DECO_PART(_PART_, _RECT_) \
-xcb_render_composite(connection(), XCB_RENDER_PICT_OP_OVER, _PART_, decorationAlpha, renderTarget,\
-                 0, 0, 0, 0, _RECT_.x(), _RECT_.y(), _RECT_.width(), _RECT_.height())
-
         if (client || deleted) {
             if (!noBorder) {
                 xcb_render_picture_t decorationAlpha = xRenderBlendPicture(data.opacity() * data.decorationOpacity());
-                RENDER_DECO_PART(top, dtr);
-                RENDER_DECO_PART(left, dlr);
-                RENDER_DECO_PART(right, drr);
-                RENDER_DECO_PART(bottom, dbr);
+                auto renderDeco = [decorationAlpha, renderTarget](xcb_render_picture_t deco, const QRect &rect) {
+                    if (deco == XCB_RENDER_PICTURE_NONE) {
+                        return;
+                    }
+                    xcb_render_composite(connection(), XCB_RENDER_PICT_OP_OVER, deco, decorationAlpha, renderTarget,
+                                         0, 0, 0, 0, rect.x(), rect.y(), rect.width(), rect.height());
+                };
+                renderDeco(top, dtr);
+                renderDeco(left, dlr);
+                renderDeco(right, drr);
+                renderDeco(bottom, dbr);
             }
             if (redirector) {
                 redirector->markAsRepainted();
             }
         }
-#undef RENDER_DECO_PART
 
         if (data.brightness() != 1.0) {
             // fake brightness change by overlaying black