xcb: Fix bgr888 VNC sessions

Introduce a slow path for 24 and 32 bit BGR. We don't care about
performance here but it has to show the correct colors.

Task-number: QTBUG-42776
Change-Id: Ic73e8ca3950b2b956f06643165dcfac51e7540f3
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@theqtcompany.com>

src/plugins/platforms/xcb/qxcbbackingstore.cpp

@@ -303,13 +303,16 @@ QXcbBackingStore::~QXcbBackingStore()
 
 QPaintDevice *QXcbBackingStore::paintDevice()
 {
-    return m_image ? m_image->image() : 0;
+    if (!m_image)
+        return 0;
+    return m_rgbImage.isNull() ? m_image->image() : &m_rgbImage;
 }
 
 void QXcbBackingStore::beginPaint(const QRegion &region)
 {
     if (!m_image)
         return;
+
     int dpr = int(m_image->image()->devicePixelRatio());
     const int windowDpr = int(window()->devicePixelRatio());
     if (windowDpr != dpr) {
@@ -317,13 +320,13 @@ void QXcbBackingStore::beginPaint(const QRegion &region)
         dpr = int(m_image->image()->devicePixelRatio());
     }
 
-    QRegion xRegion =  dpr == 1 ? region : QTransform::fromScale(dpr,dpr).map(region);
-    m_image->preparePaint(xRegion);
+    m_paintRegion = dpr == 1 ? region : QTransform::fromScale(dpr,dpr).map(region);
+    m_image->preparePaint(m_paintRegion);
 
     if (m_image->image()->hasAlphaChannel()) {
-        QPainter p(m_image->image());
+        QPainter p(paintDevice());
         p.setCompositionMode(QPainter::CompositionMode_Source);
-        const QVector<QRect> rects = xRegion.rects();
+        const QVector<QRect> rects = m_paintRegion.rects();
         const QColor blank = Qt::transparent;
         for (QVector<QRect>::const_iterator it = rects.begin(); it != rects.end(); ++it) {
             p.fillRect(*it, blank);
@@ -331,6 +334,24 @@ void QXcbBackingStore::beginPaint(const QRegion &region)
     }
 }
 
+void QXcbBackingStore::endPaint()
+{
+    QXcbWindow *platformWindow = static_cast<QXcbWindow *>(window()->handle());
+    if (!platformWindow || !platformWindow->imageNeedsRgbSwap())
+        return;
+
+    // Slow path: the paint device was m_rgbImage. Now copy with swapping red
+    // and blue into m_image.
+    const QVector<QRect> rects = m_paintRegion.rects();
+    if (rects.isEmpty())
+        return;
+    QPainter p(m_image->image());
+    for (QVector<QRect>::const_iterator it = rects.begin(); it != rects.end(); ++it) {
+        const QRect rect = *it;
+        p.drawImage(rect.topLeft(), m_rgbImage.copy(rect).rgbSwapped());
+    }
+}
+
 #ifndef QT_NO_OPENGL
 QImage QXcbBackingStore::toImage() const
 {
@@ -426,6 +447,12 @@ void QXcbBackingStore::resize(const QSize &size, const QRegion &)
     delete m_image;
     m_image = new QXcbShmImage(screen, xSize, win->depth(), win->imageFormat());
     m_image->image()->setDevicePixelRatio(dpr);
+    // Slow path for bgr888 VNC: Create an additional image, paint into that and
+    // swap R and B while copying to m_image after each paint.
+    if (win->imageNeedsRgbSwap()) {
+        m_rgbImage = QImage(xSize, win->imageFormat());
+        m_rgbImage.setDevicePixelRatio(dpr);
+    }
     Q_XCB_NOOP(connection());
 }
 

src/plugins/platforms/xcb/qxcbbackingstore.h

@@ -65,9 +65,12 @@ public:
     bool scroll(const QRegion &area, int dx, int dy) Q_DECL_OVERRIDE;
 
     void beginPaint(const QRegion &) Q_DECL_OVERRIDE;
+    void endPaint() Q_DECL_OVERRIDE;
 
 private:
     QXcbShmImage *m_image;
+    QRegion m_paintRegion;
+    QImage m_rgbImage;
 };
 
 QT_END_NAMESPACE

src/plugins/platforms/xcb/qxcbwindow.cpp

@@ -182,8 +182,10 @@ static inline bool isTransient(const QWindow *w)
            || w->type() == Qt::Popup;
 }
 
-static inline QImage::Format imageFormatForVisual(int depth, quint32 red_mask, quint32 blue_mask)
+static inline QImage::Format imageFormatForVisual(int depth, quint32 red_mask, quint32 blue_mask, bool *rgbSwap)
 {
+    if (rgbSwap)
+        *rgbSwap = false;
     switch (depth) {
     case 32:
         if (blue_mask == 0xff)
@@ -192,6 +194,11 @@ static inline QImage::Format imageFormatForVisual(int depth, quint32 red_mask, q
             return QImage::Format_A2BGR30_Premultiplied;
         if (blue_mask == 0x3ff)
             return QImage::Format_A2RGB30_Premultiplied;
+        if (red_mask == 0xff) {
+            if (rgbSwap)
+                *rgbSwap = true;
+            return QImage::Format_ARGB32_Premultiplied;
+        }
         break;
     case 30:
         if (red_mask == 0x3ff)
@@ -202,6 +209,11 @@ static inline QImage::Format imageFormatForVisual(int depth, quint32 red_mask, q
     case 24:
         if (blue_mask == 0xff)
             return QImage::Format_RGB32;
+        if (red_mask == 0xff) {
+            if (rgbSwap)
+                *rgbSwap = true;
+            return QImage::Format_RGB32;
+        }
         break;
     case 16:
         if (blue_mask == 0x1f)
@@ -296,7 +308,7 @@ void QXcbWindow::create()
         m_depth = platformScreen->screen()->root_depth;
         m_visualId = platformScreen->screen()->root_visual;
         const xcb_visualtype_t *visual = platformScreen->visualForId(m_visualId);
-        m_imageFormat = imageFormatForVisual(m_depth, visual->red_mask, visual->blue_mask);
+        m_imageFormat = imageFormatForVisual(m_depth, visual->red_mask, visual->blue_mask, &m_imageRgbSwap);
         connection()->addWindowEventListener(m_window, this);
         return;
     }
@@ -368,7 +380,7 @@ void QXcbWindow::create()
 
         if (visualInfo) {
             m_depth = visualInfo->depth;
-            m_imageFormat = imageFormatForVisual(visualInfo->depth, visualInfo->red_mask, visualInfo->blue_mask);
+            m_imageFormat = imageFormatForVisual(visualInfo->depth, visualInfo->red_mask, visualInfo->blue_mask, &m_imageRgbSwap);
             Colormap cmap = XCreateColormap(DISPLAY_FROM_XCB(this), xcb_parent_id, visualInfo->visual, AllocNone);
 
             XSetWindowAttributes a;
@@ -421,7 +433,7 @@ void QXcbWindow::create()
         }
 
         const xcb_visualtype_t *visual = platformScreen->visualForId(m_visualId);
-        m_imageFormat = imageFormatForVisual(m_depth, visual->red_mask, visual->blue_mask);
+        m_imageFormat = imageFormatForVisual(m_depth, visual->red_mask, visual->blue_mask, &m_imageRgbSwap);
 
         Q_XCB_CALL(xcb_create_window(xcb_connection(),
                                      m_depth,

src/plugins/platforms/xcb/qxcbwindow.h

@@ -113,6 +113,7 @@ public:
     xcb_window_t xcb_window() const { return m_window; }
     uint depth() const { return m_depth; }
     QImage::Format imageFormat() const { return m_imageFormat; }
+    bool imageNeedsRgbSwap() const { return m_imageRgbSwap; }
 
     bool handleGenericEvent(xcb_generic_event_t *event, long *result)  Q_DECL_OVERRIDE;
 
@@ -193,6 +194,7 @@ protected:
 
     uint m_depth;
     QImage::Format m_imageFormat;
+    bool m_imageRgbSwap;
 
     xcb_sync_int64_t m_syncValue;
     xcb_sync_counter_t m_syncCounter;