eglfs_kms_vsp2: Recover if queuing input buffers fails

Clear all queued buffers, and reinitialize the Qt layer so we can try again
next time.

Change-Id: I921f6f457666206be92aadf2fe40b855e6ebff62
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

src/plugins/platforms/eglfs/deviceintegration/eglfs_kms_vsp2/qeglfskmsvsp2screen.cpp

@@ -141,10 +141,15 @@ void QEglFSKmsVsp2Screen::initDumbFrameBuffers()
 void QEglFSKmsVsp2Screen::initVsp2()
 {
     qCDebug(qLcEglfsKmsDebug, "Initializing Vsp2 hardware");
-    const QSize screenSize = rawGeometry().size();
-    m_blendDevice.reset(new QVsp2BlendingDevice(screenSize));
+    m_blendDevice.reset(new QVsp2BlendingDevice(rawGeometry().size()));
 
     // Enable input for main buffer drawn by the compositor (always on)
+    initQtLayer();
+}
+
+void QEglFSKmsVsp2Screen::initQtLayer()
+{
+    const QSize screenSize = rawGeometry().size();
     const uint bytesPerLine = uint(screenSize.width()) * 4; //TODO: is this ok?
     bool formatSet = m_blendDevice->enableInput(m_qtLayer, QRect(QPoint(), screenSize), m_output.drm_format, bytesPerLine);
     if (!formatSet) {
@@ -298,8 +303,13 @@ void QEglFSKmsVsp2Screen::blendAndFlipDrm()
     vBlank.request.signal = 0;
     drmWaitVBlank(driFd, &vBlank);
 
-    if (!m_blendDevice->blend(backBuffer.dmabufFd))
-        qWarning() << "Vsp2: blending failed";
+    if (!m_blendDevice->blend(backBuffer.dmabufFd)) {
+        qWarning() << "Vsp2: Blending failed";
+
+        // For some reason, a failed blend may often mess up the qt layer, so reinitialize it here
+        m_blendDevice->disableInput(m_qtLayer);
+        initQtLayer();
+    }
 
     for (auto cb : m_blendFinishedCallbacks)
         cb();

src/plugins/platforms/eglfs/deviceintegration/eglfs_kms_vsp2/qeglfskmsvsp2screen.h

@@ -60,6 +60,7 @@ public:
 
     void initDumbFrameBuffers();
     void initVsp2();
+    void initQtLayer();
 
     //TODO: use a fixed index API instead of auto increment?
     int addLayer(int dmabufFd, const QSize &size, const QPoint &position, uint drmPixelFormat, uint bytesPerLine);

src/plugins/platforms/eglfs/deviceintegration/eglfs_kms_vsp2/qvsp2blendingdevice.cpp

@@ -219,11 +219,12 @@ bool QVsp2BlendingDevice::blend(int outputDmabufFd)
     if (!m_dirty)
         qWarning("Blending without being dirty, should not be necessary");
 
-    if (!m_inputs[0].enabled) {
-        qWarning("Vsp2: Can't blend with layer 0 disabled");
+    if (!hasContinuousLayers()) {
+        qWarning("Vsp2: Can't blend when layers are not enabled in order from 0 without gaps.");
         return false;
     }
 
+    bool queueingFailed = false;
     // Queue dma input buffers
     for (int i=0; i < m_inputs.size(); ++i) {
         auto &input = m_inputs[i];
@@ -233,12 +234,22 @@ bool QVsp2BlendingDevice::blend(int outputDmabufFd)
                            << "with dmabuf" << input.dmabuf.fd
                            << "and size" << input.geometry.size();
 
-                if (!disableInput(i))
-                    qWarning() << "Vsp2: Failed to disable input" << i;
+                queueingFailed = true;
             }
         }
     }
 
+    if (queueingFailed) {
+        qWarning() << "Vsp2: Trying to clean up queued buffers";
+        for (auto &input : qAsConst(m_inputs)) {
+            if (input.enabled) {
+                if (!input.rpfInput->clearBuffers())
+                    qWarning() << "Vsp2: Failed to remove buffers after an aborted blend";
+            }
+        }
+        return false;
+    }
+
     if (!m_wpfOutput->queueBuffer(outputDmabufFd, m_screenSize)) {
         qWarning() << "Vsp2: Failed to queue blending output buffer" << outputDmabufFd << m_screenSize;
         return false;
@@ -270,6 +281,17 @@ int QVsp2BlendingDevice::numInputs() const
     return m_inputs.size();
 }
 
+bool QVsp2BlendingDevice::hasContinuousLayers() const
+{
+    bool seenDisabled = false;
+    for (auto &input : qAsConst(m_inputs)) {
+        if (seenDisabled && input.enabled)
+            return false;
+        seenDisabled |= !input.enabled;
+    }
+    return m_inputs[0].enabled;
+}
+
 bool QVsp2BlendingDevice::streamOn()
 {
     for (auto &input : m_inputs) {

src/plugins/platforms/eglfs/deviceintegration/eglfs_kms_vsp2/qvsp2blendingdevice.h

@@ -63,6 +63,7 @@ public:
     bool blend(int outputDmabufFd);
     int numInputs() const;
     bool isDirty() const { return m_dirty; }
+    bool hasContinuousLayers() const;
 private:
     bool streamOn();
     bool streamOff();