Enable DWM transparency for OpenGL windows under Windows

Starting with Windows Vista, we can control the compositing window
manager using the DWM APIs. This allows us to make truly transparent
OpenGL windows.

We also have to detect and listen if compositing is enabled, otherwise
enabling transparency will cause glitches all over the place.

This (partially) fixes
Task-number: QTBUG-28214

Change-Id: I0fe1ec7adec8181b788c32de03c59142731d9e7f
Done-with: Friedemann Kleint <Friedemann.Kleint@digia.com>
Reviewed-by: Joerg Bornemann <joerg.bornemann@digia.com>
Reviewed-by: Gunnar Sletta <gunnar.sletta@digia.com>
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@digia.com>

src/plugins/platforms/windows/qtwindows_additional.h

@@ -49,6 +49,10 @@
 #    define WM_THEMECHANGED 0x031A
 #endif
 
+#ifndef WM_DWMCOMPOSITIONCHANGED
+#    define WM_DWMCOMPOSITIONCHANGED 0x31E
+#endif
+
 #ifndef GWL_HWNDPARENT
 #    define GWL_HWNDPARENT (-8)
 #endif

src/plugins/platforms/windows/qtwindowsglobal.h

@@ -105,6 +105,7 @@ enum WindowsEventType // Simplify event types
     InputMethodCloseCandidateWindowEvent = InputMethodEventFlag + 5,
     InputMethodRequest = InputMethodEventFlag + 6,
     ThemeChanged = ThemingEventFlag + 1,
+    CompositionSettingsChanged = ThemingEventFlag + 2,
     DisplayChangedEvent = 437,
     SettingChangedEvent = DisplayChangedEvent + 1,
     ContextMenu = 123,
@@ -201,6 +202,8 @@ inline QtWindows::WindowsEventType windowsEventType(UINT message, WPARAM wParamI
         return QtWindows::DisplayChangedEvent;
     case WM_THEMECHANGED:
         return QtWindows::ThemeChanged;
+    case WM_DWMCOMPOSITIONCHANGED:
+        return QtWindows::CompositionSettingsChanged;
 #ifndef QT_NO_CONTEXTMENU
     case WM_CONTEXTMENU:
         return QtWindows::ContextMenu;

src/plugins/platforms/windows/qwindowscontext.cpp

@@ -884,6 +884,9 @@ bool QWindowsContext::windowsProc(HWND hwnd, UINT message,
             theme->windowsThemeChanged(platformWindow->window());
         return true;
     }
+    case QtWindows::CompositionSettingsChanged:
+        platformWindow->handleCompositionSettingsChanged();
+        return true;
 #ifndef Q_OS_WINCE
     case QtWindows::ActivateWindowEvent:
 #ifndef QT_NO_TABLETEVENT

src/plugins/platforms/windows/qwindowswindow.cpp

@@ -56,6 +56,7 @@
 #include <QtGui/QScreen>
 #include <QtGui/QWindow>
 #include <QtGui/QRegion>
+#include <private/qsystemlibrary_p.h>
 #include <private/qwindow_p.h>
 #include <private/qguiapplication_p.h>
 #include <qpa/qwindowsysteminterface.h>
@@ -203,6 +204,69 @@ static inline QSize clientSize(HWND hwnd)
     return qSizeOfRect(rect);
 }
 
+static bool applyBlurBehindWindow(HWND hwnd)
+{
+#ifdef Q_OS_WINCE
+    Q_UNUSED(hwnd);
+    return false;
+#else
+    enum { dwmBbEnable = 0x1, dwmBbBlurRegion = 0x2 };
+
+    struct DwmBlurBehind {
+        DWORD dwFlags;
+        BOOL  fEnable;
+        HRGN  hRgnBlur;
+        BOOL  fTransitionOnMaximized;
+    };
+
+    typedef HRESULT (WINAPI *PtrDwmEnableBlurBehindWindow)(HWND, const DwmBlurBehind*);
+    typedef HRESULT (WINAPI *PtrDwmIsCompositionEnabled)(BOOL *);
+
+    // DWM API is available only from Windows Vista
+    if (QSysInfo::windowsVersion() < QSysInfo::WV_VISTA)
+        return false;
+
+    static bool functionPointersResolved = false;
+    static PtrDwmEnableBlurBehindWindow dwmBlurBehind = 0;
+    static PtrDwmIsCompositionEnabled dwmIsCompositionEnabled = 0;
+
+    if (Q_UNLIKELY(!functionPointersResolved)) {
+        QSystemLibrary library(QStringLiteral("dwmapi"));
+        if (library.load()) {
+            dwmBlurBehind = (PtrDwmEnableBlurBehindWindow)(library.resolve("DwmEnableBlurBehindWindow"));
+            dwmIsCompositionEnabled = (PtrDwmIsCompositionEnabled)(library.resolve("DwmIsCompositionEnabled"));
+        }
+
+        functionPointersResolved = true;
+    }
+
+    if (Q_UNLIKELY(!dwmBlurBehind || !dwmIsCompositionEnabled))
+        return false;
+
+    BOOL compositionEnabled;
+    if (dwmIsCompositionEnabled(&compositionEnabled) != S_OK)
+        return false;
+
+    DwmBlurBehind blurBehind = {0, 0, 0, 0};
+
+    if (compositionEnabled) {
+        blurBehind.dwFlags = dwmBbEnable | dwmBbBlurRegion;
+        blurBehind.fEnable = TRUE;
+        blurBehind.hRgnBlur = CreateRectRgn(0, 0, -1, -1);
+    } else {
+        blurBehind.dwFlags = dwmBbEnable;
+        blurBehind.fEnable = FALSE;
+    }
+
+    const bool result = dwmBlurBehind(hwnd, &blurBehind) == S_OK;
+
+    if (blurBehind.hRgnBlur)
+        DeleteObject(blurBehind.hRgnBlur);
+
+    return result;
+#endif // Q_OS_WINCE
+}
+
 // from qwidget_win.cpp, pass flags separately in case they have been "autofixed".
 static bool shouldShowMaximizeButton(const QWindow *w, Qt::WindowFlags flags)
 {
@@ -531,6 +595,10 @@ QWindowsWindow::WindowData
     result.frame = context->margins;
     result.embedded = embedded;
     result.customMargins = context->customMargins;
+
+    if (isGL && hasAlpha)
+        applyBlurBehindWindow(result.hwnd);
+
     return result;
 }
 
@@ -1155,6 +1223,13 @@ void QWindowsWindow::handleHidden()
     fireExpose(QRegion());
 }
 
+void QWindowsWindow::handleCompositionSettingsChanged()
+{
+    const QWindow *w = window();
+    if (w->surfaceType() == QWindow::OpenGLSurface && w->format().hasAlpha())
+        applyBlurBehindWindow(handle());
+}
+
 void QWindowsWindow::setGeometry(const QRect &rectIn)
 {
     QRect rect = rectIn;

src/plugins/platforms/windows/qwindowswindow.h

@@ -221,6 +221,7 @@ public:
     void handleMoved();
     void handleResized(int wParam);
     void handleHidden();
+    void handleCompositionSettingsChanged();
 
     static inline HWND handleOf(const QWindow *w);
     static inline QWindowsWindow *baseWindowOf(const QWindow *w);