From 6725dc70e11cac0902a04d3b2dc56d6698cfd130 Mon Sep 17 00:00:00 2001
From: Pierre Lamot <pierre@videolabs.io>
Date: Mon, 16 Nov 2020 10:25:48 +0100
Subject: [PATCH] qt: initialize compositor in two phases

some compositor requires to set Qt Attributes before Qt application
is started such as selecting the right graphical backend (ie: dcomp needs to set
OpenGLES). And some compositor needs know wich QPA is in use (ie: x11 vs
wayland), this can only be known after QApp is started.

Co-authored-by: Guillaume Nicolussi Castellan <nullgemm@videolabs.io>
---
 modules/gui/qt/maininterface/compositor.cpp   | 68 ++++++++++++++-----
 modules/gui/qt/maininterface/compositor.hpp   | 49 ++++++++++++-
 .../gui/qt/maininterface/compositor_dcomp.cpp | 10 ++-
 .../gui/qt/maininterface/compositor_dcomp.hpp |  4 +-
 .../gui/qt/maininterface/compositor_dummy.cpp | 10 +++
 .../gui/qt/maininterface/compositor_dummy.hpp |  3 +
 .../gui/qt/maininterface/compositor_win7.cpp  |  9 ++-
 .../gui/qt/maininterface/compositor_win7.hpp  |  3 +-
 modules/gui/qt/qt.cpp                         | 11 ++-
 9 files changed, 140 insertions(+), 27 deletions(-)

diff --git a/modules/gui/qt/maininterface/compositor.cpp b/modules/gui/qt/maininterface/compositor.cpp
index dc5b6933ba0d..f49c17adfa25 100644
--- a/modules/gui/qt/maininterface/compositor.cpp
+++ b/modules/gui/qt/maininterface/compositor.cpp
@@ -28,26 +28,62 @@
 
 namespace vlc {
 
-Compositor* Compositor::createCompositor(qt_intf_t *p_intf)
-{
-    bool ret;
-    VLC_UNUSED(ret);
+template<typename T>
+static Compositor* instanciateCompositor(qt_intf_t *p_intf) {
+    return new T(p_intf);
+}
+
+template<typename T>
+static bool preInit(qt_intf_t *p_intf) {
+    return T::preInit(p_intf);
+}
+
+struct {
+    const char* name;
+    Compositor* (*instanciate)(qt_intf_t *p_intf);
+    bool (*preInit)(qt_intf_t *p_intf);
+} static compositorList[] = {
 #ifdef _WIN32
 #ifdef HAVE_DCOMP_H
-    CompositorDirectComposition* dcomp_compositor = new CompositorDirectComposition(p_intf);
-    ret = dcomp_compositor->init();
-    if (ret)
-        return dcomp_compositor;
-    delete dcomp_compositor;
-    msg_Dbg(p_intf, "failed to create DirectComposition backend, use fallback");
+    {"dcomp", &instanciateCompositor<CompositorDirectComposition>, &preInit<CompositorDirectComposition> },
 #endif
-    CompositorWin7* win7_compositor = new CompositorWin7(p_intf);
-    if (win7_compositor->init())
-        return win7_compositor;
-    delete win7_compositor;
-    msg_Dbg(p_intf, "failed to create Win7 compositor backend, use fallback");
+    {"win7", &instanciateCompositor<CompositorWin7>, &preInit<CompositorWin7> },
 #endif
-    return new CompositorDummy(p_intf);
+    {"dummy", &instanciateCompositor<CompositorDummy>, &preInit<CompositorDummy> }
+};
+
+CompositorFactory::CompositorFactory(qt_intf_t *p_intf, const char* compositor)
+    : m_intf(p_intf)
+    , m_compositorName(compositor)
+{
+}
+
+bool CompositorFactory::preInit()
+{
+    for (; m_compositorIndex < ARRAY_SIZE(compositorList); m_compositorIndex++)
+    {
+        if (m_compositorName == "auto" || m_compositorName == compositorList[m_compositorIndex].name)
+        {
+            if (compositorList[m_compositorIndex].preInit(m_intf))
+                return true;
+        }
+    }
+    return false;
+}
+
+Compositor* CompositorFactory::createCompositor()
+{
+    for (; m_compositorIndex < ARRAY_SIZE(compositorList); m_compositorIndex++)
+    {
+        if (m_compositorName == "auto" || m_compositorName == compositorList[m_compositorIndex].name)
+        {
+            Compositor* compositor = compositorList[m_compositorIndex].instanciate(m_intf);
+            if (compositor->init())
+                return compositor;
+        }
+    }
+    msg_Err(m_intf, "no suitable compositor found");
+    return nullptr;
 }
 
 void Compositor::onWindowDestruction(vout_window_t *p_wnd)
diff --git a/modules/gui/qt/maininterface/compositor.hpp b/modules/gui/qt/maininterface/compositor.hpp
index 4e29f29e2291..2a558e1765be 100644
--- a/modules/gui/qt/maininterface/compositor.hpp
+++ b/modules/gui/qt/maininterface/compositor.hpp
@@ -48,6 +48,8 @@ public:
 public:
     virtual ~Compositor() = default;
 
+    virtual bool init() = 0;
+
     virtual MainInterface* makeMainInterface() = 0;
     virtual void destroyMainInterface() = 0;
 
@@ -57,15 +59,56 @@ public:
 
     virtual Type type() const = 0;
 
-    //factory
-    static Compositor* createCompositor(qt_intf_t *p_intf);
-
 protected:
     void onWindowDestruction(vout_window_t *p_wnd);
 
     VoutDestroyCb m_destroyCb = nullptr;
 };
 
+/**
+ * @brief The CompositorFactory class will instanciate a compositor
+ * in auto mode, compositor will be instantiated from the list by order declaration,
+ * compositor can be explicitly defined by passing its name.
+ *
+ * the usual scenario is:
+ *
+ *   - call to preInit that will try to preInit compositors from list until we find
+ *     a matching candidate
+ *
+ *   - start Qt main loop
+ *
+ *   - call to createCompositor to instantiate the compositor, if it fails it will
+ *     try to initialize the next compositors from the list
+ */
+class CompositorFactory {
+public:
+
+    CompositorFactory(qt_intf_t *p_intf, const char* compositor = "auto");
+
+    /**
+     * @brief preInit will check whether a compositor can be used, before starting Qt,
+     * each candidate will may perform some basic checks and can setup Qt enviroment variable if required
+     *
+     * @note if a compositor return true on preinit but fails to initialize afterwards, next
+     * compositor in chain will be initialized without the preinit phaze (as Qt will be already started)
+     * this might lead to an unstable configuration if incompatible operations are done in the preInit phase
+     *
+     * @return true if a compositor can be instantiated
+     */
+    bool preInit();
+
+    /**
+     * @brief createCompositor will instantiate a compositor
+     *
+     * @return the instantaied compositor, null if no compsitor can be instanciated
+     */
+    Compositor* createCompositor();
+
+private:
+    qt_intf_t* m_intf = nullptr;
+    QString m_compositorName;
+    size_t m_compositorIndex = 0;
+};
 
 }
 
diff --git a/modules/gui/qt/maininterface/compositor_dcomp.cpp b/modules/gui/qt/maininterface/compositor_dcomp.cpp
index 01026cc8605a..3966ee1baa36 100644
--- a/modules/gui/qt/maininterface/compositor_dcomp.cpp
+++ b/modules/gui/qt/maininterface/compositor_dcomp.cpp
@@ -140,6 +140,14 @@ CompositorDirectComposition::~CompositorDirectComposition()
         FreeLibrary(m_dcomp_dll);
 }
 
+bool CompositorDirectComposition::preInit(qt_intf_t * p_intf)
+{
+    //force usage of ANGLE backend
+    QApplication::setAttribute( Qt::AA_UseOpenGLES );
+
+    return true;
+}
+
 bool CompositorDirectComposition::init()
 {
     //import DirectComposition API (WIN8+)
@@ -187,8 +195,6 @@ bool CompositorDirectComposition::init()
     if (FAILED(hr))
         return false;
 
-    QApplication::setAttribute( Qt::AA_UseOpenGLES ); //force usage of ANGLE backend
-
     return true;
 }
 
diff --git a/modules/gui/qt/maininterface/compositor_dcomp.hpp b/modules/gui/qt/maininterface/compositor_dcomp.hpp
index 78e8f8a69d1a..cc5dba10ff84 100644
--- a/modules/gui/qt/maininterface/compositor_dcomp.hpp
+++ b/modules/gui/qt/maininterface/compositor_dcomp.hpp
@@ -45,7 +45,8 @@ public:
     CompositorDirectComposition(qt_intf_t *p_intf, QObject* parent = nullptr);
     ~CompositorDirectComposition();
 
-    bool init();
+    static bool preInit(qt_intf_t *);
+    bool init() override;
 
     MainInterface *makeMainInterface() override;
     void destroyMainInterface() override;
@@ -70,6 +71,7 @@ private:
     qt_intf_t *m_intf = nullptr;
 
     MainInterface* m_rootWindow = nullptr;
+
     std::unique_ptr<CompositorDCompositionUISurface> m_uiSurface;
     vout_window_t *m_window = nullptr;
     std::unique_ptr<MainUI> m_ui;
diff --git a/modules/gui/qt/maininterface/compositor_dummy.cpp b/modules/gui/qt/maininterface/compositor_dummy.cpp
index f44cbd1b3ef5..8d123de4562a 100644
--- a/modules/gui/qt/maininterface/compositor_dummy.cpp
+++ b/modules/gui/qt/maininterface/compositor_dummy.cpp
@@ -29,6 +29,16 @@ CompositorDummy::CompositorDummy(qt_intf_t *p_intf, QObject* parent)
 {
 }
 
+bool CompositorDummy::preInit(qt_intf_t *)
+{
+    return true;
+}
+
+bool CompositorDummy::init()
+{
+    return true;
+}
+
 MainInterface* CompositorDummy::makeMainInterface()
 {
     m_rootWindow = new MainInterface(m_intf);
diff --git a/modules/gui/qt/maininterface/compositor_dummy.hpp b/modules/gui/qt/maininterface/compositor_dummy.hpp
index 31c817b5b5a2..216c879f0ab8 100644
--- a/modules/gui/qt/maininterface/compositor_dummy.hpp
+++ b/modules/gui/qt/maininterface/compositor_dummy.hpp
@@ -37,6 +37,9 @@ public:
     CompositorDummy(qt_intf_t *p_intf, QObject* parent = nullptr);
     virtual ~CompositorDummy() = default;
 
+    static bool preInit(qt_intf_t*);
+    virtual bool init() override;
+
     virtual MainInterface *makeMainInterface() override;
 
     /**
diff --git a/modules/gui/qt/maininterface/compositor_win7.cpp b/modules/gui/qt/maininterface/compositor_win7.cpp
index f00605fefa45..2b4833afd45e 100644
--- a/modules/gui/qt/maininterface/compositor_win7.cpp
+++ b/modules/gui/qt/maininterface/compositor_win7.cpp
@@ -96,7 +96,7 @@ CompositorWin7::~CompositorWin7()
         delete m_stable;
 }
 
-bool CompositorWin7::init()
+bool CompositorWin7::preInit(qt_intf_t *p_intf)
 {
     //check whether D3DCompiler is available. whitout it Angle won't work
     QLibrary d3dCompilerDll;
@@ -134,13 +134,18 @@ bool CompositorWin7::init()
     //otherwise Qt will load angle and fail.
     if (!d3dCompilerDll.isLoaded() || FAILED(hr))
     {
-        msg_Info(m_intf, "no D3D support, use software backend");
+        msg_Info(p_intf, "no D3D support, use software backend");
         QQuickWindow::setSceneGraphBackend(QSGRendererInterface::Software);
     }
 
     return true;
 }
 
+bool CompositorWin7::init()
+{
+    return true;
+}
+
 MainInterface* CompositorWin7::makeMainInterface()
 {
     //Tool flag needs to be passed in the window constructor otherwise the
diff --git a/modules/gui/qt/maininterface/compositor_win7.hpp b/modules/gui/qt/maininterface/compositor_win7.hpp
index aa007baf594f..6a432d18157d 100644
--- a/modules/gui/qt/maininterface/compositor_win7.hpp
+++ b/modules/gui/qt/maininterface/compositor_win7.hpp
@@ -46,7 +46,8 @@ public:
 
     virtual ~CompositorWin7();
 
-    bool init();
+    static bool preInit(qt_intf_t *p_intf);
+    virtual bool init() override;
 
     virtual MainInterface *makeMainInterface() override;
     virtual void destroyMainInterface() override;
diff --git a/modules/gui/qt/qt.cpp b/modules/gui/qt/qt.cpp
index bbaa36dc6e23..aa6557b2678a 100644
--- a/modules/gui/qt/qt.cpp
+++ b/modules/gui/qt/qt.cpp
@@ -647,7 +647,9 @@ static void *Thread( void *obj )
 
     Q_INIT_RESOURCE( vlc );
 
-    p_intf->p_compositor = vlc::Compositor::createCompositor(p_intf);
+    vlc::CompositorFactory compositorFactory(p_intf);
+
+    compositorFactory.preInit();
 
     QApplication::setAttribute( Qt::AA_EnableHighDpiScaling );
     QApplication::setAttribute( Qt::AA_UseHighDpiPixmaps );
@@ -730,7 +732,12 @@ static void *Thread( void *obj )
 
     if( !p_intf->b_isDialogProvider )
     {
-        p_mi = p_intf->p_compositor->makeMainInterface();
+        do {
+            p_intf->p_compositor = compositorFactory.createCompositor();
+            if (! p_intf->p_compositor)
+                break;
+            p_mi = p_intf->p_compositor->makeMainInterface();
+        } while(p_mi == nullptr);
         p_intf->p_mi = p_mi;
 
         if (!p_mi)
-- 
GitLab