modules/gui/qt/maininterface/compositor.cpp

@@ -28,26 +28,62 @@
 
 namespace vlc {
 
-Compositor* Compositor::createCompositor(qt_intf_t *p_intf)
-{
-    bool ret;
-    VLC_UNUSED(ret);
+template<typename T>
+static Compositor* instanciateCompositor(qt_intf_t *p_intf) {
+    return new T(p_intf);
+}
+
+template<typename T>
+static bool preInit(qt_intf_t *p_intf) {
+    return T::preInit(p_intf);
+}
+
+struct {
+    const char* name;
+    Compositor* (*instanciate)(qt_intf_t *p_intf);
+    bool (*preInit)(qt_intf_t *p_intf);
+} static compositorList[] = {
 #ifdef _WIN32
 #ifdef HAVE_DCOMP_H
-    CompositorDirectComposition* dcomp_compositor = new CompositorDirectComposition(p_intf);
-    ret = dcomp_compositor->init();
-    if (ret)
-        return dcomp_compositor;
-    delete dcomp_compositor;
-    msg_Dbg(p_intf, "failed to create DirectComposition backend, use fallback");
+    {"dcomp", &instanciateCompositor<CompositorDirectComposition>, &preInit<CompositorDirectComposition> },
 #endif
-    CompositorWin7* win7_compositor = new CompositorWin7(p_intf);
-    if (win7_compositor->init())
-        return win7_compositor;
-    delete win7_compositor;
-    msg_Dbg(p_intf, "failed to create Win7 compositor backend, use fallback");
+    {"win7", &instanciateCompositor<CompositorWin7>, &preInit<CompositorWin7> },
 #endif
-    return new CompositorDummy(p_intf);
+    {"dummy", &instanciateCompositor<CompositorDummy>, &preInit<CompositorDummy> }
+};
+
+CompositorFactory::CompositorFactory(qt_intf_t *p_intf, const char* compositor)
+    : m_intf(p_intf)
+    , m_compositorName(compositor)
+{
+}
+
+bool CompositorFactory::preInit()
+{
+    for (; m_compositorIndex < ARRAY_SIZE(compositorList); m_compositorIndex++)
+    {
+        if (m_compositorName == "auto" || m_compositorName == compositorList[m_compositorIndex].name)
+        {
+            if (compositorList[m_compositorIndex].preInit(m_intf))
+                return true;
+        }
+    }
+    return false;
+}
+
+Compositor* CompositorFactory::createCompositor()
+{
+    for (; m_compositorIndex < ARRAY_SIZE(compositorList); m_compositorIndex++)
+    {
+        if (m_compositorName == "auto" || m_compositorName == compositorList[m_compositorIndex].name)
+        {
+            Compositor* compositor = compositorList[m_compositorIndex].instanciate(m_intf);
+            if (compositor->init())
+                return compositor;
+        }
+    }
+    msg_Err(m_intf, "no suitable compositor found");
+    return nullptr;
 }
 
 void Compositor::onWindowDestruction(vout_window_t *p_wnd)

modules/gui/qt/maininterface/compositor.hpp

@@ -48,6 +48,8 @@ public:
 public:
     virtual ~Compositor() = default;
 
+    virtual bool init() = 0;
+
     virtual MainInterface* makeMainInterface() = 0;
     virtual void destroyMainInterface() = 0;
 
@@ -57,15 +59,56 @@ public:
 
     virtual Type type() const = 0;
 
-    //factory
-    static Compositor* createCompositor(qt_intf_t *p_intf);
-
 protected:
     void onWindowDestruction(vout_window_t *p_wnd);
 
     VoutDestroyCb m_destroyCb = nullptr;
 };
 
+/**
+ * @brief The CompositorFactory class will instanciate a compositor
+ * in auto mode, compositor will be instantiated from the list by order declaration,
+ * compositor can be explicitly defined by passing its name.
+ *
+ * the usual scenario is:
+ *
+ *   - call to preInit that will try to preInit compositors from list until we find
+ *     a matching candidate
+ *
+ *   - start Qt main loop
+ *
+ *   - call to createCompositor to instantiate the compositor, if it fails it will
+ *     try to initialize the next compositors from the list
+ */
+class CompositorFactory {
+public:
+
+    CompositorFactory(qt_intf_t *p_intf, const char* compositor = "auto");
+
+    /**
+     * @brief preInit will check whether a compositor can be used, before starting Qt,
+     * each candidate will may perform some basic checks and can setup Qt enviroment variable if required
+     *
+     * @note if a compositor return true on preinit but fails to initialize afterwards, next
+     * compositor in chain will be initialized without the preinit phaze (as Qt will be already started)
+     * this might lead to an unstable configuration if incompatible operations are done in the preInit phase
+     *
+     * @return true if a compositor can be instantiated
+     */
+    bool preInit();
+
+    /**
+     * @brief createCompositor will instantiate a compositor
+     *
+     * @return the instantaied compositor, null if no compsitor can be instanciated
+     */
+    Compositor* createCompositor();
+
+private:
+    qt_intf_t* m_intf = nullptr;
+    QString m_compositorName;
+    size_t m_compositorIndex = 0;
+};
 
 }
 

modules/gui/qt/maininterface/compositor_dcomp.cpp

@@ -29,6 +29,7 @@
 #include <QApplication>
 #include <QDesktopWidget>
 #include <QQuickWidget>
+#include <QLibrary>
 
 #include <QOpenGLFunctions>
 #include <QOpenGLFramebufferObject>
@@ -140,6 +141,98 @@ CompositorDirectComposition::~CompositorDirectComposition()
         FreeLibrary(m_dcomp_dll);
 }
 
+bool CompositorDirectComposition::preInit(qt_intf_t * p_intf)
+{
+    //import DirectComposition API (WIN8+)
+    QLibrary dcompDll("DCOMP.dll");
+    if (!dcompDll.load())
+        return false;
+    DCompositionCreateDeviceFun myDCompositionCreateDevice = (DCompositionCreateDeviceFun)dcompDll.resolve("DCompositionCreateDevice");
+    if (!myDCompositionCreateDevice)
+    {
+        msg_Dbg(p_intf, "Direct Composition is not present, can't initialize direct composition");
+        return false;
+    }
+
+    //check whether D3DCompiler is available. whitout it Angle won't work
+    QLibrary d3dCompilerDll;
+    for (int i = 47; i > 41; --i)
+    {
+        d3dCompilerDll.setFileName(QString("D3DCOMPILER_%1.dll").arg(i));
+        if (d3dCompilerDll.load())
+            break;
+    }
+    if (!d3dCompilerDll.isLoaded())
+    {
+        msg_Dbg(p_intf, "can't find d3dcompiler_xx.dll, can't initialize direct composition");
+        return false;
+    }
+
+    HRESULT hr;
+    UINT creationFlags = D3D11_CREATE_DEVICE_BGRA_SUPPORT
+        //| D3D11_CREATE_DEVICE_DEBUG
+            ;
+
+    D3D_FEATURE_LEVEL requestedFeatureLevels[] = {
+        D3D_FEATURE_LEVEL_11_1,
+        D3D_FEATURE_LEVEL_11_0,
+    };
+
+    ComPtr<ID3D11Device> d3dDevice;
+    hr = D3D11CreateDevice(
+        nullptr,    // Adapter
+        D3D_DRIVER_TYPE_HARDWARE,
+        nullptr,    // Module
+        creationFlags,
+        requestedFeatureLevels,
+        ARRAY_SIZE(requestedFeatureLevels),
+        D3D11_SDK_VERSION,
+        d3dDevice.GetAddressOf(),
+        nullptr,    // Actual feature level
+        nullptr);
+
+    if (FAILED(hr))
+    {
+        msg_Dbg(p_intf, "can't create D3D11 device, can't initialize direct composition");
+        return false;
+    }
+
+    //check that we can create a shared texture
+    D3D11_FEATURE_DATA_D3D11_OPTIONS d3d11Options;
+    HRESULT checkFeatureHR = d3dDevice->CheckFeatureSupport(D3D11_FEATURE_D3D11_OPTIONS, &d3d11Options, sizeof(d3d11Options));
+
+    D3D11_TEXTURE2D_DESC texDesc = { };
+    texDesc.MipLevels = 1;
+    texDesc.SampleDesc.Count = 1;
+    texDesc.MiscFlags = 0;
+    texDesc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
+    texDesc.Usage = D3D11_USAGE_DEFAULT;
+    texDesc.CPUAccessFlags = 0;
+    texDesc.ArraySize = 1;
+    texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
+    texDesc.Height = 16;
+    texDesc.Width  = 16;
+    if (SUCCEEDED(checkFeatureHR) && d3d11Options.ExtendedResourceSharing) //D3D11.1 feature
+        texDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED | D3D11_RESOURCE_MISC_SHARED_NTHANDLE;
+    else
+        texDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
+
+    ComPtr<ID3D11Texture2D> d3dTmpTexture;
+    hr = d3dDevice->CreateTexture2D( &texDesc, NULL, &d3dTmpTexture );
+    if (FAILED(hr))
+    {
+        msg_Dbg(p_intf, "can't create shared texture, can't initialize direct composition");
+        return false;
+    }
+
+    //sanity check succeeded, we can now setup global Qt settings
+
+    //force usage of ANGLE backend
+    QApplication::setAttribute( Qt::AA_UseOpenGLES );
+
+    return true;
+}
+
 bool CompositorDirectComposition::init()
 {
     //import DirectComposition API (WIN8+)
@@ -187,8 +280,6 @@ bool CompositorDirectComposition::init()
     if (FAILED(hr))
         return false;
 
-    QApplication::setAttribute( Qt::AA_UseOpenGLES ); //force usage of ANGLE backend
-
     return true;
 }
 

modules/gui/qt/maininterface/compositor_dcomp.hpp

@@ -45,7 +45,8 @@ public:
     CompositorDirectComposition(qt_intf_t *p_intf, QObject* parent = nullptr);
     ~CompositorDirectComposition();
 
-    bool init();
+    static bool preInit(qt_intf_t *);
+    bool init() override;
 
     MainInterface *makeMainInterface() override;
     void destroyMainInterface() override;
@@ -70,6 +71,7 @@ private:
     qt_intf_t *m_intf = nullptr;
 
     MainInterface* m_rootWindow = nullptr;
+
     std::unique_ptr<CompositorDCompositionUISurface> m_uiSurface;
     vout_window_t *m_window = nullptr;
     std::unique_ptr<MainUI> m_ui;

modules/gui/qt/maininterface/compositor_dummy.cpp

@@ -29,6 +29,16 @@ CompositorDummy::CompositorDummy(qt_intf_t *p_intf, QObject* parent)
 {
 }
 
+bool CompositorDummy::preInit(qt_intf_t *)
+{
+    return true;
+}
+
+bool CompositorDummy::init()
+{
+    return true;
+}
+
 MainInterface* CompositorDummy::makeMainInterface()
 {
     m_rootWindow = new MainInterface(m_intf);

modules/gui/qt/maininterface/compositor_dummy.hpp

@@ -37,6 +37,9 @@ public:
     CompositorDummy(qt_intf_t *p_intf, QObject* parent = nullptr);
     virtual ~CompositorDummy() = default;
 
+    static bool preInit(qt_intf_t*);
+    virtual bool init() override;
+
     virtual MainInterface *makeMainInterface() override;
 
     /**

modules/gui/qt/maininterface/compositor_win7.cpp

@@ -96,7 +96,7 @@ CompositorWin7::~CompositorWin7()
         delete m_stable;
 }
 
-bool CompositorWin7::init()
+bool CompositorWin7::preInit(qt_intf_t *p_intf)
 {
     //check whether D3DCompiler is available. whitout it Angle won't work
     QLibrary d3dCompilerDll;
@@ -134,13 +134,18 @@ bool CompositorWin7::init()
     //otherwise Qt will load angle and fail.
     if (!d3dCompilerDll.isLoaded() || FAILED(hr))
     {
-        msg_Info(m_intf, "no D3D support, use software backend");
+        msg_Info(p_intf, "no D3D support, use software backend");
         QQuickWindow::setSceneGraphBackend(QSGRendererInterface::Software);
     }
 
     return true;
 }
 
+bool CompositorWin7::init()
+{
+    return true;
+}
+
 MainInterface* CompositorWin7::makeMainInterface()
 {
     //Tool flag needs to be passed in the window constructor otherwise the

modules/gui/qt/maininterface/compositor_win7.hpp

@@ -46,7 +46,8 @@ public:
 
     virtual ~CompositorWin7();
 
-    bool init();
+    static bool preInit(qt_intf_t *p_intf);
+    virtual bool init() override;
 
     virtual MainInterface *makeMainInterface() override;
     virtual void destroyMainInterface() override;

modules/gui/qt/qt.cpp

@@ -243,6 +243,9 @@ static void ShowDialog   ( intf_thread_t *, int, int, intf_dialog_args_t * );
 
 #define FULLSCREEN_CONTROL_PIXELS N_( "Fullscreen controller mouse sensitivity" )
 
+#define QT_COMPOSITOR_TEXT N_("Select Qt video intergration backend")
+#define QT_COMPOSITOR_LONGTEXT N_("Select Qt video intergration backend. Use with care, the interface may not start if an incompatible compositor is selected")
+
 static const int i_notification_list[] =
     { NOTIFICATION_NEVER, NOTIFICATION_MINIMIZED, NOTIFICATION_ALWAYS };
 
@@ -256,6 +259,27 @@ static const int i_raise_list[] =
 static const char *const psz_raise_list_text[] =
     { N_( "Never" ), N_( "Video" ), N_( "Audio" ), _( "Audio/Video" ) };
 
+static const char *const compositor_vlc[] = {
+    "auto",
+#ifdef _WIN32
+#ifdef HAVE_DCOMP_H
+    "dcomp"
+#endif
+    "win7",
+#endif
+    "dummy"
+};
+static const char *const compositor_user[] = {
+    N_("Automatic"),
+#ifdef _WIN32
+#ifdef HAVE_DCOMP_H
+    "Direct Composition",
+#endif
+    "Windows 7",
+#endif
+    N_("Dummy"),
+};
+
 /**********************************************************************/
 vlc_module_begin ()
     set_shortname( "Qt" )
@@ -300,6 +324,9 @@ vlc_module_begin ()
     add_bool( "qt-fs-controller", true, QT_FULLSCREEN_TEXT,
               nullptr )
 
+    add_string("qt-compositor", "auto", QT_COMPOSITOR_TEXT, QT_COMPOSITOR_LONGTEXT)
+            change_string_list(compositor_vlc, compositor_user)
+
     add_bool( "qt-recentplay", true, RECENTPLAY_TEXT,
               nullptr )
     add_string( "qt-recentplay-filter", "",
@@ -647,7 +674,9 @@ static void *Thread( void *obj )
 
     Q_INIT_RESOURCE( vlc );
 
-    p_intf->p_compositor = vlc::Compositor::createCompositor(p_intf);
+    vlc::CompositorFactory compositorFactory(p_intf, var_InheritString(p_intf, "qt-compositor"));
+
+    compositorFactory.preInit();
 
     QApplication::setAttribute( Qt::AA_EnableHighDpiScaling );
     QApplication::setAttribute( Qt::AA_UseHighDpiPixmaps );
@@ -730,7 +759,12 @@ static void *Thread( void *obj )
 
     if( !p_intf->b_isDialogProvider )
     {
-        p_mi = p_intf->p_compositor->makeMainInterface();
+        do {
+            p_intf->p_compositor = compositorFactory.createCompositor();
+            if (! p_intf->p_compositor)
+                break;
+            p_mi = p_intf->p_compositor->makeMainInterface();
+        } while(p_mi == nullptr);
         p_intf->p_mi = p_mi;
 
         if (!p_mi)