From d1e2e6183888391acdd65d3298545ab5765ec03f Mon Sep 17 00:00:00 2001
From: Alexandre Janniaux <ajanni@videolabs.io>
Date: Tue, 25 Mar 2025 17:10:58 +0100
Subject: [PATCH 1/2] lib: media_player: use d3d11drawable vout for callbacks

The d3d11 drawable display doesn't exist yet, but the direct3d11 display
is not reachable from a wextern window (regression from previous commit
e33e383fde533c17032614bd0cbf5be1d364917a that check window type) so we
might as well change it immediately instead of doing it in three steps.

Next commit will re-establish the usage of the display by introducing
it.
---
 lib/media_player.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/media_player.c b/lib/media_player.c
index 240ee7b55d64..6d7ba5bf07af 100644
--- a/lib/media_player.c
+++ b/lib/media_player.c
@@ -1128,7 +1128,7 @@ bool libvlc_video_set_output_callbacks(libvlc_media_player_t *mp,
     }
     else if ( engine == libvlc_video_engine_d3d11 )
     {
-        var_SetString ( mp, "vout", "direct3d11" );
+        var_SetString ( mp, "vout", "d3d11drawable" );
         var_SetString ( mp, "dec-dev", "d3d11" );
     }
     else if ( engine == libvlc_video_engine_d3d9 )
-- 
GitLab


From 7ff98c8e83f7ab391c883003ca532dc04d72fad0 Mon Sep 17 00:00:00 2001
From: Alexandre Janniaux <ajanni@videolabs.io>
Date: Tue, 25 Mar 2025 17:11:37 +0100
Subject: [PATCH 2/2] direct3d11: separate display from drawable display

Separating the display used in VLC and the libvlc application-side
display ensures we're not trying to use the latter in the VLC scenarios
and still ensures we check the kind of windowing that is being provided
to the display.

Fix a regression with e33e383fde533c17032614bd0cbf5be1d364917a
preventing the callbacks from opening the display.
---
 modules/video_output/win32/direct3d11.cpp | 38 ++++++++++++++++++-----
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/modules/video_output/win32/direct3d11.cpp b/modules/video_output/win32/direct3d11.cpp
index 9791e00e4881..18e7a5323b1d 100644
--- a/modules/video_output/win32/direct3d11.cpp
+++ b/modules/video_output/win32/direct3d11.cpp
@@ -58,7 +58,9 @@
 
 using Microsoft::WRL::ComPtr;
 
-static int  Open(vout_display_t *,
+static int  OpenDisplay(vout_display_t *,
+                 video_format_t *, vlc_video_context *);
+static int  OpenDrawable(vout_display_t *,
                  video_format_t *, vlc_video_context *);
 static void Close(vout_display_t *);
 
@@ -98,7 +100,12 @@ vlc_module_begin ()
         change_string_list(ppsz_hdr_mode, ppsz_hdr_mode_text)
 
     add_shortcut("direct3d11")
-    set_callback_display(Open, 300)
+    set_callback_display(OpenDisplay, 300)
+
+    add_submodule()
+        set_callback_display(OpenDrawable, 0)
+        add_shortcut("d3d11drawable")
+
 vlc_module_end ()
 
 enum d3d11_upscale
@@ -529,13 +536,9 @@ static const auto ops = []{
     return ops;
 }();
 
-static int Open(vout_display_t *vd,
+static int OpenGeneric(vout_display_t *vd,
                 video_format_t *fmtp, vlc_video_context *context)
 {
-    if (vd->cfg->window->type != VLC_WINDOW_TYPE_HWND &&
-        vd->cfg->window->type != VLC_WINDOW_TYPE_DCOMP)
-        return VLC_ENOTSUP;
-
     vout_display_sys_t *sys = new (std::nothrow) vout_display_sys_t();
     if (!sys)
         return VLC_ENOMEM;
@@ -639,6 +642,27 @@ error:
     return VLC_EGENERIC;
 }
 
+static int OpenDisplay(vout_display_t *vd,
+                       video_format_t *fmtp, vlc_video_context *context)
+{
+    if (vd->cfg->window->type != VLC_WINDOW_TYPE_HWND &&
+        vd->cfg->window->type != VLC_WINDOW_TYPE_DCOMP)
+        return VLC_ENOTSUP;
+
+    return OpenGeneric(vd, fmtp, context);
+}
+
+static int OpenDrawable(vout_display_t *vd,
+                        video_format_t *fmtp, vlc_video_context *context)
+{
+    if (vd->cfg->window->type != VLC_WINDOW_TYPE_DUMMY)
+        return VLC_ENOTSUP;
+
+    /* TODO: other checks for callback validity should be there. */
+
+    return OpenGeneric(vd, fmtp, context);
+}
+
 static void Close(vout_display_t *vd)
 {
     vout_display_sys_t *sys = static_cast<vout_display_sys_t *>(vd->sys);
-- 
GitLab