MFT codec: Support for loading any MFT

Adds support in Media Foundation (MFT) module to use decoders whose FCC is
not found from the hardcoded list. Improves compatibility and adds support
for common uncompressed formats decoders might prefer.
Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>

modules/codec/mft.c

@@ -92,6 +92,8 @@ struct decoder_sys_t
 
     const GUID* major_type;
     const GUID* subtype;
+    /* Container for a dynamically constructed subtype */
+    GUID custom_subtype;
 
     /* For asynchronous MFT */
     bool is_async;
@@ -183,6 +185,20 @@ static const pair_format_guid video_format_table[] =
     { 0, NULL }
 };
 
+// 8-bit luminance only
+DEFINE_MEDIATYPE_GUID (MFVideoFormat_L8, 50);
+
+/*
+ * Table to map MF Transform raw 3D3 output formats to native VLC FourCC
+ */
+static const pair_format_guid d3d_format_table[] = {
+    { VLC_CODEC_RGB32, &MFVideoFormat_RGB32  },
+    { VLC_CODEC_RGB24, &MFVideoFormat_RGB24  },
+    { VLC_CODEC_RGBA,  &MFVideoFormat_ARGB32 },
+    { VLC_CODEC_GREY,  &MFVideoFormat_L8     },
+    { 0, NULL }
+};
+
 #if defined(__MINGW64_VERSION_MAJOR) && __MINGW64_VERSION_MAJOR < 4
 DEFINE_GUID(MFAudioFormat_Dolby_AC3, 0xe06d802c, 0xdb46, 0x11cf, 0xb4, 0xd1, 0x00, 0x80, 0x5f, 0x6c, 0xbb, 0xea);
 #endif
@@ -210,6 +226,15 @@ static const GUID *FormatToGUID(const pair_format_guid table[], vlc_fourcc_t fou
     return NULL;
 }
 
+static vlc_fourcc_t GUIDToFormat(const pair_format_guid table[], const GUID* guid)
+{
+    for (int i = 0; table[i].fourcc; ++i)
+        if (IsEqualGUID(table[i].guid, guid))
+            return table[i].fourcc;
+
+    return 0;
+}
+
 /*
  * Low latency mode for Windows 8. Without this option, the H264
  * decoder will fill *all* its internal buffers before returning a
@@ -272,6 +297,16 @@ static int SetInputType(decoder_t *p_dec, DWORD stream_id, IMFMediaType **result
         hr = IMFMediaType_SetUINT64(input_media_type, &MF_MT_FRAME_SIZE, frame_size);
         if (FAILED(hr))
             goto error;
+
+        /* Some transforms like to know the frame rate and may reject the input type otherwise. */
+        UINT64 frame_ratio_num = p_dec->fmt_in.video.i_frame_rate;
+        UINT64 frame_ratio_dem = p_dec->fmt_in.video.i_frame_rate_base;
+        if(frame_ratio_num && frame_ratio_dem) {
+            UINT64 frame_rate = (frame_ratio_num << 32) | frame_ratio_dem;
+            hr = IMFMediaType_SetUINT64(input_media_type, &MF_MT_FRAME_RATE, frame_rate);
+            if(FAILED(hr))
+                goto error;
+        }
     }
     else
     {
@@ -356,7 +391,7 @@ static int SetOutputType(decoder_t *p_dec, DWORD stream_id, IMFMediaType **resul
      * preference thus we will use the first one unless YV12/I420 is
      * available for video or float32 for audio.
      */
-    int output_type_index = 0;
+    int output_type_index = -1;
     bool found = false;
     for (int i = 0; !found; ++i)
     {
@@ -380,6 +415,10 @@ static int SetOutputType(decoder_t *p_dec, DWORD stream_id, IMFMediaType **resul
         {
             if (IsEqualGUID(&subtype, &MFVideoFormat_YV12) || IsEqualGUID(&subtype, &MFVideoFormat_I420))
                 found = true;
+            /* Transform might offer output in a D3DFMT propietary FCC. If we can
+             * use it, fall back to it in case we do not find YV12 or I420 */
+            else if(output_type_index < 0 && GUIDToFormat(d3d_format_table, &subtype) > 0)
+                    output_type_index = i;
         }
         else
         {
@@ -399,9 +438,12 @@ static int SetOutputType(decoder_t *p_dec, DWORD stream_id, IMFMediaType **resul
     }
     /*
      * It's not an error if we don't find the output type we were
-     * looking for, in this case we use the first available type which
-     * is the "preferred" output type for this MFT.
+     * looking for, in this case we use the first available type.
      */
+    if(output_type_index < 0)
+        /* No output format found we prefer, just pick the first one preferred
+         * by the MFT */
+        output_type_index = 0;
 
     hr = IMFTransform_GetOutputAvailableType(p_sys->mft, stream_id, output_type_index, &output_media_type);
     if (FAILED(hr))
@@ -419,7 +461,17 @@ static int SetOutputType(decoder_t *p_dec, DWORD stream_id, IMFMediaType **resul
     if (p_dec->fmt_in.i_cat == VIDEO_ES)
     {
         video_format_Copy( &p_dec->fmt_out.video, &p_dec->fmt_in.video );
-        p_dec->fmt_out.i_codec = vlc_fourcc_GetCodec(p_dec->fmt_in.i_cat, subtype.Data1);
+
+        /* Transform might offer output in a D3DFMT propietary FCC */
+        vlc_fourcc_t fcc = GUIDToFormat(d3d_format_table, &subtype);
+        if(fcc) {
+            /* D3D formats are upside down */
+            p_dec->fmt_out.video.orientation = ORIENT_BOTTOM_LEFT;
+        } else {
+            fcc = vlc_fourcc_GetCodec(p_dec->fmt_in.i_cat, subtype.Data1);
+        }
+
+        p_dec->fmt_out.i_codec = fcc;
     }
     else
     {
@@ -1046,6 +1098,12 @@ static int FindMFT(decoder_t *p_dec)
         category = MFT_CATEGORY_VIDEO_DECODER;
         p_sys->major_type = &MFMediaType_Video;
         p_sys->subtype = FormatToGUID(video_format_table, p_dec->fmt_in.i_codec);
+        if(!p_sys->subtype) {
+            /* Codec is not well known. Construct a MF transform subtype from the fourcc */
+            p_sys->custom_subtype = MFVideoFormat_Base;
+            p_sys->custom_subtype.Data1 = p_dec->fmt_in.i_codec;
+            p_sys->subtype = &p_sys->custom_subtype;
+        }
     }
     else
     {