modules/codec/avcodec/vaapi.c

@@ -57,74 +57,6 @@ struct vaapi_vctx
     vlc_sem_t pool_sem;
 };
 
-static int GetVaProfile(const AVCodecContext *ctx, const es_format_t *fmt_in,
-                        VAProfile *va_profile, int *vlc_chroma,
-                        unsigned *pic_count)
-{
-    VAProfile i_profile;
-    unsigned count = 3;
-    int i_vlc_chroma = VLC_CODEC_VAAPI_420;
-
-    switch(ctx->codec_id)
-    {
-    case AV_CODEC_ID_MPEG1VIDEO:
-    case AV_CODEC_ID_MPEG2VIDEO:
-        i_profile = VAProfileMPEG2Main;
-        count = 4;
-        break;
-    case AV_CODEC_ID_MPEG4:
-        i_profile = VAProfileMPEG4AdvancedSimple;
-        break;
-    case AV_CODEC_ID_WMV3:
-        i_profile = VAProfileVC1Main;
-        break;
-    case AV_CODEC_ID_VC1:
-        i_profile = VAProfileVC1Advanced;
-        break;
-    case AV_CODEC_ID_H264:
-        i_profile = VAProfileH264High;
-        count = 18;
-        break;
-    case AV_CODEC_ID_HEVC:
-        if (fmt_in->i_profile == FF_PROFILE_HEVC_MAIN)
-            i_profile = VAProfileHEVCMain;
-        else if (fmt_in->i_profile == FF_PROFILE_HEVC_MAIN_10)
-        {
-            i_profile = VAProfileHEVCMain10;
-            i_vlc_chroma = VLC_CODEC_VAAPI_420_10BPP;
-        }
-        else
-            return VLC_EGENERIC;
-        count = 18;
-        break;
-    case AV_CODEC_ID_VP8:
-        i_profile = VAProfileVP8Version0_3;
-        count = 5;
-        break;
-    case AV_CODEC_ID_VP9:
-        if (ctx->profile == FF_PROFILE_VP9_0)
-            i_profile = VAProfileVP9Profile0;
-#if VA_CHECK_VERSION( 0, 39, 0 )
-        else if (ctx->profile == FF_PROFILE_VP9_2)
-        {
-            i_profile = VAProfileVP9Profile2;
-            i_vlc_chroma = VLC_CODEC_VAAPI_420_10BPP;
-        }
-#endif
-        else
-            return VLC_EGENERIC;
-        count = 10;
-        break;
-    default:
-        return VLC_EGENERIC;
-    }
-
-    *va_profile = i_profile;
-    *pic_count = count + ctx->thread_count;
-    *vlc_chroma = i_vlc_chroma;
-    return VLC_SUCCESS;
-}
-
 typedef struct {
     struct vaapi_pic_context ctx;
     AVFrame *avframe;
@@ -229,16 +161,11 @@ static int Create(vlc_va_t *va, AVCodecContext *ctx, enum AVPixelFormat hwfmt,
                   video_format_t *fmt_out, vlc_video_context **vtcx_out)
 {
     VLC_UNUSED(desc);
+    VLC_UNUSED(fmt_in);
     if ( hwfmt != AV_PIX_FMT_VAAPI || dec_device == NULL ||
         dec_device->type != VLC_DECODER_DEVICE_VAAPI)
         return VLC_EGENERIC;
 
-    VAProfile i_profile;
-    unsigned count;
-    int i_vlc_chroma;
-    if (GetVaProfile(ctx, fmt_in, &i_profile, &i_vlc_chroma, &count) != VLC_SUCCESS)
-        return VLC_EGENERIC;
-
     VADisplay va_dpy = dec_device->opaque;
 
     AVBufferRef *hwdev_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
@@ -282,6 +209,34 @@ static int Create(vlc_va_t *va, AVCodecContext *ctx, enum AVPixelFormat hwfmt,
         return VLC_EGENERIC;
     }
 
+    int vlc_chroma = 0;
+    if (hwframes_ctx->format == AV_PIX_FMT_VAAPI)
+    {
+        switch (hwframes_ctx->sw_format)
+        {
+            case AV_PIX_FMT_YUV420P:
+            case AV_PIX_FMT_NV12:
+                vlc_chroma = VLC_CODEC_VAAPI_420;
+                break;
+            case AV_PIX_FMT_P010LE:
+            case AV_PIX_FMT_P010BE:
+            case AV_PIX_FMT_YUV420P10BE:
+            case AV_PIX_FMT_YUV420P10LE:
+                vlc_chroma = VLC_CODEC_VAAPI_420_10BPP;
+                break;
+            default:
+                break;
+        }
+    }
+
+    if (vlc_chroma == 0)
+    {
+        msg_Warn(va, "ffmpeg chroma not compatible with vlc: hw: %d, sw: %d",
+                 hwframes_ctx->format, hwframes_ctx->sw_format);
+        av_buffer_unref(&hwframes_ref);
+        return VLC_EGENERIC;
+    }
+
     ctx->hw_frames_ctx = av_buffer_ref(hwframes_ref);
     if (!ctx->hw_frames_ctx)
     {
@@ -308,7 +263,7 @@ static int Create(vlc_va_t *va, AVCodecContext *ctx, enum AVPixelFormat hwfmt,
 
     msg_Info(va, "Using %s", vaQueryVendorString(va_dpy));
 
-    fmt_out->i_chroma = i_vlc_chroma;
+    fmt_out->i_chroma = vlc_chroma;
 
     va->ops = &ops;
     va->sys = vctx;