avcodec: video: fix swfmt detection when using a hw_device_ctx

(cherry picked from commit 906ffbaa) (edited)
edited:
- 3.0 uses PixelFormat
- the code to handle b_hardware_only was already merged in 3.0

Signed-off-by: Steve Lhomme <robux4@ycbcr.xyz>

modules/codec/avcodec/video.c

@@ -1725,7 +1725,8 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
     video_format_t fmt;
 
     /* Enumerate available formats */
-    enum PixelFormat swfmt = avcodec_default_get_format(p_context, pi_fmt);
+    enum PixelFormat defaultfmt = avcodec_default_get_format(p_context, pi_fmt);
+    enum PixelFormat swfmt = AV_PIX_FMT_NONE;
     bool can_hwaccel = false;
 
     for (size_t i = 0; pi_fmt[i] != AV_PIX_FMT_NONE; i++)
@@ -1738,13 +1739,22 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
         msg_Dbg( p_dec, "available %sware decoder output format %d (%s)",
                  hwaccel ? "hard" : "soft", pi_fmt[i], dsc->name );
         if (hwaccel)
+        {
+            /* The default fmt is a hw format, it can happen with some va
+             * implementations (when using a hw_device_ctx). */
+            if (defaultfmt == pi_fmt[i])
+                defaultfmt = AV_PIX_FMT_NONE;
+
             can_hwaccel = true;
+        }
+        else if (swfmt == AV_PIX_FMT_NONE && !p_sys->b_hardware_only)
+            swfmt = pi_fmt[i];
     }
 
     /* Use the default fmt in priority of any sw fmt if the default fmt is a hw
      * one */
-    if (p_sys->pix_fmt != AV_PIX_FMT_NONE && !p_sys->b_hardware_only)
-        swfmt = p_sys->pix_fmt;
+    if (defaultfmt != AV_PIX_FMT_NONE && !p_sys->b_hardware_only)
+        swfmt = defaultfmt;
 
     if (p_sys->pix_fmt == AV_PIX_FMT_NONE)
         goto no_reuse;