modules/codec/videotoolbox/decoder.c

@@ -83,13 +83,19 @@ struct frame_info_t
     picture_t *p_picture;
     int i_poc;
     int i_foc;
+    vlc_tick_t pts;
+    vlc_tick_t dts;
+    unsigned field_rate_num;
+    unsigned field_rate_den;
     bool b_flush;
     bool b_eos;
     bool b_keyframe;
+    bool b_leading;
     bool b_field;
     bool b_progressive;
     bool b_top_field_first;
     uint8_t i_num_ts;
+    uint8_t i_max_reorder;
     unsigned i_length;
     frame_info_t *p_next;
 };
@@ -99,6 +105,9 @@ struct frame_info_t
 #define H264_MAX_DPB 16
 #define VT_MAX_SEI_COUNT 16
 
+#define DEFAULT_FRAME_RATE_NUM 30000
+#define DEFAULT_FRAME_RATE_DEN 1001
+
 typedef struct decoder_sys_t
 {
     CMVideoCodecType            codec;
@@ -120,16 +129,28 @@ typedef struct decoder_sys_t
 
     /* !Codec specific callbacks */
 
-    bool                        b_vt_feed;
-    bool                        b_vt_flush;
-    bool                        b_vt_need_keyframe;
+    enum
+    {
+        STATE_BITSTREAM_WAITING_RAP = -2,
+        STATE_BITSTREAM_DISCARD_LEADING = -1,
+        STATE_BITSTREAM_SYNCED = 0,
+    } sync_state, start_sync_state;
+    enum
+    {
+        STATE_DECODER_WAITING_RAP = 0,
+        STATE_DECODER_STARTED,
+    } decoder_state;
     VTDecompressionSessionRef   session;
     CMVideoFormatDescriptionRef videoFormatDescription;
 
+    /* Decoder Callback Synchronization */
     vlc_mutex_t                 lock;
+    bool                        b_discard_decoder_output;
+
     frame_info_t               *p_pic_reorder;
     uint8_t                     i_pic_reorder;
     uint8_t                     i_pic_reorder_max;
+    bool                        b_strict_reorder;
     bool                        b_invalid_pic_reorder_max;
     bool                        b_poc_based_reorder;
 
@@ -142,7 +163,6 @@ typedef struct decoder_sys_t
 
     h264_poc_context_t          h264_pocctx;
     hevc_poc_ctx_t              hevc_pocctx;
-    bool                        b_drop_blocks;
     date_t                      pts;
 
     vlc_video_context          *vctx;
@@ -150,16 +170,79 @@ typedef struct decoder_sys_t
 } decoder_sys_t;
 
 /* Picture pacer to work-around the VT session allocating too much CVPX buffers
- * that can lead to a OOM. cf. pic_pacer_Wait usage in DecoderCallback() */
+ * that can lead to a OOM. */
 struct pic_pacer
 {
     vlc_mutex_t lock;
     vlc_cond_t  wait;
-    uint8_t     nb_field_out;
-    uint8_t     field_reorder_max;
+    uint8_t     nb_out;
+    uint8_t     allocated_max;
+    uint8_t     allocated_next;
+    uint8_t     queued_for_decode;
 };
 
-static void pic_pacer_UpdateReorderMax(struct pic_pacer *, uint8_t, uint8_t);
+#define PIC_PACER_ALLOCATABLE_MAX (1 /* callback, pre-reorder */ \
+                                 + 2 /* filters */ \
+                                 + 1 /* display */ \
+                                 + 1 /* next/prev display */)
+#define PIC_PACER_DECODE_QUEUE 4  /* max async decode before callback */
+//#define PIC_PACER_DEBUG
+static void pic_pacer_UpdateReorderMax(struct pic_pacer *, uint8_t);
+
+static void pic_pacer_Destroy(void *priv)
+{
+    (void) priv;
+}
+
+static void pic_pacer_Init(struct pic_pacer *pic_pacer)
+{
+    vlc_mutex_init(&pic_pacer->lock);
+    vlc_cond_init(&pic_pacer->wait);
+    pic_pacer->nb_out = 0;
+    pic_pacer->allocated_max = 6;
+    pic_pacer->allocated_next = pic_pacer->allocated_max;
+    pic_pacer->queued_for_decode = 0;
+}
+
+static void pic_pacer_AccountAllocation(struct pic_pacer *pic_pacer)
+{
+    vlc_mutex_lock(&pic_pacer->lock);
+    pic_pacer->nb_out += 1;
+    vlc_mutex_unlock(&pic_pacer->lock);
+}
+
+static void pic_pacer_AccountScheduledDecode(struct pic_pacer *pic_pacer)
+{
+    vlc_mutex_lock(&pic_pacer->lock);
+    pic_pacer->queued_for_decode += 1;
+    vlc_mutex_unlock(&pic_pacer->lock);
+}
+
+static void pic_pacer_AccountFinishedDecode(struct pic_pacer *pic_pacer)
+{
+    vlc_mutex_lock(&pic_pacer->lock);
+    pic_pacer->queued_for_decode -= 1;
+    vlc_cond_signal(&pic_pacer->wait);
+    vlc_mutex_unlock(&pic_pacer->lock);
+}
+
+static void pic_pacer_WaitAllocatableSlot(struct pic_pacer *pic_pacer)
+{
+    vlc_mutex_lock(&pic_pacer->lock);
+    uint8_t allocatable_total = pic_pacer->allocated_max + PIC_PACER_DECODE_QUEUE;
+
+    while( pic_pacer->queued_for_decode + pic_pacer->nb_out >= allocatable_total )
+    {
+#ifdef PIC_PACER_DEBUG
+        fprintf(stderr, "input pacing %d+%d >= %d\n",
+                pic_pacer->queued_for_decode, pic_pacer->nb_out, allocatable_total);
+#endif
+        vlc_cond_wait(&pic_pacer->wait, &pic_pacer->lock);
+        /*update*/
+        allocatable_total = pic_pacer->allocated_max + PIC_PACER_DECODE_QUEUE;
+    }
+    vlc_mutex_unlock(&pic_pacer->lock);
+}
 
 #pragma mark - start & stop
 
@@ -297,32 +380,18 @@ static bool FillReorderInfoH264(decoder_t *p_dec, const block_t *p_block,
 
                 p_info->i_num_ts = h264_get_num_ts(p_sps, &slice, sei.i_pic_struct,
                                                    p_info->i_foc, bFOC);
+                unsigned dummy;
+                h264_get_dpb_values(p_sps, &p_info->i_max_reorder, &dummy);
 
                 if (!p_info->b_progressive)
                     p_info->b_top_field_first = (sei.i_pic_struct % 2 == 1);
 
                 /* Set frame rate for timings in case of missing rate */
-                if ( (!p_dec->fmt_in->video.i_frame_rate_base ||
-                      !p_dec->fmt_in->video.i_frame_rate) &&
-                     p_sps->vui.i_time_scale && p_sps->vui.i_num_units_in_tick )
+                if (p_sps->vui.i_time_scale && p_sps->vui.i_num_units_in_tick)
                 {
-                    date_Change( &p_sys->pts, p_sps->vui.i_time_scale,
-                                              p_sps->vui.i_num_units_in_tick );
+                    p_info->field_rate_num = p_sps->vui.i_time_scale;
+                    p_info->field_rate_den = p_sps->vui.i_num_units_in_tick;
                 }
-
-                if(!p_sys->b_invalid_pic_reorder_max && i_nal_type == H264_NAL_SLICE_IDR)
-                {
-                    unsigned dummy;
-                    uint8_t i_reorder;
-                    h264_get_dpb_values(p_sps, &i_reorder, &dummy);
-                    vlc_mutex_lock(&p_sys->lock);
-                    p_sys->i_pic_reorder_max = i_reorder;
-                    pic_pacer_UpdateReorderMax(p_sys->pic_pacer,
-                                                  p_sys->i_pic_reorder_max,
-                                                  p_info->i_num_ts);
-                    vlc_mutex_unlock(&p_sys->lock);
-                }
-
             }
 
             return true; /* No need to parse further NAL */
@@ -625,27 +694,6 @@ static bool FillReorderInfoHEVC(decoder_t *p_dec, const block_t *p_block,
             if (!p_sli)
                 return false;
 
-            /* XXX: Work-around a VT bug on recent devices (iPhone X, MacBook
-             * Pro 2017). The VT session will report a BadDataErr if you send a
-             * RASL frame just after a CRA one. Indeed, RASL frames are
-             * corrupted if the decoding start at an IRAP frame (IDR/CRA), VT
-             * is likely failing to handle this case. */
-            if (!p_sys->b_vt_feed && (i_nal_type != HEVC_NAL_IDR_W_RADL &&
-                                      i_nal_type != HEVC_NAL_IDR_N_LP))
-                p_sys->b_drop_blocks = true;
-            else if (p_sys->b_drop_blocks)
-            {
-                if (i_nal_type == HEVC_NAL_RASL_N || i_nal_type == HEVC_NAL_RASL_R)
-                {
-                    hevc_rbsp_release_slice_header(p_sli);
-                    return false;
-                }
-                else
-                {
-                    p_sys->b_drop_blocks = false;
-                }
-            }
-
             p_info->b_keyframe = i_nal_type >= HEVC_NAL_BLA_W_LP;
             enum hevc_slice_type_e slice_type;
             if (hevc_get_slice_type(p_sli, &slice_type))
@@ -673,34 +721,30 @@ static bool FillReorderInfoHEVC(decoder_t *p_dec, const block_t *p_block,
                 p_info->i_poc = POC;
                 p_info->i_foc = POC; /* clearly looks wrong :/ */
                 p_info->i_num_ts = hevc_get_num_clock_ts(p_sps, sei.p_timing);
-                p_info->b_flush = (POC == 0);
+                p_info->i_max_reorder = hevc_get_max_num_reorder(p_vps);
+                p_info->b_flush = (POC == 0) ||
+                                  (i_nal_type >= HEVC_NAL_IDR_N_LP &&
+                                   i_nal_type <= HEVC_NAL_IRAP_VCL23);
                 p_info->b_field = (p_info->i_num_ts == 1);
                 p_info->b_progressive = hevc_frame_is_progressive(p_sps, sei.p_timing);
 
                 /* Set frame rate for timings in case of missing rate */
-                if ( (!p_dec->fmt_in->video.i_frame_rate_base ||
-                     !p_dec->fmt_in->video.i_frame_rate) )
-                {
-                    unsigned num, den;
-                    if (hevc_get_frame_rate(p_sps, p_vps, &num, &den))
-                        date_Change(&p_sys->pts, num, den);
-                }
+                if(hevc_get_frame_rate(p_sps, p_vps, &p_info->field_rate_num,
+                                                     &p_info->field_rate_den))
+                    p_info->field_rate_num *= 2;
 
                 if (sei.p_timing)
                     hevc_release_sei_pic_timing(sei.p_timing);
-
-                if (!p_sys->b_invalid_pic_reorder_max && p_vps)
-                {
-                    vlc_mutex_lock(&p_sys->lock);
-                    p_sys->i_pic_reorder_max = hevc_get_max_num_reorder(p_vps);
-                    pic_pacer_UpdateReorderMax(p_sys->pic_pacer,
-                                                  p_sys->i_pic_reorder_max,
-                                                  p_info->i_num_ts);
-                    vlc_mutex_unlock(&p_sys->lock);
-                }
-
             }
 
+            /* RASL Open GOP */
+            /* XXX: Work-around a VT bug on recent devices (iPhone X, MacBook
+             * Pro 2017). The VT session will report a BadDataErr if you send a
+             * RASL frame just after a CRA one. Indeed, RASL frames are
+             * corrupted if the decoding start at an IRAP frame (IDR/CRA), VT
+             * is likely failing to handle this case. */
+            p_info->b_leading = ( i_nal_type == HEVC_NAL_RASL_N || i_nal_type == HEVC_NAL_RASL_R );
+
             hevc_rbsp_release_slice_header(p_sli);
             return true; /* No need to parse further NAL */
         }
@@ -784,7 +828,7 @@ static void InsertIntoDPB(decoder_sys_t *p_sys, frame_info_t *p_info)
         else if (p_sys->b_poc_based_reorder)
             b_insert = ((*pp_lead_in)->i_foc > p_info->i_foc);
         else
-            b_insert = ((*pp_lead_in)->p_picture->date >= p_info->p_picture->date);
+            b_insert = ((*pp_lead_in)->pts >= p_info->pts);
 
         if (b_insert)
         {
@@ -801,36 +845,61 @@ static void InsertIntoDPB(decoder_sys_t *p_sys, frame_info_t *p_info)
 #endif
 }
 
-static picture_t * RemoveOneFrameFromDPB(decoder_sys_t *p_sys)
+static int RemoveOneFrameFromDPB(decoder_sys_t *p_sys, picture_t **pp_ret)
 {
     frame_info_t *p_info = p_sys->p_pic_reorder;
     if (p_info == NULL)
-        return NULL;
+    {
+        *pp_ret = NULL;
+        return VLC_EGENERIC;
+    }
 
     const int i_framepoc = p_info->i_poc;
+    const vlc_tick_t i_framepts = p_info->pts;
 
-    picture_t *p_ret = NULL;
-    picture_t **pp_ret_last = &p_ret;
+    picture_t **pp_ret_last = pp_ret;
     bool b_dequeue;
 
     do
     {
-        picture_t *p_field = p_info->p_picture;
+        /* Asynchronous fallback time init */
+        if(date_Get(&p_sys->pts) == VLC_TICK_INVALID)
+        {
+            date_Set(&p_sys->pts, p_info->pts != VLC_TICK_INVALID ?
+                                  p_info->pts : p_info->dts );
+        }
 
-        /* Compute time if missing */
-        if (p_field->date == VLC_TICK_INVALID)
-            p_field->date = date_Get(&p_sys->pts);
+        /* Compute time from output if missing */
+        if (p_info->pts == VLC_TICK_INVALID)
+            p_info->pts = date_Get(&p_sys->pts);
         else
-            date_Set(&p_sys->pts, p_field->date);
+            date_Set(&p_sys->pts, p_info->pts);
+
+        /* Update frame rate (used on interpolation) */
+        if(p_info->field_rate_num != p_sys->pts.i_divider_num ||
+           p_info->field_rate_den != p_sys->pts.i_divider_den)
+        {
+            /* no date_Change due to possible invalid num */
+            date_Init(&p_sys->pts, p_info->field_rate_num,
+                                   p_info->field_rate_den);
+            date_Set(&p_sys->pts, p_info->pts);
+        }
 
         /* Set next picture time, in case it is missing */
         if (p_info->i_length)
-            date_Set(&p_sys->pts, p_field->date + p_info->i_length);
+            date_Set(&p_sys->pts, p_info->pts + p_info->i_length);
         else
             date_Increment(&p_sys->pts, p_info->i_num_ts);
 
-        *pp_ret_last = p_field;
-        pp_ret_last = &p_field->p_next;
+        if( p_info->p_picture ) /* Can have no picture attached to entry on error */
+        {
+            if( p_info->p_picture->date == VLC_TICK_INVALID )
+                p_info->p_picture->date = p_info->pts;
+
+            /* Extract attached field to output list */
+            *pp_ret_last = p_info->p_picture;
+            pp_ret_last = &p_info->p_picture->p_next;
+        }
 
         p_sys->i_pic_reorder--;
 
@@ -843,7 +912,7 @@ static picture_t * RemoveOneFrameFromDPB(decoder_sys_t *p_sys)
             if (p_sys->b_poc_based_reorder)
                 b_dequeue = (p_info->i_poc == i_framepoc);
             else
-                b_dequeue = (p_field->date == p_info->p_picture->date);
+                b_dequeue = (p_info->pts == i_framepts);
         }
         else
         {
@@ -852,7 +921,7 @@ static picture_t * RemoveOneFrameFromDPB(decoder_sys_t *p_sys)
 
     } while(b_dequeue);
 
-    return p_ret;
+    return VLC_SUCCESS;
 }
 
 static void DrainDPBLocked(decoder_t *p_dec, bool flush)
@@ -860,10 +929,10 @@ static void DrainDPBLocked(decoder_t *p_dec, bool flush)
     decoder_sys_t *p_sys = p_dec->p_sys;
     for ( ;; )
     {
-        picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
-        if (p_fields == NULL)
+        picture_t *p_fields;
+        if(RemoveOneFrameFromDPB(p_sys, &p_fields) != VLC_SUCCESS)
             break;
-        do
+        for ( ; p_fields; )
         {
             picture_t *p_next = p_fields->p_next;
             p_fields->p_next = NULL;
@@ -872,7 +941,7 @@ static void DrainDPBLocked(decoder_t *p_dec, bool flush)
             else
                 decoder_QueueVideo(p_dec, p_fields);
             p_fields = p_next;
-        } while(p_fields != NULL);
+        }
     }
 }
 
@@ -899,59 +968,74 @@ static frame_info_t * CreateReorderInfo(decoder_t *p_dec, const block_t *p_block
         p_info->b_keyframe = true;
     }
 
+    p_info->dts = p_block->i_dts;
+    p_info->pts = p_block->i_pts;
     p_info->i_length = p_block->i_length;
 
+    if(p_dec->fmt_in->video.i_frame_rate && p_dec->fmt_in->video.i_frame_rate_base) /* demux forced rate */
+    {
+        p_info->field_rate_num = p_dec->fmt_in->video.i_frame_rate * 2;
+        p_info->field_rate_den = p_dec->fmt_in->video.i_frame_rate_base;
+    }
+    else if(!p_info->field_rate_num || !p_info->field_rate_den) /* full fallback */
+    {
+        p_info->field_rate_num = DEFAULT_FRAME_RATE_NUM * 2;
+        p_info->field_rate_den = DEFAULT_FRAME_RATE_DEN;
+    }
+
     /* required for still pictures/menus */
     p_info->b_eos = (p_block->i_flags & BLOCK_FLAG_END_OF_SEQUENCE);
 
-    if (date_Get(&p_sys->pts) == VLC_TICK_INVALID)
-        date_Set(&p_sys->pts, p_block->i_dts);
-
     return p_info;
 }
 
 static void OnDecodedFrame(decoder_t *p_dec, frame_info_t *p_info)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
-    assert(p_info->p_picture);
+
+    if(!p_sys->b_invalid_pic_reorder_max &&
+       p_info->i_max_reorder != p_sys->i_pic_reorder_max)
+    {
+        p_sys->i_pic_reorder_max = p_info->i_max_reorder;
+        pic_pacer_UpdateReorderMax(p_sys->pic_pacer, p_sys->i_pic_reorder_max);
+    }
+
     while(p_info->b_flush || p_sys->i_pic_reorder >= p_sys->i_pic_reorder_max)
     {
         /* First check if DPB sizing was correct before removing one frame */
-        if (p_sys->p_pic_reorder && !p_info->b_flush &&
+        if (p_sys->p_pic_reorder && !p_sys->b_strict_reorder && !p_info->b_flush &&
             p_sys->i_pic_reorder_max < H264_MAX_DPB)
         {
             if (p_sys->b_poc_based_reorder && p_sys->p_pic_reorder->i_foc > p_info->i_foc)
             {
                 p_sys->b_invalid_pic_reorder_max = true;
                 p_sys->i_pic_reorder_max++;
-                pic_pacer_UpdateReorderMax(p_sys->pic_pacer,
-                                              p_sys->i_pic_reorder_max, p_info->i_num_ts);
+                pic_pacer_UpdateReorderMax(p_sys->pic_pacer, p_sys->i_pic_reorder_max);
                 msg_Dbg(p_dec, "Raising max DPB to %"PRIu8, p_sys->i_pic_reorder_max);
                 break;
             }
             else if (!p_sys->b_poc_based_reorder &&
-                     p_info->p_picture->date > VLC_TICK_INVALID &&
-                     p_sys->p_pic_reorder->p_picture->date > p_info->p_picture->date)
+                     p_info->pts > VLC_TICK_INVALID &&
+                     p_sys->p_pic_reorder->pts > p_info->pts)
             {
                 p_sys->b_invalid_pic_reorder_max = true;
                 p_sys->i_pic_reorder_max++;
-                pic_pacer_UpdateReorderMax(p_sys->pic_pacer,
-                                              p_sys->i_pic_reorder_max, p_info->i_num_ts);
+                pic_pacer_UpdateReorderMax(p_sys->pic_pacer, p_sys->i_pic_reorder_max);
                 msg_Dbg(p_dec, "Raising max DPB to %"PRIu8, p_sys->i_pic_reorder_max);
                 break;
             }
         }
 
-        picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
-        if (p_fields == NULL)
+        picture_t *p_fields;
+        if(RemoveOneFrameFromDPB(p_sys, &p_fields) != VLC_SUCCESS)
             break;
-        do
+        for(; p_fields;)
         {
             picture_t *p_next = p_fields->p_next;
             p_fields->p_next = NULL;
             decoder_QueueVideo(p_dec, p_fields);
             p_fields = p_next;
-        } while(p_fields != NULL);
+        }
     }
 
     InsertIntoDPB(p_sys, p_info);
@@ -1132,21 +1216,6 @@ static CFMutableDictionaryRef CreateSessionDescriptionFormat(decoder_t *p_dec,
     return decoderConfiguration;
 }
 
-static void PtsInit(decoder_t *p_dec)
-{
-    decoder_sys_t *p_sys = p_dec->p_sys;
-
-    if (p_dec->fmt_in->video.i_frame_rate_base && p_dec->fmt_in->video.i_frame_rate)
-    {
-        date_Init(&p_sys->pts, p_dec->fmt_in->video.i_frame_rate * 2,
-                  p_dec->fmt_in->video.i_frame_rate_base);
-    }
-    else
-    {
-        date_Init(&p_sys->pts, 2 * 30000, 1001);
-    }
-}
-
 static int StartVideoToolbox(decoder_t *p_dec)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
@@ -1268,25 +1337,12 @@ static void StopVideoToolbox(decoder_t *p_dec)
         CFRelease(p_sys->videoFormatDescription);
         p_sys->videoFormatDescription = NULL;
     }
-    p_sys->b_vt_feed = false;
-    p_sys->b_drop_blocks = false;
+    p_sys->sync_state = p_sys->start_sync_state;
+    p_sys->decoder_state = STATE_DECODER_WAITING_RAP;
 }
 
 #pragma mark - module open and close
 
-static void pic_pacer_Destroy(void *priv)
-{
-    (void) priv;
-}
-
-static void pic_pacer_Init(struct pic_pacer *pic_pacer, uint8_t pic_reorder_max)
-{
-    vlc_mutex_init(&pic_pacer->lock);
-    vlc_cond_init(&pic_pacer->wait);
-    pic_pacer->nb_field_out = 0;
-    pic_pacer->field_reorder_max = pic_reorder_max * 2;
-}
-
 static int
 CreateVideoContext(decoder_t *p_dec)
 {
@@ -1320,7 +1376,7 @@ CreateVideoContext(decoder_t *p_dec)
                                          CVPX_VIDEO_CONTEXT_VIDEOTOOLBOX);
     assert(p_sys->pic_pacer);
 
-    pic_pacer_Init(p_sys->pic_pacer, p_sys->i_pic_reorder_max);
+    pic_pacer_Init(p_sys->pic_pacer);
 
     return VLC_SUCCESS;
 }
@@ -1369,6 +1425,9 @@ static int OpenDecoder(vlc_object_t *p_this)
     p_sys->i_pic_reorder_max = 4;
     p_sys->vtsession_status = VTSESSION_STATUS_OK;
     p_sys->b_cvpx_format_forced = false;
+    /* will be fixed later */
+    date_Init(&p_sys->pts, p_dec->fmt_in->video.i_frame_rate,
+                           p_dec->fmt_in->video.i_frame_rate_base);
 
     char *cvpx_chroma = var_InheritString(p_dec, "videotoolbox-cvpx-chroma");
     if (cvpx_chroma != NULL)
@@ -1402,8 +1461,6 @@ static int OpenDecoder(vlc_object_t *p_this)
         return i_ret;
     }
 
-    p_sys->b_vt_need_keyframe = false;
-
     vlc_mutex_init(&p_sys->lock);
 
     p_dec->pf_decode = DecodeBlock;
@@ -1421,8 +1478,9 @@ static int OpenDecoder(vlc_object_t *p_this)
             p_sys->pf_configure_vout = ConfigureVoutH264;
             p_sys->pf_copy_extradata = CopyDecoderExtradataH264;
             p_sys->pf_fill_reorder_info = FillReorderInfoH264;
+            p_sys->b_strict_reorder = false;
             p_sys->b_poc_based_reorder = true;
-            p_sys->b_vt_need_keyframe = true;
+            p_sys->start_sync_state = STATE_BITSTREAM_WAITING_RAP;
             break;
 
         case kCMVideoCodecType_HEVC:
@@ -1435,8 +1493,9 @@ static int OpenDecoder(vlc_object_t *p_this)
             p_sys->pf_configure_vout = ConfigureVoutHEVC;
             p_sys->pf_copy_extradata = CopyDecoderExtradataHEVC;
             p_sys->pf_fill_reorder_info = FillReorderInfoHEVC;
+            p_sys->b_strict_reorder = true;
             p_sys->b_poc_based_reorder = true;
-            p_sys->b_vt_need_keyframe = true;
+            p_sys->start_sync_state = STATE_BITSTREAM_WAITING_RAP;
             break;
 
         case kCMVideoCodecType_MPEG4Video:
@@ -1448,6 +1507,8 @@ static int OpenDecoder(vlc_object_t *p_this)
             break;
     }
 
+    p_sys->sync_state = p_sys->start_sync_state;
+
     if (p_sys->pf_codec_init && !p_sys->pf_codec_init(p_dec))
     {
         CloseDecoder(p_this);
@@ -1461,7 +1522,7 @@ static int OpenDecoder(vlc_object_t *p_this)
 
     i_ret = StartVideoToolbox(p_dec);
     if (i_ret == VLC_SUCCESS) {
-        PtsInit(p_dec);
+        date_Set(&p_sys->pts, VLC_TICK_INVALID);
         msg_Info(p_dec, "Using Video Toolbox to decode '%4.4s'",
                         (char *)&p_dec->fmt_in->i_codec);
     } else {
@@ -1769,10 +1830,8 @@ static int HandleVTStatus(decoder_t *p_dec, OSStatus status,
 static void RequestFlush(decoder_t *p_dec)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
-
-    vlc_mutex_lock(&p_sys->lock);
-    p_sys->b_vt_flush = true;
-    vlc_mutex_unlock(&p_sys->lock);
+    Drain(p_dec, true);
+    date_Set(&p_sys->pts, VLC_TICK_INVALID);
 }
 
 static void Drain(decoder_t *p_dec, bool flush)
@@ -1781,18 +1840,18 @@ static void Drain(decoder_t *p_dec, bool flush)
 
     /* draining: return last pictures of the reordered queue */
     vlc_mutex_lock(&p_sys->lock);
-    p_sys->b_vt_flush = true;
-    DrainDPBLocked(p_dec, flush);
+    p_sys->b_discard_decoder_output = flush;
     vlc_mutex_unlock(&p_sys->lock);
 
-    if (p_sys->session && p_sys->b_vt_feed)
+    if (p_sys->session && p_sys->decoder_state == STATE_DECODER_STARTED)
         VTDecompressionSessionWaitForAsynchronousFrames(p_sys->session);
 
     vlc_mutex_lock(&p_sys->lock);
-    assert(RemoveOneFrameFromDPB(p_sys) == NULL);
-    p_sys->b_vt_flush = false;
-    p_sys->b_vt_feed = false;
-    p_sys->b_drop_blocks = false;
+    DrainDPBLocked(p_dec, flush);
+    picture_t *p_output;
+    assert(RemoveOneFrameFromDPB(p_sys, &p_output) == VLC_EGENERIC);
+    p_sys->b_discard_decoder_output = false;
+    p_sys->sync_state = p_sys->start_sync_state;
     vlc_mutex_unlock(&p_sys->lock);
 }
 
@@ -1800,11 +1859,7 @@ static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
 
-    if (p_sys->b_vt_flush)
-    {
-        Drain(p_dec, true);
-        PtsInit(p_dec);
-    }
+    pic_pacer_WaitAllocatableSlot(p_sys->pic_pacer);
 
     if (p_block == NULL)
     {
@@ -1902,10 +1957,10 @@ static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
 
     if (unlikely(p_block->i_flags&(BLOCK_FLAG_CORRUPTED)))
     {
-        if (p_sys->b_vt_feed)
+        if (p_sys->sync_state == STATE_BITSTREAM_SYNCED)
         {
             Drain(p_dec, false);
-            PtsInit(p_dec);
+            date_Set(&p_sys->pts, VLC_TICK_INVALID);
         }
         goto skip;
     }
@@ -1954,10 +2009,25 @@ static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
         }
     }
 
-    if (!p_sys->b_vt_feed && p_sys->b_vt_need_keyframe && !p_info->b_keyframe)
+    if (p_sys->sync_state == STATE_BITSTREAM_WAITING_RAP)
     {
-        free(p_info);
-        goto skip;
+        if(!p_info->b_keyframe)
+        {
+            msg_Dbg(p_dec, "discarding non recovery frame %"PRId64, p_info->pts);
+            free(p_info);
+            goto skip;
+        }
+        p_sys->sync_state = STATE_BITSTREAM_DISCARD_LEADING;
+    }
+    else if(p_sys->sync_state == STATE_BITSTREAM_DISCARD_LEADING)
+    {
+        if(p_info->b_leading)
+        {
+            msg_Dbg(p_dec, "discarding skipped leading frame %"PRId64, p_info->pts);
+            free(p_info);
+            goto skip;
+        }
+        p_sys->sync_state = STATE_BITSTREAM_SYNCED;
     }
 
     CMSampleBufferRef sampleBuffer =
@@ -1978,12 +2048,20 @@ static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
     enum vtsession_status vtsession_status;
     if (HandleVTStatus(p_dec, status, &vtsession_status) == VLC_SUCCESS)
     {
-        p_sys->b_vt_feed = true;
+        pic_pacer_AccountScheduledDecode(p_sys->pic_pacer);
+
+        if(p_sys->decoder_state != STATE_DECODER_STARTED)
+        {
+            msg_Dbg(p_dec, "session accepted first frame %"PRId64, p_info->pts);
+            p_sys->decoder_state = STATE_DECODER_STARTED;
+        }
         if (p_block->i_flags & BLOCK_FLAG_END_OF_SEQUENCE)
             Drain( p_dec, false );
     }
     else
     {
+        msg_Dbg(p_dec, "session rejected frame %"PRId64" with status %d", p_info->pts, status);
+        p_sys->sync_state = p_sys->start_sync_state;
         vlc_mutex_lock(&p_sys->lock);
         p_sys->vtsession_status = vtsession_status;
         /* In case of abort, the decoder module will be reloaded next time
@@ -2080,46 +2158,45 @@ video_context_OnPicReleased(vlc_video_context *vctx, unsigned nb_fields)
 {
     struct pic_pacer *pic_pacer =
         vlc_video_context_GetCVPXPrivate(vctx, CVPX_VIDEO_CONTEXT_VIDEOTOOLBOX);
+    VLC_UNUSED(nb_fields);
 
     vlc_mutex_lock(&pic_pacer->lock);
-    assert((int) pic_pacer->nb_field_out - nb_fields >= 0);
-    pic_pacer->nb_field_out -= nb_fields;
-    vlc_cond_signal(&pic_pacer->wait);
-    vlc_mutex_unlock(&pic_pacer->lock);
-}
+    assert(pic_pacer->nb_out > 0);
+    pic_pacer->nb_out -= 1;
 
-static void
-pic_pacer_UpdateReorderMax(struct pic_pacer *pic_pacer, uint8_t pic_reorder_max,
-                              uint8_t nb_field)
-{
-    vlc_mutex_lock(&pic_pacer->lock);
+    /* our shrink condition */
+    if(pic_pacer->allocated_next < pic_pacer->allocated_max &&
+       pic_pacer->nb_out <= pic_pacer->allocated_next)
+        pic_pacer->allocated_max = pic_pacer->allocated_next;
 
-    pic_pacer->field_reorder_max = pic_reorder_max * (nb_field < 2 ? 2 : nb_field);
     vlc_cond_signal(&pic_pacer->wait);
 
     vlc_mutex_unlock(&pic_pacer->lock);
 }
 
-static int pic_pacer_Wait(struct pic_pacer *pic_pacer, const picture_t *pic)
+static void
+pic_pacer_UpdateReorderMax(struct pic_pacer *pic_pacer, uint8_t pic_reorder_max)
 {
-    const uint8_t reserved_fields = 2 * (pic->i_nb_fields < 2 ? 2 : pic->i_nb_fields);
-
     vlc_mutex_lock(&pic_pacer->lock);
 
-    /* Wait 200 ms max. We can't really know what the video output will do with
-     * output pictures (will they be rendered immediately ?), so don't wait
-     * infinitely. The output will be paced anyway by the vlc_cond_timedwait()
-     * call. */
-    vlc_tick_t deadline = vlc_tick_now() + VLC_TICK_FROM_MS(200);
-    int ret = 0;
-    while (ret == 0 && pic_pacer->field_reorder_max != 0
-        && pic_pacer->nb_field_out >= pic_pacer->field_reorder_max + reserved_fields)
-        ret = vlc_cond_timedwait(&pic_pacer->wait, &pic_pacer->lock, deadline);
-    pic_pacer->nb_field_out += pic->i_nb_fields;
+    pic_reorder_max += PIC_PACER_ALLOCATABLE_MAX;
+    bool b_growing  = pic_reorder_max > pic_pacer->allocated_max;
+#ifdef PIC_PACER_DEBUG
+    fprintf(stderr, "updating pacer max %d/%d to %d\n",
+            pic_pacer->nb_out, pic_pacer->allocated_max, pic_reorder_max);
+#endif
+    if(b_growing)
+    {
+        pic_pacer->allocated_max = pic_reorder_max;
+        pic_pacer->allocated_next = pic_reorder_max;
+        vlc_cond_signal(&pic_pacer->wait);
+    }
+    else
+    {
+        pic_pacer->allocated_next = pic_reorder_max;
+    }
 
     vlc_mutex_unlock(&pic_pacer->lock);
-
-    return ret;
 }
 
 static void DecoderCallback(void *decompressionOutputRefCon,
@@ -2136,7 +2213,7 @@ static void DecoderCallback(void *decompressionOutputRefCon,
     frame_info_t *p_info = (frame_info_t *) sourceFrameRefCon;
 
     vlc_mutex_lock(&p_sys->lock);
-    if (p_sys->b_vt_flush)
+    if (p_sys->b_discard_decoder_output)
         goto end;
 
     enum vtsession_status vtsession_status;
@@ -2177,19 +2254,17 @@ static void DecoderCallback(void *decompressionOutputRefCon,
     if (CVPixelBufferGetDataSize(imageBuffer) == 0)
         goto end;
 
-    if (likely(p_info))
-    {
-        /* Unlock the mutex because decoder_NewPicture() is blocking. Indeed,
-         * it can wait indefinitely when the input is paused. */
-
-        vlc_mutex_unlock(&p_sys->lock);
+    if (unlikely(p_info == NULL))
+        goto end;
 
-        picture_t *p_pic = decoder_NewPicture(p_dec);
-        if (!p_pic)
-            goto unlocked_end;
+    /* Unlock the mutex because decoder_NewPicture() is blocking. Indeed,
+         * it can wait indefinitely when the input is paused. */
 
-        p_info->p_picture = p_pic;
+    vlc_mutex_unlock(&p_sys->lock);
 
+    picture_t *p_pic = decoder_NewPicture(p_dec);
+    if (p_pic)
+    {
         p_pic->date = pts.value;
         p_pic->b_force = p_info->b_eos;
         p_pic->b_still = p_info->b_eos;
@@ -2201,36 +2276,31 @@ static void DecoderCallback(void *decompressionOutputRefCon,
         }
 
         if (cvpxpic_attach(p_pic, imageBuffer, p_sys->vctx,
-                           video_context_OnPicReleased) != VLC_SUCCESS)
+                           video_context_OnPicReleased) == VLC_SUCCESS)
         {
-            picture_Release(p_pic);
-            goto unlocked_end;
+            /* VT is not pacing frame allocation. If we are not fast enough to
+                 * render (release) the output pictures, the VT session can end up
+                 * allocating way too many frames. This can be problematic for 4K
+                 * 10bits. To fix this issue, we ensure that we don't have too many
+                 * output frames allocated by waiting for the vout to release them. */
+            pic_pacer_AccountAllocation(p_sys->pic_pacer);
         }
+    }
 
-        /* VT is not pacing frame allocation. If we are not fast enough to
-         * render (release) the output pictures, the VT session can end up
-         * allocating way too many frames. This can be problematic for 4K
-         * 10bits. To fix this issue, we ensure that we don't have too many
-         * output frames allocated by waiting for the vout to release them. */
-        if (pic_pacer_Wait(p_sys->pic_pacer, p_pic))
-            msg_Warn(p_dec, "pic_pacer_Wait timed out");
-
-        vlc_mutex_lock(&p_sys->lock);
+    vlc_mutex_lock(&p_sys->lock);
 
-        if (p_sys->b_vt_flush)
-        {
-            picture_Release(p_pic);
-            goto end;
-        }
+    if (p_sys->b_discard_decoder_output)
+        picture_Release(p_pic);
+    else
+        p_info->p_picture = p_pic;
 
-        OnDecodedFrame( p_dec, p_info );
-        p_info = NULL;
-    }
+    OnDecodedFrame( p_dec, p_info );
+    p_info = NULL;
 
 end:
-    vlc_mutex_unlock(&p_sys->lock);
-unlocked_end:
     free(p_info);
+    vlc_mutex_unlock(&p_sys->lock);
+    pic_pacer_AccountFinishedDecode(p_sys->pic_pacer);
     return;
 }