Commit edc1135e authored by Fiona Glaser's avatar Fiona Glaser
Browse files

Interactive encoder control: error resilience

In low-latency streaming with few clients, it is often feasible to modify encoder behavior in some fashion based on feedback from clients.
One possible application of this is error resilience: if a packet is lost, mark the associated frame (and any referenced from it) as lost.
This allows quick recovery from errors with minimal expense bit-wise.

The new i_dpb_size parameter allows a calling application to tell x264 to use a larger DPB size than required by the number of reference frames.
This lets x264 and the client keep a large buffer of old references to fall back to in case of lost frames.
If no recovery is possible even with the available buffer, x264 will force a keyframe.

This initial version does not support B-frames or intra refresh.
Recommended usage is to set keyint to a very large value, so that keyframes do not occur except as necessary for extreme error recovery.

Full documentation is in x264.h.

Move DTS/PTS calculation to before encoding each frame instead of after.
Improve documentation of x264_encoder_intra_refresh.
parent 669cc1de
......@@ -634,6 +634,8 @@ int x264_param_parse( x264_param_t *p, const char *name, const char *value )
}
OPT2("ref", "frameref")
p->i_frame_reference = atoi(value);
OPT("dpb-size")
p->i_dpb_size = atoi(value);
OPT("keyint")
{
p->i_keyint_max = atoi(value);
......
......@@ -421,6 +421,8 @@ struct x264_t
int i_cpb_delay_lookahead;
int b_queued_intra_refresh;
int64_t i_reference_invalidate_pts;
int64_t i_last_idr_pts;
/* We use only one SPS and one PPS */
x264_sps_t sps_array[1];
......
......@@ -443,6 +443,7 @@ x264_frame_t *x264_frame_pop_unused( x264_t *h, int b_fdec )
frame->b_intra_calculated = 0;
frame->b_scenecut = 1;
frame->b_keyframe = 0;
frame->b_corrupt = 0;
memset( frame->weight, 0, sizeof(frame->weight) );
memset( frame->f_weighted_cost_delta, 0, sizeof(frame->f_weighted_cost_delta) );
......
......@@ -35,6 +35,7 @@ typedef struct x264_frame
int i_type;
int i_qpplus1;
int64_t i_pts;
int64_t i_dts;
int64_t i_reordered_pts;
int i_duration; /* in SPS time_scale units (i.e 2 * timebase units) used for vfr */
int i_cpb_duration;
......@@ -143,6 +144,9 @@ typedef struct x264_frame
int i_pir_start_col;
int i_pir_end_col;
int i_frames_since_pir;
/* interactive encoder control */
int b_corrupt;
} x264_frame_t;
/* synchronized frame list */
......
......@@ -409,12 +409,16 @@ void x264_mb_predict_mv_ref16x16( x264_t *h, int i_list, int i_ref, int16_t mvc[
if( i_ref == 0 && h->frames.b_have_lowres )
{
int16_t (*lowres_mv)[2] = i_list ? h->fenc->lowres_mvs[1][h->fref1[0]->i_frame-h->fenc->i_frame-1]
: h->fenc->lowres_mvs[0][h->fenc->i_frame-h->fref0[0]->i_frame-1];
if( lowres_mv[0][0] != 0x7fff )
int idx = i_list ? h->fref1[0]->i_frame-h->fenc->i_frame-1
: h->fenc->i_frame-h->fref0[0]->i_frame-1;
if( idx <= h->param.i_bframe )
{
M32( mvc[i] ) = (M32( lowres_mv[h->mb.i_mb_xy] )*2)&0xfffeffff;
i++;
int16_t (*lowres_mv)[2] = h->fenc->lowres_mvs[i_list][idx];
if( lowres_mv[0][0] != 0x7fff )
{
M32( mvc[i] ) = (M32( lowres_mv[h->mb.i_mb_xy] )*2)&0xfffeffff;
i++;
}
}
}
......
......@@ -564,6 +564,7 @@ static int x264_validate_parameters( x264_t *h )
}
h->param.i_frame_reference = x264_clip3( h->param.i_frame_reference, 1, 16 );
h->param.i_dpb_size = x264_clip3( h->param.i_dpb_size, 1, 16 );
if( h->param.i_keyint_max <= 0 )
h->param.i_keyint_max = 1;
if( h->param.i_scenecut_threshold < 0 )
......@@ -593,10 +594,11 @@ static int x264_validate_parameters( x264_t *h )
x264_log( h, X264_LOG_WARNING, "b-pyramid normal + intra-refresh is not supported\n" );
h->param.i_bframe_pyramid = X264_B_PYRAMID_STRICT;
}
if( h->param.b_intra_refresh && h->param.i_frame_reference > 1 )
if( h->param.b_intra_refresh && (h->param.i_frame_reference > 1 || h->param.i_dpb_size > 1) )
{
x264_log( h, X264_LOG_WARNING, "ref > 1 + intra-refresh is not supported\n" );
h->param.i_frame_reference = 1;
h->param.i_dpb_size = 1;
}
if( h->param.b_intra_refresh && h->param.i_open_gop )
{
......@@ -1481,6 +1483,8 @@ static inline void x264_reference_build_list( x264_t *h, int i_poc )
for( int i = 0; h->frames.reference[i]; i++ )
{
if( h->frames.reference[i]->b_corrupt )
continue;
if( h->frames.reference[i]->i_poc < i_poc )
h->fref0[h->i_ref0++] = h->frames.reference[i];
else if( h->frames.reference[i]->i_poc > i_poc )
......@@ -2185,6 +2189,23 @@ void x264_encoder_intra_refresh( x264_t *h )
h->b_queued_intra_refresh = 1;
}
int x264_encoder_invalidate_reference( x264_t *h, int64_t pts )
{
if( h->param.i_bframe )
{
x264_log( h, X264_LOG_ERROR, "x264_encoder_invalidate_reference is not supported with B-frames enabled\n" );
return -1;
}
if( h->param.b_intra_refresh )
{
x264_log( h, X264_LOG_ERROR, "x264_encoder_invalidate_reference is not supported with intra refresh enabled\n" );
return -1;
}
h = h->thread[h->i_thread_phase];
h->i_reference_invalidate_pts = pts;
return 0;
}
/****************************************************************************
* x264_encoder_encode:
* XXX: i_poc : is the poc of the current given picture
......@@ -2330,6 +2351,29 @@ int x264_encoder_encode( x264_t *h,
h->fenc->param->param_free( h->fenc->param );
}
if( h->i_reference_invalidate_pts )
{
if( h->i_reference_invalidate_pts >= h->i_last_idr_pts )
for( int i = 0; h->frames.reference[i]; i++ )
if( h->i_reference_invalidate_pts <= h->frames.reference[i]->i_pts )
h->frames.reference[i]->b_corrupt = 1;
h->i_reference_invalidate_pts = 0;
}
if( !IS_X264_TYPE_I( h->fenc->i_type ) )
{
int valid_refs_left = 0;
for( int i = 0; h->frames.reference[i]; i++ )
if( !h->frames.reference[i]->b_corrupt )
valid_refs_left++;
/* No valid reference frames left: force an IDR. */
if( !valid_refs_left )
{
h->fenc->b_keyframe = 1;
h->fenc->i_type = X264_TYPE_IDR;
}
}
if( h->fenc->b_keyframe )
{
h->frames.i_last_keyframe = h->fenc->i_frame;
......@@ -2393,7 +2437,30 @@ int x264_encoder_encode( x264_t *h,
h->fenc->b_kept_as_ref =
h->fdec->b_kept_as_ref = i_nal_ref_idc != NAL_PRIORITY_DISPOSABLE && h->param.i_keyint_max > 1;
h->fdec->i_pts = h->fenc->i_pts *= h->i_dts_compress_multiplier;
if( h->frames.i_bframe_delay )
{
int64_t *prev_reordered_pts = thread_current->frames.i_prev_reordered_pts;
if( h->i_frame <= h->frames.i_bframe_delay )
{
if( h->i_dts_compress_multiplier == 1 )
h->fdec->i_dts = h->fenc->i_reordered_pts - h->frames.i_bframe_delay_time;
else
{
/* DTS compression */
if( h->i_frame == 1 )
thread_current->frames.i_init_delta = h->fenc->i_reordered_pts * h->i_dts_compress_multiplier;
h->fdec->i_dts = h->i_frame * thread_current->frames.i_init_delta / h->i_dts_compress_multiplier;
}
}
else
h->fdec->i_dts = prev_reordered_pts[ (h->i_frame - h->frames.i_bframe_delay) % h->frames.i_bframe_delay ];
prev_reordered_pts[ h->i_frame % h->frames.i_bframe_delay ] = h->fenc->i_reordered_pts * h->i_dts_compress_multiplier;
}
else
h->fdec->i_dts = h->fenc->i_reordered_pts;
if( h->fenc->i_type == X264_TYPE_IDR )
h->i_last_idr_pts = h->fdec->i_pts;
/* ------------------- Init ----------------------------- */
/* build ref list 0/1 */
......@@ -2616,28 +2683,9 @@ static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current,
pic_out->b_keyframe = h->fenc->b_keyframe;
pic_out->i_pts = h->fenc->i_pts *= h->i_dts_compress_multiplier;
if( h->frames.i_bframe_delay )
{
int64_t *prev_reordered_pts = thread_current->frames.i_prev_reordered_pts;
if( h->i_frame <= h->frames.i_bframe_delay )
{
if( h->i_dts_compress_multiplier == 1 )
pic_out->i_dts = h->fenc->i_reordered_pts - h->frames.i_bframe_delay_time;
else
{
/* DTS compression */
if( h->i_frame == 1 )
thread_current->frames.i_init_delta = h->fenc->i_reordered_pts * h->i_dts_compress_multiplier;
pic_out->i_dts = h->i_frame * thread_current->frames.i_init_delta / h->i_dts_compress_multiplier;
}
}
else
pic_out->i_dts = prev_reordered_pts[ (h->i_frame - h->frames.i_bframe_delay) % h->frames.i_bframe_delay ];
prev_reordered_pts[ h->i_frame % h->frames.i_bframe_delay ] = h->fenc->i_reordered_pts * h->i_dts_compress_multiplier;
}
else
pic_out->i_dts = h->fenc->i_reordered_pts;
pic_out->i_pts = h->fdec->i_pts;
pic_out->i_dts = h->fdec->i_dts;
if( pic_out->i_pts < pic_out->i_dts )
x264_log( h, X264_LOG_WARNING, "invalid DTS: PTS is less than DTS\n" );
......
......@@ -223,8 +223,8 @@ void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param )
/* extra slot with pyramid so that we don't have to override the
* order of forgetting old pictures */
sps->vui.i_max_dec_frame_buffering =
sps->i_num_ref_frames = X264_MIN(16, X264_MAX3(param->i_frame_reference, 1 + sps->vui.i_num_reorder_frames,
param->i_bframe_pyramid ? 4 : 1 ));
sps->i_num_ref_frames = X264_MIN(16, X264_MAX4(param->i_frame_reference, 1 + sps->vui.i_num_reorder_frames,
param->i_bframe_pyramid ? 4 : 1, param->i_dpb_size));
sps->i_num_ref_frames -= param->i_bframe_pyramid == X264_B_PYRAMID_STRICT;
sps->vui.b_bitstream_restriction = 1;
......
......@@ -35,7 +35,7 @@
#include <stdarg.h>
#define X264_BUILD 99
#define X264_BUILD 100
/* x264_t:
* opaque handler for encoder */
......@@ -217,6 +217,8 @@ typedef struct x264_param_t
/* Bitstream parameters */
int i_frame_reference; /* Maximum number of reference frames */
int i_dpb_size; /* Force a DPB size larger than that implied by B-frames and reference frames.
* Useful in combination with interactive error resilience. */
int i_keyint_max; /* Force an IDR keyframe at this interval */
int i_keyint_min; /* Scenecuts closer together than this are coded as I, not IDR. */
int i_scenecut_threshold; /* how aggressively to insert extra I frames */
......@@ -682,9 +684,38 @@ int x264_encoder_delayed_frames( x264_t * );
* If an intra refresh is not in progress, begin one with the next P-frame.
* If an intra refresh is in progress, begin one as soon as the current one finishes.
* Requires that b_intra_refresh be set.
*
* Useful for interactive streaming where the client can tell the server that packet loss has
* occurred. In this case, keyint can be set to an extremely high value so that intra refreshes
* only occur when calling x264_encoder_intra_refresh. */
* only occur when calling x264_encoder_intra_refresh.
*
* In multi-pass encoding, if x264_encoder_intra_refresh is called differently in each pass,
* behavior is undefined.
*
* Should not be called during an x264_encoder_encode. */
void x264_encoder_intra_refresh( x264_t * );
/* x264_encoder_invalidate_reference:
* An interactive error resilience tool, designed for use in a low-latency one-encoder-few-clients
* system. When the client has packet loss or otherwise incorrectly decodes a frame, the encoder
* can be told with this command to "forget" the frame and all frames that depend on it, referencing
* only frames that occurred before the loss. This will force a keyframe if no frames are left to
* reference after the aforementioned "forgetting".
*
* It is strongly recommended to use a large i_dpb_size in this case, which allows the encoder to
* keep around extra, older frames to fall back on in case more recent frames are all invalidated.
* Unlike increasing i_frame_reference, this does not increase the number of frames used for motion
* estimation and thus has no speed impact. It is also recommended to set a very large keyframe
* interval, so that keyframes are not used except as necessary for error recovery.
*
* x264_encoder_invalidate_reference is not currently compatible with the use of B-frames or intra
* refresh.
*
* In multi-pass encoding, if x264_encoder_invalidate_reference is called differently in each pass,
* behavior is undefined.
*
* Should not be called during an x264_encoder_encode.
*
* Returns 0 on success, negative on failure. */
int x264_encoder_invalidate_reference( x264_t *, int64_t pts );
#endif
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment