Commit 9fd7ccb2 authored by Anton Mitrofanov's avatar Anton Mitrofanov Committed by Fiona Glaser
Browse files

Fix PCM cost overflow

parent 1d70d0e5
...@@ -433,8 +433,10 @@ static void x264_mb_analyse_init( x264_t *h, x264_mb_analysis_t *a, int qp ) ...@@ -433,8 +433,10 @@ static void x264_mb_analyse_init( x264_t *h, x264_mb_analysis_t *a, int qp )
a->i_satd_i4x4 = a->i_satd_i4x4 =
a->i_satd_chroma = COST_MAX; a->i_satd_chroma = COST_MAX;
/* non-RD PCM decision is inaccurate (as is psy-rd), so don't do it */ /* non-RD PCM decision is inaccurate (as is psy-rd), so don't do it.
a->i_satd_pcm = !h->mb.i_psy_rd && a->i_mbrd ? ((uint64_t)X264_PCM_COST*a->i_lambda2 + 128) >> 8 : COST_MAX; * PCM cost can overflow with high lambda2, so cap it at COST_MAX. */
uint64_t pcm_cost = ((uint64_t)X264_PCM_COST*a->i_lambda2 + 128) >> 8;
a->i_satd_pcm = !h->mb.i_psy_rd && a->i_mbrd && pcm_cost < COST_MAX ? pcm_cost : COST_MAX;
a->b_fast_intra = 0; a->b_fast_intra = 0;
a->b_avoid_topright = 0; a->b_avoid_topright = 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment