From 7852034004194db3aa7d44e9539b80249bcfa159 Mon Sep 17 00:00:00 2001
From: Anton Mitrofanov <BugMaster@narod.ru>
Date: Fri, 10 Sep 2021 15:36:19 +0300
Subject: [PATCH] Fix integer overflow in cavlc trellis
---
common/vlc.c | 2 +-
encoder/rdo.c | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/common/vlc.c b/common/vlc.c
index 13a668b8..def68f99 100644
--- a/common/vlc.c
+++ b/common/vlc.c
@@ -37,7 +37,7 @@ void x264_cavlc_init( x264_t *h )
{
int mask = level >> 15;
int abs_level = (level^mask)-mask;
- int i_level_code = abs_level*2-mask-2;
+ int i_level_code = abs_level ? abs_level*2-mask-2 : 0;
int i_next = i_suffix;
vlc_large_t *vlc = &x264_level_token[i_suffix][level+LEVEL_TABLE_SIZE/2];
diff --git a/encoder/rdo.c b/encoder/rdo.c
index 1fa19b42..4e830002 100644
--- a/encoder/rdo.c
+++ b/encoder/rdo.c
@@ -927,7 +927,7 @@ int quant_trellis_cavlc( x264_t *h, dctcoef *dct,
ALIGNED_ARRAY_16( dctcoef, coefs,[16] );
const uint32_t *coef_weight1 = b_8x8 ? x264_dct8_weight_tab : x264_dct4_weight_tab;
const uint32_t *coef_weight2 = b_8x8 ? x264_dct8_weight2_tab : x264_dct4_weight2_tab;
- int delta_distortion[16];
+ int64_t delta_distortion[16];
int64_t score = 1ULL<<62;
int i, j;
const int f = 1<<15;
@@ -954,7 +954,7 @@ int quant_trellis_cavlc( x264_t *h, dctcoef *dct,
/* Find last non-zero coefficient. */
for( i = end; i >= start; i -= step )
- if( (unsigned)(dct[zigzag[i]] * (dc?quant_mf[0]>>1:quant_mf[zigzag[i]]) + f-1) >= 2*f )
+ if( abs(dct[zigzag[i]]) * (dc?quant_mf[0]>>1:quant_mf[zigzag[i]]) >= f )
break;
if( i < start )
@@ -987,7 +987,7 @@ int quant_trellis_cavlc( x264_t *h, dctcoef *dct,
int unquant0 = (((dc?unquant_mf[0]<<1:unquant_mf[zigzag[j]]) * (nearest_quant-1) + 128) >> 8);
int d1 = abs_coef - unquant1;
int d0 = abs_coef - unquant0;
- delta_distortion[i] = (d0*d0 - d1*d1) * (dc?256:coef_weight2[zigzag[j]]);
+ delta_distortion[i] = (int64_t)(d0*d0 - d1*d1) * (dc?256:coef_weight2[zigzag[j]]);
/* Psy trellis: bias in favor of higher AC coefficients in the reconstructed frame. */
if( h->mb.i_psy_trellis && j && !dc && !b_chroma )
@@ -1025,7 +1025,7 @@ int quant_trellis_cavlc( x264_t *h, dctcoef *dct,
while( 1 )
{
int64_t iter_score = score;
- int iter_distortion_delta = 0;
+ int64_t iter_distortion_delta = 0;
int iter_coef = -1;
int iter_mask = coef_mask;
int iter_round = round_mask;
@@ -1040,7 +1040,7 @@ int quant_trellis_cavlc( x264_t *h, dctcoef *dct,
int old_coef = coefs[i];
int new_coef = quant_coefs[round_change][i];
int cur_mask = (coef_mask&~(1 << i))|(!!new_coef << i);
- int cur_distortion_delta = delta_distortion[i] * (round_change ? -1 : 1);
+ int64_t cur_distortion_delta = delta_distortion[i] * (round_change ? -1 : 1);
int64_t cur_score = cur_distortion_delta;
coefs[i] = new_coef;
--
GitLab