Commit e5bca59c authored by Janne Grunau's avatar Janne Grunau

itx: cancel 16 out of '(x * 2896) >> 12' to avoid integer overflows

Fixes an integer overflow in inv_dct4_1d with
clusterfuzz-testcase-dav1d_fuzzer-5634807321591808 and in inv_adst16_1d
with clusterfuzz-testcase-dav1d_fuzzer-5761827623927808. Credits to oss-fuzz.
parent 1d3f6364
......@@ -42,8 +42,8 @@ inv_dct4_1d(const coef *const in, const ptrdiff_t in_s,
const int in0 = in[0 * in_s], in1 = in[1 * in_s];
const int in2 = in[2 * in_s], in3 = in[3 * in_s];
int t0 = ((in0 + in2) * 2896 + 2048) >> 12;
int t1 = ((in0 - in2) * 2896 + 2048) >> 12;
int t0 = ((in0 + in2) * 181 + 128) >> 8;
int t1 = ((in0 - in2) * 181 + 128) >> 8;
int t2 = (in1 * 1567 - in3 * 3784 + 2048) >> 12;
int t3 = (in1 * 3784 + in3 * 1567 + 2048) >> 12;
......@@ -75,8 +75,8 @@ inv_dct8_1d(const coef *const in, const ptrdiff_t in_s,
int t7 = CLIP(t7a + t6a);
t6a = CLIP(t7a - t6a);
int t5 = ((t6a - t5a) * 2896 + 2048) >> 12;
int t6 = ((t6a + t5a) * 2896 + 2048) >> 12;
int t5 = ((t6a - t5a) * 181 + 128) >> 8;
int t6 = ((t6a + t5a) * 181 + 128) >> 8;
out[0 * out_s] = CLIP(tmp[0] + t7);
out[1 * out_s] = CLIP(tmp[1] + t6);
......@@ -134,10 +134,10 @@ inv_dct16_1d(const coef *const in, const ptrdiff_t in_s,
t14 = CLIP(t14a + t13a);
t15a = CLIP(t15 + t12);
t10a = ((t13 - t10) * 2896 + 2048) >> 12;
t13a = ((t13 + t10) * 2896 + 2048) >> 12;
t11 = ((t12a - t11a) * 2896 + 2048) >> 12;
t12 = ((t12a + t11a) * 2896 + 2048) >> 12;
t10a = ((t13 - t10) * 181 + 128) >> 8;
t13a = ((t13 + t10) * 181 + 128) >> 8;
t11 = ((t12a - t11a) * 181 + 128) >> 8;
t12 = ((t12a + t11a) * 181 + 128) >> 8;
out[ 0 * out_s] = CLIP(tmp[0] + t15a);
out[ 1 * out_s] = CLIP(tmp[1] + t14);
......@@ -261,14 +261,14 @@ inv_dct32_1d(const coef *const in, const ptrdiff_t in_s,
t30a = CLIP(t30 + t25);
t31 = CLIP(t31a + t24a);
t20 = ((t27a - t20a) * 2896 + 2048) >> 12;
t27 = ((t27a + t20a) * 2896 + 2048) >> 12;
t21a = ((t26 - t21 ) * 2896 + 2048) >> 12;
t26a = ((t26 + t21 ) * 2896 + 2048) >> 12;
t22 = ((t25a - t22a) * 2896 + 2048) >> 12;
t25 = ((t25a + t22a) * 2896 + 2048) >> 12;
t23a = ((t24 - t23 ) * 2896 + 2048) >> 12;
t24a = ((t24 + t23 ) * 2896 + 2048) >> 12;
t20 = ((t27a - t20a) * 181 + 128) >> 8;
t27 = ((t27a + t20a) * 181 + 128) >> 8;
t21a = ((t26 - t21 ) * 181 + 128) >> 8;
t26a = ((t26 + t21 ) * 181 + 128) >> 8;
t22 = ((t25a - t22a) * 181 + 128) >> 8;
t25 = ((t25a + t22a) * 181 + 128) >> 8;
t23a = ((t24 - t23 ) * 181 + 128) >> 8;
t24a = ((t24 + t23 ) * 181 + 128) >> 8;
out[ 0 * out_s] = CLIP(tmp[ 0] + t31);
out[ 1 * out_s] = CLIP(tmp[ 1] + t30a);
......@@ -546,22 +546,22 @@ inv_dct64_1d(const coef *const in, const ptrdiff_t in_s,
t62 = CLIP(t62a + t49a);
t63a = CLIP(t63 + t48);
t40a = (t40 * -2896 + t55 * 2896 + 2048) >> 12;
t41 = (t41a * -2896 + t54a * 2896 + 2048) >> 12;
t42a = (t42 * -2896 + t53 * 2896 + 2048) >> 12;
t43 = (t43a * -2896 + t52a * 2896 + 2048) >> 12;
t44a = (t44 * -2896 + t51 * 2896 + 2048) >> 12;
t45 = (t45a * -2896 + t50a * 2896 + 2048) >> 12;
t46a = (t46 * -2896 + t49 * 2896 + 2048) >> 12;
t47 = (t47a * -2896 + t48a * 2896 + 2048) >> 12;
t48 = (t47a * 2896 + t48a * 2896 + 2048) >> 12;
t49a = (t46 * 2896 + t49 * 2896 + 2048) >> 12;
t50 = (t45a * 2896 + t50a * 2896 + 2048) >> 12;
t51a = (t44 * 2896 + t51 * 2896 + 2048) >> 12;
t52 = (t43a * 2896 + t52a * 2896 + 2048) >> 12;
t53a = (t42 * 2896 + t53 * 2896 + 2048) >> 12;
t54 = (t41a * 2896 + t54a * 2896 + 2048) >> 12;
t55a = (t40 * 2896 + t55 * 2896 + 2048) >> 12;
t40a = (t40 * -181 + t55 * 181 + 128) >> 8;
t41 = (t41a * -181 + t54a * 181 + 128) >> 8;
t42a = (t42 * -181 + t53 * 181 + 128) >> 8;
t43 = (t43a * -181 + t52a * 181 + 128) >> 8;
t44a = (t44 * -181 + t51 * 181 + 128) >> 8;
t45 = (t45a * -181 + t50a * 181 + 128) >> 8;
t46a = (t46 * -181 + t49 * 181 + 128) >> 8;
t47 = (t47a * -181 + t48a * 181 + 128) >> 8;
t48 = (t47a * 181 + t48a * 181 + 128) >> 8;
t49a = (t46 * 181 + t49 * 181 + 128) >> 8;
t50 = (t45a * 181 + t50a * 181 + 128) >> 8;
t51a = (t44 * 181 + t51 * 181 + 128) >> 8;
t52 = (t43a * 181 + t52a * 181 + 128) >> 8;
t53a = (t42 * 181 + t53 * 181 + 128) >> 8;
t54 = (t41a * 181 + t54a * 181 + 128) >> 8;
t55a = (t40 * 181 + t55 * 181 + 128) >> 8;
out[ 0 * out_s] = CLIP(tmp[ 0] + t63a);
out[ 1 * out_s] = CLIP(tmp[ 1] + t62);
......@@ -690,10 +690,10 @@ inv_adst8_1d(const coef *const in, const ptrdiff_t in_s,
t6 = CLIP( t4a - t6a );
t7 = CLIP( t5a - t7a );
out[3 * out_s] = -(((t2 + t3) * 2896 + 2048) >> 12);
out[4 * out_s] = ((t2 - t3) * 2896 + 2048) >> 12;
out[2 * out_s] = ((t6 + t7) * 2896 + 2048) >> 12;
out[5 * out_s] = -(((t6 - t7) * 2896 + 2048) >> 12);
out[3 * out_s] = -(((t2 + t3) * 181 + 128) >> 8);
out[4 * out_s] = ((t2 - t3) * 181 + 128) >> 8;
out[2 * out_s] = ((t6 + t7) * 181 + 128) >> 8;
out[5 * out_s] = -(((t6 - t7) * 181 + 128) >> 8);
}
static void NOINLINE
......@@ -796,14 +796,14 @@ inv_adst16_1d(const coef *const in, const ptrdiff_t in_s,
t14a = CLIP( t12 - t14 );
t15a = CLIP( t13 - t15 );
out[ 7 * out_s] = -(((t2a + t3a) * 2896 + 2048) >> 12);
out[ 8 * out_s] = ((t2a - t3a) * 2896 + 2048) >> 12;
out[ 4 * out_s] = ((t6 + t7) * 2896 + 2048) >> 12;
out[11 * out_s] = -(((t6 - t7) * 2896 + 2048) >> 12);
out[ 6 * out_s] = ((t10 + t11) * 2896 + 2048) >> 12;
out[ 9 * out_s] = -(((t10 - t11) * 2896 + 2048) >> 12);
out[ 5 * out_s] = -(((t14a + t15a) * 2896 + 2048) >> 12);
out[10 * out_s] = ((t14a - t15a) * 2896 + 2048) >> 12;
out[ 7 * out_s] = -(((t2a + t3a) * 181 + 128) >> 8);
out[ 8 * out_s] = ((t2a - t3a) * 181 + 128) >> 8;
out[ 4 * out_s] = ((t6 + t7) * 181 + 128) >> 8;
out[11 * out_s] = -(((t6 - t7) * 181 + 128) >> 8);
out[ 6 * out_s] = ((t10 + t11) * 181 + 128) >> 8;
out[ 9 * out_s] = -(((t10 - t11) * 181 + 128) >> 8);
out[ 5 * out_s] = -(((t14a + t15a) * 181 + 128) >> 8);
out[10 * out_s] = ((t14a - t15a) * 181 + 128) >> 8;
}
#define flip_inv_adst(sz) \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment