Commit 42fc4724 authored by Henrik Gramner's avatar Henrik Gramner

checkasm: Fix itx coef input ranges

Input coefficients for transforms of sizes 16x32/32x16 and above were
larger than they should be, which in rare cases could cause tests to
incorrectly fail.
parent f17c5e08
...@@ -72,15 +72,15 @@ static const char *const itx_1d_names[5] = { ...@@ -72,15 +72,15 @@ static const char *const itx_1d_names[5] = {
}; };
static const double scaling_factors[9] = { static const double scaling_factors[9] = {
4.00, /* 4x4 */ 4.0000, /* 4x4 */
4.00 * M_SQRT1_2, /* 4x8 8x4 */ 4.0000 * M_SQRT1_2, /* 4x8 8x4 */
2.00, /* 4x16 8x8 16x4 */ 2.0000, /* 4x16 8x8 16x4 */
2.00 * M_SQRT1_2, /* 8x16 16x8 */ 2.0000 * M_SQRT1_2, /* 8x16 16x8 */
1.00, /* 8x32 16x16 32x8 */ 1.0000, /* 8x32 16x16 32x8 */
1.00 * M_SQRT1_2, /* 16x32 32x16 */ 0.5000 * M_SQRT1_2, /* 16x32 32x16 */
0.50, /* 16x64 32x32 64x16 */ 0.2500, /* 16x64 32x32 64x16 */
0.50 * M_SQRT1_2, /* 32x64 64x32 */ 0.1250 * M_SQRT1_2, /* 32x64 64x32 */
0.25, /* 64x64 */ 0.0625, /* 64x64 */
}; };
/* FIXME: Ensure that those forward transforms are similar to the real AV1 /* FIXME: Ensure that those forward transforms are similar to the real AV1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment