Commit 1d3f6364 authored by Janne Grunau's avatar Janne Grunau

inv_identity: avoid signed int overflow for 12-bit files

Fixes #220.
parent c3ee7290
Pipeline #3507 passed with stages
in 5 minutes and 42 seconds
...@@ -824,7 +824,7 @@ inv_identity4_1d(const coef *const in, const ptrdiff_t in_s, ...@@ -824,7 +824,7 @@ inv_identity4_1d(const coef *const in, const ptrdiff_t in_s,
coef *const out, const ptrdiff_t out_s, const int range) coef *const out, const ptrdiff_t out_s, const int range)
{ {
for (int i = 0; i < 4; i++) for (int i = 0; i < 4; i++)
out[out_s * i] = (in[in_s * i] * 5793 + 2048) >> 12; out[out_s * i] = in[in_s * i] + ((in[in_s * i] * 1697 + 2048) >> 12);
} }
static void NOINLINE static void NOINLINE
...@@ -840,7 +840,7 @@ inv_identity16_1d(const coef *const in, const ptrdiff_t in_s, ...@@ -840,7 +840,7 @@ inv_identity16_1d(const coef *const in, const ptrdiff_t in_s,
coef *const out, const ptrdiff_t out_s, const int range) coef *const out, const ptrdiff_t out_s, const int range)
{ {
for (int i = 0; i < 16; i++) for (int i = 0; i < 16; i++)
out[out_s * i] = (in[in_s * i] * 2 * 5793 + 2048) >> 12; out[out_s * i] = 2 * in[in_s * i] + ((in[in_s * i] * 1697 + 1024) >> 11);
} }
static void NOINLINE static void NOINLINE
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment