Commit de6e3170 authored by Ronald S. Bultje's avatar Ronald S. Bultje

Deal with chroma coefficients that are exactly 0x100000

For chroma coefficients that are masked (&= 0xfffff) to no value, the
context becomes in a weird state where it has no magnitude (ctx & 0x3f
== 0), but it does have a sign (ctx & 0xc0 != 0x40). Our old code
checked just the magnitude part of the context to set the skip context
of neighbouring blocks, but libaom uses both sign and magnitude for
this purpose. Therefore, adjust our code so it does the same thing.

Luma code only checks magnitude for this purpose and is thus not
affected by this peculiarity. Fixes #325.
parent f306f969
Pipeline #12231 passed with stages
in 8 minutes and 7 seconds
......@@ -70,10 +70,10 @@ static inline unsigned get_skip_ctx(const TxfmInfo *const t_dim,
const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444;
const int not_one_blk = b_dim[2] - (!!b_dim[2] && ss_hor) > t_dim->lw ||
b_dim[3] - (!!b_dim[3] && ss_ver) > t_dim->lh;
int ca, cl;
unsigned ca, cl;
#define MERGE_CTX(dir, type, mask) \
c##dir = !!((*(const type *) dir) & mask); \
#define MERGE_CTX(dir, type, no_val) \
c##dir = *(const type *) dir != no_val; \
break
switch (t_dim->lw) {
......@@ -83,17 +83,17 @@ static inline unsigned get_skip_ctx(const TxfmInfo *const t_dim,
* and will therefore complain about the use of uninitialized variables
* when compiled in debug mode if we put the default case at the end. */
default: assert(0); /* fall-through */
case TX_4X4: MERGE_CTX(a, uint8_t, 0x3F);
case TX_8X8: MERGE_CTX(a, uint16_t, 0x3F3F);
case TX_16X16: MERGE_CTX(a, uint32_t, 0x3F3F3F3FU);
case TX_32X32: MERGE_CTX(a, uint64_t, 0x3F3F3F3F3F3F3F3FULL);
case TX_4X4: MERGE_CTX(a, uint8_t, 0x40);
case TX_8X8: MERGE_CTX(a, uint16_t, 0x4040);
case TX_16X16: MERGE_CTX(a, uint32_t, 0x40404040U);
case TX_32X32: MERGE_CTX(a, uint64_t, 0x4040404040404040ULL);
}
switch (t_dim->lh) {
default: assert(0); /* fall-through */
case TX_4X4: MERGE_CTX(l, uint8_t, 0x3F);
case TX_8X8: MERGE_CTX(l, uint16_t, 0x3F3F);
case TX_16X16: MERGE_CTX(l, uint32_t, 0x3F3F3F3FU);
case TX_32X32: MERGE_CTX(l, uint64_t, 0x3F3F3F3F3F3F3F3FULL);
case TX_4X4: MERGE_CTX(l, uint8_t, 0x40);
case TX_8X8: MERGE_CTX(l, uint16_t, 0x4040);
case TX_16X16: MERGE_CTX(l, uint32_t, 0x40404040U);
case TX_32X32: MERGE_CTX(l, uint64_t, 0x4040404040404040ULL);
}
#undef MERGE_CTX
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment