Commit d026397b authored by Fiona Glaser's avatar Fiona Glaser

Try 8x8 transform analysis even when sub8x8 partitions are present

Turn off the sub8x8 partitions, try it, and turn them back on if it didn't help.
Small compression improvement with p4x4 on (~0.1-0.5%).
Also update related comments.
parent dea5d7a5
......@@ -420,23 +420,23 @@ static ALWAYS_INLINE int x264_mb_predict_non_zero_code( x264_t *h, int idx )
i_ret = ( i_ret + 1 ) >> 1;
return i_ret & 0x7f;
}
/* intra and skip are disallowed, p8x8 is conditional. */
static const uint8_t x264_transform_allowed[X264_MBTYPE_MAX] =
{
0,0,0,0,1,2,0,1,1,1,1,1,1,1,1,1,1,1,0
};
/* x264_mb_transform_8x8_allowed:
* check whether any partition is smaller than 8x8 (or at least
* might be, according to just partition type.)
* doesn't check for cbp */
static ALWAYS_INLINE int x264_mb_transform_8x8_allowed( x264_t *h )
{
// intra and skip are disallowed
// large partitions are allowed
// direct and 8x8 are conditional
static const uint8_t partition_tab[X264_MBTYPE_MAX] = {
0,0,0,0,1,2,0,1,1,1,1,1,1,1,1,1,1,1,0,
};
if( !h->pps->b_transform_8x8_mode )
return 0;
if( h->mb.i_type != P_8x8 )
return partition_tab[h->mb.i_type];
return x264_transform_allowed[h->mb.i_type];
return M32( h->mb.i_sub_partition ) == D_L0_8x8*0x01010101;
}
......
......@@ -2851,8 +2851,15 @@ static inline void x264_mb_analyse_transform( x264_t *h )
static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *a, int *i_satd, int *i_rd )
{
if( x264_mb_transform_8x8_allowed( h ) && h->param.analyse.b_transform_8x8 )
if( h->param.analyse.b_transform_8x8 && h->pps->b_transform_8x8_mode )
{
uint32_t subpart_bak = M32( h->mb.i_sub_partition );
/* Try switching the subpartitions to 8x8 so that we can use 8x8 transform mode */
if( h->mb.i_type == P_8x8 )
M32( h->mb.i_sub_partition ) = D_L0_8x8*0x01010101;
else if( !x264_transform_allowed[h->mb.i_type] )
return;
x264_analyse_update_cache( h, a );
h->mb.b_transform_8x8 ^= 1;
/* FIXME only luma is needed for 4:2:0, but the score for comparison already includes chroma */
......@@ -2865,7 +2872,10 @@ static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *
*i_rd = i_rd8;
}
else
{
h->mb.b_transform_8x8 ^= 1;
M32( h->mb.i_sub_partition ) = subpart_bak;
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment