Commit 3cdaca1a authored by Fiona Glaser's avatar Fiona Glaser
Browse files

Fix array overreads that caused miscompilation in gcc 4.8

parent 37033444
......@@ -40,6 +40,7 @@
#define IS_DISPOSABLE(type) ( type == X264_TYPE_B )
#define FIX8(f) ((int)(f*(1<<8)+.5))
#define ALIGN(x,a) (((x)+((a)-1))&~((a)-1))
#define ARRAY_ELEMS(a) ((sizeof(a))/(sizeof(a[0])))
#define CHECKED_MALLOC( var, size )\
do {\
......
......@@ -85,44 +85,49 @@ int x264_cqm_init( x264_t *h )
int max_qp_err = -1;
int max_chroma_qp_err = -1;
int min_qp_err = QP_MAX+1;
int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4 : 2; /* Checkasm may segfault if optimized out by --chroma-format */
int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4
: h->param.analyse.b_transform_8x8 ? 2 : 0; /* Checkasm may segfault if optimized out by --chroma-format */
for( int i = 0; i < 4 + num_8x8_lists; i++ )
{
int size = i<4 ? 16 : 64;
int j;
for( j = (i<4 ? 0 : 4); j < i; j++ )
if( !memcmp( h->pps->scaling_list[i], h->pps->scaling_list[j], size*sizeof(uint8_t) ) )
break;
if( j < i )
{
h-> quant4_mf[i] = h-> quant4_mf[j];
h->dequant4_mf[i] = h->dequant4_mf[j];
h->unquant4_mf[i] = h->unquant4_mf[j];
}
else
{
CHECKED_MALLOC( h-> quant4_mf[i], (QP_MAX+1)*size*sizeof(udctcoef) );
CHECKED_MALLOC( h->dequant4_mf[i], 6*size*sizeof(int) );
CHECKED_MALLOC( h->unquant4_mf[i], (QP_MAX+1)*size*sizeof(int) );
}
for( j = (i<4 ? 0 : 4); j < i; j++ )
if( deadzone[j&3] == deadzone[i&3] &&
!memcmp( h->pps->scaling_list[i], h->pps->scaling_list[j], size*sizeof(uint8_t) ) )
break;
if( j < i )
{
h->quant4_bias[i] = h->quant4_bias[j];
h->quant4_bias0[i] = h->quant4_bias0[j];
}
else
{
CHECKED_MALLOC( h->quant4_bias[i], (QP_MAX+1)*size*sizeof(udctcoef) );
CHECKED_MALLOC( h->quant4_bias0[i], (QP_MAX+1)*size*sizeof(udctcoef) );
}
#define CQM_ALLOC( w, count )\
for( int i = 0; i < count; i++ )\
{\
int size = w*w;\
int start = w == 8 ? 4 : 0;\
int j;\
for( j = 0; j < i; j++ )\
if( !memcmp( h->pps->scaling_list[i+start], h->pps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
break;\
if( j < i )\
{\
h-> quant##w##_mf[i] = h-> quant##w##_mf[j];\
h->dequant##w##_mf[i] = h->dequant##w##_mf[j];\
h->unquant##w##_mf[i] = h->unquant##w##_mf[j];\
}\
else\
{\
CHECKED_MALLOC( h-> quant##w##_mf[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
CHECKED_MALLOC( h->dequant##w##_mf[i], 6*size*sizeof(int) );\
CHECKED_MALLOC( h->unquant##w##_mf[i], (QP_MAX+1)*size*sizeof(int) );\
}\
for( j = 0; j < i; j++ )\
if( deadzone[j] == deadzone[i] &&\
!memcmp( h->pps->scaling_list[i+start], h->pps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
break;\
if( j < i )\
{\
h->quant##w##_bias[i] = h->quant##w##_bias[j];\
h->quant##w##_bias0[i] = h->quant##w##_bias0[j];\
}\
else\
{\
CHECKED_MALLOC( h->quant##w##_bias[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
CHECKED_MALLOC( h->quant##w##_bias0[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
}\
}
CQM_ALLOC( 4, 4 )
CQM_ALLOC( 8, num_8x8_lists )
for( int q = 0; q < 6; q++ )
{
for( int i = 0; i < 16; i++ )
......@@ -204,6 +209,9 @@ int x264_cqm_init( x264_t *h )
for( int cat = 0; cat < 3 + CHROMA444; cat++ )
{
int dct8x8 = cat&1;
if( !h->param.analyse.b_transform_8x8 && dct8x8 )
continue;
int size = dct8x8 ? 64 : 16;
udctcoef *nr_offset = h->nr_offset_emergency[q][cat];
/* Denoise chroma first (due to h264's chroma QP offset), then luma, then DC. */
......
......@@ -309,7 +309,7 @@ static int check_pixel( int cpu_ref, int cpu_new )
#define TEST_PIXEL( name, align ) \
ok = 1, used_asm = 0; \
for( int i = 0; i < 8; i++ ) \
for( int i = 0; i < ARRAY_ELEMS(pixel_c.name); i++ ) \
{ \
int res_c, res_asm; \
if( pixel_asm.name[i] != pixel_ref.name[i] ) \
......@@ -1087,13 +1087,13 @@ static int check_dct( int cpu_ref, int cpu_new )
x264_zigzag_init( cpu_new, &zigzag_asm[0], &zigzag_asm[1] );
ok = 1; used_asm = 0;
TEST_INTERLEAVE( interleave_8x8_cavlc, level1, level2, dct1[0], 64 );
TEST_INTERLEAVE( interleave_8x8_cavlc, level1, level2, dct8[0], 64 );
report( "zigzag_interleave :" );
for( interlace = 0; interlace <= 1; interlace++ )
{
ok = 1; used_asm = 0;
TEST_ZIGZAG_SCAN( scan_8x8, level1, level2, dct1[0], 8 );
TEST_ZIGZAG_SCAN( scan_8x8, level1, level2, dct8[0], 8 );
TEST_ZIGZAG_SCAN( scan_4x4, level1, level2, dct1[0], 4 );
TEST_ZIGZAG_SUB( sub_4x4, level1, level2, 16 );
TEST_ZIGZAG_SUBAC( sub_4x4ac, level1, level2 );
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment