Skip to content

GitLab

Commit 6eab44d4 authored by Steven Walters's avatar Steven Walters Committed by Fiona Glaser
Browse files

Gracefully terminate in the case of a malloc failure 

Fuzz tests show that all mallocs appear to be checked correctly now.
parent 7dec1a15
Hide whitespace changes
Inline Side-by-side
common/common.c
View file @ 6eab44d4
......@@ -606,11 +606,14 @@ int x264_param_parse( x264_param_t *p, const char *name, const char *value )
****************************************************************************/
void x264_log( x264_t *h, int i_level, const char *psz_fmt, ... )
{
if( i_level <= h->param.i_log_level )
if( !h || i_level <= h->param.i_log_level )
{
va_list arg;
va_start( arg, psz_fmt );
h->param.pf_log( h->param.p_log_private, i_level, psz_fmt, arg );
if( !h )
x264_log_default( NULL, i_level, psz_fmt, arg );
else
h->param.pf_log( h->param.p_log_private, i_level, psz_fmt, arg );
va_end( arg );
}
}
......@@ -643,18 +646,21 @@ static void x264_log_default( void *p_unused, int i_level, const char *psz_fmt,
/****************************************************************************
* x264_picture_alloc:
****************************************************************************/
void x264_picture_alloc( x264_picture_t *pic, int i_csp, int i_width, int i_height )
int x264_picture_alloc( x264_picture_t *pic, int i_csp, int i_width, int i_height )
{
pic->i_type = X264_TYPE_AUTO;
pic->i_qpplus1 = 0;
pic->img.i_csp = i_csp;
pic->img.i_plane = 3;
pic->img.plane[0] = x264_malloc( 3 * i_width * i_height / 2 );
if( !pic->img.plane[0] )
return -1;
pic->img.plane[1] = pic->img.plane[0] + i_width * i_height;
pic->img.plane[2] = pic->img.plane[1] + i_width * i_height / 4;
pic->img.i_stride[0] = i_width;
pic->img.i_stride[1] = i_width / 2;
pic->img.i_stride[2] = i_width / 2;
return 0;
}
/****************************************************************************
......@@ -717,22 +723,25 @@ int x264_nal_encode( void *p_data, int *pi_data, int b_annexeb, x264_nal_t *nal
****************************************************************************/
void *x264_malloc( int i_size )
{
uint8_t *align_buf = NULL;
#ifdef SYS_MACOSX
/* Mac OS X always returns 16 bytes aligned memory */
return malloc( i_size );
align_buf = malloc( i_size );
#elif defined( HAVE_MALLOC_H )
return memalign( 16, i_size );
align_buf = memalign( 16, i_size );
#else
uint8_t * buf;
uint8_t * align_buf;
buf = (uint8_t *) malloc( i_size + 15 + sizeof( void ** ) +
sizeof( int ) );
align_buf = buf + 15 + sizeof( void ** ) + sizeof( int );
align_buf -= (intptr_t) align_buf & 15;
*( (void **) ( align_buf - sizeof( void ** ) ) ) = buf;
*( (int *) ( align_buf - sizeof( void ** ) - sizeof( int ) ) ) = i_size;
return align_buf;
uint8_t *buf = malloc( i_size + 15 + sizeof(void **) + sizeof(int) );
if( buf )
{
align_buf = buf + 15 + sizeof(void **) + sizeof(int);
align_buf -= (intptr_t) align_buf & 15;
*( (void **) ( align_buf - sizeof(void **) ) ) = buf;
*( (int *) ( align_buf - sizeof(void **) - sizeof(int) ) ) = i_size;
}
#endif
if( !align_buf )
x264_log( NULL, X264_LOG_ERROR, "malloc of size %d failed\n", i_size );
return align_buf;
}
/****************************************************************************
......@@ -750,31 +759,6 @@ void x264_free( void *p )
}
}
/****************************************************************************
* x264_realloc:
****************************************************************************/
void *x264_realloc( void *p, int i_size )
{
#ifdef HAVE_MALLOC_H
return realloc( p, i_size );
#else
int i_old_size = 0;
uint8_t * p_new;
if( p )
{
i_old_size = *( (int*) ( (uint8_t*) p - sizeof( void ** ) -
sizeof( int ) ) );
}
p_new = x264_malloc( i_size );
if( i_old_size > 0 && i_size > 0 )
{
memcpy( p_new, p, ( i_old_size < i_size ) ? i_old_size : i_size );
}
x264_free( p );
return p_new;
#endif
}
/****************************************************************************
* x264_reduce_fraction:
****************************************************************************/
......@@ -838,6 +822,8 @@ char *x264_param2string( x264_param_t *p, int b_res )
if( p->rc.psz_zones )
len += strlen(p->rc.psz_zones);
buf = s = x264_malloc( len );
if( !buf )
return NULL;
if( b_res )
{
......
common/common.h
View file @ 6eab44d4
......@@ -37,14 +37,16 @@
#define FIX8(f) ((int)(f*(1<<8)+.5))
#define CHECKED_MALLOC( var, size )\
{\
do {\
var = x264_malloc( size );\
if( !var )\
{\
x264_log( h, X264_LOG_ERROR, "malloc failed\n" );\
goto fail;\
}\
}
} while( 0 )
#define CHECKED_MALLOCZERO( var, size )\
do {\
CHECKED_MALLOC( var, size );\
memset( var, 0, size );\
} while( 0 )
#define X264_BFRAME_MAX 16
#define X264_THREAD_MAX 128
......@@ -83,7 +85,6 @@
/* x264_malloc : will do or emulate a memalign
* you have to use x264_free for buffers allocated with x264_malloc */
void *x264_malloc( int );
void *x264_realloc( void *p, int i_size );
void x264_free( void * );
/* x264_slurp_file: malloc space for the whole file and read it */
......
common/frame.c
View file @ 6eab44d4
......@@ -28,7 +28,7 @@
x264_frame_t *x264_frame_new( x264_t *h )
{
x264_frame_t *frame = x264_malloc( sizeof(x264_frame_t) );
x264_frame_t *frame;
int i, j;
int i_mb_count = h->mb.i_mb_count;
......@@ -38,9 +38,7 @@ x264_frame_t *x264_frame_new( x264_t *h )
int chroma_plane_size;
int align = h->param.cpu&X264_CPU_CACHELINE_64 ? 64 : h->param.cpu&X264_CPU_CACHELINE_32 ? 32 : 16;
if( !frame ) return NULL;
memset( frame, 0, sizeof(x264_frame_t) );
CHECKED_MALLOCZERO( frame, sizeof(x264_frame_t) );
/* allocate frame data (+64 for extra data for me) */
i_width = ALIGN( h->param.i_width, 16 );
......@@ -92,8 +90,7 @@ x264_frame_t *x264_frame_new( x264_t *h )
for( j = 0; j <= !!h->param.i_bframe; j++ )
for( i = 0; i <= h->param.i_bframe; i++ )
{
CHECKED_MALLOC( frame->lowres_mvs[j][i], 2*h->mb.i_mb_count*sizeof(int16_t) );
memset( frame->lowres_mvs[j][i], 0, 2*h->mb.i_mb_count*sizeof(int16_t) );
CHECKED_MALLOCZERO( frame->lowres_mvs[j][i], 2*h->mb.i_mb_count*sizeof(int16_t) );
CHECKED_MALLOC( frame->lowres_mv_costs[j][i], h->mb.i_mb_count*sizeof(int) );
}
CHECKED_MALLOC( frame->i_intra_cost, i_mb_count * sizeof(uint16_t) );
......@@ -149,13 +146,15 @@ x264_frame_t *x264_frame_new( x264_t *h )
CHECKED_MALLOC( frame->i_inv_qscale_factor, h->mb.i_mb_count * sizeof(uint16_t) );
}
x264_pthread_mutex_init( &frame->mutex, NULL );
x264_pthread_cond_init( &frame->cv, NULL );
if( x264_pthread_mutex_init( &frame->mutex, NULL ) )
goto fail;
if( x264_pthread_cond_init( &frame->cv, NULL ) )
goto fail;
return frame;
fail:
x264_frame_delete( frame );
x264_free( frame );
return NULL;
}
......@@ -956,7 +955,8 @@ x264_frame_t *x264_frame_pop_unused( x264_t *h )
frame = x264_frame_pop( h->frames.unused );
else
frame = x264_frame_new( h );
assert( frame->i_reference_count == 0 );
if( !frame )
return NULL;
frame->i_reference_count = 1;
frame->b_intra_calculated = 0;
return frame;
......
common/osdep.h
View file @ 6eab44d4
......@@ -90,10 +90,16 @@
#if defined(SYS_BEOS)
#include <kernel/OS.h>
#define x264_pthread_t thread_id
#define x264_pthread_create(t,u,f,d) { *(t)=spawn_thread(f,"",10,d); \
resume_thread(*(t)); }
static inline int x264_pthread_create( x264_pthread_t *t, void *a, void *(*f)(void *), void *d )
{
*t = spawn_thread( f, "", 10, d );
if( *t < B_NO_ERROR )
return -1;
resume_thread( *t );
return 0;
}
#define x264_pthread_join(t,s) { long tmp; \
wait_for_thread(t,(s)?(long*)(s):&tmp); }
wait_for_thread(t,(s)?(long*)(*(s)):&tmp); }
#ifndef usleep
#define usleep(t) snooze(t)
#endif
......@@ -105,7 +111,7 @@
#else
#define x264_pthread_t int
#define x264_pthread_create(t,u,f,d)
#define x264_pthread_create(t,u,f,d) 0
#define x264_pthread_join(t,s)
#endif //SYS_*
......@@ -125,12 +131,12 @@
#define x264_pthread_cond_wait pthread_cond_wait
#else
#define x264_pthread_mutex_t int
#define x264_pthread_mutex_init(m,f)
#define x264_pthread_mutex_init(m,f) 0
#define x264_pthread_mutex_destroy(m)
#define x264_pthread_mutex_lock(m)
#define x264_pthread_mutex_unlock(m)
#define x264_pthread_cond_t int
#define x264_pthread_cond_init(c,f)
#define x264_pthread_cond_init(c,f) 0
#define x264_pthread_cond_destroy(c)
#define x264_pthread_cond_broadcast(c)
#define x264_pthread_cond_wait(c,m)
......
common/set.c
View file @ 6eab44d4
......@@ -93,9 +93,9 @@ int x264_cqm_init( x264_t *h )
}
else
{
h-> quant4_mf[i] = x264_malloc(52*size*sizeof(uint16_t) );
h->dequant4_mf[i] = x264_malloc( 6*size*sizeof(int) );
h->unquant4_mf[i] = x264_malloc(52*size*sizeof(int) );
CHECKED_MALLOC( h-> quant4_mf[i], 52*size*sizeof(uint16_t) );
CHECKED_MALLOC( h->dequant4_mf[i], 6*size*sizeof(int) );
CHECKED_MALLOC( h->unquant4_mf[i], 52*size*sizeof(int) );
}
for( j = (i<4 ? 0 : 4); j < i; j++ )
......@@ -105,7 +105,7 @@ int x264_cqm_init( x264_t *h )
if( j < i )
h->quant4_bias[i] = h->quant4_bias[j];
else
h->quant4_bias[i] = x264_malloc(52*size*sizeof(uint16_t) );
CHECKED_MALLOC( h->quant4_bias[i], 52*size*sizeof(uint16_t) );
}
for( q = 0; q < 6; q++ )
......@@ -171,6 +171,9 @@ int x264_cqm_init( x264_t *h )
return -1;
}
return 0;
fail:
x264_cqm_delete( h );
return -1;
}
void x264_cqm_delete( x264_t *h )
......
common/visualize.c
View file @ 6eab44d4
......@@ -94,10 +94,13 @@ static void mv(int x0, int y0, int16_t dmv[2], int ref, int zoom, char *col)
/* }}} */
/* {{{ [fold] void x264_visualize_init( x264_t *h ) */
void x264_visualize_init( x264_t *h )
int x264_visualize_init( x264_t *h )
{
int mb = h->sps->i_mb_width * h->sps->i_mb_height;
h->visualize = x264_malloc(mb * sizeof(visualize_t));
CHECKED_MALLOC( h->visualize, mb * sizeof(visualize_t) );
return 0;
fail:
return -1;
}
/* }}} */
/* {{{ [fold] void x264_visualize_mb( x264_t *h ) */
......
common/visualize.h
View file @ 6eab44d4
......@@ -23,7 +23,7 @@
#include "common/common.h"
void x264_visualize_init( x264_t *h );
int x264_visualize_init( x264_t *h );
void x264_visualize_mb( x264_t *h );
void x264_visualize_show( x264_t *h );
void x264_visualize_close( x264_t *h );
......
encoder/analyse.c
View file @ 6eab44d4
......@@ -213,7 +213,7 @@ uint16_t *x264_cost_mv_fpel[92][4];
uint16_t x264_cost_ref[92][3][33];
/* initialize an array of lambda*nbits for all possible mvs */
static void x264_mb_analyse_load_costs( x264_t *h, x264_mb_analysis_t *a )
static int x264_mb_analyse_load_costs( x264_t *h, x264_mb_analysis_t *a )
{
static int16_t *p_cost_mv[92];
int i, j;
......@@ -223,7 +223,7 @@ static void x264_mb_analyse_load_costs( x264_t *h, x264_mb_analysis_t *a )
x264_emms();
/* could be faster, but isn't called many times */
/* factor of 4 from qpel, 2 from sign, and 2 because mv can be opposite from mvp */
p_cost_mv[a->i_lambda] = x264_malloc( (4*4*2048 + 1) * sizeof(int16_t) );
CHECKED_MALLOC( p_cost_mv[a->i_lambda], (4*4*2048 + 1) * sizeof(int16_t) );
p_cost_mv[a->i_lambda] += 2*4*2048;
for( i = 0; i <= 2*4*2048; i++ )
{
......@@ -243,12 +243,15 @@ static void x264_mb_analyse_load_costs( x264_t *h, x264_mb_analysis_t *a )
{
for( j=0; j<4; j++ )
{
x264_cost_mv_fpel[a->i_lambda][j] = x264_malloc( (4*2048 + 1) * sizeof(int16_t) );
CHECKED_MALLOC( x264_cost_mv_fpel[a->i_lambda][j], (4*2048 + 1) * sizeof(int16_t) );
x264_cost_mv_fpel[a->i_lambda][j] += 2*2048;
for( i = -2*2048; i < 2*2048; i++ )
x264_cost_mv_fpel[a->i_lambda][j][i] = p_cost_mv[a->i_lambda][i*4+j];
}
}
return 0;
fail:
return -1;
}
static void x264_mb_analyse_init( x264_t *h, x264_mb_analysis_t *a, int i_qp )
......@@ -2253,7 +2256,7 @@ static inline void x264_mb_analyse_qp_rd( x264_t *h, x264_mb_analysis_t *a )
/*****************************************************************************
* x264_macroblock_analyse:
*****************************************************************************/
void x264_macroblock_analyse( x264_t *h )
int x264_macroblock_analyse( x264_t *h )
{
x264_mb_analysis_t analysis;
int i_cost = COST_MAX;
......@@ -2328,12 +2331,13 @@ void x264_macroblock_analyse( x264_t *h )
int i_thresh16x8;
int i_satd_inter, i_satd_intra;
x264_mb_analyse_load_costs( h, &analysis );
if( x264_mb_analyse_load_costs( h, &analysis ) )
return -1;
x264_mb_analyse_inter_p16x16( h, &analysis );
if( h->mb.i_type == P_SKIP )
return;
return 0;
if( flags & X264_ANALYSE_PSUB16x16 )
{
......@@ -2620,7 +2624,8 @@ void x264_macroblock_analyse( x264_t *h )
int i_satd_inter = 0; // shut up uninitialized warning
h->mb.b_skip_mc = 0;
x264_mb_analyse_load_costs( h, &analysis );
if( x264_mb_analyse_load_costs( h, &analysis ) )
return -1;
/* select best inter mode */
/* direct must be first */
......@@ -2646,7 +2651,7 @@ void x264_macroblock_analyse( x264_t *h )
{
h->mb.i_type = B_SKIP;
x264_analyse_update_cache( h, &analysis );
return;
return 0;
}
}
......@@ -2871,6 +2876,7 @@ void x264_macroblock_analyse( x264_t *h )
x264_psy_trellis_init( h, 0 );
if( h->mb.b_trellis == 1 || h->mb.b_noise_reduction )
h->mb.i_skip_intra = 0;
return 0;
}
/*-------------------- Update MB from the analysis ----------------------*/
......
encoder/analyse.h
View file @ 6eab44d4
......@@ -24,7 +24,8 @@
#ifndef X264_ANALYSE_H
#define X264_ANALYSE_H
void x264_macroblock_analyse( x264_t *h );
int x264_macroblock_analyse( x264_t *h );
void x264_slicetype_decide( x264_t *h );
int x264_lowres_context_alloc( x264_t *h );
#endif
encoder/encoder.c
View file @ 6eab44d4
......@@ -43,8 +43,8 @@
#define bs_write_ue bs_write_ue_big
static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current,
x264_nal_t **pp_nal, int *pi_nal,
x264_picture_t *pic_out );
x264_nal_t **pp_nal, int *pi_nal,
x264_picture_t *pic_out );
/****************************************************************************
*
......@@ -295,17 +295,18 @@ static void x264_slice_header_write( bs_t *s, x264_slice_header_t *sh, int i_nal
/* If we are within a reasonable distance of the end of the memory allocated for the bitstream, */
/* reallocate, adding an arbitrary amount of space (100 kilobytes). */
static void x264_bitstream_check_buffer( x264_t *h )
static int x264_bitstream_check_buffer( x264_t *h )
{
uint8_t *bs_bak = h->out.p_bitstream;
if( ( h->param.b_cabac && (h->cabac.p_end - h->cabac.p < 2500) )
|| ( h->out.bs.p_end - h->out.bs.p < 2500 ) )
{
uint8_t *bs_bak = h->out.p_bitstream;
intptr_t delta;
int i;
h->out.i_bitstream += 100000;
h->out.p_bitstream = x264_realloc( h->out.p_bitstream, h->out.i_bitstream );
CHECKED_MALLOC( h->out.p_bitstream, h->out.i_bitstream );
h->mc.memcpy_aligned( h->out.p_bitstream, bs_bak, (h->out.i_bitstream - 100000) & ~15 );
delta = h->out.p_bitstream - bs_bak;
h->out.bs.p_start += delta;
......@@ -318,7 +319,12 @@ static void x264_bitstream_check_buffer( x264_t *h )
for( i = 0; i <= h->out.i_nal; i++ )
h->out.nal[i].p_payload += delta;
x264_free( bs_bak );
}
return 0;
fail:
x264_free( bs_bak );
return -1;
}
/****************************************************************************
......@@ -666,27 +672,21 @@ static void mbcmp_init( x264_t *h )
****************************************************************************/
x264_t *x264_encoder_open ( x264_param_t *param )
{
x264_t *h = x264_malloc( sizeof( x264_t ) );
x264_t *h;
char buf[1000], *p;
int i;
memset( h, 0, sizeof( x264_t ) );
CHECKED_MALLOCZERO( h, sizeof(x264_t) );
/* Create a copy of param */
memcpy( &h->param, param, sizeof( x264_param_t ) );
memcpy( &h->param, param, sizeof(x264_param_t) );
if( x264_validate_parameters( h ) < 0 )
{
x264_free( h );
return NULL;
}
goto fail;
if( h->param.psz_cqm_file )
if( x264_cqm_parse_file( h, h->param.psz_cqm_file ) < 0 )
{
x264_free( h );
return NULL;
}
goto fail;
if( h->param.rc.psz_stat_out )
h->param.rc.psz_stat_out = strdup( h->param.rc.psz_stat_out );
......@@ -737,10 +737,7 @@ x264_t *x264_encoder_open ( x264_param_t *param )
x264_validate_levels( h, 1 );
if( x264_cqm_init( h ) < 0 )
{
x264_free( h );
return NULL;
}
goto fail;
h->mb.i_mb_count = h->sps->i_mb_width * h->sps->i_mb_height;
......@@ -822,20 +819,25 @@ x264_t *x264_encoder_open ( x264_param_t *param )
h->thread[0] = h;
h->i_thread_num = 0;
for( i = 1; i < h->param.i_threads; i++ )
h->thread[i] = x264_malloc( sizeof(x264_t) );
CHECKED_MALLOC( h->thread[i], sizeof(x264_t) );
for( i = 0; i < h->param.i_threads; i++ )
{
if( i > 0 )
*h->thread[i] = *h;
h->thread[i]->fdec = x264_frame_pop_unused( h );
h->thread[i]->out.p_bitstream = x264_malloc( h->out.i_bitstream );
if( !h->thread[i]->fdec )
goto fail;
CHECKED_MALLOC( h->thread[i]->out.p_bitstream, h->out.i_bitstream );
if( x264_macroblock_cache_init( h->thread[i] ) < 0 )
return NULL;
goto fail;
}
if( x264_ratecontrol_new( h ) < 0 )
return NULL;
goto fail;
if( x264_lowres_context_alloc( h ) )
goto fail;
if( h->param.psz_dump_yuv )
{
......@@ -846,8 +848,7 @@ x264_t *x264_encoder_open ( x264_param_t *param )
else
{
x264_log( h, X264_LOG_ERROR, "can't write to fdec.yuv\n" );
x264_free( h );
return NULL;
goto fail;
}
}
......@@ -858,6 +859,9 @@ x264_t *x264_encoder_open ( x264_param_t *param )
"High 4:4:4 Predictive", h->sps->i_level_idc/10, h->sps->i_level_idc%10 );
return h;
fail:
x264_free( h );
return NULL;
}
/****************************************************************************
......@@ -938,7 +942,8 @@ int x264_encoder_headers( x264_t *h, x264_nal_t **pp_nal, int *pi_nal )
{
/* identify ourself */
x264_nal_start( h, NAL_SEI, NAL_PRIORITY_DISPOSABLE );
x264_sei_version_write( h, &h->out.bs );
if( x264_sei_version_write( h, &h->out.bs ) )
return -1;
x264_nal_end( h );
/* generate sequence parameters */
......@@ -1106,7 +1111,7 @@ static void x264_fdec_filter_row( x264_t *h, int mb_y )
}
}
static inline void x264_reference_update( x264_t *h )
static inline int x264_reference_update( x264_t *h )
{
int i;
......@@ -1119,8 +1124,10 @@ static inline void x264_reference_update( x264_t *h )
{
x264_frame_push_unused( h, h->fdec );
h->fdec = x264_frame_pop_unused( h );
if( !h->fdec )
return -1;
}
return;
return 0;
}
/* move lowres copy of the image to the ref frame */
......@@ -1139,6 +1146,9 @@ static inline void x264_reference_update( x264_t *h )
if( h->frames.reference[h->frames.i_max_dpb] )
x264_frame_push_unused( h, x264_frame_shift( h->frames.reference ) );
h->fdec = x264_frame_pop_unused( h );
if( !h->fdec )
return -1;
return 0;
}
static inline void x264_reference_reset( x264_t *h )
......@@ -1188,7 +1198,7 @@ static inline void x264_slice_init( x264_t *h, int i_nal_type, int i_global_qp )
x264_macroblock_slice_init( h );
}
static void x264_slice_write( x264_t *h )
static int x264_slice_write( x264_t *h )
{
int i_skip;
int mb_xy, i_mb_x, i_mb_y;
......@@ -1232,12 +1242,14 @@ static void x264_slice_write( x264_t *h )
* Slice I: choose I_4x4 or I_16x16 mode
* Slice P: choose between using P mode or intra (4x4 or 16x16)
* */
x264_macroblock_analyse( h );
if( x264_macroblock_analyse( h ) )
return -1;
/* encode this macroblock -> be careful it can change the mb type to P_SKIP if needed */
x264_macroblock_encode( h );
x264_bitstream_check_buffer( h );
if( x264_bitstream_check_buffer( h ) )
return -1;
if( h->param.b_cabac )
{
......@@ -1347,6 +1359,7 @@ static void x264_slice_write( x264_t *h )
+ NALU_OVERHEAD * 8
- h->stat.frame.i_tex_bits
- h->stat.frame.i_mv_bits;
return 0;
}
static void x264_thread_sync_context( x264_t *dst, x264_t *src )
......@@ -1375,7 +1388,7 @@ static void x264_thread_sync_stat( x264_t *dst, x264_t *src )
memcpy( &dst->stat.i_slice_count, &src->stat.i_slice_count, sizeof(dst->stat) - sizeof(dst->stat.frame) );
}
static int x264_slices_write( x264_t *h )
static void *x264_slices_write( x264_t *h )
{
int i_frame_size;
......@@ -1387,10 +1400,12 @@ static int x264_slices_write( x264_t *h )