Commit a0253ebe authored by Anton Mitrofanov's avatar Anton Mitrofanov Committed by Henrik Gramner
Browse files

Fix integer overflow in slicetype_path_cost

The path cost for high resolutions can exceed COST_MAX.
parent 2af27428
...@@ -1277,10 +1277,10 @@ static void vbv_lookahead( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **fram ...@@ -1277,10 +1277,10 @@ static void vbv_lookahead( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **fram
frames[next_nonb]->i_planned_type[idx] = X264_TYPE_AUTO; frames[next_nonb]->i_planned_type[idx] = X264_TYPE_AUTO;
} }
static int slicetype_path_cost( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **frames, char *path, int threshold ) static uint64_t slicetype_path_cost( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **frames, char *path, uint64_t threshold )
{ {
uint64_t cost = 0;
int loc = 1; int loc = 1;
int cost = 0;
int cur_nonb = 0; int cur_nonb = 0;
path--; /* Since the 1st path element is really the second frame */ path--; /* Since the 1st path element is really the second frame */
while( path[loc] ) while( path[loc] )
...@@ -1326,7 +1326,7 @@ static void slicetype_path( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **fra ...@@ -1326,7 +1326,7 @@ static void slicetype_path( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **fra
{ {
char paths[2][X264_LOOKAHEAD_MAX+1]; char paths[2][X264_LOOKAHEAD_MAX+1];
int num_paths = X264_MIN( h->param.i_bframe+1, length ); int num_paths = X264_MIN( h->param.i_bframe+1, length );
int best_cost = COST_MAX; uint64_t best_cost = COST_MAX64;
int best_possible = 0; int best_possible = 0;
int idx = 0; int idx = 0;
...@@ -1357,9 +1357,9 @@ static void slicetype_path( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **fra ...@@ -1357,9 +1357,9 @@ static void slicetype_path( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **fra
if( possible || !best_possible ) if( possible || !best_possible )
{ {
if( possible && !best_possible ) if( possible && !best_possible )
best_cost = COST_MAX; best_cost = COST_MAX64;
/* Calculate the actual cost of the current path */ /* Calculate the actual cost of the current path */
int cost = slicetype_path_cost( h, a, frames, paths[idx], best_cost ); uint64_t cost = slicetype_path_cost( h, a, frames, paths[idx], best_cost );
if( cost < best_cost ) if( cost < best_cost )
{ {
best_cost = cost; best_cost = cost;
...@@ -1602,9 +1602,9 @@ void x264_slicetype_analyse( x264_t *h, int intra_minigop ) ...@@ -1602,9 +1602,9 @@ void x264_slicetype_analyse( x264_t *h, int intra_minigop )
int bframes = j - last_nonb - 1; int bframes = j - last_nonb - 1;
memset( path, 'B', bframes ); memset( path, 'B', bframes );
strcpy( path+bframes, "PP" ); strcpy( path+bframes, "PP" );
int cost_p = slicetype_path_cost( h, &a, frames+last_nonb, path, COST_MAX ); uint64_t cost_p = slicetype_path_cost( h, &a, frames+last_nonb, path, COST_MAX64 );
strcpy( path+bframes, "BP" ); strcpy( path+bframes, "BP" );
int cost_b = slicetype_path_cost( h, &a, frames+last_nonb, path, cost_p ); uint64_t cost_b = slicetype_path_cost( h, &a, frames+last_nonb, path, cost_p );
if( cost_b < cost_p ) if( cost_b < cost_p )
frames[j]->i_type = X264_TYPE_B; frames[j]->i_type = X264_TYPE_B;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment