buffer overrun when bframes == X264_BFRAME_MAX

git-svn-id: svn://svn.videolan.org/x264/trunk@155 df754926-b1dd-0310-bc7b-ec298dee348c

common/common.h

@@ -235,17 +235,17 @@ struct x264_t
 
     struct
     {
-        /* Frames to be encoded */
-        x264_frame_t *current[X264_BFRAME_MAX+1];
-        /* Temporary buffer (eg B frames pending until we reach the I/P) */
-        x264_frame_t *next[X264_BFRAME_MAX+1];
+        /* Frames to be encoded (whose types have been decided) */
+        x264_frame_t *current[X264_BFRAME_MAX+3];
+        /* Temporary buffer (frames types not yet decided) */
+        x264_frame_t *next[X264_BFRAME_MAX+3];
         /* Unused frames */
-        x264_frame_t *unused[X264_BFRAME_MAX+1];
+        x264_frame_t *unused[X264_BFRAME_MAX+3];
         /* For adaptive B decision */
         x264_frame_t *last_nonb;
 
-        /* frames used for reference +1 for decoding +1 sentinel */
-        x264_frame_t *reference[16+2+1+1];
+        /* frames used for reference +1 for decoding + sentinels */
+        x264_frame_t *reference[16+2+1+2];
 
         int i_last_idr; /* Frame number of the last IDR */
 

common/macroblock.c

@@ -939,23 +939,23 @@ void x264_macroblock_cache_load( x264_t *h, int i_mb_x, int i_mb_y )
     {
         const int w = (i == 0 ? 16 : 8);
         const int i_stride = h->fdec->i_stride[i];
+        const int i_xy = i_mb_x + i_mb_y * i_stride;
         int   j;
 
         h->mb.pic.i_stride[i] = i_stride;
 
-        h->mb.pic.p_fenc[i] = &h->fenc->plane[i][ w * ( i_mb_x + i_mb_y * i_stride )];
-
-        h->mb.pic.p_fdec[i] = &h->fdec->plane[i][ w * ( i_mb_x + i_mb_y * i_stride )];
+        h->mb.pic.p_fenc[i] = &h->fenc->plane[i][w*i_xy];
+        h->mb.pic.p_fdec[i] = &h->fdec->plane[i][w*i_xy];
 
         for( j = 0; j < h->i_ref0; j++ )
         {
-            h->mb.pic.p_fref[0][j][i==0 ? 0:i+3] = &h->fref0[j]->plane[i][ w * ( i_mb_x + i_mb_y * i_stride )];
-            h->mb.pic.p_fref[0][j][i+1] = &h->fref0[j]->filtered[i+1][ 16 * ( i_mb_x + i_mb_y * h->fdec->i_stride[0] )];
+            h->mb.pic.p_fref[0][j][i==0 ? 0:i+3] = &h->fref0[j]->plane[i][w*i_xy];
+            h->mb.pic.p_fref[0][j][i+1] = &h->fref0[j]->filtered[i+1][16*i_xy];
         }
         for( j = 0; j < h->i_ref1; j++ )
         {
-            h->mb.pic.p_fref[1][j][i==0 ? 0:i+3] = &h->fref1[j]->plane[i][ w * ( i_mb_x + i_mb_y * i_stride )];
-            h->mb.pic.p_fref[1][j][i+1] = &h->fref1[j]->filtered[i+1][ 16 * ( i_mb_x + i_mb_y * h->fdec->i_stride[0] )];
+            h->mb.pic.p_fref[1][j][i==0 ? 0:i+3] = &h->fref1[j]->plane[i][w*i_xy];
+            h->mb.pic.p_fref[1][j][i+1] = &h->fref1[j]->filtered[i+1][16*i_xy];
         }
     }
 

encoder/encoder.c

@@ -461,7 +461,7 @@ x264_t *x264_encoder_open   ( x264_param_t *param )
     h->mb.i_mb_count = h->sps->i_mb_width * h->sps->i_mb_height;
 
     /* Init frames. */
-    for( i = 0; i < X264_BFRAME_MAX + 1; i++ )
+    for( i = 0; i < X264_BFRAME_MAX + 3; i++ )
     {
         h->frames.current[i] = NULL;
         h->frames.next[i]    = NULL;
@@ -478,6 +478,7 @@ x264_t *x264_encoder_open   ( x264_param_t *param )
     h->frames.reference[h->frames.i_max_dpb] = NULL;
     h->frames.i_last_idr = - h->param.i_keyint_max;
     h->frames.i_input    = 0;
+    h->frames.last_nonb  = NULL;
 
     h->i_ref0 = 0;
     h->i_ref1 = 0;
@@ -612,7 +613,7 @@ static x264_frame_t *x264_frame_get( x264_frame_t *list[X264_BFRAME_MAX+1] )
 {
     x264_frame_t *frame = list[0];
     int i;
-    for( i = 0; i < X264_BFRAME_MAX && list[i]; i++ )
+    for( i = 0; list[i]; i++ )
         list[i] = list[i+1];
     return frame;
 }
@@ -622,7 +623,7 @@ static void x264_frame_sort( x264_frame_t *list[X264_BFRAME_MAX+1], int b_dts )
     int i, b_ok;
     do {
         b_ok = 1;
-        for( i = 0; i < X264_BFRAME_MAX && list[i+1]; i++ )
+        for( i = 0; list[i+1]; i++ )
         {
             int dtype = list[i]->i_type - list[i+1]->i_type;
             int dtime = list[i]->i_frame - list[i+1]->i_frame;
@@ -1528,7 +1529,7 @@ void    x264_encoder_close  ( x264_t *h )
     }
 
     /* frames */
-    for( i = 0; i < X264_BFRAME_MAX + 1; i++ )
+    for( i = 0; i < X264_BFRAME_MAX + 3; i++ )
     {
         if( h->frames.current[i] ) x264_frame_delete( h->frames.current[i] );
         if( h->frames.next[i] )    x264_frame_delete( h->frames.next[i] );