Fix some uses of uninitialized row_satd values in VBV

Resolves some issues with QP51 in I-frames with scenecut

common/mc.c

@@ -299,9 +299,11 @@ void x264_frame_init_lowres( x264_t *h, x264_frame_t *frame )
                                   i_stride, frame->i_stride_lowres, frame->i_width_lowres, frame->i_lines_lowres );
     x264_frame_expand_border_lowres( frame );
 
-    for( y=0; y<16; y++ )
-        for( x=0; x<16; x++ )
-            frame->i_cost_est[y][x] = -1;
+    memset( frame->i_cost_est, -1, sizeof(frame->i_cost_est) );
+
+    for( x = 0; x < h->param.i_bframe + 2; x++ )
+        for( y = 0; y < h->param.i_bframe + 2; y++ )
+            frame->i_row_satds[y][x][0] = -1;
 }
 
 static void frame_init_lowres_core( uint8_t *src0, uint8_t *dst0, uint8_t *dsth, uint8_t *dstv, uint8_t *dstc,

encoder/slicetype.c

@@ -247,7 +247,8 @@ static int x264_slicetype_frame_cost( x264_t *h, x264_mb_analysis_t *a,
     /* Check whether we already evaluated this frame
      * If we have tried this frame as P, then we have also tried
      * the preceding frames as B. (is this still true?) */
-    if( frames[b]->i_cost_est[b-p0][p1-b] >= 0 )
+    /* Also check that we already calculated the row SATDs for the current frame. */
+    if( frames[b]->i_cost_est[b-p0][p1-b] >= 0 && frames[b]->i_row_satds[b-p0][p1-b][0] != -1 )
     {
         i_score = frames[b]->i_cost_est[b-p0][p1-b];
     }