Fix integer overflow in cabac psy-trellis

encoder/rdo.c

@@ -471,7 +471,7 @@ int trellis_dc_shortcut( int sign_coef, int quant_coef, int unquant_mf, int coef
 
         /* Optimize rounding for DC coefficients in DC-only luma 4x4/8x8 blocks. */
         int d = sign_coef - ((SIGN(unquant_abs_level, sign_coef) + 8)&~15);
-        uint64_t score = (uint64_t)d*d * coef_weight;
+        uint64_t score = (int64_t)d*d * coef_weight;
 
         /* code the proposed level, and count how much entropy it would take */
         if( abs_level )
@@ -734,11 +734,11 @@ int quant_trellis_cabac( x264_t *h, dctcoef *dct,
     trellis_level_t level_tree[64*8*2];
     int levels_used = 1;
     /* init trellis */
-    trellis_node_t nodes[2][8];
+    trellis_node_t nodes[2][8] = {0};
     trellis_node_t *nodes_cur = nodes[0];
     trellis_node_t *nodes_prev = nodes[1];
     trellis_node_t *bnode;
-    for( int j = 1; j < 4; j++ )
+    for( int j = 1; j < 8; j++ )
         nodes_cur[j].score = TRELLIS_SCORE_MAX;
     nodes_cur[0].score = TRELLIS_SCORE_BIAS;
     nodes_cur[0].level_idx = 0;
@@ -825,17 +825,18 @@ int quant_trellis_cabac( x264_t *h, dctcoef *dct,
                 int predicted_coef = orig_coef - sign_coef;\
                 int psy_value = abs(unquant_abs_level + SIGN(predicted_coef, sign_coef));\
                 int psy_weight = coef_weight1[zigzag[i]] * h->mb.i_psy_trellis;\
-                ssd1[k] = (uint64_t)d*d * coef_weight2[zigzag[i]] - psy_weight * psy_value;\
+                int64_t tmp = (int64_t)d*d * coef_weight2[zigzag[i]] - (int64_t)psy_weight * psy_value;\
+                ssd1[k] = (uint64_t)tmp;\
             }\
             else\
             /* FIXME: for i16x16 dc is this weight optimal? */\
-                ssd1[k] = (uint64_t)d*d * (dc?256:coef_weight2[zigzag[i]]);\
+                ssd1[k] = (int64_t)d*d * (dc?256:coef_weight2[zigzag[i]]);\
             ssd0[k] = ssd1[k];\
             if( !i && !dc && !ctx_hi )\
             {\
                 /* Optimize rounding for DC coefficients in DC-only luma 4x4/8x8 blocks. */\
                 d = sign_coef - ((SIGN(unquant_abs_level, sign_coef) + 8)&~15);\
-                ssd0[k] = (uint64_t)d*d * coef_weight2[zigzag[i]];\
+                ssd0[k] = (int64_t)d*d * coef_weight2[zigzag[i]];\
             }\
         }\
 \