Commit a6edfd66 authored by Loren Merritt's avatar Loren Merritt

fix an arithmetic overflow in trellis at high qp.


git-svn-id: svn://svn.videolan.org/x264/trunk@677 df754926-b1dd-0310-bc7b-ec298dee348c
parent 46343792
......@@ -397,8 +397,8 @@ static void quant_trellis_cabac( x264_t *h, int16_t *dct,
{
// no need to calculate ssd of 0s: it's the same in all nodes.
// no need to modify level_tree for ctx=0: it starts with an infinite loop of 0s.
const int cost_sig0 = x264_cabac_size_decision_noup( &cabac_state_sig[i], 0 )
* i_lambda2 >> ( CABAC_SIZE_BITS - LAMBDA_BITS );
const uint32_t cost_sig0 = x264_cabac_size_decision_noup( &cabac_state_sig[i], 0 )
* (uint64_t)i_lambda2 >> ( CABAC_SIZE_BITS - LAMBDA_BITS );
for( j = 1; j < 8; j++ )
{
if( nodes_cur[j].score != TRELLIS_SCORE_MAX )
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment