Commit 26c83034 authored by Loren Merritt's avatar Loren Merritt Committed by Fiona Glaser

Fix trellis 2 + subme >= 8

Trellis didn't return a boolean value as it was supposed to.
Regression in r2143-5.
parent 7d804baf
......@@ -261,9 +261,7 @@ cglobal %1, 4,15,9
inc iiq
jle .writeback_loop
%if dc
mov eax, 1
%endif
.return:
ADD rsp, pad
RET
......@@ -724,6 +722,7 @@ TRELLIS_COEF0 1
mov [nodes_curq + node_cabac_state(node_ctx) + (coeff_abs_level1_offs>>2)], r11b ; delayed from x264_cabac_size_decision2
%endif
%if %1 && node_ctx == 7
mov r6d, levelgt1_ctxm
mov [nodes_curq + node_cabac_state(node_ctx) + coeff_abs_levelgt1_offs-6], r10b
%endif
mov r6d, [nodes_prevq + node_level_idx(j)]
......@@ -786,8 +785,8 @@ TRELLIS_COEF0 1
add bitsd, r5d ; bs_size_ue_big from COEFN_SUFFIX
; n.cabac_state[levelgt1_ctx]
%if j == 7 ; && compiling support for 4:2:2
mov r5d, levelgt1_ctxm
%define coeff_abs_levelgt1_offs r5
mov r6d, levelgt1_ctxm
%define coeff_abs_levelgt1_offs r6
%endif
%if j == 7
movzx r10, byte [nodes_prevq + node_cabac_state(j) + coeff_abs_levelgt1_offs-6] ; -6 because node only stores ctx 8 and 9
......
......@@ -691,7 +691,8 @@ int quant_trellis_cabac( x264_t *h, dctcoef *dct,
{
int cost_sig = x264_cabac_size_decision_noup2( &cabac_state_sig[0], 1 )
+ x264_cabac_size_decision_noup2( &cabac_state_last[0], 1 );
return dct[0] = trellis_dc_shortcut( orig_coefs[0], quant_coefs[0], unquant_mf[0], coef_weight2[0], lambda2, cabac_state, cost_sig );
dct[0] = trellis_dc_shortcut( orig_coefs[0], quant_coefs[0], unquant_mf[0], coef_weight2[0], lambda2, cabac_state, cost_sig );
return !!dct[0];
}
#if HAVE_MMX && ARCH_X86_64
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment