macroblock.c 31.6 KB
Newer Older
Laurent Aimar's avatar
Laurent Aimar committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
/*****************************************************************************
 * macroblock.c: h264 encoder library
 *****************************************************************************
 * Copyright (C) 2003 Laurent Aimar
 * $Id: macroblock.c,v 1.1 2004/06/03 19:27:08 fenrir Exp $
 *
 * Authors: Laurent Aimar <fenrir@via.ecp.fr>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111, USA.
 *****************************************************************************/

#include <stdio.h>
#include <string.h>

27
#include "common/common.h"
Laurent Aimar's avatar
Laurent Aimar committed
28 29 30
#include "macroblock.h"


Loren Merritt's avatar
Loren Merritt committed
31 32 33 34
/* def_quant4_mf only for probe_skip; actual encoding uses matrices from set.c */
/* FIXME this seems to make better decisions with cqm=jvt, but could screw up
 * with general custom matrices. */
static const int def_quant4_mf[6][4][4] =
Laurent Aimar's avatar
Laurent Aimar committed
35
{
36 37 38 39 40 41 42 43 44 45 46 47 48 49
    { { 13107, 8066, 13107, 8066 }, { 8066, 5243, 8066, 5243 },
      { 13107, 8066, 13107, 8066 }, { 8066, 5243, 8066, 5243 } },
    { { 11916, 7490, 11916, 7490 }, { 7490, 4660, 7490, 4660 },
      { 11916, 7490, 11916, 7490 }, { 7490, 4660, 7490, 4660 } },
    { { 10082, 6554, 10082, 6554 }, { 6554, 4194, 6554, 4194 },
      { 10082, 6554, 10082, 6554 }, { 6554, 4194, 6554, 4194 } },
    { {  9362, 5825,  9362, 5825 }, { 5825, 3647, 5825, 3647 },
      {  9362, 5825,  9362, 5825 }, { 5825, 3647, 5825, 3647 } },
    { {  8192, 5243,  8192, 5243 }, { 5243, 3355, 5243, 3355 },
      {  8192, 5243,  8192, 5243 }, { 5243, 3355, 5243, 3355 } },
    { {  7282, 4559,  7282, 4559 }, { 4559, 2893, 4559, 2893 },
      {  7282, 4559,  7282, 4559 }, { 4559, 2893, 4559, 2893 } }
};

Laurent Aimar's avatar
Laurent Aimar committed
50 51 52 53
/****************************************************************************
 * Scan and Quant functions
 ****************************************************************************/

54
#define ZIG(i,y,x) level[i] = dct[x][y];
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
static inline void scan_zigzag_8x8full( int level[64], int16_t dct[8][8] )
{
    ZIG( 0,0,0) ZIG( 1,0,1) ZIG( 2,1,0) ZIG( 3,2,0)
    ZIG( 4,1,1) ZIG( 5,0,2) ZIG( 6,0,3) ZIG( 7,1,2)
    ZIG( 8,2,1) ZIG( 9,3,0) ZIG(10,4,0) ZIG(11,3,1)
    ZIG(12,2,2) ZIG(13,1,3) ZIG(14,0,4) ZIG(15,0,5)
    ZIG(16,1,4) ZIG(17,2,3) ZIG(18,3,2) ZIG(19,4,1)
    ZIG(20,5,0) ZIG(21,6,0) ZIG(22,5,1) ZIG(23,4,2)
    ZIG(24,3,3) ZIG(25,2,4) ZIG(26,1,5) ZIG(27,0,6)
    ZIG(28,0,7) ZIG(29,1,6) ZIG(30,2,5) ZIG(31,3,4)
    ZIG(32,4,3) ZIG(33,5,2) ZIG(34,6,1) ZIG(35,7,0)
    ZIG(36,7,1) ZIG(37,6,2) ZIG(38,5,3) ZIG(39,4,4)
    ZIG(40,3,5) ZIG(41,2,6) ZIG(42,1,7) ZIG(43,2,7)
    ZIG(44,3,6) ZIG(45,4,5) ZIG(46,5,4) ZIG(47,6,3)
    ZIG(48,7,2) ZIG(49,7,3) ZIG(50,6,4) ZIG(51,5,5)
    ZIG(52,4,6) ZIG(53,3,7) ZIG(54,4,7) ZIG(55,5,6)
    ZIG(56,6,5) ZIG(57,7,4) ZIG(58,7,5) ZIG(59,6,6)
    ZIG(60,5,7) ZIG(61,6,7) ZIG(62,7,6) ZIG(63,7,7)
}
Laurent Aimar's avatar
Laurent Aimar committed
74 75
static inline void scan_zigzag_4x4full( int level[16], int16_t dct[4][4] )
{
76 77 78 79
    ZIG( 0,0,0) ZIG( 1,0,1) ZIG( 2,1,0) ZIG( 3,2,0)
    ZIG( 4,1,1) ZIG( 5,0,2) ZIG( 6,0,3) ZIG( 7,1,2)
    ZIG( 8,2,1) ZIG( 9,3,0) ZIG(10,3,1) ZIG(11,2,2)
    ZIG(12,1,3) ZIG(13,2,3) ZIG(14,3,2) ZIG(15,3,3)
Laurent Aimar's avatar
Laurent Aimar committed
80 81 82
}
static inline void scan_zigzag_4x4( int level[15], int16_t dct[4][4] )
{
83 84 85 86
                ZIG( 0,0,1) ZIG( 1,1,0) ZIG( 2,2,0)
    ZIG( 3,1,1) ZIG( 4,0,2) ZIG( 5,0,3) ZIG( 6,1,2)
    ZIG( 7,2,1) ZIG( 8,3,0) ZIG( 9,3,1) ZIG(10,2,2)
    ZIG(11,1,3) ZIG(12,2,3) ZIG(13,3,2) ZIG(14,3,3)
Laurent Aimar's avatar
Laurent Aimar committed
87 88 89
}
static inline void scan_zigzag_2x2_dc( int level[4], int16_t dct[2][2] )
{
90 91 92 93
    ZIG(0,0,0)
    ZIG(1,0,1)
    ZIG(2,1,0)
    ZIG(3,1,1)
Laurent Aimar's avatar
Laurent Aimar committed
94
}
95
#undef ZIG
Laurent Aimar's avatar
Laurent Aimar committed
96

Loren Merritt's avatar
Loren Merritt committed
97
#define ZIG(i,y,x) {\
98 99 100 101
    int oe = x+y*FENC_STRIDE;\
    int od = x+y*FDEC_STRIDE;\
    level[i] = p_src[oe] - p_dst[od];\
    p_dst[od] = p_src[oe];\
Loren Merritt's avatar
Loren Merritt committed
102
}
103
static inline void sub_zigzag_4x4full( int level[16], const uint8_t *p_src, uint8_t *p_dst )
Loren Merritt's avatar
Loren Merritt committed
104 105 106 107 108 109
{
    ZIG( 0,0,0) ZIG( 1,0,1) ZIG( 2,1,0) ZIG( 3,2,0)
    ZIG( 4,1,1) ZIG( 5,0,2) ZIG( 6,0,3) ZIG( 7,1,2)
    ZIG( 8,2,1) ZIG( 9,3,0) ZIG(10,3,1) ZIG(11,2,2)
    ZIG(12,1,3) ZIG(13,2,3) ZIG(14,3,2) ZIG(15,3,3)
}
110
static inline void sub_zigzag_4x4( int level[15], const uint8_t *p_src, uint8_t *p_dst )
Loren Merritt's avatar
Loren Merritt committed
111 112 113 114 115 116 117 118
{
                ZIG( 0,0,1) ZIG( 1,1,0) ZIG( 2,2,0)
    ZIG( 3,1,1) ZIG( 4,0,2) ZIG( 5,0,3) ZIG( 6,1,2)
    ZIG( 7,2,1) ZIG( 8,3,0) ZIG( 9,3,1) ZIG(10,2,2)
    ZIG(11,1,3) ZIG(12,2,3) ZIG(13,3,2) ZIG(14,3,3)
}
#undef ZIG

119
static void quant_8x8( x264_t *h, int16_t dct[8][8], int quant_mf[6][8][8], int i_qscale, int b_intra )
120 121 122 123
{
    const int i_qbits = 16 + i_qscale / 6;
    const int i_mf = i_qscale % 6;
    const int f = ( 1 << i_qbits ) / ( b_intra ? 3 : 6 );
124
    h->quantf.quant_8x8_core( dct, quant_mf[i_mf], i_qbits, f );
125
}
126
static void quant_4x4( x264_t *h, int16_t dct[4][4], int quant_mf[6][4][4], int i_qscale, int b_intra )
Laurent Aimar's avatar
Laurent Aimar committed
127 128 129 130
{
    const int i_qbits = 15 + i_qscale / 6;
    const int i_mf = i_qscale % 6;
    const int f = ( 1 << i_qbits ) / ( b_intra ? 3 : 6 );
131
    h->quantf.quant_4x4_core( dct, quant_mf[i_mf], i_qbits, f );
Laurent Aimar's avatar
Laurent Aimar committed
132
}
133
static void quant_4x4_dc( x264_t *h, int16_t dct[4][4], int quant_mf[6][4][4], int i_qscale )
Laurent Aimar's avatar
Laurent Aimar committed
134
{
135 136 137 138
    const int i_qbits = 16 + i_qscale / 6;
    const int i_mf = i_qscale % 6;
    const int f = ( 1 << i_qbits ) / 3;
    h->quantf.quant_4x4_dc_core( dct, quant_mf[i_mf][0][0], i_qbits, f );
Laurent Aimar's avatar
Laurent Aimar committed
139
}
140
static void quant_2x2_dc( x264_t *h, int16_t dct[2][2], int quant_mf[6][4][4], int i_qscale, int b_intra )
Laurent Aimar's avatar
Laurent Aimar committed
141
{
142 143 144 145
    const int i_qbits = 16 + i_qscale / 6;
    const int i_mf = i_qscale % 6;
    const int f = ( 1 << i_qbits ) / ( b_intra ? 3 : 6 );
    h->quantf.quant_2x2_dc_core( dct, quant_mf[i_mf][0][0], i_qbits, f );
Laurent Aimar's avatar
Laurent Aimar committed
146
}
147

Laurent Aimar's avatar
Laurent Aimar committed
148 149 150 151 152 153 154 155 156 157
/* (ref: JVT-B118)
 * x264_mb_decimate_score: given dct coeffs it returns a score to see if we could empty this dct coeffs
 * to 0 (low score means set it to null)
 * Used in inter macroblock (luma and chroma)
 *  luma: for a 8x8 block: if score < 4 -> null
 *        for the complete mb: if score < 6 -> null
 *  chroma: for the complete mb: if score < 7 -> null
 */
static int x264_mb_decimate_score( int *dct, int i_max )
{
158 159 160 161 162 163 164 165 166
    static const int i_ds_table4[16] = {
        3,2,2,1,1,1,0,0,0,0,0,0,0,0,0,0 };
    static const int i_ds_table8[64] = {
        3,3,3,3,2,2,2,2,2,2,2,2,1,1,1,1,
        1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,
        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };

    const int *ds_table = (i_max == 64) ? i_ds_table8 : i_ds_table4;
Laurent Aimar's avatar
Laurent Aimar committed
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
    int i_score = 0;
    int idx = i_max - 1;

    while( idx >= 0 && dct[idx] == 0 )
        idx--;

    while( idx >= 0 )
    {
        int i_run;

        if( abs( dct[idx--] ) > 1 )
            return 9;

        i_run = 0;
        while( idx >= 0 && dct[idx] == 0 )
        {
            idx--;
            i_run++;
        }
186
        i_score += ds_table[i_run];
Laurent Aimar's avatar
Laurent Aimar committed
187 188 189 190 191 192 193
    }

    return i_score;
}

void x264_mb_encode_i4x4( x264_t *h, int idx, int i_qscale )
{
194 195 196 197
    int x = 4 * block_idx_x[idx];
    int y = 4 * block_idx_y[idx];
    uint8_t *p_src = &h->mb.pic.p_fenc[0][x+y*FENC_STRIDE];
    uint8_t *p_dst = &h->mb.pic.p_fdec[0][x+y*FDEC_STRIDE];
Laurent Aimar's avatar
Laurent Aimar committed
198 199
    int16_t dct4x4[4][4];

Loren Merritt's avatar
Loren Merritt committed
200 201
    if( h->mb.b_lossless )
    {
202
        sub_zigzag_4x4full( h->dct.block[idx].luma4x4, p_src, p_dst );
Loren Merritt's avatar
Loren Merritt committed
203 204 205
        return;
    }

206
    h->dctf.sub4x4_dct( dct4x4, p_src, FENC_STRIDE, p_dst, FDEC_STRIDE );
Loren Merritt's avatar
Loren Merritt committed
207 208 209 210 211 212

    if( h->mb.b_trellis )
        x264_quant_4x4_trellis( h, dct4x4, CQM_4IY, i_qscale, DCT_LUMA_4x4, 1 );
    else
        quant_4x4( h, dct4x4, h->quant4_mf[CQM_4IY], i_qscale, 1 );

Laurent Aimar's avatar
Laurent Aimar committed
213
    scan_zigzag_4x4full( h->dct.block[idx].luma4x4, dct4x4 );
214
    h->quantf.dequant_4x4( dct4x4, h->dequant4_mf[CQM_4IY], i_qscale );
Laurent Aimar's avatar
Laurent Aimar committed
215 216

    /* output samples to fdec */
217
    h->dctf.add4x4_idct( p_dst, FDEC_STRIDE, dct4x4 );
Laurent Aimar's avatar
Laurent Aimar committed
218 219
}

220 221
void x264_mb_encode_i8x8( x264_t *h, int idx, int i_qscale )
{
222 223 224 225
    int x = 8 * (idx&1);
    int y = 8 * (idx>>1);
    uint8_t *p_src = &h->mb.pic.p_fenc[0][x+y*FENC_STRIDE];
    uint8_t *p_dst = &h->mb.pic.p_fdec[0][x+y*FDEC_STRIDE];
226 227
    int16_t dct8x8[8][8];

228
    h->dctf.sub8x8_dct8( dct8x8, p_src, FENC_STRIDE, p_dst, FDEC_STRIDE );
Loren Merritt's avatar
Loren Merritt committed
229 230 231 232 233 234

    if( h->mb.b_trellis )
        x264_quant_8x8_trellis( h, dct8x8, CQM_8IY, i_qscale, 1 );
    else 
        quant_8x8( h, dct8x8, h->quant8_mf[CQM_8IY], i_qscale, 1 );

235
    scan_zigzag_8x8full( h->dct.luma8x8[idx], dct8x8 );
236
    h->quantf.dequant_8x8( dct8x8, h->dequant8_mf[CQM_8IY], i_qscale );
237
    h->dctf.add8x8_idct8( p_dst, FDEC_STRIDE, dct8x8 );
238 239
}

Laurent Aimar's avatar
Laurent Aimar committed
240 241 242 243 244 245 246 247 248
static void x264_mb_encode_i16x16( x264_t *h, int i_qscale )
{
    uint8_t  *p_src = h->mb.pic.p_fenc[0];
    uint8_t  *p_dst = h->mb.pic.p_fdec[0];

    int16_t dct4x4[16+1][4][4];

    int i;

Loren Merritt's avatar
Loren Merritt committed
249 250 251 252
    if( h->mb.b_lossless )
    {
        for( i = 0; i < 16; i++ )
        {
253 254 255 256 257
            int oe = block_idx_x[i]*4 + block_idx_y[i]*4*FENC_STRIDE;
            int od = block_idx_x[i]*4 + block_idx_y[i]*4*FDEC_STRIDE;
            sub_zigzag_4x4( h->dct.block[i].residual_ac, p_src+oe, p_dst+od );
            dct4x4[0][block_idx_y[i]][block_idx_x[i]] = p_src[oe] - p_dst[od];
            p_dst[od] = p_src[oe];
Loren Merritt's avatar
Loren Merritt committed
258 259 260 261 262
        }
        scan_zigzag_4x4full( h->dct.luma16x16_dc, dct4x4[0] );
        return;
    }

263
    h->dctf.sub16x16_dct( &dct4x4[1], p_src, FENC_STRIDE, p_dst, FDEC_STRIDE );
Laurent Aimar's avatar
Laurent Aimar committed
264 265 266 267 268 269
    for( i = 0; i < 16; i++ )
    {
        /* copy dc coeff */
        dct4x4[0][block_idx_y[i]][block_idx_x[i]] = dct4x4[1+i][0][0];

        /* quant/scan/dequant */
Loren Merritt's avatar
Loren Merritt committed
270 271 272 273 274
        if( h->mb.b_trellis )
            x264_quant_4x4_trellis( h, dct4x4[1+i], CQM_4IY, i_qscale, DCT_LUMA_AC, 1 );
        else
            quant_4x4( h, dct4x4[1+i], h->quant4_mf[CQM_4IY], i_qscale, 1 );

Laurent Aimar's avatar
Laurent Aimar committed
275
        scan_zigzag_4x4( h->dct.block[i].residual_ac, dct4x4[1+i] );
276
        h->quantf.dequant_4x4( dct4x4[1+i], h->dequant4_mf[CQM_4IY], i_qscale );
Laurent Aimar's avatar
Laurent Aimar committed
277 278 279
    }

    h->dctf.dct4x4dc( dct4x4[0] );
280
    quant_4x4_dc( h, dct4x4[0], h->quant4_mf[CQM_4IY], i_qscale );
Laurent Aimar's avatar
Laurent Aimar committed
281 282 283 284
    scan_zigzag_4x4full( h->dct.luma16x16_dc, dct4x4[0] );

    /* output samples to fdec */
    h->dctf.idct4x4dc( dct4x4[0] );
Loren Merritt's avatar
Loren Merritt committed
285
    x264_mb_dequant_4x4_dc( dct4x4[0], h->dequant4_mf[CQM_4IY], i_qscale );  /* XXX not inversed */
Laurent Aimar's avatar
Laurent Aimar committed
286 287 288 289 290 291 292 293

    /* calculate dct coeffs */
    for( i = 0; i < 16; i++ )
    {
        /* copy dc coeff */
        dct4x4[1+i][0][0] = dct4x4[0][block_idx_y[i]][block_idx_x[i]];
    }
    /* put pixels to fdec */
294
    h->dctf.add16x16_idct( p_dst, FDEC_STRIDE, &dct4x4[1] );
Laurent Aimar's avatar
Laurent Aimar committed
295 296
}

297
static void x264_mb_encode_8x8_chroma( x264_t *h, int b_inter, int i_qscale )
Laurent Aimar's avatar
Laurent Aimar committed
298 299 300 301 302 303 304 305 306 307 308 309
{
    int i, ch;

    for( ch = 0; ch < 2; ch++ )
    {
        uint8_t  *p_src = h->mb.pic.p_fenc[1+ch];
        uint8_t  *p_dst = h->mb.pic.p_fdec[1+ch];
        int i_decimate_score = 0;

        int16_t dct2x2[2][2];
        int16_t dct4x4[4][4][4];

Loren Merritt's avatar
Loren Merritt committed
310 311 312 313
        if( h->mb.b_lossless )
        {
            for( i = 0; i < 4; i++ )
            {
314 315 316 317 318
                int oe = block_idx_x[i]*4 + block_idx_y[i]*4*FENC_STRIDE;
                int od = block_idx_x[i]*4 + block_idx_y[i]*4*FDEC_STRIDE;
                sub_zigzag_4x4( h->dct.block[16+i+ch*4].residual_ac, p_src+oe, p_dst+od );
                h->dct.chroma_dc[ch][i] = p_src[oe] - p_dst[od];
                p_dst[od] = p_src[oe];
Loren Merritt's avatar
Loren Merritt committed
319 320 321 322
            }
            continue;
        }
            
323
        h->dctf.sub8x8_dct( dct4x4, p_src, FENC_STRIDE, p_dst, FDEC_STRIDE );
Laurent Aimar's avatar
Laurent Aimar committed
324 325 326 327 328 329
        /* calculate dct coeffs */
        for( i = 0; i < 4; i++ )
        {
            /* copy dc coeff */
            dct2x2[block_idx_y[i]][block_idx_x[i]] = dct4x4[i][0][0];

Loren Merritt's avatar
Loren Merritt committed
330
            /* no trellis; it doesn't seem to help chroma noticeably */
331
            quant_4x4( h, dct4x4[i], h->quant4_mf[CQM_4IC + b_inter], i_qscale, !b_inter );
Laurent Aimar's avatar
Laurent Aimar committed
332 333 334 335 336 337 338 339 340
            scan_zigzag_4x4( h->dct.block[16+i+ch*4].residual_ac, dct4x4[i] );

            if( b_inter )
            {
                i_decimate_score += x264_mb_decimate_score( h->dct.block[16+i+ch*4].residual_ac, 15 );
            }
        }

        h->dctf.dct2x2dc( dct2x2 );
341
        quant_2x2_dc( h, dct2x2, h->quant4_mf[CQM_4IC + b_inter], i_qscale, !b_inter );
Laurent Aimar's avatar
Laurent Aimar committed
342 343 344 345
        scan_zigzag_2x2_dc( h->dct.chroma_dc[ch], dct2x2 );

        /* output samples to fdec */
        h->dctf.idct2x2dc( dct2x2 );
Loren Merritt's avatar
Loren Merritt committed
346
        x264_mb_dequant_2x2_dc( dct2x2, h->dequant4_mf[CQM_4IC + b_inter], i_qscale );  /* XXX not inversed */
Laurent Aimar's avatar
Laurent Aimar committed
347 348 349 350

        if( b_inter && i_decimate_score < 7 )
        {
            /* Near null chroma 8x8 block so make it null (bits saving) */
Loren Merritt's avatar
Loren Merritt committed
351 352
            memset( dct4x4, 0, sizeof( dct4x4 ) );
            memset( &h->dct.block[16+ch*4], 0, 4 * sizeof( *h->dct.block ) );
Laurent Aimar's avatar
Laurent Aimar committed
353
        }
354 355 356 357 358
        else
        {
            for( i = 0; i < 4; i++ )
                h->quantf.dequant_4x4( dct4x4[i], h->dequant4_mf[CQM_4IC + b_inter], i_qscale );
        }
Laurent Aimar's avatar
Laurent Aimar committed
359 360 361 362 363

        /* calculate dct coeffs */
        for( i = 0; i < 4; i++ )
        {
            /* copy dc coeff */
364
            dct4x4[i][0][0] = dct2x2[0][i];
Laurent Aimar's avatar
Laurent Aimar committed
365
        }
366
        h->dctf.add8x8_idct( p_dst, FDEC_STRIDE, dct4x4 );
Laurent Aimar's avatar
Laurent Aimar committed
367 368 369
    }
}

370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
static void x264_macroblock_encode_skip( x264_t *h )
{
    int i;
    h->mb.i_cbp_luma = 0x00;
    h->mb.i_cbp_chroma = 0x00;

    for( i = 0; i < 16+8; i++ )
    {
        h->mb.cache.non_zero_count[x264_scan8[i]] = 0;
    }

    /* store cbp */
    h->mb.cbp[h->mb.i_mb_xy] = 0;
}

Laurent Aimar's avatar
Laurent Aimar committed
385 386 387 388 389 390
/*****************************************************************************
 * x264_macroblock_encode_pskip:
 *  Encode an already marked skip block
 *****************************************************************************/
void x264_macroblock_encode_pskip( x264_t *h )
{
391 392 393 394
    const int mvx = x264_clip3( h->mb.cache.mv[0][x264_scan8[0]][0],
                                h->mb.mv_min[0], h->mb.mv_max[0] );
    const int mvy = x264_clip3( h->mb.cache.mv[0][x264_scan8[0]][1],
                                h->mb.mv_min[1], h->mb.mv_max[1] );
Laurent Aimar's avatar
Laurent Aimar committed
395 396

    /* Motion compensation XXX probably unneeded */
397
    h->mc.mc_luma( h->mb.pic.p_fref[0][0], h->mb.pic.i_stride[0],
398 399
                   h->mb.pic.p_fdec[0],    FDEC_STRIDE,
                   mvx, mvy, 16, 16 );
Laurent Aimar's avatar
Laurent Aimar committed
400 401

    /* Chroma MC */
402
    h->mc.mc_chroma( h->mb.pic.p_fref[0][0][4], h->mb.pic.i_stride[1],
403 404
                     h->mb.pic.p_fdec[1],       FDEC_STRIDE,
                     mvx, mvy, 8, 8 );
Laurent Aimar's avatar
Laurent Aimar committed
405

406
    h->mc.mc_chroma( h->mb.pic.p_fref[0][0][5], h->mb.pic.i_stride[2],
407 408
                     h->mb.pic.p_fdec[2],       FDEC_STRIDE,
                     mvx, mvy, 8, 8 );
Laurent Aimar's avatar
Laurent Aimar committed
409

410
    x264_macroblock_encode_skip( h );
Laurent Aimar's avatar
Laurent Aimar committed
411 412 413 414 415 416 417 418
}

/*****************************************************************************
 * x264_macroblock_encode:
 *****************************************************************************/
void x264_macroblock_encode( x264_t *h )
{
    int i_cbp_dc = 0;
419
    int i_qp = h->mb.i_qp;
Laurent Aimar's avatar
Laurent Aimar committed
420 421 422 423 424 425 426 427
    int i;

    if( h->mb.i_type == P_SKIP )
    {
        /* A bit special */
        x264_macroblock_encode_pskip( h );
        return;
    }
428 429 430 431 432 433 434
    if( h->mb.i_type == B_SKIP )
    {
        /* XXX motion compensation is probably unneeded */
        x264_mb_mc( h );
        x264_macroblock_encode_skip( h );
        return;
    }
Laurent Aimar's avatar
Laurent Aimar committed
435 436 437 438

    if( h->mb.i_type == I_16x16 )
    {
        const int i_mode = h->mb.i_intra16x16_pred_mode;
439
        h->mb.b_transform_8x8 = 0;
Laurent Aimar's avatar
Laurent Aimar committed
440
        /* do the right prediction */
441
        h->predict_16x16[i_mode]( h->mb.pic.p_fdec[0] );
Laurent Aimar's avatar
Laurent Aimar committed
442 443

        /* encode the 16x16 macroblock */
444
        x264_mb_encode_i16x16( h, i_qp );
Laurent Aimar's avatar
Laurent Aimar committed
445
    }
446 447
    else if( h->mb.i_type == I_8x8 )
    {
448
        h->mb.b_transform_8x8 = 1;
449 450
        for( i = 0; i < 4; i++ )
        {
451
            uint8_t  *p_dst = &h->mb.pic.p_fdec[0][8 * (i&1) + 8 * (i>>1) * FDEC_STRIDE];
452 453
            int      i_mode = h->mb.cache.intra4x4_pred_mode[x264_scan8[4*i]];

454
            h->predict_8x8[i_mode]( p_dst, h->mb.i_neighbour8[i] );
455
            x264_mb_encode_i8x8( h, i, i_qp );
456 457
        }
    }
Laurent Aimar's avatar
Laurent Aimar committed
458 459
    else if( h->mb.i_type == I_4x4 )
    {
460
        h->mb.b_transform_8x8 = 0;
Laurent Aimar's avatar
Laurent Aimar committed
461 462
        for( i = 0; i < 16; i++ )
        {
463
            uint8_t  *p_dst = &h->mb.pic.p_fdec[0][4 * block_idx_x[i] + 4 * block_idx_y[i] * FDEC_STRIDE];
Laurent Aimar's avatar
Laurent Aimar committed
464 465
            int      i_mode = h->mb.cache.intra4x4_pred_mode[x264_scan8[i]];

466 467
            if( (h->mb.i_neighbour4[i] & (MB_TOPRIGHT|MB_TOP)) == MB_TOP )
                /* emulate missing topright samples */
468
                *(uint32_t*) &p_dst[4-FDEC_STRIDE] = p_dst[3-FDEC_STRIDE] * 0x01010101U;
469

470
            h->predict_4x4[i_mode]( p_dst );
471
            x264_mb_encode_i4x4( h, i, i_qp );
Laurent Aimar's avatar
Laurent Aimar committed
472 473 474 475 476 477 478 479 480 481
        }
    }
    else    /* Inter MB */
    {
        int i8x8, i4x4, idx;
        int i_decimate_mb = 0;

        /* Motion compensation */
        x264_mb_mc( h );

Loren Merritt's avatar
Loren Merritt committed
482 483 484 485
        if( h->mb.b_lossless )
        {
            for( i4x4 = 0; i4x4 < 16; i4x4++ )
            {
486 487 488 489 490
                int x = 4*block_idx_x[i4x4];
                int y = 4*block_idx_y[i4x4];
                sub_zigzag_4x4full( h->dct.block[i4x4].luma4x4,
                                    h->mb.pic.p_fenc[0]+x+y*FENC_STRIDE,
                                    h->mb.pic.p_fdec[0]+x+y*FDEC_STRIDE );
Loren Merritt's avatar
Loren Merritt committed
491 492 493
            }
        }
        else if( h->mb.b_transform_8x8 )
Laurent Aimar's avatar
Laurent Aimar committed
494
        {
495
            int16_t dct8x8[4][8][8];
496
            int nnz8x8[4] = {1,1,1,1};
497
            h->dctf.sub16x16_dct8( dct8x8,
498 499
                                   h->mb.pic.p_fenc[0], FENC_STRIDE,
                                   h->mb.pic.p_fdec[0], FDEC_STRIDE );
Laurent Aimar's avatar
Laurent Aimar committed
500

501
            for( idx = 0; idx < 4; idx++ )
Laurent Aimar's avatar
Laurent Aimar committed
502
            {
503 504
                if( h->mb.b_noise_reduction )
                    x264_denoise_dct( h, (int16_t*)dct8x8[idx] );
Loren Merritt's avatar
Loren Merritt committed
505 506 507 508
                if( h->mb.b_trellis )
                    x264_quant_8x8_trellis( h, dct8x8[idx], CQM_8PY, i_qp, 0 );
                else
                    quant_8x8( h, dct8x8[idx], h->quant8_mf[CQM_8PY], i_qp, 0 );
Laurent Aimar's avatar
Laurent Aimar committed
509

510
                scan_zigzag_8x8full( h->dct.luma8x8[idx], dct8x8[idx] );
Laurent Aimar's avatar
Laurent Aimar committed
511

Loren Merritt's avatar
Loren Merritt committed
512
                if( !h->mb.b_trellis )
513
                {
Loren Merritt's avatar
Loren Merritt committed
514 515 516 517 518 519
                    int i_decimate_8x8 = x264_mb_decimate_score( h->dct.luma8x8[idx], 64 );
                    i_decimate_mb += i_decimate_8x8;
                    if( i_decimate_8x8 < 4 )
                    {
                        memset( h->dct.luma8x8[idx], 0, sizeof( h->dct.luma8x8[idx] ) );
                        memset( dct8x8[idx], 0, sizeof( dct8x8[idx] ) );
520
                        nnz8x8[idx] = 0;
Loren Merritt's avatar
Loren Merritt committed
521
                    }
522
                }
Laurent Aimar's avatar
Laurent Aimar committed
523 524
            }

Loren Merritt's avatar
Loren Merritt committed
525
            if( i_decimate_mb < 6 && !h->mb.b_trellis )
526 527
                memset( h->dct.luma8x8, 0, sizeof( h->dct.luma8x8 ) );
            else
528 529 530 531 532
            {
                for( idx = 0; idx < 4; idx++ )
                    if( nnz8x8[idx] )
                    {
                        h->quantf.dequant_8x8( dct8x8[idx], h->dequant8_mf[CQM_8PY], i_qp );
533
                        h->dctf.add8x8_idct8( &h->mb.pic.p_fdec[0][(idx&1)*8 + (idx>>1)*8*FDEC_STRIDE], FDEC_STRIDE, dct8x8[idx] );
534 535
                    }
            }
536 537 538 539
        }
        else
        {
            int16_t dct4x4[16][4][4];
540
            int nnz8x8[4] = {1,1,1,1};
541
            h->dctf.sub16x16_dct( dct4x4,
542 543
                                  h->mb.pic.p_fenc[0], FENC_STRIDE,
                                  h->mb.pic.p_fdec[0], FDEC_STRIDE );
544 545

            for( i8x8 = 0; i8x8 < 4; i8x8++ )
Laurent Aimar's avatar
Laurent Aimar committed
546
            {
547 548 549 550
                int i_decimate_8x8;

                /* encode one 4x4 block */
                i_decimate_8x8 = 0;
Laurent Aimar's avatar
Laurent Aimar committed
551 552 553
                for( i4x4 = 0; i4x4 < 4; i4x4++ )
                {
                    idx = i8x8 * 4 + i4x4;
554

555 556
                    if( h->mb.b_noise_reduction )
                        x264_denoise_dct( h, (int16_t*)dct4x4[idx] );
Loren Merritt's avatar
Loren Merritt committed
557 558 559 560 561
                    if( h->mb.b_trellis )
                        x264_quant_4x4_trellis( h, dct4x4[idx], CQM_4PY, i_qp, DCT_LUMA_4x4, 0 );
                    else
                        quant_4x4( h, dct4x4[idx], h->quant4_mf[CQM_4PY], i_qp, 0 );

562 563 564
                    scan_zigzag_4x4full( h->dct.block[idx].luma4x4, dct4x4[idx] );

                    i_decimate_8x8 += x264_mb_decimate_score( h->dct.block[idx].luma4x4, 16 );
Laurent Aimar's avatar
Laurent Aimar committed
565 566
                }

567 568 569
                /* decimate this 8x8 block */
                i_decimate_mb += i_decimate_8x8;
                if( i_decimate_8x8 < 4 )
Laurent Aimar's avatar
Laurent Aimar committed
570
                {
Loren Merritt's avatar
Loren Merritt committed
571 572
                    memset( &dct4x4[i8x8*4], 0, 4 * sizeof( *dct4x4 ) );
                    memset( &h->dct.block[i8x8*4], 0, 4 * sizeof( *h->dct.block ) );
573
                    nnz8x8[i8x8] = 0;
Laurent Aimar's avatar
Laurent Aimar committed
574 575
                }
            }
576 577

            if( i_decimate_mb < 6 )
Loren Merritt's avatar
Loren Merritt committed
578
                memset( h->dct.block, 0, 16 * sizeof( *h->dct.block ) );
579
            else
580 581 582 583 584 585
            {
                for( i8x8 = 0; i8x8 < 4; i8x8++ )
                    if( nnz8x8[i8x8] )
                    {
                        for( i = 0; i < 4; i++ )
                            h->quantf.dequant_4x4( dct4x4[i8x8*4+i], h->dequant4_mf[CQM_4PY], i_qp );
586
                        h->dctf.add8x8_idct( &h->mb.pic.p_fdec[0][(i8x8&1)*8 + (i8x8>>1)*8*FDEC_STRIDE], FDEC_STRIDE, &dct4x4[i8x8*4] );
587 588
                    }
            }
Laurent Aimar's avatar
Laurent Aimar committed
589 590 591 592
        }
    }

    /* encode chroma */
593
    i_qp = i_chroma_qp_table[x264_clip3( i_qp + h->pps->i_chroma_qp_index_offset, 0, 51 )];
Laurent Aimar's avatar
Laurent Aimar committed
594 595 596
    if( IS_INTRA( h->mb.i_type ) )
    {
        const int i_mode = h->mb.i_chroma_pred_mode;
597 598
        h->predict_8x8c[i_mode]( h->mb.pic.p_fdec[1] );
        h->predict_8x8c[i_mode]( h->mb.pic.p_fdec[2] );
Laurent Aimar's avatar
Laurent Aimar committed
599 600 601
    }

    /* encode the 8x8 blocks */
602
    x264_mb_encode_8x8_chroma( h, !IS_INTRA( h->mb.i_type ), i_qp );
Laurent Aimar's avatar
Laurent Aimar committed
603 604

    /* Calculate the Luma/Chroma patern and non_zero_count */
605
    h->mb.i_cbp_luma = 0x00;
Laurent Aimar's avatar
Laurent Aimar committed
606 607 608 609 610 611 612 613
    if( h->mb.i_type == I_16x16 )
    {
        for( i = 0; i < 16; i++ )
        {
            const int nz = array_non_zero_count( h->dct.block[i].residual_ac, 15 );
            h->mb.cache.non_zero_count[x264_scan8[i]] = nz;
            if( nz > 0 )
                h->mb.i_cbp_luma = 0x0f;
614 615 616 617
        }
    }
    else if( h->mb.b_transform_8x8 )
    {
Loren Merritt's avatar
Loren Merritt committed
618 619
        /* coded_block_flag is enough for CABAC.
         * the full non_zero_count is done only in CAVLC. */
620 621 622 623 624 625 626 627
        for( i = 0; i < 4; i++ )
        {
            const int nz = array_non_zero( h->dct.luma8x8[i], 64 );
            int j;
            for( j = 0; j < 4; j++ )
                h->mb.cache.non_zero_count[x264_scan8[4*i+j]] = nz;
            if( nz > 0 )
                h->mb.i_cbp_luma |= 1 << i;
Laurent Aimar's avatar
Laurent Aimar committed
628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
        }
    }
    else
    {
        for( i = 0; i < 16; i++ )
        {
            const int nz = array_non_zero_count( h->dct.block[i].luma4x4, 16 );
            h->mb.cache.non_zero_count[x264_scan8[i]] = nz;
            if( nz > 0 )
                h->mb.i_cbp_luma |= 1 << (i/4);
        }
    }

    /* Calculate the chroma patern */
    h->mb.i_cbp_chroma = 0x00;
    for( i = 0; i < 8; i++ )
    {
        const int nz = array_non_zero_count( h->dct.block[16+i].residual_ac, 15 );
        h->mb.cache.non_zero_count[x264_scan8[16+i]] = nz;
        if( nz > 0 )
        {
            h->mb.i_cbp_chroma = 0x02;    /* dc+ac (we can't do only ac) */
        }
    }
652
    if( h->mb.i_cbp_chroma == 0x00 && array_non_zero( h->dct.chroma_dc[0], 8 ) )
Laurent Aimar's avatar
Laurent Aimar committed
653 654 655 656 657 658
    {
        h->mb.i_cbp_chroma = 0x01;    /* dc only */
    }

    if( h->param.b_cabac )
    {
659 660 661
        i_cbp_dc = ( h->mb.i_type == I_16x16 && array_non_zero( h->dct.luma16x16_dc, 16 ) )
                 | array_non_zero( h->dct.chroma_dc[0], 4 ) << 1
                 | array_non_zero( h->dct.chroma_dc[1], 4 ) << 2;
Laurent Aimar's avatar
Laurent Aimar committed
662 663 664 665 666 667 668 669 670
    }

    /* store cbp */
    h->mb.cbp[h->mb.i_mb_xy] = (i_cbp_dc << 8) | (h->mb.i_cbp_chroma << 4) | h->mb.i_cbp_luma;

    /* Check for P_SKIP
     * XXX: in the me perhaps we should take x264_mb_predict_mv_pskip into account
     *      (if multiple mv give same result)*/
    if( h->mb.i_type == P_L0 && h->mb.i_partition == D_16x16 &&
671 672
        h->mb.i_cbp_luma == 0x00 && h->mb.i_cbp_chroma== 0x00 &&
        h->mb.cache.ref[0][x264_scan8[0]] == 0 )
Laurent Aimar's avatar
Laurent Aimar committed
673
    {
674
        int mvp[2];
Laurent Aimar's avatar
Laurent Aimar committed
675

676 677 678 679 680
        x264_mb_predict_mv_pskip( h, mvp );
        if( h->mb.cache.mv[0][x264_scan8[0]][0] == mvp[0] &&
            h->mb.cache.mv[0][x264_scan8[0]][1] == mvp[1] )
        {
            h->mb.i_type = P_SKIP;
Laurent Aimar's avatar
Laurent Aimar committed
681 682
        }
    }
683 684 685 686 687

    /* Check for B_SKIP */
    if( h->mb.i_type == B_DIRECT &&
        h->mb.i_cbp_luma == 0x00 && h->mb.i_cbp_chroma== 0x00 )
    {
688
        h->mb.i_type = B_SKIP;
689
    }
Laurent Aimar's avatar
Laurent Aimar committed
690 691 692
}

/*****************************************************************************
693 694
 * x264_macroblock_probe_skip:
 *  Check if the current MB could be encoded as a [PB]_SKIP (it supposes you use
Laurent Aimar's avatar
Laurent Aimar committed
695 696
 *  the previous QP
 *****************************************************************************/
697
int x264_macroblock_probe_skip( x264_t *h, int b_bidir )
Laurent Aimar's avatar
Laurent Aimar committed
698 699 700 701 702
{
    DECLARE_ALIGNED( int16_t, dct4x4[16][4][4], 16 );
    DECLARE_ALIGNED( int16_t, dct2x2[2][2], 16 );
    DECLARE_ALIGNED( int,     dctscan[16], 16 );

703
    int i_qp = h->mb.i_qp;
Laurent Aimar's avatar
Laurent Aimar committed
704 705 706 707 708 709
    int mvp[2];
    int ch;

    int i8x8, i4x4;
    int i_decimate_mb;

710 711 712 713
    if( !b_bidir )
    {
        /* Get the MV */
        x264_mb_predict_mv_pskip( h, mvp );
714 715
        mvp[0] = x264_clip3( mvp[0], h->mb.mv_min[0], h->mb.mv_max[0] );
        mvp[1] = x264_clip3( mvp[1], h->mb.mv_min[1], h->mb.mv_max[1] );
Laurent Aimar's avatar
Laurent Aimar committed
716

717
        /* Motion compensation */
718
        h->mc.mc_luma( h->mb.pic.p_fref[0][0], h->mb.pic.i_stride[0],
719 720
                       h->mb.pic.p_fdec[0],    FDEC_STRIDE,
                       mvp[0], mvp[1], 16, 16 );
721
    }
Laurent Aimar's avatar
Laurent Aimar committed
722 723

    /* get luma diff */
724 725
    h->dctf.sub16x16_dct( dct4x4, h->mb.pic.p_fenc[0], FENC_STRIDE,
                                  h->mb.pic.p_fdec[0], FDEC_STRIDE );
Laurent Aimar's avatar
Laurent Aimar committed
726 727 728 729 730 731 732 733

    for( i8x8 = 0, i_decimate_mb = 0; i8x8 < 4; i8x8++ )
    {
        /* encode one 4x4 block */
        for( i4x4 = 0; i4x4 < 4; i4x4++ )
        {
            const int idx = i8x8 * 4 + i4x4;

734
            quant_4x4( h, dct4x4[idx], (int(*)[4][4])def_quant4_mf, i_qp, 0 );
Laurent Aimar's avatar
Laurent Aimar committed
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754
            scan_zigzag_4x4full( dctscan, dct4x4[idx] );

            i_decimate_mb += x264_mb_decimate_score( dctscan, 16 );

            if( i_decimate_mb >= 6 )
            {
                /* not as P_SKIP */
                return 0;
            }
        }
    }

    /* encode chroma */
    i_qp = i_chroma_qp_table[x264_clip3( i_qp + h->pps->i_chroma_qp_index_offset, 0, 51 )];

    for( ch = 0; ch < 2; ch++ )
    {
        uint8_t  *p_src = h->mb.pic.p_fenc[1+ch];
        uint8_t  *p_dst = h->mb.pic.p_fdec[1+ch];

755 756
        if( !b_bidir )
        {
757 758 759
            h->mc.mc_chroma( h->mb.pic.p_fref[0][0][4+ch], h->mb.pic.i_stride[1+ch],
                             h->mb.pic.p_fdec[1+ch],       FDEC_STRIDE,
                             mvp[0], mvp[1], 8, 8 );
760
        }
Laurent Aimar's avatar
Laurent Aimar committed
761

762
        h->dctf.sub8x8_dct( dct4x4, p_src, FENC_STRIDE, p_dst, FDEC_STRIDE );
Laurent Aimar's avatar
Laurent Aimar committed
763 764 765 766 767 768 769

        /* calculate dct DC */
        dct2x2[0][0] = dct4x4[0][0][0];
        dct2x2[0][1] = dct4x4[1][0][0];
        dct2x2[1][0] = dct4x4[2][0][0];
        dct2x2[1][1] = dct4x4[3][0][0];
        h->dctf.dct2x2dc( dct2x2 );
770
        quant_2x2_dc( h, dct2x2, (int(*)[4][4])def_quant4_mf, i_qp, 0 );
Laurent Aimar's avatar
Laurent Aimar committed
771 772 773 774 775 776 777 778 779
        if( dct2x2[0][0] || dct2x2[0][1] || dct2x2[1][0] || dct2x2[1][1]  )
        {
            /* can't be */
            return 0;
        }

        /* calculate dct coeffs */
        for( i4x4 = 0, i_decimate_mb = 0; i4x4 < 4; i4x4++ )
        {
780
            quant_4x4( h, dct4x4[i4x4], (int(*)[4][4])def_quant4_mf, i_qp, 0 );
Laurent Aimar's avatar
Laurent Aimar committed
781 782 783 784 785 786 787 788 789 790 791 792
            scan_zigzag_4x4( dctscan, dct4x4[i4x4] );

            i_decimate_mb += x264_mb_decimate_score( dctscan, 15 );
            if( i_decimate_mb >= 7 )
            {
                return 0;
            }
        }
    }

    return 1;
}
793 794 795 796 797 798 799 800 801

/****************************************************************************
 * DCT-domain noise reduction / adaptive deadzone
 * from libavcodec
 ****************************************************************************/

void x264_noise_reduction_update( x264_t *h )
{
    int cat, i;
802
    for( cat = 0; cat < 2; cat++ )
803
    {
804 805
        int size = cat ? 64 : 16;
        const int *weight = cat ? x264_dct8_weight2_tab : x264_dct4_weight2_tab;
806

807
        if( h->nr_count[cat] > (cat ? (1<<16) : (1<<18)) )
808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823
        {
            for( i = 0; i < size; i++ )
                h->nr_residual_sum[cat][i] >>= 1;
            h->nr_count[cat] >>= 1;
        }

        for( i = 0; i < size; i++ )
            h->nr_offset[cat][i] =
                ((uint64_t)h->param.analyse.i_noise_reduction * h->nr_count[cat]
                 + h->nr_residual_sum[cat][i]/2)
              / ((uint64_t)h->nr_residual_sum[cat][i] * weight[i]/256 + 1);
    }
}

void x264_denoise_dct( x264_t *h, int16_t *dct )
{
824
    const int cat = h->mb.b_transform_8x8;
825 826 827 828
    int i;

    h->nr_count[cat]++;

829
    for( i = (cat ? 63 : 15); i >= 1; i-- )
830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851
    {
        int level = dct[i];
        if( level )
        {
            if( level > 0 )
            {
                h->nr_residual_sum[cat][i] += level;
                level -= h->nr_offset[cat][i];
                if( level < 0 )
                    level = 0;
            }
            else
            {
                h->nr_residual_sum[cat][i] -= level;
                level += h->nr_offset[cat][i];
                if( level > 0 )
                    level = 0;
            }
            dct[i] = level;
        }
    }
}
852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936

/*****************************************************************************
 * RD only; 4 calls to this do not make up for one macroblock_encode.
 * doesn't transform chroma dc.
 *****************************************************************************/
void x264_macroblock_encode_p8x8( x264_t *h, int i8 )
{
    int i_qp = h->mb.i_qp;
    uint8_t *p_fenc = h->mb.pic.p_fenc[0] + (i8&1)*8 + (i8>>1)*8*FENC_STRIDE;
    uint8_t *p_fdec = h->mb.pic.p_fdec[0] + (i8&1)*8 + (i8>>1)*8*FDEC_STRIDE;
    int i_decimate_8x8 = 0;
    int nnz8x8 = 1;
    int ch;

    x264_mb_mc_8x8( h, i8 );

    if( h->mb.b_transform_8x8 )
    {
        int16_t dct8x8[8][8];
        h->dctf.sub8x8_dct8( dct8x8, p_fenc, FENC_STRIDE, p_fdec, FDEC_STRIDE );

        quant_8x8( h, dct8x8, h->quant8_mf[CQM_8PY], i_qp, 0 );
        scan_zigzag_8x8full( h->dct.luma8x8[i8], dct8x8 );
        i_decimate_8x8 = x264_mb_decimate_score( h->dct.luma8x8[i8], 64 );

        if( i_decimate_8x8 < 4 )
        {
            memset( h->dct.luma8x8[i8], 0, sizeof(h->dct.luma8x8[i8]) );
            nnz8x8 = 0;
        }
        if( nnz8x8 )
        {
            h->quantf.dequant_8x8( dct8x8, h->dequant8_mf[CQM_8PY], i_qp );
            h->dctf.add8x8_idct8( p_fdec, FDEC_STRIDE, dct8x8 );
        }
    }
    else
    {
        int i4, idx;
        int16_t dct4x4[4][4][4];
        h->dctf.sub8x8_dct( dct4x4, p_fenc, FENC_STRIDE, p_fdec, FDEC_STRIDE );

        for( i4 = 0; i4 < 4; i4++ )
        {
            idx = i8 * 4 + i4;

            quant_4x4( h, dct4x4[i4], h->quant4_mf[CQM_4PY], i_qp, 0 );
            scan_zigzag_4x4full( h->dct.block[idx].luma4x4, dct4x4[i4] );
            i_decimate_8x8 += x264_mb_decimate_score( h->dct.block[idx].luma4x4, 16 );
        }

        if( i_decimate_8x8 < 4 )
        {
            memset( &h->dct.block[i8*4], 0, 4 * sizeof(*h->dct.block) );
            nnz8x8 = 0;
        }
        if( nnz8x8 )
        {
            for( i4 = 0; i4 < 4; i4++ )
                 h->quantf.dequant_4x4( dct4x4[i4], h->dequant4_mf[CQM_4PY], i_qp );
            h->dctf.add8x8_idct( p_fdec, FDEC_STRIDE, dct4x4 );
        }
    }

    i_qp = i_chroma_qp_table[x264_clip3( i_qp + h->pps->i_chroma_qp_index_offset, 0, 51 )];

    for( ch = 0; ch < 2; ch++ )
    {
        int16_t dct4x4[4][4];
        p_fenc = h->mb.pic.p_fenc[1+ch] + (i8&1)*4 + (i8>>1)*4*FENC_STRIDE;
        p_fdec = h->mb.pic.p_fdec[1+ch] + (i8&1)*4 + (i8>>1)*4*FDEC_STRIDE;

        h->dctf.sub4x4_dct( dct4x4, p_fenc, FENC_STRIDE, p_fdec, FDEC_STRIDE );
        quant_4x4( h, dct4x4, h->quant4_mf[CQM_4PC], i_qp, 0 );
        scan_zigzag_4x4( h->dct.block[16+i8+ch*4].residual_ac, dct4x4 );
        h->quantf.dequant_4x4( dct4x4, h->dequant4_mf[CQM_4PC], i_qp );
        h->dctf.add4x4_idct( p_fdec, FDEC_STRIDE, dct4x4 );
    }

    if( nnz8x8 )
        h->mb.i_cbp_luma |= (1 << i8);
    else
        h->mb.i_cbp_luma &= ~(1 << i8);
    h->mb.i_cbp_chroma = 0x02;
}