macroblock.c 35.2 KB
Newer Older
Laurent Aimar's avatar
Laurent Aimar committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
/*****************************************************************************
 * macroblock.c: h264 encoder library
 *****************************************************************************
 * Copyright (C) 2003 Laurent Aimar
 * $Id: macroblock.c,v 1.1 2004/06/03 19:27:08 fenrir Exp $
 *
 * Authors: Laurent Aimar <fenrir@via.ecp.fr>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111, USA.
 *****************************************************************************/

#include <stdio.h>
#include <string.h>

27
#include "common/common.h"
Laurent Aimar's avatar
Laurent Aimar committed
28 29 30
#include "macroblock.h"


Loren Merritt's avatar
Loren Merritt committed
31 32 33 34
/* def_quant4_mf only for probe_skip; actual encoding uses matrices from set.c */
/* FIXME this seems to make better decisions with cqm=jvt, but could screw up
 * with general custom matrices. */
static const int def_quant4_mf[6][4][4] =
Laurent Aimar's avatar
Laurent Aimar committed
35
{
36 37 38 39 40 41 42 43 44 45 46 47 48 49
    { { 13107, 8066, 13107, 8066 }, { 8066, 5243, 8066, 5243 },
      { 13107, 8066, 13107, 8066 }, { 8066, 5243, 8066, 5243 } },
    { { 11916, 7490, 11916, 7490 }, { 7490, 4660, 7490, 4660 },
      { 11916, 7490, 11916, 7490 }, { 7490, 4660, 7490, 4660 } },
    { { 10082, 6554, 10082, 6554 }, { 6554, 4194, 6554, 4194 },
      { 10082, 6554, 10082, 6554 }, { 6554, 4194, 6554, 4194 } },
    { {  9362, 5825,  9362, 5825 }, { 5825, 3647, 5825, 3647 },
      {  9362, 5825,  9362, 5825 }, { 5825, 3647, 5825, 3647 } },
    { {  8192, 5243,  8192, 5243 }, { 5243, 3355, 5243, 3355 },
      {  8192, 5243,  8192, 5243 }, { 5243, 3355, 5243, 3355 } },
    { {  7282, 4559,  7282, 4559 }, { 4559, 2893, 4559, 2893 },
      {  7282, 4559,  7282, 4559 }, { 4559, 2893, 4559, 2893 } }
};

Laurent Aimar's avatar
Laurent Aimar committed
50 51 52 53
/****************************************************************************
 * Scan and Quant functions
 ****************************************************************************/

54
#define ZIG(i,y,x) level[i] = dct[x][y];
55 56

static inline void scan_zigzag_8x8full( int level[64], int16_t dct[8][8], int b_interlaced )
57
{
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
    if( b_interlaced )
    {
    ZIG( 0,0,0) ZIG( 1,1,0) ZIG( 2,2,0) ZIG( 3,0,1)
    ZIG( 4,1,1) ZIG( 5,3,0) ZIG( 6,4,0) ZIG( 7,2,1)
    ZIG( 8,0,2) ZIG( 9,3,1) ZIG(10,5,0) ZIG(11,6,0)
    ZIG(12,7,0) ZIG(13,4,1) ZIG(14,1,2) ZIG(15,0,3)
    ZIG(16,2,2) ZIG(17,5,1) ZIG(18,6,1) ZIG(19,7,1)
    ZIG(20,3,2) ZIG(21,1,3) ZIG(22,0,4) ZIG(23,2,3)
    ZIG(24,4,2) ZIG(25,5,2) ZIG(26,6,2) ZIG(27,7,2)
    ZIG(28,3,3) ZIG(29,1,4) ZIG(30,0,5) ZIG(31,2,4)
    ZIG(32,4,3) ZIG(33,5,3) ZIG(34,6,3) ZIG(35,7,3)
    ZIG(36,3,4) ZIG(37,1,5) ZIG(38,0,6) ZIG(39,2,5)
    ZIG(40,4,4) ZIG(41,5,4) ZIG(42,6,4) ZIG(43,7,4)
    ZIG(44,3,5) ZIG(45,1,6) ZIG(46,2,6) ZIG(47,4,5)
    ZIG(48,5,5) ZIG(49,6,5) ZIG(50,7,5) ZIG(51,3,6)
    ZIG(52,0,7) ZIG(53,1,7) ZIG(54,4,6) ZIG(55,5,6)
    ZIG(56,6,6) ZIG(57,7,6) ZIG(58,2,7) ZIG(59,3,7)
    ZIG(60,4,7) ZIG(61,5,7) ZIG(62,6,7) ZIG(63,7,7)
    }
    else
    {
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
    ZIG( 0,0,0) ZIG( 1,0,1) ZIG( 2,1,0) ZIG( 3,2,0)
    ZIG( 4,1,1) ZIG( 5,0,2) ZIG( 6,0,3) ZIG( 7,1,2)
    ZIG( 8,2,1) ZIG( 9,3,0) ZIG(10,4,0) ZIG(11,3,1)
    ZIG(12,2,2) ZIG(13,1,3) ZIG(14,0,4) ZIG(15,0,5)
    ZIG(16,1,4) ZIG(17,2,3) ZIG(18,3,2) ZIG(19,4,1)
    ZIG(20,5,0) ZIG(21,6,0) ZIG(22,5,1) ZIG(23,4,2)
    ZIG(24,3,3) ZIG(25,2,4) ZIG(26,1,5) ZIG(27,0,6)
    ZIG(28,0,7) ZIG(29,1,6) ZIG(30,2,5) ZIG(31,3,4)
    ZIG(32,4,3) ZIG(33,5,2) ZIG(34,6,1) ZIG(35,7,0)
    ZIG(36,7,1) ZIG(37,6,2) ZIG(38,5,3) ZIG(39,4,4)
    ZIG(40,3,5) ZIG(41,2,6) ZIG(42,1,7) ZIG(43,2,7)
    ZIG(44,3,6) ZIG(45,4,5) ZIG(46,5,4) ZIG(47,6,3)
    ZIG(48,7,2) ZIG(49,7,3) ZIG(50,6,4) ZIG(51,5,5)
    ZIG(52,4,6) ZIG(53,3,7) ZIG(54,4,7) ZIG(55,5,6)
    ZIG(56,6,5) ZIG(57,7,4) ZIG(58,7,5) ZIG(59,6,6)
    ZIG(60,5,7) ZIG(61,6,7) ZIG(62,7,6) ZIG(63,7,7)
95
    }
96
}
97 98

static inline void scan_zigzag_4x4full( int level[16], int16_t dct[4][4], int b_interlaced )
Laurent Aimar's avatar
Laurent Aimar committed
99
{
100 101 102 103 104 105 106 107 108
    if( b_interlaced )
    {
    ZIG( 0,0,0) ZIG( 1,1,0) ZIG( 2,0,1) ZIG( 3,2,0)
    ZIG( 4,3,0) ZIG( 5,1,1) ZIG( 6,2,1) ZIG( 7,3,1)
    ZIG( 8,0,2) ZIG( 9,1,2) ZIG(10,2,2) ZIG(11,3,2)
    ZIG(12,0,3) ZIG(13,1,3) ZIG(14,2,3) ZIG(15,3,3)
    }
    else
    {
109 110 111 112
    ZIG( 0,0,0) ZIG( 1,0,1) ZIG( 2,1,0) ZIG( 3,2,0)
    ZIG( 4,1,1) ZIG( 5,0,2) ZIG( 6,0,3) ZIG( 7,1,2)
    ZIG( 8,2,1) ZIG( 9,3,0) ZIG(10,3,1) ZIG(11,2,2)
    ZIG(12,1,3) ZIG(13,2,3) ZIG(14,3,2) ZIG(15,3,3)
113
    }
Laurent Aimar's avatar
Laurent Aimar committed
114
}
115 116

static inline void scan_zigzag_4x4( int level[15], int16_t dct[4][4], int b_interlaced )
Laurent Aimar's avatar
Laurent Aimar committed
117
{
118 119 120 121 122 123 124 125 126
    if( b_interlaced )
    {
                ZIG( 0,1,0) ZIG( 1,0,1) ZIG( 2,2,0)
    ZIG( 3,3,0) ZIG( 4,1,1) ZIG( 5,2,1) ZIG( 6,3,1)
    ZIG( 7,0,2) ZIG( 8,1,2) ZIG( 9,2,2) ZIG(10,3,2)
    ZIG(11,0,3) ZIG(12,1,3) ZIG(13,2,3) ZIG(14,3,3)
    }
    else
    {
127 128 129 130
                ZIG( 0,0,1) ZIG( 1,1,0) ZIG( 2,2,0)
    ZIG( 3,1,1) ZIG( 4,0,2) ZIG( 5,0,3) ZIG( 6,1,2)
    ZIG( 7,2,1) ZIG( 8,3,0) ZIG( 9,3,1) ZIG(10,2,2)
    ZIG(11,1,3) ZIG(12,2,3) ZIG(13,3,2) ZIG(14,3,3)
131
    }
Laurent Aimar's avatar
Laurent Aimar committed
132
}
133

Laurent Aimar's avatar
Laurent Aimar committed
134 135
static inline void scan_zigzag_2x2_dc( int level[4], int16_t dct[2][2] )
{
136 137 138 139
    ZIG(0,0,0)
    ZIG(1,0,1)
    ZIG(2,1,0)
    ZIG(3,1,1)
Laurent Aimar's avatar
Laurent Aimar committed
140
}
141
#undef ZIG
Laurent Aimar's avatar
Laurent Aimar committed
142

Loren Merritt's avatar
Loren Merritt committed
143
#define ZIG(i,y,x) {\
144 145 146 147
    int oe = x+y*FENC_STRIDE;\
    int od = x+y*FDEC_STRIDE;\
    level[i] = p_src[oe] - p_dst[od];\
    p_dst[od] = p_src[oe];\
Loren Merritt's avatar
Loren Merritt committed
148
}
149 150

static inline void sub_zigzag_4x4full( int level[16], const uint8_t *p_src, uint8_t *p_dst, int b_interlaced )
Loren Merritt's avatar
Loren Merritt committed
151
{
152 153 154 155 156 157 158 159 160
    if( b_interlaced )
    {
    ZIG( 0,0,0) ZIG( 1,1,0) ZIG( 2,0,1) ZIG( 3,2,0)
    ZIG( 4,3,0) ZIG( 5,1,1) ZIG( 6,2,1) ZIG( 7,3,1)
    ZIG( 8,0,2) ZIG( 9,1,2) ZIG(10,2,2) ZIG(11,3,2)
    ZIG(12,0,3) ZIG(13,1,3) ZIG(14,2,3) ZIG(15,3,3)
    }
    else
    {
Loren Merritt's avatar
Loren Merritt committed
161 162 163 164
    ZIG( 0,0,0) ZIG( 1,0,1) ZIG( 2,1,0) ZIG( 3,2,0)
    ZIG( 4,1,1) ZIG( 5,0,2) ZIG( 6,0,3) ZIG( 7,1,2)
    ZIG( 8,2,1) ZIG( 9,3,0) ZIG(10,3,1) ZIG(11,2,2)
    ZIG(12,1,3) ZIG(13,2,3) ZIG(14,3,2) ZIG(15,3,3)
165
    }
Loren Merritt's avatar
Loren Merritt committed
166
}
167 168

static inline void sub_zigzag_4x4( int level[15], const uint8_t *p_src, uint8_t *p_dst, int b_interlaced )
Loren Merritt's avatar
Loren Merritt committed
169
{
170 171 172 173 174 175 176 177 178
    if( b_interlaced )
    {
                ZIG( 0,1,0) ZIG( 1,0,1) ZIG( 2,2,0)
    ZIG( 3,3,0) ZIG( 4,1,1) ZIG( 5,2,1) ZIG( 6,3,1)
    ZIG( 7,0,2) ZIG( 8,1,2) ZIG( 9,2,2) ZIG(10,3,2)
    ZIG(11,0,3) ZIG(12,1,3) ZIG(13,2,3) ZIG(14,3,3)
    }
    else
    {
Loren Merritt's avatar
Loren Merritt committed
179 180 181 182
                ZIG( 0,0,1) ZIG( 1,1,0) ZIG( 2,2,0)
    ZIG( 3,1,1) ZIG( 4,0,2) ZIG( 5,0,3) ZIG( 6,1,2)
    ZIG( 7,2,1) ZIG( 8,3,0) ZIG( 9,3,1) ZIG(10,2,2)
    ZIG(11,1,3) ZIG(12,2,3) ZIG(13,3,2) ZIG(14,3,3)
183
    }
Loren Merritt's avatar
Loren Merritt committed
184 185 186
}
#undef ZIG

187
static void quant_8x8( x264_t *h, int16_t dct[8][8], int quant_mf[6][8][8], int i_qscale, int b_intra )
188 189 190
{
    const int i_qbits = 16 + i_qscale / 6;
    const int i_mf = i_qscale % 6;
Loren Merritt's avatar
Loren Merritt committed
191
    const int f = ( 1 << (i_qbits + b_intra) ) / 6;
192
    h->quantf.quant_8x8_core( dct, quant_mf[i_mf], i_qbits, f );
193
}
194
static void quant_4x4( x264_t *h, int16_t dct[4][4], int quant_mf[6][4][4], int i_qscale, int b_intra )
Laurent Aimar's avatar
Laurent Aimar committed
195 196 197
{
    const int i_qbits = 15 + i_qscale / 6;
    const int i_mf = i_qscale % 6;
Loren Merritt's avatar
Loren Merritt committed
198
    const int f = ( 1 << (i_qbits + b_intra) ) / 6;
199
    h->quantf.quant_4x4_core( dct, quant_mf[i_mf], i_qbits, f );
Laurent Aimar's avatar
Laurent Aimar committed
200
}
201
static void quant_4x4_dc( x264_t *h, int16_t dct[4][4], int quant_mf[6][4][4], int i_qscale )
Laurent Aimar's avatar
Laurent Aimar committed
202
{
203 204 205 206
    const int i_qbits = 16 + i_qscale / 6;
    const int i_mf = i_qscale % 6;
    const int f = ( 1 << i_qbits ) / 3;
    h->quantf.quant_4x4_dc_core( dct, quant_mf[i_mf][0][0], i_qbits, f );
Laurent Aimar's avatar
Laurent Aimar committed
207
}
208
static void quant_2x2_dc( x264_t *h, int16_t dct[2][2], int quant_mf[6][4][4], int i_qscale, int b_intra )
Laurent Aimar's avatar
Laurent Aimar committed
209
{
210 211
    const int i_qbits = 16 + i_qscale / 6;
    const int i_mf = i_qscale % 6;
Loren Merritt's avatar
Loren Merritt committed
212
    const int f = ( 1 << (i_qbits + b_intra) ) / 6;
213
    h->quantf.quant_2x2_dc_core( dct, quant_mf[i_mf][0][0], i_qbits, f );
Laurent Aimar's avatar
Laurent Aimar committed
214
}
215

Laurent Aimar's avatar
Laurent Aimar committed
216 217 218 219 220 221 222 223 224 225
/* (ref: JVT-B118)
 * x264_mb_decimate_score: given dct coeffs it returns a score to see if we could empty this dct coeffs
 * to 0 (low score means set it to null)
 * Used in inter macroblock (luma and chroma)
 *  luma: for a 8x8 block: if score < 4 -> null
 *        for the complete mb: if score < 6 -> null
 *  chroma: for the complete mb: if score < 7 -> null
 */
static int x264_mb_decimate_score( int *dct, int i_max )
{
226 227 228 229 230 231 232 233 234
    static const int i_ds_table4[16] = {
        3,2,2,1,1,1,0,0,0,0,0,0,0,0,0,0 };
    static const int i_ds_table8[64] = {
        3,3,3,3,2,2,2,2,2,2,2,2,1,1,1,1,
        1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,
        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };

    const int *ds_table = (i_max == 64) ? i_ds_table8 : i_ds_table4;
Laurent Aimar's avatar
Laurent Aimar committed
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
    int i_score = 0;
    int idx = i_max - 1;

    while( idx >= 0 && dct[idx] == 0 )
        idx--;

    while( idx >= 0 )
    {
        int i_run;

        if( abs( dct[idx--] ) > 1 )
            return 9;

        i_run = 0;
        while( idx >= 0 && dct[idx] == 0 )
        {
            idx--;
            i_run++;
        }
254
        i_score += ds_table[i_run];
Laurent Aimar's avatar
Laurent Aimar committed
255 256 257 258 259 260 261
    }

    return i_score;
}

void x264_mb_encode_i4x4( x264_t *h, int idx, int i_qscale )
{
262 263 264 265
    int x = 4 * block_idx_x[idx];
    int y = 4 * block_idx_y[idx];
    uint8_t *p_src = &h->mb.pic.p_fenc[0][x+y*FENC_STRIDE];
    uint8_t *p_dst = &h->mb.pic.p_fdec[0][x+y*FDEC_STRIDE];
Laurent Aimar's avatar
Laurent Aimar committed
266 267
    int16_t dct4x4[4][4];

Loren Merritt's avatar
Loren Merritt committed
268 269
    if( h->mb.b_lossless )
    {
270
        sub_zigzag_4x4full( h->dct.block[idx].luma4x4, p_src, p_dst, h->mb.b_interlaced );
Loren Merritt's avatar
Loren Merritt committed
271 272 273
        return;
    }

274
    h->dctf.sub4x4_dct( dct4x4, p_src, p_dst );
Loren Merritt's avatar
Loren Merritt committed
275 276 277 278 279 280

    if( h->mb.b_trellis )
        x264_quant_4x4_trellis( h, dct4x4, CQM_4IY, i_qscale, DCT_LUMA_4x4, 1 );
    else
        quant_4x4( h, dct4x4, h->quant4_mf[CQM_4IY], i_qscale, 1 );

281
    scan_zigzag_4x4full( h->dct.block[idx].luma4x4, dct4x4, h->mb.b_interlaced );
282
    h->quantf.dequant_4x4( dct4x4, h->dequant4_mf[CQM_4IY], i_qscale );
Laurent Aimar's avatar
Laurent Aimar committed
283 284

    /* output samples to fdec */
285
    h->dctf.add4x4_idct( p_dst, dct4x4 );
Laurent Aimar's avatar
Laurent Aimar committed
286 287
}

288 289
void x264_mb_encode_i8x8( x264_t *h, int idx, int i_qscale )
{
290 291 292 293
    int x = 8 * (idx&1);
    int y = 8 * (idx>>1);
    uint8_t *p_src = &h->mb.pic.p_fenc[0][x+y*FENC_STRIDE];
    uint8_t *p_dst = &h->mb.pic.p_fdec[0][x+y*FDEC_STRIDE];
294 295
    int16_t dct8x8[8][8];

296
    h->dctf.sub8x8_dct8( dct8x8, p_src, p_dst );
Loren Merritt's avatar
Loren Merritt committed
297 298 299 300 301 302

    if( h->mb.b_trellis )
        x264_quant_8x8_trellis( h, dct8x8, CQM_8IY, i_qscale, 1 );
    else 
        quant_8x8( h, dct8x8, h->quant8_mf[CQM_8IY], i_qscale, 1 );

303
    scan_zigzag_8x8full( h->dct.luma8x8[idx], dct8x8, h->mb.b_interlaced );
304
    h->quantf.dequant_8x8( dct8x8, h->dequant8_mf[CQM_8IY], i_qscale );
305
    h->dctf.add8x8_idct8( p_dst, dct8x8 );
306 307
}

Laurent Aimar's avatar
Laurent Aimar committed
308 309 310 311 312 313 314 315 316
static void x264_mb_encode_i16x16( x264_t *h, int i_qscale )
{
    uint8_t  *p_src = h->mb.pic.p_fenc[0];
    uint8_t  *p_dst = h->mb.pic.p_fdec[0];

    int16_t dct4x4[16+1][4][4];

    int i;

Loren Merritt's avatar
Loren Merritt committed
317 318 319 320
    if( h->mb.b_lossless )
    {
        for( i = 0; i < 16; i++ )
        {
321 322
            int oe = block_idx_x[i]*4 + block_idx_y[i]*4*FENC_STRIDE;
            int od = block_idx_x[i]*4 + block_idx_y[i]*4*FDEC_STRIDE;
323
            sub_zigzag_4x4( h->dct.block[i].residual_ac, p_src+oe, p_dst+od, h->mb.b_interlaced );
324
            dct4x4[0][block_idx_x[i]][block_idx_y[i]] = p_src[oe] - p_dst[od];
325
            p_dst[od] = p_src[oe];
Loren Merritt's avatar
Loren Merritt committed
326
        }
327
        scan_zigzag_4x4full( h->dct.luma16x16_dc, dct4x4[0], h->mb.b_interlaced );
Loren Merritt's avatar
Loren Merritt committed
328 329 330
        return;
    }

331
    h->dctf.sub16x16_dct( &dct4x4[1], p_src, p_dst );
Laurent Aimar's avatar
Laurent Aimar committed
332 333 334 335 336 337
    for( i = 0; i < 16; i++ )
    {
        /* copy dc coeff */
        dct4x4[0][block_idx_y[i]][block_idx_x[i]] = dct4x4[1+i][0][0];

        /* quant/scan/dequant */
Loren Merritt's avatar
Loren Merritt committed
338 339 340 341 342
        if( h->mb.b_trellis )
            x264_quant_4x4_trellis( h, dct4x4[1+i], CQM_4IY, i_qscale, DCT_LUMA_AC, 1 );
        else
            quant_4x4( h, dct4x4[1+i], h->quant4_mf[CQM_4IY], i_qscale, 1 );

343
        scan_zigzag_4x4( h->dct.block[i].residual_ac, dct4x4[1+i], h->mb.b_interlaced );
344
        h->quantf.dequant_4x4( dct4x4[1+i], h->dequant4_mf[CQM_4IY], i_qscale );
Laurent Aimar's avatar
Laurent Aimar committed
345 346 347
    }

    h->dctf.dct4x4dc( dct4x4[0] );
348
    quant_4x4_dc( h, dct4x4[0], h->quant4_mf[CQM_4IY], i_qscale );
349
    scan_zigzag_4x4full( h->dct.luma16x16_dc, dct4x4[0], h->mb.b_interlaced );
Laurent Aimar's avatar
Laurent Aimar committed
350 351 352

    /* output samples to fdec */
    h->dctf.idct4x4dc( dct4x4[0] );
Loren Merritt's avatar
Loren Merritt committed
353
    x264_mb_dequant_4x4_dc( dct4x4[0], h->dequant4_mf[CQM_4IY], i_qscale );  /* XXX not inversed */
Laurent Aimar's avatar
Laurent Aimar committed
354 355 356 357 358 359 360 361

    /* calculate dct coeffs */
    for( i = 0; i < 16; i++ )
    {
        /* copy dc coeff */
        dct4x4[1+i][0][0] = dct4x4[0][block_idx_y[i]][block_idx_x[i]];
    }
    /* put pixels to fdec */
362
    h->dctf.add16x16_idct( p_dst, &dct4x4[1] );
Laurent Aimar's avatar
Laurent Aimar committed
363 364
}

365
static void x264_mb_encode_8x8_chroma( x264_t *h, int b_inter, int i_qscale )
Laurent Aimar's avatar
Laurent Aimar committed
366 367
{
    int i, ch;
368
    int b_decimate = b_inter && (h->sh.i_type == SLICE_TYPE_B || h->param.analyse.b_dct_decimate);
Laurent Aimar's avatar
Laurent Aimar committed
369 370 371 372 373 374 375 376 377 378

    for( ch = 0; ch < 2; ch++ )
    {
        uint8_t  *p_src = h->mb.pic.p_fenc[1+ch];
        uint8_t  *p_dst = h->mb.pic.p_fdec[1+ch];
        int i_decimate_score = 0;

        int16_t dct2x2[2][2];
        int16_t dct4x4[4][4][4];

Loren Merritt's avatar
Loren Merritt committed
379 380 381 382
        if( h->mb.b_lossless )
        {
            for( i = 0; i < 4; i++ )
            {
383 384
                int oe = block_idx_x[i]*4 + block_idx_y[i]*4*FENC_STRIDE;
                int od = block_idx_x[i]*4 + block_idx_y[i]*4*FDEC_STRIDE;
385
                sub_zigzag_4x4( h->dct.block[16+i+ch*4].residual_ac, p_src+oe, p_dst+od, h->mb.b_interlaced );
386 387
                h->dct.chroma_dc[ch][i] = p_src[oe] - p_dst[od];
                p_dst[od] = p_src[oe];
Loren Merritt's avatar
Loren Merritt committed
388 389 390 391
            }
            continue;
        }
            
392
        h->dctf.sub8x8_dct( dct4x4, p_src, p_dst );
Laurent Aimar's avatar
Laurent Aimar committed
393 394 395 396 397 398
        /* calculate dct coeffs */
        for( i = 0; i < 4; i++ )
        {
            /* copy dc coeff */
            dct2x2[block_idx_y[i]][block_idx_x[i]] = dct4x4[i][0][0];

Loren Merritt's avatar
Loren Merritt committed
399
            /* no trellis; it doesn't seem to help chroma noticeably */
400
            quant_4x4( h, dct4x4[i], h->quant4_mf[CQM_4IC + b_inter], i_qscale, !b_inter );
401
            scan_zigzag_4x4( h->dct.block[16+i+ch*4].residual_ac, dct4x4[i], h->mb.b_interlaced );
Laurent Aimar's avatar
Laurent Aimar committed
402

403
            if( b_decimate )
Laurent Aimar's avatar
Laurent Aimar committed
404 405 406 407 408 409
            {
                i_decimate_score += x264_mb_decimate_score( h->dct.block[16+i+ch*4].residual_ac, 15 );
            }
        }

        h->dctf.dct2x2dc( dct2x2 );
410
        quant_2x2_dc( h, dct2x2, h->quant4_mf[CQM_4IC + b_inter], i_qscale, !b_inter );
Laurent Aimar's avatar
Laurent Aimar committed
411 412 413 414
        scan_zigzag_2x2_dc( h->dct.chroma_dc[ch], dct2x2 );

        /* output samples to fdec */
        h->dctf.idct2x2dc( dct2x2 );
Loren Merritt's avatar
Loren Merritt committed
415
        x264_mb_dequant_2x2_dc( dct2x2, h->dequant4_mf[CQM_4IC + b_inter], i_qscale );  /* XXX not inversed */
Laurent Aimar's avatar
Laurent Aimar committed
416

417
        if( b_decimate && i_decimate_score < 7 )
Laurent Aimar's avatar
Laurent Aimar committed
418 419
        {
            /* Near null chroma 8x8 block so make it null (bits saving) */
Loren Merritt's avatar
Loren Merritt committed
420
            memset( &h->dct.block[16+ch*4], 0, 4 * sizeof( *h->dct.block ) );
421 422 423
            if( !array_non_zero( (int*)dct2x2, sizeof(dct2x2)/sizeof(int) ) )
                continue;
            memset( dct4x4, 0, sizeof( dct4x4 ) );
Laurent Aimar's avatar
Laurent Aimar committed
424
        }
425 426 427 428 429
        else
        {
            for( i = 0; i < 4; i++ )
                h->quantf.dequant_4x4( dct4x4[i], h->dequant4_mf[CQM_4IC + b_inter], i_qscale );
        }
Laurent Aimar's avatar
Laurent Aimar committed
430 431

        for( i = 0; i < 4; i++ )
432
            dct4x4[i][0][0] = dct2x2[0][i];
433
        h->dctf.add8x8_idct( p_dst, dct4x4 );
Laurent Aimar's avatar
Laurent Aimar committed
434 435 436
    }
}

437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
static void x264_macroblock_encode_skip( x264_t *h )
{
    int i;
    h->mb.i_cbp_luma = 0x00;
    h->mb.i_cbp_chroma = 0x00;

    for( i = 0; i < 16+8; i++ )
    {
        h->mb.cache.non_zero_count[x264_scan8[i]] = 0;
    }

    /* store cbp */
    h->mb.cbp[h->mb.i_mb_xy] = 0;
}

Laurent Aimar's avatar
Laurent Aimar committed
452 453 454 455 456 457
/*****************************************************************************
 * x264_macroblock_encode_pskip:
 *  Encode an already marked skip block
 *****************************************************************************/
void x264_macroblock_encode_pskip( x264_t *h )
{
458 459 460 461
    const int mvx = x264_clip3( h->mb.cache.mv[0][x264_scan8[0]][0],
                                h->mb.mv_min[0], h->mb.mv_max[0] );
    const int mvy = x264_clip3( h->mb.cache.mv[0][x264_scan8[0]][1],
                                h->mb.mv_min[1], h->mb.mv_max[1] );
Laurent Aimar's avatar
Laurent Aimar committed
462 463

    /* Motion compensation XXX probably unneeded */
464
    h->mc.mc_luma( h->mb.pic.p_fref[0][0], h->mb.pic.i_stride[0],
465 466
                   h->mb.pic.p_fdec[0],    FDEC_STRIDE,
                   mvx, mvy, 16, 16 );
Laurent Aimar's avatar
Laurent Aimar committed
467 468

    /* Chroma MC */
469
    h->mc.mc_chroma( h->mb.pic.p_fref[0][0][4], h->mb.pic.i_stride[1],
470 471
                     h->mb.pic.p_fdec[1],       FDEC_STRIDE,
                     mvx, mvy, 8, 8 );
Laurent Aimar's avatar
Laurent Aimar committed
472

473
    h->mc.mc_chroma( h->mb.pic.p_fref[0][0][5], h->mb.pic.i_stride[2],
474 475
                     h->mb.pic.p_fdec[2],       FDEC_STRIDE,
                     mvx, mvy, 8, 8 );
Laurent Aimar's avatar
Laurent Aimar committed
476

477
    x264_macroblock_encode_skip( h );
Laurent Aimar's avatar
Laurent Aimar committed
478 479 480 481 482 483 484 485
}

/*****************************************************************************
 * x264_macroblock_encode:
 *****************************************************************************/
void x264_macroblock_encode( x264_t *h )
{
    int i_cbp_dc = 0;
486
    int i_qp = h->mb.i_qp;
487
    int b_decimate = h->sh.i_type == SLICE_TYPE_B || h->param.analyse.b_dct_decimate;
488
    int b_force_no_skip = 0;
Laurent Aimar's avatar
Laurent Aimar committed
489 490
    int i;

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
    if( h->sh.b_mbaff
        && h->mb.i_mb_xy == h->sh.i_first_mb + h->mb.i_mb_stride
        && IS_SKIP(h->mb.type[h->sh.i_first_mb]) )
    {
        /* The first skip is predicted to be a frame mb pair.
         * We don't yet support the aff part of mbaff, so force it to non-skip
         * so that we can pick the aff flag. */
        b_force_no_skip = 1;
        if( IS_SKIP(h->mb.i_type) )
        {
            if( h->mb.i_type == P_SKIP )
                h->mb.i_type = P_L0;
            else if( h->mb.i_type == B_SKIP )
                h->mb.i_type = B_DIRECT;
        }
    }

Laurent Aimar's avatar
Laurent Aimar committed
508 509 510 511 512 513
    if( h->mb.i_type == P_SKIP )
    {
        /* A bit special */
        x264_macroblock_encode_pskip( h );
        return;
    }
514 515 516 517 518 519 520
    if( h->mb.i_type == B_SKIP )
    {
        /* XXX motion compensation is probably unneeded */
        x264_mb_mc( h );
        x264_macroblock_encode_skip( h );
        return;
    }
Laurent Aimar's avatar
Laurent Aimar committed
521 522 523 524

    if( h->mb.i_type == I_16x16 )
    {
        const int i_mode = h->mb.i_intra16x16_pred_mode;
525
        h->mb.b_transform_8x8 = 0;
Laurent Aimar's avatar
Laurent Aimar committed
526
        /* do the right prediction */
527
        h->predict_16x16[i_mode]( h->mb.pic.p_fdec[0] );
Laurent Aimar's avatar
Laurent Aimar committed
528 529

        /* encode the 16x16 macroblock */
530
        x264_mb_encode_i16x16( h, i_qp );
Laurent Aimar's avatar
Laurent Aimar committed
531
    }
532 533
    else if( h->mb.i_type == I_8x8 )
    {
534
        DECLARE_ALIGNED( uint8_t, edge[33], 8 );
535
        h->mb.b_transform_8x8 = 1;
536 537
        for( i = 0; i < 4; i++ )
        {
538
            uint8_t  *p_dst = &h->mb.pic.p_fdec[0][8 * (i&1) + 8 * (i>>1) * FDEC_STRIDE];
539 540
            int      i_mode = h->mb.cache.intra4x4_pred_mode[x264_scan8[4*i]];

541 542
            x264_predict_8x8_filter( p_dst, edge, h->mb.i_neighbour8[i], x264_pred_i4x4_neighbors[i_mode] );
            h->predict_8x8[i_mode]( p_dst, edge );
543
            x264_mb_encode_i8x8( h, i, i_qp );
544 545
        }
    }
Laurent Aimar's avatar
Laurent Aimar committed
546 547
    else if( h->mb.i_type == I_4x4 )
    {
548
        h->mb.b_transform_8x8 = 0;
Laurent Aimar's avatar
Laurent Aimar committed
549 550
        for( i = 0; i < 16; i++ )
        {
551
            uint8_t  *p_dst = &h->mb.pic.p_fdec[0][4 * block_idx_x[i] + 4 * block_idx_y[i] * FDEC_STRIDE];
Laurent Aimar's avatar
Laurent Aimar committed
552 553
            int      i_mode = h->mb.cache.intra4x4_pred_mode[x264_scan8[i]];

554 555
            if( (h->mb.i_neighbour4[i] & (MB_TOPRIGHT|MB_TOP)) == MB_TOP )
                /* emulate missing topright samples */
556
                *(uint32_t*) &p_dst[4-FDEC_STRIDE] = p_dst[3-FDEC_STRIDE] * 0x01010101U;
557

558
            h->predict_4x4[i_mode]( p_dst );
559
            x264_mb_encode_i4x4( h, i, i_qp );
Laurent Aimar's avatar
Laurent Aimar committed
560 561 562 563 564 565 566 567 568 569
        }
    }
    else    /* Inter MB */
    {
        int i8x8, i4x4, idx;
        int i_decimate_mb = 0;

        /* Motion compensation */
        x264_mb_mc( h );

Loren Merritt's avatar
Loren Merritt committed
570 571 572 573
        if( h->mb.b_lossless )
        {
            for( i4x4 = 0; i4x4 < 16; i4x4++ )
            {
574 575 576 577
                int x = 4*block_idx_x[i4x4];
                int y = 4*block_idx_y[i4x4];
                sub_zigzag_4x4full( h->dct.block[i4x4].luma4x4,
                                    h->mb.pic.p_fenc[0]+x+y*FENC_STRIDE,
578 579
                                    h->mb.pic.p_fdec[0]+x+y*FDEC_STRIDE,
                                    h->mb.b_interlaced );
Loren Merritt's avatar
Loren Merritt committed
580 581 582
            }
        }
        else if( h->mb.b_transform_8x8 )
Laurent Aimar's avatar
Laurent Aimar committed
583
        {
584
            int16_t dct8x8[4][8][8];
585
            int nnz8x8[4] = {1,1,1,1};
586
            b_decimate &= !h->mb.b_trellis; // 8x8 trellis is inherently optimal decimation
587
            h->dctf.sub16x16_dct8( dct8x8, h->mb.pic.p_fenc[0], h->mb.pic.p_fdec[0] );
Laurent Aimar's avatar
Laurent Aimar committed
588

589
            for( idx = 0; idx < 4; idx++ )
Laurent Aimar's avatar
Laurent Aimar committed
590
            {
591 592
                if( h->mb.b_noise_reduction )
                    x264_denoise_dct( h, (int16_t*)dct8x8[idx] );
Loren Merritt's avatar
Loren Merritt committed
593 594 595 596
                if( h->mb.b_trellis )
                    x264_quant_8x8_trellis( h, dct8x8[idx], CQM_8PY, i_qp, 0 );
                else
                    quant_8x8( h, dct8x8[idx], h->quant8_mf[CQM_8PY], i_qp, 0 );
Laurent Aimar's avatar
Laurent Aimar committed
597

598
                scan_zigzag_8x8full( h->dct.luma8x8[idx], dct8x8[idx], h->mb.b_interlaced );
Laurent Aimar's avatar
Laurent Aimar committed
599

600
                if( b_decimate )
601
                {
Loren Merritt's avatar
Loren Merritt committed
602 603 604 605 606 607
                    int i_decimate_8x8 = x264_mb_decimate_score( h->dct.luma8x8[idx], 64 );
                    i_decimate_mb += i_decimate_8x8;
                    if( i_decimate_8x8 < 4 )
                    {
                        memset( h->dct.luma8x8[idx], 0, sizeof( h->dct.luma8x8[idx] ) );
                        memset( dct8x8[idx], 0, sizeof( dct8x8[idx] ) );
608
                        nnz8x8[idx] = 0;
Loren Merritt's avatar
Loren Merritt committed
609
                    }
610
                }
611 612
                else
                    nnz8x8[idx] = array_non_zero( (int*)dct8x8[idx], sizeof(*dct8x8)/sizeof(int) );
Laurent Aimar's avatar
Laurent Aimar committed
613 614
            }

615
            if( i_decimate_mb < 6 && b_decimate )
616 617
                memset( h->dct.luma8x8, 0, sizeof( h->dct.luma8x8 ) );
            else
618 619 620 621 622
            {
                for( idx = 0; idx < 4; idx++ )
                    if( nnz8x8[idx] )
                    {
                        h->quantf.dequant_8x8( dct8x8[idx], h->dequant8_mf[CQM_8PY], i_qp );
623
                        h->dctf.add8x8_idct8( &h->mb.pic.p_fdec[0][(idx&1)*8 + (idx>>1)*8*FDEC_STRIDE], dct8x8[idx] );
624 625
                    }
            }
626 627 628 629
        }
        else
        {
            int16_t dct4x4[16][4][4];
630
            int nnz8x8[4] = {1,1,1,1};
631
            h->dctf.sub16x16_dct( dct4x4, h->mb.pic.p_fenc[0], h->mb.pic.p_fdec[0] );
632 633

            for( i8x8 = 0; i8x8 < 4; i8x8++ )
Laurent Aimar's avatar
Laurent Aimar committed
634
            {
635 636 637 638
                int i_decimate_8x8;

                /* encode one 4x4 block */
                i_decimate_8x8 = 0;
Laurent Aimar's avatar
Laurent Aimar committed
639 640 641
                for( i4x4 = 0; i4x4 < 4; i4x4++ )
                {
                    idx = i8x8 * 4 + i4x4;
642

643 644
                    if( h->mb.b_noise_reduction )
                        x264_denoise_dct( h, (int16_t*)dct4x4[idx] );
Loren Merritt's avatar
Loren Merritt committed
645 646 647 648 649
                    if( h->mb.b_trellis )
                        x264_quant_4x4_trellis( h, dct4x4[idx], CQM_4PY, i_qp, DCT_LUMA_4x4, 0 );
                    else
                        quant_4x4( h, dct4x4[idx], h->quant4_mf[CQM_4PY], i_qp, 0 );

650
                    scan_zigzag_4x4full( h->dct.block[idx].luma4x4, dct4x4[idx], h->mb.b_interlaced );
651 652 653
                    
                    if( b_decimate )
                        i_decimate_8x8 += x264_mb_decimate_score( h->dct.block[idx].luma4x4, 16 );
Laurent Aimar's avatar
Laurent Aimar committed
654 655
                }

656 657
                /* decimate this 8x8 block */
                i_decimate_mb += i_decimate_8x8;
658
                if( i_decimate_8x8 < 4 && b_decimate )
Laurent Aimar's avatar
Laurent Aimar committed
659
                {
Loren Merritt's avatar
Loren Merritt committed
660 661
                    memset( &dct4x4[i8x8*4], 0, 4 * sizeof( *dct4x4 ) );
                    memset( &h->dct.block[i8x8*4], 0, 4 * sizeof( *h->dct.block ) );
662
                    nnz8x8[i8x8] = 0;
Laurent Aimar's avatar
Laurent Aimar committed
663 664
                }
            }
665

666
            if( i_decimate_mb < 6 && b_decimate )
Loren Merritt's avatar
Loren Merritt committed
667
                memset( h->dct.block, 0, 16 * sizeof( *h->dct.block ) );
668
            else
669 670 671 672 673 674
            {
                for( i8x8 = 0; i8x8 < 4; i8x8++ )
                    if( nnz8x8[i8x8] )
                    {
                        for( i = 0; i < 4; i++ )
                            h->quantf.dequant_4x4( dct4x4[i8x8*4+i], h->dequant4_mf[CQM_4PY], i_qp );
675
                        h->dctf.add8x8_idct( &h->mb.pic.p_fdec[0][(i8x8&1)*8 + (i8x8>>1)*8*FDEC_STRIDE], &dct4x4[i8x8*4] );
676 677
                    }
            }
Laurent Aimar's avatar
Laurent Aimar committed
678 679 680 681
        }
    }

    /* encode chroma */
682
    i_qp = i_chroma_qp_table[x264_clip3( i_qp + h->pps->i_chroma_qp_index_offset, 0, 51 )];
Laurent Aimar's avatar
Laurent Aimar committed
683 684 685
    if( IS_INTRA( h->mb.i_type ) )
    {
        const int i_mode = h->mb.i_chroma_pred_mode;
686 687
        h->predict_8x8c[i_mode]( h->mb.pic.p_fdec[1] );
        h->predict_8x8c[i_mode]( h->mb.pic.p_fdec[2] );
Laurent Aimar's avatar
Laurent Aimar committed
688 689 690
    }

    /* encode the 8x8 blocks */
691
    x264_mb_encode_8x8_chroma( h, !IS_INTRA( h->mb.i_type ), i_qp );
Laurent Aimar's avatar
Laurent Aimar committed
692

Loren Merritt's avatar
Loren Merritt committed
693
    /* Calculate the Luma/Chroma pattern and non_zero_count */
694
    h->mb.i_cbp_luma = 0x00;
Laurent Aimar's avatar
Laurent Aimar committed
695 696 697 698 699 700 701 702
    if( h->mb.i_type == I_16x16 )
    {
        for( i = 0; i < 16; i++ )
        {
            const int nz = array_non_zero_count( h->dct.block[i].residual_ac, 15 );
            h->mb.cache.non_zero_count[x264_scan8[i]] = nz;
            if( nz > 0 )
                h->mb.i_cbp_luma = 0x0f;
703 704 705 706
        }
    }
    else if( h->mb.b_transform_8x8 )
    {
Loren Merritt's avatar
Loren Merritt committed
707 708
        /* coded_block_flag is enough for CABAC.
         * the full non_zero_count is done only in CAVLC. */
709 710 711 712 713 714 715 716
        for( i = 0; i < 4; i++ )
        {
            const int nz = array_non_zero( h->dct.luma8x8[i], 64 );
            int j;
            for( j = 0; j < 4; j++ )
                h->mb.cache.non_zero_count[x264_scan8[4*i+j]] = nz;
            if( nz > 0 )
                h->mb.i_cbp_luma |= 1 << i;
Laurent Aimar's avatar
Laurent Aimar committed
717 718 719 720 721 722 723 724 725 726 727 728 729
        }
    }
    else
    {
        for( i = 0; i < 16; i++ )
        {
            const int nz = array_non_zero_count( h->dct.block[i].luma4x4, 16 );
            h->mb.cache.non_zero_count[x264_scan8[i]] = nz;
            if( nz > 0 )
                h->mb.i_cbp_luma |= 1 << (i/4);
        }
    }

Loren Merritt's avatar
Loren Merritt committed
730
    /* Calculate the chroma pattern */
Laurent Aimar's avatar
Laurent Aimar committed
731 732 733 734 735 736 737 738 739 740
    h->mb.i_cbp_chroma = 0x00;
    for( i = 0; i < 8; i++ )
    {
        const int nz = array_non_zero_count( h->dct.block[16+i].residual_ac, 15 );
        h->mb.cache.non_zero_count[x264_scan8[16+i]] = nz;
        if( nz > 0 )
        {
            h->mb.i_cbp_chroma = 0x02;    /* dc+ac (we can't do only ac) */
        }
    }
741
    if( h->mb.i_cbp_chroma == 0x00 && array_non_zero( h->dct.chroma_dc[0], 8 ) )
Laurent Aimar's avatar
Laurent Aimar committed
742 743 744 745 746 747
    {
        h->mb.i_cbp_chroma = 0x01;    /* dc only */
    }

    if( h->param.b_cabac )
    {
748 749 750
        i_cbp_dc = ( h->mb.i_type == I_16x16 && array_non_zero( h->dct.luma16x16_dc, 16 ) )
                 | array_non_zero( h->dct.chroma_dc[0], 4 ) << 1
                 | array_non_zero( h->dct.chroma_dc[1], 4 ) << 2;
Laurent Aimar's avatar
Laurent Aimar committed
751 752 753 754 755 756 757 758
    }

    /* store cbp */
    h->mb.cbp[h->mb.i_mb_xy] = (i_cbp_dc << 8) | (h->mb.i_cbp_chroma << 4) | h->mb.i_cbp_luma;

    /* Check for P_SKIP
     * XXX: in the me perhaps we should take x264_mb_predict_mv_pskip into account
     *      (if multiple mv give same result)*/
759
    if( !b_force_no_skip )
Laurent Aimar's avatar
Laurent Aimar committed
760
    {
761 762 763
        if( h->mb.i_type == P_L0 && h->mb.i_partition == D_16x16 &&
            h->mb.i_cbp_luma == 0x00 && h->mb.i_cbp_chroma== 0x00 &&
            h->mb.cache.ref[0][x264_scan8[0]] == 0 )
764
        {
765 766 767 768 769 770 771 772
            int mvp[2];

            x264_mb_predict_mv_pskip( h, mvp );
            if( h->mb.cache.mv[0][x264_scan8[0]][0] == mvp[0] &&
                h->mb.cache.mv[0][x264_scan8[0]][1] == mvp[1] )
            {
                h->mb.i_type = P_SKIP;
            }
Laurent Aimar's avatar
Laurent Aimar committed
773
        }
774

775 776 777 778 779 780
        /* Check for B_SKIP */
        if( h->mb.i_type == B_DIRECT &&
            h->mb.i_cbp_luma == 0x00 && h->mb.i_cbp_chroma== 0x00 )
        {
            h->mb.i_type = B_SKIP;
        }
781
    }
Laurent Aimar's avatar
Laurent Aimar committed
782 783 784
}

/*****************************************************************************
785 786
 * x264_macroblock_probe_skip:
 *  Check if the current MB could be encoded as a [PB]_SKIP (it supposes you use
Laurent Aimar's avatar
Laurent Aimar committed
787 788
 *  the previous QP
 *****************************************************************************/
789
int x264_macroblock_probe_skip( x264_t *h, int b_bidir )
Laurent Aimar's avatar
Laurent Aimar committed
790 791 792 793 794
{
    DECLARE_ALIGNED( int16_t, dct4x4[16][4][4], 16 );
    DECLARE_ALIGNED( int16_t, dct2x2[2][2], 16 );
    DECLARE_ALIGNED( int,     dctscan[16], 16 );

795
    int i_qp = h->mb.i_qp;
Laurent Aimar's avatar
Laurent Aimar committed
796 797 798 799 800 801
    int mvp[2];
    int ch;

    int i8x8, i4x4;
    int i_decimate_mb;

802 803 804 805
    if( !b_bidir )
    {
        /* Get the MV */
        x264_mb_predict_mv_pskip( h, mvp );
806 807
        mvp[0] = x264_clip3( mvp[0], h->mb.mv_min[0], h->mb.mv_max[0] );
        mvp[1] = x264_clip3( mvp[1], h->mb.mv_min[1], h->mb.mv_max[1] );
Laurent Aimar's avatar
Laurent Aimar committed
808

809
        /* Motion compensation */
810
        h->mc.mc_luma( h->mb.pic.p_fref[0][0], h->mb.pic.i_stride[0],
811 812
                       h->mb.pic.p_fdec[0],    FDEC_STRIDE,
                       mvp[0], mvp[1], 16, 16 );
813
    }
Laurent Aimar's avatar
Laurent Aimar committed
814 815

    /* get luma diff */
816 817
    h->dctf.sub16x16_dct( dct4x4, h->mb.pic.p_fenc[0],
                                  h->mb.pic.p_fdec[0] );
Laurent Aimar's avatar
Laurent Aimar committed
818 819 820 821 822 823 824 825

    for( i8x8 = 0, i_decimate_mb = 0; i8x8 < 4; i8x8++ )
    {
        /* encode one 4x4 block */
        for( i4x4 = 0; i4x4 < 4; i4x4++ )
        {
            const int idx = i8x8 * 4 + i4x4;

826
            quant_4x4( h, dct4x4[idx], (int(*)[4][4])def_quant4_mf, i_qp, 0 );
827
            scan_zigzag_4x4full( dctscan, dct4x4[idx], h->mb.b_interlaced );
Laurent Aimar's avatar
Laurent Aimar committed
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846

            i_decimate_mb += x264_mb_decimate_score( dctscan, 16 );

            if( i_decimate_mb >= 6 )
            {
                /* not as P_SKIP */
                return 0;
            }
        }
    }

    /* encode chroma */
    i_qp = i_chroma_qp_table[x264_clip3( i_qp + h->pps->i_chroma_qp_index_offset, 0, 51 )];

    for( ch = 0; ch < 2; ch++ )
    {
        uint8_t  *p_src = h->mb.pic.p_fenc[1+ch];
        uint8_t  *p_dst = h->mb.pic.p_fdec[1+ch];

847 848
        if( !b_bidir )
        {
849 850 851
            h->mc.mc_chroma( h->mb.pic.p_fref[0][0][4+ch], h->mb.pic.i_stride[1+ch],
                             h->mb.pic.p_fdec[1+ch],       FDEC_STRIDE,
                             mvp[0], mvp[1], 8, 8 );
852
        }
Laurent Aimar's avatar
Laurent Aimar committed
853

854
        h->dctf.sub8x8_dct( dct4x4, p_src, p_dst );
Laurent Aimar's avatar
Laurent Aimar committed
855 856 857 858 859 860 861

        /* calculate dct DC */
        dct2x2[0][0] = dct4x4[0][0][0];
        dct2x2[0][1] = dct4x4[1][0][0];
        dct2x2[1][0] = dct4x4[2][0][0];
        dct2x2[1][1] = dct4x4[3][0][0];
        h->dctf.dct2x2dc( dct2x2 );
862
        quant_2x2_dc( h, dct2x2, (int(*)[4][4])def_quant4_mf, i_qp, 0 );
Laurent Aimar's avatar
Laurent Aimar committed
863 864 865 866 867 868 869 870 871
        if( dct2x2[0][0] || dct2x2[0][1] || dct2x2[1][0] || dct2x2[1][1]  )
        {
            /* can't be */
            return 0;
        }

        /* calculate dct coeffs */
        for( i4x4 = 0, i_decimate_mb = 0; i4x4 < 4; i4x4++ )
        {
872
            quant_4x4( h, dct4x4[i4x4], (int(*)[4][4])def_quant4_mf, i_qp, 0 );
873
            scan_zigzag_4x4( dctscan, dct4x4[i4x4], h->mb.b_interlaced );
Laurent Aimar's avatar
Laurent Aimar committed
874 875 876 877 878 879 880 881 882 883 884

            i_decimate_mb += x264_mb_decimate_score( dctscan, 15 );
            if( i_decimate_mb >= 7 )
            {
                return 0;
            }
        }
    }

    return 1;
}
885 886 887 888 889 890 891 892 893

/****************************************************************************
 * DCT-domain noise reduction / adaptive deadzone
 * from libavcodec
 ****************************************************************************/

void x264_noise_reduction_update( x264_t *h )
{
    int cat, i;
894
    for( cat = 0; cat < 2; cat++ )
895
    {
896 897
        int size = cat ? 64 : 16;
        const int *weight = cat ? x264_dct8_weight2_tab : x264_dct4_weight2_tab;
898

899
        if( h->nr_count[cat] > (cat ? (1<<16) : (1<<18)) )
900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915
        {
            for( i = 0; i < size; i++ )
                h->nr_residual_sum[cat][i] >>= 1;
            h->nr_count[cat] >>= 1;
        }

        for( i = 0; i < size; i++ )
            h->nr_offset[cat][i] =
                ((uint64_t)h->param.analyse.i_noise_reduction * h->nr_count[cat]
                 + h->nr_residual_sum[cat][i]/2)
              / ((uint64_t)h->nr_residual_sum[cat][i] * weight[i]/256 + 1);
    }
}

void x264_denoise_dct( x264_t *h, int16_t *dct )
{
916
    const int cat = h->mb.b_transform_8x8;
917 918 919 920
    int i;

    h->nr_count[cat]++;

921
    for( i = (cat ? 63 : 15); i >= 1; i-- )
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943
    {
        int level = dct[i];
        if( level )
        {
            if( level > 0 )
            {
                h->nr_residual_sum[cat][i] += level;
                level -= h->nr_offset[cat][i];
                if( level < 0 )
                    level = 0;
            }
            else
            {
                h->nr_residual_sum[cat][i] -= level;
                level += h->nr_offset[cat][i];
                if( level > 0 )
                    level = 0;
            }
            dct[i] = level;
        }
    }
}
944 945 946 947 948 949 950 951 952 953

/*****************************************************************************
 * RD only; 4 calls to this do not make up for one macroblock_encode.
 * doesn't transform chroma dc.
 *****************************************************************************/
void x264_macroblock_encode_p8x8( x264_t *h, int i8 )
{
    int i_qp = h->mb.i_qp;
    uint8_t *p_fenc = h->mb.pic.p_fenc[0] + (i8&1)*8 + (i8>>1)*8*FENC_STRIDE;
    uint8_t *p_fdec = h->mb.pic.p_fdec[0] + (i8&1)*8 + (i8>>1)*8*FDEC_STRIDE;