set.c 12.1 KB
Newer Older
Loren Merritt's avatar
Loren Merritt committed
1
/*****************************************************************************
Fiona Glaser's avatar
Fiona Glaser committed
2
 * set.c: quantization init
Loren Merritt's avatar
Loren Merritt committed
3
 *****************************************************************************
Sean McGovern's avatar
Sean McGovern committed
4
 * Copyright (C) 2005-2011 x264 project
Fiona Glaser's avatar
Fiona Glaser committed
5 6
 *
 * Authors: Loren Merritt <lorenm@u.washington.edu>
Loren Merritt's avatar
Loren Merritt committed
7 8 9 10 11 12 13 14 15 16 17 18 19
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
20
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
Fiona Glaser's avatar
Fiona Glaser committed
21 22 23
 *
 * This program is also available under a commercial proprietary license.
 * For more information, contact us at licensing@x264.com.
Loren Merritt's avatar
Loren Merritt committed
24 25
 *****************************************************************************/

Fiona Glaser's avatar
Fiona Glaser committed
26 27
#define _ISOC99_SOURCE
#include <math.h>
Loren Merritt's avatar
Loren Merritt committed
28 29
#include "common.h"

Loren Merritt's avatar
Loren Merritt committed
30
#define SHIFT(x,s) ((s)<=0 ? (x)<<-(s) : ((x)+(1<<((s)-1)))>>(s))
Loren Merritt's avatar
Loren Merritt committed
31 32
#define DIV(n,d) (((n) + ((d)>>1)) / (d))

33
static const uint8_t dequant4_scale[6][3] =
Loren Merritt's avatar
Loren Merritt committed
34 35 36 37 38 39 40 41
{
    { 10, 13, 16 },
    { 11, 14, 18 },
    { 13, 16, 20 },
    { 14, 18, 23 },
    { 16, 20, 25 },
    { 18, 23, 29 }
};
42
static const uint16_t quant4_scale[6][3] =
Loren Merritt's avatar
Loren Merritt committed
43 44 45 46 47 48 49 50 51
{
    { 13107, 8066, 5243 },
    { 11916, 7490, 4660 },
    { 10082, 6554, 4194 },
    {  9362, 5825, 3647 },
    {  8192, 5243, 3355 },
    {  7282, 4559, 2893 },
};

52
static const uint8_t quant8_scan[16] =
Loren Merritt's avatar
Loren Merritt committed
53 54 55
{
    0,3,4,3, 3,1,5,1, 4,5,2,5, 3,1,5,1
};
56
static const uint8_t dequant8_scale[6][6] =
Loren Merritt's avatar
Loren Merritt committed
57 58 59 60 61 62 63 64
{
    { 20, 18, 32, 19, 25, 24 },
    { 22, 19, 35, 21, 28, 26 },
    { 26, 23, 42, 24, 33, 31 },
    { 28, 25, 45, 26, 35, 33 },
    { 32, 28, 51, 30, 40, 38 },
    { 36, 32, 58, 34, 46, 43 },
};
65
static const uint16_t quant8_scale[6][6] =
Loren Merritt's avatar
Loren Merritt committed
66 67 68 69 70 71 72 73 74
{
    { 13107, 11428, 20972, 12222, 16777, 15481 },
    { 11916, 10826, 19174, 11058, 14980, 14290 },
    { 10082,  8943, 15978,  9675, 12710, 11985 },
    {  9362,  8228, 14913,  8931, 11984, 11259 },
    {  8192,  7346, 13159,  7740, 10486,  9777 },
    {  7282,  6428, 11570,  6830,  9118,  8640 }
};

Loren Merritt's avatar
Loren Merritt committed
75
int x264_cqm_init( x264_t *h )
Loren Merritt's avatar
Loren Merritt committed
76 77 78 79 80
{
    int def_quant4[6][16];
    int def_quant8[6][64];
    int def_dequant4[6][16];
    int def_dequant8[6][64];
Loren Merritt's avatar
Loren Merritt committed
81 82
    int quant4_mf[4][6][16];
    int quant8_mf[2][6][64];
Loren Merritt's avatar
Loren Merritt committed
83
    int deadzone[4] = { 32 - h->param.analyse.i_luma_deadzone[1],
Loren Merritt's avatar
Loren Merritt committed
84
                        32 - h->param.analyse.i_luma_deadzone[0],
Loren Merritt's avatar
Loren Merritt committed
85 86
                        32 - 11, 32 - 21 };
    int max_qp_err = -1;
87
    int max_chroma_qp_err = -1;
88
    int min_qp_err = QP_MAX+1;
89

90
    for( int i = 0; i < 6; i++ )
91 92
    {
        int size = i<4 ? 16 : 64;
93
        int j;
94 95 96 97 98 99 100 101 102 103 104
        for( j = (i<4 ? 0 : 4); j < i; j++ )
            if( !memcmp( h->pps->scaling_list[i], h->pps->scaling_list[j], size*sizeof(uint8_t) ) )
                break;
        if( j < i )
        {
            h->  quant4_mf[i] = h->  quant4_mf[j];
            h->dequant4_mf[i] = h->dequant4_mf[j];
            h->unquant4_mf[i] = h->unquant4_mf[j];
        }
        else
        {
105
            CHECKED_MALLOC( h->  quant4_mf[i], (QP_MAX+1)*size*sizeof(udctcoef) );
106
            CHECKED_MALLOC( h->dequant4_mf[i],  6*size*sizeof(int) );
107
            CHECKED_MALLOC( h->unquant4_mf[i], (QP_MAX+1)*size*sizeof(int) );
108
        }
Loren Merritt's avatar
Loren Merritt committed
109 110 111 112 113 114 115 116

        for( j = (i<4 ? 0 : 4); j < i; j++ )
            if( deadzone[j&3] == deadzone[i&3] &&
                !memcmp( h->pps->scaling_list[i], h->pps->scaling_list[j], size*sizeof(uint8_t) ) )
                break;
        if( j < i )
            h->quant4_bias[i] = h->quant4_bias[j];
        else
117
            CHECKED_MALLOC( h->quant4_bias[i], (QP_MAX+1)*size*sizeof(udctcoef) );
118
    }
Loren Merritt's avatar
Loren Merritt committed
119

120
    for( int q = 0; q < 6; q++ )
Loren Merritt's avatar
Loren Merritt committed
121
    {
122
        for( int i = 0; i < 16; i++ )
Loren Merritt's avatar
Loren Merritt committed
123 124 125 126 127
        {
            int j = (i&1) + ((i>>2)&1);
            def_dequant4[q][i] = dequant4_scale[q][j];
            def_quant4[q][i]   =   quant4_scale[q][j];
        }
128
        for( int i = 0; i < 64; i++ )
Loren Merritt's avatar
Loren Merritt committed
129 130 131 132 133 134 135
        {
            int j = quant8_scan[((i>>1)&12) | (i&3)];
            def_dequant8[q][i] = dequant8_scale[q][j];
            def_quant8[q][i]   =   quant8_scale[q][j];
        }
    }

136
    for( int q = 0; q < 6; q++ )
Loren Merritt's avatar
Loren Merritt committed
137
    {
138 139
        for( int i_list = 0; i_list < 4; i_list++ )
            for( int i = 0; i < 16; i++ )
Loren Merritt's avatar
Loren Merritt committed
140
            {
Loren Merritt's avatar
Loren Merritt committed
141 142
                h->dequant4_mf[i_list][q][i] = def_dequant4[q][i] * h->pps->scaling_list[i_list][i];
                     quant4_mf[i_list][q][i] = DIV(def_quant4[q][i] * 16, h->pps->scaling_list[i_list][i]);
Loren Merritt's avatar
Loren Merritt committed
143
            }
144 145
        for( int i_list = 0; i_list < 2; i_list++ )
            for( int i = 0; i < 64; i++ )
Loren Merritt's avatar
Loren Merritt committed
146
            {
Loren Merritt's avatar
Loren Merritt committed
147 148
                h->dequant8_mf[i_list][q][i] = def_dequant8[q][i] * h->pps->scaling_list[4+i_list][i];
                     quant8_mf[i_list][q][i] = DIV(def_quant8[q][i] * 16, h->pps->scaling_list[4+i_list][i]);
Loren Merritt's avatar
Loren Merritt committed
149 150
            }
    }
151
    for( int q = 0; q < QP_MAX+1; q++ )
Loren Merritt's avatar
Loren Merritt committed
152
    {
153 154 155
        int j;
        for( int i_list = 0; i_list < 4; i_list++ )
            for( int i = 0; i < 16; i++ )
Loren Merritt's avatar
Loren Merritt committed
156
            {
Loren Merritt's avatar
Loren Merritt committed
157
                h->unquant4_mf[i_list][q][i] = (1ULL << (q/6 + 15 + 8)) / quant4_mf[i_list][q%6][i];
158
                h->quant4_mf[i_list][q][i] = j = SHIFT(quant4_mf[i_list][q%6][i], q/6 - 1);
159 160 161 162 163
                if( !j )
                {
                    min_qp_err = X264_MIN( min_qp_err, q );
                    continue;
                }
Loren Merritt's avatar
Loren Merritt committed
164 165
                // round to nearest, unless that would cause the deadzone to be negative
                h->quant4_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j );
166
                if( j > 0xffff && q > max_qp_err && (i_list == CQM_4IY || i_list == CQM_4PY) )
Loren Merritt's avatar
Loren Merritt committed
167
                    max_qp_err = q;
168 169
                if( j > 0xffff && q > max_chroma_qp_err && (i_list == CQM_4IC || i_list == CQM_4PC) )
                    max_chroma_qp_err = q;
Loren Merritt's avatar
Loren Merritt committed
170 171
            }
        if( h->param.analyse.b_transform_8x8 )
172 173 174 175
            for( int i_list = 0; i_list < 2; i_list++ )
                for( int i = 0; i < 64; i++ )
                {
                    h->unquant8_mf[i_list][q][i] = (1ULL << (q/6 + 16 + 8)) / quant8_mf[i_list][q%6][i];
176 177 178
                    j = SHIFT(quant8_mf[i_list][q%6][i], q/6);
                    h->quant8_mf[i_list][q][i] = (uint16_t)j;

179 180 181 182 183
                    if( !j )
                    {
                        min_qp_err = X264_MIN( min_qp_err, q );
                        continue;
                    }
184 185 186 187
                    h->quant8_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j );
                    if( j > 0xffff && q > max_qp_err )
                        max_qp_err = q;
                }
Loren Merritt's avatar
Loren Merritt committed
188 189
    }

Fiona Glaser's avatar
Fiona Glaser committed
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
    /* Emergency mode denoising. */
    x264_emms();
    CHECKED_MALLOC( h->nr_offset_emergency, sizeof(*h->nr_offset_emergency)*(QP_MAX-QP_MAX_SPEC) );
    for( int q = 0; q < QP_MAX - QP_MAX_SPEC; q++ )
        for( int cat = 0; cat <= 2; cat++ )
        {
            int dct8x8 = cat == 1;
            int size = dct8x8 ? 64 : 16;
            udctcoef *nr_offset = h->nr_offset_emergency[q][cat];
            /* Denoise chroma first (due to h264's chroma QP offset, then luma, then DC. */
            int dc_threshold =    (QP_MAX-QP_MAX_SPEC)*2/3;
            int luma_threshold =  (QP_MAX-QP_MAX_SPEC)*2/3;
            int chroma_threshold = 0;

            for( int i = 0; i < size; i++ )
            {
                int max = (1 << (7 + BIT_DEPTH)) - 1;
                /* True "emergency mode": remove all DCT coefficients */
                if( q == QP_MAX - QP_MAX_SPEC - 1 )
                {
                    nr_offset[i] = max;
                    continue;
                }

                int thresh = i == 0 ? dc_threshold : cat == 2 ? chroma_threshold : luma_threshold;
                if( q < thresh )
                {
                    nr_offset[i] = 0;
                    continue;
                }
                double pos = (double)(q-thresh+1) / (QP_MAX - QP_MAX_SPEC - thresh);

                /* XXX: this math is largely tuned for /dev/random input. */
                double start = dct8x8 ? h->unquant8_mf[CQM_8PY][QP_MAX_SPEC][i]
                                      : h->unquant4_mf[CQM_4PY][QP_MAX_SPEC][i];
                /* Formula chosen as an exponential scale to vaguely mimic the effects
                 * of a higher quantizer. */
                double bias = (pow( 2, pos*(QP_MAX - QP_MAX_SPEC)/10. )*0.003-0.003) * start;
                nr_offset[i] = X264_MIN( bias + 0.5, max );
            }
        }

232
    if( !h->mb.b_lossless )
Loren Merritt's avatar
Loren Merritt committed
233
    {
234 235 236 237 238 239 240 241 242 243 244
        while( h->chroma_qp_table[h->param.rc.i_qp_min] <= max_chroma_qp_err )
            h->param.rc.i_qp_min++;
        if( min_qp_err <= h->param.rc.i_qp_max )
            h->param.rc.i_qp_max = min_qp_err-1;
        if( max_qp_err >= h->param.rc.i_qp_min )
            h->param.rc.i_qp_min = max_qp_err+1;
        if( h->param.rc.i_qp_min > h->param.rc.i_qp_max )
        {
            x264_log( h, X264_LOG_ERROR, "Impossible QP constraints for CQM (min=%d, max=%d)\n", h->param.rc.i_qp_min, h->param.rc.i_qp_max );
            return -1;
        }
245
    }
Loren Merritt's avatar
Loren Merritt committed
246
    return 0;
247 248 249
fail:
    x264_cqm_delete( h );
    return -1;
Loren Merritt's avatar
Loren Merritt committed
250 251
}

252
#define CQM_DELETE( n, max )\
253
    for( int i = 0; i < max; i++ )\
254
    {\
255
        int j;\
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
        for( j = 0; j < i; j++ )\
            if( h->quant##n##_mf[i] == h->quant##n##_mf[j] )\
                break;\
        if( j == i )\
        {\
            x264_free( h->  quant##n##_mf[i] );\
            x264_free( h->dequant##n##_mf[i] );\
            x264_free( h->unquant##n##_mf[i] );\
        }\
        for( j = 0; j < i; j++ )\
            if( h->quant##n##_bias[i] == h->quant##n##_bias[j] )\
                break;\
        if( j == i )\
            x264_free( h->quant##n##_bias[i] );\
    }

272 273
void x264_cqm_delete( x264_t *h )
{
274 275
    CQM_DELETE( 4, 4 );
    CQM_DELETE( 8, 2 );
Fiona Glaser's avatar
Fiona Glaser committed
276
    x264_free( h->nr_offset_emergency );
277 278
}

279
static int x264_cqm_parse_jmlist( x264_t *h, const char *buf, const char *name,
280 281 282 283
                           uint8_t *cqm, const uint8_t *jvt, int length )
{
    int i;

284
    char *p = strstr( buf, name );
285 286 287 288 289 290 291 292 293 294
    if( !p )
    {
        memset( cqm, 16, length );
        return 0;
    }

    p += strlen( name );
    if( *p == 'U' || *p == 'V' )
        p++;

295
    char *nextvar = strstr( p, "INT" );
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324

    for( i = 0; i < length && (p = strpbrk( p, " \t\n," )) && (p = strpbrk( p, "0123456789" )); i++ )
    {
        int coef = -1;
        sscanf( p, "%d", &coef );
        if( i == 0 && coef == 0 )
        {
            memcpy( cqm, jvt, length );
            return 0;
        }
        if( coef < 1 || coef > 255 )
        {
            x264_log( h, X264_LOG_ERROR, "bad coefficient in list '%s'\n", name );
            return -1;
        }
        cqm[i] = coef;
    }

    if( (nextvar && p > nextvar) || i != length )
    {
        x264_log( h, X264_LOG_ERROR, "not enough coefficients in list '%s'\n", name );
        return -1;
    }

    return 0;
}

int x264_cqm_parse_file( x264_t *h, const char *filename )
{
325
    char *p;
326 327 328
    int b_error = 0;

    h->param.i_cqm_preset = X264_CQM_CUSTOM;
Loren Merritt's avatar
Loren Merritt committed
329

330
    char *buf = x264_slurp_file( filename );
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
    if( !buf )
    {
        x264_log( h, X264_LOG_ERROR, "can't open file '%s'\n", filename );
        return -1;
    }

    while( (p = strchr( buf, '#' )) != NULL )
        memset( p, ' ', strcspn( p, "\n" ) );

    b_error |= x264_cqm_parse_jmlist( h, buf, "INTRA4X4_LUMA",   h->param.cqm_4iy, x264_cqm_jvt4i, 16 );
    b_error |= x264_cqm_parse_jmlist( h, buf, "INTRA4X4_CHROMA", h->param.cqm_4ic, x264_cqm_jvt4i, 16 );
    b_error |= x264_cqm_parse_jmlist( h, buf, "INTER4X4_LUMA",   h->param.cqm_4py, x264_cqm_jvt4p, 16 );
    b_error |= x264_cqm_parse_jmlist( h, buf, "INTER4X4_CHROMA", h->param.cqm_4pc, x264_cqm_jvt4p, 16 );
    b_error |= x264_cqm_parse_jmlist( h, buf, "INTRA8X8_LUMA",   h->param.cqm_8iy, x264_cqm_jvt8i, 64 );
    b_error |= x264_cqm_parse_jmlist( h, buf, "INTER8X8_LUMA",   h->param.cqm_8py, x264_cqm_jvt8p, 64 );

    x264_free( buf );
    return b_error;
}