encoder.c 109 KB
Newer Older
Laurent Aimar's avatar
Laurent Aimar committed
1 2 3
/*****************************************************************************
 * x264: h264 encoder
 *****************************************************************************
4
 * Copyright (C) 2003-2008 x264 project
Laurent Aimar's avatar
Laurent Aimar committed
5 6
 *
 * Authors: Laurent Aimar <fenrir@via.ecp.fr>
7
 *          Loren Merritt <lorenm@u.washington.edu>
8
 *          Fiona Glaser <fiona@x264.com>
Laurent Aimar's avatar
Laurent Aimar committed
9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
22
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
Laurent Aimar's avatar
Laurent Aimar committed
23 24 25 26
 *****************************************************************************/

#include <math.h>

27 28
#include "common/common.h"
#include "common/cpu.h"
Laurent Aimar's avatar
Laurent Aimar committed
29 30 31 32 33

#include "set.h"
#include "analyse.h"
#include "ratecontrol.h"
#include "macroblock.h"
34
#include "me.h"
Laurent Aimar's avatar
Laurent Aimar committed
35

36
#ifdef HAVE_VISUALIZE
Loren Merritt's avatar
Loren Merritt committed
37 38 39
#include "common/visualize.h"
#endif

Laurent Aimar's avatar
Laurent Aimar committed
40 41
//#define DEBUG_MB_TYPE

42
#define NALU_OVERHEAD 5 // startcode + NAL type costs 5 bytes per frame
Laurent Aimar's avatar
Laurent Aimar committed
43

Fiona Glaser's avatar
Fiona Glaser committed
44 45
#define bs_write_ue bs_write_ue_big

Fiona Glaser's avatar
Fiona Glaser committed
46
static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current,
47 48
                                   x264_nal_t **pp_nal, int *pi_nal,
                                   x264_picture_t *pic_out );
Loren Merritt's avatar
Loren Merritt committed
49

Laurent Aimar's avatar
Laurent Aimar committed
50 51 52 53 54
/****************************************************************************
 *
 ******************************* x264 libs **********************************
 *
 ****************************************************************************/
55 56 57 58 59 60 61 62 63
static float x264_psnr( int64_t i_sqe, int64_t i_size )
{
    double f_mse = (double)i_sqe / ((double)65025.0 * (double)i_size);
    if( f_mse <= 0.0000000001 ) /* Max 100dB */
        return 100;

    return (float)(-10.0 * log( f_mse ) / log( 10.0 ));
}

64
static void x264_frame_dump( x264_t *h )
Laurent Aimar's avatar
Laurent Aimar committed
65
{
66
    FILE *f = fopen( h->param.psz_dump_yuv, "r+b" );
Laurent Aimar's avatar
Laurent Aimar committed
67
    int i, y;
68 69 70
    if( !f )
        return;
    /* Write the frame in display order */
71
    fseek( f, (uint64_t)h->fdec->i_frame * h->param.i_height * h->param.i_width * 3/2, SEEK_SET );
72 73 74
    for( i = 0; i < h->fdec->i_plane; i++ )
        for( y = 0; y < h->param.i_height >> !!i; y++ )
            fwrite( &h->fdec->plane[i][y*h->fdec->i_stride[i]], 1, h->param.i_width >> !!i, f );
Laurent Aimar's avatar
Laurent Aimar committed
75 76 77 78 79
    fclose( f );
}


/* Fill "default" values */
80
static void x264_slice_header_init( x264_t *h, x264_slice_header_t *sh,
Laurent Aimar's avatar
Laurent Aimar committed
81
                                    x264_sps_t *sps, x264_pps_t *pps,
Loren Merritt's avatar
Loren Merritt committed
82
                                    int i_idr_pic_id, int i_frame, int i_qp )
Laurent Aimar's avatar
Laurent Aimar committed
83
{
84
    x264_param_t *param = &h->param;
85
    int i;
86

Laurent Aimar's avatar
Laurent Aimar committed
87 88 89 90 91
    /* First we fill all field */
    sh->sps = sps;
    sh->pps = pps;

    sh->i_first_mb  = 0;
Fiona Glaser's avatar
Fiona Glaser committed
92
    sh->i_last_mb   = h->mb.i_mb_count - 1;
Laurent Aimar's avatar
Laurent Aimar committed
93 94 95 96
    sh->i_pps_id    = pps->i_id;

    sh->i_frame_num = i_frame;

97 98 99
    sh->b_mbaff = h->param.b_interlaced;
    sh->b_field_pic = 0;    /* no field support for now */
    sh->b_bottom_field = 0; /* not yet used */
Laurent Aimar's avatar
Laurent Aimar committed
100 101 102 103 104 105 106 107 108 109 110

    sh->i_idr_pic_id = i_idr_pic_id;

    /* poc stuff, fixed later */
    sh->i_poc_lsb = 0;
    sh->i_delta_poc_bottom = 0;
    sh->i_delta_poc[0] = 0;
    sh->i_delta_poc[1] = 0;

    sh->i_redundant_pic_cnt = 0;

Loren Merritt's avatar
Loren Merritt committed
111 112 113 114 115 116 117 118
    if( !h->mb.b_direct_auto_read )
    {
        if( h->mb.b_direct_auto_write )
            sh->b_direct_spatial_mv_pred = ( h->stat.i_direct_score[1] > h->stat.i_direct_score[0] );
        else
            sh->b_direct_spatial_mv_pred = ( param->analyse.i_direct_mv_pred == X264_DIRECT_PRED_SPATIAL );
    }
    /* else b_direct_spatial_mv_pred was read from the 2pass statsfile */
Laurent Aimar's avatar
Laurent Aimar committed
119 120 121 122 123

    sh->b_num_ref_idx_override = 0;
    sh->i_num_ref_idx_l0_active = 1;
    sh->i_num_ref_idx_l1_active = 1;

124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
    sh->b_ref_pic_list_reordering_l0 = h->b_ref_reorder[0];
    sh->b_ref_pic_list_reordering_l1 = h->b_ref_reorder[1];

    /* If the ref list isn't in the default order, construct reordering header */
    /* List1 reordering isn't needed yet */
    if( sh->b_ref_pic_list_reordering_l0 )
    {
        int pred_frame_num = i_frame;
        for( i = 0; i < h->i_ref0; i++ )
        {
            int diff = h->fref0[i]->i_frame_num - pred_frame_num;
            if( diff == 0 )
                x264_log( h, X264_LOG_ERROR, "diff frame num == 0\n" );
            sh->ref_pic_list_order[0][i].idc = ( diff > 0 );
            sh->ref_pic_list_order[0][i].arg = abs( diff ) - 1;
            pred_frame_num = h->fref0[i]->i_frame_num;
        }
    }

Laurent Aimar's avatar
Laurent Aimar committed
143 144
    sh->i_cabac_init_idc = param->i_cabac_init_idc;

145
    sh->i_qp = i_qp;
146
    sh->i_qp_delta = i_qp - pps->i_pic_init_qp;
Laurent Aimar's avatar
Laurent Aimar committed
147 148 149
    sh->b_sp_for_swidth = 0;
    sh->i_qs_delta = 0;

150 151 152
    /* If effective qp <= 15, deblocking would have no effect anyway */
    if( param->b_deblocking_filter
        && ( h->mb.b_variable_qp
153
        || 15 < i_qp + 2 * X264_MIN(param->i_deblocking_filter_alphac0, param->i_deblocking_filter_beta) ) )
Laurent Aimar's avatar
Laurent Aimar committed
154 155 156 157 158 159 160 161 162 163 164 165 166
    {
        sh->i_disable_deblocking_filter_idc = 0;
    }
    else
    {
        sh->i_disable_deblocking_filter_idc = 1;
    }
    sh->i_alpha_c0_offset = param->i_deblocking_filter_alphac0 << 1;
    sh->i_beta_offset = param->i_deblocking_filter_beta << 1;
}

static void x264_slice_header_write( bs_t *s, x264_slice_header_t *sh, int i_nal_ref_idc )
{
167 168
    int i;

169 170 171 172 173 174 175 176
    if( sh->b_mbaff )
    {
        assert( sh->i_first_mb % (2*sh->sps->i_mb_width) == 0 );
        bs_write_ue( s, sh->i_first_mb >> 1 );
    }
    else
        bs_write_ue( s, sh->i_first_mb );

Laurent Aimar's avatar
Laurent Aimar committed
177 178
    bs_write_ue( s, sh->i_type + 5 );   /* same type things */
    bs_write_ue( s, sh->i_pps_id );
Fiona Glaser's avatar
Fiona Glaser committed
179
    bs_write( s, sh->sps->i_log2_max_frame_num, sh->i_frame_num & ((1<<sh->sps->i_log2_max_frame_num)-1) );
Laurent Aimar's avatar
Laurent Aimar committed
180

181 182 183
    if( !sh->sps->b_frame_mbs_only )
    {
        bs_write1( s, sh->b_field_pic );
Fiona Glaser's avatar
Fiona Glaser committed
184
        if( sh->b_field_pic )
185 186 187
            bs_write1( s, sh->b_bottom_field );
    }

Laurent Aimar's avatar
Laurent Aimar committed
188 189 190 191 192 193 194
    if( sh->i_idr_pic_id >= 0 ) /* NAL IDR */
    {
        bs_write_ue( s, sh->i_idr_pic_id );
    }

    if( sh->sps->i_poc_type == 0 )
    {
Fiona Glaser's avatar
Fiona Glaser committed
195
        bs_write( s, sh->sps->i_log2_max_poc_lsb, sh->i_poc_lsb & ((1<<sh->sps->i_log2_max_poc_lsb)-1) );
Laurent Aimar's avatar
Laurent Aimar committed
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
        if( sh->pps->b_pic_order && !sh->b_field_pic )
        {
            bs_write_se( s, sh->i_delta_poc_bottom );
        }
    }
    else if( sh->sps->i_poc_type == 1 && !sh->sps->b_delta_pic_order_always_zero )
    {
        bs_write_se( s, sh->i_delta_poc[0] );
        if( sh->pps->b_pic_order && !sh->b_field_pic )
        {
            bs_write_se( s, sh->i_delta_poc[1] );
        }
    }

    if( sh->pps->b_redundant_pic_cnt )
    {
        bs_write_ue( s, sh->i_redundant_pic_cnt );
    }

    if( sh->i_type == SLICE_TYPE_B )
    {
        bs_write1( s, sh->b_direct_spatial_mv_pred );
    }
    if( sh->i_type == SLICE_TYPE_P || sh->i_type == SLICE_TYPE_SP || sh->i_type == SLICE_TYPE_B )
    {
        bs_write1( s, sh->b_num_ref_idx_override );
        if( sh->b_num_ref_idx_override )
        {
            bs_write_ue( s, sh->i_num_ref_idx_l0_active - 1 );
            if( sh->i_type == SLICE_TYPE_B )
            {
                bs_write_ue( s, sh->i_num_ref_idx_l1_active - 1 );
            }
        }
    }

    /* ref pic list reordering */
    if( sh->i_type != SLICE_TYPE_I )
    {
235 236
        bs_write1( s, sh->b_ref_pic_list_reordering_l0 );
        if( sh->b_ref_pic_list_reordering_l0 )
Laurent Aimar's avatar
Laurent Aimar committed
237
        {
238 239 240 241
            for( i = 0; i < sh->i_num_ref_idx_l0_active; i++ )
            {
                bs_write_ue( s, sh->ref_pic_list_order[0][i].idc );
                bs_write_ue( s, sh->ref_pic_list_order[0][i].arg );
Loren Merritt's avatar
Loren Merritt committed
242

243 244
            }
            bs_write_ue( s, 3 );
Laurent Aimar's avatar
Laurent Aimar committed
245 246 247 248
        }
    }
    if( sh->i_type == SLICE_TYPE_B )
    {
249 250
        bs_write1( s, sh->b_ref_pic_list_reordering_l1 );
        if( sh->b_ref_pic_list_reordering_l1 )
Laurent Aimar's avatar
Laurent Aimar committed
251
        {
252 253 254 255 256 257
            for( i = 0; i < sh->i_num_ref_idx_l1_active; i++ )
            {
                bs_write_ue( s, sh->ref_pic_list_order[1][i].idc );
                bs_write_ue( s, sh->ref_pic_list_order[1][i].arg );
            }
            bs_write_ue( s, 3 );
Laurent Aimar's avatar
Laurent Aimar committed
258 259 260
        }
    }

Dylan Yudaken's avatar
Dylan Yudaken committed
261
    if( sh->pps->b_weighted_pred && ( sh->i_type == SLICE_TYPE_P || sh->i_type == SLICE_TYPE_SP ) )
Laurent Aimar's avatar
Laurent Aimar committed
262
    {
Dylan Yudaken's avatar
Dylan Yudaken committed
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
        /* pred_weight_table() */
        bs_write_ue( s, sh->weight[0][0].i_denom );
        bs_write_ue( s, sh->weight[0][1].i_denom );
        for( i = 0; i < sh->i_num_ref_idx_l0_active; i++ )
        {
            int luma_weight_l0_flag = !!sh->weight[i][0].weightfn;
            int chroma_weight_l0_flag = !!sh->weight[i][1].weightfn || !!sh->weight[i][2].weightfn;
            bs_write1( s, luma_weight_l0_flag );
            if( luma_weight_l0_flag )
            {
                bs_write_se( s, sh->weight[i][0].i_scale );
                bs_write_se( s, sh->weight[i][0].i_offset );
            }
            bs_write1( s, chroma_weight_l0_flag );
            if( chroma_weight_l0_flag )
            {
                int j;
                for( j = 1; j < 3; j++ )
                {
                    bs_write_se( s, sh->weight[i][j].i_scale );
                    bs_write_se( s, sh->weight[i][j].i_offset );
                }
            }
        }
    }
    else if( sh->pps->b_weighted_bipred == 1 && sh->i_type == SLICE_TYPE_B )
    {
      /* TODO */
Laurent Aimar's avatar
Laurent Aimar committed
291 292 293 294 295 296 297 298 299 300 301
    }

    if( i_nal_ref_idc != 0 )
    {
        if( sh->i_idr_pic_id >= 0 )
        {
            bs_write1( s, 0 );  /* no output of prior pics flag */
            bs_write1( s, 0 );  /* long term reference flag */
        }
        else
        {
Lamont Alston's avatar
Lamont Alston committed
302 303 304 305 306 307 308 309 310 311 312
            bs_write1( s, sh->i_mmco_command_count > 0 ); /* adaptive_ref_pic_marking_mode_flag */
            if( sh->i_mmco_command_count > 0 )
            {
                int i;
                for( i = 0; i < sh->i_mmco_command_count; i++ )
                {
                    bs_write_ue( s, 1 ); /* mark short term ref as unused */
                    bs_write_ue( s, sh->mmco[i].i_difference_of_pic_nums - 1 );
                }
                bs_write_ue( s, 0 ); /* end command list */
            }
Laurent Aimar's avatar
Laurent Aimar committed
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
        }
    }

    if( sh->pps->b_cabac && sh->i_type != SLICE_TYPE_I )
    {
        bs_write_ue( s, sh->i_cabac_init_idc );
    }
    bs_write_se( s, sh->i_qp_delta );      /* slice qp delta */

    if( sh->pps->b_deblocking_filter_control )
    {
        bs_write_ue( s, sh->i_disable_deblocking_filter_idc );
        if( sh->i_disable_deblocking_filter_idc != 1 )
        {
            bs_write_se( s, sh->i_alpha_c0_offset >> 1 );
            bs_write_se( s, sh->i_beta_offset >> 1 );
        }
    }
}

333 334
/* If we are within a reasonable distance of the end of the memory allocated for the bitstream, */
/* reallocate, adding an arbitrary amount of space (100 kilobytes). */
335
static int x264_bitstream_check_buffer( x264_t *h )
336
{
337
    uint8_t *bs_bak = h->out.p_bitstream;
338 339 340 341 342 343 344
    if( ( h->param.b_cabac && (h->cabac.p_end - h->cabac.p < 2500) )
     || ( h->out.bs.p_end - h->out.bs.p < 2500 ) )
    {
        intptr_t delta;
        int i;

        h->out.i_bitstream += 100000;
345 346
        CHECKED_MALLOC( h->out.p_bitstream, h->out.i_bitstream );
        h->mc.memcpy_aligned( h->out.p_bitstream, bs_bak, (h->out.i_bitstream - 100000) & ~15 );
347 348 349 350 351 352 353 354 355 356 357 358
        delta = h->out.p_bitstream - bs_bak;

        h->out.bs.p_start += delta;
        h->out.bs.p += delta;
        h->out.bs.p_end = h->out.p_bitstream + h->out.i_bitstream;

        h->cabac.p_start += delta;
        h->cabac.p += delta;
        h->cabac.p_end = h->out.p_bitstream + h->out.i_bitstream;

        for( i = 0; i <= h->out.i_nal; i++ )
            h->out.nal[i].p_payload += delta;
359
        x264_free( bs_bak );
360
    }
361 362 363 364
    return 0;
fail:
    x264_free( bs_bak );
    return -1;
365 366
}

Laurent Aimar's avatar
Laurent Aimar committed
367 368 369 370 371 372 373 374
/****************************************************************************
 *
 ****************************************************************************
 ****************************** External API*********************************
 ****************************************************************************
 *
 ****************************************************************************/

375
static int x264_validate_parameters( x264_t *h )
Laurent Aimar's avatar
Laurent Aimar committed
376
{
377
#ifdef HAVE_MMX
378
    if( !(x264_cpu_detect() & X264_CPU_SSE) )
379
    {
380
        x264_log( h, X264_LOG_ERROR, "your cpu does not support SSE1, but x264 was compiled with asm support\n");
381 382 383 384
        x264_log( h, X264_LOG_ERROR, "to run x264, recompile without asm support (configure --disable-asm)\n");
        return -1;
    }
#endif
385
    if( h->param.i_width <= 0 || h->param.i_height <= 0 )
Laurent Aimar's avatar
Laurent Aimar committed
386
    {
387
        x264_log( h, X264_LOG_ERROR, "invalid width x height (%dx%d)\n",
388 389
                  h->param.i_width, h->param.i_height );
        return -1;
Laurent Aimar's avatar
Laurent Aimar committed
390 391
    }

Loren Merritt's avatar
Loren Merritt committed
392
    if( h->param.i_width % 2 || h->param.i_height % 2 )
Laurent Aimar's avatar
Laurent Aimar committed
393
    {
Loren Merritt's avatar
Loren Merritt committed
394
        x264_log( h, X264_LOG_ERROR, "width or height not divisible by 2 (%dx%d)\n",
395 396
                  h->param.i_width, h->param.i_height );
        return -1;
Laurent Aimar's avatar
Laurent Aimar committed
397
    }
398 399
    int i_csp = h->param.i_csp & X264_CSP_MASK;
    if( i_csp != X264_CSP_I420 && i_csp != X264_CSP_YV12 )
Laurent Aimar's avatar
Laurent Aimar committed
400
    {
401
        x264_log( h, X264_LOG_ERROR, "invalid CSP (only I420/YV12 supported)\n" );
402
        return -1;
Laurent Aimar's avatar
Laurent Aimar committed
403 404
    }

Steven Walters's avatar
Steven Walters committed
405
    if( h->param.i_threads == X264_THREADS_AUTO )
Fiona Glaser's avatar
Fiona Glaser committed
406
        h->param.i_threads = x264_cpu_num_processors() * (h->param.b_sliced_threads?2:3)/2;
Loren Merritt's avatar
Loren Merritt committed
407
    h->param.i_threads = x264_clip3( h->param.i_threads, 1, X264_THREAD_MAX );
Loren Merritt's avatar
Loren Merritt committed
408 409
    if( h->param.i_threads > 1 )
    {
Loren Merritt's avatar
Loren Merritt committed
410
#ifndef HAVE_PTHREAD
Loren Merritt's avatar
Loren Merritt committed
411
        x264_log( h, X264_LOG_WARNING, "not compiled with pthread support!\n");
Loren Merritt's avatar
Loren Merritt committed
412
        h->param.i_threads = 1;
Loren Merritt's avatar
Loren Merritt committed
413
#endif
414 415 416 417 418 419 420
        /* Avoid absurdly small thread slices as they can reduce performance
         * and VBV compliance.  Capped at an arbitrary 4 rows per thread. */
        if( h->param.b_sliced_threads )
        {
            int max_threads = (h->param.i_height+15)/16 / 4;
            h->param.i_threads = X264_MIN( h->param.i_threads, max_threads );
        }
Loren Merritt's avatar
Loren Merritt committed
421
    }
422 423
    else
        h->param.b_sliced_threads = 0;
424
    h->i_thread_frames = h->param.b_sliced_threads ? 1 : h->param.i_threads;
Loren Merritt's avatar
Loren Merritt committed
425

426 427
    if( h->param.b_interlaced )
    {
428
        if( h->param.analyse.i_me_method >= X264_ME_ESA )
429 430 431 432 433 434 435 436 437
        {
            x264_log( h, X264_LOG_WARNING, "interlace + me=esa is not implemented\n" );
            h->param.analyse.i_me_method = X264_ME_UMH;
        }
        if( h->param.analyse.i_direct_mv_pred > X264_DIRECT_PRED_SPATIAL )
        {
            x264_log( h, X264_LOG_WARNING, "interlace + direct=temporal is not implemented\n" );
            h->param.analyse.i_direct_mv_pred = X264_DIRECT_PRED_SPATIAL;
        }
Dylan Yudaken's avatar
Dylan Yudaken committed
438 439 440 441 442
        if( h->param.analyse.i_weighted_pred > 0 )
        {
            x264_log( h, X264_LOG_WARNING, "interlace + weightp is not implemented\n" );
            h->param.analyse.i_weighted_pred = X264_WEIGHTP_NONE;
        }
443 444
    }

445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
    /* Detect default ffmpeg settings and terminate with an error. */
    {
        int score = 0;
        score += h->param.analyse.i_me_range == 0;
        score += h->param.rc.i_qp_step == 3;
        score += h->param.i_keyint_max == 12;
        score += h->param.rc.i_qp_min == 2;
        score += h->param.rc.i_qp_max == 31;
        score += h->param.rc.f_qcompress == 0.5;
        score += fabs(h->param.rc.f_ip_factor - 1.25) < 0.01;
        score += fabs(h->param.rc.f_pb_factor - 1.25) < 0.01;
        score += h->param.analyse.inter == 0 && h->param.analyse.i_subpel_refine == 8;
        if( score >= 5 )
        {
            x264_log( h, X264_LOG_ERROR, "broken ffmpeg default settings detected\n" );
            x264_log( h, X264_LOG_ERROR, "use an encoding preset (vpre)\n" );
            return -1;
        }
    }

465 466
    if( h->param.rc.i_rc_method < 0 || h->param.rc.i_rc_method > 2 )
    {
467
        x264_log( h, X264_LOG_ERROR, "no ratecontrol method specified\n" );
468 469
        return -1;
    }
Loren Merritt's avatar
Loren Merritt committed
470
    h->param.rc.f_rf_constant = x264_clip3f( h->param.rc.f_rf_constant, 0, 51 );
471
    h->param.rc.i_qp_constant = x264_clip3( h->param.rc.i_qp_constant, 0, 51 );
472
    if( h->param.rc.i_rc_method == X264_RC_CRF )
Fiona Glaser's avatar
Fiona Glaser committed
473
    {
Loren Merritt's avatar
Loren Merritt committed
474
        h->param.rc.i_qp_constant = h->param.rc.f_rf_constant;
Fiona Glaser's avatar
Fiona Glaser committed
475 476
        h->param.rc.i_bitrate = 0;
    }
477 478
    if( (h->param.rc.i_rc_method == X264_RC_CQP || h->param.rc.i_rc_method == X264_RC_CRF)
        && h->param.rc.i_qp_constant == 0 )
Loren Merritt's avatar
Loren Merritt committed
479 480
    {
        h->mb.b_lossless = 1;
Loren Merritt's avatar
Loren Merritt committed
481
        h->param.i_cqm_preset = X264_CQM_FLAT;
Loren Merritt's avatar
Loren Merritt committed
482
        h->param.psz_cqm_file = NULL;
483
        h->param.rc.i_rc_method = X264_RC_CQP;
Loren Merritt's avatar
Loren Merritt committed
484 485 486
        h->param.rc.f_ip_factor = 1;
        h->param.rc.f_pb_factor = 1;
        h->param.analyse.b_psnr = 0;
487
        h->param.analyse.b_ssim = 0;
488
        h->param.analyse.i_chroma_qp_offset = 0;
489
        h->param.analyse.i_trellis = 0;
Loren Merritt's avatar
Loren Merritt committed
490
        h->param.analyse.b_fast_pskip = 0;
491
        h->param.analyse.i_noise_reduction = 0;
492
        h->param.analyse.f_psy_rd = 0;
493
        h->param.i_bframe = 0;
494 495 496
        /* 8x8dct is not useful at all in CAVLC lossless */
        if( !h->param.b_cabac )
            h->param.analyse.b_transform_8x8 = 0;
Loren Merritt's avatar
Loren Merritt committed
497
    }
Loren Merritt's avatar
Loren Merritt committed
498 499 500 501 502 503 504
    if( h->param.rc.i_rc_method == X264_RC_CQP )
    {
        float qp_p = h->param.rc.i_qp_constant;
        float qp_i = qp_p - 6*log(h->param.rc.f_ip_factor)/log(2);
        float qp_b = qp_p + 6*log(h->param.rc.f_pb_factor)/log(2);
        h->param.rc.i_qp_min = x264_clip3( (int)(X264_MIN3( qp_p, qp_i, qp_b )), 0, 51 );
        h->param.rc.i_qp_max = x264_clip3( (int)(X264_MAX3( qp_p, qp_i, qp_b ) + .999), 0, 51 );
Loren Merritt's avatar
Loren Merritt committed
505
        h->param.rc.i_aq_mode = 0;
Fiona Glaser's avatar
Fiona Glaser committed
506
        h->param.rc.b_mb_tree = 0;
Loren Merritt's avatar
Loren Merritt committed
507
    }
508 509
    h->param.rc.i_qp_max = x264_clip3( h->param.rc.i_qp_max, 0, 51 );
    h->param.rc.i_qp_min = x264_clip3( h->param.rc.i_qp_min, 0, h->param.rc.i_qp_max );
Loren Merritt's avatar
Loren Merritt committed
510

Fiona Glaser's avatar
Fiona Glaser committed
511
    int max_slices = (h->param.i_height+((16<<h->param.b_interlaced)-1))/(16<<h->param.b_interlaced);
512 513 514
    if( h->param.b_sliced_threads )
        h->param.i_slice_count = x264_clip3( h->param.i_threads, 0, max_slices );
    else
Fiona Glaser's avatar
Fiona Glaser committed
515
    {
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
        h->param.i_slice_count = x264_clip3( h->param.i_slice_count, 0, max_slices );
        h->param.i_slice_max_size = X264_MAX( h->param.i_slice_max_size, 0 );
        h->param.i_slice_max_mbs = X264_MAX( h->param.i_slice_max_mbs, 0 );
        if( h->param.b_interlaced && h->param.i_slice_max_size )
        {
            x264_log( h, X264_LOG_WARNING, "interlaced + slice-max-size is not implemented\n" );
            h->param.i_slice_max_size = 0;
        }
        if( h->param.b_interlaced && h->param.i_slice_max_mbs )
        {
            x264_log( h, X264_LOG_WARNING, "interlaced + slice-max-mbs is not implemented\n" );
            h->param.i_slice_max_mbs = 0;
        }
        if( h->param.i_slice_max_mbs || h->param.i_slice_max_size )
            h->param.i_slice_count = 0;
Fiona Glaser's avatar
Fiona Glaser committed
531 532
    }

Loren Merritt's avatar
Loren Merritt committed
533
    h->param.i_frame_reference = x264_clip3( h->param.i_frame_reference, 1, 16 );
534 535
    if( h->param.i_keyint_max <= 0 )
        h->param.i_keyint_max = 1;
Fiona Glaser's avatar
Fiona Glaser committed
536 537
    if( h->param.i_scenecut_threshold < 0 )
        h->param.i_scenecut_threshold = 0;
538 539 540 541 542
    if( !h->param.analyse.i_subpel_refine && h->param.analyse.i_direct_mv_pred > X264_DIRECT_PRED_SPATIAL )
    {
        x264_log( h, X264_LOG_WARNING, "subme=0 + direct=temporal is not supported\n" );
        h->param.analyse.i_direct_mv_pred = X264_DIRECT_PRED_SPATIAL;
    }
Loren Merritt's avatar
Loren Merritt committed
543
    h->param.i_bframe = x264_clip3( h->param.i_bframe, 0, X264_BFRAME_MAX );
544
    if( h->param.i_keyint_max == 1 )
Fiona Glaser's avatar
Fiona Glaser committed
545
    {
546
        h->param.i_bframe = 0;
Fiona Glaser's avatar
Fiona Glaser committed
547 548
        h->param.b_intra_refresh = 0;
    }
549
    h->param.i_bframe_bias = x264_clip3( h->param.i_bframe_bias, -90, 100 );
Lamont Alston's avatar
Lamont Alston committed
550 551 552
    if( h->param.i_bframe <= 1 )
        h->param.i_bframe_pyramid = X264_B_PYRAMID_NONE;
    h->param.i_bframe_pyramid = x264_clip3( h->param.i_bframe_pyramid, X264_B_PYRAMID_NONE, X264_B_PYRAMID_NORMAL );
553
    if( !h->param.i_bframe )
554
    {
555
        h->param.i_bframe_adaptive = X264_B_ADAPT_NONE;
556 557 558
        h->param.analyse.i_direct_mv_pred = 0;
        h->param.analyse.b_weighted_bipred = 0;
    }
Fiona Glaser's avatar
Fiona Glaser committed
559 560 561 562 563 564 565 566 567 568 569 570 571
    if( h->param.b_intra_refresh && h->param.i_bframe_pyramid == X264_B_PYRAMID_NORMAL )
    {
        x264_log( h, X264_LOG_WARNING, "b-pyramid normal + intra-refresh is not supported\n" );
        h->param.i_bframe_pyramid = X264_B_PYRAMID_STRICT;
    }
    if( h->param.b_intra_refresh && h->param.i_frame_reference > 1 )
    {
        x264_log( h, X264_LOG_WARNING, "ref > 1 + intra-refresh is not supported\n" );
        h->param.i_frame_reference = 1;
    }
    if( h->param.b_intra_refresh )
        h->param.i_keyint_max = X264_MIN( h->param.i_keyint_max, (h->param.i_width+15)/16 - 1 );
    h->param.i_keyint_min = x264_clip3( h->param.i_keyint_min, 1, h->param.i_keyint_max/2+1 );
Fiona Glaser's avatar
Fiona Glaser committed
572
    h->param.rc.i_lookahead = x264_clip3( h->param.rc.i_lookahead, 0, X264_LOOKAHEAD_MAX );
Fiona Glaser's avatar
Fiona Glaser committed
573 574 575 576 577 578 579
    {
        int maxrate = X264_MAX( h->param.rc.i_vbv_max_bitrate, h->param.rc.i_bitrate );
        float bufsize = maxrate ? (float)h->param.rc.i_vbv_buffer_size / maxrate : 0;
        float fps = h->param.i_fps_num > 0 && h->param.i_fps_den > 0 ? (float) h->param.i_fps_num / h->param.i_fps_den : 25.0;
        h->param.rc.i_lookahead = X264_MIN( h->param.rc.i_lookahead, X264_MAX( h->param.i_keyint_max, bufsize*fps ) );
    }

580 581 582 583 584 585
    if( !h->param.i_timebase_num || !h->param.i_timebase_den )
    {
        h->param.i_timebase_num = h->param.i_fps_den;
        h->param.i_timebase_den = h->param.i_fps_num;
    }

586 587 588
    h->param.rc.f_qcompress = x264_clip3f( h->param.rc.f_qcompress, 0.0, 1.0 );
    if( !h->param.rc.i_lookahead || h->param.i_keyint_max == 1 || h->param.rc.f_qcompress == 1 )
        h->param.rc.b_mb_tree = 0;
Fiona Glaser's avatar
Fiona Glaser committed
589 590
    if( h->param.rc.b_stat_read )
        h->param.rc.i_lookahead = 0;
Steven Walters's avatar
Steven Walters committed
591 592
#ifdef HAVE_PTHREAD
    if( h->param.i_sync_lookahead )
593 594
        h->param.i_sync_lookahead = x264_clip3( h->param.i_sync_lookahead, h->i_thread_frames + h->param.i_bframe, X264_LOOKAHEAD_MAX );
    if( h->param.rc.b_stat_read || h->i_thread_frames == 1 )
Steven Walters's avatar
Steven Walters committed
595 596 597 598
        h->param.i_sync_lookahead = 0;
#else
    h->param.i_sync_lookahead = 0;
#endif
Fiona Glaser's avatar
Fiona Glaser committed
599

Loren Merritt's avatar
Loren Merritt committed
600 601 602
    h->mb.b_direct_auto_write = h->param.analyse.i_direct_mv_pred == X264_DIRECT_PRED_AUTO
                                && h->param.i_bframe
                                && ( h->param.rc.b_stat_write || !h->param.rc.b_stat_read );
Laurent Aimar's avatar
Laurent Aimar committed
603 604 605

    h->param.i_deblocking_filter_alphac0 = x264_clip3( h->param.i_deblocking_filter_alphac0, -6, 6 );
    h->param.i_deblocking_filter_beta    = x264_clip3( h->param.i_deblocking_filter_beta, -6, 6 );
606 607
    h->param.analyse.i_luma_deadzone[0] = x264_clip3( h->param.analyse.i_luma_deadzone[0], 0, 32 );
    h->param.analyse.i_luma_deadzone[1] = x264_clip3( h->param.analyse.i_luma_deadzone[1], 0, 32 );
Laurent Aimar's avatar
Laurent Aimar committed
608

609
    h->param.i_cabac_init_idc = x264_clip3( h->param.i_cabac_init_idc, 0, 2 );
Laurent Aimar's avatar
Laurent Aimar committed
610

Loren Merritt's avatar
Loren Merritt committed
611 612 613
    if( h->param.i_cqm_preset < X264_CQM_FLAT || h->param.i_cqm_preset > X264_CQM_CUSTOM )
        h->param.i_cqm_preset = X264_CQM_FLAT;

614
    if( h->param.analyse.i_me_method < X264_ME_DIA ||
615
        h->param.analyse.i_me_method > X264_ME_TESA )
616
        h->param.analyse.i_me_method = X264_ME_HEX;
617 618 619
    if( h->param.analyse.i_me_range < 4 )
        h->param.analyse.i_me_range = 4;
    if( h->param.analyse.i_me_range > 16 && h->param.analyse.i_me_method <= X264_ME_HEX )
620
        h->param.analyse.i_me_range = 16;
621 622 623
    if( h->param.analyse.i_me_method == X264_ME_TESA &&
        (h->mb.b_lossless || h->param.analyse.i_subpel_refine <= 1) )
        h->param.analyse.i_me_method = X264_ME_ESA;
Fiona Glaser's avatar
Fiona Glaser committed
624
    h->param.analyse.i_subpel_refine = x264_clip3( h->param.analyse.i_subpel_refine, 0, 10 );
625 626 627 628
    h->param.analyse.b_mixed_references = h->param.analyse.b_mixed_references && h->param.i_frame_reference > 1;
    h->param.analyse.inter &= X264_ANALYSE_PSUB16x16|X264_ANALYSE_PSUB8x8|X264_ANALYSE_BSUB16x16|
                              X264_ANALYSE_I4x4|X264_ANALYSE_I8x8;
    h->param.analyse.intra &= X264_ANALYSE_I4x4|X264_ANALYSE_I8x8;
629 630
    if( !(h->param.analyse.inter & X264_ANALYSE_PSUB16x16) )
        h->param.analyse.inter &= ~X264_ANALYSE_PSUB8x8;
631 632 633 634 635
    if( !h->param.analyse.b_transform_8x8 )
    {
        h->param.analyse.inter &= ~X264_ANALYSE_I8x8;
        h->param.analyse.intra &= ~X264_ANALYSE_I8x8;
    }
636
    h->param.analyse.i_chroma_qp_offset = x264_clip3(h->param.analyse.i_chroma_qp_offset, -12, 12);
Loren Merritt's avatar
Loren Merritt committed
637 638
    if( !h->param.b_cabac )
        h->param.analyse.i_trellis = 0;
639
    h->param.analyse.i_trellis = x264_clip3( h->param.analyse.i_trellis, 0, 2 );
Fiona Glaser's avatar
Fiona Glaser committed
640 641 642 643 644
    if( !h->param.analyse.b_psy )
    {
        h->param.analyse.f_psy_rd = 0;
        h->param.analyse.f_psy_trellis = 0;
    }
645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
    if( !h->param.analyse.i_trellis )
        h->param.analyse.f_psy_trellis = 0;
    h->param.analyse.f_psy_rd = x264_clip3f( h->param.analyse.f_psy_rd, 0, 10 );
    h->param.analyse.f_psy_trellis = x264_clip3f( h->param.analyse.f_psy_trellis, 0, 10 );
    if( h->param.analyse.i_subpel_refine < 6 )
        h->param.analyse.f_psy_rd = 0;
    h->mb.i_psy_rd = FIX8( h->param.analyse.f_psy_rd );
    /* Psy RDO increases overall quantizers to improve the quality of luma--this indirectly hurts chroma quality */
    /* so we lower the chroma QP offset to compensate */
    /* This can be triggered repeatedly on multiple calls to parameter_validate, but since encoding
     * uses the pps chroma qp offset not the param chroma qp offset, this is not a problem. */
    if( h->mb.i_psy_rd )
        h->param.analyse.i_chroma_qp_offset -= h->param.analyse.f_psy_rd < 0.25 ? 1 : 2;
    h->mb.i_psy_trellis = FIX8( h->param.analyse.f_psy_trellis / 4 );
    /* Psy trellis has a similar effect. */
    if( h->mb.i_psy_trellis )
        h->param.analyse.i_chroma_qp_offset -= h->param.analyse.f_psy_trellis < 0.25 ? 1 : 2;
    else
        h->mb.i_psy_trellis = 0;
    h->param.analyse.i_chroma_qp_offset = x264_clip3(h->param.analyse.i_chroma_qp_offset, -12, 12);
Anton Mitrofanov's avatar
Anton Mitrofanov committed
665
    h->param.rc.i_aq_mode = x264_clip3( h->param.rc.i_aq_mode, 0, 2 );
666 667
    h->param.rc.f_aq_strength = x264_clip3f( h->param.rc.f_aq_strength, 0, 3 );
    if( h->param.rc.f_aq_strength == 0 )
668
        h->param.rc.i_aq_mode = 0;
Fiona Glaser's avatar
Fiona Glaser committed
669 670 671 672 673 674
    /* MB-tree requires AQ to be on, even if the strength is zero. */
    if( !h->param.rc.i_aq_mode && h->param.rc.b_mb_tree )
    {
        h->param.rc.i_aq_mode = 1;
        h->param.rc.f_aq_strength = 0;
    }
675
    h->param.analyse.i_noise_reduction = x264_clip3( h->param.analyse.i_noise_reduction, 0, 1<<16 );
Fiona Glaser's avatar
Fiona Glaser committed
676 677
    if( h->param.analyse.i_subpel_refine == 10 && (h->param.analyse.i_trellis != 2 || !h->param.rc.i_aq_mode) )
        h->param.analyse.i_subpel_refine = 9;
678

679 680
    {
        const x264_level_t *l = x264_levels;
681
        if( h->param.i_level_idc < 0 )
682
        {
Loren Merritt's avatar
Loren Merritt committed
683
            int maxrate_bak = h->param.rc.i_vbv_max_bitrate;
684 685 686 687 688 689
            if( h->param.rc.i_rc_method == X264_RC_ABR && h->param.rc.i_vbv_buffer_size <= 0 )
                h->param.rc.i_vbv_max_bitrate = h->param.rc.i_bitrate * 2;
            h->sps = h->sps_array;
            x264_sps_init( h->sps, h->param.i_sps_id, &h->param );
            do h->param.i_level_idc = l->level_idc;
                while( l[1].level_idc && x264_validate_levels( h, 0 ) && l++ );
Loren Merritt's avatar
Loren Merritt committed
690
            h->param.rc.i_vbv_max_bitrate = maxrate_bak;
691 692 693 694 695 696 697 698 699 700
        }
        else
        {
            while( l->level_idc && l->level_idc != h->param.i_level_idc )
                l++;
            if( l->level_idc == 0 )
            {
                x264_log( h, X264_LOG_ERROR, "invalid level_idc: %d\n", h->param.i_level_idc );
                return -1;
            }
701 702
        }
        if( h->param.analyse.i_mv_range <= 0 )
703
            h->param.analyse.i_mv_range = l->mv_range >> h->param.b_interlaced;
704
        else
705
            h->param.analyse.i_mv_range = x264_clip3(h->param.analyse.i_mv_range, 32, 512 >> h->param.b_interlaced);
706 707
    }

Dylan Yudaken's avatar
Dylan Yudaken committed
708 709 710 711
    h->param.analyse.i_weighted_pred = x264_clip3( h->param.analyse.i_weighted_pred, 0, X264_WEIGHTP_SMART );
    if( !h->param.analyse.i_weighted_pred && h->param.rc.b_mb_tree && h->param.analyse.b_psy && !h->param.b_interlaced )
        h->param.analyse.i_weighted_pred = X264_WEIGHTP_FAKE;

712
    if( h->i_thread_frames > 1 )
Loren Merritt's avatar
Loren Merritt committed
713 714 715 716 717 718 719 720 721
    {
        int r = h->param.analyse.i_mv_range_thread;
        int r2;
        if( r <= 0 )
        {
            // half of the available space is reserved and divided evenly among the threads,
            // the rest is allocated to whichever thread is far enough ahead to use it.
            // reserving more space increases quality for some videos, but costs more time
            // in thread synchronization.
722
            int max_range = (h->param.i_height + X264_THREAD_HEIGHT) / h->i_thread_frames - X264_THREAD_HEIGHT;
Loren Merritt's avatar
Loren Merritt committed
723 724 725 726 727 728 729 730 731 732 733 734
            r = max_range / 2;
        }
        r = X264_MAX( r, h->param.analyse.i_me_range );
        r = X264_MIN( r, h->param.analyse.i_mv_range );
        // round up to use the whole mb row
        r2 = (r & ~15) + ((-X264_THREAD_HEIGHT) & 15);
        if( r2 < r )
            r2 += 16;
        x264_log( h, X264_LOG_DEBUG, "using mv_range_thread = %d\n", r2 );
        h->param.analyse.i_mv_range_thread = r2;
    }

Loren Merritt's avatar
Loren Merritt committed
735 736 737 738 739
    if( h->param.rc.f_qblur < 0 )
        h->param.rc.f_qblur = 0;
    if( h->param.rc.f_complexity_blur < 0 )
        h->param.rc.f_complexity_blur = 0;

740 741
    h->param.i_sps_id &= 31;

742 743 744 745 746 747
    if( h->param.i_log_level < X264_LOG_INFO )
    {
        h->param.analyse.b_psnr = 0;
        h->param.analyse.b_ssim = 0;
    }

748 749 750
    /* ensure the booleans are 0 or 1 so they can be used in math */
#define BOOLIFY(x) h->param.x = !!h->param.x
    BOOLIFY( b_cabac );
751
    BOOLIFY( b_constrained_intra );
752
    BOOLIFY( b_deblocking_filter );
753
    BOOLIFY( b_deterministic );
754
    BOOLIFY( b_sliced_threads );
755
    BOOLIFY( b_interlaced );
Fiona Glaser's avatar
Fiona Glaser committed
756
    BOOLIFY( b_intra_refresh );
757 758 759 760
    BOOLIFY( b_visualize );
    BOOLIFY( b_aud );
    BOOLIFY( b_repeat_headers );
    BOOLIFY( b_annexb );
761
    BOOLIFY( analyse.b_transform_8x8 );
762
    BOOLIFY( analyse.b_weighted_bipred );
763
    BOOLIFY( analyse.b_chroma_me );
764
    BOOLIFY( analyse.b_mixed_references );
765
    BOOLIFY( analyse.b_fast_pskip );
766 767 768 769
    BOOLIFY( analyse.b_dct_decimate );
    BOOLIFY( analyse.b_psy );
    BOOLIFY( analyse.b_psnr );
    BOOLIFY( analyse.b_ssim );
770 771
    BOOLIFY( rc.b_stat_write );
    BOOLIFY( rc.b_stat_read );
772
    BOOLIFY( rc.b_mb_tree );
773 774
#undef BOOLIFY

775 776 777
    return 0;
}

778 779
static void mbcmp_init( x264_t *h )
{
780
    int satd = !h->mb.b_lossless && h->param.analyse.i_subpel_refine > 1;
781 782
    memcpy( h->pixf.mbcmp, satd ? h->pixf.satd : h->pixf.sad_aligned, sizeof(h->pixf.mbcmp) );
    memcpy( h->pixf.mbcmp_unaligned, satd ? h->pixf.satd : h->pixf.sad, sizeof(h->pixf.mbcmp_unaligned) );
783
    h->pixf.intra_mbcmp_x3_16x16 = satd ? h->pixf.intra_satd_x3_16x16 : h->pixf.intra_sad_x3_16x16;
Fiona Glaser's avatar
Fiona Glaser committed
784
    h->pixf.intra_mbcmp_x3_8x8c = satd ? h->pixf.intra_satd_x3_8x8c : h->pixf.intra_sad_x3_8x8c;
Fiona Glaser's avatar
Fiona Glaser committed
785
    h->pixf.intra_mbcmp_x3_4x4 = satd ? h->pixf.intra_satd_x3_4x4 : h->pixf.intra_sad_x3_4x4;
786 787 788 789
    satd &= h->param.analyse.i_me_method == X264_ME_TESA;
    memcpy( h->pixf.fpelcmp, satd ? h->pixf.satd : h->pixf.sad, sizeof(h->pixf.fpelcmp) );
    memcpy( h->pixf.fpelcmp_x3, satd ? h->pixf.satd_x3 : h->pixf.sad_x3, sizeof(h->pixf.fpelcmp_x3) );
    memcpy( h->pixf.fpelcmp_x4, satd ? h->pixf.satd_x4 : h->pixf.sad_x4, sizeof(h->pixf.fpelcmp_x4) );
790 791
}

792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809
static void x264_set_aspect_ratio( x264_t *h, x264_param_t *param, int initial )
{
    /* VUI */
    if( param->vui.i_sar_width > 0 && param->vui.i_sar_height > 0 )
    {
        int i_w = param->vui.i_sar_width;
        int i_h = param->vui.i_sar_height;
        int old_w = h->param.vui.i_sar_width;
        int old_h = h->param.vui.i_sar_height;

        x264_reduce_fraction( &i_w, &i_h );

        while( i_w > 65535 || i_h > 65535 )
        {
            i_w /= 2;
            i_h /= 2;
        }

810 811
        x264_reduce_fraction( &i_w, &i_h );

812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
        if( i_w != old_w || i_h != old_h || initial )
        {
            h->param.vui.i_sar_width = 0;
            h->param.vui.i_sar_height = 0;
            if( i_w == 0 || i_h == 0 )
                x264_log( h, X264_LOG_WARNING, "cannot create valid sample aspect ratio\n" );
            else
            {
                x264_log( h, initial?X264_LOG_INFO:X264_LOG_DEBUG, "using SAR=%d/%d\n", i_w, i_h );
                h->param.vui.i_sar_width = i_w;
                h->param.vui.i_sar_height = i_h;
            }
        }
    }
}

828 829 830
/****************************************************************************
 * x264_encoder_open:
 ****************************************************************************/
831
x264_t *x264_encoder_open( x264_param_t *param )
832
{
833
    x264_t *h;
834
    char buf[1000], *p;
835
    int i, qp, i_slicetype_length;
836

837
    CHECKED_MALLOCZERO( h, sizeof(x264_t) );
838 839

    /* Create a copy of param */
840
    memcpy( &h->param, param, sizeof(x264_param_t) );
841

842 843 844
    if( param->param_free )
        param->param_free( param );

845
    if( x264_validate_parameters( h ) < 0 )
846
        goto fail;
847

848 849
    if( h->param.psz_cqm_file )
        if( x264_cqm_parse_file( h, h->param.psz_cqm_file ) < 0 )
850
            goto fail;
851

852 853 854 855 856
    if( h->param.rc.psz_stat_out )
        h->param.rc.psz_stat_out = strdup( h->param.rc.psz_stat_out );
    if( h->param.rc.psz_stat_in )
        h->param.rc.psz_stat_in = strdup( h->param.rc.psz_stat_in );

857
    x264_set_aspect_ratio( h, &h->param, 1 );
858

Loren Merritt's avatar
Loren Merritt committed
859
    x264_reduce_fraction( &h->param.i_fps_num, &h->param.i_fps_den );
860
    x264_reduce_fraction( &h->param.i_timebase_num, &h->param.i_timebase_den );
861

Laurent Aimar's avatar
Laurent Aimar committed
862
    /* Init x264_t */
863
    h->i_frame = -1;
Laurent Aimar's avatar
Laurent Aimar committed
864 865
    h->i_frame_num = 0;
    h->i_idr_pic_id = 0;
866 867 868 869 870 871 872 873 874 875 876 877
    if( h->param.b_dts_compress )
    {
        /* h->i_dts_compress_multiplier == h->frames.i_bframe_delay + 1 */
        h->i_dts_compress_multiplier = h->param.i_bframe ? (h->param.i_bframe_pyramid ? 3 : 2) : 1;
        if( h->i_dts_compress_multiplier != 1 )
            x264_log( h, X264_LOG_DEBUG, "DTS compresion changed timebase: %d/%d -> %d/%d\n",
                      h->param.i_timebase_num, h->param.i_timebase_den,
                      h->param.i_timebase_num, h->param.i_timebase_den * h->i_dts_compress_multiplier );
        h->param.i_timebase_den *= h->i_dts_compress_multiplier;
    }
    else
        h->i_dts_compress_multiplier = 1;
Laurent Aimar's avatar
Laurent Aimar committed
878 879

    h->sps = &h->sps_array[0];
880
    x264_sps_init( h->sps, h->param.i_sps_id, &h->param );
Laurent Aimar's avatar
Laurent Aimar committed
881 882

    h->pps = &h->pps_array[0];
883
    x264_pps_init( h->pps, h->param.i_sps_id, &h->param, h->sps );
Loren Merritt's avatar
Loren Merritt committed
884

885
    x264_validate_levels( h, 1 );
886

887 888
    h->chroma_qp_table = i_chroma_qp_table + 12 + h->pps->i_chroma_qp_index_offset;

Loren Merritt's avatar
Loren Merritt committed
889
    if( x264_cqm_init( h ) < 0 )
890
        goto fail;
Loren Merritt's avatar
Loren Merritt committed
891

892
    h->mb.i_mb_count = h->sps->i_mb_width * h->sps->i_mb_height;
Laurent Aimar's avatar
Laurent Aimar committed
893 894

    /* Init frames. */
895
    if( h->param.i_bframe_adaptive == X264_B_ADAPT_TRELLIS )
896
        h->frames.i_delay = X264_MAX(h->param.i_bframe,3)*4;
897
    else
898
        h->frames.i_delay = h->param.i_bframe;
Fiona Glaser's avatar
Fiona Glaser committed
899
    if( h->param.rc.b_mb_tree || h->param.rc.i_vbv_buffer_size )
Fiona Glaser's avatar
Fiona Glaser committed
900
        h->frames.i_delay = X264_MAX( h->frames.i_delay, h->param.rc.i_lookahead );
Steven Walters's avatar
Steven Walters committed
901
    i_slicetype_length = h->frames.i_delay;
902
    h->frames.i_delay += h->i_thread_frames - 1;
903
    h->frames.i_delay = X264_MIN( h->frames.i_delay, X264_LOOKAHEAD_MAX );
Steven Walters's avatar
Steven Walters committed
904
    h->frames.i_delay += h->param.i_sync_lookahead;
905
    h->frames.i_bframe_delay = h->param.i_bframe ? (h->param.i_bframe_pyramid ? 2 : 1) : 0;
Fiona Glaser's avatar
Fiona Glaser committed
906

907 908
    h->frames.i_max_ref0 = h->param.i_frame_reference;
    h->frames.i_max_ref1 = h->sps->vui.i_num_reorder_frames;
909
    h->frames.i_max_dpb  = h->sps->vui.i_max_dec_frame_buffering;
910
    h->frames.b_have_lowres = !h->param.rc.b_stat_read
911 912
        && ( h->param.rc.i_rc_method == X264_RC_ABR
          || h->param.rc.i_rc_method == X264_RC_CRF
913
          || h->param.i_bframe_adaptive
Fiona Glaser's avatar
Fiona Glaser committed
914
          || h->param.i_scenecut_threshold
Dylan Yudaken's avatar
Dylan Yudaken committed
915 916
          || h->param.rc.b_mb_tree
          || h->param.analyse.i_weighted_pred == X264_WEIGHTP_SMART );
917
    h->frames.b_have_lowres |= h->param.rc.b_stat_read && h->param.rc.i_vbv_buffer_size > 0;
Loren Merritt's avatar
Loren Merritt committed
918
    h->frames.b_have_sub8x8_esa = !!(h->param.analyse.inter & X264_ANALYSE_PSUB8x8);
919

Fiona Glaser's avatar
Fiona Glaser committed
920
    h->frames.i_last_keyframe = - h->param.i_keyint_max;
921
    h->frames.i_input    = 0;
Steven Walters's avatar
Steven Walters committed
922 923 924

    CHECKED_MALLOCZERO( h->frames.unused[0], (h->frames.i_delay + 3) * sizeof(x264_frame_t *) );
    /* Allocate room for max refs plus a few extra just in case. */
925
    CHECKED_MALLOCZERO( h->frames.unused[1], (h->i_thread_frames + 20) * sizeof(x264_frame_t *) );
Steven Walters's avatar
Steven Walters committed
926
    CHECKED_MALLOCZERO( h->frames.current, (h->param.i_sync_lookahead + h->param.i_bframe
927
                        + h->i_thread_frames + 3) * sizeof(x264_frame_t *) );
Dylan Yudaken's avatar
Dylan Yudaken committed
928
    if( h->param.analyse.i_weighted_pred > 0 )
929
        CHECKED_MALLOCZERO( h->frames.blank_unused, h->i_thread_frames * 4 * sizeof(x264_frame_t *) );
Laurent Aimar's avatar
Laurent Aimar committed
930 931 932
    h->i_ref0 = 0;
    h->i_ref1 = 0;

933
    x264_rdo_init();
Laurent Aimar's avatar
Laurent Aimar committed
934 935 936

    /* init CPU functions */
    x264_predict_16x16_init( h->param.cpu, h->predict_16x16 );
937
    x264_predict_8x8c_init( h->param.cpu, h->predict_8x8c );
938
    x264_predict_8x8_init( h->param.cpu, h->predict_8x8, &h->predict_8x8_filter );
Laurent Aimar's avatar
Laurent Aimar committed
939
    x264_predict_4x4_init( h->param.cpu, h->predict_4x4 );
Fiona Glaser's avatar
Fiona Glaser committed
940
    if( !h->param.b_cabac )
941
        x264_init_vlc_tables();
Laurent Aimar's avatar
Laurent Aimar committed
942 943
    x264_pixel_init( h->param.cpu, &h->pixf );
    x264_dct_init( h->param.cpu, &h->dctf );
944
    x264_zigzag_init( h->param.cpu, &h->zigzagf, h->param.b_interlaced );
945
    x264_mc_init( h->param.cpu, &h->mc );
946
    x264_quant_init( h, h->param.cpu, &h->quantf );
Loren Merritt's avatar
Loren Merritt committed
947
    x264_deblock_init( h->param.cpu, &h->loopf );
948
    x264_dct_init_weights();
Laurent Aimar's avatar
Laurent Aimar committed
949

950
    mbcmp_init( h );
Loren Merritt's avatar
Loren Merritt committed
951

952 953
    p = buf + sprintf( buf, "using cpu capabilities:" );
    for( i=0; x264_cpu_names[i].flags; i++ )
954 955
    {
        if( !strcmp(x264_cpu_names[i].name, "SSE2")
956
            && h->param.cpu & (X264_CPU_SSE2_IS_FAST|X264_CPU_SSE2_IS_SLOW) )
957 958
            continue;
        if( !strcmp(x264_cpu_names[i].name, "SSE3")
959
            && (h->param.cpu & X264_CPU_SSSE3 || !(h->param.cpu & X264_CPU_CACHELINE_64)) )
960
            continue;
961
        if( !strcmp(x264_cpu_names[i].name, "SSE4.1")
962
            && (h->param.cpu & X264_CPU_SSE42) )
963
            continue;
964
        if( (h->param.cpu & x264_cpu_names[i].flags) == x264_cpu_names[i].flags
965 966
            && (!i || x264_cpu_names[i].flags != x264_cpu_names[i-1].flags) )
            p += sprintf( p, " %s", x264_cpu_names[i].name );
967
    }
968
    if( !h->param.cpu )
969 970
        p += sprintf( p, " none!" );
    x264_log( h, X264_LOG_INFO, "%s\n", buf );
Laurent Aimar's avatar