encoder.c 77.4 KB
Newer Older
Laurent Aimar's avatar
Laurent Aimar committed
1
2
3
/*****************************************************************************
 * x264: h264 encoder
 *****************************************************************************
4
 * Copyright (C) 2003-2008 x264 project
Laurent Aimar's avatar
Laurent Aimar committed
5
6
 *
 * Authors: Laurent Aimar <fenrir@via.ecp.fr>
7
 *          Loren Merritt <lorenm@u.washington.edu>
8
 *          Fiona Glaser <fiona@x264.com>
Laurent Aimar's avatar
Laurent Aimar committed
9
10
11
12
13
14
15
16
17
18
19
20
21
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
22
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
Laurent Aimar's avatar
Laurent Aimar committed
23
24
25
26
 *****************************************************************************/

#include <math.h>

27
28
#include "common/common.h"
#include "common/cpu.h"
Laurent Aimar's avatar
Laurent Aimar committed
29
30
31
32
33
34

#include "set.h"
#include "analyse.h"
#include "ratecontrol.h"
#include "macroblock.h"

Loren Merritt's avatar
Loren Merritt committed
35
36
37
38
#if VISUALIZE
#include "common/visualize.h"
#endif

Laurent Aimar's avatar
Laurent Aimar committed
39
40
//#define DEBUG_MB_TYPE

41
#define NALU_OVERHEAD 5 // startcode + NAL type costs 5 bytes per frame
Laurent Aimar's avatar
Laurent Aimar committed
42

Fiona Glaser's avatar
Fiona Glaser committed
43
44
#define bs_write_ue bs_write_ue_big

Fiona Glaser's avatar
Fiona Glaser committed
45
static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current,
46
47
                                   x264_nal_t **pp_nal, int *pi_nal,
                                   x264_picture_t *pic_out );
Loren Merritt's avatar
Loren Merritt committed
48

Laurent Aimar's avatar
Laurent Aimar committed
49
50
51
52
53
/****************************************************************************
 *
 ******************************* x264 libs **********************************
 *
 ****************************************************************************/
54
55
56
57
58
59
60
61
62
static float x264_psnr( int64_t i_sqe, int64_t i_size )
{
    double f_mse = (double)i_sqe / ((double)65025.0 * (double)i_size);
    if( f_mse <= 0.0000000001 ) /* Max 100dB */
        return 100;

    return (float)(-10.0 * log( f_mse ) / log( 10.0 ));
}

63
static void x264_frame_dump( x264_t *h )
Laurent Aimar's avatar
Laurent Aimar committed
64
{
65
    FILE *f = fopen( h->param.psz_dump_yuv, "r+b" );
Laurent Aimar's avatar
Laurent Aimar committed
66
    int i, y;
67
68
69
    if( !f )
        return;
    /* Write the frame in display order */
70
71
72
73
    fseek( f, h->fdec->i_frame * h->param.i_height * h->param.i_width * 3/2, SEEK_SET );
    for( i = 0; i < h->fdec->i_plane; i++ )
        for( y = 0; y < h->param.i_height >> !!i; y++ )
            fwrite( &h->fdec->plane[i][y*h->fdec->i_stride[i]], 1, h->param.i_width >> !!i, f );
Laurent Aimar's avatar
Laurent Aimar committed
74
75
76
77
78
    fclose( f );
}


/* Fill "default" values */
79
static void x264_slice_header_init( x264_t *h, x264_slice_header_t *sh,
Laurent Aimar's avatar
Laurent Aimar committed
80
                                    x264_sps_t *sps, x264_pps_t *pps,
Loren Merritt's avatar
Loren Merritt committed
81
                                    int i_idr_pic_id, int i_frame, int i_qp )
Laurent Aimar's avatar
Laurent Aimar committed
82
{
83
    x264_param_t *param = &h->param;
84
    int i;
85

Laurent Aimar's avatar
Laurent Aimar committed
86
87
88
89
90
    /* First we fill all field */
    sh->sps = sps;
    sh->pps = pps;

    sh->i_first_mb  = 0;
Loren Merritt's avatar
Loren Merritt committed
91
    sh->i_last_mb   = h->sps->i_mb_width * h->sps->i_mb_height;
Laurent Aimar's avatar
Laurent Aimar committed
92
93
94
95
    sh->i_pps_id    = pps->i_id;

    sh->i_frame_num = i_frame;

96
97
98
    sh->b_mbaff = h->param.b_interlaced;
    sh->b_field_pic = 0;    /* no field support for now */
    sh->b_bottom_field = 0; /* not yet used */
Laurent Aimar's avatar
Laurent Aimar committed
99
100
101
102
103
104
105
106
107
108
109

    sh->i_idr_pic_id = i_idr_pic_id;

    /* poc stuff, fixed later */
    sh->i_poc_lsb = 0;
    sh->i_delta_poc_bottom = 0;
    sh->i_delta_poc[0] = 0;
    sh->i_delta_poc[1] = 0;

    sh->i_redundant_pic_cnt = 0;

Loren Merritt's avatar
Loren Merritt committed
110
111
112
113
114
115
116
117
    if( !h->mb.b_direct_auto_read )
    {
        if( h->mb.b_direct_auto_write )
            sh->b_direct_spatial_mv_pred = ( h->stat.i_direct_score[1] > h->stat.i_direct_score[0] );
        else
            sh->b_direct_spatial_mv_pred = ( param->analyse.i_direct_mv_pred == X264_DIRECT_PRED_SPATIAL );
    }
    /* else b_direct_spatial_mv_pred was read from the 2pass statsfile */
Laurent Aimar's avatar
Laurent Aimar committed
118
119
120
121
122

    sh->b_num_ref_idx_override = 0;
    sh->i_num_ref_idx_l0_active = 1;
    sh->i_num_ref_idx_l1_active = 1;

123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
    sh->b_ref_pic_list_reordering_l0 = h->b_ref_reorder[0];
    sh->b_ref_pic_list_reordering_l1 = h->b_ref_reorder[1];

    /* If the ref list isn't in the default order, construct reordering header */
    /* List1 reordering isn't needed yet */
    if( sh->b_ref_pic_list_reordering_l0 )
    {
        int pred_frame_num = i_frame;
        for( i = 0; i < h->i_ref0; i++ )
        {
            int diff = h->fref0[i]->i_frame_num - pred_frame_num;
            if( diff == 0 )
                x264_log( h, X264_LOG_ERROR, "diff frame num == 0\n" );
            sh->ref_pic_list_order[0][i].idc = ( diff > 0 );
            sh->ref_pic_list_order[0][i].arg = abs( diff ) - 1;
            pred_frame_num = h->fref0[i]->i_frame_num;
        }
    }

Laurent Aimar's avatar
Laurent Aimar committed
142
143
    sh->i_cabac_init_idc = param->i_cabac_init_idc;

144
    sh->i_qp = i_qp;
145
    sh->i_qp_delta = i_qp - pps->i_pic_init_qp;
Laurent Aimar's avatar
Laurent Aimar committed
146
147
148
    sh->b_sp_for_swidth = 0;
    sh->i_qs_delta = 0;

149
150
151
    /* If effective qp <= 15, deblocking would have no effect anyway */
    if( param->b_deblocking_filter
        && ( h->mb.b_variable_qp
152
        || 15 < i_qp + 2 * X264_MIN(param->i_deblocking_filter_alphac0, param->i_deblocking_filter_beta) ) )
Laurent Aimar's avatar
Laurent Aimar committed
153
154
155
156
157
158
159
160
161
162
163
164
165
    {
        sh->i_disable_deblocking_filter_idc = 0;
    }
    else
    {
        sh->i_disable_deblocking_filter_idc = 1;
    }
    sh->i_alpha_c0_offset = param->i_deblocking_filter_alphac0 << 1;
    sh->i_beta_offset = param->i_deblocking_filter_beta << 1;
}

static void x264_slice_header_write( bs_t *s, x264_slice_header_t *sh, int i_nal_ref_idc )
{
166
167
    int i;

168
169
170
171
172
173
174
175
    if( sh->b_mbaff )
    {
        assert( sh->i_first_mb % (2*sh->sps->i_mb_width) == 0 );
        bs_write_ue( s, sh->i_first_mb >> 1 );
    }
    else
        bs_write_ue( s, sh->i_first_mb );

Laurent Aimar's avatar
Laurent Aimar committed
176
177
178
179
    bs_write_ue( s, sh->i_type + 5 );   /* same type things */
    bs_write_ue( s, sh->i_pps_id );
    bs_write( s, sh->sps->i_log2_max_frame_num, sh->i_frame_num );

180
181
182
183
184
185
186
    if( !sh->sps->b_frame_mbs_only )
    {
        bs_write1( s, sh->b_field_pic );
        if ( sh->b_field_pic )
            bs_write1( s, sh->b_bottom_field );
    }

Laurent Aimar's avatar
Laurent Aimar committed
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
    if( sh->i_idr_pic_id >= 0 ) /* NAL IDR */
    {
        bs_write_ue( s, sh->i_idr_pic_id );
    }

    if( sh->sps->i_poc_type == 0 )
    {
        bs_write( s, sh->sps->i_log2_max_poc_lsb, sh->i_poc_lsb );
        if( sh->pps->b_pic_order && !sh->b_field_pic )
        {
            bs_write_se( s, sh->i_delta_poc_bottom );
        }
    }
    else if( sh->sps->i_poc_type == 1 && !sh->sps->b_delta_pic_order_always_zero )
    {
        bs_write_se( s, sh->i_delta_poc[0] );
        if( sh->pps->b_pic_order && !sh->b_field_pic )
        {
            bs_write_se( s, sh->i_delta_poc[1] );
        }
    }

    if( sh->pps->b_redundant_pic_cnt )
    {
        bs_write_ue( s, sh->i_redundant_pic_cnt );
    }

    if( sh->i_type == SLICE_TYPE_B )
    {
        bs_write1( s, sh->b_direct_spatial_mv_pred );
    }
    if( sh->i_type == SLICE_TYPE_P || sh->i_type == SLICE_TYPE_SP || sh->i_type == SLICE_TYPE_B )
    {
        bs_write1( s, sh->b_num_ref_idx_override );
        if( sh->b_num_ref_idx_override )
        {
            bs_write_ue( s, sh->i_num_ref_idx_l0_active - 1 );
            if( sh->i_type == SLICE_TYPE_B )
            {
                bs_write_ue( s, sh->i_num_ref_idx_l1_active - 1 );
            }
        }
    }

    /* ref pic list reordering */
    if( sh->i_type != SLICE_TYPE_I )
    {
234
235
        bs_write1( s, sh->b_ref_pic_list_reordering_l0 );
        if( sh->b_ref_pic_list_reordering_l0 )
Laurent Aimar's avatar
Laurent Aimar committed
236
        {
237
238
239
240
            for( i = 0; i < sh->i_num_ref_idx_l0_active; i++ )
            {
                bs_write_ue( s, sh->ref_pic_list_order[0][i].idc );
                bs_write_ue( s, sh->ref_pic_list_order[0][i].arg );
Loren Merritt's avatar
Loren Merritt committed
241

242
243
            }
            bs_write_ue( s, 3 );
Laurent Aimar's avatar
Laurent Aimar committed
244
245
246
247
        }
    }
    if( sh->i_type == SLICE_TYPE_B )
    {
248
249
        bs_write1( s, sh->b_ref_pic_list_reordering_l1 );
        if( sh->b_ref_pic_list_reordering_l1 )
Laurent Aimar's avatar
Laurent Aimar committed
250
        {
251
252
253
254
255
256
            for( i = 0; i < sh->i_num_ref_idx_l1_active; i++ )
            {
                bs_write_ue( s, sh->ref_pic_list_order[1][i].idc );
                bs_write_ue( s, sh->ref_pic_list_order[1][i].arg );
            }
            bs_write_ue( s, 3 );
Laurent Aimar's avatar
Laurent Aimar committed
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
        }
    }

    if( ( sh->pps->b_weighted_pred && ( sh->i_type == SLICE_TYPE_P || sh->i_type == SLICE_TYPE_SP ) ) ||
        ( sh->pps->b_weighted_bipred == 1 && sh->i_type == SLICE_TYPE_B ) )
    {
        /* FIXME */
    }

    if( i_nal_ref_idc != 0 )
    {
        if( sh->i_idr_pic_id >= 0 )
        {
            bs_write1( s, 0 );  /* no output of prior pics flag */
            bs_write1( s, 0 );  /* long term reference flag */
        }
        else
        {
            bs_write1( s, 0 );  /* adaptive_ref_pic_marking_mode_flag */
        }
    }

    if( sh->pps->b_cabac && sh->i_type != SLICE_TYPE_I )
    {
        bs_write_ue( s, sh->i_cabac_init_idc );
    }
    bs_write_se( s, sh->i_qp_delta );      /* slice qp delta */

    if( sh->pps->b_deblocking_filter_control )
    {
        bs_write_ue( s, sh->i_disable_deblocking_filter_idc );
        if( sh->i_disable_deblocking_filter_idc != 1 )
        {
            bs_write_se( s, sh->i_alpha_c0_offset >> 1 );
            bs_write_se( s, sh->i_beta_offset >> 1 );
        }
    }
}

296
297
/* If we are within a reasonable distance of the end of the memory allocated for the bitstream, */
/* reallocate, adding an arbitrary amount of space (100 kilobytes). */
298
static int x264_bitstream_check_buffer( x264_t *h )
299
{
300
    uint8_t *bs_bak = h->out.p_bitstream;
301
302
303
304
305
306
307
    if( ( h->param.b_cabac && (h->cabac.p_end - h->cabac.p < 2500) )
     || ( h->out.bs.p_end - h->out.bs.p < 2500 ) )
    {
        intptr_t delta;
        int i;

        h->out.i_bitstream += 100000;
308
309
        CHECKED_MALLOC( h->out.p_bitstream, h->out.i_bitstream );
        h->mc.memcpy_aligned( h->out.p_bitstream, bs_bak, (h->out.i_bitstream - 100000) & ~15 );
310
311
312
313
314
315
316
317
318
319
320
321
        delta = h->out.p_bitstream - bs_bak;

        h->out.bs.p_start += delta;
        h->out.bs.p += delta;
        h->out.bs.p_end = h->out.p_bitstream + h->out.i_bitstream;

        h->cabac.p_start += delta;
        h->cabac.p += delta;
        h->cabac.p_end = h->out.p_bitstream + h->out.i_bitstream;

        for( i = 0; i <= h->out.i_nal; i++ )
            h->out.nal[i].p_payload += delta;
322
        x264_free( bs_bak );
323
    }
324
325
326
327
    return 0;
fail:
    x264_free( bs_bak );
    return -1;
328
329
}

Laurent Aimar's avatar
Laurent Aimar committed
330
331
332
333
334
335
336
337
/****************************************************************************
 *
 ****************************************************************************
 ****************************** External API*********************************
 ****************************************************************************
 *
 ****************************************************************************/

338
static int x264_validate_parameters( x264_t *h )
Laurent Aimar's avatar
Laurent Aimar committed
339
{
340
341
342
343
344
345
346
347
#ifdef HAVE_MMX
    if( !(x264_cpu_detect() & X264_CPU_MMXEXT) )
    {
        x264_log( h, X264_LOG_ERROR, "your cpu does not support MMXEXT, but x264 was compiled with asm support\n");
        x264_log( h, X264_LOG_ERROR, "to run x264, recompile without asm support (configure --disable-asm)\n");
        return -1;
    }
#endif
348
    if( h->param.i_width <= 0 || h->param.i_height <= 0 )
Laurent Aimar's avatar
Laurent Aimar committed
349
    {
350
        x264_log( h, X264_LOG_ERROR, "invalid width x height (%dx%d)\n",
351
352
                  h->param.i_width, h->param.i_height );
        return -1;
Laurent Aimar's avatar
Laurent Aimar committed
353
354
    }

Loren Merritt's avatar
Loren Merritt committed
355
    if( h->param.i_width % 2 || h->param.i_height % 2 )
Laurent Aimar's avatar
Laurent Aimar committed
356
    {
Loren Merritt's avatar
Loren Merritt committed
357
        x264_log( h, X264_LOG_ERROR, "width or height not divisible by 2 (%dx%d)\n",
358
359
                  h->param.i_width, h->param.i_height );
        return -1;
Laurent Aimar's avatar
Laurent Aimar committed
360
    }
361
    if( h->param.i_csp != X264_CSP_I420 )
Laurent Aimar's avatar
Laurent Aimar committed
362
    {
363
        x264_log( h, X264_LOG_ERROR, "invalid CSP (only I420 supported)\n" );
364
        return -1;
Laurent Aimar's avatar
Laurent Aimar committed
365
366
    }

367
    if( h->param.i_threads == 0 )
Loren Merritt's avatar
Loren Merritt committed
368
369
        h->param.i_threads = x264_cpu_num_processors() * 3/2;
    h->param.i_threads = x264_clip3( h->param.i_threads, 1, X264_THREAD_MAX );
Loren Merritt's avatar
Loren Merritt committed
370
371
    if( h->param.i_threads > 1 )
    {
Loren Merritt's avatar
Loren Merritt committed
372
#ifndef HAVE_PTHREAD
Loren Merritt's avatar
Loren Merritt committed
373
        x264_log( h, X264_LOG_WARNING, "not compiled with pthread support!\n");
Loren Merritt's avatar
Loren Merritt committed
374
        h->param.i_threads = 1;
Loren Merritt's avatar
Loren Merritt committed
375
#endif
Loren Merritt's avatar
Loren Merritt committed
376
    }
Loren Merritt's avatar
Loren Merritt committed
377

378
379
    if( h->param.b_interlaced )
    {
380
        if( h->param.analyse.i_me_method >= X264_ME_ESA )
381
382
383
384
385
386
387
388
389
390
391
        {
            x264_log( h, X264_LOG_WARNING, "interlace + me=esa is not implemented\n" );
            h->param.analyse.i_me_method = X264_ME_UMH;
        }
        if( h->param.analyse.i_direct_mv_pred > X264_DIRECT_PRED_SPATIAL )
        {
            x264_log( h, X264_LOG_WARNING, "interlace + direct=temporal is not implemented\n" );
            h->param.analyse.i_direct_mv_pred = X264_DIRECT_PRED_SPATIAL;
        }
    }

392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
    /* Detect default ffmpeg settings and terminate with an error. */
    {
        int score = 0;
        score += h->param.analyse.i_me_range == 0;
        score += h->param.rc.i_qp_step == 3;
        score += h->param.i_keyint_max == 12;
        score += h->param.rc.i_qp_min == 2;
        score += h->param.rc.i_qp_max == 31;
        score += h->param.rc.f_qcompress == 0.5;
        score += fabs(h->param.rc.f_ip_factor - 1.25) < 0.01;
        score += fabs(h->param.rc.f_pb_factor - 1.25) < 0.01;
        score += h->param.analyse.inter == 0 && h->param.analyse.i_subpel_refine == 8;
        if( score >= 5 )
        {
            x264_log( h, X264_LOG_ERROR, "broken ffmpeg default settings detected\n" );
            x264_log( h, X264_LOG_ERROR, "use an encoding preset (vpre)\n" );
            return -1;
        }
    }

412
413
    if( h->param.rc.i_rc_method < 0 || h->param.rc.i_rc_method > 2 )
    {
414
        x264_log( h, X264_LOG_ERROR, "no ratecontrol method specified\n" );
415
416
        return -1;
    }
Loren Merritt's avatar
Loren Merritt committed
417
    h->param.rc.f_rf_constant = x264_clip3f( h->param.rc.f_rf_constant, 0, 51 );
418
    h->param.rc.i_qp_constant = x264_clip3( h->param.rc.i_qp_constant, 0, 51 );
419
    if( h->param.rc.i_rc_method == X264_RC_CRF )
Fiona Glaser's avatar
Fiona Glaser committed
420
    {
Loren Merritt's avatar
Loren Merritt committed
421
        h->param.rc.i_qp_constant = h->param.rc.f_rf_constant;
Fiona Glaser's avatar
Fiona Glaser committed
422
423
        h->param.rc.i_bitrate = 0;
    }
424
425
    if( (h->param.rc.i_rc_method == X264_RC_CQP || h->param.rc.i_rc_method == X264_RC_CRF)
        && h->param.rc.i_qp_constant == 0 )
Loren Merritt's avatar
Loren Merritt committed
426
427
    {
        h->mb.b_lossless = 1;
Loren Merritt's avatar
Loren Merritt committed
428
        h->param.i_cqm_preset = X264_CQM_FLAT;
Loren Merritt's avatar
Loren Merritt committed
429
        h->param.psz_cqm_file = NULL;
430
        h->param.rc.i_rc_method = X264_RC_CQP;
Loren Merritt's avatar
Loren Merritt committed
431
432
433
        h->param.rc.f_ip_factor = 1;
        h->param.rc.f_pb_factor = 1;
        h->param.analyse.b_psnr = 0;
434
        h->param.analyse.b_ssim = 0;
435
        h->param.analyse.i_chroma_qp_offset = 0;
436
        h->param.analyse.i_trellis = 0;
Loren Merritt's avatar
Loren Merritt committed
437
        h->param.analyse.b_fast_pskip = 0;
438
        h->param.analyse.i_noise_reduction = 0;
439
        h->param.analyse.f_psy_rd = 0;
440
        h->param.i_bframe = 0;
441
442
443
        /* 8x8dct is not useful at all in CAVLC lossless */
        if( !h->param.b_cabac )
            h->param.analyse.b_transform_8x8 = 0;
Loren Merritt's avatar
Loren Merritt committed
444
    }
Loren Merritt's avatar
Loren Merritt committed
445
446
447
448
449
450
451
    if( h->param.rc.i_rc_method == X264_RC_CQP )
    {
        float qp_p = h->param.rc.i_qp_constant;
        float qp_i = qp_p - 6*log(h->param.rc.f_ip_factor)/log(2);
        float qp_b = qp_p + 6*log(h->param.rc.f_pb_factor)/log(2);
        h->param.rc.i_qp_min = x264_clip3( (int)(X264_MIN3( qp_p, qp_i, qp_b )), 0, 51 );
        h->param.rc.i_qp_max = x264_clip3( (int)(X264_MAX3( qp_p, qp_i, qp_b ) + .999), 0, 51 );
Loren Merritt's avatar
Loren Merritt committed
452
        h->param.rc.i_aq_mode = 0;
Fiona Glaser's avatar
Fiona Glaser committed
453
        h->param.rc.b_mb_tree = 0;
Loren Merritt's avatar
Loren Merritt committed
454
    }
455
456
    h->param.rc.i_qp_max = x264_clip3( h->param.rc.i_qp_max, 0, 51 );
    h->param.rc.i_qp_min = x264_clip3( h->param.rc.i_qp_min, 0, h->param.rc.i_qp_max );
Loren Merritt's avatar
Loren Merritt committed
457

458
459
    if( ( h->param.i_width % 16 || h->param.i_height % 16 )
        && h->param.i_height != 1080 && !h->mb.b_lossless )
Loren Merritt's avatar
Loren Merritt committed
460
    {
461
462
463
        // There's nothing special about 1080 in that the warning still applies to it,
        // but chances are the user can't help it if his content is already 1080p,
        // so there's no point in warning in that case.
Loren Merritt's avatar
Loren Merritt committed
464
        x264_log( h, X264_LOG_WARNING,
Loren Merritt's avatar
Loren Merritt committed
465
466
467
468
                  "width or height not divisible by 16 (%dx%d), compression will suffer.\n",
                  h->param.i_width, h->param.i_height );
    }

Loren Merritt's avatar
Loren Merritt committed
469
    h->param.i_frame_reference = x264_clip3( h->param.i_frame_reference, 1, 16 );
470
471
    if( h->param.i_keyint_max <= 0 )
        h->param.i_keyint_max = 1;
Fiona Glaser's avatar
Fiona Glaser committed
472
473
    if( h->param.i_scenecut_threshold < 0 )
        h->param.i_scenecut_threshold = 0;
474
    h->param.i_keyint_min = x264_clip3( h->param.i_keyint_min, 1, h->param.i_keyint_max/2+1 );
475
476
477
478
479
    if( !h->param.analyse.i_subpel_refine && h->param.analyse.i_direct_mv_pred > X264_DIRECT_PRED_SPATIAL )
    {
        x264_log( h, X264_LOG_WARNING, "subme=0 + direct=temporal is not supported\n" );
        h->param.analyse.i_direct_mv_pred = X264_DIRECT_PRED_SPATIAL;
    }
Loren Merritt's avatar
Loren Merritt committed
480
    h->param.i_bframe = x264_clip3( h->param.i_bframe, 0, X264_BFRAME_MAX );
481
    h->param.i_bframe_bias = x264_clip3( h->param.i_bframe_bias, -90, 100 );
482
    h->param.b_bframe_pyramid = h->param.b_bframe_pyramid && h->param.i_bframe > 1;
483
    if( !h->param.i_bframe )
484
    {
485
        h->param.i_bframe_adaptive = X264_B_ADAPT_NONE;
486
487
488
        h->param.analyse.i_direct_mv_pred = 0;
        h->param.analyse.b_weighted_bipred = 0;
    }
Fiona Glaser's avatar
Fiona Glaser committed
489
    h->param.rc.i_lookahead = x264_clip3( h->param.rc.i_lookahead, 0, X264_LOOKAHEAD_MAX );
Fiona Glaser's avatar
Fiona Glaser committed
490
491
492
493
494
495
496
    {
        int maxrate = X264_MAX( h->param.rc.i_vbv_max_bitrate, h->param.rc.i_bitrate );
        float bufsize = maxrate ? (float)h->param.rc.i_vbv_buffer_size / maxrate : 0;
        float fps = h->param.i_fps_num > 0 && h->param.i_fps_den > 0 ? (float) h->param.i_fps_num / h->param.i_fps_den : 25.0;
        h->param.rc.i_lookahead = X264_MIN( h->param.rc.i_lookahead, X264_MAX( h->param.i_keyint_max, bufsize*fps ) );
    }

Fiona Glaser's avatar
Fiona Glaser committed
497
498
499
500
501
502
503
    if( h->param.rc.b_stat_read )
        h->param.rc.i_lookahead = 0;
    else if( !h->param.rc.i_lookahead )
        h->param.rc.b_mb_tree = 0;
    if( h->param.rc.f_qcompress == 1 )
        h->param.rc.b_mb_tree = 0;

Loren Merritt's avatar
Loren Merritt committed
504
505
506
    h->mb.b_direct_auto_write = h->param.analyse.i_direct_mv_pred == X264_DIRECT_PRED_AUTO
                                && h->param.i_bframe
                                && ( h->param.rc.b_stat_write || !h->param.rc.b_stat_read );
Laurent Aimar's avatar
Laurent Aimar committed
507
508
509

    h->param.i_deblocking_filter_alphac0 = x264_clip3( h->param.i_deblocking_filter_alphac0, -6, 6 );
    h->param.i_deblocking_filter_beta    = x264_clip3( h->param.i_deblocking_filter_beta, -6, 6 );
510
511
    h->param.analyse.i_luma_deadzone[0] = x264_clip3( h->param.analyse.i_luma_deadzone[0], 0, 32 );
    h->param.analyse.i_luma_deadzone[1] = x264_clip3( h->param.analyse.i_luma_deadzone[1], 0, 32 );
Laurent Aimar's avatar
Laurent Aimar committed
512

513
    h->param.i_cabac_init_idc = x264_clip3( h->param.i_cabac_init_idc, 0, 2 );
Laurent Aimar's avatar
Laurent Aimar committed
514

Loren Merritt's avatar
Loren Merritt committed
515
516
517
    if( h->param.i_cqm_preset < X264_CQM_FLAT || h->param.i_cqm_preset > X264_CQM_CUSTOM )
        h->param.i_cqm_preset = X264_CQM_FLAT;

518
    if( h->param.analyse.i_me_method < X264_ME_DIA ||
519
        h->param.analyse.i_me_method > X264_ME_TESA )
520
        h->param.analyse.i_me_method = X264_ME_HEX;
521
522
523
    if( h->param.analyse.i_me_range < 4 )
        h->param.analyse.i_me_range = 4;
    if( h->param.analyse.i_me_range > 16 && h->param.analyse.i_me_method <= X264_ME_HEX )
524
        h->param.analyse.i_me_range = 16;
525
526
527
    if( h->param.analyse.i_me_method == X264_ME_TESA &&
        (h->mb.b_lossless || h->param.analyse.i_subpel_refine <= 1) )
        h->param.analyse.i_me_method = X264_ME_ESA;
Fiona Glaser's avatar
Fiona Glaser committed
528
    h->param.analyse.i_subpel_refine = x264_clip3( h->param.analyse.i_subpel_refine, 0, 10 );
529
530
531
532
    h->param.analyse.b_mixed_references = h->param.analyse.b_mixed_references && h->param.i_frame_reference > 1;
    h->param.analyse.inter &= X264_ANALYSE_PSUB16x16|X264_ANALYSE_PSUB8x8|X264_ANALYSE_BSUB16x16|
                              X264_ANALYSE_I4x4|X264_ANALYSE_I8x8;
    h->param.analyse.intra &= X264_ANALYSE_I4x4|X264_ANALYSE_I8x8;
533
534
    if( !(h->param.analyse.inter & X264_ANALYSE_PSUB16x16) )
        h->param.analyse.inter &= ~X264_ANALYSE_PSUB8x8;
535
536
537
538
539
    if( !h->param.analyse.b_transform_8x8 )
    {
        h->param.analyse.inter &= ~X264_ANALYSE_I8x8;
        h->param.analyse.intra &= ~X264_ANALYSE_I8x8;
    }
540
    h->param.analyse.i_chroma_qp_offset = x264_clip3(h->param.analyse.i_chroma_qp_offset, -12, 12);
Loren Merritt's avatar
Loren Merritt committed
541
542
    if( !h->param.b_cabac )
        h->param.analyse.i_trellis = 0;
543
    h->param.analyse.i_trellis = x264_clip3( h->param.analyse.i_trellis, 0, 2 );
Fiona Glaser's avatar
Fiona Glaser committed
544
545
546
547
548
    if( !h->param.analyse.b_psy )
    {
        h->param.analyse.f_psy_rd = 0;
        h->param.analyse.f_psy_trellis = 0;
    }
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
    if( !h->param.analyse.i_trellis )
        h->param.analyse.f_psy_trellis = 0;
    h->param.analyse.f_psy_rd = x264_clip3f( h->param.analyse.f_psy_rd, 0, 10 );
    h->param.analyse.f_psy_trellis = x264_clip3f( h->param.analyse.f_psy_trellis, 0, 10 );
    if( h->param.analyse.i_subpel_refine < 6 )
        h->param.analyse.f_psy_rd = 0;
    h->mb.i_psy_rd = FIX8( h->param.analyse.f_psy_rd );
    /* Psy RDO increases overall quantizers to improve the quality of luma--this indirectly hurts chroma quality */
    /* so we lower the chroma QP offset to compensate */
    /* This can be triggered repeatedly on multiple calls to parameter_validate, but since encoding
     * uses the pps chroma qp offset not the param chroma qp offset, this is not a problem. */
    if( h->mb.i_psy_rd )
        h->param.analyse.i_chroma_qp_offset -= h->param.analyse.f_psy_rd < 0.25 ? 1 : 2;
    h->mb.i_psy_trellis = FIX8( h->param.analyse.f_psy_trellis / 4 );
    /* Psy trellis has a similar effect. */
    if( h->mb.i_psy_trellis )
        h->param.analyse.i_chroma_qp_offset -= h->param.analyse.f_psy_trellis < 0.25 ? 1 : 2;
    else
        h->mb.i_psy_trellis = 0;
    h->param.analyse.i_chroma_qp_offset = x264_clip3(h->param.analyse.i_chroma_qp_offset, -12, 12);
Anton Mitrofanov's avatar
Anton Mitrofanov committed
569
    h->param.rc.i_aq_mode = x264_clip3( h->param.rc.i_aq_mode, 0, 2 );
570
571
    h->param.rc.f_aq_strength = x264_clip3f( h->param.rc.f_aq_strength, 0, 3 );
    if( h->param.rc.f_aq_strength == 0 )
572
        h->param.rc.i_aq_mode = 0;
Fiona Glaser's avatar
Fiona Glaser committed
573
574
575
576
577
578
579
580
581
582
583
    /* MB-tree requires AQ to be on, even if the strength is zero. */
    if( !h->param.rc.i_aq_mode && h->param.rc.b_mb_tree )
    {
        h->param.rc.i_aq_mode = 1;
        h->param.rc.f_aq_strength = 0;
    }
    if( h->param.rc.b_mb_tree && h->param.b_bframe_pyramid )
    {
        x264_log( h, X264_LOG_WARNING, "b-pyramid + mb-tree is not supported\n" );
        h->param.b_bframe_pyramid = 0;
    }
584
    h->param.analyse.i_noise_reduction = x264_clip3( h->param.analyse.i_noise_reduction, 0, 1<<16 );
Fiona Glaser's avatar
Fiona Glaser committed
585
586
    if( h->param.analyse.i_subpel_refine == 10 && (h->param.analyse.i_trellis != 2 || !h->param.rc.i_aq_mode) )
        h->param.analyse.i_subpel_refine = 9;
587

588
589
    {
        const x264_level_t *l = x264_levels;
590
        if( h->param.i_level_idc < 0 )
591
        {
Loren Merritt's avatar
Loren Merritt committed
592
            int maxrate_bak = h->param.rc.i_vbv_max_bitrate;
593
594
595
596
597
598
            if( h->param.rc.i_rc_method == X264_RC_ABR && h->param.rc.i_vbv_buffer_size <= 0 )
                h->param.rc.i_vbv_max_bitrate = h->param.rc.i_bitrate * 2;
            h->sps = h->sps_array;
            x264_sps_init( h->sps, h->param.i_sps_id, &h->param );
            do h->param.i_level_idc = l->level_idc;
                while( l[1].level_idc && x264_validate_levels( h, 0 ) && l++ );
Loren Merritt's avatar
Loren Merritt committed
599
            h->param.rc.i_vbv_max_bitrate = maxrate_bak;
600
601
602
603
604
605
606
607
608
609
        }
        else
        {
            while( l->level_idc && l->level_idc != h->param.i_level_idc )
                l++;
            if( l->level_idc == 0 )
            {
                x264_log( h, X264_LOG_ERROR, "invalid level_idc: %d\n", h->param.i_level_idc );
                return -1;
            }
610
611
        }
        if( h->param.analyse.i_mv_range <= 0 )
612
            h->param.analyse.i_mv_range = l->mv_range >> h->param.b_interlaced;
613
        else
614
            h->param.analyse.i_mv_range = x264_clip3(h->param.analyse.i_mv_range, 32, 512 >> h->param.b_interlaced);
615
616
    }

Loren Merritt's avatar
Loren Merritt committed
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
    if( h->param.i_threads > 1 )
    {
        int r = h->param.analyse.i_mv_range_thread;
        int r2;
        if( r <= 0 )
        {
            // half of the available space is reserved and divided evenly among the threads,
            // the rest is allocated to whichever thread is far enough ahead to use it.
            // reserving more space increases quality for some videos, but costs more time
            // in thread synchronization.
            int max_range = (h->param.i_height + X264_THREAD_HEIGHT) / h->param.i_threads - X264_THREAD_HEIGHT;
            r = max_range / 2;
        }
        r = X264_MAX( r, h->param.analyse.i_me_range );
        r = X264_MIN( r, h->param.analyse.i_mv_range );
        // round up to use the whole mb row
        r2 = (r & ~15) + ((-X264_THREAD_HEIGHT) & 15);
        if( r2 < r )
            r2 += 16;
        x264_log( h, X264_LOG_DEBUG, "using mv_range_thread = %d\n", r2 );
        h->param.analyse.i_mv_range_thread = r2;
    }

Loren Merritt's avatar
Loren Merritt committed
640
641
642
643
644
    if( h->param.rc.f_qblur < 0 )
        h->param.rc.f_qblur = 0;
    if( h->param.rc.f_complexity_blur < 0 )
        h->param.rc.f_complexity_blur = 0;

645
646
    h->param.i_sps_id &= 31;

647
648
649
650
651
652
    if( h->param.i_log_level < X264_LOG_INFO )
    {
        h->param.analyse.b_psnr = 0;
        h->param.analyse.b_ssim = 0;
    }

653
654
655
656
    /* ensure the booleans are 0 or 1 so they can be used in math */
#define BOOLIFY(x) h->param.x = !!h->param.x
    BOOLIFY( b_cabac );
    BOOLIFY( b_deblocking_filter );
657
    BOOLIFY( b_interlaced );
658
659
660
661
662
    BOOLIFY( analyse.b_transform_8x8 );
    BOOLIFY( analyse.b_chroma_me );
    BOOLIFY( analyse.b_fast_pskip );
    BOOLIFY( rc.b_stat_write );
    BOOLIFY( rc.b_stat_read );
663
    BOOLIFY( rc.b_mb_tree );
664
665
#undef BOOLIFY

666
667
668
    return 0;
}

669
670
static void mbcmp_init( x264_t *h )
{
671
    int satd = !h->mb.b_lossless && h->param.analyse.i_subpel_refine > 1;
672
673
    memcpy( h->pixf.mbcmp, satd ? h->pixf.satd : h->pixf.sad_aligned, sizeof(h->pixf.mbcmp) );
    memcpy( h->pixf.mbcmp_unaligned, satd ? h->pixf.satd : h->pixf.sad, sizeof(h->pixf.mbcmp_unaligned) );
674
    h->pixf.intra_mbcmp_x3_16x16 = satd ? h->pixf.intra_satd_x3_16x16 : h->pixf.intra_sad_x3_16x16;
Fiona Glaser's avatar
Fiona Glaser committed
675
    h->pixf.intra_mbcmp_x3_8x8c = satd ? h->pixf.intra_satd_x3_8x8c : h->pixf.intra_sad_x3_8x8c;
Fiona Glaser's avatar
Fiona Glaser committed
676
    h->pixf.intra_mbcmp_x3_4x4 = satd ? h->pixf.intra_satd_x3_4x4 : h->pixf.intra_sad_x3_4x4;
677
678
679
680
    satd &= h->param.analyse.i_me_method == X264_ME_TESA;
    memcpy( h->pixf.fpelcmp, satd ? h->pixf.satd : h->pixf.sad, sizeof(h->pixf.fpelcmp) );
    memcpy( h->pixf.fpelcmp_x3, satd ? h->pixf.satd_x3 : h->pixf.sad_x3, sizeof(h->pixf.fpelcmp_x3) );
    memcpy( h->pixf.fpelcmp_x4, satd ? h->pixf.satd_x4 : h->pixf.sad_x4, sizeof(h->pixf.fpelcmp_x4) );
681
682
}

683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
static void x264_set_aspect_ratio( x264_t *h, x264_param_t *param, int initial )
{
    /* VUI */
    if( param->vui.i_sar_width > 0 && param->vui.i_sar_height > 0 )
    {
        int i_w = param->vui.i_sar_width;
        int i_h = param->vui.i_sar_height;
        int old_w = h->param.vui.i_sar_width;
        int old_h = h->param.vui.i_sar_height;

        x264_reduce_fraction( &i_w, &i_h );

        while( i_w > 65535 || i_h > 65535 )
        {
            i_w /= 2;
            i_h /= 2;
        }

        if( i_w != old_w || i_h != old_h || initial )
        {
            h->param.vui.i_sar_width = 0;
            h->param.vui.i_sar_height = 0;
            if( i_w == 0 || i_h == 0 )
                x264_log( h, X264_LOG_WARNING, "cannot create valid sample aspect ratio\n" );
            else
            {
                x264_log( h, initial?X264_LOG_INFO:X264_LOG_DEBUG, "using SAR=%d/%d\n", i_w, i_h );
                h->param.vui.i_sar_width = i_w;
                h->param.vui.i_sar_height = i_h;
            }
        }
    }
}

717
718
719
720
721
/****************************************************************************
 * x264_encoder_open:
 ****************************************************************************/
x264_t *x264_encoder_open   ( x264_param_t *param )
{
722
    x264_t *h;
723
    char buf[1000], *p;
724
725
    int i;

726
    CHECKED_MALLOCZERO( h, sizeof(x264_t) );
727
728

    /* Create a copy of param */
729
    memcpy( &h->param, param, sizeof(x264_param_t) );
730

731
732
733
    if( param->param_free )
        param->param_free( param );

734
    if( x264_validate_parameters( h ) < 0 )
735
        goto fail;
736

737
738
    if( h->param.psz_cqm_file )
        if( x264_cqm_parse_file( h, h->param.psz_cqm_file ) < 0 )
739
            goto fail;
740

741
742
743
744
745
    if( h->param.rc.psz_stat_out )
        h->param.rc.psz_stat_out = strdup( h->param.rc.psz_stat_out );
    if( h->param.rc.psz_stat_in )
        h->param.rc.psz_stat_in = strdup( h->param.rc.psz_stat_in );

746
    x264_set_aspect_ratio( h, param, 1 );
747

Loren Merritt's avatar
Loren Merritt committed
748
    x264_reduce_fraction( &h->param.i_fps_num, &h->param.i_fps_den );
749

Laurent Aimar's avatar
Laurent Aimar committed
750
751
752
753
754
755
    /* Init x264_t */
    h->i_frame = 0;
    h->i_frame_num = 0;
    h->i_idr_pic_id = 0;

    h->sps = &h->sps_array[0];
756
    x264_sps_init( h->sps, h->param.i_sps_id, &h->param );
Laurent Aimar's avatar
Laurent Aimar committed
757
758

    h->pps = &h->pps_array[0];
759
    x264_pps_init( h->pps, h->param.i_sps_id, &h->param, h->sps);
Loren Merritt's avatar
Loren Merritt committed
760

761
    x264_validate_levels( h, 1 );
762

Loren Merritt's avatar
Loren Merritt committed
763
    if( x264_cqm_init( h ) < 0 )
764
        goto fail;
Loren Merritt's avatar
Loren Merritt committed
765

766
    h->mb.i_mb_count = h->sps->i_mb_width * h->sps->i_mb_height;
Laurent Aimar's avatar
Laurent Aimar committed
767
768

    /* Init frames. */
769
    if( h->param.i_bframe_adaptive == X264_B_ADAPT_TRELLIS )
770
        h->frames.i_delay = X264_MAX(h->param.i_bframe,3)*4;
771
    else
772
        h->frames.i_delay = h->param.i_bframe;
Fiona Glaser's avatar
Fiona Glaser committed
773
    if( h->param.rc.b_mb_tree || h->param.rc.i_vbv_buffer_size )
Fiona Glaser's avatar
Fiona Glaser committed
774
        h->frames.i_delay = X264_MAX( h->frames.i_delay, h->param.rc.i_lookahead );
775
776
    h->frames.i_delay += h->param.i_threads - 1;
    h->frames.i_delay = X264_MIN( h->frames.i_delay, X264_LOOKAHEAD_MAX );
Fiona Glaser's avatar
Fiona Glaser committed
777

778
779
    h->frames.i_max_ref0 = h->param.i_frame_reference;
    h->frames.i_max_ref1 = h->sps->vui.i_num_reorder_frames;
780
    h->frames.i_max_dpb  = h->sps->vui.i_max_dec_frame_buffering;
781
    h->frames.b_have_lowres = !h->param.rc.b_stat_read
782
783
        && ( h->param.rc.i_rc_method == X264_RC_ABR
          || h->param.rc.i_rc_method == X264_RC_CRF
784
          || h->param.i_bframe_adaptive
Fiona Glaser's avatar
Fiona Glaser committed
785
786
          || h->param.i_scenecut_threshold
          || h->param.rc.b_mb_tree );
787
    h->frames.b_have_lowres |= (h->param.rc.b_stat_read && h->param.rc.i_vbv_buffer_size > 0);
Loren Merritt's avatar
Loren Merritt committed
788
    h->frames.b_have_sub8x8_esa = !!(h->param.analyse.inter & X264_ANALYSE_PSUB8x8);
789

790
    h->frames.i_last_idr = - h->param.i_keyint_max;
791
    h->frames.i_input    = 0;
792
    h->frames.last_nonb  = NULL;
Laurent Aimar's avatar
Laurent Aimar committed
793
794
795
796

    h->i_ref0 = 0;
    h->i_ref1 = 0;

797
798
    h->chroma_qp_table = i_chroma_qp_table + 12 + h->pps->i_chroma_qp_index_offset;

Loren Merritt's avatar
Loren Merritt committed
799
    x264_rdo_init( );
Laurent Aimar's avatar
Laurent Aimar committed
800
801
802

    /* init CPU functions */
    x264_predict_16x16_init( h->param.cpu, h->predict_16x16 );
803
    x264_predict_8x8c_init( h->param.cpu, h->predict_8x8c );
804
    x264_predict_8x8_init( h->param.cpu, h->predict_8x8, &h->predict_8x8_filter );
Laurent Aimar's avatar
Laurent Aimar committed
805
    x264_predict_4x4_init( h->param.cpu, h->predict_4x4 );
Fiona Glaser's avatar
Fiona Glaser committed
806
    if( !h->param.b_cabac )
807
        x264_init_vlc_tables();
Laurent Aimar's avatar
Laurent Aimar committed
808
809
    x264_pixel_init( h->param.cpu, &h->pixf );
    x264_dct_init( h->param.cpu, &h->dctf );
810
    x264_zigzag_init( h->param.cpu, &h->zigzagf, h->param.b_interlaced );
811
    x264_mc_init( h->param.cpu, &h->mc );
812
    x264_quant_init( h, h->param.cpu, &h->quantf );
Loren Merritt's avatar
Loren Merritt committed
813
    x264_deblock_init( h->param.cpu, &h->loopf );
814
    x264_dct_init_weights();
Laurent Aimar's avatar
Laurent Aimar committed
815

816
    mbcmp_init( h );
Loren Merritt's avatar
Loren Merritt committed
817

818
819
    p = buf + sprintf( buf, "using cpu capabilities:" );
    for( i=0; x264_cpu_names[i].flags; i++ )
820
821
822
823
824
825
826
    {
        if( !strcmp(x264_cpu_names[i].name, "SSE2")
            && param->cpu & (X264_CPU_SSE2_IS_FAST|X264_CPU_SSE2_IS_SLOW) )
            continue;
        if( !strcmp(x264_cpu_names[i].name, "SSE3")
            && (param->cpu & X264_CPU_SSSE3 || !(param->cpu & X264_CPU_CACHELINE_64)) )
            continue;
827
828
829
        if( !strcmp(x264_cpu_names[i].name, "SSE4.1")
            && (param->cpu & X264_CPU_SSE42) )
            continue;
830
831
832
        if( (param->cpu & x264_cpu_names[i].flags) == x264_cpu_names[i].flags
            && (!i || x264_cpu_names[i].flags != x264_cpu_names[i-1].flags) )
            p += sprintf( p, " %s", x264_cpu_names[i].name );
833
    }
834
835
836
    if( !param->cpu )
        p += sprintf( p, " none!" );
    x264_log( h, X264_LOG_INFO, "%s\n", buf );
837

Loren Merritt's avatar
Loren Merritt committed
838
    h->out.i_nal = 0;
839
840
841
    h->out.i_bitstream = X264_MAX( 1000000, h->param.i_width * h->param.i_height * 4
        * ( h->param.rc.i_rc_method == X264_RC_ABR ? pow( 0.95, h->param.rc.i_qp_min )
          : pow( 0.95, h->param.rc.i_qp_constant ) * X264_MAX( 1, h->param.rc.f_ip_factor )));
Loren Merritt's avatar
Loren Merritt committed
842

Loren Merritt's avatar
Loren Merritt committed
843
    h->thread[0] = h;
844
    h->i_thread_num = 0;
845
    for( i = 1; i < h->param.i_threads; i++ )
846
        CHECKED_MALLOC( h->thread[i], sizeof(x264_t) );
Loren Merritt's avatar
Loren Merritt committed
847

Loren Merritt's avatar
Loren Merritt committed
848
849
850
851
    for( i = 0; i < h->param.i_threads; i++ )
    {
        if( i > 0 )
            *h->thread[i] = *h;
852
        h->thread[i]->fdec = x264_frame_pop_unused( h );
853
854
855
        if( !h->thread[i]->fdec )
            goto fail;
        CHECKED_MALLOC( h->thread[i]->out.p_bitstream, h->out.i_bitstream );
Loren Merritt's avatar
Loren Merritt committed
856
        if( x264_macroblock_cache_init( h->thread[i] ) < 0 )
857
            goto fail;
Loren Merritt's avatar
Loren Merritt committed
858
859
860
    }

    if( x264_ratecontrol_new( h ) < 0 )
861
862
863
864
        goto fail;

    if( x264_lowres_context_alloc( h ) )
        goto fail;
Loren Merritt's avatar
Loren Merritt committed
865

866
    if( h->param.psz_dump_yuv )
867
868
    {
        /* create or truncate the reconstructed video file */
869
        FILE *f = fopen( h->param.psz_dump_yuv, "w" );
870
871
872
873
874
        if( f )
            fclose( f );
        else
        {
            x264_log( h, X264_LOG_ERROR, "can't write to fdec.yuv\n" );
875
            goto fail;
876
877
878
        }
    }

879
880
881
882
883
884
    x264_log( h, X264_LOG_INFO, "profile %s, level %d.%d\n",
        h->sps->i_profile_idc == PROFILE_BASELINE ? "Baseline" :
        h->sps->i_profile_idc == PROFILE_MAIN ? "Main" :
        h->sps->i_profile_idc == PROFILE_HIGH ? "High" :
        "High 4:4:4 Predictive", h->sps->i_level_idc/10, h->sps->i_level_idc%10 );

Laurent Aimar's avatar
Laurent Aimar committed
885
    return h;
886
887
888
fail:
    x264_free( h );
    return NULL;
Laurent Aimar's avatar
Laurent Aimar committed
889
890
}

891
892
893
894
895
/****************************************************************************
 * x264_encoder_reconfig:
 ****************************************************************************/
int x264_encoder_reconfig( x264_t *h, x264_param_t *param )
{
896
    h = h->thread[h->i_thread_phase%h->param.i_threads];
897
    x264_set_aspect_ratio( h, param, 0 );
898
899
900
#define COPY(var) h->param.var = param->var
    COPY( i_frame_reference ); // but never uses more refs than initially specified
    COPY( i_bframe_bias );
Fiona Glaser's avatar
Fiona Glaser committed
901
    if( h->param.i_scenecut_threshold )
902
        COPY( i_scenecut_threshold ); // can't turn it on or off, only vary the threshold
903
904
905
906
907
908
    COPY( b_deblocking_filter );
    COPY( i_deblocking_filter_alphac0 );
    COPY( i_deblocking_filter_beta );
    COPY( analyse.intra );
    COPY( analyse.inter );
    COPY( analyse.i_direct_mv_pred );
Fiona Glaser's avatar
Fiona Glaser committed
909
910
911
    /* Scratch buffer prevents me_range from being increased for esa/tesa */
    if( h->param.analyse.i_me_method < X264_ME_ESA || param->analyse.i_me_range < h->param.analyse.i_me_range )
        COPY( analyse.i_me_range );
912
    COPY( analyse.i_noise_reduction );
913
914
915
    /* We can't switch out of subme=0 during encoding. */
    if( h->param.analyse.i_subpel_refine )
        COPY( analyse.i_subpel_refine );
916
917
918
919
920
    COPY( analyse.i_trellis );
    COPY( analyse.b_chroma_me );
    COPY( analyse.b_dct_decimate );
    COPY( analyse.b_fast_pskip );
    COPY( analyse.b_mixed_references );
921
922
    COPY( analyse.f_psy_rd );
    COPY( analyse.f_psy_trellis );
923
    // can only twiddle these if they were enabled to begin with:
924
925
    if( h->param.analyse.i_me_method >= X264_ME_ESA || param->analyse.i_me_method < X264_ME_ESA )
        COPY( analyse.i_me_method );
Loren Merritt's avatar
Loren Merritt committed
926
927
    if( h->param.analyse.i_me_method >= X264_ME_ESA && !h->frames.b_have_sub8x8_esa )
        h->param.analyse.inter &= ~X264_ANALYSE_PSUB8x8;
928
    if( h->pps->b_transform_8x8_mode )
929
930
931
932
        COPY( analyse.b_transform_8x8 );
    if( h->frames.i_max_ref1 > 1 )
        COPY( b_bframe_pyramid );
#undef COPY
Loren Merritt's avatar
Loren Merritt committed
933

934
    mbcmp_init( h );
Loren Merritt's avatar
Loren Merritt committed
935

936
937
938
    return x264_validate_parameters( h );
}

Laurent Aimar's avatar
Laurent Aimar committed
939
940
941
942
943
944
945
946
947
/* internal usage */
static void x264_nal_start( x264_t *h, int i_type, int i_ref_idc )
{
    x264_nal_t *nal = &h->out.nal[h->out.i_nal];

    nal->i_ref_idc = i_ref_idc;
    nal->i_type    = i_type;

    nal->i_payload= 0;
948
    nal->p_payload= &h->out.p_bitstream[bs_pos( &h->out.bs ) / 8];
Laurent Aimar's avatar
Laurent Aimar committed
949
950
951
952
}
static void x264_nal_end( x264_t *h )
{
    x264_nal_t *nal = &h->out.nal[h->out.i_nal];
953
    nal->i_payload = &h->out.p_bitstream[bs_pos( &h->out.bs ) / 8] - nal->p_payload;
Laurent Aimar's avatar
Laurent Aimar committed
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
    h->out.i_nal++;
}

/****************************************************************************
 * x264_encoder_headers:
 ****************************************************************************/
int x264_encoder_headers( x264_t *h, x264_nal_t **pp_nal, int *pi_nal )
{
    /* init bitstream context */
    h->out.i_nal = 0;
    bs_init( &h->out.bs, h->out.p_bitstream, h->out.i_bitstream );

    /* Put SPS and PPS */
    if( h->i_frame == 0 )
    {
969
970
        /* identify ourself */
        x264_nal_start( h, NAL_SEI, NAL_PRIORITY_DISPOSABLE );
971
972
        if( x264_sei_version_write( h, &h->out.bs ) )
            return -1;
973
974
        x264_nal_end( h );

Laurent Aimar's avatar
Laurent Aimar committed
975
976
977
978
979
980
981
982
983
984
985
986
987
        /* generate sequence parameters */
        x264_nal_start( h, NAL_SPS, NAL_PRIORITY_HIGHEST );
        x264_sps_write( &h->out.bs, h->sps );
        x264_nal_end( h );

        /* generate picture parameters */
        x264_nal_start( h, NAL_PPS, NAL_PRIORITY_HIGHEST );
        x264_pps_write( &h->out.bs, h->pps );
        x264_nal_end( h );
    }
    /* now set output*/
    *pi_nal = h->out.i_nal;
    *pp_nal = &h->out.nal[0];
988
    h->out.i_nal = 0;
Laurent Aimar's avatar
Laurent Aimar committed
989
990
991
992

    return 0;
}

Loren Merritt's avatar
Loren Merritt committed
993
static inline void x264_reference_build_list( x264_t *h, int i_poc )
Laurent Aimar's avatar
Laurent Aimar committed
994
995
996
997
998
999
1000
{
    int i;
    int b_ok;

    /* build ref list 0/1 */
    h->i_ref0 = 0;
    h->i_ref1 = 0;