videotoolbox.m 52.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
/*****************************************************************************
 * videotoolbox.m: Video Toolbox decoder
 *****************************************************************************
 * Copyright © 2014-2015 VideoLabs SAS
 *
 * Authors: Felix Paul Kühne <fkuehne # videolan.org>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#pragma mark preamble

#ifdef HAVE_CONFIG_H
# import "config.h"
#endif

#import <vlc_common.h>
#import <vlc_plugin.h>
#import <vlc_codec.h>
32
#import "hxxx_helper.h"
33
#import <vlc_bits.h>
34
#import <vlc_boxes.h>
35
36
37
38
39
#import "../packetizer/h264_nal.h"
#import "../packetizer/h264_slice.h"
#import "../packetizer/hxxx_nal.h"
#import "../packetizer/hxxx_sei.h"
#import "../video_chroma/copy.h"
40
41

#import <VideoToolbox/VideoToolbox.h>
42
#import <VideoToolbox/VTErrors.h>
43
44
45
46
47
48
49
50

#import <Foundation/Foundation.h>
#import <TargetConditionals.h>

#import <sys/types.h>
#import <sys/sysctl.h>
#import <mach/machine.h>

51
52
#if TARGET_OS_IPHONE
#import <UIKit/UIKit.h>
53
54
55
56
57
58

/* support iOS SDKs < v9.1 */
#ifndef CPUFAMILY_ARM_TWISTER
#define CPUFAMILY_ARM_TWISTER 0x92fb37c8
#endif

59
60
#endif

61
62
63
64
65
#pragma mark - module descriptor

static int OpenDecoder(vlc_object_t *);
static void CloseDecoder(vlc_object_t *);

66
#if MAC_OS_X_VERSION_MIN_ALLOWED < 1090
67
68
69
70
71
const CFStringRef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder = CFSTR("EnableHardwareAcceleratedVideoDecoder");
const CFStringRef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder = CFSTR("RequireHardwareAcceleratedVideoDecoder");
#endif

#define VT_REQUIRE_HW_DEC N_("Use Hardware decoders only")
72
73
#define VT_TEMPO_DEINTERLACE N_("Deinterlacing")
#define VT_TEMPO_DEINTERLACE_LONG N_("If interlaced content is detected, temporal deinterlacing is enabled at the expense of a pipeline delay.")
74
75
76
77
78
79
80

vlc_module_begin()
set_category(CAT_INPUT)
set_subcategory(SUBCAT_INPUT_VCODEC)
set_description(N_("VideoToolbox video decoder"))
set_capability("decoder",800)
set_callbacks(OpenDecoder, CloseDecoder)
81

82
add_bool("videotoolbox-temporal-deinterlacing", true, VT_TEMPO_DEINTERLACE, VT_TEMPO_DEINTERLACE_LONG, false)
83
84
85
86
87
add_bool("videotoolbox-hw-decoder-only", false, VT_REQUIRE_HW_DEC, VT_REQUIRE_HW_DEC, false)
vlc_module_end()

#pragma mark - local prototypes

88
static int ESDSCreate(decoder_t *, uint8_t *, uint32_t);
89
static int avcCFromAnnexBCreate(decoder_t *);
90
static int ExtradataInfoCreate(decoder_t *, CFStringRef, void *, size_t);
91
static int HandleVTStatus(decoder_t *, OSStatus);
92
static int DecodeBlock(decoder_t *, block_t *);
93
static void Flush(decoder_t *);
94
static void DecoderCallback(void *, void *, OSStatus, VTDecodeInfoFlags,
95
                            CVPixelBufferRef, CMTime, CMTime);
96
97
98
99
void VTDictionarySetInt32(CFMutableDictionaryRef, CFStringRef, int);
static void copy420YpCbCr8Planar(picture_t *, CVPixelBufferRef buffer,
                                 unsigned i_width, unsigned i_height);
static BOOL deviceSupportsAdvancedProfiles();
100
static BOOL deviceSupportsAdvancedLevels();
101

102
103
104
105
struct picture_sys_t {
    CFTypeRef pixelBuffer;
};

106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
typedef struct frame_info_t frame_info_t;

struct frame_info_t
{
    picture_t *p_picture;
    int i_poc;
    int i_foc;
    bool b_flush;
    bool b_field;
    bool b_progressive;
    uint8_t i_num_ts;
    unsigned i_length;
    frame_info_t *p_next;
};

121
122
#pragma mark - decoder structure

123
124
#define H264_MAX_DPB 16

125
126
127
struct decoder_sys_t
{
    CMVideoCodecType            codec;
128
    struct                      hxxx_helper hh;
129

130
131
    bool                        b_vt_feed;
    bool                        b_vt_flush;
132
133
    VTDecompressionSessionRef   session;
    CMVideoFormatDescriptionRef videoFormatDescription;
134
135
    CFMutableDictionaryRef      decoderConfiguration;
    CFMutableDictionaryRef      destinationPixelBufferAttributes;
136
    CFMutableDictionaryRef      extradataInfo;
137

138
    vlc_mutex_t                 lock;
139
140
141
    frame_info_t               *p_pic_reorder;
    uint8_t                     i_pic_reorder;
    uint8_t                     i_pic_reorder_max;
142
    bool                        b_invalid_pic_reorder_max;
143
    bool                        b_poc_based_reorder;
144
    bool                        b_enable_temporal_processing;
145
146

    bool                        b_format_propagated;
147
    bool                        b_abort;
148
149
150

    poc_context_t               pocctx;
    date_t                      pts;
151
152
153
154
};

#pragma mark - start & stop

155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
static void GetSPSPPS(uint8_t i_pps_id, void *priv,
                      const h264_sequence_parameter_set_t **pp_sps,
                      const h264_picture_parameter_set_t **pp_pps)
{
    decoder_sys_t *p_sys = priv;

    *pp_pps = p_sys->hh.h264.pps_list[i_pps_id].h264_pps;
    if(*pp_pps == NULL)
        *pp_sps = NULL;
    else
        *pp_sps = p_sys->hh.h264.sps_list[(*pp_pps)->i_sps_id].h264_sps;
}

struct sei_callback_s
{
    uint8_t i_pic_struct;
    const h264_sequence_parameter_set_t *p_sps;
};

static bool ParseH264SEI(const hxxx_sei_data_t *p_sei_data, void *priv)
{

    if(p_sei_data->i_type == HXXX_SEI_PIC_TIMING)
    {
        struct sei_callback_s *s = priv;
        if(s->p_sps && s->p_sps->vui.b_valid)
        {
            if(s->p_sps->vui.b_hrd_parameters_present_flag)
            {
                bs_read(p_sei_data->p_bs, s->p_sps->vui.i_cpb_removal_delay_length_minus1 + 1);
                bs_read(p_sei_data->p_bs, s->p_sps->vui.i_dpb_output_delay_length_minus1 + 1);
            }

            if(s->p_sps->vui.b_pic_struct_present_flag)
               s->i_pic_struct = bs_read( p_sei_data->p_bs, 4);
        }
        return false;
    }

    return true;
}

static bool ParseH264NAL(decoder_sys_t *p_sys,
                         const uint8_t *p_buffer, size_t i_buffer,
                         uint8_t i_nal_length_size, frame_info_t *p_info)
{
    hxxx_iterator_ctx_t itctx;
    hxxx_iterator_init(&itctx, p_buffer, i_buffer, i_nal_length_size);

    const uint8_t *p_nal; size_t i_nal;
    const uint8_t *p_sei_nal = NULL; size_t i_sei_nal = 0;
    while(hxxx_iterate_next(&itctx, &p_nal, &i_nal))
    {
        if(i_nal < 2)
            continue;

        const enum h264_nal_unit_type_e i_nal_type = p_nal[0] & 0x1F;

        if (i_nal_type <= H264_NAL_SLICE_IDR && i_nal_type != H264_NAL_UNKNOWN)
        {
            h264_slice_t slice;
            if(!h264_decode_slice(p_nal, i_nal, GetSPSPPS, p_sys, &slice))
                return false;

            const h264_sequence_parameter_set_t *p_sps;
            const h264_picture_parameter_set_t *p_pps;
            GetSPSPPS(slice.i_pic_parameter_set_id, p_sys, &p_sps, &p_pps);
            if(p_sps)
            {
224
225
226
227
228
229
230
231
232
                if(!p_sys->b_invalid_pic_reorder_max && i_nal_type == H264_NAL_SLICE_IDR)
                {
                    unsigned dummy;
                    uint8_t i_reorder;
                    h264_get_dpb_values(p_sps, &i_reorder, &dummy);
                    vlc_mutex_lock(&p_sys->lock);
                    p_sys->i_pic_reorder_max = i_reorder;
                    vlc_mutex_unlock(&p_sys->lock);
                }
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400

                int bFOC;
                h264_compute_poc(p_sps, &slice, &p_sys->pocctx,
                                 &p_info->i_poc, &p_info->i_foc, &bFOC);

                p_info->b_flush = (slice.type == H264_SLICE_TYPE_I) || slice.has_mmco5;
                p_info->b_field = slice.i_field_pic_flag;
                p_info->b_progressive = !p_sps->mb_adaptive_frame_field_flag &&
                                        !slice.i_field_pic_flag;

                struct sei_callback_s sei;
                sei.p_sps = p_sps;
                sei.i_pic_struct = UINT8_MAX;

                if(p_sei_nal)
                    HxxxParseSEI(p_sei_nal, i_sei_nal, 1, ParseH264SEI, &sei);

                p_info->i_num_ts = h264_get_num_ts(p_sps, &slice, sei.i_pic_struct,
                                                   p_info->i_foc, bFOC);
            }

            return true; /* No need to parse further NAL */
        }
        else if(i_nal_type == H264_NAL_SEI)
        {
            p_sei_nal = p_nal;
            i_sei_nal = i_nal;
        }
    }

    return false;
}

static void InsertIntoDPB(decoder_sys_t *p_sys, frame_info_t *p_info)
{
    frame_info_t **pp_lead_in = &p_sys->p_pic_reorder;

    for( ;; pp_lead_in = & ((*pp_lead_in)->p_next))
    {
        bool b_insert;
        if(*pp_lead_in == NULL)
            b_insert = true;
        else if(p_sys->b_poc_based_reorder)
            b_insert = ((*pp_lead_in)->i_foc > p_info->i_foc);
        else
            b_insert = ((*pp_lead_in)->p_picture->date >= p_info->p_picture->date);

        if(b_insert)
        {
            p_info->p_next = *pp_lead_in;
            *pp_lead_in = p_info;
            p_sys->i_pic_reorder += (p_info->b_field) ? 1 : 2;
            break;
        }
    }
#if 0
    for(frame_info_t *p_in=p_sys->p_pic_reorder; p_in; p_in = p_in->p_next)
        printf(" %d", p_in->i_foc);
    printf("\n");
#endif
}

static picture_t * RemoveOneFrameFromDPB(decoder_sys_t *p_sys)
{
    frame_info_t *p_info = p_sys->p_pic_reorder;
    if(p_info == NULL)
        return NULL;

    const int i_framepoc = p_info->i_poc;

    picture_t *p_ret = NULL;
    picture_t **pp_ret_last = &p_ret;
    bool b_dequeue;

    do
    {
        picture_t *p_field = p_info->p_picture;

        /* Compute time if missing */
        if (p_field->date == VLC_TS_INVALID)
            p_field->date = date_Get(&p_sys->pts);
        else
            date_Set(&p_sys->pts, p_field->date);

        /* Set next picture time, in case it is missing */
        if (p_info->i_length)
            date_Set(&p_sys->pts, p_field->date + p_info->i_length);
        else
            date_Increment(&p_sys->pts, p_info->i_num_ts);

        *pp_ret_last = p_field;
        pp_ret_last = &p_field->p_next;

        p_sys->i_pic_reorder -= (p_info->b_field) ? 1 : 2;

        p_sys->p_pic_reorder = p_info->p_next;
        free(p_info);
        p_info = p_sys->p_pic_reorder;

        if (p_info)
        {
            if (p_sys->b_poc_based_reorder)
                b_dequeue = (p_info->i_poc == i_framepoc);
            else
                b_dequeue = (p_field->date == p_info->p_picture->date);
        }
        else b_dequeue = false;

    } while(b_dequeue);

    return p_ret;
}

static void FlushDPB(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;
    for( ;; )
    {
        picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
        if (p_fields == NULL)
            break;
        do
        {
            picture_t *p_next = p_fields->p_next;
            p_fields->p_next = NULL;
            decoder_QueueVideo(p_dec, p_fields);
            p_fields = p_next;
        } while(p_fields != NULL);
    }
}

static frame_info_t * CreateReorderInfo(decoder_sys_t *p_sys, const block_t *p_block)
{
    frame_info_t *p_info = calloc(1, sizeof(*p_info));
    if (!p_info)
        return NULL;

    if (p_sys->b_poc_based_reorder)
    {
        if(p_sys->codec != kCMVideoCodecType_H264 ||
           !ParseH264NAL(p_sys, p_block->p_buffer, p_block->i_buffer, 4, p_info))
        {
            assert(p_sys->codec == kCMVideoCodecType_H264);
            free(p_info);
            return NULL;
        }
    }
    else
    {
        p_info->i_num_ts = 2;
        p_info->b_progressive = true;
        p_info->b_field = false;
    }

    p_info->i_length = p_block->i_length;

    if (date_Get(&p_sys->pts) == VLC_TS_INVALID)
        date_Set(&p_sys->pts, p_block->i_dts);

    return p_info;
}

static void OnDecodedFrame(decoder_t *p_dec, frame_info_t *p_info)
{
    decoder_sys_t *p_sys = p_dec->p_sys;
    assert(p_info->p_picture);
    while(p_info->b_flush || p_sys->i_pic_reorder == (p_sys->i_pic_reorder_max * 2))
    {
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
        /* First check if DPB sizing was correct before removing one frame */
        if (p_sys->p_pic_reorder && !p_info->b_flush &&
            p_sys->i_pic_reorder_max < H264_MAX_DPB)
        {
            if(p_sys->b_poc_based_reorder && p_sys->p_pic_reorder->i_foc > p_info->i_foc)
            {
                p_sys->b_invalid_pic_reorder_max = true;
                p_sys->i_pic_reorder_max++;
                msg_Info(p_dec, "Raising max DPB to %"PRIu8, p_sys->i_pic_reorder_max);
                break;
            }
            else if (!p_sys->b_poc_based_reorder &&
                     p_info->p_picture->date > VLC_TS_INVALID &&
                     p_sys->p_pic_reorder->p_picture->date > p_info->p_picture->date)
            {
                p_sys->b_invalid_pic_reorder_max = true;
                p_sys->i_pic_reorder_max++;
                msg_Info(p_dec, "Raising max DPB to %"PRIu8, p_sys->i_pic_reorder_max);
                break;
            }
        }

423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
        picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
        if (p_fields == NULL)
            break;
        do
        {
            picture_t *p_next = p_fields->p_next;
            p_fields->p_next = NULL;
            decoder_QueueVideo(p_dec, p_fields);
            p_fields = p_next;
        } while(p_fields != NULL);
    }

    InsertIntoDPB(p_sys, p_info);
}

438
439
static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
{
440
    uint8_t i_profile = 0xFF, i_level = 0xFF;
441
442
443
444
445
446
447
448
449
450
451
452
453
454
    bool b_ret = false;
    CMVideoCodecType codec;

    /* check for the codec we can and want to decode */
    switch (p_dec->fmt_in.i_codec) {
        case VLC_CODEC_H264:
            codec = kCMVideoCodecType_H264;

            b_ret = h264_get_profile_level(&p_dec->fmt_in, &i_profile, &i_level, NULL);
            if (!b_ret) {
                msg_Warn(p_dec, "H264 profile and level parsing failed because it didn't arrive yet");
                return kCMVideoCodecType_H264;
            }

455
            msg_Dbg(p_dec, "trying to decode MPEG-4 Part 10: profile %" PRIx8 ", level %" PRIx8, i_profile, i_level);
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470

            switch (i_profile) {
                case PROFILE_H264_BASELINE:
                case PROFILE_H264_MAIN:
                case PROFILE_H264_HIGH:
                    break;

                case PROFILE_H264_HIGH_10:
                {
                    if (deviceSupportsAdvancedProfiles())
                        break;
                }

                default:
                {
471
                    msg_Dbg(p_dec, "unsupported H264 profile %" PRIx8, i_profile);
472
473
474
475
476
477
478
                    return -1;
                }
            }

#if !TARGET_OS_IPHONE
            /* a level higher than 5.2 was not tested, so don't dare to
             * try to decode it*/
479
            if (i_level > 52) {
480
                msg_Dbg(p_dec, "unsupported H264 level %" PRIx8, i_level);
481
                return -1;
482
            }
483
484
#else
            /* on SoC A8, 4.2 is the highest specified profile */
485
486
487
            if (i_level > 42) {
                /* on Twister, we can do up to 5.2 */
                if (!deviceSupportsAdvancedLevels() || i_level > 52) {
488
                    msg_Dbg(p_dec, "unsupported H264 level %" PRIx8, i_level);
489
490
491
                    return -1;
                }
            }
492
493
494
495
#endif

            break;
        case VLC_CODEC_MP4V:
496
497
498
499
500
501
502
        {
            if (p_dec->fmt_in.i_original_fourcc == VLC_FOURCC( 'X','V','I','D' )) {
                msg_Warn(p_dec, "XVID decoding not implemented, fallback on software");
                return -1;
            }

            msg_Dbg(p_dec, "Will decode MP4V with original FourCC '%4.4s'", (char *)&p_dec->fmt_in.i_original_fourcc);
503
            codec = kCMVideoCodecType_MPEG4Video;
504

505
            break;
506
        }
507
#if !TARGET_OS_IPHONE
508
509
510
511
        case VLC_CODEC_H263:
            codec = kCMVideoCodecType_H263;
            break;

512
            /* there are no DV or ProRes decoders on iOS, so bailout early */
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
        case VLC_CODEC_PRORES:
            /* the VT decoder can't differenciate between the ProRes flavors, so we do it */
            switch (p_dec->fmt_in.i_original_fourcc) {
                case VLC_FOURCC( 'a','p','4','c' ):
                case VLC_FOURCC( 'a','p','4','h' ):
                    codec = kCMVideoCodecType_AppleProRes4444;
                    break;

                case VLC_FOURCC( 'a','p','c','h' ):
                    codec = kCMVideoCodecType_AppleProRes422HQ;
                    break;

                case VLC_FOURCC( 'a','p','c','s' ):
                    codec = kCMVideoCodecType_AppleProRes422LT;
                    break;

                case VLC_FOURCC( 'a','p','c','o' ):
                    codec = kCMVideoCodecType_AppleProRes422Proxy;
                    break;

                default:
                    codec = kCMVideoCodecType_AppleProRes422;
                    break;
            }
            if (codec != 0)
                break;

540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
        case VLC_CODEC_DV:
            /* the VT decoder can't differenciate between PAL and NTSC, so we need to do it */
            switch (p_dec->fmt_in.i_original_fourcc) {
                case VLC_FOURCC( 'd', 'v', 'c', ' '):
                case VLC_FOURCC( 'd', 'v', ' ', ' '):
                    msg_Dbg(p_dec, "Decoding DV NTSC");
                    codec = kCMVideoCodecType_DVCNTSC;
                    break;

                case VLC_FOURCC( 'd', 'v', 's', 'd'):
                case VLC_FOURCC( 'd', 'v', 'c', 'p'):
                case VLC_FOURCC( 'D', 'V', 'S', 'D'):
                    msg_Dbg(p_dec, "Decoding DV PAL");
                    codec = kCMVideoCodecType_DVCPAL;
                    break;

                default:
                    break;
            }
            if (codec != 0)
                break;
#endif
562
563
            /* mpgv / mp2v needs fixing, so disable it for now */
#if 0
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
        case VLC_CODEC_MPGV:
            codec = kCMVideoCodecType_MPEG1Video;
            break;
        case VLC_CODEC_MP2V:
            codec = kCMVideoCodecType_MPEG2Video;
            break;
#endif

        default:
#ifndef NDEBUG
            msg_Err(p_dec, "'%4.4s' is not supported", (char *)&p_dec->fmt_in.i_codec);
#endif
            return -1;
    }

    return codec;
}

582
static int StartVideoToolbox(decoder_t *p_dec)
583
584
585
586
{
    decoder_sys_t *p_sys = p_dec->p_sys;
    OSStatus status;

587
588
589
590
591
592
593
594
595
    assert(p_sys->extradataInfo != nil);

    p_sys->decoderConfiguration =
        CFDictionaryCreateMutable(kCFAllocatorDefault, 2,
                                  &kCFTypeDictionaryKeyCallBacks,
                                  &kCFTypeDictionaryValueCallBacks);
    if (p_sys->decoderConfiguration == NULL)
        return VLC_ENOMEM;

596
    CFDictionarySetValue(p_sys->decoderConfiguration,
597
598
                         kCVImageBufferChromaLocationBottomFieldKey,
                         kCVImageBufferChromaLocation_Left);
599
    CFDictionarySetValue(p_sys->decoderConfiguration,
600
601
602
                         kCVImageBufferChromaLocationTopFieldKey,
                         kCVImageBufferChromaLocation_Left);

603
604
605
    CFDictionarySetValue(p_sys->decoderConfiguration,
                         kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
                         p_sys->extradataInfo);
606
607

    /* pixel aspect ratio */
608
609
610
611
612
613
614
615
616
    CFMutableDictionaryRef pixelaspectratio =
        CFDictionaryCreateMutable(kCFAllocatorDefault, 2,
                                  &kCFTypeDictionaryKeyCallBacks,
                                  &kCFTypeDictionaryValueCallBacks);

    const unsigned i_video_width = p_dec->fmt_out.video.i_width;
    const unsigned i_video_height = p_dec->fmt_out.video.i_height;
    const unsigned i_sar_num = p_dec->fmt_out.video.i_sar_num;
    const unsigned i_sar_den = p_dec->fmt_out.video.i_sar_den;
617

618
619
620

    date_Init( &p_sys->pts, p_dec->fmt_in.video.i_frame_rate * 2, p_dec->fmt_in.video.i_frame_rate_base );

621
622
623
624
625
626
    VTDictionarySetInt32(pixelaspectratio,
                         kCVImageBufferPixelAspectRatioHorizontalSpacingKey,
                         i_sar_num);
    VTDictionarySetInt32(pixelaspectratio,
                         kCVImageBufferPixelAspectRatioVerticalSpacingKey,
                         i_sar_den);
627
    CFDictionarySetValue(p_sys->decoderConfiguration,
628
629
630
631
632
633
634
                         kCVImageBufferPixelAspectRatioKey,
                         pixelaspectratio);
    CFRelease(pixelaspectratio);

    /* enable HW accelerated playback, since this is optional on OS X
     * note that the backend may still fallback on software mode if no
     * suitable hardware is available */
635
    CFDictionarySetValue(p_sys->decoderConfiguration,
636
637
638
639
640
641
                         kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
                         kCFBooleanTrue);

    /* on OS X, we can force VT to fail if no suitable HW decoder is available,
     * preventing the aforementioned SW fallback */
    if (var_InheritInteger(p_dec, "videotoolbox-hw-decoder-only"))
642
        CFDictionarySetValue(p_sys->decoderConfiguration,
643
644
645
                             kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
                             kCFBooleanTrue);

646
647
648
649
650
651
652
653
654
    if (p_sys->b_enable_temporal_processing)
    {
        msg_Dbg(p_dec, "Interlaced content detected, inserting temporal deinterlacer");
        CFDictionarySetValue(p_sys->decoderConfiguration,
                             kVTDecompressionPropertyKey_FieldMode,
                             kVTDecompressionProperty_FieldMode_DeinterlaceFields);
        CFDictionarySetValue(p_sys->decoderConfiguration,
                             kVTDecompressionPropertyKey_DeinterlaceMode,
                             kVTDecompressionProperty_DeinterlaceMode_Temporal);
655
656
    }

657
658
659
660
661
    /* create video format description */
    status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
                                            p_sys->codec,
                                            i_video_width,
                                            i_video_height,
662
                                            p_sys->decoderConfiguration,
663
664
                                            &p_sys->videoFormatDescription);
    if (status) {
665
        CFRelease(p_sys->decoderConfiguration);
666
        msg_Err(p_dec, "video format description creation failed (%i)", (int)status);
667
668
669
670
        return VLC_EGENERIC;
    }

    /* destination pixel buffer attributes */
671
672
673
674
    p_sys->destinationPixelBufferAttributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
                                                                        2,
                                                                        &kCFTypeDictionaryKeyCallBacks,
                                                                        &kCFTypeDictionaryValueCallBacks);
675

676
#if !TARGET_OS_IPHONE
677
    CFDictionarySetValue(p_sys->destinationPixelBufferAttributes,
678
                         kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey,
679
                         kCFBooleanTrue);
680
#else
681
    CFDictionarySetValue(p_sys->destinationPixelBufferAttributes,
682
                         kCVPixelBufferOpenGLESCompatibilityKey,
683
                         kCFBooleanTrue);
684
#endif
685

686
    VTDictionarySetInt32(p_sys->destinationPixelBufferAttributes,
687
688
                         kCVPixelBufferWidthKey,
                         i_video_width);
689
    VTDictionarySetInt32(p_sys->destinationPixelBufferAttributes,
690
691
                         kCVPixelBufferHeightKey,
                         i_video_height);
692
    VTDictionarySetInt32(p_sys->destinationPixelBufferAttributes,
693
694
695
                         kCVPixelBufferBytesPerRowAlignmentKey,
                         i_video_width * 2);

696
697
698
699
    /* setup decoder callback record */
    VTDecompressionOutputCallbackRecord decoderCallbackRecord;
    decoderCallbackRecord.decompressionOutputCallback = DecoderCallback;
    decoderCallbackRecord.decompressionOutputRefCon = p_dec;
700

701
702
703
704
705
706
    /* create decompression session */
    status = VTDecompressionSessionCreate(kCFAllocatorDefault,
                                          p_sys->videoFormatDescription,
                                          p_sys->decoderConfiguration,
                                          p_sys->destinationPixelBufferAttributes,
                                          &decoderCallbackRecord, &p_sys->session);
707

708
709
    if (HandleVTStatus(p_dec, status) != VLC_SUCCESS)
        return VLC_EGENERIC;
710

711
    return VLC_SUCCESS;
712
713
}

714
static void StopVideoToolbox(decoder_t *p_dec, bool b_reset_format)
715
716
717
{
    decoder_sys_t *p_sys = p_dec->p_sys;

718
    if (p_sys->session != nil)
719
720
721
722
723
724
725
726
727
728
    {
        VTDecompressionSessionInvalidate(p_sys->session);
        CFRelease(p_sys->session);
        p_sys->session = nil;

        if (b_reset_format)
        {
            p_sys->b_format_propagated = false;
            p_dec->fmt_out.i_codec = 0;
        }
729
        FlushDPB( p_dec );
730
    }
731

732
    if (p_sys->videoFormatDescription != nil) {
733
        CFRelease(p_sys->videoFormatDescription);
734
735
        p_sys->videoFormatDescription = nil;
    }
736
737
738
739
740
741
742
743
    if (p_sys->decoderConfiguration != nil) {
        CFRelease(p_sys->decoderConfiguration);
        p_sys->decoderConfiguration = nil;
    }
    if (p_sys->destinationPixelBufferAttributes != nil) {
        CFRelease(p_sys->destinationPixelBufferAttributes);
        p_sys->destinationPixelBufferAttributes = nil;
    }
744
745
}

746
static int RestartVideoToolbox(decoder_t *p_dec, bool b_reset_format)
747
748
749
750
751
{
    decoder_sys_t *p_sys = p_dec->p_sys;

    msg_Dbg(p_dec, "Restarting decoder session");

752
    if (p_sys->session != nil)
753
        StopVideoToolbox(p_dec, b_reset_format);
754

755
    return StartVideoToolbox(p_dec);
756
757
}

758
759
#pragma mark - module open and close

760
761
762
763
764
765
766
static int SetupDecoderExtradata(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;
    CFMutableDictionaryRef extradata_info = NULL;

    if (p_sys->codec == kCMVideoCodecType_H264)
    {
767
768
769
770
771
772
773
        hxxx_helper_init(&p_sys->hh, VLC_OBJECT(p_dec),
                         p_dec->fmt_in.i_codec, true);
        int i_ret = hxxx_helper_set_extra(&p_sys->hh, p_dec->fmt_in.p_extra,
                                          p_dec->fmt_in.i_extra);
        if (i_ret != VLC_SUCCESS)
            return i_ret;

774
775
776
777
778
779
780
        if (p_dec->fmt_in.p_extra)
        {
            int i_ret = ExtradataInfoCreate(p_dec, CFSTR("avcC"),
                                            p_dec->fmt_in.p_extra,
                                            p_dec->fmt_in.i_extra);
            if (i_ret != VLC_SUCCESS)
                return i_ret;
781
782
        }
        /* else: AnnexB case, we'll get extradata from first input blocks */
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
    }
    else if (p_sys->codec == kCMVideoCodecType_MPEG4Video)
    {
        if (!p_dec->fmt_in.i_extra)
            return VLC_EGENERIC;
        int i_ret = ESDSCreate(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra);
        if (i_ret != VLC_SUCCESS)
            return i_ret;
    }
    else
    {
        int i_ret = ExtradataInfoCreate(p_dec, NULL, NULL, 0);
        if (i_ret != VLC_SUCCESS)
            return i_ret;
    }

    return VLC_SUCCESS;
}

802
803
804
static int OpenDecoder(vlc_object_t *p_this)
{
    decoder_t *p_dec = (decoder_t *)p_this;
805
806
807
808
809
810
811

#if TARGET_OS_IPHONE
    if (unlikely([[UIDevice currentDevice].systemVersion floatValue] < 8.0)) {
        msg_Warn(p_dec, "decoder skipped as OS is too old");
        return VLC_EGENERIC;
    }
#endif
812

813
814
815
    if (p_dec->fmt_in.i_cat != VIDEO_ES)
        return VLC_EGENERIC;

816
817
818
819
    /* Fail if this module already failed to decode this ES */
    if (var_Type(p_dec, "videotoolbox-failed") != 0)
        return VLC_EGENERIC;

820
    /* check quickly if we can digest the offered data */
821
    CMVideoCodecType codec;
822
    codec = CodecPrecheck(p_dec);
823
    if (codec == -1)
824
825
826
827
828
829
830
831
832
        return VLC_EGENERIC;

    /* now that we see a chance to decode anything, allocate the
     * internals and start the decoding session */
    decoder_sys_t *p_sys;
    p_sys = malloc(sizeof(*p_sys));
    if (!p_sys)
        return VLC_ENOMEM;
    p_dec->p_sys = p_sys;
833
    p_sys->session = nil;
834
835
    p_sys->b_vt_feed = false;
    p_sys->b_vt_flush = false;
836
    p_sys->codec = codec;
837
    p_sys->videoFormatDescription = nil;
838
839
    p_sys->decoderConfiguration = nil;
    p_sys->destinationPixelBufferAttributes = nil;
840
    p_sys->extradataInfo = nil;
841
842
    p_sys->p_pic_reorder = NULL;
    p_sys->i_pic_reorder = 0;
843
    p_sys->i_pic_reorder_max = 4;
844
    p_sys->b_invalid_pic_reorder_max = false;
845
    p_sys->b_poc_based_reorder = false;
846
    p_sys->b_format_propagated = false;
847
    p_sys->b_abort = false;
848
    p_sys->b_enable_temporal_processing = false;
849
    h264_poc_context_init( &p_sys->pocctx );
850
    vlc_mutex_init(&p_sys->lock);
851

852
853
854
    /* return our proper VLC internal state */
    p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
    p_dec->fmt_out.video = p_dec->fmt_in.video;
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
    if (!p_dec->fmt_out.video.i_sar_num || !p_dec->fmt_out.video.i_sar_den)
    {
        p_dec->fmt_out.video.i_sar_num = 1;
        p_dec->fmt_out.video.i_sar_den = 1;
    }
    if (!p_dec->fmt_out.video.i_visible_width
     || !p_dec->fmt_out.video.i_visible_height)
    {
        p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
        p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
    }
    else
    {
        p_dec->fmt_out.video.i_width = p_dec->fmt_out.video.i_visible_width;
        p_dec->fmt_out.video.i_height = p_dec->fmt_out.video.i_visible_height;
    }
871
872
873

    p_dec->fmt_out.i_codec = 0;

874
875
876
    if( codec == kCMVideoCodecType_H264 )
        p_sys->b_poc_based_reorder = true;

877
878
879
880
881
882
883
884
885
886
    int i_ret = SetupDecoderExtradata(p_dec);
    if (i_ret != VLC_SUCCESS)
        goto error;

    if (p_sys->extradataInfo != nil)
    {
        i_ret = StartVideoToolbox(p_dec);
        if (i_ret != VLC_SUCCESS)
            goto error;
    } /* else: late opening */
887

888
889
    p_dec->pf_decode = DecodeBlock;
    p_dec->pf_flush  = Flush;
890
891
892
893

    msg_Info(p_dec, "Using Video Toolbox to decode '%4.4s'", (char *)&p_dec->fmt_in.i_codec);

    return VLC_SUCCESS;
894
895
896
897

error:
    CloseDecoder(p_this);
    return i_ret;
898
899
900
901
902
903
904
}

static void CloseDecoder(vlc_object_t *p_this)
{
    decoder_t *p_dec = (decoder_t *)p_this;
    decoder_sys_t *p_sys = p_dec->p_sys;

905
    StopVideoToolbox(p_dec, true);
906

907
908
909
    if (p_sys->codec == kCMVideoCodecType_H264)
        hxxx_helper_clean(&p_sys->hh);

910
    vlc_mutex_destroy(&p_sys->lock);
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
    free(p_sys);
}

#pragma mark - helpers

static BOOL deviceSupportsAdvancedProfiles()
{
#if TARGET_IPHONE_SIMULATOR
    return NO;
#endif
#if TARGET_OS_IPHONE
    size_t size;
    cpu_type_t type;

    size = sizeof(type);
    sysctlbyname("hw.cputype", &type, &size, NULL, 0);

    /* Support for H264 profile HIGH 10 was introduced with the first 64bit Apple ARM SoC, the A7 */
    if (type == CPU_TYPE_ARM64)
        return YES;

    return NO;
#else
    return NO;
#endif
}

938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
static BOOL deviceSupportsAdvancedLevels()
{
#if TARGET_IPHONE_SIMULATOR
    return YES;
#endif
#if TARGET_OS_IPHONE
    size_t size;
    int32_t cpufamily;

    size = sizeof(cpufamily);
    sysctlbyname("hw.cpufamily", &cpufamily, &size, NULL, 0);

    /* Proper 4K decoding requires a Twister SoC
     * Everything below will kill the decoder daemon */
    if (cpufamily == CPUFAMILY_ARM_TWISTER) {
        return YES;
    }

    return NO;
#else
    return YES;
#endif
}

962
963
964
965
966
967
968
969
static inline void bo_add_mp4_tag_descr(bo_t *p_bo, uint8_t tag, uint32_t size)
{
    bo_add_8(p_bo, tag);
    for (int i = 3; i>0; i--)
        bo_add_8(p_bo, (size>>(7*i)) | 0x80);
    bo_add_8(p_bo, size & 0x7F);
}

970
static int ESDSCreate(decoder_t *p_dec, uint8_t *p_buf, uint32_t i_buf_size)
971
972
973
974
975
976
977
978
{
    int full_size = 3 + 5 +13 + 5 + i_buf_size + 3;
    int config_size = 13 + 5 + i_buf_size;
    int padding = 12;

    bo_t bo;
    bool status = bo_init(&bo, 1024);
    if (status != true)
979
        return VLC_EGENERIC;
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005

    bo_add_8(&bo, 0);       // Version
    bo_add_24be(&bo, 0);    // Flags

    // elementary stream description tag
    bo_add_mp4_tag_descr(&bo, 0x03, full_size);
    bo_add_16be(&bo, 0);    // esid
    bo_add_8(&bo, 0);       // stream priority (0-3)

    // decoder configuration description tag
    bo_add_mp4_tag_descr(&bo, 0x04, config_size);
    bo_add_8(&bo, 32);      // object type identification (32 == MPEG4)
    bo_add_8(&bo, 0x11);    // stream type
    bo_add_24be(&bo, 0);    // buffer size
    bo_add_32be(&bo, 0);    // max bitrate
    bo_add_32be(&bo, 0);    // avg bitrate

    // decoder specific description tag
    bo_add_mp4_tag_descr(&bo, 0x05, i_buf_size);
    bo_add_mem(&bo, i_buf_size, p_buf);

    // sync layer configuration description tag
    bo_add_8(&bo, 0x06);    // tag
    bo_add_8(&bo, 0x01);    // length
    bo_add_8(&bo, 0x02);    // no SL

1006
1007
    int i_ret = ExtradataInfoCreate(p_dec, CFSTR("esds"), bo.b->p_buffer,
                                    bo.b->i_buffer);
Thomas Guillem's avatar
Thomas Guillem committed
1008
    bo_deinit(&bo);
1009
1010
1011
    return i_ret;
}

1012
static int avcCFromAnnexBCreate(decoder_t *p_dec)
1013
1014
1015
{
    decoder_sys_t *p_sys = p_dec->p_sys;

1016
    if (p_sys->hh.h264.i_sps_count == 0 || p_sys->hh.h264.i_pps_count == 0)
1017
1018
        return VLC_EGENERIC;

1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
    unsigned i_h264_width, i_h264_height, i_video_width, i_video_height;
    int i_sar_num, i_sar_den, i_ret;
    i_ret = h264_helper_get_current_picture_size(&p_sys->hh,
                                                 &i_h264_width, &i_h264_height,
                                                 &i_video_width, &i_video_height);
    if (i_ret != VLC_SUCCESS)
        return i_ret;
    i_ret = h264_helper_get_current_sar(&p_sys->hh, &i_sar_num, &i_sar_den);
    if (i_ret != VLC_SUCCESS)
        return i_ret;
1029
1030
1031
1032
1033

    p_dec->fmt_out.video.i_visible_width =
    p_dec->fmt_out.video.i_width = i_video_width;
    p_dec->fmt_out.video.i_visible_height =
    p_dec->fmt_out.video.i_height = i_video_height;
1034
1035
    p_dec->fmt_out.video.i_sar_num = i_sar_num;
    p_dec->fmt_out.video.i_sar_den = i_sar_den;
1036

1037
    block_t *p_avcC = h264_helper_get_avcc_config(&p_sys->hh);
1038
1039
1040
    if (!p_avcC)
        return VLC_EGENERIC;

1041
1042
    i_ret = ExtradataInfoCreate(p_dec, CFSTR("avcC"), p_avcC->p_buffer,
                                p_avcC->i_buffer);
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
    block_Release(p_avcC);
    return i_ret;
}

static int ExtradataInfoCreate(decoder_t *p_dec, CFStringRef name, void *p_data,
                               size_t i_data)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

    p_sys->extradataInfo =
        CFDictionaryCreateMutable(kCFAllocatorDefault, 1,
                                  &kCFTypeDictionaryKeyCallBacks,
                                  &kCFTypeDictionaryValueCallBacks);
    if (p_sys->extradataInfo == nil)
        return VLC_EGENERIC;

    if (p_data == NULL)
        return VLC_SUCCESS;

    CFDataRef extradata = CFDataCreate(kCFAllocatorDefault, p_data, i_data);
    if (extradata == nil)
    {
        CFRelease(p_sys->extradataInfo);
        p_sys->extradataInfo = nil;
        return VLC_EGENERIC;
    }
    CFDictionarySetValue(p_sys->extradataInfo, name, extradata);
    CFRelease(extradata);
    return VLC_SUCCESS;
1072
1073
1074
1075
}

static CMSampleBufferRef VTSampleBufferCreate(decoder_t *p_dec,
                                              CMFormatDescriptionRef fmt_desc,
Thomas Guillem's avatar
Thomas Guillem committed
1076
                                              block_t *p_block)
1077
1078
1079
1080
{
    OSStatus status;
    CMBlockBufferRef  block_buf = NULL;
    CMSampleBufferRef sample_buf = NULL;
1081
1082
1083
1084
1085
    CMTime pts;
    if(!p_dec->p_sys->b_poc_based_reorder && p_block->i_pts == VLC_TS_INVALID)
        pts = CMTimeMake(p_block->i_dts, CLOCK_FREQ);
    else
        pts = CMTimeMake(p_block->i_pts, CLOCK_FREQ);
1086

Thomas Guillem's avatar
Thomas Guillem committed
1087
1088
    CMSampleTimingInfo timeInfoArray[1] = { {
        .duration = CMTimeMake(p_block->i_length, 1),
1089
        .presentationTimeStamp = pts,
Thomas Guillem's avatar
Thomas Guillem committed
1090
1091
        .decodeTimeStamp = CMTimeMake(p_block->i_dts, CLOCK_FREQ),
    } };
1092
1093

    status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
Thomas Guillem's avatar
Thomas Guillem committed
1094
1095
                                                p_block->p_buffer,  // memoryBlock
                                                p_block->i_buffer,  // blockLength
1096
1097
1098
                                                kCFAllocatorNull,   // blockAllocator
                                                NULL,               // customBlockSource
                                                0,                  // offsetToData
Thomas Guillem's avatar
Thomas Guillem committed
1099
                                                p_block->i_buffer,  // dataLength
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
                                                false,              // flags
                                                &block_buf);

    if (!status) {
        status = CMSampleBufferCreate(kCFAllocatorDefault,  // allocator
                                      block_buf,            // dataBuffer
                                      TRUE,                 // dataReady
                                      0,                    // makeDataReadyCallback
                                      0,                    // makeDataReadyRefcon
                                      fmt_desc,             // formatDescription
                                      1,                    // numSamples
                                      1,                    // numSampleTimingEntries
                                      timeInfoArray,        // sampleTimingArray
                                      0,                    // numSampleSizeEntries
                                      NULL,                 // sampleSizeArray
                                      &sample_buf);
        if (status != noErr)
1117
            msg_Warn(p_dec, "sample buffer creation failure %i", (int)status);
1118
    } else
1119
        msg_Warn(p_dec, "cm block buffer creation failure %i", (int)status);
1120

1121
    if (block_buf != nil)
1122
        CFRelease(block_buf);
1123
    block_buf = nil;
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143

    return sample_buf;
}

void VTDictionarySetInt32(CFMutableDictionaryRef dict, CFStringRef key, int value)
{
    CFNumberRef number;
    number = CFNumberCreate(NULL, kCFNumberSInt32Type, &value);
    CFDictionarySetValue(dict, key, number);
    CFRelease(number);
}

static void copy420YpCbCr8Planar(picture_t *p_pic,
                                 CVPixelBufferRef buffer,
                                 unsigned i_width,
                                 unsigned i_height)
{
    uint8_t *pp_plane[2];
    size_t pi_pitch[2];

1144
    if (!buffer || i_width == 0 || i_height == 0)
1145
1146
        return;

1147
    CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
1148
1149
1150
1151
1152
1153

    for (int i = 0; i < 2; i++) {
        pp_plane[i] = CVPixelBufferGetBaseAddressOfPlane(buffer, i);
        pi_pitch[i] = CVPixelBufferGetBytesPerRowOfPlane(buffer, i);
    }

1154
    CopyFromNv12ToI420(p_pic, pp_plane, pi_pitch, i_height);
1155

1156
    CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
1157
1158
}

1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
static int HandleVTStatus(decoder_t *p_dec, OSStatus status)
{
#define VTERRCASE(x) \
    case x: msg_Err(p_dec, "vt session error: '" #x "'"); break;

    switch (status)
    {
        case 0:
            return VLC_SUCCESS;

        VTERRCASE(kVTPropertyNotSupportedErr)
        VTERRCASE(kVTPropertyReadOnlyErr)
        VTERRCASE(kVTParameterErr)
        VTERRCASE(kVTInvalidSessionErr)
        VTERRCASE(kVTAllocationFailedErr)
        VTERRCASE(kVTPixelTransferNotSupportedErr)
        VTERRCASE(kVTCouldNotFindVideoDecoderErr)
        VTERRCASE(kVTCouldNotCreateInstanceErr)
        VTERRCASE(kVTCouldNotFindVideoEncoderErr)
        VTERRCASE(kVTVideoDecoderBadDataErr)
        VTERRCASE(kVTVideoDecoderUnsupportedDataFormatErr)
        VTERRCASE(kVTVideoDecoderMalfunctionErr)
        VTERRCASE(kVTVideoEncoderMalfunctionErr)
        VTERRCASE(kVTVideoDecoderNotAvailableNowErr)
        VTERRCASE(kVTImageRotationNotSupportedErr)
        VTERRCASE(kVTVideoEncoderNotAvailableNowErr)
        VTERRCASE(kVTFormatDescriptionChangeNotSupportedErr)
        VTERRCASE(kVTInsufficientSourceColorDataErr)
        VTERRCASE(kVTCouldNotCreateColorCorrectionDataErr)
        VTERRCASE(kVTColorSyncTransformConvertFailedErr)
        VTERRCASE(kVTVideoDecoderAuthorizationErr)
        VTERRCASE(kVTVideoEncoderAuthorizationErr)
        VTERRCASE(kVTColorCorrectionPixelTransferFailedErr)
        VTERRCASE(kVTMultiPassStorageIdentifierMismatchErr)
        VTERRCASE(kVTMultiPassStorageInvalidErr)
        VTERRCASE(kVTFrameSiloInvalidTimeStampErr)
        VTERRCASE(kVTFrameSiloInvalidTimeRangeErr)
        VTERRCASE(kVTCouldNotFindTemporalFilterErr)
        VTERRCASE(kVTPixelTransferNotPermittedErr)
Thomas Guillem's avatar
Thomas Guillem committed
1198
1199
1200
1201
        case -12219:
            msg_Err(p_dec, "vt session error: "
                    "'kVTColorCorrectionImageRotationFailedErr'");
            break;
1202
1203
1204
1205
1206
1207
1208
        default:
            msg_Err(p_dec, "unknown vt session error (%i)", (int)status);
    }
#undef VTERRCASE
    return VLC_EGENERIC;
}

1209
1210
#pragma mark - actual decoding

1211
1212
1213
1214
static void Flush(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

1215
1216
1217
1218
    /* There is no Flush in VT api, ask to restart VT from next DecodeBlock if
     * we already feed some input blocks (it's better to not restart here in
     * order to avoid useless restart just before a close). */
    p_sys->b_vt_flush = p_sys->b_vt_feed;
1219
1220
}

1221
1222
1223
1224
1225
1226
1227
1228
static void Drain(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

    /* draining: return last pictures of the reordered queue */
    if (p_sys->session)
        VTDecompressionSessionWaitForAsynchronousFrames(p_sys->session);

1229
1230
1231
    vlc_mutex_lock(&p_sys->lock);
    FlushDPB( p_dec );
    vlc_mutex_unlock(&p_sys->lock);
1232
1233
}

1234
static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
1235
1236
{
    decoder_sys_t *p_sys = p_dec->p_sys;
1237
    frame_info_t *p_info = NULL;
1238

1239
1240
1241
1242
1243
    if (p_sys->b_vt_flush) {
        RestartVideoToolbox(p_dec, false);
        p_sys->b_vt_flush = false;
    }

1244
    if (p_block == NULL)
Thomas Guillem's avatar
Thomas Guillem committed
1245
    {
1246
        Drain(p_dec);
1247
1248
        return VLCDEC_SUCCESS;
    }
1249

1250
1251
1252
1253
1254
1255
1256
    vlc_mutex_lock(&p_sys->lock);
    if (p_sys->b_abort) { /* abort from output thread (DecoderCallback) */
        vlc_mutex_unlock(&p_sys->lock);
        goto reload;
    }
    vlc_mutex_unlock(&p_sys->lock);

Thomas Guillem's avatar
Thomas Guillem committed
1257
1258
1259
    if (unlikely(p_block->i_flags&(BLOCK_FLAG_CORRUPTED)))
    {
        if (p_sys->b_vt_feed)
1260
1261
        {
            Drain(p_dec);
Thomas Guillem's avatar
Thomas Guillem committed
1262
            RestartVideoToolbox(p_dec, false);
1263
        }
Thomas Guillem's avatar
Thomas Guillem committed
1264
1265
1266
        goto skip;
    }

1267
1268
1269
1270
1271
1272
1273
1274
1275
    bool b_config_changed = false;
    if (p_sys->codec == kCMVideoCodecType_H264 && p_sys->hh.pf_process_block)
    {
        p_block = p_sys->hh.pf_process_block(&p_sys->hh, p_block, &b_config_changed);
        if (!p_block)
            return VLCDEC_SUCCESS;
    }

    if (b_config_changed)
1276
    {
Thomas Guillem's avatar
Thomas Guillem committed
1277
1278
        /* decoding didn't start yet, which is ok for H264, let's see
         * if we can use this block to get going */
1279
1280
1281
        assert(p_sys->codec == kCMVideoCodecType_H264 && !p_sys->hh.b_is_xvcC);
        if (p_sys->session)
        {
1282
1283
            msg_Dbg(p_dec, "SPS/PPS changed: draining H264 decoder");
            Drain(p_dec);
1284
1285
1286
            msg_Dbg(p_dec, "SPS/PPS changed: restarting H264 decoder");
            StopVideoToolbox(p_dec, true);
        }
1287

1288
        int i_ret = avcCFromAnnexBCreate(p_dec);
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
        if (i_ret == VLC_SUCCESS)
        {
            if ((p_block->i_flags & BLOCK_FLAG_TOP_FIELD_FIRST
             || p_block->i_flags & BLOCK_FLAG_BOTTOM_FIELD_FIRST)
             && var_InheritBool(p_dec, "videotoolbox-temporal-deinterlacing"))
                p_sys->b_enable_temporal_processing = true;

            msg_Dbg(p_dec, "Got SPS/PPS: late opening of H264 decoder");
            StartVideoToolbox(p_dec);
        }
        if (!p_sys->session)
            goto skip;
Thomas Guillem's avatar
Thomas Guillem committed
1301
    }
1302

1303
1304
1305
1306

    p_info = CreateReorderInfo(p_sys, p_block);
    if(unlikely(!p_info))
        goto skip;
1307

Thomas Guillem's avatar
Thomas Guillem committed
1308
1309
1310
1311
    CMSampleBufferRef sampleBuffer =
        VTSampleBufferCreate(p_dec, p_sys->videoFormatDescription, p_block);
    if (unlikely(!sampleBuffer))
        goto skip;
1312

Thomas Guillem's avatar
Thomas Guillem committed
1313
1314
1315
1316
1317
1318
1319
    VTDecodeInfoFlags flagOut;
    VTDecodeFrameFlags decoderFlags = kVTDecodeFrame_EnableAsynchronousDecompression;
    if (unlikely(p_sys->b_enable_temporal_processing))
        decoderFlags |= kVTDecodeFrame_EnableTemporalProcessing;

    OSStatus status =
        VTDecompressionSessionDecodeFrame(p_sys->session, sampleBuffer,
1320
                                          decoderFlags, p_info, &flagOut);
1321
    if (HandleVTStatus(p_dec, status) == VLC_SUCCESS)
1322
    {
Thomas Guillem's avatar
Thomas Guillem committed
1323
        p_sys->b_vt_feed = true;
1324
1325
        p_info = NULL;
    }
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
    else
    {
        switch (status)
        {
            case kCVReturnInvalidArgument:
                msg_Err(p_dec, "decoder failure: invalid argument");
                /* The decoder module will be reloaded next time since we already
                 * modified the input block */
                vlc_mutex_lock(&p_sys->lock);
                p_dec->p_sys->b_abort = true;
                vlc_mutex_unlock(&p_sys->lock);
                break;
            case -8969 /* codecBadDataErr */:
            case kVTVideoDecoderBadDataErr:
1340
1341
1342
                if (RestartVideoToolbox(p_dec, true) == VLC_SUCCESS)
                {
                    status = VTDecompressionSessionDecodeFrame(p_sys->session,
1343
                                    sampleBuffer, decoderFlags, p_info, &flagOut);
1344
1345

                    if (status != 0)
1346
1347
                    {
                        free( p_info );
1348
                        StopVideoToolbox(p_dec, true);
1349
                    }
1350
                }
1351
1352
1353
1354
1355
1356
1357
                break;
            case -8960 /* codecErr */:
            case kVTVideoDecoderMalfunctionErr:
            case kVTInvalidSessionErr:
                RestartVideoToolbox(p_dec, true);
                break;
        }
1358
        p_info = NULL;
1359
    }
1360
    CFRelease(sampleBuffer);
1361
1362

skip:
1363
    free(p_info);
Thomas Guillem's avatar
Thomas Guillem committed
1364
    block_Release(p_block);
1365
    return VLCDEC_SUCCESS;
1366
1367
1368
1369

reload:
    /* Add an empty variable so that videotoolbox won't be loaded again for
     * this ES */
1370
1371
    var_Create(p_dec, "videotoolbox-failed", VLC_VAR_VOID);
    return VLCDEC_RELOAD;
1372
1373
}

1374
static int UpdateVideoFormat(decoder_t *p_dec, CVPixelBufferRef imageBuffer)
1375
{
1376
1377
1378
1379
1380
1381
1382
    CFDictionaryRef attachments = CVBufferGetAttachments(imageBuffer, kCVAttachmentMode_ShouldPropagate);
    NSDictionary *attachmentDict = (NSDictionary *)attachments;
#ifndef NDEBUG
    NSLog(@"%@", attachments);
#endif
    if (attachmentDict == nil || attachmentDict.count == 0)
        return -1;
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438

    NSString *colorSpace = attachmentDict[(NSString *)kCVImageBufferYCbCrMatrixKey];
    if (colorSpace != nil) {
        if ([colorSpace isEqualToString:(NSString *)kCVImageBufferYCbCrMatrix_ITU_R_601_4])
            p_dec->fmt_out.video.space = COLOR_SPACE_BT601;
        else if ([colorSpace isEqualToString:(NSString *)kCVImageBufferYCbCrMatrix_ITU_R_709_2])
            p_dec->fmt_out.video.space = COLOR_SPACE_BT709;
        else
            p_dec->fmt_out.video.space = COLOR_SPACE_UNDEF;
    }

    NSString *colorprimary = attachmentDict[(NSString *)kCVImageBufferColorPrimariesKey];
    if (colorprimary != nil) {
        if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_SMPTE_C] ||
            [colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_EBU_3213])
            p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_BT601_625;
        else if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_ITU_R_709_2])
            p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_BT709;
        else if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_P22])
            p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_DCI_P3;
        else
            p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_UNDEF;
    }

    NSString *transfer = attachmentDict[(NSString *)kCVImageBufferTransferFunctionKey];
    if (transfer != nil) {
        if ([transfer isEqualToString:(NSString *)kCVImageBufferTransferFunction_ITU_R_709_2] ||
            [transfer isEqualToString:(NSString *)kCVImageBufferTransferFunction_SMPTE_240M_1995])