mediacodec.c 51.2 KB
Newer Older
1
/*****************************************************************************
2
 * mediacodec.c: Video decoder module using the Android MediaCodec API
3
4
5
6
7
8
9
10
11
12
13
14
 *****************************************************************************
 * Copyright (C) 2012 Martin Storsjo
 *
 * Authors: Martin Storsjo <martin@martin.st>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Jean-Baptiste Kempf's avatar
LGPL    
Jean-Baptiste Kempf committed
15
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

/*****************************************************************************
 * Preamble
 *****************************************************************************/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

#include <stdint.h>
31
#include <assert.h>
32
33

#include <vlc_common.h>
Thomas Guillem's avatar
Thomas Guillem committed
34
#include <vlc_aout.h>
35
36
37
#include <vlc_plugin.h>
#include <vlc_codec.h>
#include <vlc_block_helper.h>
38
#include <vlc_memory.h>
39
#include <vlc_timestamp_helper.h>
40
#include <vlc_threads.h>
41
#include <vlc_bits.h>
42

Thomas Guillem's avatar
Thomas Guillem committed
43
#include "mediacodec.h"
44
#include "../codec/hxxx_helper.h"
45
46
47
#include <OMX_Core.h>
#include <OMX_Component.h>
#include "omxil_utils.h"
48
#include "../../video_output/android/display.h"
49

50
51
#define BLOCK_FLAG_CSD (0x01 << BLOCK_FLAG_PRIVATE_SHIFT)

52
#define DECODE_FLAG_RESTART (0x01)
53
#define DECODE_FLAG_DRAIN (0x02)
54
/**
55
 * Callback called when a new block is processed from DecodeBlock.
56
57
 * It returns -1 in case of error, 0 if block should be dropped, 1 otherwise.
 */
58
typedef int (*dec_on_new_block_cb)(decoder_t *, block_t **);
59

60
61
62
63
64
/**
 * Callback called when decoder is flushing.
 */
typedef void (*dec_on_flush_cb)(decoder_t *);

65
/**
66
 * Callback called when DecodeBlock try to get an output buffer (pic or block).
67
68
 * It returns -1 in case of error, or the number of output buffer returned.
 */
69
70
typedef int (*dec_process_output_cb)(decoder_t *, mc_api_out *, picture_t **,
                                     block_t **);
71

72
73
struct decoder_sys_t
{
74
    mc_api api;
75

76
    /* Codec Specific Data buffer: sent in DecodeBlock after a start or a flush
77
     * with the BUFFER_FLAG_CODEC_CONFIG flag.*/
78
79
    #define MAX_CSD_COUNT 3
    block_t *pp_csd[MAX_CSD_COUNT];
80
81
    size_t i_csd_count;
    size_t i_csd_send;
82

83
84
    bool b_has_format;

85
    int64_t i_preroll_end;
Thomas Guillem's avatar
Thomas Guillem committed
86
    int     i_quirks;
87

88
    /* Specific Audio/Video callbacks */
89
    dec_on_new_block_cb     pf_on_new_block;
90
    dec_on_flush_cb         pf_on_flush;
91
    dec_process_output_cb   pf_process_output;
92

93
    vlc_mutex_t     lock;
94
95
    vlc_thread_t    out_thread;
    /* Cond used to signal the output thread */
96
    vlc_cond_t      cond;
97
    /* Cond used to signal the decoder thread */
98
    vlc_cond_t      dec_cond;
99
    /* Set to true by pf_flush to signal the output thread to flush */
100
    bool            b_flush_out;
101
    /* If true, the output thread will start to dequeue output pictures */
102
    bool            b_output_ready;
103
104
    /* If true, the first input block was successfully dequeued */
    bool            b_input_dequeued;
105
    bool            b_aborted;
106
    bool            b_drained;
107
    bool            b_adaptive;
108
    int             i_decode_flags;
109

110
111
112
113
    union
    {
        struct
        {
114
            void *p_surface, *p_jsurface;
115
            unsigned i_angle;
116
            unsigned int i_stride, i_slice_height;
117
            int i_pixel_format;
118
            struct hxxx_helper hh;
119
            /* stores the inflight picture for each output buffer or NULL */
120
            picture_sys_t** pp_inflight_pictures;
121
122
123
            unsigned int i_inflight_pictures;
            timestamp_fifo_t *timestamp_fifo;
        } video;
Thomas Guillem's avatar
Thomas Guillem committed
124
125
126
127
        struct {
            date_t i_end_date;
            int i_channels;
            bool b_extract;
128
129
            /* Some audio decoders need a valid channel count */
            bool b_need_channels;
Thomas Guillem's avatar
Thomas Guillem committed
130
131
            int pi_extraction[AOUT_CHAN_MAX];
        } audio;
132
    };
133
134
135
136
137
};

/*****************************************************************************
 * Local prototypes
 *****************************************************************************/
Thomas Guillem's avatar
Thomas Guillem committed
138
static int  OpenDecoderJni(vlc_object_t *);
Thomas Guillem's avatar
Thomas Guillem committed
139
static int  OpenDecoderNdk(vlc_object_t *);
140
static void CleanDecoder(decoder_t *);
141
142
static void CloseDecoder(vlc_object_t *);

143
static int Video_OnNewBlock(decoder_t *, block_t **);
144
static int VideoHXXX_OnNewBlock(decoder_t *, block_t **);
145
static int VideoVC1_OnNewBlock(decoder_t *, block_t **);
146
static void Video_OnFlush(decoder_t *);
147
148
static int Video_ProcessOutput(decoder_t *, mc_api_out *, picture_t **,
                               block_t **);
149
static int DecodeBlock(decoder_t *, block_t *);
150

151
static int Audio_OnNewBlock(decoder_t *, block_t **);
152
static void Audio_OnFlush(decoder_t *);
153
154
static int Audio_ProcessOutput(decoder_t *, mc_api_out *, picture_t **,
                               block_t **);
155
156
157
158
159

static void DecodeFlushLocked(decoder_t *);
static void DecodeFlush(decoder_t *);
static void StopMediaCodec(decoder_t *);
static void *OutThread(void *);
160

161
static void InvalidateAllPictures(decoder_t *);
162
static void RemoveInflightPictures(decoder_t *);
163

164
165
166
/*****************************************************************************
 * Module descriptor
 *****************************************************************************/
167
168
169
#define DIRECTRENDERING_TEXT "Android direct rendering"
#define DIRECTRENDERING_LONGTEXT \
    "Enable Android direct rendering using opaque buffers."
170

Thomas Guillem's avatar
Thomas Guillem committed
171
172
173
#define MEDIACODEC_AUDIO_TEXT "Use MediaCodec for audio decoding"
#define MEDIACODEC_AUDIO_LONGTEXT "Still experimental."

174
175
#define MEDIACODEC_TUNNELEDPLAYBACK_TEXT "Use a tunneled surface for playback"

176
177
#define CFG_PREFIX "mediacodec-"

178
vlc_module_begin ()
179
180
181
182
183
    set_description("Video decoder using Android MediaCodec via NDK")
    set_category(CAT_INPUT)
    set_subcategory(SUBCAT_INPUT_VCODEC)
    set_section(N_("Decoding"), NULL)
    set_capability("decoder", 0) /* Only enabled via commandline arguments */
184
185
    add_bool(CFG_PREFIX "dr", true,
             DIRECTRENDERING_TEXT, DIRECTRENDERING_LONGTEXT, true)
Thomas Guillem's avatar
Thomas Guillem committed
186
187
    add_bool(CFG_PREFIX "audio", false,
             MEDIACODEC_AUDIO_TEXT, MEDIACODEC_AUDIO_LONGTEXT, true)
188
189
    add_bool(CFG_PREFIX "tunneled-playback", false,
             MEDIACODEC_TUNNELEDPLAYBACK_TEXT, NULL, true)
190
191
    set_callbacks(OpenDecoderNdk, CloseDecoder)
    add_shortcut("mediacodec_ndk")
Thomas Guillem's avatar
Thomas Guillem committed
192
    add_submodule ()
193
194
195
196
        set_description("Video decoder using Android MediaCodec via JNI")
        set_capability("decoder", 0)
        set_callbacks(OpenDecoderJni, CloseDecoder)
        add_shortcut("mediacodec_jni")
197
198
vlc_module_end ()

199
200
201
202
static void CSDFree(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

203
204
    for (unsigned int i = 0; i < p_sys->i_csd_count; ++i)
        block_Release(p_sys->pp_csd[i]);
205
206
207
    p_sys->i_csd_count = 0;
}

208
209
/* Init the p_sys->p_csd that will be sent from DecodeBlock */
static void CSDInit(decoder_t *p_dec, block_t *p_blocks, size_t i_count)
210
211
{
    decoder_sys_t *p_sys = p_dec->p_sys;
212
    assert(i_count >= 0 && i_count <= 3);
213

214
    CSDFree(p_dec);
215

216
    for (size_t i = 0; i < i_count; ++i)
217
    {
218
219
        assert(p_blocks != NULL);
        p_sys->pp_csd[i] = p_blocks;
220
        p_sys->pp_csd[i]->i_flags = BLOCK_FLAG_CSD;
221
222
        p_blocks = p_blocks->p_next;
        p_sys->pp_csd[i]->p_next = NULL;
223
    }
224

225
    p_sys->i_csd_count = i_count;
226
    p_sys->i_csd_send = 0;
227
228
}

229
static int CSDDup(decoder_t *p_dec, const void *p_buf, size_t i_buf)
230
{
231
232
233
234
    block_t *p_block = block_Alloc(i_buf);
    if (!p_block)
        return VLC_ENOMEM;
    memcpy(p_block->p_buffer, p_buf, i_buf);
235

236
237
    CSDInit(p_dec, p_block, 1);
    return VLC_SUCCESS;
238
239
}

240
/* Fill the p_sys->p_csd struct with H264 Parameter Sets */
241
static int H264SetCSD(decoder_t *p_dec, bool *p_size_changed)
242
{
243
244
245
    decoder_sys_t *p_sys = p_dec->p_sys;
    struct hxxx_helper *hh = &p_sys->video.hh;
    assert(hh->h264.i_sps_count > 0 || hh->h264.i_pps_count > 0);
246

247
    block_t *p_spspps_blocks = h264_helper_get_annexb_config(hh);
248

249
250
    if (p_spspps_blocks != NULL)
        CSDInit(p_dec, p_spspps_blocks, 2);
251

252
253
    unsigned i_w, i_h, i_vw, i_vh;
    h264_helper_get_current_picture_size(hh, &i_w, &i_h, &i_vw, &i_vh);
254

255
256
257
    if (p_size_changed)
        *p_size_changed = (i_w != p_dec->fmt_out.video.i_width
                        || i_h != p_dec->fmt_out.video.i_height);
258

259
260
261
262
263
264
    p_dec->fmt_out.video.i_visible_width =
    p_dec->fmt_out.video.i_width = i_w;
    p_dec->fmt_out.video.i_visible_height =
    p_dec->fmt_out.video.i_height = i_h;
    return VLC_SUCCESS;
}
265

266
267
268
269
270
271
/* Fill the p_sys->p_csd struct with HEVC Parameter Sets */
static int HEVCSetCSD(decoder_t *p_dec, bool *p_size_changed)
{
    (void) p_size_changed;
    decoder_sys_t *p_sys = p_dec->p_sys;
    struct hxxx_helper *hh = &p_sys->video.hh;
272

273
    assert(hh->hevc.i_annexb_config_nal > 0);
274

275
276
    return CSDDup(p_dec, hh->hevc.p_annexb_config_nal,
                  hh->hevc.i_annexb_config_nal);
277
278
}

279
280
281
static int ParseVideoExtraH264(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
{
    decoder_sys_t *p_sys = p_dec->p_sys;
282
    struct hxxx_helper *hh = &p_sys->video.hh;
283

284
285
    int i_ret = hxxx_helper_set_extra(hh, p_extra, i_extra);
    if (i_ret != VLC_SUCCESS)
286
        return i_ret;
287

288
289
290
291
292
293
    if (!hh->b_is_xvcC && p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_ADAPTIVE)
    {
        p_sys->b_adaptive = true;
        return VLC_SUCCESS;
    }

294
    assert(hh->pf_process_block != NULL);
295
    p_sys->pf_on_new_block = VideoHXXX_OnNewBlock;
296
297
298
299

    if (hh->h264.i_sps_count > 0 || hh->h264.i_pps_count > 0)
        return H264SetCSD(p_dec, NULL);
    return VLC_SUCCESS;
300
301
302
303
304
}

static int ParseVideoExtraHEVC(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
{
    decoder_sys_t *p_sys = p_dec->p_sys;
305
    struct hxxx_helper *hh = &p_sys->video.hh;
306

307
    int i_ret = hxxx_helper_set_extra(hh, p_extra, i_extra);
308
    if (i_ret != VLC_SUCCESS)
309
310
        return i_ret;

311
312
313
314
315
316
317
318
319
320
321
    if (!hh->b_is_xvcC)
    {
        if (p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_ADAPTIVE)
        {
            p_sys->b_adaptive = true;
            return VLC_SUCCESS;
        }
        else /* TODO */
            return VLC_EGENERIC;
    }

322
    assert(hh->pf_process_block != NULL);
323
    p_sys->pf_on_new_block = VideoHXXX_OnNewBlock;
324

325
326
    if (hh->hevc.i_annexb_config_nal > 0)
        return HEVCSetCSD(p_dec, NULL);
327
328
329
    return VLC_SUCCESS;
}

330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
static int ParseVideoExtraVc1(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
{
    int offset = 0;

    if (i_extra < 4)
        return VLC_EGENERIC;

    /* Initialisation data starts with : 0x00 0x00 0x01 0x0f */
    /* Skipping unecessary data */
    static const uint8_t vc1_start_code[4] = {0x00, 0x00, 0x01, 0x0f};
    for (; offset < i_extra - 4 ; ++offset)
    {
        if (!memcmp(&p_extra[offset], vc1_start_code, 4))
            break;
    }

    /* Could not find the sequence header start code */
    if (offset >= i_extra - 4)
        return VLC_EGENERIC;

350
    p_dec->p_sys->pf_on_new_block = VideoVC1_OnNewBlock;
351
    return CSDDup(p_dec, p_extra + offset, i_extra - offset);
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
}

static int ParseVideoExtraWmv3(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
{
    /* WMV3 initialisation data :
     * 8 fixed bytes
     * 4 extradata bytes
     * 4 height bytes (little endian)
     * 4 width bytes (little endian)
     * 16 fixed bytes */

    if (i_extra < 4)
        return VLC_EGENERIC;

    uint8_t p_data[36] = {
        0x8e, 0x01, 0x00, 0xc5, /* Fixed bytes values */
        0x04, 0x00, 0x00, 0x00, /* Same */
        0x00, 0x00, 0x00, 0x00, /* extradata emplacement */
        0x00, 0x00, 0x00, 0x00, /* height emplacement (little endian) */
        0x00, 0x00, 0x00, 0x00, /* width emplacement (little endian) */
        0x0c, 0x00, 0x00, 0x00, /* Fixed byte pattern */
        0x00, 0x00, 0x00, 0x00,
        0x00, 0x00, 0x00, 0x00,
        0x00, 0x00, 0x00, 0x00
    };

    /* Adding extradata */
    memcpy(&p_data[8], p_extra, 4);
    /* Adding height and width, little endian */
    SetDWLE(&(p_data[12]), p_dec->fmt_in.video.i_height);
    SetDWLE(&(p_data[16]), p_dec->fmt_in.video.i_width);

384
    p_dec->p_sys->pf_on_new_block = VideoVC1_OnNewBlock;
385
    return CSDDup(p_dec, p_data, sizeof(p_data));
386
387
}

388
static int ParseVideoExtra(decoder_t *p_dec)
389
{
390
391
    uint8_t *p_extra = p_dec->fmt_in.p_extra;
    int i_extra = p_dec->fmt_in.i_extra;
392

393
    switch (p_dec->fmt_in.i_codec)
394
    {
395
396
397
398
399
    case VLC_CODEC_H264:
        return ParseVideoExtraH264(p_dec, p_extra, i_extra);
    case VLC_CODEC_HEVC:
        return ParseVideoExtraHEVC(p_dec, p_extra, i_extra);
    case VLC_CODEC_WMV3:
400
        return ParseVideoExtraWmv3(p_dec, p_extra, i_extra);
401
    case VLC_CODEC_VC1:
402
        return ParseVideoExtraVc1(p_dec, p_extra, i_extra);
403
404
    default:
        return VLC_SUCCESS;
405
    }
406
407
}

408
409
410
411
412
static int UpdateOpaqueVout(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;
    picture_t *p_dummy_hwpic;

413
    if (p_sys->api.b_support_rotation)
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
    {
        switch (p_dec->fmt_in.video.orientation)
        {
            case ORIENT_ROTATED_90:
                p_sys->video.i_angle = 90;
                break;
            case ORIENT_ROTATED_180:
                p_sys->video.i_angle = 180;
                break;
            case ORIENT_ROTATED_270:
                p_sys->video.i_angle = 270;
                break;
            default:
                p_sys->video.i_angle = 0;
                break;
        }

        /* If MediaCodec can handle the rotation, reset the orientation to
         * Normal in order to ask the vout not to rotate. */
        if (p_sys->video.i_angle != 0)
            video_format_ApplyRotation(&p_dec->fmt_out.video,
                                       &p_dec->fmt_in.video);
    }
    else
        p_sys->video.i_angle = 0;

440
441
442
443
444
445
446
447
448
449
    /* Direct rendering: Request a valid OPAQUE Vout in order to get
     * the surface attached to it */
    if (decoder_UpdateVideoFormat(p_dec) != 0
     || (p_dummy_hwpic = decoder_NewPicture(p_dec)) == NULL)
    {
        p_sys->video.p_surface = p_sys->video.p_jsurface = NULL;
        return VLC_EGENERIC;
    }

    assert(p_dummy_hwpic->p_sys);
450
451
    assert(p_dummy_hwpic->p_sys->hw.p_surface);
    assert(p_dummy_hwpic->p_sys->hw.p_jsurface);
452

453
454
    p_sys->video.p_surface = p_dummy_hwpic->p_sys->hw.p_surface;
    p_sys->video.p_jsurface = p_dummy_hwpic->p_sys->hw.p_jsurface;
455
456
457
458
    picture_Release(p_dummy_hwpic);
    return VLC_SUCCESS;
}

459
/*****************************************************************************
460
 * StartMediaCodec: Create the mediacodec instance
461
 *****************************************************************************/
Thomas Guillem's avatar
Thomas Guillem committed
462
static int StartMediaCodec(decoder_t *p_dec)
463
464
{
    decoder_sys_t *p_sys = p_dec->p_sys;
465
    union mc_api_args args;
466

467
468
    if ((p_sys->api.i_quirks & MC_API_QUIRKS_NEED_CSD) && !p_sys->i_csd_count
     && !p_sys->b_adaptive)
469
        return VLC_ENOOBJ;
470

471
    if (p_dec->fmt_in.i_cat == VIDEO_ES)
472
    {
473
474
        if (!p_sys->b_adaptive
         && (!p_dec->fmt_out.video.i_width || !p_dec->fmt_out.video.i_height))
475
            return VLC_ENOOBJ;
476
477
        args.video.i_width = p_dec->fmt_out.video.i_width;
        args.video.i_height = p_dec->fmt_out.video.i_height;
478
        args.video.i_angle = p_sys->video.i_angle;
479

480
481
        args.video.p_surface = p_sys->video.p_surface;
        args.video.p_jsurface = p_sys->video.p_jsurface;
482
        args.video.b_tunneled_playback = args.video.p_surface ?
483
                var_InheritBool(p_dec, CFG_PREFIX "tunneled-playback") : false;
484
485
486
        if (p_sys->b_adaptive)
            msg_Dbg(p_dec, "mediacodec configured for adaptative playback");
        args.video.b_adaptive_playback = p_sys->b_adaptive;
487
    }
Thomas Guillem's avatar
Thomas Guillem committed
488
489
    else
    {
490
        date_Set(&p_sys->audio.i_end_date, VLC_TS_INVALID);
Thomas Guillem's avatar
Thomas Guillem committed
491
492

        args.audio.i_sample_rate    = p_dec->fmt_in.audio.i_rate;
493
        args.audio.i_channel_count  = p_dec->p_sys->audio.i_channels;
Thomas Guillem's avatar
Thomas Guillem committed
494
    }
495

496
    return p_sys->api.start(&p_sys->api, &args);
497
498
499
500
501
}

/*****************************************************************************
 * StopMediaCodec: Close the mediacodec instance
 *****************************************************************************/
Thomas Guillem's avatar
Thomas Guillem committed
502
static void StopMediaCodec(decoder_t *p_dec)
503
504
505
{
    decoder_sys_t *p_sys = p_dec->p_sys;

506
    /* Remove all pictures that are currently in flight in order
507
     * to prevent the vout from using destroyed output buffers. */
508
    if (p_sys->api.b_direct_rendering)
509
        RemoveInflightPictures(p_dec);
510

511
    p_sys->api.stop(&p_sys->api);
512
513
}

514
515
516
/*****************************************************************************
 * OpenDecoder: Create the decoder instance
 *****************************************************************************/
Thomas Guillem's avatar
Thomas Guillem committed
517
static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
518
{
Thomas Guillem's avatar
Thomas Guillem committed
519
    decoder_t *p_dec = (decoder_t *)p_this;
520
    decoder_sys_t *p_sys;
521
    int i_ret;
522
    size_t i_h264_profile = 0;
523
    const char *mime = NULL;
524

Thomas Guillem's avatar
Thomas Guillem committed
525
526
527
    /* Video or Audio if "mediacodec-audio" bool is true */
    if (p_dec->fmt_in.i_cat != VIDEO_ES && (p_dec->fmt_in.i_cat != AUDIO_ES
     || !var_InheritBool(p_dec, CFG_PREFIX "audio")))
528
529
        return VLC_EGENERIC;

530
531
532
533
    /* Fail if this module already failed to decode this ES */
    if (var_Type(p_dec, "mediacodec-failed") != 0)
        return VLC_EGENERIC;

534
    if (p_dec->fmt_in.i_cat == VIDEO_ES)
535
    {
536
537
538
539
540
541
542
543
544
        switch (p_dec->fmt_in.i_codec) {
        case VLC_CODEC_HEVC: mime = "video/hevc"; break;
        case VLC_CODEC_H264: mime = "video/avc"; break;
        case VLC_CODEC_H263: mime = "video/3gpp"; break;
        case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
        case VLC_CODEC_WMV3: mime = "video/x-ms-wmv"; break;
        case VLC_CODEC_VC1:  mime = "video/wvc1"; break;
        case VLC_CODEC_VP8:  mime = "video/x-vnd.on2.vp8"; break;
        case VLC_CODEC_VP9:  mime = "video/x-vnd.on2.vp9"; break;
Thomas Guillem's avatar
Thomas Guillem committed
545
546
547
548
549
550
        /* FIXME: mpeg2 is disabled: sar num/den can't be updated from
         * MediaCodec. Use avcodec instead that will update it. The proper
         * solution is to update sar from a mpeg2 packetizer.
         *
         * case VLC_CODEC_MPGV: mime = "video/mpeg2"; break;
         */
551
552
        }
    }
Thomas Guillem's avatar
Thomas Guillem committed
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
    else
    {
        switch (p_dec->fmt_in.i_codec) {
        case VLC_CODEC_AMR_NB: mime = "audio/3gpp"; break;
        case VLC_CODEC_AMR_WB: mime = "audio/amr-wb"; break;
        case VLC_CODEC_MPGA:
        case VLC_CODEC_MP3:    mime = "audio/mpeg"; break;
        case VLC_CODEC_MP2:    mime = "audio/mpeg-L2"; break;
        case VLC_CODEC_MP4A:   mime = "audio/mp4a-latm"; break;
        case VLC_CODEC_QCELP:  mime = "audio/qcelp"; break;
        case VLC_CODEC_VORBIS: mime = "audio/vorbis"; break;
        case VLC_CODEC_OPUS:   mime = "audio/opus"; break;
        case VLC_CODEC_ALAW:   mime = "audio/g711-alaw"; break;
        case VLC_CODEC_MULAW:  mime = "audio/g711-mlaw"; break;
        case VLC_CODEC_FLAC:   mime = "audio/flac"; break;
        case VLC_CODEC_GSM:    mime = "audio/gsm"; break;
        case VLC_CODEC_A52:    mime = "audio/ac3"; break;
        case VLC_CODEC_EAC3:   mime = "audio/eac3"; break;
        case VLC_CODEC_ALAC:   mime = "audio/alac"; break;
        case VLC_CODEC_DTS:    mime = "audio/vnd.dts"; break;
        /* case VLC_CODEC_: mime = "audio/mpeg-L1"; break; */
        /* case VLC_CODEC_: mime = "audio/aac-adts"; break; */
        }
    }
577
578
579
580
581
    if (!mime)
    {
        msg_Dbg(p_dec, "codec %4.4s not supported",
                (char *)&p_dec->fmt_in.i_codec);
        return VLC_EGENERIC;
582
583
    }

584
585
    /* Allocate the memory needed to store the decoder's structure */
    if ((p_sys = calloc(1, sizeof(*p_sys))) == NULL)
Thomas Guillem's avatar
Thomas Guillem committed
586
        return VLC_ENOMEM;
587
588
589
590
591

    p_sys->api.p_obj = p_this;
    p_sys->api.i_codec = p_dec->fmt_in.i_codec;
    p_sys->api.i_cat = p_dec->fmt_in.i_cat;
    p_sys->api.psz_mime = mime;
592
593

    if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
594
595
    {
        uint8_t i_profile;
596
        if (h264_get_profile_level(&p_dec->fmt_in, &i_profile, NULL, NULL))
597
598
599
            i_h264_profile = i_profile;
    }

600
    if (pf_init(&p_sys->api) != 0)
Thomas Guillem's avatar
Thomas Guillem committed
601
    {
602
        free(p_sys);
Thomas Guillem's avatar
Thomas Guillem committed
603
604
        return VLC_EGENERIC;
    }
605
    if (p_sys->api.configure(&p_sys->api, i_h264_profile) != 0)
606
    {
607
608
609
610
        /* If the device can't handle video/wvc1,
         * it can probably handle video/x-ms-wmv */
        if (!strcmp(mime, "video/wvc1") && p_dec->fmt_in.i_codec == VLC_CODEC_VC1)
        {
611
612
            p_sys->api.psz_mime = "video/x-ms-wmv";
            if (p_sys->api.configure(&p_sys->api, i_h264_profile) != 0)
613
            {
614
615
                p_sys->api.clean(&p_sys->api);
                free(p_sys);
616
                return (VLC_EGENERIC);
617
618
619
620
            }
        }
        else
        {
621
622
            p_sys->api.clean(&p_sys->api);
            free(p_sys);
623
624
            return VLC_EGENERIC;
        }
625
    }
626

627
    p_dec->p_sys = p_sys;
628
629
630
631

    p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
    p_dec->fmt_out.video = p_dec->fmt_in.video;
    p_dec->fmt_out.audio = p_dec->fmt_in.audio;
632
633
    p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
    p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
634

635
636
637
638
    vlc_mutex_init(&p_sys->lock);
    vlc_cond_init(&p_sys->cond);
    vlc_cond_init(&p_sys->dec_cond);

639
    if (p_dec->fmt_in.i_cat == VIDEO_ES)
640
    {
641
642
643
644
        switch (p_dec->fmt_in.i_codec)
        {
        case VLC_CODEC_H264:
        case VLC_CODEC_HEVC:
645
646
            hxxx_helper_init(&p_sys->video.hh, VLC_OBJECT(p_dec),
                             p_dec->fmt_in.i_codec, false);
647
648
            break;
        }
649
        p_sys->pf_on_new_block = Video_OnNewBlock;
650
        p_sys->pf_on_flush = Video_OnFlush;
651
        p_sys->pf_process_output = Video_ProcessOutput;
652

653
654
        p_sys->video.timestamp_fifo = timestamp_FifoNew(32);
        if (!p_sys->video.timestamp_fifo)
655
            goto bailout;
656

657
658
        TAB_INIT(p_sys->video.i_inflight_pictures,
                 p_sys->video.pp_inflight_pictures);
659
660
661
662
663
664

        if (var_InheritBool(p_dec, CFG_PREFIX "dr"))
        {
            /* Direct rendering: Request a valid OPAQUE Vout in order to get
             * the surface attached to it */
            p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
665

666
667
668
669
670
671
            if (UpdateOpaqueVout(p_dec) != VLC_SUCCESS)
            {
                msg_Err(p_dec, "Opaque Vout request failed");
                goto bailout;
            }
        }
672
    }
Thomas Guillem's avatar
Thomas Guillem committed
673
674
    else
    {
675
        p_sys->pf_on_new_block = Audio_OnNewBlock;
676
        p_sys->pf_on_flush = Audio_OnFlush;
677
        p_sys->pf_process_output = Audio_ProcessOutput;
678
        p_sys->audio.i_channels = p_dec->fmt_in.audio.i_channels;
Thomas Guillem's avatar
Thomas Guillem committed
679

680
        if ((p_sys->api.i_quirks & MC_API_AUDIO_QUIRKS_NEED_CHANNELS)
681
         && !p_sys->audio.i_channels)
682
        {
683
684
            msg_Warn(p_dec, "codec need a valid channel count");
            goto bailout;
685
        }
Thomas Guillem's avatar
Thomas Guillem committed
686
    }
687

688
689
690
691
    /* Try first to configure specific Video CSD */
    if (p_dec->fmt_in.i_cat == VIDEO_ES)
        if (ParseVideoExtra(p_dec) != VLC_SUCCESS)
            goto bailout;
692

693
    /* Set default CSD if ParseVideoExtra failed to configure one */
694
695
696
    if (!p_sys->i_csd_count && p_dec->fmt_in.i_extra
     && CSDDup(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra) != VLC_SUCCESS)
        goto bailout;
697

698
699
    i_ret = StartMediaCodec(p_dec);
    switch (i_ret)
700
    {
701
702
703
    case VLC_SUCCESS:
        break;
    case VLC_ENOOBJ:
704
705
706
707
708
709
        switch (p_dec->fmt_in.i_codec)
        {
        case VLC_CODEC_H264:
        case VLC_CODEC_HEVC:
            msg_Warn(p_dec, "late opening for codec %4.4s",
                     (const char *)&p_dec->fmt_in.i_codec);
710
            break;
711
712
713
714
715
716
        default:
            msg_Warn(p_dec, "late opening with %4.4s not handled",
                     (const char *) &p_dec->fmt_in.i_codec);
            goto bailout;
        }
        break;
717
    default:
718
        msg_Err(p_dec, "StartMediaCodec failed");
719
720
721
722
723
724
725
726
727
728
729
        goto bailout;
    }

    if (vlc_clone(&p_sys->out_thread, OutThread, p_dec,
                  VLC_THREAD_PRIORITY_LOW))
    {
        msg_Err(p_dec, "vlc_clone failed");
        vlc_mutex_unlock(&p_sys->lock);
        goto bailout;
    }

730
731
    p_dec->pf_decode = DecodeBlock;
    p_dec->pf_flush  = DecodeFlush;
732

733
734
    return VLC_SUCCESS;

735
bailout:
736
    CleanDecoder(p_dec);
737
    return VLC_EGENERIC;
Thomas Guillem's avatar
Thomas Guillem committed
738
}
739

Thomas Guillem's avatar
Thomas Guillem committed
740
741
742
743
744
static int OpenDecoderNdk(vlc_object_t *p_this)
{
    return OpenDecoder(p_this, MediaCodecNdk_Init);
}

Thomas Guillem's avatar
Thomas Guillem committed
745
746
747
static int OpenDecoderJni(vlc_object_t *p_this)
{
    return OpenDecoder(p_this, MediaCodecJni_Init);
748
749
}

750
751
752
753
static void AbortDecoderLocked(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

754
    if (!p_sys->b_aborted)
755
    {
756
        p_sys->b_aborted = true;
757
758
759
760
        vlc_cancel(p_sys->out_thread);
    }
}

761
static void CleanDecoder(decoder_t *p_dec)
762
763
764
{
    decoder_sys_t *p_sys = p_dec->p_sys;

765
766
767
    vlc_mutex_destroy(&p_sys->lock);
    vlc_cond_destroy(&p_sys->cond);
    vlc_cond_destroy(&p_sys->dec_cond);
768

Thomas Guillem's avatar
Thomas Guillem committed
769
    StopMediaCodec(p_dec);
770

771
    CSDFree(p_dec);
772
    p_sys->api.clean(&p_sys->api);
773
774
775

    if (p_dec->fmt_in.i_cat == VIDEO_ES)
    {
776
777
778
779
        if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
         || p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
            hxxx_helper_clean(&p_sys->video.hh);

780
781
        if (p_sys->video.timestamp_fifo)
            timestamp_FifoRelease(p_sys->video.timestamp_fifo);
782
    }
783
784
785
    free(p_sys);
}

786
787
788
789
790
791
792
793
794
/*****************************************************************************
 * CloseDecoder: Close the decoder instance
 *****************************************************************************/
static void CloseDecoder(vlc_object_t *p_this)
{
    decoder_t *p_dec = (decoder_t *)p_this;
    decoder_sys_t *p_sys = p_dec->p_sys;

    vlc_mutex_lock(&p_sys->lock);
795
    /* Unblock output thread waiting in dequeue_out */
796
    DecodeFlushLocked(p_dec);
797
798
    /* Cancel the output thread */
    AbortDecoderLocked(p_dec);
799
    vlc_mutex_unlock(&p_sys->lock);
800

801
802
803
804
805
    vlc_join(p_sys->out_thread, NULL);

    CleanDecoder(p_dec);
}

806
807
808
/*****************************************************************************
 * vout callbacks
 *****************************************************************************/
809
static void ReleasePicture(decoder_t *p_dec, unsigned i_index, bool b_render)
810
{
Thomas Guillem's avatar
Thomas Guillem committed
811
    decoder_sys_t *p_sys = p_dec->p_sys;
812

813
    p_sys->api.release_out(&p_sys->api, i_index, b_render);
814
815
816
817
818
819
}

static void InvalidateAllPictures(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

820
821
    for (unsigned int i = 0; i < p_sys->video.i_inflight_pictures; ++i)
        AndroidOpaquePicture_Release(p_sys->video.pp_inflight_pictures[i],
822
                                     false);
823
824
}

825
static int InsertInflightPicture(decoder_t *p_dec, picture_sys_t *p_picsys)
826
827
828
{
    decoder_sys_t *p_sys = p_dec->p_sys;

829
    if (!p_picsys->hw.p_dec)
830
    {
831
832
        p_picsys->hw.p_dec = p_dec;
        p_picsys->hw.pf_release = ReleasePicture;
833
        TAB_APPEND_CAST((picture_sys_t **),
834
835
                        p_sys->video.i_inflight_pictures,
                        p_sys->video.pp_inflight_pictures,
836
837
                        p_picsys);
    } /* else already attached */
838
839
840
    return 0;
}

841
842
843
844
static void RemoveInflightPictures(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

845
846
847
848
    for (unsigned int i = 0; i < p_sys->video.i_inflight_pictures; ++i)
        AndroidOpaquePicture_DetachDecoder(p_sys->video.pp_inflight_pictures[i]);
    TAB_CLEAN(p_sys->video.i_inflight_pictures,
              p_sys->video.pp_inflight_pictures);
849
850
}

851
852
static int Video_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
                               picture_t **pp_out_pic, block_t **pp_out_block)
853
854
{
    decoder_sys_t *p_sys = p_dec->p_sys;
855
856
    (void) pp_out_block;
    assert(pp_out_pic);
857

858
    if (p_out->type == MC_OUT_TYPE_BUF)
859
    {
860
861
        picture_t *p_pic = NULL;

862
863
864
865
866
        /* If the oldest input block had no PTS, the timestamp of
         * the frame returned by MediaCodec might be wrong so we
         * overwrite it with the corresponding dts. Call FifoGet
         * first in order to avoid a gap if buffers are released
         * due to an invalid format or a preroll */
867
        int64_t forced_ts = timestamp_FifoGet(p_sys->video.timestamp_fifo);
868

869
        if (!p_sys->b_has_format) {
870
            msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
871
            return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
872
873
        }

874
        if (p_out->buf.i_ts <= p_sys->i_preroll_end)
875
            return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
876

877
        if (!p_sys->api.b_direct_rendering && p_out->buf.p_ptr == NULL)
Thomas Guillem's avatar
Thomas Guillem committed
878
879
880
        {
            /* This can happen when receiving an EOS buffer */
            msg_Warn(p_dec, "Invalid buffer, dropping frame");
881
            return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
Thomas Guillem's avatar
Thomas Guillem committed
882
883
        }

884
885
886
        p_pic = decoder_NewPicture(p_dec);
        if (!p_pic) {
            msg_Warn(p_dec, "NewPicture failed");
887
            return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
888
        }
889
890

        if (forced_ts == VLC_TS_INVALID)
891
            p_pic->date = p_out->buf.i_ts;
892
893
        else
            p_pic->date = forced_ts;
894
        p_pic->b_progressive = true;
895

896
        if (p_sys->api.b_direct_rendering)
897
        {
898
            p_pic->p_sys->hw.i_index = p_out->buf.i_index;
899
            InsertInflightPicture(p_dec, p_pic->p_sys);
900
901
902
903
904
905
        } else {
            unsigned int chroma_div;
            GetVlcChromaSizes(p_dec->fmt_out.i_codec,
                              p_dec->fmt_out.video.i_width,
                              p_dec->fmt_out.video.i_height,
                              NULL, NULL, &chroma_div);
906
907
908
            CopyOmxPicture(p_sys->video.i_pixel_format, p_pic,
                           p_sys->video.i_slice_height, p_sys->video.i_stride,
                           (uint8_t *)p_out->buf.p_ptr, chroma_div, NULL);
Thomas Guillem's avatar
Thomas Guillem committed
909

910
            if (p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false))
911
912
913
914
            {
                picture_Release(p_pic);
                return -1;
            }