mediacodec.c 54.4 KB
Newer Older
1
/*****************************************************************************
2
 * mediacodec.c: Video decoder module using the Android MediaCodec API
3 4 5 6 7 8 9 10 11 12 13 14
 *****************************************************************************
 * Copyright (C) 2012 Martin Storsjo
 *
 * Authors: Martin Storsjo <martin@martin.st>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Jean-Baptiste Kempf's avatar
LGPL  
Jean-Baptiste Kempf committed
15
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

/*****************************************************************************
 * Preamble
 *****************************************************************************/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

#include <stdint.h>
31
#include <assert.h>
32 33

#include <vlc_common.h>
Thomas Guillem's avatar
Thomas Guillem committed
34
#include <vlc_aout.h>
35 36 37
#include <vlc_plugin.h>
#include <vlc_codec.h>
#include <vlc_block_helper.h>
38
#include <vlc_memory.h>
39
#include <vlc_timestamp_helper.h>
40
#include <vlc_threads.h>
41

Thomas Guillem's avatar
Thomas Guillem committed
42
#include "mediacodec.h"
43 44
#include "../../packetizer/h264_nal.h"
#include "../../packetizer/hevc_nal.h"
45
#include "../../packetizer/hxxx_nal.h"
46 47 48
#include <OMX_Core.h>
#include <OMX_Component.h>
#include "omxil_utils.h"
49
#include "../../video_output/android/android_window.h"
50

51 52
#define BLOCK_FLAG_CSD (0x01 << BLOCK_FLAG_PRIVATE_SHIFT)

53 54 55 56 57 58 59
/* Codec Specific Data */
struct csd
{
    uint8_t *p_buf;
    size_t i_size;
};

60 61
#define NEWBLOCK_FLAG_RESTART (0x01)
#define NEWBLOCK_FLAG_FLUSH (0x02)
62 63 64 65
/**
 * Callback called when a new block is processed from DecodeCommon.
 * It returns -1 in case of error, 0 if block should be dropped, 1 otherwise.
 */
66
typedef int (*dec_on_new_block_cb)(decoder_t *, block_t **, int *);
67

68 69 70 71 72
/**
 * Callback called when decoder is flushing.
 */
typedef void (*dec_on_flush_cb)(decoder_t *);

73 74 75 76
/**
 * Callback called when DecodeCommon try to get an output buffer (pic or block).
 * It returns -1 in case of error, or the number of output buffer returned.
 */
77
typedef int (*dec_process_output_cb)(decoder_t *, mc_api_out *, picture_t **, block_t **);
78

79 80
struct decoder_sys_t
{
Thomas Guillem's avatar
Thomas Guillem committed
81
    mc_api *api;
82

83
    /* Codec Specific Data buffer: sent in DecodeCommon after a start or a flush
84
     * with the BUFFER_FLAG_CODEC_CONFIG flag.*/
85
    block_t **pp_csd;
86 87
    size_t i_csd_count;
    size_t i_csd_send;
88

89
    bool b_update_format;
90 91
    bool b_has_format;

92
    int64_t i_preroll_end;
Thomas Guillem's avatar
Thomas Guillem committed
93
    int     i_quirks;
94

95
    /* Specific Audio/Video callbacks */
96
    dec_on_new_block_cb     pf_on_new_block;
97
    dec_on_flush_cb         pf_on_flush;
98
    dec_process_output_cb   pf_process_output;
99

100
    vlc_mutex_t     lock;
101 102
    vlc_thread_t    out_thread;
    /* Cond used to signal the output thread */
103
    vlc_cond_t      cond;
104
    /* Cond used to signal the decoder thread */
105
    vlc_cond_t      dec_cond;
106
    /* Set to true by pf_flush to signal the output thread to flush */
107
    bool            b_flush_out;
108
    /* If true, the output thread will start to dequeue output pictures */
109
    bool            b_output_ready;
110 111
    /* If true, the first input block was successfully dequeued */
    bool            b_input_dequeued;
112
    bool            b_aborted;
113
    /* TODO: remove */
114
    bool            b_error_signaled;
115

116 117 118 119 120
    union
    {
        struct
        {
            AWindowHandler *p_awh;
121 122
            unsigned int i_stride, i_slice_height, i_width, i_height;
            int i_pixel_format;
123
            uint8_t i_nal_length_size;
124
            size_t i_h264_profile;
125
            bool b_first_mp4v_iframe;
126
            /* stores the inflight picture for each output buffer or NULL */
127
            picture_sys_t** pp_inflight_pictures;
128 129 130
            unsigned int i_inflight_pictures;
            timestamp_fifo_t *timestamp_fifo;
        } video;
Thomas Guillem's avatar
Thomas Guillem committed
131 132 133 134
        struct {
            date_t i_end_date;
            int i_channels;
            bool b_extract;
135 136
            /* Some audio decoders need a valid channel count */
            bool b_need_channels;
Thomas Guillem's avatar
Thomas Guillem committed
137 138
            int pi_extraction[AOUT_CHAN_MAX];
        } audio;
139
    } u;
140 141 142 143 144
};

/*****************************************************************************
 * Local prototypes
 *****************************************************************************/
Thomas Guillem's avatar
Thomas Guillem committed
145
static int  OpenDecoderJni(vlc_object_t *);
Thomas Guillem's avatar
Thomas Guillem committed
146
static int  OpenDecoderNdk(vlc_object_t *);
147
static void CleanDecoder(decoder_t *);
148 149
static void CloseDecoder(vlc_object_t *);

150
static int Video_OnNewBlock(decoder_t *, block_t **, int *);
151
static void Video_OnFlush(decoder_t *);
152
static int Video_ProcessOutput(decoder_t *, mc_api_out *, picture_t **, block_t **);
153
static picture_t *DecodeVideo(decoder_t *, block_t **);
154

155
static int Audio_OnNewBlock(decoder_t *, block_t **, int *);
156
static void Audio_OnFlush(decoder_t *);
157
static int Audio_ProcessOutput(decoder_t *, mc_api_out *, picture_t **, block_t **);
Thomas Guillem's avatar
Thomas Guillem committed
158
static block_t *DecodeAudio(decoder_t *, block_t **);
159 160 161 162 163

static void DecodeFlushLocked(decoder_t *);
static void DecodeFlush(decoder_t *);
static void StopMediaCodec(decoder_t *);
static void *OutThread(void *);
164

165
static void InvalidateAllPictures(decoder_t *);
166
static void RemoveInflightPictures(decoder_t *);
167

168 169 170
/*****************************************************************************
 * Module descriptor
 *****************************************************************************/
171 172 173 174
#define DIRECTRENDERING_TEXT N_("Android direct rendering")
#define DIRECTRENDERING_LONGTEXT N_(\
        "Enable Android direct rendering using opaque buffers.")

Thomas Guillem's avatar
Thomas Guillem committed
175 176 177
#define MEDIACODEC_AUDIO_TEXT "Use MediaCodec for audio decoding"
#define MEDIACODEC_AUDIO_LONGTEXT "Still experimental."

178 179
#define MEDIACODEC_TUNNELEDPLAYBACK_TEXT "Use a tunneled surface for playback"

180 181
#define CFG_PREFIX "mediacodec-"

182
vlc_module_begin ()
183
    set_description( N_("Video decoder using Android MediaCodec via NDK") )
184 185 186 187
    set_category( CAT_INPUT )
    set_subcategory( SUBCAT_INPUT_VCODEC )
    set_section( N_("Decoding") , NULL )
    set_capability( "decoder", 0 ) /* Only enabled via commandline arguments */
188 189
    add_bool(CFG_PREFIX "dr", true,
             DIRECTRENDERING_TEXT, DIRECTRENDERING_LONGTEXT, true)
Thomas Guillem's avatar
Thomas Guillem committed
190 191
    add_bool(CFG_PREFIX "audio", false,
             MEDIACODEC_AUDIO_TEXT, MEDIACODEC_AUDIO_LONGTEXT, true)
192 193
    add_bool(CFG_PREFIX "tunneled-playback", false,
             MEDIACODEC_TUNNELEDPLAYBACK_TEXT, NULL, true)
Thomas Guillem's avatar
Thomas Guillem committed
194 195 196
    set_callbacks( OpenDecoderNdk, CloseDecoder )
    add_shortcut( "mediacodec_ndk" )
    add_submodule ()
197
        set_description( N_("Video decoder using Android MediaCodec via JNI") )
Thomas Guillem's avatar
Thomas Guillem committed
198 199 200
        set_capability( "decoder", 0 )
        set_callbacks( OpenDecoderJni, CloseDecoder )
        add_shortcut( "mediacodec_jni" )
201 202
vlc_module_end ()

203 204 205 206
static void CSDFree(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

207
    if (p_sys->pp_csd)
208 209
    {
        for (unsigned int i = 0; i < p_sys->i_csd_count; ++i)
210 211 212
            block_Release(p_sys->pp_csd[i]);
        free(p_sys->pp_csd);
        p_sys->pp_csd = NULL;
213 214 215 216
    }
    p_sys->i_csd_count = 0;
}

217
/* Create the p_sys->p_csd that will be sent from DecodeCommon */
218 219 220 221
static int CSDDup(decoder_t *p_dec, const struct csd *p_csd, size_t i_count)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

222
    CSDFree(p_dec);
223

224 225
    p_sys->pp_csd = malloc(i_count * sizeof(block_t *));
    if (!p_sys->pp_csd)
226 227
        return VLC_ENOMEM;

228
    for (size_t i = 0; i < i_count; ++i)
229
    {
230 231
        p_sys->pp_csd[i] = block_Alloc(p_csd[i].i_size);
        if (!p_sys->pp_csd[i])
232 233 234 235
        {
            CSDFree(p_dec);
            return VLC_ENOMEM;
        }
236 237 238
        p_sys->pp_csd[i]->i_flags = BLOCK_FLAG_CSD;
        memcpy(p_sys->pp_csd[i]->p_buffer, p_csd[i].p_buf, p_csd[i].i_size);
        p_sys->i_csd_count++;
239
    }
240 241

    p_sys->i_csd_send = 0;
242 243 244 245
    return VLC_SUCCESS;
}

static bool CSDCmp(decoder_t *p_dec, struct csd *p_csd, size_t i_csd_count)
246
{
247 248 249 250 251 252
    decoder_sys_t *p_sys = p_dec->p_sys;

    if (p_sys->i_csd_count != i_csd_count)
        return false;
    for (size_t i = 0; i < i_csd_count; ++i)
    {
253 254 255
        if (p_sys->pp_csd[i]->i_buffer != p_csd[i].i_size
         || memcmp(p_sys->pp_csd[i]->p_buffer, p_csd[i].p_buf,
                   p_csd[i].i_size) != 0)
256 257 258 259
            return false;
    }
    return true;
}
260

261 262 263 264 265 266 267 268 269 270
/* Fill the p_sys->p_csd struct with H264 Parameter Sets */
static int H264SetCSD(decoder_t *p_dec, void *p_buf, size_t i_size,
                      bool *p_size_changed)
{
    decoder_sys_t *p_sys = p_dec->p_sys;
    uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL;
    size_t i_sps_size = 0, i_pps_size = 0;

    /* Check if p_buf contains a valid SPS PPS */
    if (h264_get_spspps(p_buf, i_size, &p_sps_buf, &i_sps_size,
271
                        &p_pps_buf, &i_pps_size) == 0 )
272
    {
273 274 275
        struct csd csd[2];
        int i_csd_count = 0;

276 277 278 279 280 281 282 283 284
        const uint8_t *p_buffer = p_sps_buf;
        size_t i_buffer = i_sps_size;
        if(!hxxx_strip_AnnexB_startcode(&p_buffer, &i_buffer))
            return VLC_EGENERIC;

        h264_sequence_parameter_set_t *p_sps = h264_decode_sps(p_buffer, i_buffer, true);
        if( !p_sps )
            return VLC_EGENERIC;

285 286 287
        unsigned vsize[4];
        (void) h264_get_picture_size( p_sps, &vsize[0], &vsize[1], &vsize[2], &vsize[3] );
        /* FIXME: what to do with visible width/height ? */
288

289
        if (i_sps_size)
290
        {
291 292 293
            csd[i_csd_count].p_buf = p_sps_buf;
            csd[i_csd_count].i_size = i_sps_size;
            i_csd_count++;
294
        }
295
        if (i_pps_size)
296
        {
297 298 299
            csd[i_csd_count].p_buf = p_pps_buf;
            csd[i_csd_count].i_size = i_pps_size;
            i_csd_count++;
300
        }
301 302 303 304

        /* Compare the SPS PPS with the old one */
        if (!CSDCmp(p_dec, csd, i_csd_count))
        {
305 306
            msg_Warn(p_dec, "New SPS/PPS found, id: %d size: %ux%u sps: %d pps: %d",
                     p_sps->i_id, vsize[0], vsize[1],
307 308 309 310 311
                     i_sps_size, i_pps_size);

            /* In most use cases, p_sys->p_csd[0] contains a SPS, and
             * p_sys->p_csd[1] contains a PPS */
            if (CSDDup(p_dec, csd, i_csd_count))
312 313
            {
                h264_release_sps( p_sps );
314
                return VLC_ENOMEM;
315
            }
316 317

            if (p_size_changed)
318 319
                *p_size_changed = (vsize[0] != p_sys->u.video.i_width
                                || vsize[1] != p_sys->u.video.i_height);
320

321 322
            p_sys->u.video.i_width = vsize[0];
            p_sys->u.video.i_height = vsize[1];
323 324

            h264_release_sps( p_sps );
325 326 327

            return VLC_SUCCESS;
        }
328 329

        h264_release_sps( p_sps );
330
    }
331

332
    return VLC_EGENERIC;
333 334
}

335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
static int ParseVideoExtraVc1(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
{
    int offset = 0;
    struct csd csd;

    if (i_extra < 4)
        return VLC_EGENERIC;

    /* Initialisation data starts with : 0x00 0x00 0x01 0x0f */
    /* Skipping unecessary data */
    static const uint8_t vc1_start_code[4] = {0x00, 0x00, 0x01, 0x0f};
    for (; offset < i_extra - 4 ; ++offset)
    {
        if (!memcmp(&p_extra[offset], vc1_start_code, 4))
            break;
    }

    /* Could not find the sequence header start code */
    if (offset >= i_extra - 4)
        return VLC_EGENERIC;

    csd.i_size = i_extra - offset;
    csd.p_buf = p_extra + offset;

    return CSDDup(p_dec, &csd, 1);
}

static int ParseVideoExtraWmv3(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
{
    /* WMV3 initialisation data :
     * 8 fixed bytes
     * 4 extradata bytes
     * 4 height bytes (little endian)
     * 4 width bytes (little endian)
     * 16 fixed bytes */

    if (i_extra < 4)
        return VLC_EGENERIC;

    uint8_t p_data[36] = {
        0x8e, 0x01, 0x00, 0xc5, /* Fixed bytes values */
        0x04, 0x00, 0x00, 0x00, /* Same */
        0x00, 0x00, 0x00, 0x00, /* extradata emplacement */
        0x00, 0x00, 0x00, 0x00, /* height emplacement (little endian) */
        0x00, 0x00, 0x00, 0x00, /* width emplacement (little endian) */
        0x0c, 0x00, 0x00, 0x00, /* Fixed byte pattern */
        0x00, 0x00, 0x00, 0x00,
        0x00, 0x00, 0x00, 0x00,
        0x00, 0x00, 0x00, 0x00
    };

    struct csd csd;

    csd.i_size = sizeof(p_data);
    /* Adding extradata */
    memcpy(&p_data[8], p_extra, 4);
    /* Adding height and width, little endian */
    SetDWLE(&(p_data[12]), p_dec->fmt_in.video.i_height);
    SetDWLE(&(p_data[16]), p_dec->fmt_in.video.i_width);

    csd.p_buf = p_data;
    return CSDDup(p_dec, &csd, 1);
}

399
static int ParseVideoExtra(decoder_t *p_dec)
400 401
{
    decoder_sys_t *p_sys = p_dec->p_sys;
402 403
    uint8_t *p_extra = p_dec->fmt_in.p_extra;
    int i_extra = p_dec->fmt_in.i_extra;
404 405 406 407 408 409

    if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
     || p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
    {
        if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
        {
410 411 412 413 414 415 416 417 418 419 420 421 422 423
            if (h264_isavcC(p_extra, i_extra))
            {
                size_t i_size = 0;
                uint8_t *p_buf = h264_avcC_to_AnnexB_NAL(p_extra, i_extra, &i_size,
                                                         &p_sys->u.video.i_nal_length_size);
                if(p_buf)
                {
                    H264SetCSD(p_dec, p_buf, i_size, NULL);
                    free(p_buf);
                }
            }
            else
                H264SetCSD(p_dec, p_extra, i_extra, NULL);
        }
424
        else  /* FIXME and refactor: CSDDup vs CSDless SetCSD */
425
        {
426
            if (hevc_ishvcC(p_extra, i_extra))
427 428
            {
                struct csd csd;
429 430 431 432 433 434 435
                csd.p_buf = hevc_hvcC_to_AnnexB_NAL(p_extra, i_extra, &csd.i_size,
                                                    &p_sys->u.video.i_nal_length_size);
                if(csd.p_buf)
                {
                    CSDDup(p_dec, &csd, 1);
                    free(csd.p_buf);
                }
436
            }
437
            /* FIXME: what to do with AnnexB ? */
438 439
        }
    }
440 441
    else if (p_dec->fmt_in.i_codec == VLC_CODEC_WMV3)
    {
442
        return ParseVideoExtraWmv3(p_dec, p_extra, i_extra);
443 444 445
    }
    else if (p_dec->fmt_in.i_codec == VLC_CODEC_VC1)
    {
446
        return ParseVideoExtraVc1(p_dec, p_extra, i_extra);
447
    }
448 449 450
    return VLC_SUCCESS;
}

451
/*****************************************************************************
452
 * StartMediaCodec: Create the mediacodec instance
453
 *****************************************************************************/
Thomas Guillem's avatar
Thomas Guillem committed
454
static int StartMediaCodec(decoder_t *p_dec)
455 456
{
    decoder_sys_t *p_sys = p_dec->p_sys;
457
    union mc_api_args args;
458

459
    if (((p_sys->api->i_quirks & MC_API_QUIRKS_NEED_CSD) && !p_sys->pp_csd))
460
    {
461 462 463
        msg_Warn(p_dec, "waiting for extra data for codec %4.4s",
                 (const char *)&p_dec->fmt_in.i_codec);
        return VLC_ENOOBJ;
464
    }
465

466
    if (p_dec->fmt_in.i_cat == VIDEO_ES)
467
    {
468 469
        if (!p_sys->u.video.i_width || !p_sys->u.video.i_height)
        {
470 471 472
            msg_Warn(p_dec, "waiting for a valid video size for codec %4.4s",
                     (const char *)&p_dec->fmt_in.i_codec);
            return VLC_ENOOBJ;
473 474 475
        }
        args.video.i_width = p_sys->u.video.i_width;
        args.video.i_height = p_sys->u.video.i_height;
476

477
        switch (p_dec->fmt_in.video.orientation)
478
        {
479 480 481 482 483 484 485 486 487 488 489
            case ORIENT_ROTATED_90:
                args.video.i_angle = 90;
                break;
            case ORIENT_ROTATED_180:
                args.video.i_angle = 180;
                break;
            case ORIENT_ROTATED_270:
                args.video.i_angle = 270;
                break;
            default:
                args.video.i_angle = 0;
490
        }
491

492
        /* Configure again if h264 profile changed */
493 494 495 496 497 498 499
        if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
         && !p_sys->u.video.i_h264_profile)
        {
            h264_get_profile_level(&p_dec->fmt_in,
                                   &p_sys->u.video.i_h264_profile, NULL, NULL);
            if (p_sys->u.video.i_h264_profile)
            {
500 501
                if (p_sys->api->configure(p_sys->api,
                                          p_sys->u.video.i_h264_profile) != 0 )
502 503 504
                    return VLC_EGENERIC;
            }
        }
505

506
        if (!p_sys->u.video.p_awh && var_InheritBool(p_dec, CFG_PREFIX "dr"))
507 508 509 510 511 512 513 514 515 516 517
        {
            if ((p_sys->u.video.p_awh = AWindowHandler_new(VLC_OBJECT(p_dec))))
            {
                /* Direct rendering:
                 * The surface must be released by the Vout before calling
                 * start. Request a valid OPAQUE Vout to release any non-OPAQUE
                 * Vout that will release the surface.
                 */
                p_dec->fmt_out.video.i_width = p_sys->u.video.i_width;
                p_dec->fmt_out.video.i_height = p_sys->u.video.i_height;
                p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
518 519 520 521 522 523 524 525
                if (decoder_UpdateVideoFormat(p_dec) != 0)
                {
                    msg_Err(p_dec, "Opaque Vout request failed: "
                                   "fallback to non opaque");

                    AWindowHandler_destroy(p_sys->u.video.p_awh);
                    p_sys->u.video.p_awh = NULL;
                }
526 527
            }
        }
528
        args.video.p_awh = p_sys->u.video.p_awh;
529 530
        args.video.b_tunneled_playback = args.video.p_awh ?
                var_InheritBool(p_dec, CFG_PREFIX "tunneled-playback") : false;
531
    }
Thomas Guillem's avatar
Thomas Guillem committed
532 533 534 535 536 537 538
    else
    {
        date_Set(&p_sys->u.audio.i_end_date, VLC_TS_INVALID);

        args.audio.i_sample_rate    = p_dec->fmt_in.audio.i_rate;
        args.audio.i_channel_count  = p_dec->p_sys->u.audio.i_channels;
    }
539

540
    return p_sys->api->start(p_sys->api, &args);
541 542 543 544 545
}

/*****************************************************************************
 * StopMediaCodec: Close the mediacodec instance
 *****************************************************************************/
Thomas Guillem's avatar
Thomas Guillem committed
546
static void StopMediaCodec(decoder_t *p_dec)
547 548 549
{
    decoder_sys_t *p_sys = p_dec->p_sys;

550
    /* Remove all pictures that are currently in flight in order
551
     * to prevent the vout from using destroyed output buffers. */
Thomas Guillem's avatar
Thomas Guillem committed
552
    if (p_sys->api->b_direct_rendering)
553
        RemoveInflightPictures(p_dec);
554

Thomas Guillem's avatar
Thomas Guillem committed
555
    p_sys->api->stop(p_sys->api);
556
    if (p_dec->fmt_in.i_cat == VIDEO_ES && p_sys->u.video.p_awh)
557
        AWindowHandler_releaseSurface(p_sys->u.video.p_awh, AWindow_Video);
558 559
}

560 561 562
/*****************************************************************************
 * OpenDecoder: Create the decoder instance
 *****************************************************************************/
Thomas Guillem's avatar
Thomas Guillem committed
563
static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
564
{
Thomas Guillem's avatar
Thomas Guillem committed
565
    decoder_t *p_dec = (decoder_t *)p_this;
566
    decoder_sys_t *p_sys;
Thomas Guillem's avatar
Thomas Guillem committed
567
    mc_api *api;
568
    int i_ret;
569
    size_t i_h264_profile = 0;
570
    const char *mime = NULL;
571

Thomas Guillem's avatar
Thomas Guillem committed
572 573 574
    /* Video or Audio if "mediacodec-audio" bool is true */
    if (p_dec->fmt_in.i_cat != VIDEO_ES && (p_dec->fmt_in.i_cat != AUDIO_ES
     || !var_InheritBool(p_dec, CFG_PREFIX "audio")))
575 576
        return VLC_EGENERIC;

577
    if (p_dec->fmt_in.i_cat == VIDEO_ES)
578
    {
579
        if (!p_dec->fmt_in.video.i_width || !p_dec->fmt_in.video.i_height)
580
        {
581 582
            /* We can handle h264 without a valid video size
             * TODO handle VC1 with no size */
583 584 585 586 587 588
            if (p_dec->fmt_in.i_codec != VLC_CODEC_H264)
            {
                msg_Dbg(p_dec, "resolution (%dx%d) not supported",
                        p_dec->fmt_in.video.i_width, p_dec->fmt_in.video.i_height);
                return VLC_EGENERIC;
            }
589
        }
590 591 592 593 594 595 596 597 598 599

        switch (p_dec->fmt_in.i_codec) {
        case VLC_CODEC_HEVC: mime = "video/hevc"; break;
        case VLC_CODEC_H264: mime = "video/avc"; break;
        case VLC_CODEC_H263: mime = "video/3gpp"; break;
        case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
        case VLC_CODEC_WMV3: mime = "video/x-ms-wmv"; break;
        case VLC_CODEC_VC1:  mime = "video/wvc1"; break;
        case VLC_CODEC_VP8:  mime = "video/x-vnd.on2.vp8"; break;
        case VLC_CODEC_VP9:  mime = "video/x-vnd.on2.vp9"; break;
600
        case VLC_CODEC_MPGV: mime = "video/mpeg2"; break;
601 602
        }
    }
Thomas Guillem's avatar
Thomas Guillem committed
603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
    else
    {
        switch (p_dec->fmt_in.i_codec) {
        case VLC_CODEC_AMR_NB: mime = "audio/3gpp"; break;
        case VLC_CODEC_AMR_WB: mime = "audio/amr-wb"; break;
        case VLC_CODEC_MPGA:
        case VLC_CODEC_MP3:    mime = "audio/mpeg"; break;
        case VLC_CODEC_MP2:    mime = "audio/mpeg-L2"; break;
        case VLC_CODEC_MP4A:   mime = "audio/mp4a-latm"; break;
        case VLC_CODEC_QCELP:  mime = "audio/qcelp"; break;
        case VLC_CODEC_VORBIS: mime = "audio/vorbis"; break;
        case VLC_CODEC_OPUS:   mime = "audio/opus"; break;
        case VLC_CODEC_ALAW:   mime = "audio/g711-alaw"; break;
        case VLC_CODEC_MULAW:  mime = "audio/g711-mlaw"; break;
        case VLC_CODEC_FLAC:   mime = "audio/flac"; break;
        case VLC_CODEC_GSM:    mime = "audio/gsm"; break;
        case VLC_CODEC_A52:    mime = "audio/ac3"; break;
        case VLC_CODEC_EAC3:   mime = "audio/eac3"; break;
        case VLC_CODEC_ALAC:   mime = "audio/alac"; break;
        case VLC_CODEC_DTS:    mime = "audio/vnd.dts"; break;
        /* case VLC_CODEC_: mime = "audio/mpeg-L1"; break; */
        /* case VLC_CODEC_: mime = "audio/aac-adts"; break; */
        }
    }
627 628 629 630 631
    if (!mime)
    {
        msg_Dbg(p_dec, "codec %4.4s not supported",
                (char *)&p_dec->fmt_in.i_codec);
        return VLC_EGENERIC;
632 633
    }

634
    if ((api = calloc(1, sizeof(mc_api))) == NULL)
Thomas Guillem's avatar
Thomas Guillem committed
635 636
        return VLC_ENOMEM;
    api->p_obj = p_this;
637 638 639 640 641 642 643
    api->i_codec = p_dec->fmt_in.i_codec;
    api->i_cat = p_dec->fmt_in.i_cat;
    api->psz_mime = mime;

    if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
        h264_get_profile_level(&p_dec->fmt_in, &i_h264_profile, NULL, NULL);
    if (pf_init(api) != 0)
Thomas Guillem's avatar
Thomas Guillem committed
644 645 646 647
    {
        free(api);
        return VLC_EGENERIC;
    }
648 649
    if (api->configure(api, i_h264_profile) != 0)
    {
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
        /* If the device can't handle video/wvc1,
         * it can probably handle video/x-ms-wmv */
        if (!strcmp(mime, "video/wvc1") && p_dec->fmt_in.i_codec == VLC_CODEC_VC1)
        {
            api->psz_mime = "video/x-ms-wmv";
            if (api->configure(api, i_h264_profile) != 0)
            {
                api->clean(api);
                free(api);
                return (VLC_EGENERIC);
            }
        }
        else
        {
            api->clean(api);
            free(api);
            return VLC_EGENERIC;
        }
668
    }
669 670

    /* Allocate the memory needed to store the decoder's structure */
671
    if ((p_sys = calloc(1, sizeof(*p_sys))) == NULL)
Thomas Guillem's avatar
Thomas Guillem committed
672 673 674
    {
        api->clean(api);
        free(api);
675
        return VLC_ENOMEM;
Thomas Guillem's avatar
Thomas Guillem committed
676
    }
677 678
    p_sys->api = api;
    p_dec->p_sys = p_sys;
679 680 681 682

    p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
    p_dec->fmt_out.video = p_dec->fmt_in.video;
    p_dec->fmt_out.audio = p_dec->fmt_in.audio;
683

684 685 686 687
    vlc_mutex_init(&p_sys->lock);
    vlc_cond_init(&p_sys->cond);
    vlc_cond_init(&p_sys->dec_cond);

688
    if (p_dec->fmt_in.i_cat == VIDEO_ES)
689
    {
690
        p_sys->pf_on_new_block = Video_OnNewBlock;
691
        p_sys->pf_on_flush = Video_OnFlush;
692
        p_sys->pf_process_output = Video_ProcessOutput;
693 694
        p_sys->u.video.i_width = p_dec->fmt_in.video.i_width;
        p_sys->u.video.i_height = p_dec->fmt_in.video.i_height;
695
        p_sys->u.video.i_h264_profile = i_h264_profile;
696

697 698
        p_sys->u.video.timestamp_fifo = timestamp_FifoNew(32);
        if (!p_sys->u.video.timestamp_fifo)
699
            goto bailout;
700

701 702
        TAB_INIT( p_sys->u.video.i_inflight_pictures,
                  p_sys->u.video.pp_inflight_pictures );
703
    }
Thomas Guillem's avatar
Thomas Guillem committed
704 705
    else
    {
706
        p_sys->pf_on_new_block = Audio_OnNewBlock;
707
        p_sys->pf_on_flush = Audio_OnFlush;
708
        p_sys->pf_process_output = Audio_ProcessOutput;
709
        p_sys->u.audio.i_channels = p_dec->fmt_in.audio.i_channels;
Thomas Guillem's avatar
Thomas Guillem committed
710

711
        if ((p_sys->api->i_quirks & MC_API_AUDIO_QUIRKS_NEED_CHANNELS)
Thomas Guillem's avatar
Thomas Guillem committed
712
         && !p_sys->u.audio.i_channels)
713
        {
714 715
            msg_Warn(p_dec, "codec need a valid channel count");
            goto bailout;
716
        }
Thomas Guillem's avatar
Thomas Guillem committed
717
    }
718 719

    if (p_dec->fmt_in.i_extra)
Thomas Guillem's avatar
Thomas Guillem committed
720
    {
721 722 723 724 725 726 727
        /* Try first to configure specific Video CSD */
        if (p_dec->fmt_in.i_cat == VIDEO_ES)
            if (ParseVideoExtra(p_dec) != VLC_SUCCESS)
                return VLC_EGENERIC;

        /* Set default CSD if ParseVideoExtra failed to configure one */
        if (!p_sys->pp_csd)
728
        {
729 730 731 732 733
            struct csd csd;

            csd.p_buf = p_dec->fmt_in.p_extra;
            csd.i_size = p_dec->fmt_in.i_extra;
            CSDDup(p_dec, &csd, 1);
734
        }
Thomas Guillem's avatar
Thomas Guillem committed
735
    }
736

737 738
    i_ret = StartMediaCodec(p_dec);
    switch (i_ret)
739
    {
740 741 742 743 744 745 746 747
    case VLC_SUCCESS:
        break;
    case VLC_ENOOBJ:
        if (p_dec->fmt_in.i_codec == VLC_CODEC_MP4V)
            msg_Warn(p_dec, "late opening with MPEG4 not handled"); /* TODO */
        else
            break;
    default:
748
        msg_Err(p_dec, "StartMediaCodec failed");
749 750 751 752 753 754 755 756 757 758 759
        goto bailout;
    }

    if (vlc_clone(&p_sys->out_thread, OutThread, p_dec,
                  VLC_THREAD_PRIORITY_LOW))
    {
        msg_Err(p_dec, "vlc_clone failed");
        vlc_mutex_unlock(&p_sys->lock);
        goto bailout;
    }

760 761 762 763
    p_dec->pf_decode_video = DecodeVideo;
    p_dec->pf_decode_audio = DecodeAudio;
    p_dec->pf_flush        = DecodeFlush;

764 765
    return VLC_SUCCESS;

766
bailout:
767
    CleanDecoder(p_dec);
768
    return VLC_EGENERIC;
Thomas Guillem's avatar
Thomas Guillem committed
769
}
770

Thomas Guillem's avatar
Thomas Guillem committed
771 772 773 774 775
static int OpenDecoderNdk(vlc_object_t *p_this)
{
    return OpenDecoder(p_this, MediaCodecNdk_Init);
}

Thomas Guillem's avatar
Thomas Guillem committed
776 777 778
static int OpenDecoderJni(vlc_object_t *p_this)
{
    return OpenDecoder(p_this, MediaCodecJni_Init);
779 780
}

781 782 783 784
static void AbortDecoderLocked(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

785
    if (!p_sys->b_aborted)
786
    {
787
        p_sys->b_aborted = true;
788 789 790 791
        vlc_cancel(p_sys->out_thread);
    }
}

792
static void CleanDecoder(decoder_t *p_dec)
793 794 795
{
    decoder_sys_t *p_sys = p_dec->p_sys;

796 797 798
    vlc_mutex_destroy(&p_sys->lock);
    vlc_cond_destroy(&p_sys->cond);
    vlc_cond_destroy(&p_sys->dec_cond);
799

Thomas Guillem's avatar
Thomas Guillem committed
800
    StopMediaCodec(p_dec);
801

802
    CSDFree(p_dec);
Thomas Guillem's avatar
Thomas Guillem committed
803
    p_sys->api->clean(p_sys->api);
804 805 806 807 808 809 810 811

    if (p_dec->fmt_in.i_cat == VIDEO_ES)
    {
        if (p_sys->u.video.timestamp_fifo)
            timestamp_FifoRelease(p_sys->u.video.timestamp_fifo);
        if (p_sys->u.video.p_awh)
            AWindowHandler_destroy(p_sys->u.video.p_awh);
    }
Thomas Guillem's avatar
Thomas Guillem committed
812
    free(p_sys->api);
813 814 815
    free(p_sys);
}

816 817 818 819 820 821 822 823 824
/*****************************************************************************
 * CloseDecoder: Close the decoder instance
 *****************************************************************************/
static void CloseDecoder(vlc_object_t *p_this)
{
    decoder_t *p_dec = (decoder_t *)p_this;
    decoder_sys_t *p_sys = p_dec->p_sys;

    vlc_mutex_lock(&p_sys->lock);
825
    /* Unblock output thread waiting in dequeue_out */
826
    DecodeFlushLocked(p_dec);
827 828
    /* Cancel the output thread */
    AbortDecoderLocked(p_dec);
829
    vlc_mutex_unlock(&p_sys->lock);
830

831 832 833 834 835
    vlc_join(p_sys->out_thread, NULL);

    CleanDecoder(p_dec);
}

836 837 838
/*****************************************************************************
 * vout callbacks
 *****************************************************************************/
839
static void ReleasePicture(decoder_t *p_dec, unsigned i_index, bool b_render)
840
{
Thomas Guillem's avatar
Thomas Guillem committed
841
    decoder_sys_t *p_sys = p_dec->p_sys;
842

Thomas Guillem's avatar
Thomas Guillem committed
843
    p_sys->api->release_out(p_sys->api, i_index, b_render);
844 845 846 847 848 849
}

static void InvalidateAllPictures(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

850 851 852
    for (unsigned int i = 0; i < p_sys->u.video.i_inflight_pictures; ++i)
        AndroidOpaquePicture_Release(p_sys->u.video.pp_inflight_pictures[i],
                                     false);
853 854
}

855
static int InsertInflightPicture(decoder_t *p_dec, picture_sys_t *p_picsys)
856 857 858
{
    decoder_sys_t *p_sys = p_dec->p_sys;

859 860 861 862 863 864 865 866 867
    if (!p_picsys->priv.hw.p_dec)
    {
        p_picsys->priv.hw.p_dec = p_dec;
        p_picsys->priv.hw.pf_release = ReleasePicture;
        TAB_APPEND_CAST((picture_sys_t **),
                        p_sys->u.video.i_inflight_pictures,
                        p_sys->u.video.pp_inflight_pictures,
                        p_picsys);
    } /* else already attached */
868 869 870
    return 0;
}

871 872 873 874 875 876 877 878 879 880
static void RemoveInflightPictures(decoder_t *p_dec)
{
    decoder_sys_t *p_sys = p_dec->p_sys;

    for (unsigned int i = 0; i < p_sys->u.video.i_inflight_pictures; ++i)
        AndroidOpaquePicture_DetachDecoder(p_sys->u.video.pp_inflight_pictures[i]);
    TAB_CLEAN(p_sys->u.video.i_inflight_pictures,
              p_sys->u.video.pp_inflight_pictures);
}

881 882
static int Video_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
                               picture_t **pp_out_pic, block_t **pp_out_block)
883 884
{
    decoder_sys_t *p_sys = p_dec->p_sys;
885 886
    (void) pp_out_block;
    assert(pp_out_pic);
887

888
    if (p_out->type == MC_OUT_TYPE_BUF)
889
    {
890 891
        picture_t *p_pic = NULL;