packetizer: h264: extract spsext with sps/pps

spsext should be in avcC for high profiles

modules/codec/omxil/mediacodec.c

@@ -268,12 +268,14 @@ static int H264SetCSD(decoder_t *p_dec, void *p_buf, size_t i_size,
                       bool *p_size_changed)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
-    uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL;
-    size_t i_sps_size = 0, i_pps_size = 0;
+    uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL, *p_ext_buf = NULL;
+    size_t i_sps_size = 0, i_pps_size = 0, i_ext_size = 0;
 
     /* Check if p_buf contains a valid SPS PPS */
-    if (h264_get_spspps(p_buf, i_size, &p_sps_buf, &i_sps_size,
-                        &p_pps_buf, &i_pps_size) == 0 )
+    if (h264_get_spspps(p_buf, i_size,
+                        &p_sps_buf, &i_sps_size,
+                        &p_pps_buf, &i_pps_size,
+                        &p_ext_buf, &i_ext_size) == 0 )
     {
         struct csd csd[2];
         int i_csd_count = 0;

modules/codec/videotoolbox.m

@@ -329,8 +329,8 @@ static int StartVideoToolbox(decoder_t *p_dec, block_t *p_block)
             i_ret = VLC_SUCCESS;
         }
 
-        uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL;
-        size_t i_sps_size = 0, i_pps_size = 0;
+        uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL, *p_ext_buf = NULL;
+        size_t i_sps_size = 0, i_pps_size = 0, i_ext_size = 0;
         if (!p_buf) {
             msg_Warn(p_dec, "no valid extradata or conversion failed");
             return VLC_EGENERIC;
@@ -338,12 +338,10 @@ static int StartVideoToolbox(decoder_t *p_dec, block_t *p_block)
 
         /* get the SPS and PPS units from the NAL unit which is either
          * part of the demuxer's avvC atom or the mid stream data block */
-        i_ret = h264_get_spspps(p_buf,
-                                i_buf,
-                                &p_sps_buf,
-                                &i_sps_size,
-                                &p_pps_buf,
-                                &i_pps_size);
+        i_ret = h264_get_spspps(p_buf, i_buf,
+                                &p_sps_buf, &i_sps_size,
+                                &p_pps_buf, &i_pps_size,
+                                &p_ext_buf, &i_ext_size);
         if(p_alloc_buf)
             free(p_alloc_buf);
         if (i_ret != VLC_SUCCESS) {

modules/mux/mp4/libmp4mux.c

@@ -807,11 +807,15 @@ static bo_t *GetAvcCTag(es_format_t *p_fmt)
         return NULL;
     uint8_t *p_sps = NULL;
     uint8_t *p_pps = NULL;
+    uint8_t *p_ext = NULL;
     size_t   i_sps_size = 0;
     size_t   i_pps_size = 0;
+    size_t   i_ext_size = 0;
 
     if( h264_get_spspps(p_fmt->p_extra, p_fmt->i_extra,
-                        &p_sps, &i_sps_size, &p_pps, &i_pps_size) != 0 || !p_sps || !p_pps )
+                        &p_sps, &i_sps_size,
+                        &p_pps, &i_pps_size,
+                        &p_ext, &i_ext_size ) != 0 || !p_sps || !p_pps )
     {
         bo_free(avcC);
         return NULL;

modules/packetizer/h264_nal.c

@@ -159,10 +159,11 @@ void h264_AVC_to_AnnexB( uint8_t *p_buf, uint32_t i_len,
 
 int h264_get_spspps( uint8_t *p_buf, size_t i_buf,
                      uint8_t **pp_sps, size_t *p_sps_size,
-                     uint8_t **pp_pps, size_t *p_pps_size )
+                     uint8_t **pp_pps, size_t *p_pps_size,
+                     uint8_t **pp_ext, size_t *p_ext_size )
 {
-    uint8_t *p_sps = NULL, *p_pps = NULL;
-    size_t i_sps_size = 0, i_pps_size = 0;
+    uint8_t *p_sps = NULL, *p_pps = NULL, *p_ext = NULL;
+    size_t i_sps_size = 0, i_pps_size = 0, i_ext_size = 0;
     int i_nal_type = H264_NAL_UNKNOWN;
     bool b_first_nal = true;
     bool b_has_zero_byte = false;
@@ -181,8 +182,10 @@ int h264_get_spspps( uint8_t *p_buf, size_t i_buf,
                     i_sps_size = p_buf - p_sps - (b_has_zero_byte ? 1 : 0);
                 if( i_nal_type == H264_NAL_PPS )
                     i_pps_size = p_buf - p_pps - (b_has_zero_byte ? 1 : 0);
+                if( i_nal_type == H264_NAL_SPS_EXT )
+                    i_ext_size = p_buf - p_pps - (b_has_zero_byte ? 1 : 0);
 
-                if( i_sps_size && i_pps_size )
+                if( i_sps_size && i_pps_size && i_ext_size ) /* early end */
                     break;
             }
 
@@ -201,6 +204,8 @@ int h264_get_spspps( uint8_t *p_buf, size_t i_buf,
                 p_sps = p_buf - 1;
             if( i_nal_type == H264_NAL_PPS && !p_pps )
                 p_pps = p_buf - 1;
+            if( i_nal_type == H264_NAL_SPS_EXT && !p_ext )
+                p_ext = p_buf - 1;
 
             /* cf. 7.4.1.2.3 */
             if( i_nal_type > 18 || ( i_nal_type >= 10 && i_nal_type <= 12 ) )
@@ -228,6 +233,8 @@ int h264_get_spspps( uint8_t *p_buf, size_t i_buf,
             i_sps_size = p_buf - p_sps;
         if( !i_pps_size && i_nal_type == H264_NAL_PPS )
             i_pps_size = p_buf - p_pps;
+        if( !i_ext_size && i_nal_type == H264_NAL_SPS_EXT )
+            i_ext_size = p_buf - p_ext;
     }
     if( ( !p_sps || !i_sps_size ) && ( !p_pps || !i_pps_size ) )
         return -1;
@@ -235,6 +242,8 @@ int h264_get_spspps( uint8_t *p_buf, size_t i_buf,
     *p_sps_size = i_sps_size;
     *pp_pps = p_pps;
     *p_pps_size = i_pps_size;
+    *pp_ext = p_ext;
+    *p_ext_size = i_ext_size;
 
     return 0;
 }

modules/packetizer/h264_nal.h

@@ -159,7 +159,8 @@ void h264_AVC_to_AnnexB( uint8_t *p_buf, uint32_t i_len,
  * Returns 0 if a SPS and/or a PPS is found */
 int h264_get_spspps( uint8_t *p_buf, size_t i_buf,
                      uint8_t **pp_sps, size_t *p_sps_size,
-                     uint8_t **pp_pps, size_t *p_pps_size );
+                     uint8_t **pp_pps, size_t *p_pps_size,
+                     uint8_t **pp_ext, size_t *p_ext_size );
 
 /* Create a AVCDecoderConfigurationRecord from SPS/PPS
  * Returns a valid block_t on success, must be freed with block_Release */