Move bitstream end check to macroblock level

Additionally, instead of silently truncating the frame upon reaching the end of the buffer, reallocate a larger buffer instead.

common/bs.h

@@ -48,8 +48,6 @@ static inline int bs_pos( bs_t *s )
 
 static inline void bs_write( bs_t *s, int i_count, uint32_t i_bits )
 {
-    if( s->p >= s->p_end - 4 )
-        return;
     while( i_count > 0 )
     {
         if( i_count < 32 )
@@ -72,16 +70,13 @@ static inline void bs_write( bs_t *s, int i_count, uint32_t i_bits )
 
 static inline void bs_write1( bs_t *s, uint32_t i_bit )
 {
-    if( s->p < s->p_end )
+    *s->p <<= 1;
+    *s->p |= i_bit;
+    s->i_left--;
+    if( s->i_left == 0 )
     {
-        *s->p <<= 1;
-        *s->p |= i_bit;
-        s->i_left--;
-        if( s->i_left == 0 )
-        {
-            s->p++;
-            s->i_left = 8;
-        }
+        s->p++;
+        s->i_left = 8;
     }
 }
 

common/cabac.c

@@ -866,8 +866,6 @@ static inline void x264_cabac_putbyte( x264_cabac_t *cb )
         {
             int carry = out >> 8;
             int bytes_outstanding = cb->i_bytes_outstanding;
-            if( cb->p + bytes_outstanding + 1 >= cb->p_end )
-                return;
             // this can't modify before the beginning of the stream because
             // that would correspond to a probability > 1.
             // it will write before the beginning of the stream, which is ok
@@ -955,9 +953,6 @@ void x264_cabac_encode_flush( x264_t *h, x264_cabac_t *cb )
     cb->i_queue = 8;
     x264_cabac_putbyte( cb );
 
-    if( cb->p + cb->i_bytes_outstanding + 1 >= cb->p_end )
-        return; //FIXME throw an error instead of silently truncating the frame
-
     while( cb->i_bytes_outstanding > 0 )
     {
         *(cb->p++) = 0xff;

common/x86/cabac-a.asm

@@ -124,7 +124,6 @@ cglobal x264_cabac_encode_decision_asm, 0,7
     mov   [r0+cb.queue], t3d
     cmp   t3d, 8
     jge .putbyte
-.ret:
     REP_RET
 .putbyte:
     ; alive: t0=cb t3=queue t6=low
@@ -144,9 +143,6 @@ cglobal x264_cabac_encode_decision_asm, 0,7
     je .postpone
     mov   t5d, [r0+cb.bytes_outstanding]
     shr   t1d, 8 ; carry
-    lea   t6, [t4+t5+1]
-    cmp   t6, [r0+cb.end]
-    jge .ret
     add   [t4-1], t1b
     test  t5d, t5d
     jz .no_outstanding

encoder/encoder.c

@@ -290,6 +290,34 @@ static void x264_slice_header_write( bs_t *s, x264_slice_header_t *sh, int i_nal
     }
 }
 
+/* If we are within a reasonable distance of the end of the memory allocated for the bitstream, */
+/* reallocate, adding an arbitrary amount of space (100 kilobytes). */
+static void x264_bitstream_check_buffer( x264_t *h )
+{
+    if( ( h->param.b_cabac && (h->cabac.p_end - h->cabac.p < 2500) )
+     || ( h->out.bs.p_end - h->out.bs.p < 2500 ) )
+    {
+        uint8_t *bs_bak = h->out.p_bitstream;
+        intptr_t delta;
+        int i;
+
+        h->out.i_bitstream += 100000;
+        h->out.p_bitstream = x264_realloc( h->out.p_bitstream, h->out.i_bitstream );
+        delta = h->out.p_bitstream - bs_bak;
+
+        h->out.bs.p_start += delta;
+        h->out.bs.p += delta;
+        h->out.bs.p_end = h->out.p_bitstream + h->out.i_bitstream;
+
+        h->cabac.p_start += delta;
+        h->cabac.p += delta;
+        h->cabac.p_end = h->out.p_bitstream + h->out.i_bitstream;
+
+        for( i = 0; i <= h->out.i_nal; i++ )
+            h->out.nal[i].p_payload += delta;
+    }
+}
+
 /****************************************************************************
  *
  ****************************************************************************
@@ -1090,6 +1118,8 @@ static void x264_slice_write( x264_t *h )
         /* encode this macroblock -> be careful it can change the mb type to P_SKIP if needed */
         x264_macroblock_encode( h );
 
+        x264_bitstream_check_buffer( h );
+
         if( h->param.b_cabac )
         {
             if( mb_xy > h->sh.i_first_mb && !(h->sh.b_mbaff && (i_mb_y&1)) )