common/base.c

@@ -99,13 +99,18 @@ void x264_log_internal( int i_level, const char *psz_fmt, ... )
 /****************************************************************************
  * x264_malloc:
  ****************************************************************************/
-void *x264_malloc( int i_size )
+void *x264_malloc( int64_t i_size )
 {
+#define HUGE_PAGE_SIZE 2*1024*1024
+#define HUGE_PAGE_THRESHOLD HUGE_PAGE_SIZE*7/8 /* FIXME: Is this optimal? */
+    if( i_size < 0 || i_size > (SIZE_MAX - HUGE_PAGE_SIZE) /*|| i_size > (SIZE_MAX - NATIVE_ALIGN - sizeof(void **))*/ )
+    {
+        x264_log_internal( X264_LOG_ERROR, "invalid size of malloc: %"PRId64"\n", i_size );
+        return NULL;
+    }
     uint8_t *align_buf = NULL;
 #if HAVE_MALLOC_H
 #if HAVE_THP
-#define HUGE_PAGE_SIZE 2*1024*1024
-#define HUGE_PAGE_THRESHOLD HUGE_PAGE_SIZE*7/8 /* FIXME: Is this optimal? */
     /* Attempt to allocate huge pages to reduce TLB misses. */
     if( i_size >= HUGE_PAGE_THRESHOLD )
     {
@@ -118,8 +123,6 @@ void *x264_malloc( int i_size )
         }
     }
     else
-#undef HUGE_PAGE_SIZE
-#undef HUGE_PAGE_THRESHOLD
 #endif
         align_buf = memalign( NATIVE_ALIGN, i_size );
 #else
@@ -132,8 +135,10 @@ void *x264_malloc( int i_size )
     }
 #endif
     if( !align_buf )
-        x264_log_internal( X264_LOG_ERROR, "malloc of size %d failed\n", i_size );
+        x264_log_internal( X264_LOG_ERROR, "malloc of size %"PRId64" failed\n", i_size );
     return align_buf;
+#undef HUGE_PAGE_SIZE
+#undef HUGE_PAGE_THRESHOLD
 }
 
 /****************************************************************************
@@ -242,12 +247,12 @@ REALIGN_STACK int x264_picture_alloc( x264_picture_t *pic, int i_csp, int i_widt
     pic->img.i_csp = i_csp;
     pic->img.i_plane = csp_tab[csp].planes;
     int depth_factor = i_csp & X264_CSP_HIGH_DEPTH ? 2 : 1;
-    int plane_offset[3] = {0};
-    int frame_size = 0;
+    int64_t plane_offset[3] = {0};
+    int64_t frame_size = 0;
     for( int i = 0; i < pic->img.i_plane; i++ )
     {
         int stride = (((int64_t)i_width * csp_tab[csp].width_fix8[i]) >> 8) * depth_factor;
-        int plane_size = (((int64_t)i_height * csp_tab[csp].height_fix8[i]) >> 8) * stride;
+        int64_t plane_size = (((int64_t)i_height * csp_tab[csp].height_fix8[i]) >> 8) * stride;
         pic->img.i_stride[i] = stride;
         plane_offset[i] = frame_size;
         frame_size += plane_size;

common/base.h

@@ -263,7 +263,7 @@ X264_API void x264_log_internal( int i_level, const char *psz_fmt, ... );
 
 /* x264_malloc : will do or emulate a memalign
  * you have to use x264_free for buffers allocated with x264_malloc */
-X264_API void *x264_malloc( int );
+X264_API void *x264_malloc( int64_t );
 X264_API void  x264_free( void * );
 
 /* x264_slurp_file: malloc space for the whole file and read it */
@@ -296,12 +296,12 @@ do {\
 
 #define PREALLOC_INIT\
     int    prealloc_idx = 0;\
-    size_t prealloc_size = 0;\
+    int64_t prealloc_size = 0;\
     uint8_t **preallocs[PREALLOC_BUF_SIZE];
 
 #define PREALLOC( var, size )\
 do {\
-    var = (void*)prealloc_size;\
+    var = (void*)(intptr_t)prealloc_size;\
     preallocs[prealloc_idx++] = (uint8_t**)&var;\
     prealloc_size += ALIGN(size, NATIVE_ALIGN);\
 } while( 0 )

common/frame.c

@@ -162,7 +162,7 @@ static x264_frame_t *frame_new( x264_t *h, int b_fdec )
 
     for( int p = 0; p < luma_plane_count; p++ )
     {
-        int luma_plane_size = align_plane_size( frame->i_stride[p] * (frame->i_lines[p] + 2*i_padv), disalign );
+        int64_t luma_plane_size = align_plane_size( frame->i_stride[p] * (frame->i_lines[p] + 2*i_padv), disalign );
         if( h->param.analyse.i_subpel_refine && b_fdec )
             luma_plane_size *= 4;
 
@@ -205,7 +205,7 @@ static x264_frame_t *frame_new( x264_t *h, int b_fdec )
     {
         if( h->frames.b_have_lowres )
         {
-            int luma_plane_size = align_plane_size( frame->i_stride_lowres * (frame->i_lines[0]/2 + 2*PADV), disalign );
+            int64_t luma_plane_size = align_plane_size( frame->i_stride_lowres * (frame->i_lines[0]/2 + 2*PADV), disalign );
 
             PREALLOC( frame->buffer_lowres, (4 * luma_plane_size + padh_align) * sizeof(pixel) );
 
@@ -244,7 +244,7 @@ static x264_frame_t *frame_new( x264_t *h, int b_fdec )
 
     for( int p = 0; p < luma_plane_count; p++ )
     {
-        int luma_plane_size = align_plane_size( frame->i_stride[p] * (frame->i_lines[p] + 2*i_padv), disalign );
+        int64_t luma_plane_size = align_plane_size( frame->i_stride[p] * (frame->i_lines[p] + 2*i_padv), disalign );
         if( h->param.analyse.i_subpel_refine && b_fdec )
         {
             for( int i = 0; i < 4; i++ )
@@ -274,7 +274,7 @@ static x264_frame_t *frame_new( x264_t *h, int b_fdec )
     {
         if( h->frames.b_have_lowres )
         {
-            int luma_plane_size = align_plane_size( frame->i_stride_lowres * (frame->i_lines[0]/2 + 2*PADV), disalign );
+            int64_t luma_plane_size = align_plane_size( frame->i_stride_lowres * (frame->i_lines[0]/2 + 2*PADV), disalign );
             for( int i = 0; i < 4; i++ )
                 frame->lowres[i] = frame->buffer_lowres + frame->i_stride_lowres * PADV + PADH + padh_align + i * luma_plane_size;
 

common/opencl.c

@@ -131,8 +131,11 @@ static cl_program opencl_cache_load( x264_t *h, const char *dev_name, const char
     uint8_t *binary = NULL;
 
     fseek( fp, 0, SEEK_END );
-    size_t size = ftell( fp );
-    rewind( fp );
+    int64_t file_size = ftell( fp );
+    fseek( fp, 0, SEEK_SET );
+    if( file_size < 0 || file_size > SIZE_MAX )
+        goto fail;
+    size_t size = file_size;
     CHECKED_MALLOC( binary, size );
 
     if( fread( binary, 1, size, fp ) != size )

configure

@@ -128,7 +128,7 @@ cl_ldflags() {
 }
 
 cc_check() {
-    if [ -z "$3" ]; then
+    if [ -z "$3$4" ]; then
         if [ -z "$1$2" ]; then
             log_check "whether $CC works"
         elif [ -z "$1" ]; then
@@ -138,7 +138,11 @@ cc_check() {
         fi
     elif [ -z "$1" ]; then
         if [ -z "$2" ]; then
-            log_check "whether $CC supports $3"
+            if [ -z "$3" ]; then
+                log_check "whether $CC supports $4"
+            else
+                log_check "whether $CC supports $3"
+            fi
         else
             log_check "whether $CC supports $3 with $2"
         fi
@@ -149,6 +153,9 @@ cc_check() {
     for arg in $1; do
         echo "#include <$arg>" >> conftest.c
     done
+    if [ -n "$4" ]; then
+        echo "$4" >> conftest.c
+    fi
     echo "int main (void) { $3 return 0; }" >> conftest.c
     if [ $compiler_style = MS ]; then
         cc_cmd="$CC conftest.c $(cc_cflags $CFLAGS $CFLAGSCLI $CHECK_CFLAGS $2) -link $(cl_ldflags $2 $LDFLAGSCLI $LDFLAGS)"
@@ -613,6 +620,15 @@ if [ $compiler = GNU ]; then
     if cc_check '' -Werror=unknown-warning-option ; then
         CHECK_CFLAGS="$CHECK_CFLAGS -Werror=unknown-warning-option"
     fi
+    if cc_check '' -Werror=unknown-attributes ; then
+        CHECK_CFLAGS="$CHECK_CFLAGS -Werror=unknown-attributes"
+    fi
+    if cc_check '' -Werror=attributes ; then
+        CHECK_CFLAGS="$CHECK_CFLAGS -Werror=attributes"
+    fi
+    if cc_check '' -Werror=ignored-attributes ; then
+        CHECK_CFLAGS="$CHECK_CFLAGS -Werror=ignored-attributes"
+    fi
 fi
 
 libm=""
@@ -888,34 +904,36 @@ if [ $shared = yes -a \( $ARCH = "X86_64" -o $ARCH = "PPC" -o $ARCH = "ALPHA" -o
     pic="yes"
 fi
 
-if [ $compiler = GNU -a \( $ARCH = X86 -o $ARCH = X86_64 \) ] ; then
-    if cc_check '' -mpreferred-stack-boundary=6 ; then
-        CFLAGS="$CFLAGS -mpreferred-stack-boundary=6"
-        stack_alignment=64
-    elif cc_check '' -mstack-alignment=64 ; then
-        CFLAGS="$CFLAGS -mstack-alignment=64"
-        stack_alignment=64
-    elif [ $stack_alignment -lt 16 ] ; then
-        if cc_check '' -mpreferred-stack-boundary=4 ; then
-            CFLAGS="$CFLAGS -mpreferred-stack-boundary=4"
-            stack_alignment=16
-        elif cc_check '' -mstack-alignment=16 ; then
-            CFLAGS="$CFLAGS -mstack-alignment=16"
-            stack_alignment=16
+if cc_check '' '' '' '__attribute__((force_align_arg_pointer))' ; then
+    if [ $compiler = GNU -a \( $ARCH = X86 -o $ARCH = X86_64 \) ] ; then
+        if cc_check '' -mpreferred-stack-boundary=6 ; then
+            CFLAGS="$CFLAGS -mpreferred-stack-boundary=6"
+            stack_alignment=64
+        elif cc_check '' -mstack-alignment=64 ; then
+            CFLAGS="$CFLAGS -mstack-alignment=64"
+            stack_alignment=64
+        elif [ $stack_alignment -lt 16 ] ; then
+            if cc_check '' -mpreferred-stack-boundary=4 ; then
+                CFLAGS="$CFLAGS -mpreferred-stack-boundary=4"
+                stack_alignment=16
+            elif cc_check '' -mstack-alignment=16 ; then
+                CFLAGS="$CFLAGS -mstack-alignment=16"
+                stack_alignment=16
+            fi
         fi
-    fi
-elif [ $compiler = ICC -a $ARCH = X86 ]; then
-    # icc on linux has various degrees of mod16 stack support
-    if [ $SYS = LINUX ]; then
-        # >= 12 defaults to a mod16 stack
-        if cpp_check "" "" "__INTEL_COMPILER >= 1200" ; then
-            stack_alignment=16
-        # 11 <= x < 12 is capable of keeping a mod16 stack, but defaults to not doing so.
-        elif cpp_check "" "" "__INTEL_COMPILER >= 1100" ; then
-            CFLAGS="$CFLAGS -falign-stack=assume-16-byte"
-            stack_alignment=16
+    elif [ $compiler = ICC -a $ARCH = X86 ]; then
+        # icc on linux has various degrees of mod16 stack support
+        if [ $SYS = LINUX ]; then
+            # >= 12 defaults to a mod16 stack
+            if cpp_check "" "" "__INTEL_COMPILER >= 1200" ; then
+                stack_alignment=16
+            # 11 <= x < 12 is capable of keeping a mod16 stack, but defaults to not doing so.
+            elif cpp_check "" "" "__INTEL_COMPILER >= 1100" ; then
+                CFLAGS="$CFLAGS -falign-stack=assume-16-byte"
+                stack_alignment=16
+            fi
+            # < 11 is completely incapable of keeping a mod16 stack
         fi
-        # < 11 is completely incapable of keeping a mod16 stack
     fi
 fi
 

encoder/encoder.c

@@ -370,6 +370,8 @@ static int bitstream_check_buffer_internal( x264_t *h, int size, int b_cabac, in
     if( (b_cabac && (h->cabac.p_end - h->cabac.p < size)) ||
         (h->out.bs.p_end - h->out.bs.p < size) )
     {
+        if( size > INT_MAX - h->out.i_bitstream )
+            return -1;
         int buf_size = h->out.i_bitstream + size;
         uint8_t *buf = x264_malloc( buf_size );
         if( !buf )
@@ -470,7 +472,9 @@ static int validate_parameters( x264_t *h, int b_open )
     }
 #endif
 
-    if( h->param.i_width <= 0 || h->param.i_height <= 0 )
+#define MAX_RESOLUTION 16384
+    if( h->param.i_width <= 0 || h->param.i_height <= 0 ||
+        h->param.i_width > MAX_RESOLUTION || h->param.i_height > MAX_RESOLUTION )
     {
         x264_log( h, X264_LOG_ERROR, "invalid width x height (%dx%d)\n",
                   h->param.i_width, h->param.i_height );
@@ -862,8 +866,8 @@ static int validate_parameters( x264_t *h, int b_open )
     h->param.rc.f_rf_constant_max = x264_clip3f( h->param.rc.f_rf_constant_max, -QP_BD_OFFSET, 51 );
     h->param.rc.i_qp_constant = x264_clip3( h->param.rc.i_qp_constant, -1, QP_MAX );
     h->param.analyse.i_subpel_refine = x264_clip3( h->param.analyse.i_subpel_refine, 0, 11 );
-    h->param.rc.f_ip_factor = X264_MAX( h->param.rc.f_ip_factor, 0.01f );
-    h->param.rc.f_pb_factor = X264_MAX( h->param.rc.f_pb_factor, 0.01f );
+    h->param.rc.f_ip_factor = x264_clip3f( h->param.rc.f_ip_factor, 0.01, 10.0 );
+    h->param.rc.f_pb_factor = x264_clip3f( h->param.rc.f_pb_factor, 0.01, 10.0 );
     if( h->param.rc.i_rc_method == X264_RC_CRF )
     {
         h->param.rc.i_qp_constant = h->param.rc.f_rf_constant + QP_BD_OFFSET;
@@ -1647,9 +1651,13 @@ x264_t *x264_encoder_open( x264_param_t *param )
     }
 
     h->out.i_nal = 0;
-    h->out.i_bitstream = X264_MAX( 1000000, h->param.i_width * h->param.i_height * 4
-        * ( h->param.rc.i_rc_method == X264_RC_ABR ? pow( 0.95, h->param.rc.i_qp_min )
-          : pow( 0.95, h->param.rc.i_qp_constant ) * X264_MAX( 1, h->param.rc.f_ip_factor )));
+    h->out.i_bitstream = x264_clip3f(
+        h->param.i_width * h->param.i_height * 4
+        * ( h->param.rc.i_rc_method == X264_RC_ABR
+            ? pow( 0.95, h->param.rc.i_qp_min )
+            : pow( 0.95, h->param.rc.i_qp_constant ) * X264_MAX( 1, h->param.rc.f_ip_factor ) ),
+        1000000, INT_MAX/3
+    );
 
     h->nal_buffer_size = h->out.i_bitstream * 3/2 + 4 + 64; /* +4 for startcode, +64 for nal_escape assembly padding */
     CHECKED_MALLOC( h->nal_buffer, h->nal_buffer_size );
@@ -1940,11 +1948,13 @@ static int nal_end( x264_t *h )
 }
 
 static int check_encapsulated_buffer( x264_t *h, x264_t *h0, int start,
-                                      int previous_nal_size, int necessary_size )
+                                      int64_t previous_nal_size, int64_t necessary_size )
 {
     if( h0->nal_buffer_size < necessary_size )
     {
         necessary_size *= 2;
+        if( necessary_size > INT_MAX )
+            return -1;
         uint8_t *buf = x264_malloc( necessary_size );
         if( !buf )
             return -1;
@@ -1966,12 +1976,14 @@ static int check_encapsulated_buffer( x264_t *h, x264_t *h0, int start,
 static int encoder_encapsulate_nals( x264_t *h, int start )
 {
     x264_t *h0 = h->thread[0];
-    int nal_size = 0, previous_nal_size = 0;
+    int64_t nal_size = 0, previous_nal_size = 0;
 
     if( h->param.nalu_process )
     {
         for( int i = start; i < h->out.i_nal; i++ )
             nal_size += h->out.nal[i].i_payload;
+        if( nal_size > INT_MAX )
+            return -1;
         return nal_size;
     }
 
@@ -1982,7 +1994,7 @@ static int encoder_encapsulate_nals( x264_t *h, int start )
         nal_size += h->out.nal[i].i_payload;
 
     /* Worst-case NAL unit escaping: reallocate the buffer if it's too small. */
-    int necessary_size = previous_nal_size + nal_size * 3/2 + h->out.i_nal * 4 + 4 + 64;
+    int64_t necessary_size = previous_nal_size + nal_size * 3/2 + h->out.i_nal * 4 + 4 + 64;
     for( int i = start; i < h->out.i_nal; i++ )
         necessary_size += h->out.nal[i].i_padding;
     if( check_encapsulated_buffer( h, h0, start, previous_nal_size, necessary_size ) )
@@ -3879,7 +3891,7 @@ static int encoder_frame_end( x264_t *h, x264_t *thread_current,
      * We don't know the size of the last slice until encapsulation so we add filler to the encapsulated NAL */
     if( h->param.i_avcintra_class )
     {
-        if( check_encapsulated_buffer( h, h->thread[0], h->out.i_nal, frame_size, frame_size + filler ) < 0 )
+        if( check_encapsulated_buffer( h, h->thread[0], h->out.i_nal, frame_size, (int64_t)frame_size + filler ) < 0 )
             return -1;
 
         x264_nal_t *nal = &h->out.nal[h->out.i_nal-1];

encoder/ratecontrol.c

@@ -713,8 +713,9 @@ void x264_ratecontrol_init_reconfigurable( x264_t *h, int b_init )
         rc->vbv_max_rate = vbv_max_bitrate;
         rc->buffer_size = vbv_buffer_size;
         rc->single_frame_vbv = rc->buffer_rate * 1.1 > rc->buffer_size;
-        rc->cbr_decay = 1.0 - rc->buffer_rate / rc->buffer_size
-                      * 0.5 * X264_MAX(0, 1.5 - rc->buffer_rate * rc->fps / rc->bitrate);
+        if( rc->b_abr && h->param.rc.i_rc_method == X264_RC_ABR )
+            rc->cbr_decay = 1.0 - rc->buffer_rate / rc->buffer_size
+                          * 0.5 * X264_MAX(0, 1.5 - rc->buffer_rate * rc->fps / rc->bitrate);
         if( h->param.rc.i_rc_method == X264_RC_CRF && h->param.rc.f_rf_constant_max )
         {
             rc->rate_factor_max_increment = h->param.rc.f_rf_constant_max - h->param.rc.f_rf_constant;

filters/video/resize.c

@@ -483,6 +483,10 @@ static int init( hnd_t *handle, cli_vid_filter_t *filter, video_info_t *info, x2
     int src_pix_fmt_inv = convert_csp_to_pix_fmt( info->csp ^ X264_CSP_HIGH_DEPTH );
     int dst_pix_fmt_inv = convert_csp_to_pix_fmt( h->dst_csp ^ X264_CSP_HIGH_DEPTH );
 
+    FAIL_IF_ERROR( h->dst.width <= 0 || h->dst.height <= 0 ||
+                   h->dst.width > MAX_RESOLUTION || h->dst.height > MAX_RESOLUTION,
+                   "invalid width x height (%dx%d)\n", h->dst.width, h->dst.height );
+
     /* confirm swscale can support this conversion */
     FAIL_IF_ERROR( src_pix_fmt == AV_PIX_FMT_NONE && src_pix_fmt_inv != AV_PIX_FMT_NONE,
                    "input colorspace %s with bit depth %d is not supported\n", av_get_pix_fmt_name( src_pix_fmt_inv ),

input/input.c

@@ -65,22 +65,22 @@ int x264_cli_csp_depth_factor( int csp )
     return (csp & X264_CSP_HIGH_DEPTH) ? 2 : 1;
 }
 
-uint64_t x264_cli_pic_plane_size( int csp, int width, int height, int plane )
+int64_t x264_cli_pic_plane_size( int csp, int width, int height, int plane )
 {
     int csp_mask = csp & X264_CSP_MASK;
     if( x264_cli_csp_is_invalid( csp ) || plane < 0 || plane >= x264_cli_csps[csp_mask].planes )
         return 0;
-    uint64_t size = (uint64_t)width * height;
+    int64_t size = (int64_t)width * height;
     size *= x264_cli_csps[csp_mask].width[plane] * x264_cli_csps[csp_mask].height[plane];
     size *= x264_cli_csp_depth_factor( csp );
     return size;
 }
 
-uint64_t x264_cli_pic_size( int csp, int width, int height )
+int64_t x264_cli_pic_size( int csp, int width, int height )
 {
     if( x264_cli_csp_is_invalid( csp ) )
         return 0;
-    uint64_t size = 0;
+    int64_t size = 0;
     int csp_mask = csp & X264_CSP_MASK;
     for( int i = 0; i < x264_cli_csps[csp_mask].planes; i++ )
         size += x264_cli_pic_plane_size( csp, width, height, i );
@@ -107,7 +107,7 @@ static int cli_pic_init_internal( cli_pic_t *pic, int csp, int width, int height
 
         if( alloc )
         {
-            size_t size = (size_t)(height * x264_cli_csps[csp_mask].height[i]) * stride;
+            int64_t size = (int64_t)(height * x264_cli_csps[csp_mask].height[i]) * stride;
             pic->img.plane[i] = x264_malloc( size );
             if( !pic->img.plane[i] )
                 return -1;
@@ -182,11 +182,13 @@ int x264_cli_mmap_init( cli_mmap_t *h, FILE *fh )
  * in segfaults. We have to pad the buffer size as a workaround to avoid that. */
 #define MMAP_PADDING 64
 
-void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, size_t size )
+void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, int64_t size )
 {
 #if defined(_WIN32) || HAVE_MMAP
     uint8_t *base;
     int align = offset & h->align_mask;
+    if( size < 0 || size > (SIZE_MAX - MMAP_PADDING - align) )
+        return NULL;
     offset -= align;
     size   += align;
 #ifdef _WIN32
@@ -196,10 +198,10 @@ void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, size_t size )
     {
         /* It's not possible to do the POSIX mmap() remapping trick on Windows, so if the padding crosses a
          * page boundary past the end of the file we have to copy the entire frame into a padded buffer. */
-        if( (base = MapViewOfFile( h->map_handle, FILE_MAP_READ, offset >> 32, offset, size )) )
+        if( (base = MapViewOfFile( h->map_handle, FILE_MAP_READ, (uint64_t)offset >> 32, offset, size )) )
         {
             uint8_t *buf = NULL;
-            HANDLE anon_map = CreateFileMappingW( INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, padded_size, NULL );
+            HANDLE anon_map = CreateFileMappingW( INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, (uint64_t)padded_size >> 32, padded_size, NULL );
             if( anon_map )
             {
                 if( (buf = MapViewOfFile( anon_map, FILE_MAP_WRITE, 0, 0, 0 )) )
@@ -213,7 +215,7 @@ void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, size_t size )
             return buf;
         }
     }
-    else if( (base = MapViewOfFile( h->map_handle, FILE_MAP_READ, offset >> 32, offset, padded_size )) )
+    else if( (base = MapViewOfFile( h->map_handle, FILE_MAP_READ, (uint64_t)offset >> 32, offset, padded_size )) )
     {
         /* PrefetchVirtualMemory() is only available on Windows 8 and newer. */
         if( h->prefetch_virtual_memory )
@@ -249,13 +251,15 @@ void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, size_t size )
     return NULL;
 }
 
-int x264_cli_munmap( cli_mmap_t *h, void *addr, size_t size )
+int x264_cli_munmap( cli_mmap_t *h, void *addr, int64_t size )
 {
 #if defined(_WIN32) || HAVE_MMAP
     void *base = (void*)((intptr_t)addr & ~h->align_mask);
 #ifdef _WIN32
     return !UnmapViewOfFile( base );
 #else
+    if( size < 0 || size > (SIZE_MAX - MMAP_PADDING - ((intptr_t)addr - (intptr_t)base)) )
+        return -1;
     return munmap( base, size + MMAP_PADDING + (intptr_t)addr - (intptr_t)base );
 #endif
 #endif

input/input.h

@@ -132,8 +132,8 @@ int      x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height );
 int      x264_cli_pic_alloc_aligned( cli_pic_t *pic, int csp, int width, int height );
 int      x264_cli_pic_init_noalloc( cli_pic_t *pic, int csp, int width, int height );
 void     x264_cli_pic_clean( cli_pic_t *pic );
-uint64_t x264_cli_pic_plane_size( int csp, int width, int height, int plane );
-uint64_t x264_cli_pic_size( int csp, int width, int height );
+int64_t  x264_cli_pic_plane_size( int csp, int width, int height, int plane );
+int64_t  x264_cli_pic_size( int csp, int width, int height );
 const x264_cli_csp_t *x264_cli_get_csp( int csp );
 
 typedef struct
@@ -151,8 +151,8 @@ typedef struct
 } cli_mmap_t;
 
 int x264_cli_mmap_init( cli_mmap_t *h, FILE *fh );
-void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, size_t size );
-int x264_cli_munmap( cli_mmap_t *h, void *addr, size_t size );
+void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, int64_t size );
+int x264_cli_munmap( cli_mmap_t *h, void *addr, int64_t size );
 void x264_cli_mmap_close( cli_mmap_t *h );
 
 #endif

input/raw.c

@@ -33,8 +33,8 @@ typedef struct
 {
     FILE *fh;
     int next_frame;
-    uint64_t plane_size[4];
-    uint64_t frame_size;
+    int64_t plane_size[4];
+    int64_t frame_size;
     int bit_depth;
     cli_mmap_t mmap;
     int use_mmap;
@@ -96,7 +96,7 @@ static int open_file( char *psz_filename, hnd_t *p_handle, video_info_t *info, c
     if( x264_is_regular_file( h->fh ) )
     {
         fseek( h->fh, 0, SEEK_END );
-        uint64_t size = ftell( h->fh );
+        int64_t size = ftell( h->fh );
         fseek( h->fh, 0, SEEK_SET );
         info->num_frames = size / h->frame_size;
         FAIL_IF_ERROR( !info->num_frames, "empty input file\n" );
@@ -129,9 +129,9 @@ static int read_frame_internal( cli_pic_t *pic, raw_hnd_t *h, int bit_depth_uc )
             /* upconvert non 16bit high depth planes to 16bit using the same
              * algorithm as used in the depth filter. */
             uint16_t *plane = (uint16_t*)pic->img.plane[i];
-            uint64_t pixel_count = h->plane_size[i];
+            int64_t pixel_count = h->plane_size[i];
             int lshift = 16 - h->bit_depth;
-            for( uint64_t j = 0; j < pixel_count; j++ )
+            for( int64_t j = 0; j < pixel_count; j++ )
                 plane[j] = plane[j] << lshift;
         }
     }

input/timecode.c

@@ -105,7 +105,7 @@ static int parse_tcfile( FILE *tcfile_in, timecode_hnd_t *h, video_info_t *info
 #define NO_TIMECODE_LINE (buff[0] == '#' || buff[0] == '\n' || buff[0] == '\r')
     if( tcfv == 1 )
     {
-        uint64_t file_pos;
+        int64_t file_pos;
         double assume_fps, seq_fps;
         int start, end = -1;
         int prev_start = -1, prev_end = -1;
@@ -221,7 +221,7 @@ static int parse_tcfile( FILE *tcfile_in, timecode_hnd_t *h, video_info_t *info
     }
     else    /* tcfv == 2 */
     {
-        uint64_t file_pos = ftell( tcfile_in );
+        int64_t file_pos = ftell( tcfile_in );
 
         h->stored_pts_num = 0;
         while( fgets( buff, sizeof(buff), tcfile_in ) != NULL )

input/y4m.c

@@ -34,8 +34,8 @@ typedef struct
     int next_frame;
     int seq_header_len;
     int frame_header_len;
-    uint64_t frame_size;
-    uint64_t plane_size[3];
+    int64_t frame_size;
+    int64_t plane_size[3];
     int bit_depth;
     cli_mmap_t mmap;
     int use_mmap;
@@ -213,7 +213,7 @@ static int open_file( char *psz_filename, hnd_t *p_handle, video_info_t *info, c
 
     if( x264_is_regular_file( h->fh ) )
     {
-        uint64_t init_pos = ftell( h->fh );
+        int64_t init_pos = ftell( h->fh );
 
         /* Find out the length of the frame header */
         int len = 1;
@@ -224,7 +224,7 @@ static int open_file( char *psz_filename, hnd_t *p_handle, video_info_t *info, c
         h->frame_size += len;
 
         fseek( h->fh, 0, SEEK_END );
-        uint64_t i_size = ftell( h->fh );
+        int64_t i_size = ftell( h->fh );
         fseek( h->fh, init_pos, SEEK_SET );
         info->num_frames = (i_size - h->seq_header_len) / h->frame_size;
         FAIL_IF_ERROR( !info->num_frames, "empty input file\n" );
@@ -285,9 +285,9 @@ static int read_frame_internal( cli_pic_t *pic, y4m_hnd_t *h, int bit_depth_uc )
             /* upconvert non 16bit high depth planes to 16bit using the same
              * algorithm as used in the depth filter. */
             uint16_t *plane = (uint16_t*)pic->img.plane[i];
-            uint64_t pixel_count = h->plane_size[i];
+            int64_t pixel_count = h->plane_size[i];
             int lshift = 16 - h->bit_depth;
-            for( uint64_t j = 0; j < pixel_count; j++ )
+            for( int64_t j = 0; j < pixel_count; j++ )
                 plane[j] = plane[j] << lshift;
         }
     }

output/flv.c

@@ -332,7 +332,7 @@ static int close_file( hnd_t handle, int64_t largest_pts, int64_t second_largest
     if( x264_is_regular_file( c->fp ) && total_duration > 0 )
     {
         double framerate;
-        uint64_t filesize = ftell( c->fp );
+        int64_t filesize = ftell( c->fp );
 
         if( p_flv->i_framerate_pos )
         {
@@ -342,7 +342,7 @@ static int close_file( hnd_t handle, int64_t largest_pts, int64_t second_largest
 
         CHECK( rewrite_amf_double( c->fp, p_flv->i_duration_pos, total_duration ) );
         CHECK( rewrite_amf_double( c->fp, p_flv->i_filesize_pos, filesize ) );
-        CHECK( rewrite_amf_double( c->fp, p_flv->i_bitrate_pos, filesize * 8 / ( total_duration * 1000 ) ) );
+        CHECK( rewrite_amf_double( c->fp, p_flv->i_bitrate_pos, filesize * 8.0 / ( total_duration * 1000 ) ) );
     }
     ret = 0;
 

x264.c

@@ -1667,6 +1667,10 @@ generic_option:
                   info.height, info.interlaced ? 'i' : 'p', info.sar_width, info.sar_height,
                   info.fps_num, info.fps_den, info.vfr ? 'v' : 'c' );
 
+    FAIL_IF_ERROR( info.width <= 0 || info.height <= 0 ||
+                   info.width > MAX_RESOLUTION || info.height > MAX_RESOLUTION,
+                   "invalid width x height (%dx%d)\n", info.width, info.height );
+
     if( tcfile_name )
     {
         FAIL_IF_ERROR( b_user_fps, "--fps + --tcfile-in is incompatible.\n" );

x264.h

@@ -45,7 +45,7 @@ extern "C" {
 
 #include "x264_config.h"
 
-#define X264_BUILD 158
+#define X264_BUILD 159
 
 #ifdef _WIN32
 #   define X264_DLL_IMPORT __declspec(dllimport)

x264cli.h

@@ -32,6 +32,8 @@
 /* In microseconds */
 #define UPDATE_INTERVAL 250000
 
+#define MAX_RESOLUTION 16384
+
 typedef void *hnd_t;
 
 extern const char * const x264_avcintra_class_names[];