Commit 4e541fa9 authored by Rafaël Carré's avatar Rafaël Carré

__vout_AllocatePicture() : check all possibilities of integer overflow

Thanks to nefrir and mfwitten on irc for their help
parent 7c081ad1
......@@ -570,8 +570,31 @@ int __vout_AllocatePicture( vlc_object_t *p_this, picture_t *p_pic,
}
/* Calculate how big the new image should be */
size_t i_bytes = (size_t)p_pic->format.i_bits_per_pixel *
i_width_aligned * i_height_aligned / 8;
/*
* bytes = width_aligned * height_aligned * bpp / 8
* We need to check for an integer overflow at each multiplication since
* height & width (and bpp?) could be arbitrary large
*/
size_t i_bytes = 0;
/* i_width_aligned is a multiple of 16, so we can divide by 8 now */
size_t i_width_aligned_divided = i_width_aligned / 8;
if( i_width_aligned_divided <= (SIZE_MAX/i_height_aligned) )
{
size_t i_pixels_divided = i_width_aligned_divided * i_height_aligned;
size_t i_bpp = p_pic->format.i_bits_per_pixel;
if( i_pixels_divided <= (SIZE_MAX/i_bpp) )
{
i_bytes = i_pixels_divided * i_bpp;
}
}
if( i_bytes == 0 )
{
p_pic->i_planes = 0;
return VLC_ENOMEM;
}
p_pic->p_data = vlc_memalign( &p_pic->p_data_orig, 16, i_bytes );
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment