Suggest fuzzer for x264_encoder_encode()
I suggest this fuzzer for continuous vulnerability checks.
extern "C" {
#ifdef _WIN32
#include <fcntl.h> /* _O_BINARY */
#include <io.h> /* _setmode() */
#endif
#include <stdint.h>
#include <stdio.h>
#include <x264.h>
}
#include "FuzzedDataProvider.h"
#include <fstream>
#include <gtest/gtest.h>
bool save_file(std::string Path, std::string Content) {
std::ofstream OFS(Path);
if (!OFS.is_open())
return false;
OFS << Content;
return true;
}
void fuzz_x264(FuzzedDataProvider &provider) {
x264_param_t param;
x264_picture_t pic;
x264_picture_t pic_out;
x264_t *h;
int i_frame = 0;
int i_frame_size;
x264_nal_t *nal;
int i_nal;
int width = provider.ConsumeIntegralInRange(1, 1024);
int height = provider.ConsumeIntegralInRange(1, 1024);
int luma_size = width * height;
int chroma_size = luma_size / 4;
if (!width || !height)
return;
auto input = provider.ConsumeRemainingBytesAsString();
const char *file_name = "input";
save_file(file_name, input);
FILE *fp = fopen(file_name, "rb");
if (!fp)
goto fail;
/* Get default params for preset/tuning */
if (x264_param_default_preset(¶m, "medium", NULL) < 0)
goto fail;
/* Configure non-default params */
param.i_bitdepth = 8;
param.i_csp = X264_CSP_I420;
param.i_width = width;
param.i_height = height;
param.b_vfr_input = 0;
param.b_repeat_headers = 1;
param.b_annexb = 1;
/* Apply profile restrictions. */
if (x264_param_apply_profile(¶m, "high") < 0)
goto fail;
if (x264_picture_alloc(&pic, param.i_csp, param.i_width, param.i_height) < 0)
goto fail;
#undef fail
#define fail fail2
h = x264_encoder_open(¶m);
if (!h)
goto fail;
#undef fail
#define fail fail3
/* Encode frames */
for (;; i_frame++) {
/* Read input frame */
if (fread(pic.img.plane[0], 1, luma_size, fp) != (unsigned)luma_size)
break;
if (fread(pic.img.plane[1], 1, chroma_size, fp) != (unsigned)chroma_size)
break;
if (fread(pic.img.plane[2], 1, chroma_size, fp) != (unsigned)chroma_size)
break;
pic.i_pts = i_frame;
i_frame_size = x264_encoder_encode(h, &nal, &i_nal, &pic, &pic_out);
if (i_frame_size < 0)
goto fail;
else if (i_frame_size) {
if (!fwrite(nal->p_payload, i_frame_size, 1, stdout))
goto fail;
}
}
/* Flush delayed frames */
while (x264_encoder_delayed_frames(h)) {
i_frame_size = x264_encoder_encode(h, &nal, &i_nal, NULL, &pic_out);
if (i_frame_size < 0)
goto fail;
else if (i_frame_size) {
if (!fwrite(nal->p_payload, i_frame_size, 1, stdout))
goto fail;
}
}
#undef fail
fail3:
x264_encoder_close(h);
fail2:
x264_picture_clean(&pic);
fail:
x264_param_cleanup(¶m);
if (fp)
fclose(fp);
}
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, uint32_t size) {
FuzzedDataProvider provider(data, size);
fuzz_x264(provider);
return 0;
}