Skip to content

Wrong handle passed to nalu_process callback

It seems that the callback set in nalu_process gets called with the wrong encoder handle pointer (the x264_t * argument).

The following example code segfaults when the callback cb calls x264_nal_encode(h, workspace, nal) with h being said handle pointer. If h is replaced by the one produced by x264_encoder_open, things are fine.

A git bisection lead me to 71ed44c7 as the first bad commit, the preceding commit is good. Current master HEAD is bad.

Example code:

#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <x264.h>

#define WS_SIZE 10000000
uint8_t * workspace;
x264_t * h_override;

void cb(x264_t * h, x264_nal_t * nal, void * opaque)
{
  assert((nal->i_payload*3)/2 + 5 + 64 < WS_SIZE); // Probably not reentrant, just here to communicate to the reader.
  printf("%p %p\n", h, h_override);
  x264_nal_encode(h, workspace, nal); // Segfault here.
  //x264_nal_encode(h_override, workspace, nal); // This works.
  
  // Removed: Process nal.
  }
}

int main(int argc, char ** argv)
{
  uint8_t * fake_frame = malloc(1280*720*3);
  memset(fake_frame, 0, 1280*720*3);

  workspace = malloc(WS_SIZE);
  ok = true;

  x264_param_t param;
  int status = x264_param_default_preset(&param, "ultrafast", "zerolatency");
  assert(status == 0);

  param.i_csp = X264_CSP_RGB;
  param.i_width = 1280;
  param.i_height = 720;
  param.i_threads = 1;
  param.i_lookahead_threads = 1;
  param.i_frame_total = 0;
  param.i_fps_num = 30;
  param.i_fps_den = 1;
  param.i_slice_max_size = 1024;
  param.b_annexb = 1;
  param.nalu_process = cb;

  status = x264_param_apply_profile(&param, "high444");
  assert(status == 0);

  x264_t * h = x264_encoder_open(&param);
  assert(h);
  h_override = h;

  x264_picture_t pic;
  status = x264_picture_alloc(&pic, param.i_csp, param.i_width, param.i_height);
  assert(pic.img.i_plane == 1);

  x264_picture_t pic_out;
  x264_nal_t * nal; // Not used. We process NALs in cb.
  int i_nal;

  for (int i = 0; i < 100; ++i)
  {
    pic.i_pts = i;
    pic.img.plane[0] = fake_frame;
    status = x264_encoder_encode(h, &nal, &i_nal, &pic, &pic_out);
  }
  
  x264_encoder_close(h);
  x264_picture_clean(&pic);
  free(workspace);
  free(fake_frame);
  return 0;
}

Thanks to BugMaster on IRC for helping me diagnose this.

To upload designs, you'll need to enable LFS and have an admin enable hashed storage. More information