diff --git a/src/renderer.c b/src/renderer.c
index aa3f01ebc69361bb4127c9c49f124d844c76d0bc..e64f7fa2c26b89e804191d7e1e2e2f89b8718bd9 100644
--- a/src/renderer.c
+++ b/src/renderer.c
@@ -1310,6 +1310,7 @@ static bool pass_scale_main(struct pl_renderer *rr, struct pass_state *pass,
     const struct pl_image *image = &pass->image;
     bool need_fbo = image->num_overlays > 0;
     need_fbo |= rr->peak_detect_state && !params->allow_delayed_peak_detect;
+    need_fbo |= src.new_w != img->w || src.new_h != img->h;
 
     struct sampler_info info = sample_src_info(rr, &src, params);
     bool use_sigmoid = info.dir == SAMPLER_UP && params->sigmoid_params;
@@ -1332,7 +1333,6 @@ static bool pass_scale_main(struct pl_renderer *rr, struct pass_state *pass,
     }
 
     if (info.dir == SAMPLER_NOOP && !need_fbo) {
-        pl_assert(src.new_w == img->w && src.new_h == img->h);
         PL_TRACE(rr, "Skipping main scaler (would be no-op)");
         return true;
     }