diff --git a/src/renderer.c b/src/renderer.c
index 536e496dc2b8507892d1dd8c81be69ae8e3394ff..0cb1441e966b8e9c88c1b9c065ffa107eb2532ba 100644
--- a/src/renderer.c
+++ b/src/renderer.c
@@ -1312,7 +1312,6 @@ static bool pass_scale_main(struct pl_renderer *rr, struct pass_state *pass,
 
     bool need_fbo = image->num_overlays > 0;
     need_fbo |= rr->peak_detect_state && !params->allow_delayed_peak_detect;
-    need_fbo |= src.new_w != img->w || src.new_h != img->h;
 
     struct sampler_info info = sample_src_info(rr, &src, params);
     bool use_sigmoid = info.dir == SAMPLER_UP && params->sigmoid_params;
@@ -1335,6 +1334,7 @@ static bool pass_scale_main(struct pl_renderer *rr, struct pass_state *pass,
     }
 
     if (info.dir == SAMPLER_NOOP && !need_fbo) {
+        pl_assert(src.new_w == img->w && src.new_h == img->h);
         PL_TRACE(rr, "Skipping main scaler (would be no-op)");
         return true;
     }