opengl: handle output resize in a generic way

Resizing the window resulted in a call to a function on the specific
"renderer" filter.

Instead, add a filter callback to handle output resizing, and propagate
the requested input size backwards.

This will allow to properly handle resizing if an upscaler is inserted
before the renderer.

Co-authored-by: Maxime Meissonnier <mmeisson@outlook.fr>

modules/video_output/opengl/filter.h

@@ -62,6 +62,28 @@ struct vlc_gl_filter_ops {
      * Free filter resources
      */
     void (*close)(struct vlc_gl_filter *filter);
+
+    /**
+     * Request a (responsive) filter to adapt its output size (optional)
+     *
+     * A responsive filter is a filter for which the size of the produced
+     * pictures depends on the output (e.g. display) size rather than the
+     * input. This is for example the case for a renderer.
+     *
+     * A new output size is requested (size_out). The filter is authorized to
+     * change the size_out to enforce its own constraints.
+     *
+     * In addition, it may request to the previous filter (if any) an optimal
+     * size it wants to receive. If set to non-zero value, this previous filter
+     * will receive this size as its requested size (and so on).
+     *
+     * \retval true if the resize is accepted (possibly with a modified
+     *              size_out)
+     * \retval false if the resize is rejected (included on error)
+     */
+    int (*request_output_size)(struct vlc_gl_filter *filter,
+                               struct vlc_gl_tex_size *size_out,
+                               struct vlc_gl_tex_size *optimal_in);
 };
 
 /**

modules/video_output/opengl/filters.c

@@ -517,3 +517,45 @@ vlc_gl_filters_SetViewport(struct vlc_gl_filters *filters, int x, int y,
     filters->viewport.width = width;
     filters->viewport.height = height;
 }
+
+int
+vlc_gl_filters_SetOutputSize(struct vlc_gl_filters *filters, unsigned width,
+                             unsigned height)
+{
+    bool resized = false;
+    struct vlc_gl_tex_size req = { width, height };
+
+    struct vlc_gl_filter_priv *priv;
+    vlc_list_reverse_foreach(priv, &filters->list, node)
+    {
+        struct vlc_gl_filter *filter = &priv->filter;
+        if (!filter->ops->request_output_size) {
+            /* Could not propagate further */
+            break;
+        }
+
+        struct vlc_gl_tex_size optimal_in = {0};
+        int ret =
+            filter->ops->request_output_size(filter, &req, &optimal_in);
+        if (ret != VLC_SUCCESS)
+            break;
+
+        /* The filter may have modified the requested size */
+        priv->size_out = req;
+
+        /* Recreate the framebuffers/textures with the new size */
+        vlc_gl_filter_ApplyOutputSize(filter);
+
+        resized = true;
+
+        if (!optimal_in.width || !optimal_in.height)
+            /* No specific input size requested, do not propagate further */
+            break;
+
+        /* Request the previous filter to output at the optimal input size of
+         * the current filter. */
+        req = optimal_in;
+    }
+
+    return resized ? VLC_SUCCESS : VLC_EGENERIC;
+}

modules/video_output/opengl/filters.h

@@ -106,4 +106,11 @@ void
 vlc_gl_filters_SetViewport(struct vlc_gl_filters *filters, int x, int y,
                            unsigned width, unsigned height);
 
+/**
+ * Change the output size
+ */
+int
+vlc_gl_filters_SetOutputSize(struct vlc_gl_filters *filters, unsigned width,
+                             unsigned height);
+
 #endif

modules/video_output/opengl/renderer.c

@@ -359,7 +359,7 @@ vlc_gl_renderer_SetViewpoint(struct vlc_gl_renderer *renderer,
     return VLC_SUCCESS;
 }
 
-void
+static void
 vlc_gl_renderer_SetOutputSize(struct vlc_gl_renderer *renderer, unsigned width,
                               unsigned height)
 {
@@ -368,6 +368,8 @@ vlc_gl_renderer_SetOutputSize(struct vlc_gl_renderer *renderer, unsigned width,
     /* Each time the window size changes, we must recompute the minimum zoom
      * since the aspect ration changes.
      * We must also set the new current zoom value. */
+    renderer->target_width = width;
+    renderer->target_height = height;
     renderer->f_sar = f_sar;
     UpdateFOVy(renderer);
     UpdateZ(renderer);
@@ -376,6 +378,23 @@ vlc_gl_renderer_SetOutputSize(struct vlc_gl_renderer *renderer, unsigned width,
     getViewpointMatrixes(renderer, fmt->projection_mode);
 }
 
+static int
+RequestOutputSize(struct vlc_gl_filter *filter,
+                  struct vlc_gl_tex_size *req,
+                  struct vlc_gl_tex_size *optimal_in)
+{
+    struct vlc_gl_renderer *renderer = filter->sys;
+
+    vlc_gl_renderer_SetOutputSize(renderer, req->width, req->height);
+
+    /* The optimal input size is the size for which the renderer do not need to
+     * scale */
+    optimal_in->width = renderer->target_width;
+    optimal_in->height = renderer->target_height;
+
+    return VLC_SUCCESS;
+}
+
 static int BuildSphere(GLfloat **vertexCoord, GLfloat **textureCoord, unsigned *nbVertices,
                        GLushort **indices, unsigned *nbIndices)
 {
@@ -762,6 +781,7 @@ vlc_gl_renderer_Open(struct vlc_gl_filter *filter,
     static const struct vlc_gl_filter_ops filter_ops = {
         .draw = Draw,
         .close = Close,
+        .request_output_size = RequestOutputSize,
     };
     filter->ops = &filter_ops;
     filter->sys = renderer;

modules/video_output/opengl/renderer.h

@@ -94,6 +94,11 @@ struct vlc_gl_renderer
     float f_fovy; /* to avoid recalculating them when needed.      */
     float f_z;    /* Position of the camera on the shpere radius vector */
     float f_sar;
+
+    /* Original size from which f_sar is computed
+     *     f_sar = (float) target_width / target_height */
+    unsigned target_width;
+    unsigned target_height;
 };
 
 vlc_gl_filter_open_fn vlc_gl_renderer_Open;
@@ -102,8 +107,4 @@ int
 vlc_gl_renderer_SetViewpoint(struct vlc_gl_renderer *renderer,
                              const vlc_viewpoint_t *p_vp);
 
-void
-vlc_gl_renderer_SetOutputSize(struct vlc_gl_renderer *renderer, unsigned width,
-                              unsigned height);
-
 #endif /* include-guard */

modules/video_output/opengl/vout_helper.c

@@ -244,7 +244,11 @@ int vout_display_opengl_SetViewpoint(vout_display_opengl_t *vgl,
 void vout_display_opengl_SetOutputSize(vout_display_opengl_t *vgl,
                                        unsigned width, unsigned height)
 {
-    vlc_gl_renderer_SetOutputSize(vgl->renderer, width, height);
+    int ret = vlc_gl_filters_SetOutputSize(vgl->filters, width, height);
+    /* The renderer, last filter in the chain, necessarily accepts the new
+     * output size */
+    assert(ret == VLC_SUCCESS);
+    (void) ret;
 }
 
 void vout_display_opengl_Viewport(vout_display_opengl_t *vgl, int x, int y,