cropadd: handle metadata rotated videos

This uses the rotate nearly identical to the newer formatcrop however has an easier time as it actually chops all the pixels so doesn't run into the aspect/sizing issues formatcrop currently does.

modules/video_filter/croppadd.c

@@ -118,6 +118,28 @@ typedef struct
     int i_paddleft;
     int i_paddright;
 } filter_sys_t;
+#define IDX_TOP 0
+#define IDX_LEFT 1
+#define IDX_BOTTOM 2
+#define IDX_RIGHT 3
+
+struct transform {
+    unsigned idx_top;
+    unsigned idx_left;
+    /* idx_bottom is idx_top XOR 2
+       idx_right is idx_left XOR 2 */
+};
+
+static const struct transform transforms[8] = {
+    [ORIENT_TOP_LEFT]     = { IDX_TOP,    IDX_LEFT },
+    [ORIENT_TOP_RIGHT]    = { IDX_TOP,    IDX_RIGHT },
+    [ORIENT_BOTTOM_LEFT]  = { IDX_BOTTOM, IDX_LEFT },
+    [ORIENT_BOTTOM_RIGHT] = { IDX_BOTTOM, IDX_RIGHT },
+    [ORIENT_LEFT_TOP]     = { IDX_LEFT,   IDX_TOP },
+    [ORIENT_LEFT_BOTTOM]  = { IDX_LEFT,   IDX_BOTTOM },
+    [ORIENT_RIGHT_TOP]    = { IDX_RIGHT,  IDX_TOP },
+    [ORIENT_RIGHT_BOTTOM] = { IDX_RIGHT,  IDX_BOTTOM },
+};
 
 /*****************************************************************************
  * OpenFilter: probe the filter and return score
@@ -169,6 +191,23 @@ static int OpenFilter( filter_t *p_filter )
     GET_OPTION( paddleft )
     GET_OPTION( paddright )
 
+    video_format_t *fmt = &p_filter->fmt_in.video;
+    video_orientation_t orientation = fmt->orientation;
+    const struct transform *tx = &transforms[orientation];
+    /* In the same order as IDX_ constants values */
+    unsigned crop[] = { p_sys->i_croptop, p_sys->i_cropleft, p_sys->i_cropbottom, p_sys->i_cropright };
+    unsigned padd[] = { p_sys->i_paddtop, p_sys->i_paddleft, p_sys->i_paddbottom, p_sys->i_paddright };
+
+    p_sys->i_croptop = crop[tx->idx_top];
+    p_sys->i_cropleft = crop[tx->idx_left];
+    p_sys->i_cropbottom = crop[tx->idx_top ^ 2];
+    p_sys->i_cropright = crop[tx->idx_left ^ 2];
+
+    p_sys->i_paddtop = padd[tx->idx_top];
+    p_sys->i_paddleft = padd[tx->idx_left];
+    p_sys->i_paddbottom = padd[tx->idx_top ^ 2];
+    p_sys->i_paddright = padd[tx->idx_left ^ 2];
+
     p_filter->fmt_out.video.i_height =
     p_filter->fmt_out.video.i_visible_height =
         p_filter->fmt_in.video.i_visible_height