Try 8x8 transform analysis even when sub8x8 partitions are present

Turn off the sub8x8 partitions, try it, and turn them back on if it didn't help.
Small compression improvement with p4x4 on (~0.1-0.5%).
Also update related comments.

common/macroblock.h

@@ -420,23 +420,23 @@ static ALWAYS_INLINE int x264_mb_predict_non_zero_code( x264_t *h, int idx )
         i_ret = ( i_ret + 1 ) >> 1;
     return i_ret & 0x7f;
 }
+
+/* intra and skip are disallowed, p8x8 is conditional. */
+static const uint8_t x264_transform_allowed[X264_MBTYPE_MAX] =
+{
+    0,0,0,0,1,2,0,1,1,1,1,1,1,1,1,1,1,1,0
+};
+
 /* x264_mb_transform_8x8_allowed:
  *      check whether any partition is smaller than 8x8 (or at least
  *      might be, according to just partition type.)
  *      doesn't check for cbp */
 static ALWAYS_INLINE int x264_mb_transform_8x8_allowed( x264_t *h )
 {
-    // intra and skip are disallowed
-    // large partitions are allowed
-    // direct and 8x8 are conditional
-    static const uint8_t partition_tab[X264_MBTYPE_MAX] = {
-        0,0,0,0,1,2,0,1,1,1,1,1,1,1,1,1,1,1,0,
-    };
-
     if( !h->pps->b_transform_8x8_mode )
         return 0;
     if( h->mb.i_type != P_8x8 )
-        return partition_tab[h->mb.i_type];
+        return x264_transform_allowed[h->mb.i_type];
     return M32( h->mb.i_sub_partition ) == D_L0_8x8*0x01010101;
 }
 

encoder/analyse.c

@@ -2851,8 +2851,15 @@ static inline void x264_mb_analyse_transform( x264_t *h )
 
 static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *a, int *i_satd, int *i_rd )
 {
-    if( x264_mb_transform_8x8_allowed( h ) && h->param.analyse.b_transform_8x8 )
+    if( h->param.analyse.b_transform_8x8 && h->pps->b_transform_8x8_mode )
     {
+        uint32_t subpart_bak = M32( h->mb.i_sub_partition );
+        /* Try switching the subpartitions to 8x8 so that we can use 8x8 transform mode */
+        if( h->mb.i_type == P_8x8 )
+            M32( h->mb.i_sub_partition ) = D_L0_8x8*0x01010101;
+        else if( !x264_transform_allowed[h->mb.i_type] )
+            return;
+
         x264_analyse_update_cache( h, a );
         h->mb.b_transform_8x8 ^= 1;
         /* FIXME only luma is needed for 4:2:0, but the score for comparison already includes chroma */
@@ -2865,7 +2872,10 @@ static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *
             *i_rd = i_rd8;
         }
         else
+        {
             h->mb.b_transform_8x8 ^= 1;
+            M32( h->mb.i_sub_partition ) = subpart_bak;
+        }
     }
 }