SDL2: backport SDL_CalculateYUVSize() that checks for YUV size overflow (#6972)

src/render/SDL_yuv_sw.c

@@ -57,30 +57,11 @@ SDL_SW_CreateYUVTexture(Uint32 format, int w, int h)
     swdata->w = w;
     swdata->h = h;
     {
-        const int sz_plane = w * h;
-        const int sz_plane_chroma = ((w + 1) / 2) * ((h + 1) / 2);
-        const int sz_plane_packed = ((w + 1) / 2) * h;
-        int dst_size = 0;
-        switch (format) {
-        case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U  (3 planes) */
-        case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V  (3 planes) */
-            dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
-            break;
-
-        case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
-        case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
-        case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
-            dst_size = 4 * sz_plane_packed;
-            break;
-
-        case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved  (2 planes) */
-        case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved  (2 planes) */
-            dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
-            break;
-
-        default:
-            SDL_assert(0 && "We should never get here (caught above)");
-            break;
+        size_t dst_size;
+        if (SDL_CalculateYUVSize(format, w, h, &dst_size, NULL) < 0) {
+            SDL_SW_DestroyYUVTexture(swdata);
+            SDL_OutOfMemory();
+            return NULL;
         }
         swdata->pixels = (Uint8 *)SDL_SIMDAlloc(dst_size);
         if (!swdata->pixels) {

src/video/SDL_surface.c

@@ -39,13 +39,15 @@ SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32);
 /*
  * Calculate the pad-aligned scanline width of a surface.
  * Return SDL_SIZE_MAX on overflow.
+ *
+ * for FOURCC, use SDL_CalculateYUVSize()
  */
 static size_t
 SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal)
 {
     size_t pitch;
 
-    if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) {
+    if (SDL_BITSPERPIXEL(format) >= 8) {
         if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) {
             return SDL_SIZE_MAX;
         }
@@ -93,11 +95,16 @@ SDL_CreateRGBSurfaceWithFormat(Uint32 flags, int width, int height, int depth,
         return NULL;
     }
 
-    pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
-    if (pitch > SDL_MAX_SINT32) {
-        /* Overflow... */
-        SDL_OutOfMemory();
+    if (SDL_ISPIXELFORMAT_FOURCC(format)) {
+        SDL_SetError("invalid format");
         return NULL;
+    } else {
+        pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
+        if (pitch > SDL_MAX_SINT32) {
+            /* Overflow... */
+            SDL_OutOfMemory();
+            return NULL;
+        }
     }
 
     /* Allocate the surface */
@@ -269,7 +276,12 @@ SDL_CreateRGBSurfaceWithFormatFrom(void *pixels,
         return NULL;
     }
 
-    minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+    if (SDL_ISPIXELFORMAT_FOURCC(format)) {
+        SDL_SetError("invalid format");
+        return NULL;
+    } else {
+        minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+    }
 
     if (pitch < 0 || (pitch > 0 && ((size_t)pitch) < minimalPitch)) {
         SDL_InvalidParamError("pitch");

src/video/SDL_yuv.c

@@ -31,6 +31,10 @@
 
 static SDL_YUV_CONVERSION_MODE SDL_YUV_ConversionMode = SDL_YUV_CONVERSION_BT601;
 
+#if SDL_HAVE_YUV
+static SDL_bool IsPlanar2x2Format(Uint32 format);
+#endif
+
 void SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_MODE mode)
 {
     SDL_YUV_ConversionMode = mode;
@@ -54,6 +58,133 @@ SDL_YUV_CONVERSION_MODE SDL_GetYUVConversionModeForResolution(int width, int hei
     return mode;
 }
 
+/*
+ * Calculate YUV size and pitch. Check for overflow.
+ * Output 'pitch' that can be used with SDL_ConvertPixels()
+ *
+ * return 0 on success, -1 on error
+ */
+int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch)
+{
+#if SDL_HAVE_YUV
+    int sz_plane = 0, sz_plane_chroma = 0, sz_plane_packed = 0;
+
+    if (IsPlanar2x2Format(format) == SDL_TRUE) {
+        {
+            /* sz_plane == w * h; */
+            size_t s1;
+            if (SDL_size_mul_overflow(w, h, &s1) < 0) {
+                return -1;
+            }
+            sz_plane = (int) s1;
+        }
+
+        {
+            /* sz_plane_chroma == ((w + 1) / 2) * ((h + 1) / 2); */
+            size_t s1, s2, s3;
+            if (SDL_size_add_overflow(w, 1, &s1) < 0) {
+                return -1;
+            }
+            s1 = s1 / 2;
+            if (SDL_size_add_overflow(h, 1, &s2) < 0) {
+                return -1;
+            }
+            s2 = s2 / 2;
+            if (SDL_size_mul_overflow(s1, s2, &s3) < 0) {
+                return -1;
+            }
+            sz_plane_chroma = (int) s3;
+        }
+    } else {
+        /* sz_plane_packed == ((w + 1) / 2) * h; */
+        size_t s1, s2;
+        if (SDL_size_add_overflow(w, 1, &s1) < 0) {
+            return -1;
+        }
+        s1 = s1 / 2;
+        if (SDL_size_mul_overflow(s1, h, &s2) < 0) {
+            return -1;
+        }
+        sz_plane_packed = (int) s2;
+    }
+
+    switch (format) {
+    case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U  (3 planes) */
+    case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V  (3 planes) */
+
+        if (pitch) {
+            *pitch = w;
+        }
+
+        if (size) {
+            /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
+            size_t s1, s2;
+            if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
+                return -1;
+            }
+            if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
+                return -1;
+            }
+            *size = (int)s2;
+        }
+        break;
+
+    case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
+    case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
+    case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
+
+        if (pitch) {
+            /* pitch == ((w + 1) / 2) * 4; */
+           size_t p1, p2;
+           if (SDL_size_add_overflow(w, 1, &p1) < 0) {
+               return -1;
+           }
+           p1 = p1 / 2;
+           if (SDL_size_mul_overflow(p1, 4, &p2) < 0) {
+               return -1;
+           }
+           *pitch = (int) p2;
+        }
+
+        if (size) {
+            /* dst_size == 4 * sz_plane_packed; */
+            size_t s1;
+            if (SDL_size_mul_overflow(sz_plane_packed, 4, &s1) < 0) {
+                return -1;
+            }
+            *size = (int) s1;
+        }
+        break;
+
+    case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved  (2 planes) */
+    case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved  (2 planes) */
+        if (pitch) {
+            *pitch = w;
+        }
+
+        if (size) {
+            /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
+            size_t s1, s2;
+            if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
+                return -1;
+            }
+            if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
+                return -1;
+            }
+            *size = (int) s2;
+        }
+        break;
+
+    default:
+        return -1;
+    }
+
+    return 0;
+#else
+    return -1;
+#endif
+}
+
 #if SDL_HAVE_YUV
 
 static int GetYUVConversionType(int width, int height, YCbCrType *yuv_type)

src/video/SDL_yuv_c.h

@@ -30,6 +30,9 @@ extern int SDL_ConvertPixels_YUV_to_RGB(int width, int height, Uint32 src_format
 extern int SDL_ConvertPixels_RGB_to_YUV(int width, int height, Uint32 src_format, const void *src, int src_pitch, Uint32 dst_format, void *dst, int dst_pitch);
 extern int SDL_ConvertPixels_YUV_to_YUV(int width, int height, Uint32 src_format, const void *src, int src_pitch, Uint32 dst_format, void *dst, int dst_pitch);
 
+
+extern int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch);
+
 #endif /* SDL_yuv_c_h_ */
 
 /* vi: set ts=4 sw=4 expandtab: */