- simplication and factorization around CalculateSize and Pitch, RGB/YUV

- update SDL_CalculateYUVSize pitch to size_t

src/video/SDL_pixels_c.h

@@ -30,6 +30,7 @@
 
 /* Pixel format functions */
 extern int SDL_InitFormat(SDL_PixelFormat *format, Uint32 pixel_format);
+extern int SDL_CalculateSize(Uint32 format, int width, int height, size_t *size, size_t *pitch, SDL_bool minimalPitch);
 
 /* Blit mapping functions */
 extern SDL_BlitMap *SDL_AllocBlitMap(void);

src/video/SDL_surface.c

@@ -42,32 +42,71 @@ SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32);
  *
  * for FOURCC, use SDL_CalculateYUVSize()
  */
-static size_t
-SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal)
+static int
+SDL_CalculateRGBSize(Uint32 format, size_t width, size_t height, size_t *size, size_t *pitch, SDL_bool minimal)
 {
-    size_t pitch;
-
     if (SDL_BITSPERPIXEL(format) >= 8) {
-        if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) {
-            return SDL_SIZE_MAX;
+        if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), pitch)) {
+            return -1;
         }
     } else {
-        if (SDL_size_mul_overflow(width, SDL_BITSPERPIXEL(format), &pitch)) {
-            return SDL_SIZE_MAX;
+        if (SDL_size_mul_overflow(width, SDL_BITSPERPIXEL(format), pitch)) {
+            return -1;
         }
-        if (SDL_size_add_overflow(pitch, 7, &pitch)) {
-            return SDL_SIZE_MAX;
+        if (SDL_size_add_overflow(*pitch, 7, pitch)) {
+            return -1;
         }
-        pitch /= 8;
+        *pitch /= 8;
     }
     if (!minimal) {
         /* 4-byte aligning for speed */
-        if (SDL_size_add_overflow(pitch, 3, &pitch)) {
-            return SDL_SIZE_MAX;
+        if (SDL_size_add_overflow(*pitch, 3, pitch)) {
+            return -1;
         }
-        pitch &= ~3;
+        *pitch &= ~3;
+    }
+
+    if (SDL_size_mul_overflow(height, *pitch, size)) {
+        /* Overflow... */
+        return -1;
     }
-    return pitch;
+
+    return 0;
+}
+
+int SDL_CalculateSize(Uint32 format, int width, int height, size_t *size, size_t *pitch, SDL_bool minimalPitch)
+{
+    size_t p = 0, sz = 0;
+
+    if (size) {
+        *size = 0;
+    }
+
+    if (pitch) {
+        *pitch = 0;
+    }
+
+    if (SDL_ISPIXELFORMAT_FOURCC(format)) {
+        if (SDL_CalculateYUVSize(format, width, height, &sz, &p) < 0) {
+            /* Overflow... */
+            return -1;
+        }
+    } else {
+        if (SDL_CalculateRGBSize(format, width, height, &sz, &p, minimalPitch) < 0) {
+            /* Overflow... */
+            return -1;
+        }
+    }
+
+    if (size) {
+        *size = sz;
+    }
+
+    if (pitch) {
+        *pitch = p;
+    }
+
+    return 0;
 }
 
 /*
@@ -77,7 +116,7 @@ SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal)
 SDL_Surface *
 SDL_CreateSurface(int width, int height, Uint32 format)
 {
-    size_t pitch;
+    size_t pitch, size;
     SDL_Surface *surface;
 
     if (width < 0) {
@@ -90,21 +129,10 @@ SDL_CreateSurface(int width, int height, Uint32 format)
         return NULL;
     }
 
-    if (SDL_ISPIXELFORMAT_FOURCC(format)) {
-        int p;
-        if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) {
-            /* Overflow... */
-            SDL_OutOfMemory();
-            return NULL;
-        }
-        pitch = p;
-    } else {
-        pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
-        if (pitch > SDL_MAX_SINT32) {
-            /* Overflow... */
-            SDL_OutOfMemory();
-            return NULL;
-        }
+    if (SDL_CalculateSize(format, width, height, &size, &pitch, SDL_FALSE /* not minimal pitch */) < 0) {
+        /* Overflow... */
+        SDL_OutOfMemory();
+        return NULL;
     }
 
     /* Allocate the surface */
@@ -146,25 +174,6 @@ SDL_CreateSurface(int width, int height, Uint32 format)
 
     /* Get the pixels */
     if (surface->w && surface->h) {
-        size_t size;
-        if (SDL_ISPIXELFORMAT_FOURCC(surface->format->format)) {
-            /* Get correct size and pitch for YUV formats */
-            if (SDL_CalculateYUVSize(surface->format->format, surface->w, surface->h, &size, &surface->pitch) < 0) {
-                /* Overflow... */
-                SDL_DestroySurface(surface);
-                SDL_OutOfMemory();
-                return NULL;
-            }
-        } else {
-            /* Assumptions checked in surface_size_assumptions assert above */
-            if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) {
-                /* Overflow... */
-                SDL_DestroySurface(surface);
-                SDL_OutOfMemory();
-                return NULL;
-            }
-        }
-
         surface->pixels = SDL_aligned_alloc(SDL_SIMDGetAlignment(), size);
         if (!surface->pixels) {
             SDL_DestroySurface(surface);
@@ -219,15 +228,10 @@ SDL_CreateSurfaceFrom(void *pixels,
     } else {
         size_t minimalPitch;
 
-        if (SDL_ISPIXELFORMAT_FOURCC(format)) {
-            int p;
-            if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) {
-                SDL_InvalidParamError("pitch");
-                return NULL;
-            }
-            minimalPitch = p;
-        } else {
-            minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+        if (SDL_CalculateSize(format, width, height, NULL, &minimalPitch, SDL_TRUE /* minimal pitch */) < 0) {
+            /* Overflow... */
+            SDL_OutOfMemory();
+            return NULL;
         }
 
         if (pitch < 0 || (size_t)pitch < minimalPitch) {

src/video/SDL_yuv.c

@@ -62,7 +62,7 @@ SDL_YUV_CONVERSION_MODE SDL_GetYUVConversionModeForResolution(int width, int hei
  *
  * return 0 on success, -1 on error
  */
-int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch)
+int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, size_t *pitch)
 {
 #if SDL_HAVE_YUV
     int sz_plane = 0, sz_plane_chroma = 0, sz_plane_packed = 0;
@@ -141,7 +141,7 @@ int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch)
            if (SDL_size_mul_overflow(p1, 4, &p2) < 0) {
                return -1;
            }
-           *pitch = (int) p2;
+           *pitch = p2;
         }
 
         if (size) {

src/video/SDL_yuv_c.h

@@ -31,6 +31,6 @@ extern int SDL_ConvertPixels_RGB_to_YUV(int width, int height, Uint32 src_format
 extern int SDL_ConvertPixels_YUV_to_YUV(int width, int height, Uint32 src_format, const void *src, int src_pitch, Uint32 dst_format, void *dst, int dst_pitch);
 
 
-extern int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch);
+extern int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, size_t *pitch);
 
 #endif /* SDL_yuv_c_h_ */