Fixed potential overflow in software triangle rendering

src/render/software/SDL_triangle.c

@@ -22,6 +22,8 @@
 
 #if SDL_VIDEO_RENDER_SW && !defined(SDL_RENDER_DISABLED)
 
+#include <limits.h>
+
 #include "SDL_triangle.h"
 
 #include "../../video/SDL_blit.h"
@@ -222,7 +224,8 @@ int SDL_SW_FillTriangle(SDL_Surface *dst, SDL_Point *d0, SDL_Point *d1, SDL_Poin
     Uint8 *dst_ptr;
     int dst_pitch;
 
-    int area, is_clockwise;
+    Sint64 area;
+    int is_clockwise;
 
     int d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x;
     Sint64 w0_row, w1_row, w2_row;
@@ -308,7 +311,9 @@ int SDL_SW_FillTriangle(SDL_Surface *dst, SDL_Point *d0, SDL_Point *d1, SDL_Poin
     }
 
     is_clockwise = area > 0;
-    area = SDL_abs(area);
+    if (area < 0) {
+        area = -area;
+    }
 
     {
         int val;
@@ -463,7 +468,8 @@ int SDL_SW_BlitTriangle(
     int *src_ptr;
     int src_pitch;
 
-    int area, is_clockwise;
+    Sint64 area, tmp64;
+    int is_clockwise;
 
     int d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x;
     int s2s0_x, s2s1_x, s2s0_y, s2s1_y;
@@ -475,8 +481,11 @@ int SDL_SW_BlitTriangle(
 
     int has_modulation;
 
-    if (!src || !dst) {
-        return -1;
+    if (!src) {
+        return SDL_InvalidParamError("src");
+    }
+    if (!src) {
+        return SDL_InvalidParamError("dst");
     }
 
     area = cross_product(d0, d1, d2->x, d2->y);
@@ -578,7 +587,9 @@ int SDL_SW_BlitTriangle(
     src_pitch = src->pitch;
 
     is_clockwise = area > 0;
-    area = SDL_abs(area);
+    if (area < 0) {
+        area = -area;
+    }
 
     {
         int val;
@@ -627,8 +638,20 @@ int SDL_SW_BlitTriangle(
     bias_w2 = (is_top_left(d0, d1, is_clockwise) ? 0 : -1);
 
     /* precompute constant 's2->x * area' used in TRIANGLE_GET_TEXTCOORD */
-    s2_x_area.x = s2->x * area;
-    s2_x_area.y = s2->y * area;
+    tmp64 = s2->x * area;
+    if (tmp64 >= INT_MIN && tmp64 <= INT_MAX) {
+        s2_x_area.x = (int)tmp64;
+    } else {
+        ret = SDL_SetError("triangle area overflow");
+        goto end;
+    }
+    tmp64 = s2->y * area;
+    if (tmp64 >= INT_MIN && tmp64 <= INT_MAX) {
+        s2_x_area.y = (int)tmp64;
+    } else {
+        ret = SDL_SetError("triangle area overflow");
+        goto end;
+    }
 
     if (blend != SDL_BLENDMODE_NONE || src->format->format != dst->format->format || has_modulation || !is_uniform) {
         /* Use SDL_BlitTriangle_Slow */
@@ -674,9 +697,18 @@ int SDL_SW_BlitTriangle(
         tmp_info.dst = dst_ptr;
         tmp_info.dst_pitch = dst_pitch;
 
-        SDL_BlitTriangle_Slow(&tmp_info, s2_x_area, dstrect, area, bias_w0, bias_w1, bias_w2,
+#define CHECK_INT_RANGE(X) \
+    if ((X) < INT_MIN || (X) > INT_MAX) { \
+        ret = SDL_SetError("integer overflow (%s = %" SDL_PRIs64 ")", #X, X); \
+        goto end; \
+    }
+        CHECK_INT_RANGE(area);
+        CHECK_INT_RANGE(w0_row);
+        CHECK_INT_RANGE(w1_row);
+        CHECK_INT_RANGE(w2_row);
+        SDL_BlitTriangle_Slow(&tmp_info, s2_x_area, dstrect, (int)area, bias_w0, bias_w1, bias_w2,
                               d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x,
-                              s2s0_x, s2s1_x, s2s0_y, s2s1_y, w0_row, w1_row, w2_row,
+                              s2s0_x, s2s1_x, s2s0_y, s2s1_y, (int)w0_row, (int)w1_row, (int)w2_row,
                               c0, c1, c2, is_uniform);
 
         goto end;