Fixed bug 2979 - SDL_ConvertSurface does not convert color keys consistently

Edmund Horner

When a 16-bit "565 format" surface has a colour key set, it will blit with correct transparency.  If, however, it has its colour key set then is converted to a 32-bit ARGB format surface, the colour key in the converted image will not necessarily be the same pixel value as the transparent pixels.  It may not blit correctly, because the colour key does not match the right pixels.

In my case, with an image using 0xB54A for transparency, the colour key was converted to 180,170,82; but the corresponding pixels (with the same original value) were converted to 180,169,82.  Blitting the converted image did not use transparency where expected.

I have attached a test case.  The bug has been replicated on both x86_64 Linux (SDL 2.0.2), and 32-bit MS C++ 2010 on Windows (SDL 2.0.0).

src/video/SDL_surface.c

@@ -970,13 +970,32 @@ SDL_ConvertSurface(SDL_Surface * surface, const SDL_PixelFormat * format,
         }
 
         if (set_colorkey_by_color) {
-            /* Set the colorkey by color, which needs to be unique */
-            Uint8 keyR, keyG, keyB, keyA;
+            SDL_Surface *tmp;
+            SDL_Surface *tmp2;
+            int converted_colorkey = 0;
+
+            /* Create a dummy surface to get the colorkey converted */
+            tmp = SDL_CreateRGBSurface(0, 1, 1,
+                                   surface->format->BitsPerPixel, surface->format->Rmask,
+                                   surface->format->Gmask, surface->format->Bmask,
+                                   surface->format->Amask);
+
+            SDL_FillRect(tmp, NULL, surface->map->info.colorkey);
+
+            tmp->map->info.flags &= ~SDL_COPY_COLORKEY;
+
+            /* Convertion of the colorkey */
+            tmp2 = SDL_ConvertSurface(tmp, format, 0);
+
+            /* Get the converted colorkey */
+            memcpy(&converted_colorkey, tmp2->pixels, tmp2->format->BytesPerPixel);
+
+            SDL_FreeSurface(tmp);
+            SDL_FreeSurface(tmp2);
+
+            /* Set the converted colorkey on the new surface */
+            SDL_SetColorKey(convert, 1, converted_colorkey);
 
-            SDL_GetRGBA(surface->map->info.colorkey, surface->format, &keyR,
-                        &keyG, &keyB, &keyA);
-            SDL_SetColorKey(convert, 1,
-                            SDL_MapRGBA(convert->format, keyR, keyG, keyB, keyA));
             /* This is needed when converting for 3D texture upload */
             SDL_ConvertColorkeyToAlpha(convert);
         }