Don't report windows being maximized when fullscreen on X11

This is a functional state for some window managers (tested using stock Ubuntu 22.04.1), and removing that state, e.g. using SDL_RestoreWindow(), results in a window centered and floating, and not visually covering the rest of the desktop.

src/video/x11/SDL_x11events.c

@@ -1460,7 +1460,7 @@ X11_DispatchEvent(_THIS, XEvent *xevent)
                    without ever mapping / unmapping them, so we handle that here,
                    because they use the NETWM protocol to notify us of changes.
                  */
-                const Uint32 flags = X11_GetNetWMState(_this, xevent->xproperty.window);
+                const Uint32 flags = X11_GetNetWMState(_this, data->window, xevent->xproperty.window);
                 const Uint32 changed = flags ^ data->window->flags;
 
                 if ((changed & SDL_WINDOW_HIDDEN) || (changed & SDL_WINDOW_FULLSCREEN)) {

src/video/x11/SDL_x11window.c

@@ -173,7 +173,7 @@ X11_SetNetWMState(_THIS, Window xwindow, Uint32 flags)
 }
 
 Uint32
-X11_GetNetWMState(_THIS, Window xwindow)
+X11_GetNetWMState(_THIS, SDL_Window *window, Window xwindow)
 {
     SDL_VideoData *videodata = (SDL_VideoData *) _this->driverdata;
     Display *display = videodata->display;
@@ -211,14 +211,28 @@ X11_GetNetWMState(_THIS, Window xwindow)
                 fullscreen = 1;
             }
         }
-        if (maximized == 3) {
-            flags |= SDL_WINDOW_MAXIMIZED;
-        }
 
         if (fullscreen == 1) {
             flags |= SDL_WINDOW_FULLSCREEN;
         }
 
+        if (maximized == 3) {
+            /* Fullscreen windows are maximized on some window managers,
+               and this is functional behavior - if maximized is removed,
+               the windows remain floating centered and not covering the
+               rest of the desktop. So we just won't change the maximize
+               state for fullscreen windows here, otherwise SDL would think
+               we're always maximized when fullscreen and not restore the
+               correct state when leaving fullscreen.
+            */
+            if (fullscreen) {
+                flags |= (window->flags & SDL_WINDOW_MAXIMIZED);
+            } else {
+                flags |= SDL_WINDOW_MAXIMIZED;
+            }
+        }
+
+
         /* If the window is unmapped, numItems will be zero and _NET_WM_STATE_HIDDEN
          * will not be set. Do an additional check to see if the window is unmapped
          * and mark it as SDL_WINDOW_HIDDEN if it is.
@@ -306,7 +320,7 @@ SetupWindowData(_THIS, SDL_Window * window, Window w, BOOL created)
         data->colormap = attrib.colormap;
     }
 
-    window->flags |= X11_GetNetWMState(_this, w);
+    window->flags |= X11_GetNetWMState(_this, window, w);
 
     {
         Window FocalWindow;
@@ -1252,6 +1266,14 @@ SetWindowMaximized(_THIS, SDL_Window * window, SDL_bool maximized)
         window->flags |= SDL_WINDOW_MAXIMIZED;
     } else {
         window->flags &= ~SDL_WINDOW_MAXIMIZED;
+
+        if ((window->flags & SDL_WINDOW_FULLSCREEN) != 0) {
+            /* Fullscreen windows are maximized on some window managers,
+               and this is functional behavior, so don't remove that state
+               now, we'll take care of it when we leave fullscreen mode.
+             */
+            return;
+        }
     }
 
     if (X11_IsWindowMapped(_this, window)) {

src/video/x11/SDL_x11window.h

@@ -81,7 +81,7 @@ typedef struct
 } SDL_WindowData;
 
 extern void X11_SetNetWMState(_THIS, Window xwindow, Uint32 flags);
-extern Uint32 X11_GetNetWMState(_THIS, Window xwindow);
+extern Uint32 X11_GetNetWMState(_THIS, SDL_Window *window, Window xwindow);
 
 extern int X11_CreateWindow(_THIS, SDL_Window * window);
 extern int X11_CreateWindowFrom(_THIS, SDL_Window * window, const void *data);