cocoa: Set appropriate flags on fullscreen spaces windows

Don't mark fullscreen windows as having a title bar if the window is borderless, or it can end up in a weird, pseudo-decorated state when leaving fullscreen if the borders were toggled back on while the window was fullscreen.

Setting the window styling when about to leave fullscreen caused issues with this as well and is no longer needed, as it is ensured that the window's resizable state won't change while in a fullscreen space, or in a transition period.

src/video/cocoa/SDL_cocoawindow.m

@@ -1140,8 +1140,19 @@ static SDL_bool Cocoa_IsZoomed(SDL_Window *window)
 - (void)windowWillEnterFullScreen:(NSNotification *)aNotification
 {
     SDL_Window *window = _data.window;
+    NSUInteger flags = NSWindowStyleMaskClosable | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable;
 
-    SetWindowStyle(window, (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable));
+    /* Don't set the titled flag on a fullscreen window if the windowed-mode window
+     * is borderless, or the window can wind up in a weird, pseudo-decorated state
+     * when leaving fullscreen if the border flag was toggled on.
+     */
+    if (!(window->flags & SDL_WINDOW_BORDERLESS)) {
+        flags |= NSWindowStyleMaskTitled;
+    } else {
+        flags |= NSWindowStyleMaskBorderless;
+    }
+
+    SetWindowStyle(window, flags);
 
     _data.was_zoomed = !!(window->flags & SDL_WINDOW_MAXIMIZED);
 
@@ -1205,27 +1216,24 @@ static SDL_bool Cocoa_IsZoomed(SDL_Window *window)
 
     isFullscreenSpace = NO;
     inFullscreenTransition = YES;
-
-    /* As of macOS 10.11, the window seems to need to be resizable when exiting
-       a Space, in order for it to resize back to its windowed-mode size.
-       As of macOS 10.15, the window decorations can go missing sometimes after
-       certain fullscreen-desktop->exlusive-fullscreen->windowed mode flows
-       sometimes. Making sure the style mask always uses the windowed mode style
-       when returning to windowed mode from a space (instead of using a pending
-       fullscreen mode style mask) seems to work around that issue.
-     */
-    SetWindowStyle(window, GetWindowWindowedStyle(window) | NSWindowStyleMaskResizable);
 }
 
 - (void)windowDidFailToExitFullScreen:(NSNotification *)aNotification
 {
     SDL_Window *window = _data.window;
+    NSUInteger flags = NSWindowStyleMaskClosable | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable;
 
     if (window->is_destroying) {
         return;
     }
 
-    SetWindowStyle(window, (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable));
+    if (!(window->flags & SDL_WINDOW_BORDERLESS)) {
+        flags |= NSWindowStyleMaskTitled;
+    } else {
+        flags |= NSWindowStyleMaskBorderless;
+    }
+    
+    SetWindowStyle(window, flags);
 
     isFullscreenSpace = YES;
     inFullscreenTransition = NO;
@@ -2456,7 +2464,7 @@ void Cocoa_SetWindowBordered(SDL_VideoDevice *_this, SDL_Window *window, SDL_boo
 void Cocoa_SetWindowResizable(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool resizable)
 {
     @autoreleasepool {
-        /* Don't set this if we're in a space!
+        /* Don't set this if we're in or transitioning to/from a space!
          * The window will get permanently stuck if resizable is false.
          * -flibit
          */
@@ -2464,7 +2472,7 @@ void Cocoa_SetWindowResizable(SDL_VideoDevice *_this, SDL_Window *window, SDL_bo
         Cocoa_WindowListener *listener = data.listener;
         NSWindow *nswindow = data.nswindow;
         SDL_CocoaVideoData *videodata = data.videodata;
-        if (![listener isInFullscreenSpace]) {
+        if (![listener isInFullscreenSpace] && ![listener isInFullscreenSpaceTransition]) {
             SetWindowStyle(window, GetWindowStyle(window));
         }
         if (videodata.allow_spaces) {