wayland: Enforce or override libdecor minimum window size

libdecor plugins can change the min/max window size values internally to enforce a minimum window size, and errors and crashes can result if the window size is below the internal limit.

On versions of libdecor >= 0.1.1, the minimum width and height can be queried and the minimum required window size will be enforced. The application requested window size is still respected, however, the actual window may be slightly larger than the drawable area to accommodate the required libdecor minimum size.

On version 0.1.0 of libdecor, which lacks the function to retrieve the minimum size, the internal limits are overridden before committing a frame, so that the internal limits always match the window size as a workaround, even if the window is technically smaller than the plugin would normally allow.

(cherry picked from commit 423a82cd4b65cf72668551093bfdf58d49bce9ce)

cmake/sdlchecks.cmake

@@ -734,6 +734,17 @@ macro(CheckWayland)
             else()
               list(APPEND EXTRA_LIBS ${PKG_LIBDECOR_LIBRARIES})
             endif()
+
+            cmake_push_check_state()
+            list(APPEND CMAKE_REQUIRED_FLAGS ${PKG_LIBDECOR_CFLAGS})
+            list(APPEND CMAKE_REQUIRED_INCLUDES ${PKG_LIBDECOR_INCLUDE_DIRS})
+            list(APPEND CMAKE_REQUIRED_LIBRARIES ${PKG_LIBDECOR_LINK_LIBRARIES})
+            check_symbol_exists(libdecor_frame_get_max_content_size "libdecor.h" HAVE_LIBDECOR_FRAME_GET_MAX_CONTENT_SIZE)
+            check_symbol_exists(libdecor_frame_get_min_content_size "libdecor.h" HAVE_LIBDECOR_FRAME_GET_MIN_CONTENT_SIZE)
+            if(HAVE_LIBDECOR_FRAME_GET_MAX_CONTENT_SIZE AND HAVE_LIBDECOR_FRAME_GET_MIN_CONTENT_SIZE)
+              set(SDL_HAVE_LIBDECOR_GET_MIN_MAX 1)
+            endif()
+            cmake_pop_check_state()
         endif()
       endif()
 

include/SDL_config.h.cmake

@@ -541,6 +541,8 @@
 #cmakedefine SDL_VIDEO_VITA_PVR @SDL_VIDEO_VITA_PVR@
 #cmakedefine SDL_VIDEO_VITA_PVR_OGL @SDL_VIDEO_VITA_PVR_OGL@
 
+#cmakedefine SDL_HAVE_LIBDECOR_GET_MIN_MAX @SDL_HAVE_LIBDECOR_GET_MIN_MAX@
+
 #if !defined(HAVE_STDINT_H) && !defined(_STDINT_H_)
 /* Most everything except Visual Studio 2008 and earlier has stdint.h now */
 #if defined(_MSC_VER) && (_MSC_VER < 1600)

src/video/wayland/SDL_waylanddyn.c

@@ -58,7 +58,7 @@ static waylanddynlib waylandlibs[] = {
     { NULL, SDL_VIDEO_DRIVER_WAYLAND_DYNAMIC_LIBDECOR }
 };
 
-static void *WAYLAND_GetSym(const char *fnname, int *pHasModule)
+static void *WAYLAND_GetSym(const char *fnname, int *pHasModule, SDL_bool required)
 {
     int i;
     void *fn = NULL;
@@ -78,7 +78,7 @@ static void *WAYLAND_GetSym(const char *fnname, int *pHasModule)
         SDL_Log("WAYLAND: Symbol '%s' NOT FOUND!\n", fnname);
 #endif
 
-    if (fn == NULL) {
+    if (fn == NULL && required) {
         *pHasModule = 0; /* kill this module. */
     }
 
@@ -92,9 +92,10 @@ static void *WAYLAND_GetSym(const char *fnname, int *pHasModule)
 #endif /* SDL_VIDEO_DRIVER_WAYLAND_DYNAMIC */
 
 /* Define all the function pointers and wrappers... */
-#define SDL_WAYLAND_MODULE(modname)     int SDL_WAYLAND_HAVE_##modname = 0;
-#define SDL_WAYLAND_SYM(rc, fn, params) SDL_DYNWAYLANDFN_##fn WAYLAND_##fn = NULL;
-#define SDL_WAYLAND_INTERFACE(iface)    const struct wl_interface *WAYLAND_##iface = NULL;
+#define SDL_WAYLAND_MODULE(modname)         int SDL_WAYLAND_HAVE_##modname = 0;
+#define SDL_WAYLAND_SYM(rc, fn, params)     SDL_DYNWAYLANDFN_##fn WAYLAND_##fn = NULL;
+#define SDL_WAYLAND_SYM_OPT(rc, fn, params) SDL_DYNWAYLANDFN_##fn WAYLAND_##fn = NULL;
+#define SDL_WAYLAND_INTERFACE(iface)        const struct wl_interface *WAYLAND_##iface = NULL;
 #include "SDL_waylandsym.h"
 
 static int wayland_load_refcount = 0;
@@ -109,9 +110,10 @@ void SDL_WAYLAND_UnloadSymbols(void)
 #endif
 
             /* set all the function pointers to NULL. */
-#define SDL_WAYLAND_MODULE(modname)     SDL_WAYLAND_HAVE_##modname = 0;
-#define SDL_WAYLAND_SYM(rc, fn, params) WAYLAND_##fn = NULL;
-#define SDL_WAYLAND_INTERFACE(iface)    WAYLAND_##iface = NULL;
+#define SDL_WAYLAND_MODULE(modname)         SDL_WAYLAND_HAVE_##modname = 0;
+#define SDL_WAYLAND_SYM(rc, fn, params)     WAYLAND_##fn = NULL;
+#define SDL_WAYLAND_SYM_OPT(rc, fn, params) WAYLAND_##fn = NULL;
+#define SDL_WAYLAND_INTERFACE(iface)        WAYLAND_##iface = NULL;
 #include "SDL_waylandsym.h"
 
 #ifdef SDL_VIDEO_DRIVER_WAYLAND_DYNAMIC
@@ -145,9 +147,10 @@ int SDL_WAYLAND_LoadSymbols(void)
 #define SDL_WAYLAND_MODULE(modname) SDL_WAYLAND_HAVE_##modname = 1; /* default yes */
 #include "SDL_waylandsym.h"
 
-#define SDL_WAYLAND_MODULE(modname)     thismod = &SDL_WAYLAND_HAVE_##modname;
-#define SDL_WAYLAND_SYM(rc, fn, params) WAYLAND_##fn = (SDL_DYNWAYLANDFN_##fn)WAYLAND_GetSym(#fn, thismod);
-#define SDL_WAYLAND_INTERFACE(iface)    WAYLAND_##iface = (struct wl_interface *)WAYLAND_GetSym(#iface, thismod);
+#define SDL_WAYLAND_MODULE(modname)         thismod = &SDL_WAYLAND_HAVE_##modname;
+#define SDL_WAYLAND_SYM(rc, fn, params)     WAYLAND_##fn = (SDL_DYNWAYLANDFN_##fn)WAYLAND_GetSym(#fn, thismod, SDL_TRUE);
+#define SDL_WAYLAND_SYM_OPT(rc, fn, params) WAYLAND_##fn = (SDL_DYNWAYLANDFN_##fn)WAYLAND_GetSym(#fn, thismod, SDL_FALSE);
+#define SDL_WAYLAND_INTERFACE(iface)        WAYLAND_##iface = (struct wl_interface *)WAYLAND_GetSym(#iface, thismod, SDL_TRUE);
 #include "SDL_waylandsym.h"
 
         if (SDL_WAYLAND_HAVE_WAYLAND_CLIENT) {
@@ -161,9 +164,10 @@ int SDL_WAYLAND_LoadSymbols(void)
 
 #else /* no dynamic WAYLAND */
 
-#define SDL_WAYLAND_MODULE(modname)     SDL_WAYLAND_HAVE_##modname = 1; /* default yes */
-#define SDL_WAYLAND_SYM(rc, fn, params) WAYLAND_##fn = fn;
-#define SDL_WAYLAND_INTERFACE(iface)    WAYLAND_##iface = &iface;
+#define SDL_WAYLAND_MODULE(modname)         SDL_WAYLAND_HAVE_##modname = 1; /* default yes */
+#define SDL_WAYLAND_SYM(rc, fn, params)     WAYLAND_##fn = fn;
+#define SDL_WAYLAND_SYM_OPT(rc, fn, params) WAYLAND_##fn = fn;
+#define SDL_WAYLAND_INTERFACE(iface)        WAYLAND_##iface = &iface;
 #include "SDL_waylandsym.h"
 
 #endif

src/video/wayland/SDL_waylanddyn.h

@@ -71,6 +71,9 @@ void SDL_WAYLAND_UnloadSymbols(void);
 #define SDL_WAYLAND_SYM(rc, fn, params)        \
     typedef rc(*SDL_DYNWAYLANDFN_##fn) params; \
     extern SDL_DYNWAYLANDFN_##fn WAYLAND_##fn;
+#define SDL_WAYLAND_SYM_OPT(rc, fn, params)    \
+    typedef rc(*SDL_DYNWAYLANDFN_##fn) params; \
+    extern SDL_DYNWAYLANDFN_##fn WAYLAND_##fn;
 #define SDL_WAYLAND_INTERFACE(iface) extern const struct wl_interface *WAYLAND_##iface;
 #include "SDL_waylandsym.h"
 
@@ -137,7 +140,9 @@ void SDL_WAYLAND_UnloadSymbols(void);
 #define libdecor_frame_set_title                (*WAYLAND_libdecor_frame_set_title)
 #define libdecor_frame_set_app_id               (*WAYLAND_libdecor_frame_set_app_id)
 #define libdecor_frame_set_max_content_size     (*WAYLAND_libdecor_frame_set_max_content_size)
+#define libdecor_frame_get_max_content_size     (*WAYLAND_libdecor_frame_get_max_content_size)
 #define libdecor_frame_set_min_content_size     (*WAYLAND_libdecor_frame_set_min_content_size)
+#define libdecor_frame_get_min_content_size     (*WAYLAND_libdecor_frame_get_min_content_size)
 #define libdecor_frame_resize                   (*WAYLAND_libdecor_frame_resize)
 #define libdecor_frame_move                     (*WAYLAND_libdecor_frame_move)
 #define libdecor_frame_commit                   (*WAYLAND_libdecor_frame_commit)

src/video/wayland/SDL_waylandsym.h

@@ -29,6 +29,10 @@
 #define SDL_WAYLAND_SYM(rc,fn,params)
 #endif
 
+#ifndef SDL_WAYLAND_SYM_OPT
+#define SDL_WAYLAND_SYM_OPT(rc,fn,params)
+#endif
+
 #ifndef SDL_WAYLAND_INTERFACE
 #define SDL_WAYLAND_INTERFACE(iface)
 #endif
@@ -212,10 +216,19 @@ SDL_WAYLAND_SYM(bool, libdecor_configuration_get_content_size, (struct libdecor_
 SDL_WAYLAND_SYM(bool, libdecor_configuration_get_window_state, (struct libdecor_configuration *,\
                                                                 enum libdecor_window_state *))
 SDL_WAYLAND_SYM(int, libdecor_dispatch, (struct libdecor *, int))
+
+/* Only found in libdecor 0.1.1 or higher, so failure to load them is not fatal. */
+SDL_WAYLAND_SYM_OPT(void, libdecor_frame_get_min_content_size, (struct libdecor_frame *,\
+                                                            int *,\
+                                                            int *))
+SDL_WAYLAND_SYM_OPT(void, libdecor_frame_get_max_content_size, (struct libdecor_frame *,\
+                                                            int *,\
+                                                            int *))
 #endif
 
 #undef SDL_WAYLAND_MODULE
 #undef SDL_WAYLAND_SYM
+#undef SDL_WAYLAND_SYM_OPT
 #undef SDL_WAYLAND_INTERFACE
 
 /* *INDENT-ON* */ /* clang-format on */

src/video/wayland/SDL_waylandwindow.c

@@ -270,8 +270,9 @@ static void ConfigureWindowGeometry(SDL_Window *window)
                 wl_surface_set_buffer_scale(data->surface, (int32_t)SDL_ceilf(data->scale_factor));
             }
 
-            data->window_width = window->w;
-            data->window_height = window->h;
+            /* Clamp the physical window size to the system minimum required size. */
+            data->window_width = SDL_max(window->w, data->system_min_required_width);
+            data->window_height = SDL_max(window->h, data->system_min_required_height);
 
             data->pointer_scale_x = 1.0f;
             data->pointer_scale_y = 1.0f;
@@ -767,6 +768,47 @@ static const struct zxdg_toplevel_decoration_v1_listener decoration_listener = {
 };
 
 #ifdef HAVE_LIBDECOR_H
+/*
+ * XXX: Hack for older versions of libdecor that lack the function to query the
+ *      minimum content size limit. The internal limits must always be overridden
+ *      to ensure that very small windows don't cause errors or crashes.
+ *
+ *      On versions of libdecor that expose the function to get the minimum content
+ *      size limit, this function is a no-op.
+ *
+ *      Can be removed if the minimum required version of libdecor is raised
+ *      to a version that guarantees the availability of this function.
+ */
+static void OverrideLibdecorLimits(SDL_Window *window)
+{
+#if defined(SDL_VIDEO_DRIVER_WAYLAND_DYNAMIC_LIBDECOR)
+    if (libdecor_frame_get_min_content_size == NULL) {
+        SetMinMaxDimensions(window, SDL_FALSE);
+    }
+#elif !defined(SDL_HAVE_LIBDECOR_GET_MIN_MAX)
+    SetMinMaxDimensions(window, SDL_FALSE);
+#endif
+}
+
+/*
+ * NOTE: Retrieves the minimum content size limits, if the function for doing so is available.
+ *       On versions of libdecor that lack the minimum content size retrieval function, this
+ *       function is a no-op.
+ *
+ *       Can be replaced with a direct call if the minimum required version of libdecor is raised
+ *       to a version that guarantees the availability of this function.
+ */
+static void LibdecorGetMinContentSize(struct libdecor_frame *frame, int *min_w, int *min_h)
+{
+#if defined(SDL_VIDEO_DRIVER_WAYLAND_DYNAMIC_LIBDECOR)
+    if (libdecor_frame_get_min_content_size != NULL) {
+        libdecor_frame_get_min_content_size(frame, min_w, min_h);
+    }
+#elif defined(SDL_HAVE_LIBDECOR_GET_MIN_MAX)
+    libdecor_frame_get_min_content_size(frame, min_w, min_h);
+#endif
+}
+
 static void decoration_frame_configure(struct libdecor_frame *frame,
                                        struct libdecor_configuration *configuration,
                                        void *user_data)
@@ -850,6 +892,8 @@ static void decoration_frame_configure(struct libdecor_frame *frame,
         width = window->windowed.w;
         height = window->windowed.h;
         wind->floating_resize_pending = SDL_FALSE;
+
+        OverrideLibdecorLimits(window);
     } else {
         /*
          * XXX: libdecor can send bogus content sizes that are +/- the height
@@ -889,13 +933,17 @@ static void decoration_frame_configure(struct libdecor_frame *frame,
 
     /* Do the resize on the SDL side (this will set window->w/h)... */
     Wayland_HandleResize(window, width, height, scale_factor);
-    wind->shell_surface.libdecor.initial_configure_seen = SDL_TRUE;
 
     /* ... then commit the changes on the libdecor side. */
     state = libdecor_state_new(wind->window_width, wind->window_height);
     libdecor_frame_commit(frame, state, configuration);
     libdecor_state_free(state);
 
+    if (!wind->shell_surface.libdecor.initial_configure_seen) {
+        LibdecorGetMinContentSize(frame, &wind->system_min_required_width, &wind->system_min_required_height);
+        wind->shell_surface.libdecor.initial_configure_seen = SDL_TRUE;
+    }
+
     /* Update the resize capability. Since this will change the capabilities and
      * commit a new frame state with the last known content dimension, this has
      * to be called after the new state has been committed and the new content
@@ -1404,6 +1452,20 @@ void Wayland_ShowWindow(_THIS, SDL_Window *window)
         if (data->shell_surface.libdecor.frame && window->flags & SDL_WINDOW_BORDERLESS) {
             Wayland_SetWindowBordered(_this, window, SDL_FALSE);
         }
+
+        /* Libdecor plugins can enforce minimum window sizes, so adjust if the initial window size is too small. */
+        if (window->windowed.w < data->system_min_required_width ||
+            window->windowed.h < data->system_min_required_height) {
+
+            /* Warn if the window frame will be larger than the content surface. */
+            SDL_LogWarn(SDL_LOG_CATEGORY_VIDEO,
+                        "Window dimensions (%i, %i) are smaller than the system enforced minimum (%i, %i); window borders will be larger than the content surface.",
+                        window->windowed.w, window->windowed.h, data->system_min_required_width, data->system_min_required_height);
+
+            data->window_width = SDL_max(window->windowed.w, data->system_min_required_width);
+            data->window_height = SDL_max(window->windowed.h, data->system_min_required_height);
+            CommitLibdecorFrame(window);
+        }
     } else
 #endif
     {
@@ -2112,12 +2174,15 @@ void Wayland_SetWindowSize(_THIS, SDL_Window *window)
 
 #ifdef HAVE_LIBDECOR_H
     /* we must not resize the window while we have a static (non-floating) size */
-    if (wind->shell_surface_type == WAYLAND_SURFACE_LIBDECOR &&
-        wind->shell_surface.libdecor.frame &&
-        !libdecor_frame_is_floating(wind->shell_surface.libdecor.frame)) {
-        /* Commit the resize when we re-enter floating state */
-        wind->floating_resize_pending = SDL_TRUE;
-        return;
+    if (wind->shell_surface_type == WAYLAND_SURFACE_LIBDECOR) {
+        if (wind->shell_surface.libdecor.frame &&
+            !libdecor_frame_is_floating(wind->shell_surface.libdecor.frame)) {
+            /* Commit the resize when we re-enter floating state */
+            wind->floating_resize_pending = SDL_TRUE;
+            return;
+        }
+
+        OverrideLibdecorLimits(window);
     }
 #endif
 

src/video/wayland/SDL_waylandwindow.h

@@ -108,6 +108,8 @@ typedef struct
     int drawable_width, drawable_height;
     int fs_output_width, fs_output_height;
     int window_width, window_height;
+    int system_min_required_width;
+    int system_min_required_height;
     SDL_bool needs_resize_event;
     SDL_bool floating_resize_pending;
     SDL_bool was_floating;