wayland: Factor out the general SHM buffer allocation/free functions from the cursor code

It will be necessary for a forthcoming toplevel icon setting protocol that will likely soon call for implementations, and is cleaner in general.

src/video/wayland/SDL_waylandmouse.c

@@ -23,13 +23,6 @@
 
 #ifdef SDL_VIDEO_DRIVER_WAYLAND
 
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <signal.h>
-#include <errno.h>
-
 #include "../SDL_sysvideo.h"
 #include "../SDL_video_c.h"
 
@@ -40,6 +33,7 @@
 
 #include "wayland-cursor.h"
 #include "SDL_waylandmouse.h"
+#include "SDL_waylandshmbuffer.h"
 
 #include "cursor-shape-v1-client-protocol.h"
 
@@ -51,18 +45,16 @@ static int Wayland_SetRelativeMouseMode(SDL_bool enabled);
 
 typedef struct
 {
-    struct wl_buffer *buffer;
+    struct Wayland_SHMBuffer shmBuffer;
     struct wl_surface *surface;
 
     int hot_x, hot_y;
     int w, h;
 
-    /* shm_data is non-NULL for custom cursors.
+    /* shmBuffer.shm_data is non-NULL for custom cursors.
      * When shm_data is NULL, system_cursor must be valid
      */
     SDL_SystemCursor system_cursor;
-    int shm_data_size;
-    void *shm_data;
 } Wayland_CursorData;
 
 static int dbus_cursor_size;
@@ -343,7 +335,7 @@ static SDL_bool wayland_get_system_cursor(SDL_VideoData *vdata, Wayland_CursorDa
     }
 
     /* ... Set the cursor data, finally. */
-    cdata->buffer = WAYLAND_wl_cursor_image_get_buffer(cursor->images[0]);
+    cdata->shmBuffer.wl_buffer = WAYLAND_wl_cursor_image_get_buffer(cursor->images[0]);
     cdata->hot_x = cursor->images[0]->hotspot_x;
     cdata->hot_y = cursor->images[0]->hotspot_y;
     cdata->w = cursor->images[0]->width;
@@ -351,134 +343,6 @@ static SDL_bool wayland_get_system_cursor(SDL_VideoData *vdata, Wayland_CursorDa
     return SDL_TRUE;
 }
 
-static int set_tmp_file_size(int fd, off_t size)
-{
-#ifdef HAVE_POSIX_FALLOCATE
-    sigset_t set, old_set;
-    int ret;
-
-    /* SIGALRM can potentially block a large posix_fallocate() operation
-     * from succeeding, so block it.
-     */
-    sigemptyset(&set);
-    sigaddset(&set, SIGALRM);
-    sigprocmask(SIG_BLOCK, &set, &old_set);
-
-    do {
-        ret = posix_fallocate(fd, 0, size);
-    } while (ret == EINTR);
-
-    sigprocmask(SIG_SETMASK, &old_set, NULL);
-
-    if (ret == 0) {
-        return 0;
-    } else if (ret != EINVAL && errno != EOPNOTSUPP) {
-        return -1;
-    }
-#endif
-
-    if (ftruncate(fd, size) < 0) {
-        return -1;
-    }
-    return 0;
-}
-
-static int wayland_create_tmp_file(off_t size)
-{
-    int fd;
-
-#ifdef HAVE_MEMFD_CREATE
-    fd = memfd_create("SDL", MFD_CLOEXEC | MFD_ALLOW_SEALING);
-    if (fd >= 0) {
-        fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_SEAL);
-    } else
-#endif
-    {
-        static const char template[] = "/sdl-shared-XXXXXX";
-        char *xdg_path;
-        char tmp_path[PATH_MAX];
-
-        xdg_path = SDL_getenv("XDG_RUNTIME_DIR");
-        if (!xdg_path) {
-            return -1;
-        }
-
-        SDL_strlcpy(tmp_path, xdg_path, PATH_MAX);
-        SDL_strlcat(tmp_path, template, PATH_MAX);
-
-        fd = mkostemp(tmp_path, O_CLOEXEC);
-        if (fd < 0) {
-            return -1;
-        }
-
-        /* Need to manually unlink the temp files, or they can persist after close and fill up the temp storage. */
-        unlink(tmp_path);
-    }
-
-    if (set_tmp_file_size(fd, size) < 0) {
-        close(fd);
-        return -1;
-    }
-
-    return fd;
-}
-
-static void mouse_buffer_release(void *data, struct wl_buffer *buffer)
-{
-}
-
-static const struct wl_buffer_listener mouse_buffer_listener = {
-    mouse_buffer_release
-};
-
-static int create_buffer_from_shm(Wayland_CursorData *d,
-                                  int width,
-                                  int height,
-                                  uint32_t format)
-{
-    SDL_VideoDevice *vd = SDL_GetVideoDevice();
-    SDL_VideoData *data = vd->driverdata;
-    struct wl_shm_pool *shm_pool;
-
-    const int stride = width * 4;
-    d->shm_data_size = stride * height;
-
-    const int shm_fd = wayland_create_tmp_file(d->shm_data_size);
-    if (shm_fd < 0) {
-        return SDL_SetError("Creating mouse cursor buffer failed.");
-    }
-
-    d->shm_data = mmap(NULL,
-                       d->shm_data_size,
-                       PROT_READ | PROT_WRITE,
-                       MAP_SHARED,
-                       shm_fd,
-                       0);
-    if (d->shm_data == MAP_FAILED) {
-        d->shm_data = NULL;
-        close(shm_fd);
-        return SDL_SetError("mmap() failed.");
-    }
-
-    SDL_assert(d->shm_data != NULL);
-
-    shm_pool = wl_shm_create_pool(data->shm, shm_fd, d->shm_data_size);
-    d->buffer = wl_shm_pool_create_buffer(shm_pool,
-                                          0,
-                                          width,
-                                          height,
-                                          stride,
-                                          format);
-    wl_buffer_add_listener(d->buffer,
-                           &mouse_buffer_listener,
-                           d);
-
-    wl_shm_pool_destroy(shm_pool);
-    close(shm_fd);
-
-    return 0;
-}
-
 static SDL_Cursor *Wayland_CreateCursor(SDL_Surface *surface, int hot_x, int hot_y)
 {
     SDL_Cursor *cursor = SDL_calloc(1, sizeof(*cursor));
@@ -493,10 +357,7 @@ static SDL_Cursor *Wayland_CreateCursor(SDL_Surface *surface, int hot_x, int hot
         cursor->driverdata = (void *)data;
 
         /* Allocate shared memory buffer for this cursor */
-        if (create_buffer_from_shm(data,
-                                   surface->w,
-                                   surface->h,
-                                   WL_SHM_FORMAT_ARGB8888) < 0) {
+        if (Wayland_AllocSHMBuffer(surface->w, surface->h, &data->shmBuffer) != 0) {
             SDL_free(cursor->driverdata);
             SDL_free(cursor);
             return NULL;
@@ -505,7 +366,7 @@ static SDL_Cursor *Wayland_CreateCursor(SDL_Surface *surface, int hot_x, int hot
         /* Wayland requires premultiplied alpha for its surfaces. */
         SDL_PremultiplyAlpha(surface->w, surface->h,
                              surface->format->format, surface->pixels, surface->pitch,
-                             SDL_PIXELFORMAT_ARGB8888, data->shm_data, surface->w * 4);
+                             SDL_PIXELFORMAT_ARGB8888, data->shmBuffer.shm_data, surface->w * 4);
 
         data->surface = wl_compositor_create_surface(wd->compositor);
         wl_surface_set_user_data(data->surface, NULL);
@@ -555,15 +416,10 @@ static SDL_Cursor *Wayland_CreateDefaultCursor(void)
 static void Wayland_FreeCursorData(Wayland_CursorData *d)
 {
     /* Buffers for system cursors must not be destroyed. */
-    if (d->buffer) {
-        if (d->shm_data) {
-            wl_buffer_destroy(d->buffer);
-
-            munmap(d->shm_data, d->shm_data_size);
-            d->shm_data_size = 0;
-            d->shm_data = NULL;
-        }
-        d->buffer = NULL;
+    if (d->shmBuffer.shm_data) {
+        Wayland_ReleaseSHMBuffer(&d->shmBuffer);
+    } else {
+        d->shmBuffer.wl_buffer = NULL;
     }
 
     if (d->surface) {
@@ -678,7 +534,7 @@ static int Wayland_ShowCursor(SDL_Cursor *cursor)
         Wayland_CursorData *data = cursor->driverdata;
 
         /* TODO: High-DPI custom cursors? -flibit */
-        if (!data->shm_data) {
+        if (!data->shmBuffer.shm_data) {
             if (input->cursor_shape) {
                 Wayland_SetSystemCursorShape(input, data->system_cursor);
                 return 0;
@@ -693,7 +549,7 @@ static int Wayland_ShowCursor(SDL_Cursor *cursor)
                               data->surface,
                               data->hot_x / scale,
                               data->hot_y / scale);
-        wl_surface_attach(data->surface, data->buffer, 0, 0);
+        wl_surface_attach(data->surface, data->shmBuffer.wl_buffer, 0, 0);
         wl_surface_damage(data->surface, 0, 0, data->w, data->h);
         wl_surface_commit(data->surface);
 

src/video/wayland/SDL_waylandmouse.h

@@ -20,9 +20,9 @@
 */
 
 #include "SDL_internal.h"
-#include "SDL_waylandvideo.h"
 
-#ifdef SDL_VIDEO_DRIVER_WAYLAND
+#ifndef SDL_waylandmouse_h_
+#define SDL_waylandmouse_h_
 
 extern void Wayland_InitMouse(void);
 extern void Wayland_FiniMouse(SDL_VideoData *data);

src/video/wayland/SDL_waylandshmbuffer.c

@@ -0,0 +1,170 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+#include "SDL_internal.h"
+
+#ifdef SDL_VIDEO_DRIVER_WAYLAND
+
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <signal.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "SDL_waylandshmbuffer.h"
+#include "SDL_waylandvideo.h"
+
+static int SetTempFileSize(int fd, off_t size)
+{
+#ifdef HAVE_POSIX_FALLOCATE
+    sigset_t set, old_set;
+    int ret;
+
+    /* SIGALRM can potentially block a large posix_fallocate() operation
+     * from succeeding, so block it.
+     */
+    sigemptyset(&set);
+    sigaddset(&set, SIGALRM);
+    sigprocmask(SIG_BLOCK, &set, &old_set);
+
+    do {
+        ret = posix_fallocate(fd, 0, size);
+    } while (ret == EINTR);
+
+    sigprocmask(SIG_SETMASK, &old_set, NULL);
+
+    if (ret == 0) {
+        return 0;
+    } else if (ret != EINVAL && errno != EOPNOTSUPP) {
+        return -1;
+    }
+#endif
+
+    if (ftruncate(fd, size) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+static int CreateTempFD(off_t size)
+{
+    int fd;
+
+#ifdef HAVE_MEMFD_CREATE
+    fd = memfd_create("SDL", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    if (fd >= 0) {
+        fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_SEAL);
+    } else
+#endif
+    {
+        static const char template[] = "/sdl-shared-XXXXXX";
+        char *xdg_path;
+        char tmp_path[PATH_MAX];
+
+        xdg_path = SDL_getenv("XDG_RUNTIME_DIR");
+        if (!xdg_path) {
+            return -1;
+        }
+
+        SDL_strlcpy(tmp_path, xdg_path, PATH_MAX);
+        SDL_strlcat(tmp_path, template, PATH_MAX);
+
+        fd = mkostemp(tmp_path, O_CLOEXEC);
+        if (fd < 0) {
+            return -1;
+        }
+
+        /* Need to manually unlink the temp files, or they can persist after close and fill up the temp storage. */
+        unlink(tmp_path);
+    }
+
+    if (SetTempFileSize(fd, size) < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return fd;
+}
+
+static void buffer_handle_release(void *data, struct wl_buffer *wl_buffer)
+{
+    /* NOP */
+}
+
+static struct wl_buffer_listener buffer_listener = {
+    buffer_handle_release
+};
+
+int Wayland_AllocSHMBuffer(int width, int height, struct Wayland_SHMBuffer *shmBuffer)
+{
+    SDL_VideoDevice *vd = SDL_GetVideoDevice();
+    SDL_VideoData *data = vd->driverdata;
+    struct wl_shm_pool *shm_pool;
+    const Uint32 SHM_FMT = WL_SHM_FORMAT_ARGB8888;
+
+    if (!shmBuffer) {
+        return SDL_InvalidParamError("shmBuffer");
+    }
+
+    const int stride = width * 4;
+    shmBuffer->shm_data_size = stride * height;
+
+    const int shm_fd = CreateTempFD(shmBuffer->shm_data_size);
+    if (shm_fd < 0) {
+        return SDL_SetError("Creating SHM buffer failed.");
+    }
+
+    shmBuffer->shm_data = mmap(NULL, shmBuffer->shm_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0);
+    if (shmBuffer->shm_data == MAP_FAILED) {
+        shmBuffer->shm_data = NULL;
+        close(shm_fd);
+        return SDL_SetError("mmap() failed.");
+    }
+
+    SDL_assert(shmBuffer->shm_data != NULL);
+
+    shm_pool = wl_shm_create_pool(data->shm, shm_fd, shmBuffer->shm_data_size);
+    shmBuffer->wl_buffer = wl_shm_pool_create_buffer(shm_pool, 0, width, height, stride, SHM_FMT);
+    wl_buffer_add_listener(shmBuffer->wl_buffer, &buffer_listener, shmBuffer);
+
+    wl_shm_pool_destroy(shm_pool);
+    close(shm_fd);
+
+    return 0;
+}
+
+void Wayland_ReleaseSHMBuffer(struct Wayland_SHMBuffer *shmBuffer)
+{
+    if (shmBuffer) {
+        if (shmBuffer->wl_buffer) {
+            wl_buffer_destroy(shmBuffer->wl_buffer);
+            shmBuffer->wl_buffer = NULL;
+        }
+        if (shmBuffer->shm_data) {
+            munmap(shmBuffer->shm_data, shmBuffer->shm_data_size);
+            shmBuffer->shm_data = NULL;
+        }
+        shmBuffer->shm_data_size = 0;
+    }
+}
+
+#endif

src/video/wayland/SDL_waylandshmbuffer.h

@@ -0,0 +1,38 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+#include "SDL_internal.h"
+
+#ifndef SDL_waylandshmbuffer_h_
+#define SDL_waylandshmbuffer_h_
+
+struct Wayland_SHMBuffer
+{
+    struct wl_buffer *wl_buffer;
+    void *shm_data;
+    int shm_data_size;
+};
+
+/* Allocates an SHM buffer with the format WL_SHM_FORMAT_ARGB8888 */
+extern int Wayland_AllocSHMBuffer(int width, int height, struct Wayland_SHMBuffer *shmBuffer);
+extern void Wayland_ReleaseSHMBuffer(struct Wayland_SHMBuffer *shmBuffer);
+
+#endif