fix(gamescope): release cursor SHM buffers during long sessions (#1189)

* fix(gamescope): avoid retaining pointer cursor SHM commits

* test(gamescope): verify cursor release and pending window control

* test(gamescope): keep live probe inert during test discovery

* test: match Gamescope process name in cursor probe

* test: clarify cursor-role probe serial behavior

---------

Co-authored-by: siyuan.qsy <siyuan.qsy@alibaba-inc.com>
Co-authored-by: xXJsonDeruloXx <danielhimebauch@gmail.com>

tools/gamescope/PATCHES.md

@@ -66,3 +66,22 @@ library list, before anything is published.
 
 Sixteen more of Armada's patches are DRM/lease/HDR-on-KMS work for a native display, which this
 app's Wayland-hosted gamescope never reaches, or need a newer gamescope than the runtime has.
+
+- `0116-release-cursor-shm-buffers.patch` - this app: pointer cursor surfaces have no
+  X11/xdg window association, so routing their commits through `g_PendingCommits`
+  retained every old SHM buffer until the surface was destroyed. Ignore cursor-role
+  commits before taking the buffer lock; X11 cursor images still follow the existing
+  MouseCursor path. Window commits awaiting association are unchanged.
+
+The live regression probe is `tools/tests/test_gamescope_cursor_shm.py`. Run it
+inside an active Gamescope session with access to that process's `/proc` fd
+directory. It submits 20 distinct SHM cursor buffers and requires at most the
+current buffer to remain open, then none after surface destruction. The
+`--unassigned-control` variant omits the cursor role and verifies that all 20
+commits awaiting a window association remain queued until surface destruction.
+
+Device validation (2026-10-20): the stock component retained 20/20 cursor buffers;
+the patched component retained 1/20, then 0 after destruction. The unassigned
+control retained 20/20 and then 0. In Backpack Battles, two batches of ten touch
+drags leaked 188 and 197 SHM descriptors before the fix; the same batches added
+zero after the fix (112 total descriptors before and after each batch).

tools/gamescope/patches/0116-release-cursor-shm-buffers.patch

@@ -0,0 +1,15 @@
+--- a/src/wlserver.cpp
++++ b/src/wlserver.cpp
+@@ -203,6 +203,12 @@
+ }
+ 
+ void xwayland_surface_commit(struct wlr_surface *wlr_surface) {
++	// Cursor surfaces never acquire an X11/xdg window association. Keeping their
++	// commits in g_PendingCommits pins every old SHM buffer until the cursor
++	// surface is destroyed. X11 cursor images are handled by MouseCursor.
++	if ( wlr_surface->role && strcmp( wlr_surface->role->name, "wl_pointer-cursor" ) == 0 )
++		return;
++
+ 	wlr_surface->current.committed = 0;
+ 
+ 	wlserver_x11_surface_info *wlserver_x11_surface_info = get_wl_surface_info(wlr_surface)->x11_surface;

tools/tests/test_gamescope_cursor_shm.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Live regression probe; run inside a running Gamescope session.
+
+Requires Python 3, WAYLAND_DISPLAY, XDG_RUNTIME_DIR and access to Gamescope's
+/proc/<pid>/fd. Creates only a temporary surface with the cursor role; it does
+not take pointer focus or change game/save files. Affected Gamescope retains all
+20 SHM allocations; fixed Gamescope retains at most the current cursor buffer.
+"""
+import os
+import socket
+import struct
+import time
+import json
+import array
+import argparse
+from pathlib import Path
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--unassigned-control', action='store_true', help='verify that window buffers awaiting association remain queued')
+    args = parser.parse_args()
+    s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    s.settimeout(2)
+    s.connect(os.path.join(os.environ['XDG_RUNTIME_DIR'], os.environ['WAYLAND_DISPLAY']))
+    U = lambda *v: struct.pack('<' + 'I' * len(v), *v)
+
+    def string(v):
+        b = v.encode() + b'\x00'
+        return U(len(b)) + b + b'\x00' * (-len(b) % 4)
+
+    def send(obj, op, p=b'', fd=None):
+        b = U(obj, len(p) + 8 << 16 | op) + p
+        if fd is None:
+            s.sendall(b)
+        else:
+            s.sendmsg([b], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [fd]))])
+    nextid = 2
+    pids = []
+    for process in Path('/proc').iterdir():
+        if not process.name.isdigit():
+            continue
+        try:
+            if (process / 'comm').read_text().strip() == 'gamescope':
+                pids.append(process)
+        except (FileNotFoundError, PermissionError):
+            pass
+    assert len(pids) == 1, f'expected one Gamescope process, found {[p.name for p in pids]}'
+    fd_dir = pids[0] / 'fd'
+
+    def alloc():
+        nonlocal nextid
+        nextid += 1
+        return nextid
+    pending = b''
+    globals = {}
+
+    def roundtrip():
+        nonlocal pending
+        cb = alloc()
+        send(1, 0, U(cb))
+        while True:
+            while len(pending) >= 8:
+                obj, h = struct.unpack_from('<II', pending)
+                n = h >> 16
+                op = h & 65535
+                if len(pending) < n:
+                    break
+                p = pending[8:n]
+                pending = pending[n:]
+                if obj == 1 and op == 0:
+                    raise RuntimeError('wayland error ' + repr(p))
+                if obj == 2 and op == 0:
+                    name, l = struct.unpack_from('<II', p)
+                    itf = p[8:8 + l - 1].decode()
+                    ver = struct.unpack_from('<I', p, 8 + (l + 3) // 4 * 4)[0]
+                    globals[itf] = (name, ver)
+                if obj == cb:
+                    return
+            chunk = s.recv(65536)
+            if not chunk:
+                raise RuntimeError('Wayland connection closed during roundtrip')
+            pending += chunk
+
+    def bind(itf, version):
+        obj = alloc()
+        name, ver = globals[itf]
+        send(2, 0, U(name) + string(itf) + U(min(ver, version), obj))
+        return obj
+
+    def count():
+        n = 0
+        for f in fd_dir.iterdir():
+            try:
+                n += 'dd-shm-probe' in os.readlink(f)
+            except FileNotFoundError:
+                pass
+        return n
+    send(1, 1, U(2))
+    roundtrip()
+    comp = bind('wl_compositor', 4)
+    shm = bind('wl_shm', 1)
+    seat = bind('wl_seat', 1)
+    pointer = alloc()
+    send(seat, 0, U(pointer))
+    surface = alloc()
+    send(comp, 0, U(surface))
+    if not args.unassigned_control:
+        # wlroots 0.20.2 assigns the cursor role before forwarding this serial
+        # to Gamescope, so zero avoids needing pointer focus but still tests the
+        # role guard in xwayland_surface_commit.
+        send(pointer, 0, U(0, surface, 0, 0))
+    roundtrip()
+    before = count()
+    for i in range(20):
+        fd = os.memfd_create('dd-shm-probe', os.MFD_CLOEXEC)
+        os.ftruncate(fd, 64 * 64 * 4)
+        pool = alloc()
+        send(shm, 0, U(pool, 64 * 64 * 4), fd)
+        os.close(fd)
+        buffer = alloc()
+        send(pool, 0, U(buffer, 0, 64, 64, 64 * 4, 0))
+        send(pool, 1)
+        send(surface, 1, U(buffer, 0, 0))
+        send(surface, 2, U(0, 0, 64, 64))
+        send(surface, 6)
+        send(buffer, 0)
+        roundtrip()
+    after_commits = count()
+    send(surface, 0)
+    roundtrip()
+    time.sleep(0.2)
+    after_surface_destroy = count()
+    s.close()
+    print(json.dumps(dict(before=before, after_commits=after_commits, after_surface_destroy=after_surface_destroy, growth=after_commits - before)))
+    if args.unassigned_control:
+        assert after_commits - before == 20, 'unassigned window commits were unexpectedly discarded'
+    else:
+        assert after_commits - before <= 1, f'cursor buffers leaked: {after_commits - before} retained after 20 commits'
+    assert after_surface_destroy == before, 'cursor buffers remain after destroying the test surface'
+
+
+if __name__ == "__main__":
+    main()