From ffa40a15d5655fb7fb2aa22cdf51d13f7ba95af1 Mon Sep 17 00:00:00 2001 From: MaxsTechReview Date: Fri, 23 Oct 2026 02:12:49 -0400 Subject: [PATCH] perf: run Wine's arm64 SIGUSR2 context switch without a kernel signal (#1371) --- .github/workflows/build.yml | 2 +- README.md | 2 +- tools/linuxfs/preload/usr2.c | 218 ++++++++++++++++++++++ tools/tests/test_usr2_preload.py | 301 +++++++++++++++++++++++++++++++ 4 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 tools/linuxfs/preload/usr2.c create mode 100644 tools/tests/test_usr2_preload.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aacad45f03..ebfd545ca2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -601,7 +601,7 @@ jobs: - name: Install Python test dependencies run: | sudo apt-get update -qq - sudo apt-get install -y -qq python3-yaml + sudo apt-get install -y -qq python3-yaml gcc-aarch64-linux-gnu qemu-user - name: Steam import and game environment tests run: python3 -m unittest discover -s tools/tests -p 'test_*.py' diff --git a/README.md b/README.md index 62e7939526..7515dffdd6 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Compatibility and performance vary by device; hardware validation is limited. De ## Credits and licence -GPL-3.0. Runtime, shim, input, and controller work build on WinNative and Bannerlator (maxjivi05). Epic Games and GOG support build on [Phobos’s](https://github.com/phobos665) adaptation of [Legendary](https://github.com/legendary-gl/legendary) and [gogdl](https://github.com/Heroic-Games-Launcher/heroic-gogdl), respectively. The registry entries a GOG game's install script (`goggame-.script`) sets are read the way [Heroic Games Launcher](https://github.com/Heroic-Games-Launcher/HeroicGamesLauncher)'s GOG setup (GPL-3.0) reads them. LSFG frame generation is from the work of Camille LaVey and the [Eden](https://eden-emu.dev) emulator project, following [lsfg-vk](https://github.com/PancakeTAS/lsfg-vk), ported to WinNative and DroidDeck by [@maxjivi05](https://github.com/maxjivi05); it needs your own copy of [Lossless Scaling](https://store.steampowered.com/app/993090/) and ships none of its shaders. x86 AppImages, and any whose own runtime cannot unpack them, are unpacked with [uruntime](https://github.com/VHSgunzo/uruntime) by VHSgunzo (MIT), shipped unmodified with its licence, which also starts the arm64 emulator AppImages whose own runtime cannot run its extractor on Android. The Steam client's audio goes through [@The412Banner](https://github.com/The412Banner)'s [DirectAudio](https://github.com/The412Banner/directaudio) PulseAudio sink (LGPL-2.1-or-later), built from `tools/directaudio/sink` with its request size kept to its ring so clients are not asked to queue 100 ms ahead. mmap ring-buffer controller input was developed by ribbit384 ([WinNative #345](https://github.com/WinNative-Emu/WinNative/pull/345)). Why the x86-64 Steam client refused the arm64 client's sign-in was checked by reading Steam's saved token with the `ConnectCache` format from [deadlock-api-ingest](https://github.com/deadlock-api/deadlock-api-ingest). x86 programs that link OpenSSL 3 under FEX use the copy in the [Steam Linux Runtime 4.0](https://gitlab.steamos.cloud/steamrt/steamrt) platform. Hooks written with `WriteProcessMemory` into code [FEX](https://github.com/FEX-Emu/FEX) has translated go through `/proc//mem` again when the kernel refuses `process_vm_writev`, as [Wine](https://gitlab.winehq.org/wine/wine)'s `server/ptrace.c` wrote them before Wine 11. x86 Linux games under FEX get the same `/etc/asound.conf` that [pressure-vessel](https://gitlab.steamos.cloud/steamrt/steam-runtime-tools) writes into the Steam Linux Runtime, so ALSA plays through PulseAudio. Games render with [Mesa](https://gitlab.freedesktop.org/mesa/mesa) Turnip's own per-render-pass choice between tile memory and system memory, while the Steam client keeps `TU_DEBUG=sysmem`; [FEX](https://github.com/FEX-Emu/FEX)'s code disk cache was checked in its source and is already on through Proton. The proot fast path picks the guest name of a host file bound under several names in the order [PRoot](https://github.com/proot-me/proot)'s `path/binding.c` (GPL-2.0-or-later) sorts its bindings, so links and `/proc/self/fd` read back as they do through proot. The session's [PipeWire](https://gitlab.freedesktop.org/pipewire/pipewire) (MIT) opens its sockets with the per-socket access settings documented in its `module-access.c`, and [WirePlumber](https://gitlab.freedesktop.org/pipewire/wireplumber) (MIT) keeps those clients at full permissions through its `access.rules`. Wine write watches use the userfaultfd support in [Proton](https://github.com/ValveSoftware/wine)'s `dlls/ntdll/unix/virtual.c` (LGPL-2.1-or-later): libblsession gives it the arm64 syscall number and answers its `PAGEMAP_SCAN` from `/proc/self/pagemap` where Android refuses the ioctl. See [LICENSE](LICENSE). Steam and Proton belong to Valve Corporation; this project is not affiliated with Valve. +GPL-3.0. Runtime, shim, input, and controller work build on WinNative and Bannerlator (maxjivi05). Epic Games and GOG support build on [Phobos’s](https://github.com/phobos665) adaptation of [Legendary](https://github.com/legendary-gl/legendary) and [gogdl](https://github.com/Heroic-Games-Launcher/heroic-gogdl), respectively. The registry entries a GOG game's install script (`goggame-.script`) sets are read the way [Heroic Games Launcher](https://github.com/Heroic-Games-Launcher/HeroicGamesLauncher)'s GOG setup (GPL-3.0) reads them. LSFG frame generation is from the work of Camille LaVey and the [Eden](https://eden-emu.dev) emulator project, following [lsfg-vk](https://github.com/PancakeTAS/lsfg-vk), ported to WinNative and DroidDeck by [@maxjivi05](https://github.com/maxjivi05); it needs your own copy of [Lossless Scaling](https://store.steampowered.com/app/993090/) and ships none of its shaders. x86 AppImages, and any whose own runtime cannot unpack them, are unpacked with [uruntime](https://github.com/VHSgunzo/uruntime) by VHSgunzo (MIT), shipped unmodified with its licence, which also starts the arm64 emulator AppImages whose own runtime cannot run its extractor on Android. The Steam client's audio goes through [@The412Banner](https://github.com/The412Banner)'s [DirectAudio](https://github.com/The412Banner/directaudio) PulseAudio sink (LGPL-2.1-or-later), built from `tools/directaudio/sink` with its request size kept to its ring so clients are not asked to queue 100 ms ahead. mmap ring-buffer controller input was developed by ribbit384 ([WinNative #345](https://github.com/WinNative-Emu/WinNative/pull/345)). Why the x86-64 Steam client refused the arm64 client's sign-in was checked by reading Steam's saved token with the `ConnectCache` format from [deadlock-api-ingest](https://github.com/deadlock-api/deadlock-api-ingest). x86 programs that link OpenSSL 3 under FEX use the copy in the [Steam Linux Runtime 4.0](https://gitlab.steamos.cloud/steamrt/steamrt) platform. Hooks written with `WriteProcessMemory` into code [FEX](https://github.com/FEX-Emu/FEX) has translated go through `/proc//mem` again when the kernel refuses `process_vm_writev`, as [Wine](https://gitlab.winehq.org/wine/wine)'s `server/ptrace.c` wrote them before Wine 11. x86 Linux games under FEX get the same `/etc/asound.conf` that [pressure-vessel](https://gitlab.steamos.cloud/steamrt/steam-runtime-tools) writes into the Steam Linux Runtime, so ALSA plays through PulseAudio. Games render with [Mesa](https://gitlab.freedesktop.org/mesa/mesa) Turnip's own per-render-pass choice between tile memory and system memory, while the Steam client keeps `TU_DEBUG=sysmem`; [FEX](https://github.com/FEX-Emu/FEX)'s code disk cache was checked in its source and is already on through Proton. The proot fast path picks the guest name of a host file bound under several names in the order [PRoot](https://github.com/proot-me/proot)'s `path/binding.c` (GPL-2.0-or-later) sorts its bindings, so links and `/proc/self/fd` read back as they do through proot. The session's [PipeWire](https://gitlab.freedesktop.org/pipewire/pipewire) (MIT) opens its sockets with the per-socket access settings documented in its `module-access.c`, and [WirePlumber](https://gitlab.freedesktop.org/pipewire/wireplumber) (MIT) keeps those clients at full permissions through its `access.rules`. Wine write watches use the userfaultfd support in [Proton](https://github.com/ValveSoftware/wine)'s `dlls/ntdll/unix/virtual.c` (LGPL-2.1-or-later): libblsession gives it the arm64 syscall number and answers its `PAGEMAP_SCAN` from `/proc/self/pagemap` where Android refuses the ioctl. Wine on arm64 sets a thread's full context by raising `SIGUSR2` from `syscall_dispatcher_return_slowpath` in [Wine](https://gitlab.winehq.org/wine/wine)'s `dlls/ntdll/unix/signal_arm64.c` (LGPL-2.1-or-later); libblsession runs that handler in place on a frame laid out as the [Linux](https://github.com/torvalds/linux) kernel's `arch/arm64/kernel/signal.c` builds it and returns through `rt_sigreturn`, so proot does not stop the thread for it. See [LICENSE](LICENSE). Steam and Proton belong to Valve Corporation; this project is not affiliated with Valve. XR glasses support builds on WinNative's VITURE glasses work. VITURE protocol details were checked against [XRLinuxDriver](https://github.com/wheaney/XRLinuxDriver) by [@wheaney](https://github.com/wheaney), including the VITURE protocol header it ships, [ar-drivers-rs](https://github.com/badicsalex/ar-drivers-rs) by [@badicsalex](https://github.com/badicsalex) and the [go-macos/iokit VITURE package](https://pkg.go.dev/github.com/go-macos/iokit/viture); the optional VITURE SDK belongs to VITURE and is not included. RayNeo support adapts [@informalTechCode](https://github.com/informalTechCode)'s [RayNeo-Air-Series-3DoF-Scaffold](https://github.com/informalTechCode/RayNeo-Air-Series-3DoF-Scaffold) (Apache-2.0) and references [@verncat](https://github.com/verncat)'s [RayNeo-Air-3S-Pro-OpenVR](https://github.com/verncat/RayNeo-Air-3S-Pro-OpenVR) (MIT) and [Monado](https://gitlab.freedesktop.org/monado/monado) (BSL-1.0); see [docs/third-party/rayneo](docs/third-party/rayneo/README.md). XREAL 1S support references [XRLinuxDriver](https://github.com/wheaney/XRLinuxDriver) and [xrealOneDeviceKit](https://github.com/wheaney/xrealOneDeviceKit) by @wheaney, [xreal_one_driver](https://github.com/rohitsangwan01/xreal_one_driver) by [@rohitsangwan01](https://github.com/rohitsangwan01) (MIT) and [xr-tools](https://github.com/0xcaff/xr-tools) by [@0xcaff](https://github.com/0xcaff). Rokid support follows the Rokid protocol notes in [ar-drivers-rs](https://github.com/badicsalex/ar-drivers-rs) and @badicsalex's [Void Computing write-ups](https://voidcomputing.hu/blog/worse-better-prettier/), and [Monado](https://gitlab.freedesktop.org/monado/monado)'s Rokid driver. The REDMAGIC glasses-display findings draw on [@mekhontsev](https://github.com/mekhontsev)'s [MagicDesk investigation](https://github.com/mekhontsev/magicdesk/blob/main/docs/xr-resolution-fix.md) and [@Coding-BR](https://github.com/Coding-BR)'s [decompiled Nubia kernel](https://github.com/Coding-BR/android_kernel_nubia_sm8850_qwjujube). VITURE, RayNeo, XREAL, Rokid and REDMAGIC are trademarks of their owners; this project is not affiliated with them. diff --git a/tools/linuxfs/preload/usr2.c b/tools/linuxfs/preload/usr2.c new file mode 100644 index 0000000000..049518c288 --- /dev/null +++ b/tools/linuxfs/preload/usr2.c @@ -0,0 +1,218 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__aarch64__) +#include +#include + +#define BL_STR2(x) #x +#define BL_STR(x) BL_STR2(x) + +#define BL_HWCAP_GCS (1UL << 32) +#define BL_FRAME_SIZE 4688 +#define BL_REGS 312 +#define BL_SP 560 +#define BL_PC 568 +#define BL_PSTATE 576 +#define BL_FPSIMD 592 + +struct bl_sigframe { + siginfo_t info; + ucontext_t uc; +}; + +struct bl_kernel_sigaction { + void (*handler)(int, siginfo_t *, void *); + unsigned long flags; + void *restorer; + uint64_t mask; +}; + +struct bl_usr2_call { + void (*handler)(int, siginfo_t *, void *); + uint64_t old_mask; +}; + +_Static_assert(sizeof(struct bl_sigframe) == BL_FRAME_SIZE, "rt_sigframe size"); +_Static_assert(offsetof(struct bl_sigframe, uc.uc_mcontext.regs) == BL_REGS, "regs offset"); +_Static_assert(offsetof(struct bl_sigframe, uc.uc_mcontext.sp) == BL_SP, "sp offset"); +_Static_assert(offsetof(struct bl_sigframe, uc.uc_mcontext.pc) == BL_PC, "pc offset"); +_Static_assert(offsetof(struct bl_sigframe, uc.uc_mcontext.pstate) == BL_PSTATE, "pstate offset"); +_Static_assert(offsetof(struct bl_sigframe, uc.uc_mcontext.__reserved) == BL_FPSIMD, "reserved offset"); +_Static_assert(sizeof(struct fpsimd_context) + sizeof(struct _aarch64_ctx) <= + sizeof(((mcontext_t *)0)->__reserved), "reserved space"); + +__attribute__((visibility("hidden"))) void bl_usr2_frame(struct bl_usr2_call *call); +__attribute__((visibility("hidden"), used)) void bl_usr2_run(struct bl_usr2_call *call, struct bl_sigframe *frame); + +__asm__(".text\n" + ".p2align 2\n" + ".type bl_usr2_frame, %function\n" + "bl_usr2_frame:\n" + " mov x16, #" BL_STR(BL_FRAME_SIZE) "\n" + " sub sp, sp, x16\n" + " add x17, sp, #" BL_STR(BL_REGS) "\n" + " stp x0, x1, [x17, #0]\n" + " stp x2, x3, [x17, #16]\n" + " stp x4, x5, [x17, #32]\n" + " stp x6, x7, [x17, #48]\n" + " stp x8, x9, [x17, #64]\n" + " stp x10, x11, [x17, #80]\n" + " stp x12, x13, [x17, #96]\n" + " stp x14, x15, [x17, #112]\n" + " stp xzr, xzr, [x17, #128]\n" + " stp x18, x19, [x17, #144]\n" + " stp x20, x21, [x17, #160]\n" + " stp x22, x23, [x17, #176]\n" + " stp x24, x25, [x17, #192]\n" + " stp x26, x27, [x17, #208]\n" + " stp x28, x29, [x17, #224]\n" + " str x30, [x17, #240]\n" + " str xzr, [x17, #0]\n" + " add x16, sp, x16\n" + " str x16, [sp, #" BL_STR(BL_SP) "]\n" + " adr x16, 1f\n" + " str x16, [sp, #" BL_STR(BL_PC) "]\n" + " mrs x16, nzcv\n" + " str x16, [sp, #" BL_STR(BL_PSTATE) "]\n" + " add x17, sp, #" BL_STR(BL_FPSIMD) "\n" + " mrs x16, fpsr\n" + " str w16, [x17, #8]\n" + " mrs x16, fpcr\n" + " str w16, [x17, #12]\n" + " stp q0, q1, [x17, #16]\n" + " stp q2, q3, [x17, #48]\n" + " stp q4, q5, [x17, #80]\n" + " stp q6, q7, [x17, #112]\n" + " stp q8, q9, [x17, #144]\n" + " stp q10, q11, [x17, #176]\n" + " stp q12, q13, [x17, #208]\n" + " stp q14, q15, [x17, #240]\n" + " stp q16, q17, [x17, #272]\n" + " stp q18, q19, [x17, #304]\n" + " stp q20, q21, [x17, #336]\n" + " stp q22, q23, [x17, #368]\n" + " stp q24, q25, [x17, #400]\n" + " stp q26, q27, [x17, #432]\n" + " stp q28, q29, [x17, #464]\n" + " stp q30, q31, [x17, #496]\n" + " mov x1, sp\n" + " bl bl_usr2_run\n" + " mov x8, #" BL_STR(__NR_rt_sigreturn) "\n" + " svc #0\n" + " brk #0\n" + "1:\n" + " ret\n" + ".size bl_usr2_frame, .-bl_usr2_frame\n"); + +static int (*usr2_real_raise)(int); + +static long usr2_sys(long nr, long a, long b, long c, long d) { + register long x8 __asm__("x8") = nr; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + register long x3 __asm__("x3") = d; + __asm__ volatile("svc #0" : "+r"(x0) : "r"(x8), "r"(x1), "r"(x2), "r"(x3) : "memory"); + return x0; +} + +static uintptr_t usr2_lo, usr2_hi; + +static int usr2_find_ntdll(struct dl_phdr_info *info, size_t size, void *data) { + (void)size; + (void)data; + const char *slash = strrchr(info->dlpi_name, '/'); + if (strcmp(slash ? slash + 1 : info->dlpi_name, "ntdll.so") != 0) return 0; + for (int i = 0; i < info->dlpi_phnum; i++) { + const ElfW(Phdr) *ph = &info->dlpi_phdr[i]; + if (ph->p_type != PT_LOAD || !(ph->p_flags & PF_X)) continue; + uintptr_t lo = info->dlpi_addr + ph->p_vaddr; + __atomic_store_n(&usr2_hi, lo + ph->p_memsz, __ATOMIC_RELAXED); + __atomic_store_n(&usr2_lo, lo, __ATOMIC_RELEASE); + return 1; + } + return 0; +} + +static int usr2_from_ntdll(void *caller) { + uintptr_t pc = (uintptr_t)caller, lo = __atomic_load_n(&usr2_lo, __ATOMIC_ACQUIRE); + if (!lo) { + dl_iterate_phdr(usr2_find_ntdll, NULL); + lo = __atomic_load_n(&usr2_lo, __ATOMIC_ACQUIRE); + if (!lo) return 0; + } + return pc >= lo && pc < __atomic_load_n(&usr2_hi, __ATOMIC_RELAXED); +} + +static uint64_t usr2_ssbs(void) { + uint64_t value = 0; + if (getauxval(AT_HWCAP) & HWCAP_SSBS) __asm__ volatile("mrs %0, s3_3_c4_c2_6" : "=r"(value)); + return value & (1UL << 12); +} + +void bl_usr2_run(struct bl_usr2_call *call, struct bl_sigframe *frame) { + struct fpsimd_context *fp = (struct fpsimd_context *)frame->uc.uc_mcontext.__reserved; + struct _aarch64_ctx *end = (struct _aarch64_ctx *)(fp + 1); + + memset(&frame->info, 0, sizeof(frame->info)); + frame->info.si_signo = SIGUSR2; + frame->info.si_code = SI_TKILL; + frame->info.si_pid = (pid_t)usr2_sys(__NR_getpid, 0, 0, 0, 0); + frame->info.si_uid = (uid_t)usr2_sys(__NR_getuid, 0, 0, 0, 0); + frame->uc.uc_flags = 0; + frame->uc.uc_link = NULL; + if (usr2_sys(__NR_sigaltstack, 0, (long)&frame->uc.uc_stack, 0, 0) != 0) { + memset(&frame->uc.uc_stack, 0, sizeof(frame->uc.uc_stack)); + frame->uc.uc_stack.ss_flags = SS_DISABLE; + } + memset(&frame->uc.uc_sigmask, 0, sizeof(frame->uc.uc_sigmask)); + memcpy(&frame->uc.uc_sigmask, &call->old_mask, sizeof(call->old_mask)); + frame->uc.uc_mcontext.fault_address = 0; + frame->uc.uc_mcontext.pstate |= usr2_ssbs(); + fp->head.magic = FPSIMD_MAGIC; + fp->head.size = sizeof(*fp); + end->magic = 0; + end->size = 0; + call->handler(SIGUSR2, &frame->info, &frame->uc); +} + +int raise(int sig) { + int (*real)(int) = __atomic_load_n(&usr2_real_raise, __ATOMIC_ACQUIRE); + if (!real) { + real = (int (*)(int))dlsym(RTLD_NEXT, "raise"); + __atomic_store_n(&usr2_real_raise, real, __ATOMIC_RELEASE); + } + if (sig != SIGUSR2 || (getauxval(AT_HWCAP) & BL_HWCAP_GCS) || !usr2_from_ntdll(__builtin_return_address(0))) { + return real(sig); + } + + struct bl_kernel_sigaction action; + if (usr2_sys(__NR_rt_sigaction, SIGUSR2, 0, (long)&action, sizeof(action.mask)) != 0) return real(sig); + if (!(action.flags & SA_SIGINFO) || (action.flags & SA_RESETHAND) || (uintptr_t)action.handler <= 1) { + return real(sig); + } + + uint64_t bit = 1ULL << (SIGUSR2 - 1); + uint64_t block = action.mask | ((action.flags & SA_NODEFER) ? 0 : bit); + struct bl_usr2_call call = {action.handler, 0}; + if (usr2_sys(__NR_rt_sigprocmask, SIG_BLOCK, (long)&block, (long)&call.old_mask, sizeof(block)) != 0) { + return real(sig); + } + if (call.old_mask & bit) { + usr2_sys(__NR_rt_sigprocmask, SIG_SETMASK, (long)&call.old_mask, 0, sizeof(call.old_mask)); + return real(sig); + } + bl_usr2_frame(&call); + return 0; +} +#endif diff --git a/tools/tests/test_usr2_preload.py b/tools/tests/test_usr2_preload.py new file mode 100644 index 0000000000..4a180df393 --- /dev/null +++ b/tools/tests/test_usr2_preload.py @@ -0,0 +1,301 @@ +import ctypes +import os +from pathlib import Path +import platform +import shutil +import subprocess +import tempfile +import unittest + +PRELOAD = Path(__file__).resolve().parents[1] / "linuxfs/preload" + +NTDLL = r""" +#include +__attribute__((noinline)) void syscall_dispatcher_return_slowpath(void) { raise(SIGUSR2); } +void slowpath(void) { + syscall_dispatcher_return_slowpath(); + __asm__ volatile("" ::: "memory"); +} +void slowpath_other(int sig) { + raise(sig); + __asm__ volatile("" ::: "memory"); +} +""" + +PROGRAM = r""" +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void slowpath(void); +void slowpath_other(int sig); + +static char altstack[65536]; +static char landing_stack[65536] __attribute__((aligned(16))); +static volatile int mode; +static int calls; +static volatile int on_alt, usr2_blocked, usr1_blocked, code_ok, stack_ok; +static volatile long handler_sp; +static jmp_buf back; +static uint64_t saved[31]; +static unsigned __int128 saved_v[32]; +static uint32_t saved_fpsr; + +void landing(void); +__asm__(".text\n.globl landing\n.type landing, %function\nlanding:\n" + " sub sp, sp, #784\n" + " stp x0, x1, [sp, #0]\n stp x2, x3, [sp, #16]\n stp x4, x5, [sp, #32]\n stp x6, x7, [sp, #48]\n" + " stp x8, x9, [sp, #64]\n stp x10, x11, [sp, #80]\n stp x12, x13, [sp, #96]\n stp x14, x15, [sp, #112]\n" + " stp x16, x17, [sp, #128]\n stp x18, x19, [sp, #144]\n stp x20, x21, [sp, #160]\n stp x22, x23, [sp, #176]\n" + " stp x24, x25, [sp, #192]\n stp x26, x27, [sp, #208]\n stp x28, x29, [sp, #224]\n str x30, [sp, #240]\n" + " add x9, sp, #256\n" + " stp q0, q1, [x9, #0]\n stp q2, q3, [x9, #32]\n stp q4, q5, [x9, #64]\n stp q6, q7, [x9, #96]\n" + " stp q8, q9, [x9, #128]\n stp q10, q11, [x9, #160]\n stp q12, q13, [x9, #192]\n stp q14, q15, [x9, #224]\n" + " stp q16, q17, [x9, #256]\n stp q18, q19, [x9, #288]\n stp q20, q21, [x9, #320]\n stp q22, q23, [x9, #352]\n" + " stp q24, q25, [x9, #384]\n stp q26, q27, [x9, #416]\n stp q28, q29, [x9, #448]\n stp q30, q31, [x9, #480]\n" + " mrs x10, fpsr\n str w10, [sp, #768]\n" + " mov x0, sp\n bl landed\n brk #0\n"); + +void landed(uint64_t *regs) { + memcpy(saved, regs, sizeof(saved)); + memcpy(saved_v, regs + 32, sizeof(saved_v)); + saved_fpsr = *(uint32_t *)(regs + 96); + longjmp(back, 1); +} + +static struct fpsimd_context *fpsimd_of(ucontext_t *uc) { + struct _aarch64_ctx *c = (struct _aarch64_ctx *)uc->uc_mcontext.__reserved; + while (c->magic) { + if (c->magic == FPSIMD_MAGIC) return (struct fpsimd_context *)c; + c = (struct _aarch64_ctx *)((char *)c + c->size); + } + return NULL; +} + +static void handler(int sig, siginfo_t *info, void *ctx) { + ucontext_t *uc = ctx; + sigset_t now; + long sp = (long)&now; + __atomic_add_fetch(&calls, 1, __ATOMIC_RELAXED); + handler_sp = sp; + on_alt = sp >= (long)altstack && sp < (long)altstack + (long)sizeof(altstack); + pthread_sigmask(SIG_BLOCK, NULL, &now); + usr2_blocked = sigismember(&now, SIGUSR2); + usr1_blocked = sigismember(&now, SIGUSR1); + code_ok = sig == SIGUSR2 && info->si_signo == SIGUSR2 && info->si_code == SI_TKILL && info->si_pid == getpid(); + stack_ok = uc->uc_stack.ss_sp == altstack && uc->uc_stack.ss_size == sizeof(altstack) && !(uc->uc_stack.ss_flags & SS_DISABLE); + if (mode == 1) { + struct fpsimd_context *fp = fpsimd_of(uc); + for (int i = 0; i <= 28; i++) uc->uc_mcontext.regs[i] = 0x1000 + i; + uc->uc_mcontext.regs[29] = 0x2900; + uc->uc_mcontext.regs[30] = 0x3000; + uc->uc_mcontext.sp = (uint64_t)(landing_stack + sizeof(landing_stack)); + uc->uc_mcontext.pc = (uint64_t)landing; + uc->uc_mcontext.pstate = 0x60000000; + for (int i = 0; i < 32; i++) fp->vregs[i] = ((unsigned __int128)(0x7700 + i) << 64) | (0x5500 + i); + fp->fpsr = 0x8; + } +} + +static void usr1(int sig, siginfo_t *info, void *ctx) { + (void)sig; + (void)info; + (void)ctx; +} + +uint64_t kept_x[10]; +double kept_d[8]; + +static int keep_callee_saved(void) { + uint64_t *out = kept_x; + double *d = kept_d; + __asm__ volatile( + "mov x19, #19\n mov x20, #20\n mov x21, #21\n mov x22, #22\n mov x23, #23\n" + "mov x24, #24\n mov x25, #25\n mov x26, #26\n mov x27, #27\n mov x28, #28\n" + "fmov d8, #1.0\n fmov d9, #2.0\n fmov d10, #3.0\n fmov d11, #4.0\n" + "fmov d12, #5.0\n fmov d13, #6.0\n fmov d14, #7.0\n fmov d15, #8.0\n" + "bl slowpath\n" + "adrp x0, kept_x\n add x0, x0, :lo12:kept_x\n" + "stp x19, x20, [x0, #0]\n stp x21, x22, [x0, #16]\n stp x23, x24, [x0, #32]\n" + "stp x25, x26, [x0, #48]\n stp x27, x28, [x0, #64]\n" + "adrp x0, kept_d\n add x0, x0, :lo12:kept_d\n" + "stp d8, d9, [x0, #0]\n stp d10, d11, [x0, #16]\n stp d12, d13, [x0, #32]\n stp d14, d15, [x0, #48]\n" + : + : + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", + "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "x30", + "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31", "memory", "cc"); + for (int i = 0; i < 10; i++) + if (out[i] != (uint64_t)(19 + i)) return 0; + for (int i = 0; i < 8; i++) + if (d[i] != (double)(i + 1)) return 0; + return 1; +} + +static void *stress(void *arg) { + (void)arg; + for (int i = 0; i < 2000; i++) slowpath(); + return NULL; +} + +int main(int argc, char **argv) { + stack_t ss = {.ss_sp = altstack, .ss_size = sizeof(altstack), .ss_flags = 0}; + struct sigaction sa; + sigset_t mask, now; + sigaltstack(&ss, NULL); + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = handler; + sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_RESTART; + sigemptyset(&sa.sa_mask); + sigaddset(&sa.sa_mask, SIGUSR1); + sigaction(SIGUSR2, &sa, NULL); + sa.sa_sigaction = usr1; + sigemptyset(&sa.sa_mask); + sigaction(SIGUSR1, &sa, NULL); + + if (argc > 1 && strcmp(argv[1], "bench") == 0) { + int n = argc > 2 ? atoi(argv[2]) : 20000; + struct timespec a, b; + clock_gettime(CLOCK_MONOTONIC, &a); + for (int i = 0; i < n; i++) slowpath(); + clock_gettime(CLOCK_MONOTONIC, &b); + printf("%.2f us per raise, alt %d\n", ((b.tv_sec - a.tv_sec) * 1e9 + (b.tv_nsec - a.tv_nsec)) / 1e3 / n, on_alt); + return 0; + } + + mode = 0; + calls = 0; + int kept = keep_callee_saved(); + pthread_sigmask(SIG_BLOCK, NULL, &now); + printf("returned calls=%d kept=%d usr2_blocked=%d usr1_blocked=%d code=%d stack=%d after_usr2=%d after_usr1=%d\n", + calls, kept, usr2_blocked, usr1_blocked, code_ok, stack_ok, sigismember(&now, SIGUSR2), + sigismember(&now, SIGUSR1)); + printf("path %s\n", on_alt ? "kernel" : "inline"); + + mode = 1; + calls = 0; + if (!setjmp(back)) { + slowpath(); + printf("redirect did not happen\n"); + } else { + int ok = 1; + for (int i = 0; i <= 28; i++) + if (saved[i] != (uint64_t)(0x1000 + i)) ok = 0; + if (saved[29] != 0x2900 || saved[30] != 0x3000) ok = 0; + for (int i = 0; i < 32; i++) + if (saved_v[i] != (((unsigned __int128)(0x7700 + i) << 64) | (0x5500 + i))) ok = 0; + printf("redirect calls=%d regs=%d fpsr=%u\n", calls, ok, saved_fpsr & 0x8); + } + mode = 0; + + sigemptyset(&mask); + sigaddset(&mask, SIGUSR2); + pthread_sigmask(SIG_BLOCK, &mask, NULL); + calls = 0; + slowpath(); + int while_blocked = calls; + pthread_sigmask(SIG_UNBLOCK, &mask, NULL); + printf("blocked during=%d after=%d\n", while_blocked, calls); + + calls = 0; + slowpath_other(SIGUSR1); + raise(SIGUSR2); + printf("other calls=%d main_path=%s\n", calls, on_alt ? "kernel" : "inline"); + + calls = 0; + pthread_t t[4]; + for (int i = 0; i < 4; i++) pthread_create(&t[i], NULL, stress, NULL); + for (int i = 0; i < 4; i++) pthread_join(t[i], NULL); + printf("threads calls=%d\n", calls); + return 0; +} +""" + + +def has_gcs_hwcap(): + try: + getauxval = ctypes.CDLL(None).getauxval + except (AttributeError, OSError): + return False + getauxval.restype = ctypes.c_ulong + getauxval.argtypes = [ctypes.c_ulong] + return bool(getauxval(16) & (1 << 32)) + + +def runner(): + if platform.machine() in ("aarch64", "arm64") and shutil.which("gcc") and not has_gcs_hwcap(): + return "gcc", [] + if shutil.which("aarch64-linux-gnu-gcc") and shutil.which("qemu-aarch64") and Path("/usr/aarch64-linux-gnu").is_dir(): + return "aarch64-linux-gnu-gcc", ["qemu-aarch64", "-cpu", "neoverse-n1", "-L", "/usr/aarch64-linux-gnu"] + return None, None + + +class Usr2PreloadTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.cc, cls.prefix = runner() + if cls.cc is None: + raise unittest.SkipTest("no aarch64 compiler and runner") + cls.tmp = tempfile.TemporaryDirectory() + tmp = Path(cls.tmp.name) + (tmp / "ntdll.c").write_text(NTDLL) + (tmp / "prog.c").write_text(PROGRAM) + sources = sorted(str(p) for p in PRELOAD.glob("*.c")) + subprocess.run([cls.cc, "-shared", "-fPIC", "-O2", "-Wall", "-pthread", "-o", str(tmp / "libblsession.so"), + *sources, "-ldl"], check=True) + subprocess.run([cls.cc, "-shared", "-fPIC", "-O2", "-o", str(tmp / "ntdll.so"), str(tmp / "ntdll.c")], check=True) + subprocess.run([cls.cc, "-O1", "-U_FORTIFY_SOURCE", "-pthread", "-o", str(tmp / "prog"), str(tmp / "prog.c"), str(tmp / "ntdll.so"), + "-Wl,-rpath," + str(tmp)], check=True) + cls.program = tmp / "prog" + cls.preload = tmp / "libblsession.so" + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def run_program(self, preload): + env = dict(os.environ) + env.pop("LD_PRELOAD", None) + cmd = list(self.prefix) + [str(self.program)] + if preload: + if self.prefix: + cmd = list(self.prefix) + ["-E", "LD_PRELOAD=" + str(self.preload), str(self.program)] + else: + env["LD_PRELOAD"] = str(self.preload) + out = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=120) + self.assertEqual(out.returncode, 0, out.stderr) + return out.stdout.splitlines() + + def test_a_ntdll_raise_runs_the_handler_without_a_kernel_signal(self): + lines = self.run_program(True) + self.assertIn("path inline", lines) + self.assertIn("returned calls=1 kept=1 usr2_blocked=1 usr1_blocked=1 code=1 stack=1 after_usr2=0 after_usr1=0", lines) + + def test_b_context_the_handler_writes_is_the_one_that_resumes(self): + self.assertIn("redirect calls=1 regs=1 fpsr=8", self.run_program(True)) + + def test_c_behaves_like_the_kernel_signal(self): + stock = [line for line in self.run_program(False) if not line.startswith("path ")] + inline = [line for line in self.run_program(True) if not line.startswith("path ")] + self.assertIn("returned calls=1 kept=1 usr2_blocked=1 usr1_blocked=1 code=1 stack=1 after_usr2=0 after_usr1=0", stock) + self.assertEqual(stock, inline) + + def test_d_blocked_other_signals_and_other_callers_use_the_kernel(self): + lines = self.run_program(True) + self.assertIn("blocked during=0 after=1", lines) + self.assertIn("other calls=1 main_path=kernel", lines) + self.assertIn("threads calls=8000", lines) + + +if __name__ == "__main__": + unittest.main()