From a0c86abcd31c5d2f97c275b24d300dd1e69964a2 Mon Sep 17 00:00:00 2001 From: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:43:12 -0500 Subject: [PATCH] feat: emulate LSE atomics so the Steam client starts on ARMv8.0 CPUs --- .github/workflows/build.yml | 3 + .../droiddeck/launcher/core/CpuFeatures.kt | 17 + .../launcher/session/SessionFiles.kt | 9 +- .../launcher/core/CpuFeaturesTest.kt | 48 ++ docs/arm-v8.0-steam-client.md | 114 +++ tools/build_local.sh | 1 + tools/linuxfs/check-preloads.sh | 4 + tools/linuxfs/lse-emu/README.md | 15 + tools/linuxfs/lse-emu/build.sh | 14 + tools/linuxfs/lse-emu/lse_emu.c | 756 ++++++++++++++++++ .../overlay/usr/local/bin/droiddeck-session | 5 + tools/release/runtime_inputs.py | 1 + tools/tests/lse_emu_test.c | 128 +++ tools/tests/test_lse_emu.py | 69 ++ 14 files changed, 1183 insertions(+), 1 deletion(-) create mode 100644 app/src/main/java/com/droiddeck/launcher/core/CpuFeatures.kt create mode 100644 app/src/test/java/com/droiddeck/launcher/core/CpuFeaturesTest.kt create mode 100644 docs/arm-v8.0-steam-client.md create mode 100644 tools/linuxfs/lse-emu/README.md create mode 100644 tools/linuxfs/lse-emu/build.sh create mode 100644 tools/linuxfs/lse-emu/lse_emu.c create mode 100644 tools/tests/lse_emu_test.c create mode 100644 tools/tests/test_lse_emu.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 42ce804dd..a69109d29 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -295,6 +295,9 @@ jobs: aarch64-linux-gnu-gcc -O2 -Wall -o "$D/usr/local/bin/droiddeck-glprobe" tools/venus/glprobe.c -ldl aarch64-linux-gnu-strip --strip-unneeded "$D/usr/local/bin/droiddeck-glprobe" + - name: Build the LSE atomics emulator for ARMv8.0 CPUs + run: bash tools/linuxfs/lse-emu/build.sh app/src/main/assets/linuxfs + - name: Check the preloads run: | set -euo pipefail diff --git a/app/src/main/java/com/droiddeck/launcher/core/CpuFeatures.kt b/app/src/main/java/com/droiddeck/launcher/core/CpuFeatures.kt new file mode 100644 index 000000000..e8cd318bc --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/core/CpuFeatures.kt @@ -0,0 +1,17 @@ +package com.droiddeck.launcher.core + +import java.io.File + +object CpuFeatures { + internal fun lacksAtomics(cpuInfo: String): Boolean { + val cores = cpuInfo.lineSequence() + .filter { it.substringBefore(':').trim() == "Features" } + .map { it.substringAfter(':').trim().split(Regex("\\s+")) } + .toList() + return cores.isNotEmpty() && cores.any { "atomics" !in it } + } + + val lacksAtomics: Boolean by lazy { + runCatching { lacksAtomics(File("/proc/cpuinfo").readText()) }.getOrDefault(false) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt index 3cefd44c0..ec70ddf36 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt @@ -5,6 +5,7 @@ import android.content.Context import android.os.Environment import android.util.Log import com.droiddeck.launcher.R +import com.droiddeck.launcher.core.CpuFeatures import com.droiddeck.launcher.core.FileUtils import com.droiddeck.launcher.runtime.BwrapSpawner import com.droiddeck.launcher.runtime.DesktopCatalog @@ -18,6 +19,10 @@ object SessionFiles { private const val NO_PAD_SWITCH = "Download/droiddeck-no-pad" const val DIRECTAUDIO_DIR = "usr/local/lib/directaudio" private const val STAGED_LEDGER = "usr/local/.droiddeck-staged" + internal const val LSE_EMU = "usr/local/lib/droiddeck-lse/liblse_emu.so" + + internal fun lsePreload(lacksAtomics: Boolean, staged: Boolean): String = + if (lacksAtomics && staged) "/$LSE_EMU\n" else "" val DIRECTAUDIO_SETS = arrayOf("linux-wine11", "linux-wine11-systhread") internal val DESKTOP_FILES = arrayOf( @@ -195,6 +200,7 @@ object SessionFiles { "usr/local/lib/droiddeck-pipewire.tzst" to "usr/local/lib/droiddeck-pipewire.tzst", "usr/local/share/licenses/uruntime/LICENSE" to "usr/local/share/licenses/uruntime/LICENSE", "usr/local/lib/droiddeck-dirt/ws2_32.dll" to "usr/local/lib/droiddeck-dirt/ws2_32.dll", + LSE_EMU to LSE_EMU, ) + mangoapp + xclipse.map { it to it } + maliCompat + msitools.map { it to it } + vr + gpuLoad + fexPreloads).filter { (asset, _) -> val dir = asset.substringBeforeLast('/') runCatching { context.assets.list("linuxfs/$dir")?.contains(asset.substringAfterLast('/')) == true }.getOrDefault(false) @@ -252,7 +258,8 @@ object SessionFiles { } ledger.save() for (old in arrayOf("aarch64-unix", "aarch64-windows", "i386-windows")) File(root, "$DIRECTAUDIO_DIR/lib/wine/$old").deleteRecursively() - val preload = StringBuilder("/usr/local/lib/libblsession.so\n") + val preload = StringBuilder(lsePreload(CpuFeatures.lacksAtomics, File(root, LSE_EMU).isFile)) + preload.append("/usr/local/lib/libblsession.so\n") if (!File(Environment.getExternalStorageDirectory(), NO_PAD_SWITCH).exists()) { preload.append("/usr/local/lib/libfakeinput.so\n") } diff --git a/app/src/test/java/com/droiddeck/launcher/core/CpuFeaturesTest.kt b/app/src/test/java/com/droiddeck/launcher/core/CpuFeaturesTest.kt new file mode 100644 index 000000000..cbb7c374a --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/core/CpuFeaturesTest.kt @@ -0,0 +1,48 @@ +package com.droiddeck.launcher.core + +import com.droiddeck.launcher.session.SessionFiles +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class CpuFeaturesTest { + @Test fun anArmv80CpuLacksAtomics() { + val cpuInfo = """ + processor : 0 + Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid + processor : 1 + Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid + """.trimIndent() + assertTrue(CpuFeatures.lacksAtomics(cpuInfo)) + } + + @Test fun oneCoreWithoutAtomicsIsEnough() { + val cpuInfo = """ + processor : 0 + Features : fp asimd atomics + processor : 4 + Features : fp asimd + """.trimIndent() + assertTrue(CpuFeatures.lacksAtomics(cpuInfo)) + } + + @Test fun anArmv82CpuHasAtomics() { + val cpuInfo = """ + processor : 0 + Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp + """.trimIndent() + assertFalse(CpuFeatures.lacksAtomics(cpuInfo)) + } + + @Test fun noFeatureLinesMeansNoEmulation() { + assertFalse(CpuFeatures.lacksAtomics("processor : 0\n")) + assertFalse(CpuFeatures.lacksAtomics("")) + } + + @Test fun theEmulatorIsPreloadedFirstOnlyWhenNeededAndStaged() { + assertEquals("/usr/local/lib/droiddeck-lse/liblse_emu.so\n", SessionFiles.lsePreload(lacksAtomics = true, staged = true)) + assertEquals("", SessionFiles.lsePreload(lacksAtomics = true, staged = false)) + assertEquals("", SessionFiles.lsePreload(lacksAtomics = false, staged = true)) + } +} diff --git a/docs/arm-v8.0-steam-client.md b/docs/arm-v8.0-steam-client.md new file mode 100644 index 000000000..b4240f4bf --- /dev/null +++ b/docs/arm-v8.0-steam-client.md @@ -0,0 +1,114 @@ +# Steam client on ARMv8.0 CPUs + +## The crash + +Valve's native arm64 Steam client stopped starting on ARMv8.0 CPUs after the April 15 2026 +build (client version 1776387948). The process dies with SIGILL right away, the session log +shows `"$steam_root/steamrtarm64/steam" ... Illegal instruction` and `steam exited rc=132`, +and the screen stays black. + +The faulting instruction is an ARMv8.1 LSE atomic. The GDB trace in +[ValveSoftware/steam-for-linux#13288](https://github.com/ValveSoftware/steam-for-linux/issues/13288) +stops on `ldaddal x1, x0, [x0]`, with `swpa` and `casal` next to it. Earlier builds were most +likely compiled with `-moutline-atomics`, which picks LL/SC or LSE at run time. Newer builds +inline LSE. Valve has not replied on that issue, so the cause comes from the reporter's +disassembly, not from Valve. + +Affected cores are the ARMv8.0 ones without LSE: Cortex-A35, A53, A57, A72 and A73. A55, A75 +and later have it. On Android that covers the Snapdragon 4xx/6xx parts built on A53/A73 (for example the +Snapdragon 685 with Adreno 610 in +[Droid-Deck/DroidDeck#380](https://github.com/Droid-Deck/DroidDeck/issues/380)), plus the +Nintendo Switch, Raspberry Pi 4 and older, and the Jetson Nano outside Android. +`/proc/cpuinfo` shows it: the `Features` line has no `atomics`. + +LSE is an ARMv8.1 feature. LSE-Emu users report the current client running on a Cortex-A57, +so nothing later than LSE seems to be required yet. + +## What other projects do + +- [SildurFX/Switchdeck](https://github.com/SildurFX/Switchdeck) (Switch, L4T Linux) lets the + client update once, then overlays `linuxarm64`, `steamrtarm64`, `steamui` and a `steam.cfg` + from the April build. It works there because the install script controls the whole system. + For us a pinned client is replaced by the self updater on its next start, which is why #380 + was closed. +- [SildurFX/LSE-Emu](https://github.com/SildurFX/LSE-Emu) is a preload library by the same + author, written to run the current arm64 client on the Switch. A SIGILL handler performs + each LSE instruction with an LL/SC loop. With `LSE_EMU_PATCH=1` the first trap at a site + writes an LL/SC stub within branch range and replaces the instruction with a `b` to it, so + the site runs natively after that. It covers the full LSE set plus `LDAPR`. The author + reports Steam, FEX, Proton games and ARMSX2 running on a Cortex-A57, with ARMSX2 close to + native ARMv8.0 speed once patched. GPL-3.0, same as this repo. +- [Azkali/proton-arm64-a57](https://github.com/Azkali/proton-arm64-a57) rebuilds Proton 11 + arm64 (Wine, DXVK, vkd3d-proton, FEX's WoW64 and ARM64EC DLLs) with + `-march=armv8-a -mtune=cortex-a57`. Stock Proton 11 arm64 also SIGILLs on ARMv8.0 + (`wineboot` rc=132), so the client is not the only problem for games. +- Some installers fall back to Valve's x86-64 client under FEX on CPUs without LSE. We already + have that path (`BL_STEAM_CLIENT_ARCH=x86_64`, `droiddeck-steam-x64`), but it is slower and + FEX itself has to be built without LSE. +- [FrankNat996/armor](https://github.com/FrankNat996/armor) notes a separate problem: the arm64 + client's ELF segments are 4K aligned, so it SIGSEGVs at once on 16K page kernels. + +## This change + +The session vendors LSE-Emu (`tools/linuxfs/lse-emu`, upstream commit `8bc9548`). CI and +`tools/build_local.sh` build it as `usr/local/lib/droiddeck-lse/liblse_emu.so` with +`-march=armv8-a -mno-outline-atomics`. The build fails if the library contains LSE +instructions itself. + +When `/proc/cpuinfo` reports any core without `atomics`, `SessionFiles` puts the library first +in the rootfs `/etc/ld.so.preload`. `BwrapSpawner` already turns that file into `LD_PRELOAD` +for sandboxed processes. On CPUs with LSE nothing changes: the file is staged but not +preloaded, and the library would also stay idle on its own because it checks `HWCAP_ATOMICS`. + +`droiddeck-session` sees the library in `ld.so.preload` and exports `LSE_EMU=1` and +`LSE_EMU_PATCH=1`, so the Steam client and everything it starts get the emulation. With a +session debug folder it also sets `LSE_EMU_LOG=$BL_DEBUG_DIR/lse_emu.log`, and `session.log` +gets a line starting with `== cpu: no LSE atomics`. + +The app targets SDK 28, so the app domain still has `execmod` and the in-place patching can +make file-backed code pages writable. If the target SDK goes up, `mprotect` on those pages is +refused, every site logs `mprotect-failed`, and execution stays on the slower trap path. + +## Tests + +`tools/tests/test_lse_emu.py` builds the library and a program that uses every LSE family +(`ldadd` in all four sizes, `ldclr`, `ldeor`, `ldset`, `swp`, `ldsmax`, `ldumin`, `casal`, +`caspal`) from four threads, and runs it under `qemu-aarch64 -cpu cortex-a57`: + +- without the library the program dies with SIGILL +- trapping alone runs every form and keeps the threaded counters exact +- patching runs every form and patches every site, single threaded +- without `LSE_EMU=1`, and on `-cpu neoverse-n1`, the library stays out of the way +- a SIGILL handler the program installs itself still runs for an unrelated `udf` + +Threaded patching segfaults under qemu-user, writing to the stub arena, which is mapped and +writable. qemu write-protects pages it has translated code from, and that races when other +threads run code on them. It needs checking on real hardware. + +## Testing on a device + +On an ARMv8.0 phone or handheld (no `atomics` in `/proc/cpuinfo`): + +1. Install the test APK and start a Steam session. +2. `session.log` should have `== cpu: no LSE atomics` and no `steam exited rc=132`. +3. In the session debug folder, `lse_emu.log` should show `start`, `patched` and `exit` lines. + `cut -d' ' -f2 lse_emu.log | sort | uniq -c` summarises it. `unhandled` means an + instruction the library does not cover, and `mprotect-failed` or `no-arena` mean sites left + on the trap path. +4. Compare UI responsiveness against an ARMv8.2 device, and note any `lse_emu: unhandled` + lines in `session.log`. + +## Still open + +- Games. Proton 11 arm64 and Valve's FEX build need the same treatment. The preload reaches + them when they inherit `LD_PRELOAD`, but pressure-vessel and static binaries can drop it, + and FEX installs its own signal handlers with raw syscalls. An ARMv8.0 Proton build like + Azkali's is the cleaner fix. +- In #380 the session first died with SIGILL in the app's own `wl-compositor` thread, inside + `turnip-sdk36/vulkan.ad08XX.so`, which is the A8xx driver build loaded on an Adreno 610. + Picking the 6xx/7xx Turnip fixed it, and Auto picks that build once the driver release list + is loaded. Nobody has checked yet whether our bundled driver builds avoid LSE. The preload + cannot help there because the compositor is a bionic thread in the app process. +- JIT code (anonymous memory) is never patched and always traps. +- If Valve starts requiring something past LSE, the library prints + `lse_emu: unhandled pc=... insn=...` and the client crashes as before. diff --git a/tools/build_local.sh b/tools/build_local.sh index 041d6b313..153bf7c55 100755 --- a/tools/build_local.sh +++ b/tools/build_local.sh @@ -148,6 +148,7 @@ docker run --rm --platform "${docker_platform}" \ external/MaliBridge/scripts/build-client-linux.sh "$d" aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc -O2 -Wall -o "$d/usr/local/bin/droiddeck-glprobe" tools/venus/glprobe.c -ldl aarch64-linux-gnu-strip --strip-unneeded "$d/usr/local/bin/droiddeck-glprobe" + bash tools/linuxfs/lse-emu/build.sh "$d" aarch64-linux-gnu-gcc -O2 -Wall -Wextra -o "$d/usr/local/bin/droiddeck-clipboard" tools/linuxfs/clipboard/clipboard.c -ldl aarch64-linux-gnu-strip --strip-unneeded "$d/usr/local/bin/droiddeck-clipboard" for script in tools/linuxfs/overlay/usr/local/bin/droiddeck-* tools/linuxfs/overlay/usr/local/bin/steam-compatibility; do diff --git a/tools/linuxfs/check-preloads.sh b/tools/linuxfs/check-preloads.sh index bc7ca42ca..c0e7b0e81 100644 --- a/tools/linuxfs/check-preloads.sh +++ b/tools/linuxfs/check-preloads.sh @@ -31,5 +31,9 @@ for sym in vkNegotiateLoaderLayerInterfaceVersion vkGetInstanceProcAddr vkGetDev grep -qx "$sym" <<<"$XCLIPSE" || { echo "::error::the Xclipse layer does not export $sym"; exit 1; } done test -f "$D/usr/local/lib/droiddeck-xclipse/VkLayer_droiddeck_xclipse.json" +LSE=$(syms "$D/usr/local/lib/droiddeck-lse/liblse_emu.so") +for sym in sigaction signal sigprocmask pthread_sigmask prctl syscall; do + grep -qx "$sym" <<<"$LSE" || { echo "::error::liblse_emu.so does not export $sym"; exit 1; } +done test -f "$D/usr/local/bin/droiddeck-session" test -f "$D/usr/local/bin/droiddeck-proton-extra" diff --git a/tools/linuxfs/lse-emu/README.md b/tools/linuxfs/lse-emu/README.md new file mode 100644 index 000000000..e406fee81 --- /dev/null +++ b/tools/linuxfs/lse-emu/README.md @@ -0,0 +1,15 @@ +# LSE-Emu + +`lse_emu.c` is an unmodified copy of [SildurFX/LSE-Emu](https://github.com/SildurFX/LSE-Emu) +at commit `8bc9548ac6fd24a8d506b266b17ba8d0e71a9b01` (GPL-3.0-or-later). + +It is an `LD_PRELOAD` library that emulates the ARMv8.1 LSE atomic instructions (and `LDAPR`) +on ARMv8.0 CPUs. A SIGILL handler performs each trapped instruction with an LL/SC loop. With +`LSE_EMU_PATCH=1` the first trap at a site writes an LL/SC stub nearby and replaces the +instruction with a branch to it, so later runs do not trap. + +The session stages it as `/usr/local/lib/droiddeck-lse/liblse_emu.so` and only lists it in +`/etc/ld.so.preload` when `/proc/cpuinfo` has no `atomics` flag. See +`docs/arm-v8.0-steam-client.md` for why. + +To update, copy `lse_emu.c` from a newer upstream commit and change the commit above. diff --git a/tools/linuxfs/lse-emu/build.sh b/tools/linuxfs/lse-emu/build.sh new file mode 100644 index 000000000..4c4a6d5e0 --- /dev/null +++ b/tools/linuxfs/lse-emu/build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +D=${1:?asset directory} +CC=${CC:-aarch64-linux-gnu-gcc} +STRIP=${STRIP:-aarch64-linux-gnu-strip} +here=$(cd "$(dirname "$0")" && pwd) +out="$D/usr/local/lib/droiddeck-lse/liblse_emu.so" +mkdir -p "$(dirname "$out")" +"$CC" -shared -fPIC -O2 -march=armv8-a -mno-outline-atomics -Wall -Wextra -Wno-unused-parameter \ + -Wl,-z,now,-z,max-page-size=65536 -o "$out" "$here/lse_emu.c" -ldl +"$STRIP" --strip-unneeded "$out" +if "${OBJDUMP:-aarch64-linux-gnu-objdump}" -d "$out" | grep -qE $'\t(ld(add|clr|eor|set|smax|smin|umax|umin)|swp|cas)[a-z]*\t'; then + echo "::error::liblse_emu.so itself contains LSE instructions"; exit 1 +fi diff --git a/tools/linuxfs/lse-emu/lse_emu.c b/tools/linuxfs/lse-emu/lse_emu.c new file mode 100644 index 000000000..630f4e887 --- /dev/null +++ b/tools/linuxfs/lse-emu/lse_emu.c @@ -0,0 +1,756 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// https://github.com/SildurFX/LSE-Emu +// Copyright (C) 2026 SildurFX + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef HWCAP_ATOMICS +#define HWCAP_ATOMICS (1UL << 8) +#endif +#ifndef SA_RESTORER +#define SA_RESTORER 0x04000000 +#endif +#ifndef MAP_FIXED_NOREPLACE +#define MAP_FIXED_NOREPLACE 0x100000 +#endif + +/* ======================= raw syscalls (no PLT in handler) ================= */ + +static long sc(long n, long a, long b, long c, long d, long e, long f) { + register long x8 asm("x8") = n, x0 asm("x0") = a, x1 asm("x1") = b; + register long x2 asm("x2") = c, x3 asm("x3") = d, x4 asm("x4") = e, x5 asm("x5") = f; + asm volatile("svc 0" : "+r"(x0) + : "r"(x8), "r"(x1), "r"(x2), "r"(x3), "r"(x4), "r"(x5) + : "memory", "cc"); + return x0; +} + +/* ======================= slow path: emulation ============================= */ + +#define DEFINE_RMW(NAME, UT, ST) \ +static UT NAME(void *p, unsigned op, UT v) { \ + UT *q = (UT *)p; \ + switch (op) { \ + case 0: return __atomic_fetch_add(q, v, __ATOMIC_SEQ_CST); \ + case 1: return __atomic_fetch_and(q, (UT)~v, __ATOMIC_SEQ_CST); \ + case 2: return __atomic_fetch_xor(q, v, __ATOMIC_SEQ_CST); \ + case 3: return __atomic_fetch_or (q, v, __ATOMIC_SEQ_CST); \ + case 8: return __atomic_exchange_n(q, v, __ATOMIC_SEQ_CST); \ + } \ + UT old = __atomic_load_n(q, __ATOMIC_RELAXED), nw; \ + do { \ + switch (op) { \ + case 4: nw = ((ST)old > (ST)v) ? old : v; break; \ + case 5: nw = ((ST)old < (ST)v) ? old : v; break; \ + case 6: nw = (old > v) ? old : v; break; \ + default: nw = (old < v) ? old : v; break; \ + } \ + } while (!__atomic_compare_exchange_n(q, &old, nw, 1, \ + __ATOMIC_SEQ_CST, __ATOMIC_RELAXED));\ + return old; \ +} +DEFINE_RMW(rmw8, uint8_t, int8_t) +DEFINE_RMW(rmw16, uint16_t, int16_t) +DEFINE_RMW(rmw32, uint32_t, int32_t) +DEFINE_RMW(rmw64, uint64_t, int64_t) + +#define DEFINE_CAS(NAME, UT) \ +static UT NAME(void *p, UT exp, UT des) { \ + UT e = exp; \ + __atomic_compare_exchange_n((UT *)p, &e, des, 0, \ + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \ + return e; \ +} +DEFINE_CAS(cas8, uint8_t) +DEFINE_CAS(cas16, uint16_t) +DEFINE_CAS(cas32, uint32_t) +DEFINE_CAS(cas64, uint64_t) + +static void casp128(void *p, uint64_t *o0, uint64_t *o1, + uint64_t e0, uint64_t e1, uint64_t d0, uint64_t d1) { + uint64_t a, b, t, s; + asm volatile( + "1: ldaxp %0, %1, [%6]\n" + " eor %2, %0, %7\n" + " eor %3, %1, %8\n" + " orr %2, %2, %3\n" + " cbnz %2, 2f\n" + " stlxp %w2, %9, %10, [%6]\n" + " cbnz %w2, 1b\n" + "2: clrex\n" + : "=&r"(a), "=&r"(b), "=&r"(t), "=&r"(s) + : "0"(0), "1"(0), "r"(p), "r"(e0), "r"(e1), "r"(d0), "r"(d1) + : "memory", "cc"); + *o0 = a; *o1 = b; +} + +static int emulate(ucontext_t *uc, uint32_t i) { + uint64_t *x = (uint64_t *)uc->uc_mcontext.regs; + unsigned rt = i & 31, rn = (i >> 5) & 31, rs = (i >> 16) & 31; + unsigned size = i >> 30; + void *p = (void *)(rn == 31 ? uc->uc_mcontext.sp : x[rn]); +#define RD(r) ((r) >= 31 ? 0 : x[r]) +#define WR(r, v) do { if ((r) < 31) x[r] = (v); } while (0) + + if ((i & 0x3f200c00) == 0x38200000) { // LD, SWP, LDAPR + unsigned o3 = (i >> 15) & 1, opc = (i >> 12) & 7, op; + uint64_t v, old; + if (!o3) op = opc; + else if (opc == 0) op = 8; + else if (opc == 4 && rs == 31 && + (i & (1u << 23)) && !(i & (1u << 22))) { + switch (size) { + case 0: v = __atomic_load_n((uint8_t *)p, __ATOMIC_ACQUIRE); break; + case 1: v = __atomic_load_n((uint16_t *)p, __ATOMIC_ACQUIRE); break; + case 2: v = __atomic_load_n((uint32_t *)p, __ATOMIC_ACQUIRE); break; + default: v = __atomic_load_n((uint64_t *)p, __ATOMIC_ACQUIRE); break; + } + WR(rt, v); + return 1; + } else return 0; + v = RD(rs); + switch (size) { + case 0: old = rmw8 (p, op, (uint8_t)v); break; + case 1: old = rmw16(p, op, (uint16_t)v); break; + case 2: old = rmw32(p, op, (uint32_t)v); break; + default: old = rmw64(p, op, v); break; + } + WR(rt, old); + return 1; + } + if ((i & 0x3fa07c00) == 0x08a07c00) { // CAS + uint64_t e = RD(rs), d = RD(rt), old; + switch (size) { + case 0: old = cas8 (p, (uint8_t)e, (uint8_t)d); break; + case 1: old = cas16(p, (uint16_t)e, (uint16_t)d); break; + case 2: old = cas32(p, (uint32_t)e, (uint32_t)d); break; + default: old = cas64(p, e, d); break; + } + WR(rs, old); + return 1; + } + if ((i & 0xbfa07c00) == 0x08207c00) { // CASP + if ((rs & 1) || (rt & 1)) return 0; + uint64_t e0 = RD(rs), e1 = RD(rs + 1), d0 = RD(rt), d1 = RD(rt + 1); + if (size == 0) { + uint64_t e = (uint32_t)e0 | ((uint64_t)(uint32_t)e1 << 32); + uint64_t d = (uint32_t)d0 | ((uint64_t)(uint32_t)d1 << 32); + uint64_t o = cas64(p, e, d); + WR(rs, (uint32_t)o); WR(rs + 1, o >> 32); + } else { + uint64_t o0, o1; + casp128(p, &o0, &o1, e0, e1, d0, d1); + WR(rs, o0); WR(rs + 1, o1); + } + return 1; + } + return 0; +} + +/* ======================= fast path: stub generator ======================== */ + +#define STP2(a,b) (0xA9BE0000u | ((b) << 10) | (31u << 5) | (a)) // stp a,b,[sp,#-32]! +#define LDP2(a,b) (0xA8C20000u | ((b) << 10) | (31u << 5) | (a)) // ldp a,b,[sp],#32 +#define STR16(c) (0xF9000800u | (31u << 5) | (c)) // str c,[sp,#16] +#define LDR16(c) (0xF9400800u | (31u << 5) | (c)) // ldr c,[sp,#16] +#define MOVX(d,m) (0xAA0003E0u | ((m) << 16) | (d)) // mov xd,xm +#define W(x) do { uint32_t _v = (x); w[n++] = _v; } while (0) + +static uint32_t ldx(unsigned sz, unsigned acq, unsigned rt, unsigned rn) { + return (sz << 30) | 0x085F7C00u | (acq << 15) | (rn << 5) | rt; +} +static uint32_t stx(unsigned sz, unsigned rel, unsigned rs, unsigned rt, unsigned rn) { + return (sz << 30) | 0x08007C00u | (rs << 16) | (rel << 15) | (rn << 5) | rt; +} +static uint32_t alu(unsigned op, unsigned d, unsigned n, unsigned m) { + static const uint32_t base[4] = { 0x8B000000u, 0x8A200000u, 0xCA000000u, 0xAA000000u }; + return base[op] | (m << 16) | (n << 5) | d; // add, bic, eor, orr +} + +// CASP{A}{L} Xs,Xs+1,Xt,Xt+1,[Xn] (64-bit pair): ldxp/ldaxp + compare + stxp/stlxp. Four scratch registers, no flag changes, 15 words. +static int gen_casp(uint32_t *w, uintptr_t stub, uintptr_t ret, unsigned sz, + unsigned rs, unsigned rt, unsigned rn, unsigned acq, unsigned rel) { + // sz 1: 64-bit pair (LDXP x), sz 0: 32-bit pair (LDXP w, 8-byte aligned) + const uint32_t LDXP = sz ? 0xC87F0000u : 0x887F0000u, STXP = sz ? 0xC8200000u : 0x88200000u; + const uint32_t EOR = sz ? 0xCA000000u : 0x4A000000u, ORR = sz ? 0xAA000000u : 0x2A000000u; + unsigned t[4], k = 0; int n = 0; + for (unsigned r = 9; r <= 17 && k < 4; r++) + if (r != rs && r != rs + 1 && r != rt && r != rt + 1 && r != rn) t[k++] = r; + if (k < 4) return 0; + unsigned t0 = t[0], t1 = t[1], t2 = t[2], t3 = t[3]; + W(STP2(t0, t1)); + W(0xA9010000u | (t3 << 10) | (31u << 5) | t2); // stp t2,t3,[sp,#16] + int loop = n; + W(LDXP | (acq << 15) | (t1 << 10) | (rn << 5) | t0); // ldxp/ldaxp t0,t1,[rn] + W(EOR | (rs << 16) | (t0 << 5) | t2); // eor t2,t0,rs + W(EOR | ((rs + 1) << 16) | (t1 << 5) | t3); // eor t3,t1,rs+1 + W(ORR | (t3 << 16) | (t2 << 5) | t2); // orr t2,t2,t3 + W(0xB5000000u | (3u << 5) | t2); // cbnz t2, clrex (+3) + W(STXP | (t2 << 16) | (rel << 15) | ((rt + 1) << 10) | (rn << 5) | rt); // stxp/stlxp w(t2),rt,rt+1,[rn] + W(0x35000000u | (((unsigned)(loop - n) & 0x7FFFF) << 5) | t2); // cbnz w(t2), loop + W(0xD5033F5Fu); // clrex + W(MOVX(rs, t0)); + W(MOVX(rs + 1, t1)); + W(0xA9410000u | (t3 << 10) | (31u << 5) | t2); // ldp t2,t3,[sp,#16] + W(LDP2(t0, t1)); + { int64_t off = (int64_t)ret - (int64_t)(stub + 4 * (uintptr_t)n); + W(0x14000000u | (((uint32_t)(off >> 2)) & 0x03FFFFFFu)); } + return n; +} + +// Builds the stub for one instruction. Returns word count, 0 if unsupported. +static int gen(uint32_t *w, uintptr_t stub, uintptr_t ret, uint32_t i) { + unsigned size = i >> 30, rt = i & 31, rn = (i >> 5) & 31, rs = (i >> 16) & 31; + unsigned kind, op = 0, acq, rel; // kind: 0 ldop, 1 swp, 2 cas, 3 casp, 4 min/max + int n = 0; + + if ((i & 0x3FFFFC00u) == 0x38BFC000u) { // LDAPR[B|H]: stub = LDAR[B|H] (same operands, stronger ordering) ; b ret + W((i & 0xC0000000u) | 0x08DFFC00u | (i & 0x3FFu)); + { int64_t off = (int64_t)ret - (int64_t)(stub + 4 * (uintptr_t)n); + W(0x14000000u | (((uint32_t)(off >> 2)) & 0x03FFFFFFu)); } + return n; + } + if ((i & 0x3f200c00) == 0x38200000) { + unsigned o3 = (i >> 15) & 1, opc = (i >> 12) & 7; + if (!o3 && opc <= 3) { kind = 0; op = opc; } + else if (!o3) { kind = 4; op = opc; } // LDSMAX LDSMIN LDUMAX LDUMIN (opc 4..7) + else if (o3 && opc == 0) kind = 1; + else return 0; + acq = (i >> 23) & 1; rel = (i >> 22) & 1; + } else if ((i & 0x3fa07c00) == 0x08a07c00) { + kind = 2; acq = (i >> 22) & 1; rel = (i >> 15) & 1; + } else if ((i & 0xbfa07c00) == 0x08207c00) { // CASP, 32-bit (size 0) and 64-bit (size 1) pairs + if (size > 1 || (rs & 1) || (rt & 1) || rs >= 30 || rt >= 30) return 0; + kind = 3; acq = (i >> 22) & 1; rel = (i >> 15) & 1; + } else return 0; + if (rn == 31) return 0; // sp-based: leave to trap path + if (kind == 3) return gen_casp(w, stub, ret, size, rs, rt, rn, acq, rel); + + unsigned t[3], k = 0; // three scratch regs from x9..x17 + for (unsigned r = 9; r <= 17 && k < 3; r++) + if (r != rs && r != rt && r != rn) t[k++] = r; + unsigned t0 = t[0], t1 = t[1], t2 = t[2]; + + W(STP2(t0, t1)); + W(STR16(t2)); + int loop = n; + W(ldx(size, acq, t0, rn)); // t0 = old + if (kind == 1) { + W(stx(size, rel, t2, rs, rn)); + W(0x35000000u | (((unsigned)(loop - n) & 0x7FFFF) << 5) | t2); + } else if (kind == 4) { // LDSMAX/LDSMIN/LDUMAX/LDUMIN + static const unsigned cc[4] = { 0xC /*gt*/, 0xB /*lt*/, 0x8 /*hi*/, 0x3 /*lo*/ }; + unsigned sgn = !(op & 2), sf = (size == 3), cmpn = t0; + if (size < 2 && sgn) { // sign-extend the loaded value: sxtb/sxth wt1,wt0 + W((size == 1 ? 0x13003C00u : 0x13001C00u) | (t0 << 5) | t1); + cmpn = t1; + } + if (size < 2) // cmp wcmpn, wrs, {u,s}xt{b,h} + W(0x6B20001Fu | (rs << 16) | ((sgn ? (size == 1 ? 5u : 4u) : (size == 1 ? 1u : 0u)) << 13) | (cmpn << 5)); + else // cmp t0, rs + W((sf ? 0xEB00001Fu : 0x6B00001Fu) | (rs << 16) | (t0 << 5)); + W((sf ? 0x9A800000u : 0x1A800000u) | (rs << 16) | (cc[op - 4] << 12) | (t0 << 5) | t1); // csel t1,t0,rs,cond + W(stx(size, rel, t2, t1, rn)); + W(0x35000000u | (((unsigned)(loop - n) & 0x7FFFF) << 5) | t2); + } else if (kind == 0) { + W(alu(op, t1, t0, rs)); // t1 = new + W(stx(size, rel, t2, t1, rn)); + W(0x35000000u | (((unsigned)(loop - n) & 0x7FFFF) << 5) | t2); + } else { + if (size == 3) W(0xCA000000u | (rs << 16) | (t0 << 5) | t1); + else if (size == 2) W(0x4A000000u | (rs << 16) | (t0 << 5) | t1); // 32-bit eor + else { + W((size == 1 ? 0x53003C00u : 0x53001C00u) | (rs << 5) | t1); // uxth/uxtb + W(0xCA000000u | (t1 << 16) | (t0 << 5) | t1); + } + W(0xB5000000u | (3u << 5) | t1); // cbnz t1, fail (+3) + W(stx(size, rel, t2, rt, rn)); + W(0x35000000u | (((unsigned)(loop - n) & 0x7FFFF) << 5) | t2); + W(0xD5033F5Fu); // fail: clrex + } + if (kind == 2) { if (rs != 31) W(MOVX(rs, t0)); } + else if (rt != 31) W(MOVX(rt, t0)); + W(LDR16(t2)); + W(LDP2(t0, t1)); + { int64_t off = (int64_t)ret - (int64_t)(stub + 4 * (uintptr_t)n); + W(0x14000000u | (((uint32_t)(off >> 2)) & 0x03FFFFFFu)); } + return n; +} + +/* ---- stub arenas (must sit within +-128 MiB of the patched code) ---- */ + +#define ARENA_SZ 0x10000 +#define SLOT 64 +#define MAX_ARENAS 256 +#define REACH 0x7000000 // 112 MiB, safely inside the 128 MiB branch range +static struct { uintptr_t base, used; } arena[MAX_ARENAS]; +static volatile int narena; + +static int near_(uintptr_t b, uintptr_t pc) { + intptr_t d = (intptr_t)b - (intptr_t)pc; + return d > -(intptr_t)REACH && d < (intptr_t)REACH; +} +static int in_arena(uintptr_t a) { + int n = __atomic_load_n(&narena, __ATOMIC_ACQUIRE); // pairs with the release store in alloc_slot + for (int k = 0; k < n; k++) + if (a >= arena[k].base && a < arena[k].base + ARENA_SZ) return 1; + return 0; +} +static unsigned long hexv(const char **p); + +//Find a free 64 KiB-aligned hole within REACH of pc by walking /proc/self/maps called with the patch lock held, so the static buffers are safe +static uintptr_t find_gap(uintptr_t pc, const uintptr_t *tr, int ntr) { + static char buf[4096], line[512]; + long fd = sc(SYS_openat, -100, (long)"/proc/self/maps", O_RDONLY | O_CLOEXEC, 0, 0, 0); + if (fd < 0) return 0; + uintptr_t lo = pc > (uintptr_t)REACH + 0x100000 ? pc - REACH : 0x100000; + uintptr_t hi = pc + REACH; + uintptr_t prev = 0, best = 0, bestd = ~(uintptr_t)0; + int ll = 0; long n; + while ((n = sc(SYS_read, fd, (long)buf, sizeof buf, 0, 0, 0)) > 0) + for (long k = 0; k < n; k++) { + char c = buf[k]; + if (c != '\n') { if (ll < 511) line[ll++] = c; continue; } + line[ll] = 0; ll = 0; + const char *p = line; + uintptr_t s0 = hexv(&p); if (*p != '-') continue; p++; + uintptr_t e0 = hexv(&p); + uintptr_t g0 = prev > lo ? prev : lo; + g0 = (g0 + 0xFFFF) & ~(uintptr_t)0xFFFF; + uintptr_t g1 = (s0 < hi ? s0 : hi) & ~(uintptr_t)0xFFFF; + if (g1 > g0 && g1 - g0 >= ARENA_SZ) { + uintptr_t rlo = g0, rhi = g1 - ARENA_SZ; // valid block start range + uintptr_t cand = pc & ~(uintptr_t)0xFFFF; + if (cand < rlo) cand = rlo; + if (cand > rhi) cand = rhi; + // skip blocks a previous mmap refused, walking outwards inside this hole + for (int st = 0; st < 16; st++) { + uintptr_t pick = 0; int found = 0; + for (int w = 0; w < 2 && !found; w++) { + if (w == 1 && st == 0) continue; + uintptr_t c; + if (w == 0) { if (cand + (uintptr_t)st * ARENA_SZ > rhi) continue; c = cand + (uintptr_t)st * ARENA_SZ; } + else { if (cand < rlo + (uintptr_t)st * ARENA_SZ) continue; c = cand - (uintptr_t)st * ARENA_SZ; } + int is_bad = 0; + for (int t = 0; t < ntr; t++) if (tr[t] == c) is_bad = 1; + if (!is_bad) { pick = c; found = 1; } + } + if (found) { + uintptr_t d = pick > pc ? pick - pc : pc - pick; + if (d < bestd) { bestd = d; best = pick; } + break; + } + } + } + if (e0 > prev) prev = e0; + } + sc(SYS_close, fd, 0, 0, 0, 0, 0); + return best; +} + +static uintptr_t alloc_slot(uintptr_t pc) { + for (int k = 0; k < narena; k++) + if (arena[k].used + SLOT <= ARENA_SZ && near_(arena[k].base, pc)) { + uintptr_t s = arena[k].base + arena[k].used; + arena[k].used += SLOT; + return s; + } + if (narena >= MAX_ARENAS) return 0; + // free hole nearest to the site, found by walking /proc/self/maps + uintptr_t tried[16]; int ntried = 0; // blocks mmap refused, don't offer them again + for (int attempt = 0; attempt < 12; attempt++) { + uintptr_t hint = find_gap(pc, tried, ntried); + if (!hint) return 0; + long r = sc(SYS_mmap, hint, ARENA_SZ, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if ((unsigned long)r > (unsigned long)-4096) { tried[ntried++] = hint; continue; } + if ((uintptr_t)r != hint) { //kernel without MAP_FIXED_NOREPLACE moved it + sc(SYS_munmap, r, ARENA_SZ, 0, 0, 0, 0); + tried[ntried++] = hint; continue; + } + arena[narena].base = (uintptr_t)r; + arena[narena].used = SLOT; + int idx = narena; + __atomic_store_n(&narena, idx + 1, __ATOMIC_RELEASE); // publish the slot only after base/used are written + return arena[idx].base; + } + return 0; +} + +/* ---- known executable file-backed ranges (parsed from /proc/self/maps) ---- */ + +#define MAX_RANGES 512 +static struct { uintptr_t lo, hi; } rng[MAX_RANGES]; +static volatile int nrng; +static int misses; + +static int in_ranges(uintptr_t pc) { + for (int k = 0; k < nrng; k++) if (pc >= rng[k].lo && pc < rng[k].hi) return 1; + return 0; +} +static unsigned long hexv(const char **p) { + unsigned long v = 0; + for (;; (*p)++) { + char c = **p; int d; + if (c >= '0' && c <= '9') d = c - '0'; + else if (c >= 'a' && c <= 'f') d = c - 'a' + 10; + else break; + v = (v << 4) | d; + } + return v; +} +static void parse_line(const char *p) { + unsigned long lo = hexv(&p); if (*p != '-') return; p++; + unsigned long hi = hexv(&p); if (*p != ' ') return; p++; + if (!(p[0] == 'r' && p[1] == '-' && p[2] == 'x' && p[3] == 'p')) return; + for (int f = 0; f < 4 && *p; f++) { while (*p && *p != ' ') p++; while (*p == ' ') p++; } + if (*p != '/') return; + if (nrng < MAX_RANGES) { rng[nrng].lo = lo; rng[nrng].hi = hi; nrng = nrng + 1; } +} +static void refresh_ranges(void) { + static char buf[4096], line[512]; + long fd = sc(SYS_openat, -100, (long)"/proc/self/maps", O_RDONLY | O_CLOEXEC, 0, 0, 0); + if (fd < 0) return; + nrng = 0; + int ll = 0; long n; + while ((n = sc(SYS_read, fd, (long)buf, sizeof buf, 0, 0, 0)) > 0) + for (long k = 0; k < n; k++) { + char c = buf[k]; + if (c == '\n') { line[ll] = 0; parse_line(line); ll = 0; } + else if (ll < 511) line[ll++] = c; + } + sc(SYS_close, fd, 0, 0, 0, 0, 0); +} + +/* ---- debug log (LSE_EMU_LOG=1) ---- */ + +static int add_str(char *b, int n, const char *s); +static int add_dec(char *b, int n, unsigned long v); + +static volatile int logon; +static char logpath[200] = "/tmp/lse_emu.log"; +static unsigned long st_traps, st_patched, st_failed; + +static void logwrite(const char *b, int n) { // raw syscalls, O_APPEND: safe from the handler + long fd = sc(SYS_openat, -100, (long)logpath, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC, 0666, 0, 0); + if (fd < 0) return; + sc(SYS_write, fd, (long)b, n, 0, 0, 0); + sc(SYS_close, fd, 0, 0, 0, 0, 0); +} +static int add_hex(char *b, int n, uint64_t v, int digits) { + for (int k = (digits - 1) * 4; k >= 0; k -= 4) b[n++] = "0123456789abcdef"[(v >> k) & 15]; + return n; +} +// One line per site, patched is logged as it happens, failures are de-duplicated. +static void logsite(const char *why, uintptr_t pc, uint32_t insn) { + if (!logon) return; + static uintptr_t seen[256]; static volatile int ns; + if (why[0] != 'p') { // only "patched" is logged every time + for (int k = 0; k < ns; k++) if (seen[k] == pc) return; + if (ns >= 256) return; + seen[ns] = pc; ns = ns + 1; + } + char b[160]; int n = 0; + n = add_str(b, n, "pid="); n = add_dec(b, n, (unsigned long)sc(SYS_getpid, 0, 0, 0, 0, 0, 0)); + n = add_str(b, n, " "); n = add_str(b, n, why); + n = add_str(b, n, " pc=0x"); n = add_hex(b, n, pc, 16); + n = add_str(b, n, " insn=0x"); n = add_hex(b, n, insn, 8); + b[n++] = '\n'; + logwrite(b, n); +} +static void logstats(const char *tag) { + if (!logon) return; + char b[160]; int n = 0; + n = add_str(b, n, "pid="); n = add_dec(b, n, (unsigned long)sc(SYS_getpid, 0, 0, 0, 0, 0, 0)); + n = add_str(b, n, " "); n = add_str(b, n, tag); + n = add_str(b, n, " traps="); n = add_dec(b, n, st_traps); + n = add_str(b, n, " patched="); n = add_dec(b, n, st_patched); + n = add_str(b, n, " failed="); n = add_dec(b, n, st_failed); + n = add_str(b, n, " arenas="); n = add_dec(b, n, (unsigned long)narena); + b[n++] = '\n'; + logwrite(b, n); +} + +/* ---- patching ---- */ + +static volatile int plock, patch_enabled, stats; +static uintptr_t bad[256]; + +// Try-lock, a handler that interrupts try_patch on the same thread and traps again must never spin on a lock it already owns. A skipped site is simply retried on its next trap. +static int trylock(void) { return !__atomic_exchange_n(&plock, 1, __ATOMIC_ACQUIRE); } +static void unlock(void) { __atomic_store_n(&plock, 0, __ATOMIC_RELEASE); } + +// Overwrite one instruction word with a branch (single aligned release store), then flush the icache. +static int patch_word(uintptr_t pc, uint32_t v) { + uintptr_t page = pc & ~(uintptr_t)4095; + if (sc(SYS_mprotect, page, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, 0, 0, 0)) return -1; + __atomic_store_n((uint32_t *)pc, v, __ATOMIC_RELEASE); + __builtin___clear_cache((char *)pc, (char *)pc + 4); + sc(SYS_mprotect, page, 4096, PROT_READ | PROT_EXEC, 0, 0, 0); + return 0; +} + +static void try_patch(uintptr_t pc, uint32_t insn) { + uint32_t w[16]; + if (bad[(pc >> 2) & 255] == pc) return; + if (!gen(w, 0, 0, insn)) { logsite("unsupported-form", pc, insn); return; } // not a patchable form + + if (!trylock()) return; + if (*(volatile uint32_t *)pc != insn) goto out; // already patched by another thread + if (!in_ranges(pc)) { + if (misses++ < 16) refresh_ranges(); + if (!in_ranges(pc)) { logsite("not-file-backed", pc, insn); bad[(pc >> 2) & 255] = pc; goto out; } + } + uintptr_t slot = alloc_slot(pc); + if (!slot) { logsite("no-arena", pc, insn); bad[(pc >> 2) & 255] = pc; st_failed++; goto out; } + int n = gen(w, slot, pc + 4, insn); + + for (int k = 0; k < n; k++) ((volatile uint32_t *)slot)[k] = w[k]; + __builtin___clear_cache((char *)slot, (char *)slot + 4 * n); + + long off = (long)slot - (long)pc; + uint32_t b = 0x14000000u | (((uint32_t)(off >> 2)) & 0x03FFFFFFu); + if (patch_word(pc, b)) { + logsite("mprotect-failed", pc, insn); + bad[(pc >> 2) & 255] = pc; st_failed++; goto out; + } + st_patched++; + logsite("patched", pc, insn); +out: + unlock(); +} + +/* ======================= libc hooks (unchanged logic) ===================== */ + +static int (*real_sigaction)(int, const struct sigaction *, struct sigaction *); +static int (*real_sigprocmask)(int, const sigset_t *, sigset_t *); +static int (*real_pthread_sigmask)(int, const sigset_t *, sigset_t *); +static long (*real_syscall)(long, ...); + +static void resolve(void) { + if (!real_sigaction) real_sigaction = dlsym(RTLD_NEXT, "sigaction"); + if (!real_sigprocmask) real_sigprocmask = dlsym(RTLD_NEXT, "sigprocmask"); + if (!real_pthread_sigmask) real_pthread_sigmask = dlsym(RTLD_NEXT, "pthread_sigmask"); + if (!real_syscall) real_syscall = dlsym(RTLD_NEXT, "syscall"); +} + +static volatile int active, app_valid; +static void * volatile app_h; +static volatile unsigned long app_f; +struct ksa { void *h; unsigned long f; void *restorer; uint64_t mask; }; + +int sigaction(int sig, const struct sigaction *act, struct sigaction *old) { + resolve(); + if (!active || sig != SIGILL) return real_sigaction(sig, act, old); + if (old) { + memset(old, 0, sizeof *old); + if (app_valid) { old->sa_sigaction = app_h; old->sa_flags = (int)(app_f & ~(unsigned long)SA_RESTORER); } + else old->sa_handler = SIG_DFL; + } + if (act) { app_h = (void *)act->sa_sigaction; app_f = (unsigned long)act->sa_flags; app_valid = 1; } + return 0; +} + +sighandler_t signal(int sig, sighandler_t h) { + struct sigaction sa, o; + memset(&sa, 0, sizeof sa); + sa.sa_handler = h; sa.sa_flags = SA_RESTART; + sigemptyset(&sa.sa_mask); + if (sigaction(sig, &sa, &o) < 0) return SIG_ERR; + return o.sa_handler; +} + +int sigprocmask(int how, const sigset_t *set, sigset_t *old) { + resolve(); + if (active && set && how != SIG_UNBLOCK) { + sigset_t t = *set; sigdelset(&t, SIGILL); + return real_sigprocmask(how, &t, old); + } + return real_sigprocmask(how, set, old); +} +int pthread_sigmask(int how, const sigset_t *set, sigset_t *old) { + resolve(); + if (active && set && how != SIG_UNBLOCK) { + sigset_t t = *set; sigdelset(&t, SIGILL); + return real_pthread_sigmask(how, &t, old); + } + return real_pthread_sigmask(how, set, old); +} + +// a process that turns on seccomp must not be patched afterwards +int prctl(int opt, ...) { + va_list ap; va_start(ap, opt); + unsigned long a1 = va_arg(ap, unsigned long), a2 = va_arg(ap, unsigned long); + unsigned long a3 = va_arg(ap, unsigned long), a4 = va_arg(ap, unsigned long); + va_end(ap); + resolve(); + if (opt == PR_SET_SECCOMP) patch_enabled = 0; + return (int)real_syscall(SYS_prctl, (long)opt, a1, a2, a3, a4); +} + +long syscall(long n, ...) { + unsigned long a[6]; + va_list ap; va_start(ap, n); + for (int k = 0; k < 6; k++) a[k] = va_arg(ap, unsigned long); + va_end(ap); + resolve(); + if (active) { +#ifdef SYS_seccomp + if (n == SYS_seccomp) patch_enabled = 0; +#endif + if (n == SYS_prctl && a[0] == PR_SET_SECCOMP) patch_enabled = 0; + if (n == SYS_rt_sigaction && a[0] == SIGILL) { + struct ksa *act = (struct ksa *)a[1], *old = (struct ksa *)a[2]; + if (old) { + memset(old, 0, sizeof *old); + if (app_valid) { old->h = app_h; old->f = app_f; } + } + if (act) { app_h = act->h; app_f = act->f; app_valid = 1; } + return 0; + } + if (n == SYS_rt_sigprocmask && a[1] && a[3] == 8 && a[0] != SIG_UNBLOCK) { + uint64_t m = *(uint64_t *)a[1]; + m &= ~(1ULL << (SIGILL - 1)); + return real_syscall(n, a[0], &m, a[2], a[3], a[4], a[5]); + } + } + return real_syscall(n, a[0], a[1], a[2], a[3], a[4], a[5]); +} + +/* ======================= handler ========================================== */ + +static void put_hex(uint64_t v) { + char b[18]; b[0] = '0'; b[1] = 'x'; + for (int k = 0; k < 16; k++) b[2 + k] = "0123456789abcdef"[(v >> (60 - 4 * k)) & 15]; + (void)!write(2, b, 18); +} + +static void on_sigill(int sig, siginfo_t *si, void *ctx) { + ucontext_t *uc = (ucontext_t *)ctx; + uintptr_t pc = uc->uc_mcontext.pc; + uint32_t insn = __atomic_load_n((uint32_t *)pc, __ATOMIC_ACQUIRE); // pairs with the release store in patch_word + + if ((insn & 0xFC000000u) == 0x14000000u) { // another thread patched it first + intptr_t off = ((int32_t)(insn << 6)) >> 4; + if (in_arena(pc + off)) return; // just re-execute the new branch + } + + if (emulate(uc, insn)) { + uc->uc_mcontext.pc = pc + 4; + st_traps++; + if (patch_enabled) try_patch(pc, insn); + else if (logon) logsite("no-patch", pc, insn); + if (logon && !(st_traps & 0x3FFF)) logstats("stats"); // every 16384 traps + return; + } + + logsite("unhandled", pc, insn); + (void)!write(2, "lse_emu: unhandled pc=", 22); + put_hex(pc); + (void)!write(2, " insn=", 6); + put_hex(insn); + (void)!write(2, "\n", 1); + + void *h = app_h; + if (app_valid && h != (void *)SIG_DFL && h != (void *)SIG_IGN) { + if (app_f & SA_SIGINFO) ((void (*)(int, siginfo_t *, void *))h)(sig, si, ctx); + else ((void (*)(int))h)(sig); + return; + } + struct sigaction dfl; memset(&dfl, 0, sizeof dfl); + dfl.sa_handler = SIG_DFL; + real_sigaction(SIGILL, &dfl, NULL); +} + +/* ======================= init / fini ====================================== */ + +static int add_str(char *b, int n, const char *s) { while (*s) b[n++] = *s++; return n; } +static int add_dec(char *b, int n, unsigned long v) { + char t[21]; int k = 0; + if (!v) t[k++] = '0'; + while (v) { t[k++] = '0' + v % 10; v /= 10; } + while (k) b[n++] = t[--k]; + return n; +} + +static int skipped(void) { // LSE_EMU_SKIP=name:name + const char *s = getenv("LSE_EMU_SKIP"); + if (!s || !*s) return 0; + char exe[512]; ssize_t r = readlink("/proc/self/exe", exe, sizeof exe - 1); + if (r <= 0) return 0; + exe[r] = 0; + char tok[128]; + while (*s) { + int k = 0; + while (*s && *s != ':' && k < 127) tok[k++] = *s++; + tok[k] = 0; + if (*s == ':') s++; + if (k && strstr(exe, tok)) return 1; + } + return 0; +} + +__attribute__((constructor)) static void init(void) { + resolve(); + if (getauxval(AT_HWCAP) & HWCAP_ATOMICS) return; // CPU has real LSE + + // Exit silently if LSE_EMU is missing or not set to "1" + const char *emu_env = getenv("LSE_EMU"); + if (!emu_env || emu_env[0] != '1') { + return; + } + + struct sigaction sa; memset(&sa, 0, sizeof sa); + sa.sa_sigaction = on_sigill; + sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_NODEFER; + sigemptyset(&sa.sa_mask); + real_sigaction(SIGILL, &sa, NULL); + active = 1; + + const char *p = getenv("LSE_EMU_PATCH"); + stats = getenv("LSE_EMU_STATS") != NULL; + if (p && p[0] == '1' && !skipped()) { + refresh_ranges(); + patch_enabled = 1; + } + + const char *lg = getenv("LSE_EMU_LOG"); // =1 -> /tmp/lse_emu.log, =/path -> that file + if (lg && lg[0] && lg[0] != '0') { + if (lg[0] == '/') { size_t l = strlen(lg); if (l < sizeof logpath) memcpy(logpath, lg, l + 1); } + logon = 1; + char b[700], exe[480]; int n = 0; + ssize_t r = readlink("/proc/self/exe", exe, sizeof exe - 1); + exe[r > 0 ? r : 0] = 0; + n = add_str(b, n, "pid="); n = add_dec(b, n, (unsigned long)getpid()); + n = add_str(b, n, " start patch="); n = add_dec(b, n, (unsigned long)patch_enabled); + n = add_str(b, n, " exe="); n = add_str(b, n, exe); + b[n++] = '\n'; + logwrite(b, n); + } +} + +__attribute__((destructor)) static void fini(void) { + if (!active) return; + logstats("exit"); + if (!stats) return; + char b[160]; int n = 0; + n = add_str(b, n, "lse_emu: pid="); n = add_dec(b, n, (unsigned long)getpid()); + n = add_str(b, n, " traps="); n = add_dec(b, n, st_traps); + n = add_str(b, n, " patched="); n = add_dec(b, n, st_patched); + n = add_str(b, n, " failed="); n = add_dec(b, n, st_failed); + b[n++] = '\n'; + (void)!write(2, b, n); +} \ No newline at end of file diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-session b/tools/linuxfs/overlay/usr/local/bin/droiddeck-session index 415f9472f..c68d740aa 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-session +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-session @@ -11,6 +11,10 @@ if [ "${BL_RAISE_NOFILE:-0}" = 1 ] && [ -z "${BL_INSIDE:-}" ]; then unset bl_nofile_was fi export HOME=${HOME:-/root} +if grep -qs '/liblse_emu\.so$' /etc/ld.so.preload; then + export LSE_EMU=1 LSE_EMU_PATCH=1 + [ -n "${BL_DEBUG_DIR:-}" ] && export LSE_EMU_LOG=${LSE_EMU_LOG:-$BL_DEBUG_DIR/lse_emu.log} +fi mode=${1:-desktop} shift || true step() { echo "== STEP $(date -u +%H:%M:%S.%3N) $*"; } @@ -30,6 +34,7 @@ if [ -n "${BL_LOG:-}" ] && [ -z "${BL_INSIDE:-}" ]; then echo "== uid=$(id -u) size=${BL_WIDTH:-?}x${BL_HEIGHT:-?} icd=${VK_ICD_FILENAMES:-none}" echo "== gamescope: $(gamescope --version 2>&1 | head -1) at $(command -v gamescope)" echo "== rootfs: $(cat /.version 2>/dev/null || echo unversioned)" + [ "${LSE_EMU:-}" = 1 ] && echo "== cpu: no LSE atomics (ARMv8.0); liblse_emu.so emulates them (log: ${LSE_EMU_LOG:-off})" fi start_diagnostic_inventory() { diff --git a/tools/release/runtime_inputs.py b/tools/release/runtime_inputs.py index 806c49f79..e9d2bbda8 100644 --- a/tools/release/runtime_inputs.py +++ b/tools/release/runtime_inputs.py @@ -89,6 +89,7 @@ def required(): ['msiinfo', 'cabextract', '7z', '7z.so', 'libmsi-1.0.so.0', 'libgsf-1.so.114', 'libgcab-1.0.so.0']] paths += ['assets/linuxfs/usr/local/lib/droiddeck-xclipse/' + name for name in ['libVkLayer_droiddeck_xclipse.so', 'VkLayer_droiddeck_xclipse.json']] + paths += ['assets/linuxfs/usr/local/lib/droiddeck-lse/liblse_emu.so'] paths += ['assets/linuxfs/usr/local/lib/droiddeck-vr/' + name for name in ['droiddeck/bin/linuxarm64/driver_droiddeck.so', 'droiddeck/bin/linuxarm64/xr-display-mode', 'droiddeck/driver.vrdrivermanifest', diff --git a/tools/tests/lse_emu_test.c b/tools/tests/lse_emu_test.c new file mode 100644 index 000000000..9db8cf6e7 --- /dev/null +++ b/tools/tests/lse_emu_test.c @@ -0,0 +1,128 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#define MAX_THREADS 8 +#define ROUNDS 20000 + +static uint64_t counter64; +static uint32_t counter32; +static uint16_t counter16; +static uint8_t counter8; +static uint64_t spin; + +static void check(int ok, const char *what) { + if (!ok) { + printf("FAIL %s\n", what); + exit(1); + } +} + +static uint64_t ldaddal64(uint64_t *p, uint64_t v) { + uint64_t old; + __asm__ volatile("ldaddal %x1, %x0, [%2]" : "=r"(old) : "r"(v), "r"(p) : "memory"); + return old; +} + +static uint32_t ldadd32(uint32_t *p, uint32_t v) { + uint32_t old; + __asm__ volatile("ldadd %w1, %w0, [%2]" : "=r"(old) : "r"(v), "r"(p) : "memory"); + return old; +} + +static uint16_t ldaddlh(uint16_t *p, uint16_t v) { + uint16_t old; + __asm__ volatile("ldaddlh %w1, %w0, [%2]" : "=r"(old) : "r"(v), "r"(p) : "memory"); + return old; +} + +static uint8_t ldaddab(uint8_t *p, uint8_t v) { + uint8_t old; + __asm__ volatile("ldaddab %w1, %w0, [%2]" : "=r"(old) : "r"(v), "r"(p) : "memory"); + return old; +} + +static uint64_t casal64(uint64_t *p, uint64_t expected, uint64_t desired) { + __asm__ volatile("casal %x0, %x2, [%1]" : "+r"(expected) : "r"(p), "r"(desired) : "memory"); + return expected; +} + +static void *worker(void *arg) { + (void)arg; + for (int i = 0; i < ROUNDS; i++) { + ldaddal64(&counter64, 1); + ldadd32(&counter32, 1); + ldaddlh(&counter16, 1); + ldaddab(&counter8, 1); + uint64_t seen = __atomic_load_n(&spin, __ATOMIC_RELAXED); + while (casal64(&spin, seen, seen + 1) != seen) seen = __atomic_load_n(&spin, __ATOMIC_RELAXED); + } + return NULL; +} + +static volatile sig_atomic_t app_handler_ran; + +static void app_sigill(int sig, siginfo_t *info, void *context) { + (void)sig; + (void)info; + app_handler_ran = 1; + ((ucontext_t *)context)->uc_mcontext.pc += 4; +} + +int main(int argc, char **argv) { + int threads_wanted = argc > 1 ? atoi(argv[1]) : 4; + check(threads_wanted >= 1 && threads_wanted <= MAX_THREADS, "thread count"); + uint64_t v = 0xF0; + uint64_t old; + __asm__ volatile("ldclral %x1, %x0, [%2]" : "=r"(old) : "r"((uint64_t)0x30), "r"(&v) : "memory"); + check(old == 0xF0 && v == 0xC0, "ldclral"); + __asm__ volatile("ldeora %x1, %x0, [%2]" : "=r"(old) : "r"((uint64_t)0xFF), "r"(&v) : "memory"); + check(old == 0xC0 && v == 0x3F, "ldeora"); + __asm__ volatile("ldsetl %x1, %x0, [%2]" : "=r"(old) : "r"((uint64_t)0x100), "r"(&v) : "memory"); + check(old == 0x3F && v == 0x13F, "ldsetl"); + __asm__ volatile("swpal %x1, %x0, [%2]" : "=r"(old) : "r"((uint64_t)7), "r"(&v) : "memory"); + check(old == 0x13F && v == 7, "swpal"); + + int64_t s = -5; + int64_t sold; + __asm__ volatile("ldsmax %x1, %x0, [%2]" : "=r"(sold) : "r"((int64_t)-9), "r"(&s) : "memory"); + check(sold == -5 && s == -5, "ldsmax keeps the larger value"); + uint32_t u = 10; + uint32_t uold; + __asm__ volatile("ldumin %w1, %w0, [%2]" : "=r"(uold) : "r"((uint32_t)3), "r"(&u) : "memory"); + check(uold == 10 && u == 3, "ldumin"); + + uint64_t pair[2] __attribute__((aligned(16))) = {1, 2}; + register uint64_t e0 __asm__("x4") = 1, e1 __asm__("x5") = 2, d0 __asm__("x6") = 3, d1 __asm__("x7") = 4; + __asm__ volatile("caspal x4, x5, x6, x7, [%4]" : "+r"(e0), "+r"(e1) : "r"(d0), "r"(d1), "r"(pair) : "memory"); + check(e0 == 1 && e1 == 2 && pair[0] == 3 && pair[1] == 4, "caspal"); + + uint64_t missed = casal64(&v, 99, 1); + check(missed == 7 && v == 7, "casal leaves memory alone on a mismatch"); + + pthread_t threads[MAX_THREADS]; + for (int i = 0; i < threads_wanted; i++) pthread_create(&threads[i], NULL, worker, NULL); + for (int i = 0; i < threads_wanted; i++) pthread_join(threads[i], NULL); + uint64_t total = (uint64_t)threads_wanted * ROUNDS; + check(counter64 == total, "64-bit counter"); + check(counter32 == (uint32_t)total, "32-bit counter"); + check(counter16 == (uint16_t)total, "16-bit counter"); + check(counter8 == (uint8_t)total, "8-bit counter"); + check(spin == total, "casal loop"); + + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = app_sigill; + sa.sa_flags = SA_SIGINFO; + sigaction(SIGILL, &sa, NULL); + __asm__ volatile("udf #0"); + check(app_handler_ran, "the program's own SIGILL handler still runs"); + + printf("lse ok\n"); + return 0; +} diff --git a/tools/tests/test_lse_emu.py b/tools/tests/test_lse_emu.py new file mode 100644 index 000000000..c35f3c776 --- /dev/null +++ b/tools/tests/test_lse_emu.py @@ -0,0 +1,69 @@ +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +TOOLS = Path(__file__).resolve().parents[1] +SYSROOT = Path("/usr/aarch64-linux-gnu") + + +def toolchain(): + if not (shutil.which("aarch64-linux-gnu-gcc") and shutil.which("qemu-aarch64") and SYSROOT.is_dir()): + raise unittest.SkipTest("needs aarch64-linux-gnu-gcc, libc6-dev-arm64-cross and qemu-user") + + +class LseEmuTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + toolchain() + cls.tmp = tempfile.TemporaryDirectory(prefix="droiddeck-lse-") + out = Path(cls.tmp.name) + subprocess.run(["bash", str(TOOLS / "linuxfs/lse-emu/build.sh"), str(out)], check=True) + cls.library = out / "usr/local/lib/droiddeck-lse/liblse_emu.so" + cls.program = out / "lse-test" + subprocess.run(["aarch64-linux-gnu-gcc", "-O1", "-march=armv8.1-a", "-Wall", "-Wextra", "-Werror", "-pthread", + str(Path(__file__).with_name("lse_emu_test.c")), "-o", str(cls.program)], check=True) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def run_on(self, cpu, *env, threads=4): + command = ["qemu-aarch64", "-cpu", cpu, "-L", str(SYSROOT)] + for item in env: + command += ["-E", item] + return subprocess.run(command + [str(self.program), str(threads)], capture_output=True, text=True, timeout=120) + + def test_an_armv8_0_cpu_faults_without_the_emulator(self): + result = self.run_on("cortex-a57") + self.assertEqual(result.returncode, -4, result.stdout + result.stderr) + + def test_trapping_alone_runs_every_form_on_an_armv8_0_cpu(self): + result = self.run_on("cortex-a57", f"LD_PRELOAD={self.library}", "LSE_EMU=1", "LSE_EMU_STATS=1") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("lse ok", result.stdout) + self.assertRegex(result.stderr, r"traps=[1-9]\d* patched=0 ") + + def test_patching_runs_every_form_on_an_armv8_0_cpu(self): + # qemu-user write-protects translated code pages and races when other threads run them while the + # stub arena is written, so threaded patching is only meaningful on hardware. + result = self.run_on("cortex-a57", f"LD_PRELOAD={self.library}", "LSE_EMU=1", "LSE_EMU_PATCH=1", + "LSE_EMU_STATS=1", threads=1) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("lse ok", result.stdout) + self.assertRegex(result.stderr, r"patched=[1-9]\d* failed=0") + + def test_it_stays_off_without_the_switch(self): + result = self.run_on("cortex-a57", f"LD_PRELOAD={self.library}") + self.assertEqual(result.returncode, -4, result.stdout + result.stderr) + + def test_it_stays_off_on_a_cpu_with_lse(self): + result = self.run_on("neoverse-n1", f"LD_PRELOAD={self.library}", "LSE_EMU=1", "LSE_EMU_STATS=1") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("traps=", result.stderr) + + +if __name__ == "__main__": + unittest.main()