From 183cbd9351955f34cbf73cf9a35d53f9fbde23c9 Mon Sep 17 00:00:00 2001 From: MaxsTechReview Date: Sat, 21 Nov 2026 20:11:18 -0500 Subject: [PATCH] fix(preload): start apps that check for a single thread and stop the first-call allocation hang (#2995) * fix(preload): look up the native glibc symbol version first so wrapped calls never allocate On x86 the first dlvsym lookup asked for GLIBC_2.17, which only exists on arm64. The failed lookup makes glibc allocate, which deadlocks when the first wrapped call happens inside a program's own allocator setup. Each architecture now asks for its base version first. * fix(zram): start the page-out worker per request instead of keeping a thread in every process Every process that loaded the session preload carried a resident zr-worker thread from its constructor on. Programs that check they are single-threaded before forking or before unshare(CLONE_NEWUSER) failed, for example Chromium's zygote, which could not start its GPU process. The signal handler now starts the worker with a raw clone on a stack the preload maps itself. The worker makes only raw system calls and exits when no request is left, so starting it from the handler is safe and idle processes stay single-threaded. When the worker cannot be started, the process trims inline in small steps as before. * fix(preload): start the page-out worker with the same clone flags as a thread Programs that filter clone to the flag set threads use, as browser sandboxes do, now accept the worker. The kernel clears the stack-busy word and wakes waiters when the worker exits. * fix(preload): page out inline in single-threaded and self-sandboxed processes A process with one thread may be about to check that it is single-threaded before fork or unshare, and a process that added a seccomp filter after it started may trap system calls into its own handler. Neither gets a worker thread now; both trim inline in small steps as before. * fix(preload): check the signalled thread's own seccomp filters before starting the worker Read Seccomp_filters from /proc/thread-self/status so a thread that installed its own filter without TSYNC trims inline instead of calling clone under that filter, and fall back to the inline path when the signal mask cannot be changed. * test(preload): start parked threads with SIGURG already blocked * test(preload): count only page-out workers when checking for leftover threads --- docs/development/low-memory.md | 14 +- tools/linuxfs/preload/inputudev.c | 8 + tools/linuxfs/preload/zram.c | 312 +++++++++++++++++++++----- tools/tests/test_inputudev_preload.py | 73 +++++- tools/tests/test_zram_preload.py | 189 +++++++++++++--- 5 files changed, 509 insertions(+), 87 deletions(-) diff --git a/docs/development/low-memory.md b/docs/development/low-memory.md index 218d1fd8e..50e78363e 100644 --- a/docs/development/low-memory.md +++ b/docs/development/low-memory.md @@ -99,10 +99,16 @@ available. ## Threads -Page-out never runs on a thread of the game. The preload's signal handler only wakes its -`zr-worker` thread, which does the `madvise` calls. A forked process -without a worker trims inline in steps of at most 32 MiB per request and looks at no more -than 512 MiB of its address space each time. The page-out for a suspend and the read-back +When a worker can be started, page-out runs outside the game's threads. The preload's signal +handler starts a `zr-worker` thread for the request, which does the `madvise` calls and exits once no request +is left, so a process that is not paging out has no thread the preload added. Programs that +check they are single-threaded before they fork or call `unshare(CLONE_NEWUSER)`, such as +Chromium's zygote, keep working. The worker is a bare `clone` with the flags of a normal +thread, on a stack the preload maps itself, and makes only raw system calls, so starting it +from the signal handler is safe and seccomp filters that only allow thread clones accept it. A +process that has a single thread, or whose signalled thread added a seccomp filter after the +process started, as browser sandboxes do, gets no worker. When there is no worker, the process trims inline in steps of at most 32 MiB per request and +looks at no more than 512 MiB of its address space each time. The page-out for a suspend and the read-back after it are not limited there: they run once, while the process is being stopped or started, and cover all of it. The monitor is a shell loop in the session, outside every game process. diff --git a/tools/linuxfs/preload/inputudev.c b/tools/linuxfs/preload/inputudev.c index 823614bfd..3dcb5a104 100644 --- a/tools/linuxfs/preload/inputudev.c +++ b/tools/linuxfs/preload/inputudev.c @@ -1045,7 +1045,15 @@ static void *own_udev_symbol(const char *name) { } static void *dlvsym_any(const char *name) { +#if defined(__aarch64__) + static const char *versions[] = {"GLIBC_2.17", "GLIBC_2.34"}; +#elif defined(__x86_64__) + static const char *versions[] = {"GLIBC_2.2.5", "GLIBC_2.34"}; +#elif defined(__i386__) + static const char *versions[] = {"GLIBC_2.0", "GLIBC_2.34"}; +#else static const char *versions[] = {"GLIBC_2.17", "GLIBC_2.2.5", "GLIBC_2.0", "GLIBC_2.34"}; +#endif size_t i; void *resolved = NULL; diff --git a/tools/linuxfs/preload/zram.c b/tools/linuxfs/preload/zram.c index 551a6e148..ad13d8258 100644 --- a/tools/linuxfs/preload/zram.c +++ b/tools/linuxfs/preload/zram.c @@ -1,8 +1,9 @@ #define _GNU_SOURCE -#include +#pragma GCC optimize("no-stack-protector", "no-tree-loop-distribute-patterns") #include #include #include +#include #include #include #include @@ -10,6 +11,9 @@ #include #include #include +#if defined(__i386__) +#include +#endif #include @@ -31,13 +35,156 @@ #define ZR_INLINE_CHUNKS 512UL #define ZR_FAILED (~0UL) #define ZR_FUTEX_WAIT 0 -#define ZR_FUTEX_WAKE 1 +#define ZR_STACK (128UL << 10) +#define ZR_THREAD (CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | \ + CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID) +#define ZR_TEXT(x) #x +#define ZR_NR(x) ZR_TEXT(x) +#define ZR_CALL(n, a, b, c, d, e, f, ...) \ + zr_call((long)(n), (long)(a), (long)(b), (long)(c), (long)(d), (long)(e), (long)(f)) +#define zr_sys(...) ZR_CALL(__VA_ARGS__, 0, 0, 0, 0, 0, 0, 0) + +#if defined(__aarch64__) +#define ZR_MMAP SYS_mmap +__asm__(".pushsection .text\n" + ".p2align 2\n" + ".hidden zr_call\n" + ".type zr_call, %function\n" + "zr_call:\n" + " mov x8, x0\n" + " mov x0, x1\n" + " mov x1, x2\n" + " mov x2, x3\n" + " mov x3, x4\n" + " mov x4, x5\n" + " mov x5, x6\n" + " svc #0\n" + " ret\n" + ".size zr_call, . - zr_call\n" + ".hidden zr_clone\n" + ".type zr_clone, %function\n" + "zr_clone:\n" + " mov x5, x2\n" + " mov x2, x3\n" + " mov x3, x4\n" + " mov x4, x5\n" + " mov x8, #" ZR_NR(SYS_clone) "\n" + " svc #0\n" + " cbnz x0, 1f\n" + " mov x29, xzr\n" + " mov x30, xzr\n" + " bl zr_thread\n" + " mov x0, xzr\n" + " mov x8, #" ZR_NR(SYS_exit) "\n" + " svc #0\n" + " brk #0\n" + "1:\n" + " ret\n" + ".size zr_clone, . - zr_clone\n" + ".popsection\n"); +#elif defined(__x86_64__) +#define ZR_MMAP SYS_mmap +__asm__(".pushsection .text\n" + ".p2align 4\n" + ".hidden zr_call\n" + ".type zr_call, @function\n" + "zr_call:\n" + " mov %rdi, %rax\n" + " mov %rsi, %rdi\n" + " mov %rdx, %rsi\n" + " mov %rcx, %rdx\n" + " mov %r8, %r10\n" + " mov %r9, %r8\n" + " mov 8(%rsp), %r9\n" + " syscall\n" + " ret\n" + ".size zr_call, . - zr_call\n" + ".hidden zr_clone\n" + ".type zr_clone, @function\n" + "zr_clone:\n" + " mov %rdx, %r10\n" + " mov %rcx, %rdx\n" + " mov $" ZR_NR(SYS_clone) ", %eax\n" + " syscall\n" + " test %rax, %rax\n" + " jnz 1f\n" + " xor %ebp, %ebp\n" + " call zr_thread\n" + " xor %edi, %edi\n" + " mov $" ZR_NR(SYS_exit) ", %eax\n" + " syscall\n" + " hlt\n" + "1:\n" + " ret\n" + ".size zr_clone, . - zr_clone\n" + ".popsection\n"); +#elif defined(__i386__) +#define ZR_MMAP SYS_mmap2 +__asm__(".pushsection .text\n" + ".p2align 4\n" + ".hidden zr_call\n" + ".type zr_call, @function\n" + "zr_call:\n" + " push %ebp\n" + " push %edi\n" + " push %esi\n" + " push %ebx\n" + " mov 20(%esp), %eax\n" + " mov 24(%esp), %ebx\n" + " mov 28(%esp), %ecx\n" + " mov 32(%esp), %edx\n" + " mov 36(%esp), %esi\n" + " mov 40(%esp), %edi\n" + " mov 44(%esp), %ebp\n" + " int $0x80\n" + " pop %ebx\n" + " pop %esi\n" + " pop %edi\n" + " pop %ebp\n" + " ret\n" + ".size zr_call, . - zr_call\n" + ".hidden zr_clone\n" + ".type zr_clone, @function\n" + "zr_clone:\n" + " push %ebx\n" + " push %esi\n" + " push %edi\n" + " mov 16(%esp), %ebx\n" + " mov 20(%esp), %ecx\n" + " mov 24(%esp), %edi\n" + " mov 28(%esp), %edx\n" + " mov 32(%esp), %esi\n" + " mov $" ZR_NR(SYS_clone) ", %eax\n" + " int $0x80\n" + " test %eax, %eax\n" + " jnz 1f\n" + " xor %ebp, %ebp\n" + " call zr_thread\n" + " xor %ebx, %ebx\n" + " mov $" ZR_NR(SYS_exit) ", %eax\n" + " int $0x80\n" + " hlt\n" + "1:\n" + " pop %edi\n" + " pop %esi\n" + " pop %ebx\n" + " ret\n" + ".size zr_clone, . - zr_clone\n" + ".popsection\n"); +#else +#error "zram.c needs a raw system call for this architecture" +#endif + +long zr_call(long n, long a, long b, long c, long d, long e, long f) __attribute__((visibility("hidden"))); +long zr_clone(long flags, char *stack, int *child, int *parent, void *tls) __attribute__((visibility("hidden"))); +void zr_thread(void) __attribute__((visibility("hidden"))); -static long (*zr_sys)(long, ...); static unsigned long zr_page; -static int zr_busy; -static int zr_pending; -static int zr_worker; +static unsigned zr_requests; +static int zr_stack_busy; +static int zr_tid; +static long zr_filters; +static char *zr_stack; static char zr_maps[8192]; static uint64_t zr_entries[ZR_BATCH]; static struct iovec zr_iov[ZR_BATCH]; @@ -54,6 +201,18 @@ static uint16_t zr_gen = 1; static unsigned long zr_cursor; static unsigned long zr_visits; +static int zr_prefix(const char *text, const char *prefix) { + while (*prefix) + if (*text++ != *prefix++) return 0; + return 1; +} + +static char *zr_find(char *text, char c, size_t n) { + for (; n > 0; n--, text++) + if (*text == c) return text; + return NULL; +} + static int zr_open(const char *path) { return (int)zr_sys(SYS_openat, AT_FDCWD, path, O_RDONLY | O_CLOEXEC, 0); } @@ -103,7 +262,7 @@ static int zr_eligible(const char *line, int compress, unsigned long *start, uns while (*p && *p != ' ') p++; while (*p == ' ') p++; } - return *p == 0 || strncmp(p, "[heap]", 6) == 0 || strncmp(p, "[anon:", 6) == 0; + return *p == 0 || zr_prefix(p, "[heap]") || zr_prefix(p, "[anon:"); } static void zr_read_back(int pagemap, unsigned long start, unsigned long end) { @@ -137,7 +296,7 @@ static unsigned long zr_count(int pagemap, unsigned long start, unsigned long en do { if (zr_sys(SYS_lseek, pagemap, (long)(start / zr_page * sizeof(uint64_t)), SEEK_SET) < 0) return ZR_FAILED; got = zr_sys(SYS_read, pagemap, zr_entries, size); - } while (got < 0 && errno == EINTR); + } while (got == -EINTR); if (got != size) return ZR_FAILED; for (long i = 0; i < got / (long)sizeof(uint64_t); i++) if (zr_entries[i] & bit) count++; @@ -157,8 +316,7 @@ static struct zr_slot *zr_slot(unsigned long addr, int create) { } } if (!create) return NULL; - memset(&zr_slots[home], 0, sizeof(zr_slots[home])); - zr_slots[home].key = key; + zr_slots[home] = (struct zr_slot){.key = key}; return &zr_slots[home]; } @@ -228,13 +386,13 @@ static unsigned long zr_walk(int mode, unsigned long want, unsigned long lo, uns size_t have = 0; while (done < want && zr_visits > 0) { long n = zr_sys(SYS_read, maps, zr_maps + have, sizeof(zr_maps) - 1 - have); - if (n < 0 && errno == EINTR) continue; + if (n == -EINTR) continue; if (n <= 0) break; have += (size_t)n; zr_maps[have] = 0; char *line = zr_maps; char *nl; - while (done < want && zr_visits > 0 && (nl = memchr(line, '\n', have - (size_t)(line - zr_maps))) != NULL) { + while (done < want && zr_visits > 0 && (nl = zr_find(line, '\n', have - (size_t)(line - zr_maps))) != NULL) { *nl = 0; unsigned long s, e; if (zr_eligible(line, mode != 0, &s, &e) && e > s) { @@ -245,7 +403,7 @@ static unsigned long zr_walk(int mode, unsigned long want, unsigned long lo, uns line = nl + 1; } have -= (size_t)(line - zr_maps); - memmove(zr_maps, line, have); + for (size_t i = 0; i < have; i++) zr_maps[i] = line[i]; if (have == sizeof(zr_maps) - 1) have = 0; } if (pagemap >= 0) zr_sys(SYS_close, pagemap); @@ -291,15 +449,88 @@ static void zr_request(int worker) { zr_ack((long)(done * (zr_page / 1024)), 1); } -static void *zr_worker_main(void *unused) { - (void)unused; - prctl(PR_SET_NAME, "zr-worker", 0, 0, 0); - for (;;) { - while (!__atomic_exchange_n(&zr_pending, 0, __ATOMIC_SEQ_CST)) - zr_sys(SYS_futex, &zr_pending, ZR_FUTEX_WAIT, 0, NULL, NULL, 0); - zr_request(1); +static void zr_drain(int worker) { + unsigned seen; + do { + seen = __atomic_load_n(&zr_requests, __ATOMIC_ACQUIRE); + zr_request(worker); + } while (!__atomic_compare_exchange_n(&zr_requests, &seen, 0, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)); +} + +void zr_thread(void) { + zr_sys(SYS_prctl, PR_SET_NAME, "zr-worker"); + zr_drain(1); +} + +static void *zr_tls(void *desc) { +#if defined(__aarch64__) + (void)desc; + return __builtin_thread_pointer(); +#elif defined(__x86_64__) + (void)desc; + void *self; + __asm__("mov %%fs:0, %0" : "=r"(self)); + return self; +#else + struct user_desc *entry = desc; + unsigned short selector; + __asm__("mov %%gs, %0" : "=r"(selector)); + entry->entry_number = selector >> 3; + return zr_sys(SYS_get_thread_area, entry) == 0 ? entry : NULL; +#endif +} + +static long zr_field(const char *text, const char *key) { + for (const char *line = text; *line; line++) { + if ((line == text || line[-1] == '\n') && zr_prefix(line, key)) { + long value = 0; + for (const char *k = key; *k; k++) line++; + while (*line == ' ' || *line == '\t') line++; + for (; *line >= '0' && *line <= '9'; line++) value = value * 10 + (*line - '0'); + return value; + } } - return NULL; + return -1; +} + +static int zr_status(long *threads, long *filters) { + int fd = zr_open("/proc/thread-self/status"); + if (fd < 0) return 0; + long have = 0, n; + while (have < (long)sizeof(zr_maps) - 1 && + ((n = zr_sys(SYS_read, fd, zr_maps + have, sizeof(zr_maps) - 1 - (size_t)have)) > 0 || n == -EINTR)) + if (n > 0) have += n; + zr_sys(SYS_close, fd); + zr_maps[have] = 0; + *threads = zr_field(zr_maps, "Threads:"); + *filters = zr_field(zr_maps, "Seccomp_filters:"); + return have > 0; +} + +static int zr_start(void) { + long threads, filters; + if (!zr_status(&threads, &filters) || threads < 2 || filters > zr_filters) return 0; + if (zr_stack == NULL) { + long base = zr_sys(ZR_MMAP, 0, ZR_STACK, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK | MAP_NORESERVE, -1, 0); + if ((unsigned long)base > -4096UL) return 0; + zr_stack = (char *)base; + } + struct { long sec, nsec; } wait = {0, 5000000}; + for (int i = 0; i < 20 && __atomic_load_n(&zr_stack_busy, __ATOMIC_ACQUIRE); i++) + zr_sys(SYS_futex, &zr_stack_busy, ZR_FUTEX_WAIT, 1, &wait, NULL, 0); + if (__atomic_load_n(&zr_stack_busy, __ATOMIC_ACQUIRE)) return 0; + uint64_t desc[4] = {0}; + void *tls = zr_tls(desc); + if (tls == NULL) return 0; + uint64_t all = ~0ULL, previous; + if (zr_sys(SYS_rt_sigprocmask, SIG_SETMASK, &all, &previous, sizeof(all)) != 0) return 0; + __atomic_store_n(&zr_stack_busy, 1, __ATOMIC_RELEASE); + long tid = zr_clone(ZR_THREAD, zr_stack + ZR_STACK, &zr_stack_busy, &zr_tid, tls); + zr_sys(SYS_rt_sigprocmask, SIG_SETMASK, &previous, NULL, sizeof(previous)); + if (tid > 0) return 1; + __atomic_store_n(&zr_stack_busy, 0, __ATOMIC_RELEASE); + return 0; } static void zr_handler(int sig, siginfo_t *info, void *context) { @@ -307,34 +538,23 @@ static void zr_handler(int sig, siginfo_t *info, void *context) { (void)context; if (info == NULL || info->si_code != SI_USER) return; int saved = errno; - __atomic_store_n(&zr_pending, 1, __ATOMIC_SEQ_CST); - if (__atomic_load_n(&zr_worker, __ATOMIC_ACQUIRE)) { - zr_sys(SYS_futex, &zr_pending, ZR_FUTEX_WAKE, 1, NULL, NULL, 0); - } else while (__atomic_load_n(&zr_pending, __ATOMIC_SEQ_CST) - && !__atomic_exchange_n(&zr_busy, 1, __ATOMIC_SEQ_CST)) { - __atomic_store_n(&zr_pending, 0, __ATOMIC_SEQ_CST); - zr_request(0); - __atomic_store_n(&zr_busy, 0, __ATOMIC_SEQ_CST); - } + if (__atomic_fetch_add(&zr_requests, 1, __ATOMIC_ACQ_REL) == 0 && !zr_start()) zr_drain(0); errno = saved; } static void zr_child(void) { - __atomic_store_n(&zr_busy, 0, __ATOMIC_SEQ_CST); - __atomic_store_n(&zr_pending, 0, __ATOMIC_SEQ_CST); - __atomic_store_n(&zr_worker, 0, __ATOMIC_RELEASE); + __atomic_store_n(&zr_requests, 0, __ATOMIC_RELEASE); + __atomic_store_n(&zr_stack_busy, 0, __ATOMIC_RELEASE); } __attribute__((constructor)) static void zr_init(void) { - void *libc = dlopen("libc.so.6", RTLD_LAZY | RTLD_NOLOAD); - if (libc == NULL) return; - zr_sys = (long (*)(long, ...))dlsym(libc, "syscall"); - if (zr_sys == NULL) return; int fd = zr_open(ZR_CTL); if (fd < 0) return; zr_sys(SYS_close, fd); long page = sysconf(_SC_PAGESIZE); zr_page = page > 0 ? (unsigned long)page : 4096UL; + long threads; + if (!zr_status(&threads, &zr_filters)) zr_filters = -1; struct sigaction old; if (sigaction(SIGURG, NULL, &old) != 0) return; if ((old.sa_flags & SA_SIGINFO) || old.sa_handler != SIG_DFL) return; @@ -343,21 +563,5 @@ __attribute__((constructor)) static void zr_init(void) { sa.sa_sigaction = zr_handler; sa.sa_flags = SA_SIGINFO | SA_RESTART; sigemptyset(&sa.sa_mask); - if (sigaction(SIGURG, &sa, NULL) == 0) { - pthread_atfork(NULL, NULL, zr_child); - pthread_attr_t attr; - if (pthread_attr_init(&attr) == 0) { - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_t thread; - sigset_t blocked; - sigset_t previous; - if (sigfillset(&blocked) == 0 && - pthread_sigmask(SIG_SETMASK, &blocked, &previous) == 0) { - if (pthread_create(&thread, &attr, zr_worker_main, NULL) == 0) - __atomic_store_n(&zr_worker, 1, __ATOMIC_RELEASE); - pthread_sigmask(SIG_SETMASK, &previous, NULL); - } - pthread_attr_destroy(&attr); - } - } + if (sigaction(SIGURG, &sa, NULL) == 0) pthread_atfork(NULL, NULL, zr_child); } diff --git a/tools/tests/test_inputudev_preload.py b/tools/tests/test_inputudev_preload.py index c1d970fd1..8e4cf66e8 100644 --- a/tools/tests/test_inputudev_preload.py +++ b/tools/tests/test_inputudev_preload.py @@ -77,6 +77,66 @@ char *realpath(const char *path, char *resolved) { } """ +LOCKED = r""" +#include +#include + +extern int busy; + +__attribute__((constructor)) static void start(void) { + busy = 1; + syscall(SYS_getpid); + busy = 0; +} +""" + +ARENA = r""" +#include +#include +#include + +static _Alignas(64) unsigned char arena[1 << 23]; +static size_t used; +int busy; +static int busy_allocations; + +void *memalign(size_t align, size_t size) { + uintptr_t start; + if (busy) busy_allocations++; + if (align < 16) align = 16; + start = ((uintptr_t)(arena + used) + 16 + align - 1) & ~(uintptr_t)(align - 1); + if (start + size > (uintptr_t)(arena + sizeof(arena))) return NULL; + ((size_t *)start)[-1] = size; + used = start + size - (uintptr_t)arena; + return (void *)start; +} + +void *malloc(size_t size) { return memalign(16, size); } +void *aligned_alloc(size_t align, size_t size) { return memalign(align, size); } +int posix_memalign(void **out, size_t align, size_t size) { return (*out = memalign(align, size)) ? 0 : 12; } +void free(void *p) { (void)p; } + +void *calloc(size_t n, size_t size) { + void *p = malloc(n * size); + if (p) memset(p, 0, n * size); + return p; +} + +void *realloc(void *p, size_t size) { + void *q = malloc(size); + if (p && q) { + size_t old = ((size_t *)p)[-1]; + memcpy(q, p, old < size ? old : size); + } + return q; +} + +int main(void) { + printf("allocations %d\n", busy_allocations); + return 0; +} +""" + UDEV = r""" #include #include @@ -414,7 +474,7 @@ class InputUdevPreloadTest(unittest.TestCase): if "__GLIBC__" not in compile_c("-dM", "-E", "-x", "c", "-include", "limits.h", "/dev/null"): raise unittest.SkipTest("needs a glibc toolchain") sources = (("once.c", ONCE), ("first.c", FIRST), ("second.c", SECOND), ("chain.c", CHAIN), ("mark.c", MARK), - ("udev.c", UDEV), ("program.c", PROGRAM)) + ("udev.c", UDEV), ("program.c", PROGRAM), ("locked.c", LOCKED), ("arena.c", ARENA)) for name, text in sources: (build / name).write_text(text) cls.lib = cls.dir / "libddsession.so" @@ -427,6 +487,11 @@ class InputUdevPreloadTest(unittest.TestCase): compile_c("-shared", "-fPIC", "-O2", "-o", str(cls.udev), str(build / "udev.c")) cls.program = cls.dir / "program" compile_c("-O0", "-o", str(cls.program), str(build / "program.c"), str(cls.dir / "libfirst.so"), "-ldl") + compile_c("-shared", "-fPIC", "-O0", "-o", str(cls.dir / "liblocked.so"), str(build / "locked.c")) + cls.arena = cls.dir / "arena" + compile_c("-O0", "-rdynamic", "-o", str(cls.arena), str(build / "arena.c"), "-Wl,--no-as-needed", + str(cls.dir / "liblocked.so"), + f"-Wl,-rpath,{cls.dir}") cls.pads = cls.pad_dir("pads", "event0") @classmethod @@ -449,6 +514,12 @@ class InputUdevPreloadTest(unittest.TestCase): def test_a_lookup_from_another_preloads_initialiser_does_not_reenter_it(self): self.assertEqual(self.run_program("once", "libonce.so"), ["readlink 1"]) + def test_a_first_wrapped_call_inside_the_programs_allocator_does_not_allocate(self): + result = subprocess.run([str(self.arena)], env=dict(os.environ, LD_PRELOAD=str(self.lib)), capture_output=True, + text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), ["allocations 0"]) + def test_the_real_lookup_still_sees_the_original_caller(self): self.assertEqual(self.run_program("next", "libfirst.so", "libsecond.so"), ["next 2"]) diff --git a/tools/tests/test_zram_preload.py b/tools/tests/test_zram_preload.py index bf1e8391c..8a9f81348 100644 --- a/tools/tests/test_zram_preload.py +++ b/tools/tests/test_zram_preload.py @@ -18,7 +18,14 @@ PROGRAM = r""" #include #include #include +#include +#include +#include +#include +#include #include +#include +#include #include #include #include @@ -31,14 +38,15 @@ static unsigned char *shared; static volatile sig_atomic_t urg_tid; static volatile sig_atomic_t usr_tid; static int urg_count, usr_count; -static int worker_tid; static int worker_delivery; _Static_assert(__atomic_always_lock_free(sizeof(int), 0), "signal state must be lock-free"); static void signal_tid(int sig) { sig_atomic_t tid = (sig_atomic_t)syscall(SYS_gettid); - if (tid == __atomic_load_n(&worker_tid, __ATOMIC_RELAXED)) + char name[16] = {0}; + prctl(PR_GET_NAME, name, 0, 0, 0); + if (!strcmp(name, "zr-worker")) __atomic_store_n(&worker_delivery, 1, __ATOMIC_RELAXED); if (sig == SIGURG) { __atomic_store_n(&urg_tid, tid, __ATOMIC_RELAXED); @@ -123,6 +131,75 @@ static long trim(char mode, long kb) { exit(5); } +static int workers(void) { + DIR *dir = opendir("/proc/self/task"); + struct dirent *entry; + int count = 0; + while (dir && (entry = readdir(dir)) != NULL) { + if (entry->d_name[0] == '.') continue; + char path[64], name[32] = {0}; + snprintf(path, sizeof(path), "/proc/self/task/%s/comm", entry->d_name); + FILE *comm = fopen(path, "r"); + if (comm && fgets(name, sizeof(name), comm) && !strcmp(name, "zr-worker\n")) count++; + if (comm) fclose(comm); + } + if (dir) closedir(dir); + return count; +} + +static void without_threads(void) { + struct rlimit limit; + if (getrlimit(RLIMIT_NPROC, &limit)) _exit(15); + limit.rlim_cur = 0; + if (setrlimit(RLIMIT_NPROC, &limit)) _exit(15); +} + +static void *park(void *arg) { + (void)arg; + for (;;) pause(); + return NULL; +} + +static void parked(void) { + pthread_t thread; + sigset_t urg, previous; + sigemptyset(&urg); + sigaddset(&urg, SIGURG); + pthread_sigmask(SIG_BLOCK, &urg, &previous); + if (pthread_create(&thread, NULL, park, NULL)) _exit(16); + pthread_sigmask(SIG_SETMASK, &previous, NULL); +} + +static void filter_clone(unsigned long allowed) { + struct sock_filter rules[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_clone, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, allowed, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = {sizeof(rules) / sizeof(rules[0]), rules}; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) _exit(17); +} + +static volatile int sandbox_ready; + +static void *sandbox(void *arg) { + (void)arg; + sigset_t urg; + sigemptyset(&urg); + sigaddset(&urg, SIGURG); + filter_clone(0); + pthread_sigmask(SIG_UNBLOCK, &urg, NULL); + sandbox_ready = 1; + for (;;) pause(); + return NULL; +} + +static const unsigned long thread_clone = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | + CLONE_SYSVSEM | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID; + static uint64_t sum(const unsigned char *p, size_t n) { uint64_t s = 0; for (size_t i = 0; i < n; i += 64) s = s * 31 + p[i]; @@ -155,6 +232,7 @@ int main(int argc, char **argv) { memset(shared, 0, size); pthread_t t[2]; for (uintptr_t i = 0; i < 2; i++) pthread_create(&t[i], NULL, writer, (void *)i); + parked(); struct sigaction sa; sigaction(SIGURG, NULL, &sa); printf("handler %d\n", (sa.sa_flags & SA_SIGINFO) ? 1 : 0); @@ -164,27 +242,21 @@ int main(int argc, char **argv) { memset(&signal_action, 0, sizeof(signal_action)); signal_action.sa_handler = signal_tid; sigemptyset(&signal_action.sa_mask); - sigaction(SIGURG, &signal_action, NULL); sigaction(SIGUSR1, &signal_action, NULL); + int fd = open(ctl, O_WRONLY | O_TRUNC); + if (fd < 0 || write(fd, "1", 1) != 1) return 10; + close(fd); printf("ready %d\n", getpid()); fflush(stdout); - int id; - if (scanf("%d", &id) != 1) return 10; - __atomic_store_n(&worker_tid, id, __ATOMIC_RELAXED); - printf("armed 1\n"); - fflush(stdout); for (int round = 1; round <= 100; round++) { int i; - for (i = 0; i < 5000 && - (__atomic_load_n(&urg_count, __ATOMIC_ACQUIRE) < round || - __atomic_load_n(&usr_count, __ATOMIC_ACQUIRE) < round); i++) usleep(1000); + for (i = 0; i < 5000 && __atomic_load_n(&usr_count, __ATOMIC_ACQUIRE) < round; i++) usleep(1000); if (i == 5000) return 11; printf("received %d\n", round); fflush(stdout); } stop = 1; for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); - printf("urg %d\n", __atomic_load_n(&urg_tid, __ATOMIC_RELAXED)); printf("usr %d\n", __atomic_load_n(&usr_tid, __ATOMIC_RELAXED)); printf("worker %d\n", __atomic_load_n(&worker_delivery, __ATOMIC_RELAXED)); return 0; @@ -199,6 +271,13 @@ int main(int argc, char **argv) { int status_code; waitpid(child, &status_code, 0); printf("childack %d\n", WIFEXITED(status_code) && WEXITSTATUS(status_code) == 0); + } else if (!strcmp(argv[4], "threads")) { + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + printf("idle %d\n", workers()); + request('1'); + printf("after %d\n", workers()); + return 0; } else if (!strcmp(argv[4], "trim")) { stop = 1; for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); @@ -271,8 +350,47 @@ int main(int argc, char **argv) { _exit(0); } waitpid(child, NULL, 0); + child = fork(); + if (child < 0) exit(6); + if (child == 0) { + parked(); + without_threads(); + for (size_t i = 0; i < size; i += page) data[i]++; + printf("nothread_trim %ld\n", trim('2', 1L << 30)); + fflush(stdout); + _exit(0); + } + waitpid(child, NULL, 0); printf("worker_trim %ld\n", trim('2', 1L << 30)); return 0; + } else if (!strcmp(argv[4], "sandboxed")) { + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + filter_clone(thread_clone); + printf("sandboxed_trim %ld\n", trim('2', 1L << 30)); + return 0; + } else if (!strcmp(argv[4], "thread-sandboxed")) { + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + sigset_t urg; + sigemptyset(&urg); + sigaddset(&urg, SIGURG); + pthread_sigmask(SIG_BLOCK, &urg, NULL); + pthread_t thread; + if (pthread_create(&thread, NULL, sandbox, NULL)) exit(16); + while (!sandbox_ready) usleep(1000); + printf("thread_sandboxed_trim %ld\n", trim('2', 1L << 30)); + return 0; + } else if (!strcmp(argv[4], "filtered")) { + fflush(stdout); + filter_clone(thread_clone); + execl("/proc/self/exe", argv[0], argv[1], argv[2], "0", "filtered-trim", (char *)NULL); + return 18; + } else if (!strcmp(argv[4], "filtered-trim")) { + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + printf("filtered_trim %ld\n", trim('2', 1L << 30)); + return 0; } else if (!strcmp(argv[4], "burst")) { send_request('0'); usleep(10000); @@ -283,6 +401,7 @@ int main(int argc, char **argv) { pid_t child = fork(); if (child < 0) exit(8); if (child == 0) { + without_threads(); if (!strcmp(argv[4], "timed-inline-single")) { if (munmap(data, size) || munmap(shared, size)) _exit(12); data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -431,6 +550,11 @@ class ZramPreloadTest(unittest.TestCase): self.assertEqual([1], out["intact"]) self.assertEqual(6, len(out["out"])) + def test_a_process_keeps_no_extra_thread_while_nothing_is_paged_out(self): + out = self.run_program(None, 8, 0, "threads") + self.assertEqual([0], out["idle"]) + self.assertEqual([0], out["after"]) + def test_process_signals_do_not_reach_zram_worker(self): environment = dict(os.environ, LD_PRELOAD=str(self.lib)) process = subprocess.Popen([str(self.steam), str(self.ctl), "1", "0", "signals"], @@ -439,30 +563,16 @@ class ZramPreloadTest(unittest.TestCase): process.stdout.readline() ready = process.stdout.readline().split() self.assertEqual("ready", ready[0]) - worker = None - task_dir = Path("/proc") / ready[1] / "task" - for task in task_dir.iterdir(): - if (task / "comm").read_text().strip() == "zr-worker": - worker = int(task.name) - break - self.assertIsNotNone(worker) - self.assertEqual(os.sched_getscheduler(process.pid), os.sched_getscheduler(worker)) - process.stdin.write(f"{worker}\n") - process.stdin.flush() - self.assertEqual("armed 1", process.stdout.readline().strip()) - for _ in range(100): + for round in range(100): os.kill(process.pid, signal.SIGURG) os.kill(process.pid, signal.SIGUSR1) - self.assertEqual(f"received {_ + 1}", process.stdout.readline().strip()) + self.assertEqual(f"received {round + 1}", process.stdout.readline().strip()) process.stdin.close() process.wait(timeout=10) output = process.stdout.read() values = dict(line.split() for line in output.splitlines()) - self.assertNotEqual(0, int(values["urg"])) self.assertNotEqual(0, int(values["usr"])) self.assertEqual(0, int(values["worker"])) - self.assertNotEqual(worker, int(values["urg"])) - self.assertNotEqual(worker, int(values["usr"])) finally: if process.poll() is None: process.kill() @@ -537,10 +647,33 @@ class ZramPreloadTest(unittest.TestCase): @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") def test_process_without_a_worker_trims_in_small_steps(self): + if os.geteuid() == 0: + self.skipTest("the thread limit does not apply to root") + out = self.run_program(None, 48, 0, "trim-fork") + self.assertTrue(16 * 1024 < out["nothread_trim"][0] <= 32 * 1024, out) + self.assertGreater(out["worker_trim"][0], 64 * 1024, out) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_a_single_threaded_process_trims_without_starting_a_thread(self): out = self.run_program(None, 48, 0, "trim-fork") self.assertTrue(16 * 1024 < out["inline_trim"][0] <= 32 * 1024, out) self.assertGreater(out["worker_trim"][0], 64 * 1024, out) + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_a_process_that_sandboxed_itself_trims_without_starting_a_thread(self): + out = self.run_program(None, 48, 0, "sandboxed") + self.assertTrue(16 * 1024 < out["sandboxed_trim"][0] <= 32 * 1024, out) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_a_thread_that_sandboxed_itself_trims_without_starting_a_thread(self): + out = self.run_program(None, 48, 0, "thread-sandboxed") + self.assertTrue(16 * 1024 < out["thread_sandboxed_trim"][0] <= 32 * 1024, out) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_the_worker_passes_a_filter_that_only_allows_thread_clones(self): + out = self.run_program(None, 48, 0, "filtered") + self.assertGreater(out["filtered_trim"][0], 40 * 1024, out) + def test_worker_acknowledges_back_to_back_requests(self): out = self.run_program(None, 48, 0, "burst") self.assertEqual([1], out["burstack"])