diff --git a/docs/development/low-memory.md b/docs/development/low-memory.md index 218d1fd8e..50e78363e 100644 --- a/docs/development/low-memory.md +++ b/docs/development/low-memory.md @@ -99,10 +99,16 @@ available. ## Threads -Page-out never runs on a thread of the game. The preload's signal handler only wakes its -`zr-worker` thread, which does the `madvise` calls. A forked process -without a worker trims inline in steps of at most 32 MiB per request and looks at no more -than 512 MiB of its address space each time. The page-out for a suspend and the read-back +When a worker can be started, page-out runs outside the game's threads. The preload's signal +handler starts a `zr-worker` thread for the request, which does the `madvise` calls and exits once no request +is left, so a process that is not paging out has no thread the preload added. Programs that +check they are single-threaded before they fork or call `unshare(CLONE_NEWUSER)`, such as +Chromium's zygote, keep working. The worker is a bare `clone` with the flags of a normal +thread, on a stack the preload maps itself, and makes only raw system calls, so starting it +from the signal handler is safe and seccomp filters that only allow thread clones accept it. A +process that has a single thread, or whose signalled thread added a seccomp filter after the +process started, as browser sandboxes do, gets no worker. When there is no worker, the process trims inline in steps of at most 32 MiB per request and +looks at no more than 512 MiB of its address space each time. The page-out for a suspend and the read-back after it are not limited there: they run once, while the process is being stopped or started, and cover all of it. The monitor is a shell loop in the session, outside every game process. diff --git a/tools/linuxfs/preload/inputudev.c b/tools/linuxfs/preload/inputudev.c index 823614bfd..3dcb5a104 100644 --- a/tools/linuxfs/preload/inputudev.c +++ b/tools/linuxfs/preload/inputudev.c @@ -1045,7 +1045,15 @@ static void *own_udev_symbol(const char *name) { } static void *dlvsym_any(const char *name) { +#if defined(__aarch64__) + static const char *versions[] = {"GLIBC_2.17", "GLIBC_2.34"}; +#elif defined(__x86_64__) + static const char *versions[] = {"GLIBC_2.2.5", "GLIBC_2.34"}; +#elif defined(__i386__) + static const char *versions[] = {"GLIBC_2.0", "GLIBC_2.34"}; +#else static const char *versions[] = {"GLIBC_2.17", "GLIBC_2.2.5", "GLIBC_2.0", "GLIBC_2.34"}; +#endif size_t i; void *resolved = NULL; diff --git a/tools/linuxfs/preload/zram.c b/tools/linuxfs/preload/zram.c index 551a6e148..ad13d8258 100644 --- a/tools/linuxfs/preload/zram.c +++ b/tools/linuxfs/preload/zram.c @@ -1,8 +1,9 @@ #define _GNU_SOURCE -#include +#pragma GCC optimize("no-stack-protector", "no-tree-loop-distribute-patterns") #include #include #include +#include #include #include #include @@ -10,6 +11,9 @@ #include #include #include +#if defined(__i386__) +#include +#endif #include @@ -31,13 +35,156 @@ #define ZR_INLINE_CHUNKS 512UL #define ZR_FAILED (~0UL) #define ZR_FUTEX_WAIT 0 -#define ZR_FUTEX_WAKE 1 +#define ZR_STACK (128UL << 10) +#define ZR_THREAD (CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | \ + CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID) +#define ZR_TEXT(x) #x +#define ZR_NR(x) ZR_TEXT(x) +#define ZR_CALL(n, a, b, c, d, e, f, ...) \ + zr_call((long)(n), (long)(a), (long)(b), (long)(c), (long)(d), (long)(e), (long)(f)) +#define zr_sys(...) ZR_CALL(__VA_ARGS__, 0, 0, 0, 0, 0, 0, 0) + +#if defined(__aarch64__) +#define ZR_MMAP SYS_mmap +__asm__(".pushsection .text\n" + ".p2align 2\n" + ".hidden zr_call\n" + ".type zr_call, %function\n" + "zr_call:\n" + " mov x8, x0\n" + " mov x0, x1\n" + " mov x1, x2\n" + " mov x2, x3\n" + " mov x3, x4\n" + " mov x4, x5\n" + " mov x5, x6\n" + " svc #0\n" + " ret\n" + ".size zr_call, . - zr_call\n" + ".hidden zr_clone\n" + ".type zr_clone, %function\n" + "zr_clone:\n" + " mov x5, x2\n" + " mov x2, x3\n" + " mov x3, x4\n" + " mov x4, x5\n" + " mov x8, #" ZR_NR(SYS_clone) "\n" + " svc #0\n" + " cbnz x0, 1f\n" + " mov x29, xzr\n" + " mov x30, xzr\n" + " bl zr_thread\n" + " mov x0, xzr\n" + " mov x8, #" ZR_NR(SYS_exit) "\n" + " svc #0\n" + " brk #0\n" + "1:\n" + " ret\n" + ".size zr_clone, . - zr_clone\n" + ".popsection\n"); +#elif defined(__x86_64__) +#define ZR_MMAP SYS_mmap +__asm__(".pushsection .text\n" + ".p2align 4\n" + ".hidden zr_call\n" + ".type zr_call, @function\n" + "zr_call:\n" + " mov %rdi, %rax\n" + " mov %rsi, %rdi\n" + " mov %rdx, %rsi\n" + " mov %rcx, %rdx\n" + " mov %r8, %r10\n" + " mov %r9, %r8\n" + " mov 8(%rsp), %r9\n" + " syscall\n" + " ret\n" + ".size zr_call, . - zr_call\n" + ".hidden zr_clone\n" + ".type zr_clone, @function\n" + "zr_clone:\n" + " mov %rdx, %r10\n" + " mov %rcx, %rdx\n" + " mov $" ZR_NR(SYS_clone) ", %eax\n" + " syscall\n" + " test %rax, %rax\n" + " jnz 1f\n" + " xor %ebp, %ebp\n" + " call zr_thread\n" + " xor %edi, %edi\n" + " mov $" ZR_NR(SYS_exit) ", %eax\n" + " syscall\n" + " hlt\n" + "1:\n" + " ret\n" + ".size zr_clone, . - zr_clone\n" + ".popsection\n"); +#elif defined(__i386__) +#define ZR_MMAP SYS_mmap2 +__asm__(".pushsection .text\n" + ".p2align 4\n" + ".hidden zr_call\n" + ".type zr_call, @function\n" + "zr_call:\n" + " push %ebp\n" + " push %edi\n" + " push %esi\n" + " push %ebx\n" + " mov 20(%esp), %eax\n" + " mov 24(%esp), %ebx\n" + " mov 28(%esp), %ecx\n" + " mov 32(%esp), %edx\n" + " mov 36(%esp), %esi\n" + " mov 40(%esp), %edi\n" + " mov 44(%esp), %ebp\n" + " int $0x80\n" + " pop %ebx\n" + " pop %esi\n" + " pop %edi\n" + " pop %ebp\n" + " ret\n" + ".size zr_call, . - zr_call\n" + ".hidden zr_clone\n" + ".type zr_clone, @function\n" + "zr_clone:\n" + " push %ebx\n" + " push %esi\n" + " push %edi\n" + " mov 16(%esp), %ebx\n" + " mov 20(%esp), %ecx\n" + " mov 24(%esp), %edi\n" + " mov 28(%esp), %edx\n" + " mov 32(%esp), %esi\n" + " mov $" ZR_NR(SYS_clone) ", %eax\n" + " int $0x80\n" + " test %eax, %eax\n" + " jnz 1f\n" + " xor %ebp, %ebp\n" + " call zr_thread\n" + " xor %ebx, %ebx\n" + " mov $" ZR_NR(SYS_exit) ", %eax\n" + " int $0x80\n" + " hlt\n" + "1:\n" + " pop %edi\n" + " pop %esi\n" + " pop %ebx\n" + " ret\n" + ".size zr_clone, . - zr_clone\n" + ".popsection\n"); +#else +#error "zram.c needs a raw system call for this architecture" +#endif + +long zr_call(long n, long a, long b, long c, long d, long e, long f) __attribute__((visibility("hidden"))); +long zr_clone(long flags, char *stack, int *child, int *parent, void *tls) __attribute__((visibility("hidden"))); +void zr_thread(void) __attribute__((visibility("hidden"))); -static long (*zr_sys)(long, ...); static unsigned long zr_page; -static int zr_busy; -static int zr_pending; -static int zr_worker; +static unsigned zr_requests; +static int zr_stack_busy; +static int zr_tid; +static long zr_filters; +static char *zr_stack; static char zr_maps[8192]; static uint64_t zr_entries[ZR_BATCH]; static struct iovec zr_iov[ZR_BATCH]; @@ -54,6 +201,18 @@ static uint16_t zr_gen = 1; static unsigned long zr_cursor; static unsigned long zr_visits; +static int zr_prefix(const char *text, const char *prefix) { + while (*prefix) + if (*text++ != *prefix++) return 0; + return 1; +} + +static char *zr_find(char *text, char c, size_t n) { + for (; n > 0; n--, text++) + if (*text == c) return text; + return NULL; +} + static int zr_open(const char *path) { return (int)zr_sys(SYS_openat, AT_FDCWD, path, O_RDONLY | O_CLOEXEC, 0); } @@ -103,7 +262,7 @@ static int zr_eligible(const char *line, int compress, unsigned long *start, uns while (*p && *p != ' ') p++; while (*p == ' ') p++; } - return *p == 0 || strncmp(p, "[heap]", 6) == 0 || strncmp(p, "[anon:", 6) == 0; + return *p == 0 || zr_prefix(p, "[heap]") || zr_prefix(p, "[anon:"); } static void zr_read_back(int pagemap, unsigned long start, unsigned long end) { @@ -137,7 +296,7 @@ static unsigned long zr_count(int pagemap, unsigned long start, unsigned long en do { if (zr_sys(SYS_lseek, pagemap, (long)(start / zr_page * sizeof(uint64_t)), SEEK_SET) < 0) return ZR_FAILED; got = zr_sys(SYS_read, pagemap, zr_entries, size); - } while (got < 0 && errno == EINTR); + } while (got == -EINTR); if (got != size) return ZR_FAILED; for (long i = 0; i < got / (long)sizeof(uint64_t); i++) if (zr_entries[i] & bit) count++; @@ -157,8 +316,7 @@ static struct zr_slot *zr_slot(unsigned long addr, int create) { } } if (!create) return NULL; - memset(&zr_slots[home], 0, sizeof(zr_slots[home])); - zr_slots[home].key = key; + zr_slots[home] = (struct zr_slot){.key = key}; return &zr_slots[home]; } @@ -228,13 +386,13 @@ static unsigned long zr_walk(int mode, unsigned long want, unsigned long lo, uns size_t have = 0; while (done < want && zr_visits > 0) { long n = zr_sys(SYS_read, maps, zr_maps + have, sizeof(zr_maps) - 1 - have); - if (n < 0 && errno == EINTR) continue; + if (n == -EINTR) continue; if (n <= 0) break; have += (size_t)n; zr_maps[have] = 0; char *line = zr_maps; char *nl; - while (done < want && zr_visits > 0 && (nl = memchr(line, '\n', have - (size_t)(line - zr_maps))) != NULL) { + while (done < want && zr_visits > 0 && (nl = zr_find(line, '\n', have - (size_t)(line - zr_maps))) != NULL) { *nl = 0; unsigned long s, e; if (zr_eligible(line, mode != 0, &s, &e) && e > s) { @@ -245,7 +403,7 @@ static unsigned long zr_walk(int mode, unsigned long want, unsigned long lo, uns line = nl + 1; } have -= (size_t)(line - zr_maps); - memmove(zr_maps, line, have); + for (size_t i = 0; i < have; i++) zr_maps[i] = line[i]; if (have == sizeof(zr_maps) - 1) have = 0; } if (pagemap >= 0) zr_sys(SYS_close, pagemap); @@ -291,15 +449,88 @@ static void zr_request(int worker) { zr_ack((long)(done * (zr_page / 1024)), 1); } -static void *zr_worker_main(void *unused) { - (void)unused; - prctl(PR_SET_NAME, "zr-worker", 0, 0, 0); - for (;;) { - while (!__atomic_exchange_n(&zr_pending, 0, __ATOMIC_SEQ_CST)) - zr_sys(SYS_futex, &zr_pending, ZR_FUTEX_WAIT, 0, NULL, NULL, 0); - zr_request(1); +static void zr_drain(int worker) { + unsigned seen; + do { + seen = __atomic_load_n(&zr_requests, __ATOMIC_ACQUIRE); + zr_request(worker); + } while (!__atomic_compare_exchange_n(&zr_requests, &seen, 0, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)); +} + +void zr_thread(void) { + zr_sys(SYS_prctl, PR_SET_NAME, "zr-worker"); + zr_drain(1); +} + +static void *zr_tls(void *desc) { +#if defined(__aarch64__) + (void)desc; + return __builtin_thread_pointer(); +#elif defined(__x86_64__) + (void)desc; + void *self; + __asm__("mov %%fs:0, %0" : "=r"(self)); + return self; +#else + struct user_desc *entry = desc; + unsigned short selector; + __asm__("mov %%gs, %0" : "=r"(selector)); + entry->entry_number = selector >> 3; + return zr_sys(SYS_get_thread_area, entry) == 0 ? entry : NULL; +#endif +} + +static long zr_field(const char *text, const char *key) { + for (const char *line = text; *line; line++) { + if ((line == text || line[-1] == '\n') && zr_prefix(line, key)) { + long value = 0; + for (const char *k = key; *k; k++) line++; + while (*line == ' ' || *line == '\t') line++; + for (; *line >= '0' && *line <= '9'; line++) value = value * 10 + (*line - '0'); + return value; + } } - return NULL; + return -1; +} + +static int zr_status(long *threads, long *filters) { + int fd = zr_open("/proc/thread-self/status"); + if (fd < 0) return 0; + long have = 0, n; + while (have < (long)sizeof(zr_maps) - 1 && + ((n = zr_sys(SYS_read, fd, zr_maps + have, sizeof(zr_maps) - 1 - (size_t)have)) > 0 || n == -EINTR)) + if (n > 0) have += n; + zr_sys(SYS_close, fd); + zr_maps[have] = 0; + *threads = zr_field(zr_maps, "Threads:"); + *filters = zr_field(zr_maps, "Seccomp_filters:"); + return have > 0; +} + +static int zr_start(void) { + long threads, filters; + if (!zr_status(&threads, &filters) || threads < 2 || filters > zr_filters) return 0; + if (zr_stack == NULL) { + long base = zr_sys(ZR_MMAP, 0, ZR_STACK, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK | MAP_NORESERVE, -1, 0); + if ((unsigned long)base > -4096UL) return 0; + zr_stack = (char *)base; + } + struct { long sec, nsec; } wait = {0, 5000000}; + for (int i = 0; i < 20 && __atomic_load_n(&zr_stack_busy, __ATOMIC_ACQUIRE); i++) + zr_sys(SYS_futex, &zr_stack_busy, ZR_FUTEX_WAIT, 1, &wait, NULL, 0); + if (__atomic_load_n(&zr_stack_busy, __ATOMIC_ACQUIRE)) return 0; + uint64_t desc[4] = {0}; + void *tls = zr_tls(desc); + if (tls == NULL) return 0; + uint64_t all = ~0ULL, previous; + if (zr_sys(SYS_rt_sigprocmask, SIG_SETMASK, &all, &previous, sizeof(all)) != 0) return 0; + __atomic_store_n(&zr_stack_busy, 1, __ATOMIC_RELEASE); + long tid = zr_clone(ZR_THREAD, zr_stack + ZR_STACK, &zr_stack_busy, &zr_tid, tls); + zr_sys(SYS_rt_sigprocmask, SIG_SETMASK, &previous, NULL, sizeof(previous)); + if (tid > 0) return 1; + __atomic_store_n(&zr_stack_busy, 0, __ATOMIC_RELEASE); + return 0; } static void zr_handler(int sig, siginfo_t *info, void *context) { @@ -307,34 +538,23 @@ static void zr_handler(int sig, siginfo_t *info, void *context) { (void)context; if (info == NULL || info->si_code != SI_USER) return; int saved = errno; - __atomic_store_n(&zr_pending, 1, __ATOMIC_SEQ_CST); - if (__atomic_load_n(&zr_worker, __ATOMIC_ACQUIRE)) { - zr_sys(SYS_futex, &zr_pending, ZR_FUTEX_WAKE, 1, NULL, NULL, 0); - } else while (__atomic_load_n(&zr_pending, __ATOMIC_SEQ_CST) - && !__atomic_exchange_n(&zr_busy, 1, __ATOMIC_SEQ_CST)) { - __atomic_store_n(&zr_pending, 0, __ATOMIC_SEQ_CST); - zr_request(0); - __atomic_store_n(&zr_busy, 0, __ATOMIC_SEQ_CST); - } + if (__atomic_fetch_add(&zr_requests, 1, __ATOMIC_ACQ_REL) == 0 && !zr_start()) zr_drain(0); errno = saved; } static void zr_child(void) { - __atomic_store_n(&zr_busy, 0, __ATOMIC_SEQ_CST); - __atomic_store_n(&zr_pending, 0, __ATOMIC_SEQ_CST); - __atomic_store_n(&zr_worker, 0, __ATOMIC_RELEASE); + __atomic_store_n(&zr_requests, 0, __ATOMIC_RELEASE); + __atomic_store_n(&zr_stack_busy, 0, __ATOMIC_RELEASE); } __attribute__((constructor)) static void zr_init(void) { - void *libc = dlopen("libc.so.6", RTLD_LAZY | RTLD_NOLOAD); - if (libc == NULL) return; - zr_sys = (long (*)(long, ...))dlsym(libc, "syscall"); - if (zr_sys == NULL) return; int fd = zr_open(ZR_CTL); if (fd < 0) return; zr_sys(SYS_close, fd); long page = sysconf(_SC_PAGESIZE); zr_page = page > 0 ? (unsigned long)page : 4096UL; + long threads; + if (!zr_status(&threads, &zr_filters)) zr_filters = -1; struct sigaction old; if (sigaction(SIGURG, NULL, &old) != 0) return; if ((old.sa_flags & SA_SIGINFO) || old.sa_handler != SIG_DFL) return; @@ -343,21 +563,5 @@ __attribute__((constructor)) static void zr_init(void) { sa.sa_sigaction = zr_handler; sa.sa_flags = SA_SIGINFO | SA_RESTART; sigemptyset(&sa.sa_mask); - if (sigaction(SIGURG, &sa, NULL) == 0) { - pthread_atfork(NULL, NULL, zr_child); - pthread_attr_t attr; - if (pthread_attr_init(&attr) == 0) { - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_t thread; - sigset_t blocked; - sigset_t previous; - if (sigfillset(&blocked) == 0 && - pthread_sigmask(SIG_SETMASK, &blocked, &previous) == 0) { - if (pthread_create(&thread, &attr, zr_worker_main, NULL) == 0) - __atomic_store_n(&zr_worker, 1, __ATOMIC_RELEASE); - pthread_sigmask(SIG_SETMASK, &previous, NULL); - } - pthread_attr_destroy(&attr); - } - } + if (sigaction(SIGURG, &sa, NULL) == 0) pthread_atfork(NULL, NULL, zr_child); } diff --git a/tools/tests/test_inputudev_preload.py b/tools/tests/test_inputudev_preload.py index c1d970fd1..8e4cf66e8 100644 --- a/tools/tests/test_inputudev_preload.py +++ b/tools/tests/test_inputudev_preload.py @@ -77,6 +77,66 @@ char *realpath(const char *path, char *resolved) { } """ +LOCKED = r""" +#include +#include + +extern int busy; + +__attribute__((constructor)) static void start(void) { + busy = 1; + syscall(SYS_getpid); + busy = 0; +} +""" + +ARENA = r""" +#include +#include +#include + +static _Alignas(64) unsigned char arena[1 << 23]; +static size_t used; +int busy; +static int busy_allocations; + +void *memalign(size_t align, size_t size) { + uintptr_t start; + if (busy) busy_allocations++; + if (align < 16) align = 16; + start = ((uintptr_t)(arena + used) + 16 + align - 1) & ~(uintptr_t)(align - 1); + if (start + size > (uintptr_t)(arena + sizeof(arena))) return NULL; + ((size_t *)start)[-1] = size; + used = start + size - (uintptr_t)arena; + return (void *)start; +} + +void *malloc(size_t size) { return memalign(16, size); } +void *aligned_alloc(size_t align, size_t size) { return memalign(align, size); } +int posix_memalign(void **out, size_t align, size_t size) { return (*out = memalign(align, size)) ? 0 : 12; } +void free(void *p) { (void)p; } + +void *calloc(size_t n, size_t size) { + void *p = malloc(n * size); + if (p) memset(p, 0, n * size); + return p; +} + +void *realloc(void *p, size_t size) { + void *q = malloc(size); + if (p && q) { + size_t old = ((size_t *)p)[-1]; + memcpy(q, p, old < size ? old : size); + } + return q; +} + +int main(void) { + printf("allocations %d\n", busy_allocations); + return 0; +} +""" + UDEV = r""" #include #include @@ -414,7 +474,7 @@ class InputUdevPreloadTest(unittest.TestCase): if "__GLIBC__" not in compile_c("-dM", "-E", "-x", "c", "-include", "limits.h", "/dev/null"): raise unittest.SkipTest("needs a glibc toolchain") sources = (("once.c", ONCE), ("first.c", FIRST), ("second.c", SECOND), ("chain.c", CHAIN), ("mark.c", MARK), - ("udev.c", UDEV), ("program.c", PROGRAM)) + ("udev.c", UDEV), ("program.c", PROGRAM), ("locked.c", LOCKED), ("arena.c", ARENA)) for name, text in sources: (build / name).write_text(text) cls.lib = cls.dir / "libddsession.so" @@ -427,6 +487,11 @@ class InputUdevPreloadTest(unittest.TestCase): compile_c("-shared", "-fPIC", "-O2", "-o", str(cls.udev), str(build / "udev.c")) cls.program = cls.dir / "program" compile_c("-O0", "-o", str(cls.program), str(build / "program.c"), str(cls.dir / "libfirst.so"), "-ldl") + compile_c("-shared", "-fPIC", "-O0", "-o", str(cls.dir / "liblocked.so"), str(build / "locked.c")) + cls.arena = cls.dir / "arena" + compile_c("-O0", "-rdynamic", "-o", str(cls.arena), str(build / "arena.c"), "-Wl,--no-as-needed", + str(cls.dir / "liblocked.so"), + f"-Wl,-rpath,{cls.dir}") cls.pads = cls.pad_dir("pads", "event0") @classmethod @@ -449,6 +514,12 @@ class InputUdevPreloadTest(unittest.TestCase): def test_a_lookup_from_another_preloads_initialiser_does_not_reenter_it(self): self.assertEqual(self.run_program("once", "libonce.so"), ["readlink 1"]) + def test_a_first_wrapped_call_inside_the_programs_allocator_does_not_allocate(self): + result = subprocess.run([str(self.arena)], env=dict(os.environ, LD_PRELOAD=str(self.lib)), capture_output=True, + text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), ["allocations 0"]) + def test_the_real_lookup_still_sees_the_original_caller(self): self.assertEqual(self.run_program("next", "libfirst.so", "libsecond.so"), ["next 2"]) diff --git a/tools/tests/test_zram_preload.py b/tools/tests/test_zram_preload.py index bf1e8391c..8a9f81348 100644 --- a/tools/tests/test_zram_preload.py +++ b/tools/tests/test_zram_preload.py @@ -18,7 +18,14 @@ PROGRAM = r""" #include #include #include +#include +#include +#include +#include +#include #include +#include +#include #include #include #include @@ -31,14 +38,15 @@ static unsigned char *shared; static volatile sig_atomic_t urg_tid; static volatile sig_atomic_t usr_tid; static int urg_count, usr_count; -static int worker_tid; static int worker_delivery; _Static_assert(__atomic_always_lock_free(sizeof(int), 0), "signal state must be lock-free"); static void signal_tid(int sig) { sig_atomic_t tid = (sig_atomic_t)syscall(SYS_gettid); - if (tid == __atomic_load_n(&worker_tid, __ATOMIC_RELAXED)) + char name[16] = {0}; + prctl(PR_GET_NAME, name, 0, 0, 0); + if (!strcmp(name, "zr-worker")) __atomic_store_n(&worker_delivery, 1, __ATOMIC_RELAXED); if (sig == SIGURG) { __atomic_store_n(&urg_tid, tid, __ATOMIC_RELAXED); @@ -123,6 +131,75 @@ static long trim(char mode, long kb) { exit(5); } +static int workers(void) { + DIR *dir = opendir("/proc/self/task"); + struct dirent *entry; + int count = 0; + while (dir && (entry = readdir(dir)) != NULL) { + if (entry->d_name[0] == '.') continue; + char path[64], name[32] = {0}; + snprintf(path, sizeof(path), "/proc/self/task/%s/comm", entry->d_name); + FILE *comm = fopen(path, "r"); + if (comm && fgets(name, sizeof(name), comm) && !strcmp(name, "zr-worker\n")) count++; + if (comm) fclose(comm); + } + if (dir) closedir(dir); + return count; +} + +static void without_threads(void) { + struct rlimit limit; + if (getrlimit(RLIMIT_NPROC, &limit)) _exit(15); + limit.rlim_cur = 0; + if (setrlimit(RLIMIT_NPROC, &limit)) _exit(15); +} + +static void *park(void *arg) { + (void)arg; + for (;;) pause(); + return NULL; +} + +static void parked(void) { + pthread_t thread; + sigset_t urg, previous; + sigemptyset(&urg); + sigaddset(&urg, SIGURG); + pthread_sigmask(SIG_BLOCK, &urg, &previous); + if (pthread_create(&thread, NULL, park, NULL)) _exit(16); + pthread_sigmask(SIG_SETMASK, &previous, NULL); +} + +static void filter_clone(unsigned long allowed) { + struct sock_filter rules[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_clone, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, allowed, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = {sizeof(rules) / sizeof(rules[0]), rules}; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) _exit(17); +} + +static volatile int sandbox_ready; + +static void *sandbox(void *arg) { + (void)arg; + sigset_t urg; + sigemptyset(&urg); + sigaddset(&urg, SIGURG); + filter_clone(0); + pthread_sigmask(SIG_UNBLOCK, &urg, NULL); + sandbox_ready = 1; + for (;;) pause(); + return NULL; +} + +static const unsigned long thread_clone = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | + CLONE_SYSVSEM | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID; + static uint64_t sum(const unsigned char *p, size_t n) { uint64_t s = 0; for (size_t i = 0; i < n; i += 64) s = s * 31 + p[i]; @@ -155,6 +232,7 @@ int main(int argc, char **argv) { memset(shared, 0, size); pthread_t t[2]; for (uintptr_t i = 0; i < 2; i++) pthread_create(&t[i], NULL, writer, (void *)i); + parked(); struct sigaction sa; sigaction(SIGURG, NULL, &sa); printf("handler %d\n", (sa.sa_flags & SA_SIGINFO) ? 1 : 0); @@ -164,27 +242,21 @@ int main(int argc, char **argv) { memset(&signal_action, 0, sizeof(signal_action)); signal_action.sa_handler = signal_tid; sigemptyset(&signal_action.sa_mask); - sigaction(SIGURG, &signal_action, NULL); sigaction(SIGUSR1, &signal_action, NULL); + int fd = open(ctl, O_WRONLY | O_TRUNC); + if (fd < 0 || write(fd, "1", 1) != 1) return 10; + close(fd); printf("ready %d\n", getpid()); fflush(stdout); - int id; - if (scanf("%d", &id) != 1) return 10; - __atomic_store_n(&worker_tid, id, __ATOMIC_RELAXED); - printf("armed 1\n"); - fflush(stdout); for (int round = 1; round <= 100; round++) { int i; - for (i = 0; i < 5000 && - (__atomic_load_n(&urg_count, __ATOMIC_ACQUIRE) < round || - __atomic_load_n(&usr_count, __ATOMIC_ACQUIRE) < round); i++) usleep(1000); + for (i = 0; i < 5000 && __atomic_load_n(&usr_count, __ATOMIC_ACQUIRE) < round; i++) usleep(1000); if (i == 5000) return 11; printf("received %d\n", round); fflush(stdout); } stop = 1; for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); - printf("urg %d\n", __atomic_load_n(&urg_tid, __ATOMIC_RELAXED)); printf("usr %d\n", __atomic_load_n(&usr_tid, __ATOMIC_RELAXED)); printf("worker %d\n", __atomic_load_n(&worker_delivery, __ATOMIC_RELAXED)); return 0; @@ -199,6 +271,13 @@ int main(int argc, char **argv) { int status_code; waitpid(child, &status_code, 0); printf("childack %d\n", WIFEXITED(status_code) && WEXITSTATUS(status_code) == 0); + } else if (!strcmp(argv[4], "threads")) { + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + printf("idle %d\n", workers()); + request('1'); + printf("after %d\n", workers()); + return 0; } else if (!strcmp(argv[4], "trim")) { stop = 1; for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); @@ -271,8 +350,47 @@ int main(int argc, char **argv) { _exit(0); } waitpid(child, NULL, 0); + child = fork(); + if (child < 0) exit(6); + if (child == 0) { + parked(); + without_threads(); + for (size_t i = 0; i < size; i += page) data[i]++; + printf("nothread_trim %ld\n", trim('2', 1L << 30)); + fflush(stdout); + _exit(0); + } + waitpid(child, NULL, 0); printf("worker_trim %ld\n", trim('2', 1L << 30)); return 0; + } else if (!strcmp(argv[4], "sandboxed")) { + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + filter_clone(thread_clone); + printf("sandboxed_trim %ld\n", trim('2', 1L << 30)); + return 0; + } else if (!strcmp(argv[4], "thread-sandboxed")) { + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + sigset_t urg; + sigemptyset(&urg); + sigaddset(&urg, SIGURG); + pthread_sigmask(SIG_BLOCK, &urg, NULL); + pthread_t thread; + if (pthread_create(&thread, NULL, sandbox, NULL)) exit(16); + while (!sandbox_ready) usleep(1000); + printf("thread_sandboxed_trim %ld\n", trim('2', 1L << 30)); + return 0; + } else if (!strcmp(argv[4], "filtered")) { + fflush(stdout); + filter_clone(thread_clone); + execl("/proc/self/exe", argv[0], argv[1], argv[2], "0", "filtered-trim", (char *)NULL); + return 18; + } else if (!strcmp(argv[4], "filtered-trim")) { + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + printf("filtered_trim %ld\n", trim('2', 1L << 30)); + return 0; } else if (!strcmp(argv[4], "burst")) { send_request('0'); usleep(10000); @@ -283,6 +401,7 @@ int main(int argc, char **argv) { pid_t child = fork(); if (child < 0) exit(8); if (child == 0) { + without_threads(); if (!strcmp(argv[4], "timed-inline-single")) { if (munmap(data, size) || munmap(shared, size)) _exit(12); data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -431,6 +550,11 @@ class ZramPreloadTest(unittest.TestCase): self.assertEqual([1], out["intact"]) self.assertEqual(6, len(out["out"])) + def test_a_process_keeps_no_extra_thread_while_nothing_is_paged_out(self): + out = self.run_program(None, 8, 0, "threads") + self.assertEqual([0], out["idle"]) + self.assertEqual([0], out["after"]) + def test_process_signals_do_not_reach_zram_worker(self): environment = dict(os.environ, LD_PRELOAD=str(self.lib)) process = subprocess.Popen([str(self.steam), str(self.ctl), "1", "0", "signals"], @@ -439,30 +563,16 @@ class ZramPreloadTest(unittest.TestCase): process.stdout.readline() ready = process.stdout.readline().split() self.assertEqual("ready", ready[0]) - worker = None - task_dir = Path("/proc") / ready[1] / "task" - for task in task_dir.iterdir(): - if (task / "comm").read_text().strip() == "zr-worker": - worker = int(task.name) - break - self.assertIsNotNone(worker) - self.assertEqual(os.sched_getscheduler(process.pid), os.sched_getscheduler(worker)) - process.stdin.write(f"{worker}\n") - process.stdin.flush() - self.assertEqual("armed 1", process.stdout.readline().strip()) - for _ in range(100): + for round in range(100): os.kill(process.pid, signal.SIGURG) os.kill(process.pid, signal.SIGUSR1) - self.assertEqual(f"received {_ + 1}", process.stdout.readline().strip()) + self.assertEqual(f"received {round + 1}", process.stdout.readline().strip()) process.stdin.close() process.wait(timeout=10) output = process.stdout.read() values = dict(line.split() for line in output.splitlines()) - self.assertNotEqual(0, int(values["urg"])) self.assertNotEqual(0, int(values["usr"])) self.assertEqual(0, int(values["worker"])) - self.assertNotEqual(worker, int(values["urg"])) - self.assertNotEqual(worker, int(values["usr"])) finally: if process.poll() is None: process.kill() @@ -537,10 +647,33 @@ class ZramPreloadTest(unittest.TestCase): @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") def test_process_without_a_worker_trims_in_small_steps(self): + if os.geteuid() == 0: + self.skipTest("the thread limit does not apply to root") + out = self.run_program(None, 48, 0, "trim-fork") + self.assertTrue(16 * 1024 < out["nothread_trim"][0] <= 32 * 1024, out) + self.assertGreater(out["worker_trim"][0], 64 * 1024, out) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_a_single_threaded_process_trims_without_starting_a_thread(self): out = self.run_program(None, 48, 0, "trim-fork") self.assertTrue(16 * 1024 < out["inline_trim"][0] <= 32 * 1024, out) self.assertGreater(out["worker_trim"][0], 64 * 1024, out) + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_a_process_that_sandboxed_itself_trims_without_starting_a_thread(self): + out = self.run_program(None, 48, 0, "sandboxed") + self.assertTrue(16 * 1024 < out["sandboxed_trim"][0] <= 32 * 1024, out) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_a_thread_that_sandboxed_itself_trims_without_starting_a_thread(self): + out = self.run_program(None, 48, 0, "thread-sandboxed") + self.assertTrue(16 * 1024 < out["thread_sandboxed_trim"][0] <= 32 * 1024, out) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_the_worker_passes_a_filter_that_only_allows_thread_clones(self): + out = self.run_program(None, 48, 0, "filtered") + self.assertGreater(out["filtered_trim"][0], 40 * 1024, out) + def test_worker_acknowledges_back_to_back_requests(self): out = self.run_program(None, 48, 0, "burst") self.assertEqual([1], out["burstack"])