From 812e4923028ede9119841d311b1b71f21e314ce1 Mon Sep 17 00:00:00 2001 From: MaxsTechReview Date: Tue, 17 Nov 2026 11:24:05 -0500 Subject: [PATCH] fix(zram): page game memory out on a worker thread, not the signalled game thread (#2335) * fix: move zram page-out off signal thread * fix: drain pending zram requests * Time the async and inline page-out runs on the same workload * fix(zram): keep pageout worker responsive under load * test(zram): synchronize signal delivery before fixture shutdown * test(zram): compare matched single-request timings * test(zram): correct single request timing control * test(zram): calibrate single request timing together * test(zram): fail the timed run when the acknowledgement never arrives --- tools/linuxfs/preload/zram.c | 52 +++++++- tools/tests/test_zram_preload.py | 219 ++++++++++++++++++++++++++++++- 2 files changed, 263 insertions(+), 8 deletions(-) diff --git a/tools/linuxfs/preload/zram.c b/tools/linuxfs/preload/zram.c index af62c29413..e271668bea 100644 --- a/tools/linuxfs/preload/zram.c +++ b/tools/linuxfs/preload/zram.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -20,11 +21,14 @@ #endif #define ZR_SWAPPED (1ULL << 62) #define ZR_BATCH 512 +#define ZR_FUTEX_WAIT 0 +#define ZR_FUTEX_WAKE 1 static long (*zr_sys)(long, ...); static unsigned long zr_page; static int zr_busy; static int zr_pending; +static int zr_worker; static char zr_maps[8192]; static uint64_t zr_entries[ZR_BATCH]; static struct iovec zr_iov[ZR_BATCH]; @@ -143,19 +147,36 @@ static void zr_ack(int done) { zr_sys(SYS_close, fd); } +static void zr_request(void) { + int mode = zr_mode(); + if (mode == 1) zr_ack(0); + if (mode >= 0) zr_walk(mode); + if (mode == 1) zr_ack(1); +} + +static void *zr_worker_main(void *unused) { + (void)unused; + prctl(PR_SET_NAME, "zr-worker", 0, 0, 0); + for (;;) { + while (!__atomic_exchange_n(&zr_pending, 0, __ATOMIC_SEQ_CST)) + zr_sys(SYS_futex, &zr_pending, ZR_FUTEX_WAIT, 0, NULL, NULL, 0); + zr_request(); + } + return NULL; +} + static void zr_handler(int sig, siginfo_t *info, void *context) { (void)sig; (void)context; if (info == NULL || info->si_code != SI_USER) return; int saved = errno; __atomic_store_n(&zr_pending, 1, __ATOMIC_SEQ_CST); - while (__atomic_load_n(&zr_pending, __ATOMIC_SEQ_CST) - && !__atomic_exchange_n(&zr_busy, 1, __ATOMIC_SEQ_CST)) { + if (__atomic_load_n(&zr_worker, __ATOMIC_ACQUIRE)) { + zr_sys(SYS_futex, &zr_pending, ZR_FUTEX_WAKE, 1, NULL, NULL, 0); + } else while (__atomic_load_n(&zr_pending, __ATOMIC_SEQ_CST) + && !__atomic_exchange_n(&zr_busy, 1, __ATOMIC_SEQ_CST)) { __atomic_store_n(&zr_pending, 0, __ATOMIC_SEQ_CST); - int mode = zr_mode(); - if (mode == 1) zr_ack(0); - if (mode >= 0) zr_walk(mode); - if (mode == 1) zr_ack(1); + zr_request(); __atomic_store_n(&zr_busy, 0, __ATOMIC_SEQ_CST); } errno = saved; @@ -164,6 +185,7 @@ static void zr_handler(int sig, siginfo_t *info, void *context) { static void zr_child(void) { __atomic_store_n(&zr_busy, 0, __ATOMIC_SEQ_CST); __atomic_store_n(&zr_pending, 0, __ATOMIC_SEQ_CST); + __atomic_store_n(&zr_worker, 0, __ATOMIC_RELEASE); } __attribute__((constructor)) static void zr_init(void) { @@ -184,5 +206,21 @@ __attribute__((constructor)) static void zr_init(void) { sa.sa_sigaction = zr_handler; sa.sa_flags = SA_SIGINFO | SA_RESTART; sigemptyset(&sa.sa_mask); - if (sigaction(SIGURG, &sa, NULL) == 0) pthread_atfork(NULL, NULL, zr_child); + if (sigaction(SIGURG, &sa, NULL) == 0) { + pthread_atfork(NULL, NULL, zr_child); + pthread_attr_t attr; + if (pthread_attr_init(&attr) == 0) { + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + pthread_t thread; + sigset_t blocked; + sigset_t previous; + if (sigfillset(&blocked) == 0 && + pthread_sigmask(SIG_SETMASK, &blocked, &previous) == 0) { + if (pthread_create(&thread, &attr, zr_worker_main, NULL) == 0) + __atomic_store_n(&zr_worker, 1, __ATOMIC_RELEASE); + pthread_sigmask(SIG_SETMASK, &previous, NULL); + } + pthread_attr_destroy(&attr); + } + } } diff --git a/tools/tests/test_zram_preload.py b/tools/tests/test_zram_preload.py index 88d5c73020..59bc500ee4 100644 --- a/tools/tests/test_zram_preload.py +++ b/tools/tests/test_zram_preload.py @@ -1,5 +1,6 @@ import os from pathlib import Path +import signal import shutil import subprocess import tempfile @@ -9,6 +10,7 @@ PRELOAD = Path(__file__).resolve().parents[1] / "linuxfs/preload" PROGRAM = r""" #define _GNU_SOURCE +#include #include #include #include @@ -17,12 +19,36 @@ PROGRAM = r""" #include #include #include +#include +#include +#include #include static const char *ctl; static size_t size, page; static volatile int stop; static unsigned char *shared; +static volatile sig_atomic_t urg_tid; +static volatile sig_atomic_t usr_tid; +static int urg_count, usr_count; +static int worker_tid; +static int worker_delivery; + +_Static_assert(__atomic_always_lock_free(sizeof(int), 0), "signal state must be lock-free"); + +static void signal_tid(int sig) { + sig_atomic_t tid = (sig_atomic_t)syscall(SYS_gettid); + if (tid == __atomic_load_n(&worker_tid, __ATOMIC_RELAXED)) + __atomic_store_n(&worker_delivery, 1, __ATOMIC_RELAXED); + if (sig == SIGURG) { + __atomic_store_n(&urg_tid, tid, __ATOMIC_RELAXED); + __atomic_add_fetch(&urg_count, 1, __ATOMIC_RELEASE); + } + if (sig == SIGUSR1) { + __atomic_store_n(&usr_tid, tid, __ATOMIC_RELAXED); + __atomic_add_fetch(&usr_count, 1, __ATOMIC_RELEASE); + } +} static long status(const char *key) { FILE *f = fopen("/proc/self/status", "r"); @@ -34,13 +60,37 @@ static long status(const char *key) { return v; } -static void request(char mode) { +static void send_request(char mode) { int fd = open(ctl, O_WRONLY | O_TRUNC); if (fd < 0 || write(fd, &mode, 1) != 1) exit(3); close(fd); kill(getpid(), SIGURG); } +static void wait_ack(void) { + char ack[256]; + snprintf(ack, sizeof(ack), "%s.ack.%d", ctl, getpid()); + for (int i = 0; i < 1200; i++) { + FILE *f = fopen(ack, "r"); + int done = 0; + if (f) { + done = fgetc(f) == '1'; + fclose(f); + } + if (done) return; + usleep(10000); + } + exit(5); +} + +static void request(char mode) { + send_request(mode); + if (mode == '1') { + wait_ack(); + } + usleep(1000000); +} + static uint64_t sum(const unsigned char *p, size_t n) { uint64_t s = 0; for (size_t i = 0; i < n; i += 64) s = s * 31 + p[i]; @@ -77,6 +127,106 @@ int main(int argc, char **argv) { sigaction(SIGURG, NULL, &sa); printf("handler %d\n", (sa.sa_flags & SA_SIGINFO) ? 1 : 0); if (argc > 4) { + if (!strcmp(argv[4], "signals")) { + struct sigaction signal_action; + memset(&signal_action, 0, sizeof(signal_action)); + signal_action.sa_handler = signal_tid; + sigemptyset(&signal_action.sa_mask); + sigaction(SIGURG, &signal_action, NULL); + sigaction(SIGUSR1, &signal_action, NULL); + printf("ready %d\n", getpid()); + fflush(stdout); + int id; + if (scanf("%d", &id) != 1) return 10; + __atomic_store_n(&worker_tid, id, __ATOMIC_RELAXED); + printf("armed 1\n"); + fflush(stdout); + for (int round = 1; round <= 100; round++) { + int i; + for (i = 0; i < 5000 && + (__atomic_load_n(&urg_count, __ATOMIC_ACQUIRE) < round || + __atomic_load_n(&usr_count, __ATOMIC_ACQUIRE) < round); i++) usleep(1000); + if (i == 5000) return 11; + printf("received %d\n", round); + fflush(stdout); + } + stop = 1; + for (int i = 0; i < 2; i++) pthread_join(t[i], NULL); + printf("urg %d\n", __atomic_load_n(&urg_tid, __ATOMIC_RELAXED)); + printf("usr %d\n", __atomic_load_n(&usr_tid, __ATOMIC_RELAXED)); + printf("worker %d\n", __atomic_load_n(&worker_delivery, __ATOMIC_RELAXED)); + return 0; + } else if (!strcmp(argv[4], "fork")) { + pid_t child = fork(); + if (child < 0) exit(6); + if (child == 0) { + request('0'); + request('1'); + _exit(0); + } + int status_code; + waitpid(child, &status_code, 0); + printf("childack %d\n", WIFEXITED(status_code) && WEXITSTATUS(status_code) == 0); + } else if (!strcmp(argv[4], "burst")) { + send_request('0'); + usleep(10000); + send_request('1'); + wait_ack(); + printf("burstack 1\n"); + } else if (!strcmp(argv[4], "timed-inline") || !strcmp(argv[4], "timed-inline-single")) { + pid_t child = fork(); + if (child < 0) exit(8); + if (child == 0) { + if (!strcmp(argv[4], "timed-inline-single")) { + if (munmap(data, size) || munmap(shared, size)) _exit(12); + data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + shared = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (data == MAP_FAILED || shared == MAP_FAILED) _exit(13); + for (size_t i = 0; i < size; i++) data[i] = (unsigned char)(i * 7 + i / 4096); + memset(shared, 0, size); + } + struct timespec start, end; + int fd = open(ctl, O_WRONLY | O_TRUNC); + if (fd < 0 || write(fd, "1", 1) != 1) _exit(9); + close(fd); + clock_gettime(CLOCK_MONOTONIC, &start); + int requests = !strcmp(argv[4], "timed-inline-single") ? 1 : 64; + for (int i = 0; i < requests; i++) { + send_request('1'); + } + clock_gettime(CLOCK_MONOTONIC, &end); + long elapsed = (end.tv_sec - start.tv_sec) * 1000L + + (end.tv_nsec - start.tv_nsec) / 1000000L; + printf("inline %ld\n", elapsed); + printf("inline_swap %ld\n", status("VmSwap:")); + fflush(stdout); + _exit(0); + } + int status_code; + if (waitpid(child, &status_code, 0) != child || !WIFEXITED(status_code) || WEXITSTATUS(status_code)) exit(14); + } else if (!strcmp(argv[4], "timed")) { + struct timespec previous, current, deadline; + long max_gap = 0; + int fd = open(ctl, O_WRONLY | O_TRUNC); + if (fd < 0 || write(fd, "1", 1) != 1) exit(7); + close(fd); + clock_gettime(CLOCK_MONOTONIC, &previous); + kill(getpid(), SIGURG); + deadline = previous; + deadline.tv_sec += 1; + for (;;) { + clock_gettime(CLOCK_MONOTONIC, ¤t); + long gap = (current.tv_sec - previous.tv_sec) * 1000L + + (current.tv_nsec - previous.tv_nsec) / 1000000L; + if (gap > max_gap) max_gap = gap; + previous = current; + if (current.tv_sec > deadline.tv_sec || + (current.tv_sec == deadline.tv_sec && current.tv_nsec >= deadline.tv_nsec)) break; + } + wait_ack(); + printf("gap %ld\n", max_gap); + printf("out %ld\n", status("VmSwap:")); + } else { unsigned char *ro = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); memset(ro, 0x5a, size); mprotect(ro, size, PROT_READ); @@ -84,6 +234,7 @@ int main(int argc, char **argv) { printf("elsewhere %ld\n", status("VmSwap:")); request('0'); printf("back %ld\n", status("VmSwap:")); + } } for (int c = 0; c < cycles; c++) { request('1'); @@ -173,6 +324,72 @@ class ZramPreloadTest(unittest.TestCase): self.assertEqual([1], out["intact"]) self.assertEqual(6, len(out["out"])) + def test_process_signals_do_not_reach_zram_worker(self): + environment = dict(os.environ, LD_PRELOAD=str(self.lib)) + process = subprocess.Popen([str(self.steam), str(self.ctl), "1", "0", "signals"], + env=environment, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + try: + process.stdout.readline() + ready = process.stdout.readline().split() + self.assertEqual("ready", ready[0]) + worker = None + task_dir = Path("/proc") / ready[1] / "task" + for task in task_dir.iterdir(): + if (task / "comm").read_text().strip() == "zr-worker": + worker = int(task.name) + break + self.assertIsNotNone(worker) + self.assertEqual(os.sched_getscheduler(process.pid), os.sched_getscheduler(worker)) + process.stdin.write(f"{worker}\n") + process.stdin.flush() + self.assertEqual("armed 1", process.stdout.readline().strip()) + for _ in range(100): + os.kill(process.pid, signal.SIGURG) + os.kill(process.pid, signal.SIGUSR1) + self.assertEqual(f"received {_ + 1}", process.stdout.readline().strip()) + output, _ = process.communicate(timeout=10) + values = dict(line.split() for line in output.splitlines()) + self.assertNotEqual(0, int(values["urg"])) + self.assertNotEqual(0, int(values["usr"])) + self.assertEqual(0, int(values["worker"])) + self.assertNotEqual(worker, int(values["urg"])) + self.assertNotEqual(worker, int(values["usr"])) + finally: + if process.poll() is None: + process.kill() + process.wait() + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_signal_does_not_stall_the_main_thread(self): + out = self.run_program(None, 256, 0, "timed") + self.assertLess(out["gap"][0], 250) + self.assertGreater(out["out"][0], 32 * 1024) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_inline_pageout_exceeds_async_timing_tolerance(self): + out = self.run_program(None, 256, 0, "timed-inline") + self.assertGreater(out["inline"][0], 250) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_single_request_timing_control(self): + asynchronous = self.run_program(None, 48, 0, "timed") + self.assertGreater(asynchronous["out"][0], 32 * 1024) + inline = self.run_program(None, 48, 0, "timed-inline-single") + self.assertGreater(inline["inline_swap"][0], 32 * 1024) + if inline["inline"][0] <= 250: + self.skipTest("single inline request did not establish the 250 ms timing threshold") + self.assertGreater(inline["inline"][0], 250) + self.assertLess(asynchronous["gap"][0], 250) + + @unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap") + def test_forked_child_still_acknowledges_pageout(self): + out = self.run_program(None, 48, 0, "fork") + self.assertEqual([1], out["childack"]) + + def test_worker_acknowledges_back_to_back_requests(self): + out = self.run_program(None, 48, 0, "burst") + self.assertEqual([1], out["burstack"]) + if __name__ == "__main__": unittest.main()