fix(zram): page game memory out on a worker thread, not the signalled game thread (#2335)

* fix: move zram page-out off signal thread

* fix: drain pending zram requests

* Time the async and inline page-out runs on the same workload

* fix(zram): keep pageout worker responsive under load

* test(zram): synchronize signal delivery before fixture shutdown

* test(zram): compare matched single-request timings

* test(zram): correct single request timing control

* test(zram): calibrate single request timing together

* test(zram): fail the timed run when the acknowledgement never arrives
This commit is contained in:
MaxsTechReview authored and GitHub committed 2026-11-17 11:24:05 -05:00
1 parent f84e9d5191
commit 812e492302
2 files changed
+263 -8

No files matched your search

+45 -7
View File
@@ -7,6 +7,7 @@
#include <stdint.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/prctl.h>
#include <sys/syscall.h>
#include <sys/uio.h>
#include <unistd.h>
@@ -20,11 +21,14 @@
#endif
#define ZR_SWAPPED (1ULL << 62)
#define ZR_BATCH 512
#define ZR_FUTEX_WAIT 0
#define ZR_FUTEX_WAKE 1
static long (*zr_sys)(long, ...);
static unsigned long zr_page;
static int zr_busy;
static int zr_pending;
static int zr_worker;
static char zr_maps[8192];
static uint64_t zr_entries[ZR_BATCH];
static struct iovec zr_iov[ZR_BATCH];
@@ -143,19 +147,36 @@ static void zr_ack(int done) {
zr_sys(SYS_close, fd);
}
static void zr_request(void) {
int mode = zr_mode();
if (mode == 1) zr_ack(0);
if (mode >= 0) zr_walk(mode);
if (mode == 1) zr_ack(1);
}
static void *zr_worker_main(void *unused) {
(void)unused;
prctl(PR_SET_NAME, "zr-worker", 0, 0, 0);
for (;;) {
while (!__atomic_exchange_n(&zr_pending, 0, __ATOMIC_SEQ_CST))
zr_sys(SYS_futex, &zr_pending, ZR_FUTEX_WAIT, 0, NULL, NULL, 0);
zr_request();
}
return NULL;
}
static void zr_handler(int sig, siginfo_t *info, void *context) {
(void)sig;
(void)context;
if (info == NULL || info->si_code != SI_USER) return;
int saved = errno;
__atomic_store_n(&zr_pending, 1, __ATOMIC_SEQ_CST);
while (__atomic_load_n(&zr_pending, __ATOMIC_SEQ_CST)
&& !__atomic_exchange_n(&zr_busy, 1, __ATOMIC_SEQ_CST)) {
if (__atomic_load_n(&zr_worker, __ATOMIC_ACQUIRE)) {
zr_sys(SYS_futex, &zr_pending, ZR_FUTEX_WAKE, 1, NULL, NULL, 0);
} else while (__atomic_load_n(&zr_pending, __ATOMIC_SEQ_CST)
&& !__atomic_exchange_n(&zr_busy, 1, __ATOMIC_SEQ_CST)) {
__atomic_store_n(&zr_pending, 0, __ATOMIC_SEQ_CST);
int mode = zr_mode();
if (mode == 1) zr_ack(0);
if (mode >= 0) zr_walk(mode);
if (mode == 1) zr_ack(1);
zr_request();
__atomic_store_n(&zr_busy, 0, __ATOMIC_SEQ_CST);
}
errno = saved;
@@ -164,6 +185,7 @@ static void zr_handler(int sig, siginfo_t *info, void *context) {
static void zr_child(void) {
__atomic_store_n(&zr_busy, 0, __ATOMIC_SEQ_CST);
__atomic_store_n(&zr_pending, 0, __ATOMIC_SEQ_CST);
__atomic_store_n(&zr_worker, 0, __ATOMIC_RELEASE);
}
__attribute__((constructor)) static void zr_init(void) {
@@ -184,5 +206,21 @@ __attribute__((constructor)) static void zr_init(void) {
sa.sa_sigaction = zr_handler;
sa.sa_flags = SA_SIGINFO | SA_RESTART;
sigemptyset(&sa.sa_mask);
if (sigaction(SIGURG, &sa, NULL) == 0) pthread_atfork(NULL, NULL, zr_child);
if (sigaction(SIGURG, &sa, NULL) == 0) {
pthread_atfork(NULL, NULL, zr_child);
pthread_attr_t attr;
if (pthread_attr_init(&attr) == 0) {
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
pthread_t thread;
sigset_t blocked;
sigset_t previous;
if (sigfillset(&blocked) == 0 &&
pthread_sigmask(SIG_SETMASK, &blocked, &previous) == 0) {
if (pthread_create(&thread, &attr, zr_worker_main, NULL) == 0)
__atomic_store_n(&zr_worker, 1, __ATOMIC_RELEASE);
pthread_sigmask(SIG_SETMASK, &previous, NULL);
}
pthread_attr_destroy(&attr);
}
}
}
+218 -1
View File
@@ -1,5 +1,6 @@
import os
from pathlib import Path
import signal
import shutil
import subprocess
import tempfile
@@ -9,6 +10,7 @@ PRELOAD = Path(__file__).resolve().parents[1] / "linuxfs/preload"
PROGRAM = r"""
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <signal.h>
@@ -17,12 +19,36 @@ PROGRAM = r"""
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>
static const char *ctl;
static size_t size, page;
static volatile int stop;
static unsigned char *shared;
static volatile sig_atomic_t urg_tid;
static volatile sig_atomic_t usr_tid;
static int urg_count, usr_count;
static int worker_tid;
static int worker_delivery;
_Static_assert(__atomic_always_lock_free(sizeof(int), 0), "signal state must be lock-free");
static void signal_tid(int sig) {
sig_atomic_t tid = (sig_atomic_t)syscall(SYS_gettid);
if (tid == __atomic_load_n(&worker_tid, __ATOMIC_RELAXED))
__atomic_store_n(&worker_delivery, 1, __ATOMIC_RELAXED);
if (sig == SIGURG) {
__atomic_store_n(&urg_tid, tid, __ATOMIC_RELAXED);
__atomic_add_fetch(&urg_count, 1, __ATOMIC_RELEASE);
}
if (sig == SIGUSR1) {
__atomic_store_n(&usr_tid, tid, __ATOMIC_RELAXED);
__atomic_add_fetch(&usr_count, 1, __ATOMIC_RELEASE);
}
}
static long status(const char *key) {
FILE *f = fopen("/proc/self/status", "r");
@@ -34,13 +60,37 @@ static long status(const char *key) {
return v;
}
static void request(char mode) {
static void send_request(char mode) {
int fd = open(ctl, O_WRONLY | O_TRUNC);
if (fd < 0 || write(fd, &mode, 1) != 1) exit(3);
close(fd);
kill(getpid(), SIGURG);
}
static void wait_ack(void) {
char ack[256];
snprintf(ack, sizeof(ack), "%s.ack.%d", ctl, getpid());
for (int i = 0; i < 1200; i++) {
FILE *f = fopen(ack, "r");
int done = 0;
if (f) {
done = fgetc(f) == '1';
fclose(f);
}
if (done) return;
usleep(10000);
}
exit(5);
}
static void request(char mode) {
send_request(mode);
if (mode == '1') {
wait_ack();
}
usleep(1000000);
}
static uint64_t sum(const unsigned char *p, size_t n) {
uint64_t s = 0;
for (size_t i = 0; i < n; i += 64) s = s * 31 + p[i];
@@ -77,6 +127,106 @@ int main(int argc, char **argv) {
sigaction(SIGURG, NULL, &sa);
printf("handler %d\n", (sa.sa_flags & SA_SIGINFO) ? 1 : 0);
if (argc > 4) {
if (!strcmp(argv[4], "signals")) {
struct sigaction signal_action;
memset(&signal_action, 0, sizeof(signal_action));
signal_action.sa_handler = signal_tid;
sigemptyset(&signal_action.sa_mask);
sigaction(SIGURG, &signal_action, NULL);
sigaction(SIGUSR1, &signal_action, NULL);
printf("ready %d\n", getpid());
fflush(stdout);
int id;
if (scanf("%d", &id) != 1) return 10;
__atomic_store_n(&worker_tid, id, __ATOMIC_RELAXED);
printf("armed 1\n");
fflush(stdout);
for (int round = 1; round <= 100; round++) {
int i;
for (i = 0; i < 5000 &&
(__atomic_load_n(&urg_count, __ATOMIC_ACQUIRE) < round ||
__atomic_load_n(&usr_count, __ATOMIC_ACQUIRE) < round); i++) usleep(1000);
if (i == 5000) return 11;
printf("received %d\n", round);
fflush(stdout);
}
stop = 1;
for (int i = 0; i < 2; i++) pthread_join(t[i], NULL);
printf("urg %d\n", __atomic_load_n(&urg_tid, __ATOMIC_RELAXED));
printf("usr %d\n", __atomic_load_n(&usr_tid, __ATOMIC_RELAXED));
printf("worker %d\n", __atomic_load_n(&worker_delivery, __ATOMIC_RELAXED));
return 0;
} else if (!strcmp(argv[4], "fork")) {
pid_t child = fork();
if (child < 0) exit(6);
if (child == 0) {
request('0');
request('1');
_exit(0);
}
int status_code;
waitpid(child, &status_code, 0);
printf("childack %d\n", WIFEXITED(status_code) && WEXITSTATUS(status_code) == 0);
} else if (!strcmp(argv[4], "burst")) {
send_request('0');
usleep(10000);
send_request('1');
wait_ack();
printf("burstack 1\n");
} else if (!strcmp(argv[4], "timed-inline") || !strcmp(argv[4], "timed-inline-single")) {
pid_t child = fork();
if (child < 0) exit(8);
if (child == 0) {
if (!strcmp(argv[4], "timed-inline-single")) {
if (munmap(data, size) || munmap(shared, size)) _exit(12);
data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
shared = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (data == MAP_FAILED || shared == MAP_FAILED) _exit(13);
for (size_t i = 0; i < size; i++) data[i] = (unsigned char)(i * 7 + i / 4096);
memset(shared, 0, size);
}
struct timespec start, end;
int fd = open(ctl, O_WRONLY | O_TRUNC);
if (fd < 0 || write(fd, "1", 1) != 1) _exit(9);
close(fd);
clock_gettime(CLOCK_MONOTONIC, &start);
int requests = !strcmp(argv[4], "timed-inline-single") ? 1 : 64;
for (int i = 0; i < requests; i++) {
send_request('1');
}
clock_gettime(CLOCK_MONOTONIC, &end);
long elapsed = (end.tv_sec - start.tv_sec) * 1000L +
(end.tv_nsec - start.tv_nsec) / 1000000L;
printf("inline %ld\n", elapsed);
printf("inline_swap %ld\n", status("VmSwap:"));
fflush(stdout);
_exit(0);
}
int status_code;
if (waitpid(child, &status_code, 0) != child || !WIFEXITED(status_code) || WEXITSTATUS(status_code)) exit(14);
} else if (!strcmp(argv[4], "timed")) {
struct timespec previous, current, deadline;
long max_gap = 0;
int fd = open(ctl, O_WRONLY | O_TRUNC);
if (fd < 0 || write(fd, "1", 1) != 1) exit(7);
close(fd);
clock_gettime(CLOCK_MONOTONIC, &previous);
kill(getpid(), SIGURG);
deadline = previous;
deadline.tv_sec += 1;
for (;;) {
clock_gettime(CLOCK_MONOTONIC, &current);
long gap = (current.tv_sec - previous.tv_sec) * 1000L +
(current.tv_nsec - previous.tv_nsec) / 1000000L;
if (gap > max_gap) max_gap = gap;
previous = current;
if (current.tv_sec > deadline.tv_sec ||
(current.tv_sec == deadline.tv_sec && current.tv_nsec >= deadline.tv_nsec)) break;
}
wait_ack();
printf("gap %ld\n", max_gap);
printf("out %ld\n", status("VmSwap:"));
} else {
unsigned char *ro = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
memset(ro, 0x5a, size);
mprotect(ro, size, PROT_READ);
@@ -84,6 +234,7 @@ int main(int argc, char **argv) {
printf("elsewhere %ld\n", status("VmSwap:"));
request('0');
printf("back %ld\n", status("VmSwap:"));
}
}
for (int c = 0; c < cycles; c++) {
request('1');
@@ -173,6 +324,72 @@ class ZramPreloadTest(unittest.TestCase):
self.assertEqual([1], out["intact"])
self.assertEqual(6, len(out["out"]))
def test_process_signals_do_not_reach_zram_worker(self):
environment = dict(os.environ, LD_PRELOAD=str(self.lib))
process = subprocess.Popen([str(self.steam), str(self.ctl), "1", "0", "signals"],
env=environment, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
try:
process.stdout.readline()
ready = process.stdout.readline().split()
self.assertEqual("ready", ready[0])
worker = None
task_dir = Path("/proc") / ready[1] / "task"
for task in task_dir.iterdir():
if (task / "comm").read_text().strip() == "zr-worker":
worker = int(task.name)
break
self.assertIsNotNone(worker)
self.assertEqual(os.sched_getscheduler(process.pid), os.sched_getscheduler(worker))
process.stdin.write(f"{worker}\n")
process.stdin.flush()
self.assertEqual("armed 1", process.stdout.readline().strip())
for _ in range(100):
os.kill(process.pid, signal.SIGURG)
os.kill(process.pid, signal.SIGUSR1)
self.assertEqual(f"received {_ + 1}", process.stdout.readline().strip())
output, _ = process.communicate(timeout=10)
values = dict(line.split() for line in output.splitlines())
self.assertNotEqual(0, int(values["urg"]))
self.assertNotEqual(0, int(values["usr"]))
self.assertEqual(0, int(values["worker"]))
self.assertNotEqual(worker, int(values["urg"]))
self.assertNotEqual(worker, int(values["usr"]))
finally:
if process.poll() is None:
process.kill()
process.wait()
@unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap")
def test_signal_does_not_stall_the_main_thread(self):
out = self.run_program(None, 256, 0, "timed")
self.assertLess(out["gap"][0], 250)
self.assertGreater(out["out"][0], 32 * 1024)
@unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap")
def test_inline_pageout_exceeds_async_timing_tolerance(self):
out = self.run_program(None, 256, 0, "timed-inline")
self.assertGreater(out["inline"][0], 250)
@unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap")
def test_single_request_timing_control(self):
asynchronous = self.run_program(None, 48, 0, "timed")
self.assertGreater(asynchronous["out"][0], 32 * 1024)
inline = self.run_program(None, 48, 0, "timed-inline-single")
self.assertGreater(inline["inline_swap"][0], 32 * 1024)
if inline["inline"][0] <= 250:
self.skipTest("single inline request did not establish the 250 ms timing threshold")
self.assertGreater(inline["inline"][0], 250)
self.assertLess(asynchronous["gap"][0], 250)
@unittest.skipUnless(swap_total_kb() > 256 * 1024, "needs swap")
def test_forked_child_still_acknowledges_pageout(self):
out = self.run_program(None, 48, 0, "fork")
self.assertEqual([1], out["childack"])
def test_worker_acknowledges_back_to_back_requests(self):
out = self.run_program(None, 48, 0, "burst")
self.assertEqual([1], out["burstack"])
if __name__ == "__main__":
unittest.main()