perf(storage): prefer verified direct SD library access

This commit is contained in:
xXJsonDeruloXx committed 2026-10-04 17:33:36 -04:00
1 parent 4fc2162203
commit 7f141314c6
6 files changed
+65 -457

No files matched your search

-282
View File
@@ -1,282 +0,0 @@
#define _GNU_SOURCE
#include <dirent.h>
#include <dlfcn.h>
#include <errno.h>
#include <pthread.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/statfs.h>
#include <time.h>
#include <unistd.h>
#define FUSE_SUPER_MAGIC 0x65735546
#define CACHE_SLOTS 32
#define CACHE_TTL_NS 2000000000ULL
struct listing {
dev_t dev;
ino_t ino;
struct timespec mtime;
struct timespec ctime;
uint64_t expires;
size_t count;
unsigned int readers;
struct dirent64 *entries;
};
struct reader {
DIR *dir;
struct listing *listing;
size_t pos;
size_t capacity;
int cached;
int capture_failed;
struct listing capture;
struct reader *next;
};
static struct listing listings[CACHE_SLOTS];
static struct reader *readers;
static pthread_mutex_t cache_lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_once_t hooks_once = PTHREAD_ONCE_INIT;
static struct dirent64 *(*real_readdir64)(DIR *);
static int (*real_closedir)(DIR *);
static void (*real_rewinddir)(DIR *);
static long (*real_telldir)(DIR *);
static void (*real_seekdir)(DIR *, long);
static void lock_for_fork(void) { pthread_mutex_lock(&cache_lock); }
static void unlock_after_fork(void) { pthread_mutex_unlock(&cache_lock); }
static void init_hooks(void) {
real_readdir64 = dlsym(RTLD_NEXT, "readdir64");
real_closedir = dlsym(RTLD_NEXT, "closedir");
real_rewinddir = dlsym(RTLD_NEXT, "rewinddir");
real_telldir = dlsym(RTLD_NEXT, "telldir");
real_seekdir = dlsym(RTLD_NEXT, "seekdir");
pthread_atfork(lock_for_fork, unlock_after_fork, unlock_after_fork);
}
static int is_steam_runtime(void) {
return !strcmp(program_invocation_short_name, "steam") &&
strstr(program_invocation_name, "/Steam/steamrtarm64/") != NULL;
}
static uint64_t monotonic_ns(void) {
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
return (uint64_t)now.tv_sec * 1000000000ULL + now.tv_nsec;
}
static int same_directory(const struct listing *listing, const struct stat *st, uint64_t now) {
return listing->entries && listing->expires > now && listing->dev == st->st_dev &&
listing->ino == st->st_ino && listing->mtime.tv_sec == st->st_mtim.tv_sec &&
listing->mtime.tv_nsec == st->st_mtim.tv_nsec &&
listing->ctime.tv_sec == st->st_ctim.tv_sec &&
listing->ctime.tv_nsec == st->st_ctim.tv_nsec;
}
static struct reader *find_reader(DIR *dir) {
for (struct reader *reader = readers; reader; reader = reader->next)
if (reader->dir == dir) return reader;
return NULL;
}
static void release_reader(DIR *dir) {
struct reader **link = &readers;
while (*link) {
if ((*link)->dir == dir) {
struct reader *reader = *link;
if (reader->cached) reader->listing->readers--;
else free(reader->capture.entries);
*link = reader->next;
free(reader);
return;
}
link = &(*link)->next;
}
}
static struct listing *available_listing(void) {
struct listing *oldest = NULL;
for (size_t i = 0; i < CACHE_SLOTS; i++) {
struct listing *listing = &listings[i];
if (listing->readers == 0 && (!oldest || listing->expires < oldest->expires))
oldest = listing;
}
return oldest;
}
/* Capture only what the caller reads. Completing a stable listing makes it reusable, without
* forcing callers that stop early to scan the whole directory or holding a global lock over I/O. */
static struct reader *reader_for(DIR *dir) {
int fd = dirfd(dir);
struct statfs fs;
struct stat before;
if (!real_telldir || !real_rewinddir || real_telldir(dir) != 0 || fd < 0 ||
fstatfs(fd, &fs) != 0 || fs.f_type != FUSE_SUPER_MAGIC || fstat(fd, &before) != 0)
return NULL;
struct reader *reader = calloc(1, sizeof(*reader));
if (!reader) return NULL;
reader->dir = dir;
reader->capture = (struct listing){
.dev = before.st_dev, .ino = before.st_ino,
.mtime = before.st_mtim, .ctime = before.st_ctim,
};
reader->listing = &reader->capture;
uint64_t now = monotonic_ns();
pthread_mutex_lock(&cache_lock);
for (size_t i = 0; i < CACHE_SLOTS; i++) {
if (same_directory(&listings[i], &before, now)) {
reader->listing = &listings[i];
reader->listing->readers++;
reader->cached = 1;
break;
}
}
reader->next = readers;
readers = reader;
pthread_mutex_unlock(&cache_lock);
return reader;
}
static void capture_entry(struct reader *reader, const struct dirent64 *entry) {
struct listing *listing = &reader->capture;
if (listing->count == reader->capacity) {
size_t capacity = reader->capacity ? reader->capacity * 2 : 64;
if (capacity < reader->capacity || capacity > SIZE_MAX / sizeof(*entry)) {
reader->capture_failed = 1;
return;
}
struct dirent64 *grown = realloc(listing->entries, capacity * sizeof(*entry));
if (!grown) { reader->capture_failed = 1; return; }
listing->entries = grown;
reader->capacity = capacity;
}
size_t bytes = entry->d_reclen < sizeof(*entry) ? entry->d_reclen : sizeof(*entry);
memset(&listing->entries[listing->count], 0, sizeof(*entry));
memcpy(&listing->entries[listing->count++], entry, bytes);
}
static void publish_listing(struct reader *reader) {
struct stat after;
struct listing *listing = &reader->capture;
if (fstat(dirfd(reader->dir), &after) != 0 ||
listing->dev != after.st_dev || listing->ino != after.st_ino ||
listing->mtime.tv_sec != after.st_mtim.tv_sec ||
listing->mtime.tv_nsec != after.st_mtim.tv_nsec ||
listing->ctime.tv_sec != after.st_ctim.tv_sec ||
listing->ctime.tv_nsec != after.st_ctim.tv_nsec)
return;
pthread_mutex_lock(&cache_lock);
struct listing *slot = available_listing();
if (slot) {
free(slot->entries);
*slot = *listing;
slot->expires = monotonic_ns() + CACHE_TTL_NS;
slot->readers = 1;
reader->listing = slot;
reader->pos = slot->count;
reader->cached = 1;
listing->entries = NULL;
}
pthread_mutex_unlock(&cache_lock);
}
struct dirent64 *readdir64(DIR *dir) {
pthread_once(&hooks_once, init_hooks);
if (!real_readdir64) { errno = ENOSYS; return NULL; }
if (!is_steam_runtime()) return real_readdir64(dir);
int saved_errno = errno;
pthread_mutex_lock(&cache_lock);
struct reader *reader = find_reader(dir);
pthread_mutex_unlock(&cache_lock);
if (!reader) reader = reader_for(dir);
if (reader && reader->cached) {
errno = saved_errno;
return reader->pos < reader->listing->count ?
&reader->listing->entries[reader->pos++] : NULL;
}
errno = 0;
struct dirent64 *entry = real_readdir64(dir);
int error = errno;
if (reader && !reader->capture_failed) {
if (entry) capture_entry(reader, entry);
else {
if (!error) publish_listing(reader);
reader->capture_failed = 1;
}
}
errno = error ? error : saved_errno;
return entry;
}
/* 32- and 64-bit dirent layouts are identical in the supported 64-bit runtimes. */
#if __SIZEOF_POINTER__ == 8
struct dirent *readdir(DIR *dir) {
return (struct dirent *)readdir64(dir);
}
#endif
int closedir(DIR *dir) {
pthread_once(&hooks_once, init_hooks);
if (!real_closedir) { errno = ENOSYS; return -1; }
int saved_errno = errno;
pthread_mutex_lock(&cache_lock);
release_reader(dir);
pthread_mutex_unlock(&cache_lock);
errno = saved_errno;
return real_closedir(dir);
}
void rewinddir(DIR *dir) {
pthread_once(&hooks_once, init_hooks);
int saved_errno = errno;
pthread_mutex_lock(&cache_lock);
release_reader(dir);
pthread_mutex_unlock(&cache_lock);
if (real_rewinddir) real_rewinddir(dir);
errno = saved_errno;
}
long telldir(DIR *dir) {
pthread_once(&hooks_once, init_hooks);
int saved_errno = errno;
pthread_mutex_lock(&cache_lock);
struct reader *reader = find_reader(dir);
long offset = reader && reader->cached ? (reader->pos ? reader->listing->entries[reader->pos - 1].d_off : 0) : -1;
pthread_mutex_unlock(&cache_lock);
if ((!reader || !reader->cached) && real_telldir) return real_telldir(dir);
if (!reader || !reader->cached) { errno = ENOSYS; return -1; }
errno = saved_errno;
return offset;
}
void seekdir(DIR *dir, long offset) {
pthread_once(&hooks_once, init_hooks);
int saved_errno = errno;
pthread_mutex_lock(&cache_lock);
struct reader *reader = find_reader(dir);
if (reader && !reader->cached) {
release_reader(dir);
reader = NULL;
}
if (reader) {
size_t pos = 0;
if (offset) {
for (; pos < reader->listing->count; pos++)
if (reader->listing->entries[pos].d_off == offset) { pos++; break; }
if (!pos || reader->listing->entries[pos - 1].d_off != offset)
release_reader(dir), reader = NULL;
}
if (reader) reader->pos = pos;
}
pthread_mutex_unlock(&cache_lock);
if (real_seekdir) real_seekdir(dir, offset);
errno = saved_errno;
}
+6 -4
View File
@@ -10,7 +10,7 @@
* It decides nothing proot would decide differently. The guest path is mapped literally - the
* longest binding whose guest path is a prefix, else the rootfs - and the parent directory is
* opened O_PATH and checked against /proc/self/fd. A case-only difference is accepted only on
* FUSE and only after each differently spelled component is verified as a directory rather than
* FUSE or FAT and only after each differently spelled component is verified as a directory rather than
* a symlink. The last component is looked at without following it; other symlinks go to proot.
* So do "..", a trailing "/", anything under /proc, any flag not handled here and any failure
* to decide: the real libc call runs, proot traps it and answers as it always has.
@@ -34,6 +34,7 @@
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <linux/magic.h>
#include <pthread.h>
#include <stdarg.h>
#include <stdint.h>
@@ -257,10 +258,11 @@ static long canonical_dir(const char *dir) {
out[n] = 0;
if (strcmp(out, dir) != 0) {
struct statfs fs;
int fuse = sc(SYS_fstatfs, fd, (long)&fs, 0, 0, 0) == 0 && fs.f_type == 0x65735546;
int case_alias = sc(SYS_fstatfs, fd, (long)&fs, 0, 0, 0) == 0 &&
(fs.f_type == FUSE_SUPER_MAGIC || fs.f_type == EXFAT_SUPER_MAGIC || fs.f_type == MSDOS_SUPER_MAGIC);
sc(SYS_close, fd, 0, 0, 0, 0);
if (!fuse || strcasecmp(out, dir) != 0) return 0;
/* FUSE case aliases return the stored spelling from /proc/self/fd. Prove the final
if (!case_alias || strcasecmp(out, dir) != 0) return 0;
/* FUSE and FAT case aliases return the stored spelling from /proc/self/fd. Prove the final
* component and each parent so a case-only symlink path still goes to proot. */
struct stat st;
if (sc(SYS_newfstatat, AT_FDCWD, (long)dir, (long)&st, AT_SYMLINK_NOFOLLOW, 0) != 0 ||
-166
View File
@@ -1,166 +0,0 @@
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import unittest
SOURCE = Path(__file__).resolve().parents[1] / "linuxfs/preload/readdir_cache.c"
# Mark test directories as FUSE and count reads below the cache. Blocking one stream verifies
# that its disk I/O does not hold up a different stream; the harness has a deadlock timeout.
SHIM = r"""
#define _GNU_SOURCE
#include <dirent.h>
#include <dlfcn.h>
#include <pthread.h>
#include <sys/statfs.h>
static int calls, blocked_fd = -1, entered;
static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t condition = PTHREAD_COND_INITIALIZER;
int fstatfs(int fd, struct statfs *st) {
int (*real)(int, struct statfs *) = dlsym(RTLD_NEXT, "fstatfs");
int result = real(fd, st);
if (!result) st->f_type = 0x65735546;
return result;
}
int probe_calls(void) { return __atomic_load_n(&calls, __ATOMIC_RELAXED); }
void probe_block(int fd) { blocked_fd = fd; entered = 0; }
void probe_wait(void) {
pthread_mutex_lock(&lock);
while (!entered) pthread_cond_wait(&condition, &lock);
pthread_mutex_unlock(&lock);
}
void probe_release(void) {
pthread_mutex_lock(&lock);
blocked_fd = -1;
pthread_cond_broadcast(&condition);
pthread_mutex_unlock(&lock);
}
struct dirent64 *readdir64(DIR *dir) {
struct dirent64 *(*real)(DIR *) = dlsym(RTLD_NEXT, "readdir64");
__atomic_add_fetch(&calls, 1, __ATOMIC_RELAXED);
pthread_mutex_lock(&lock);
if (dirfd(dir) == blocked_fd) {
entered = 1;
pthread_cond_broadcast(&condition);
while (blocked_fd >= 0) pthread_cond_wait(&condition, &lock);
}
pthread_mutex_unlock(&lock);
return real(dir);
}
"""
PROGRAM = r"""
#define _GNU_SOURCE
#include <dirent.h>
#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#define CHECK(c) do { if (!(c)) { fprintf(stderr, "failed line %d\n", __LINE__); exit(1); } } while (0)
static int count(const char *path, const char *wanted, int *found) {
DIR *dir = opendir(path);
CHECK(dir);
int n = 0;
struct dirent *entry;
errno = 73;
while ((entry = readdir(dir))) {
n++;
if (wanted && !strcmp(entry->d_name, wanted)) *found = 1;
}
CHECK(errno == 73);
CHECK(closedir(dir) == 0);
return n;
}
static void *read_one(void *dir) { CHECK(readdir(dir)); return NULL; }
int main(int argc, char **argv) {
CHECK(argc == 3);
alarm(10);
int (*calls)(void) = dlsym(RTLD_DEFAULT, "probe_calls");
void (*block)(int) = dlsym(RTLD_DEFAULT, "probe_block");
void (*wait)(void) = dlsym(RTLD_DEFAULT, "probe_wait");
void (*release)(void) = dlsym(RTLD_DEFAULT, "probe_release");
DIR *dir = opendir(argv[1]);
CHECK(dir && readdir(dir));
CHECK(calls() == 1); /* An early lookup must not read all 5,000 files. */
long offset = telldir(dir);
char expected[256];
struct dirent *entry = readdir(dir);
CHECK(entry);
strcpy(expected, entry->d_name);
seekdir(dir, offset);
CHECK((entry = readdir(dir)) && !strcmp(entry->d_name, expected));
CHECK(closedir(dir) == 0);
int before = calls();
CHECK(count(argv[1], NULL, NULL) == 5002 && calls() > before);
before = calls();
CHECK(count(argv[1], NULL, NULL) == 5002 && calls() == before);
dir = opendir(argv[1]);
CHECK(readdir(dir));
offset = telldir(dir);
CHECK((entry = readdir(dir)));
strcpy(expected, entry->d_name);
seekdir(dir, offset);
CHECK((entry = readdir(dir)) && !strcmp(entry->d_name, expected));
char path[4096];
snprintf(path, sizeof(path), "%s/created", argv[1]);
int fd = open(path, O_CREAT | O_WRONLY | O_EXCL, 0600);
CHECK(fd >= 0 && close(fd) == 0);
rewinddir(dir);
int found = 0, n = 0;
errno = 73;
while ((entry = readdir(dir))) { n++; if (!strcmp(entry->d_name, "created")) found = 1; }
CHECK(n == 5003 && found && errno == 73 && closedir(dir) == 0);
CHECK(unlink(path) == 0);
found = 0;
CHECK(count(argv[1], "created", &found) == 5002 && !found);
dir = opendir(argv[2]);
CHECK(dir);
block(dirfd(dir));
pthread_t worker;
CHECK(pthread_create(&worker, NULL, read_one, dir) == 0);
wait();
CHECK(count(argv[1], NULL, NULL) == 5002); /* Must complete while the other stream is blocked. */
release();
CHECK(pthread_join(worker, NULL) == 0 && closedir(dir) == 0);
puts("lazy reads, complete large listings, seek, mutation and concurrent streams passed");
return 0;
}
"""
@unittest.skipUnless(sys.platform.startswith("linux") and shutil.which("cc"), "Linux and a C compiler are needed")
class ReaddirCacheTest(unittest.TestCase):
def test_streaming_cache_preserves_directory_behavior(self):
with tempfile.TemporaryDirectory() as temporary:
base = Path(temporary)
exe = base / "Steam/steamrtarm64/steam"
exe.parent.mkdir(parents=True)
large, other = base / "large", base / "other"
large.mkdir()
other.mkdir()
for i in range(5000):
(large / str(i)).touch()
for name, source in (("shim", SHIM), ("probe", PROGRAM)):
(base / f"{name}.c").write_text(source)
cache, shim = base / "cache.so", base / "shim.so"
for target, source in ((cache, SOURCE), (shim, base / "shim.c")):
subprocess.run(["cc", "-O2", "-Wall", "-Wextra", "-Werror", "-fPIC", "-shared", "-pthread",
"-o", str(target), str(source), "-ldl"], check=True)
subprocess.run(["cc", "-O2", "-Wall", "-Wextra", "-Werror", "-pthread", "-o", str(exe),
str(base / "probe.c"), "-ldl"], check=True)
result = subprocess.run([str(exe), str(large), str(other)],
env=dict(os.environ, LD_PRELOAD=f"{cache}:{shim}"),
capture_output=True, text=True, timeout=15)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
if __name__ == "__main__":
unittest.main()