mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-20 13:26:02 +00:00
perf: answer Steam's missing-file checks on SD and USB libraries from the directory cache (#2212)
* perf: answer Steam's missing-file checks on FUSE libraries from the directory cache Before creating and committing each file, the Steam client checks whether it already exists under its exact, lower-cased and folded spellings. On an SD or USB library reached through Android's FUSE layer every one of those misses is a round trip to MediaProvider (Brawlhalla: about 65,000 misses, 92 of the 161 s of preallocation and 60 of the 93 s of commit). The client already caches the listings of its steamapps directories on FUSE and keeps them exact through its own changes, inotify and a short TTL. access() now answers a name that is absent from a live listing of its directory without asking the filesystem. Directory paths match case-insensitively only when the library's filesystem folds case, and names compare case-insensitively, so an existing file is never reported missing. * fix: reuse a listing for a differently cased folder only when the whole path folds case Checking that steamapps alone folds case let a listing of one folder answer for a different folder below it whose name differs only in case. The fill now compares the listed folder with its fully case-swapped path below steamapps, so a listing is reused across case only when the filesystem folds every level. * fix(dircache): match a folded listing only below the same prefix and skip it during a tracked change * fix(linuxfs): a folder renamed or removed above a cached listing drops that listing * fix(linuxfs): watch the folders above a cached listing so a rename or removal by any process drops it * fix(linuxfs): also watch the folders above steamapps, and stop a stuck probe from hanging the tests * fix(linuxfs): a listing under a symlink or an unwatchable folder no longer answers absent lookups * fix(linuxfs): a symlink check that cannot be watched also stops absent answers * fix(linuxfs): a differently cased lookup reuses a listing only for the same folder
This commit is contained in:
1 parent
3526309f88
commit
a33954b8bd
3 files changed
+458
-21
No files matched your search
@@ -22,8 +22,12 @@
|
||||
#define MAX_LISTINGS 16
|
||||
#define MAX_ENTRIES 100000
|
||||
#define MAX_HANDLES 64
|
||||
#define MAX_UP 32
|
||||
#define FUSE_MAGIC 0x65735546
|
||||
#define WATCHED (IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO | IN_DELETE_SELF | IN_MOVE_SELF | IN_ONLYDIR)
|
||||
#define ABOVE (IN_DELETE_SELF | IN_MOVE_SELF | IN_ONLYDIR | IN_MASK_ADD)
|
||||
#define GONE (IN_DELETE_SELF | IN_MOVE_SELF | IN_IGNORED)
|
||||
#define LINKED (IN_DELETE_SELF | IN_MOVE_SELF | IN_DONT_FOLLOW | IN_MASK_ADD)
|
||||
|
||||
#if __SIZEOF_POINTER__ == 8
|
||||
|
||||
@@ -42,10 +46,14 @@ struct listing {
|
||||
int refs;
|
||||
int detached;
|
||||
int wd;
|
||||
int up[MAX_UP], nup;
|
||||
int blind;
|
||||
size_t n, cap;
|
||||
struct entry *e;
|
||||
char *recs;
|
||||
size_t len;
|
||||
char *where;
|
||||
int folds;
|
||||
};
|
||||
|
||||
struct handle {
|
||||
@@ -55,6 +63,8 @@ struct handle {
|
||||
size_t pos;
|
||||
int fd;
|
||||
int wd;
|
||||
int up[MAX_UP], nup;
|
||||
int blind;
|
||||
int dirty;
|
||||
char *path;
|
||||
struct dirent64 out;
|
||||
@@ -147,6 +157,7 @@ static void destroy(struct listing *l) {
|
||||
for (size_t i = 0; i < l->n; i++) free(l->e[i].name);
|
||||
free(l->e);
|
||||
free(l->recs);
|
||||
free(l->where);
|
||||
free(l);
|
||||
}
|
||||
|
||||
@@ -155,15 +166,26 @@ static void unref(struct listing *l) {
|
||||
}
|
||||
|
||||
|
||||
static int under(const int *up, int nup, int wd) {
|
||||
for (int i = 0; i < nup; i++)
|
||||
if (up[i] == wd) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void release(int wd) {
|
||||
if (wd < 0 || notify < 0) return;
|
||||
for (int i = 0; i < MAX_LISTINGS; i++)
|
||||
if (table[i] && table[i]->wd == wd) return;
|
||||
if (table[i] && (table[i]->wd == wd || under(table[i]->up, table[i]->nup, wd))) return;
|
||||
for (int i = 0; i < high_water; i++)
|
||||
if (handles[i] && handles[i]->wd == wd) return;
|
||||
if (handles[i] && (handles[i]->wd == wd || under(handles[i]->up, handles[i]->nup, wd))) return;
|
||||
inotify_rm_watch(notify, wd);
|
||||
}
|
||||
|
||||
static void let_go(int wd, const int *up, int nup) {
|
||||
release(wd);
|
||||
for (int i = 0; i < nup; i++) release(up[i]);
|
||||
}
|
||||
|
||||
static void detach(struct listing *l) {
|
||||
__atomic_sub_fetch(&cached_listings, 1, __ATOMIC_RELEASE);
|
||||
l->detached = 1;
|
||||
@@ -172,17 +194,32 @@ static void detach(struct listing *l) {
|
||||
|
||||
static void drop(int slot) {
|
||||
struct listing *l = table[slot];
|
||||
int wd = l->wd;
|
||||
int wd = l->wd, up[MAX_UP], nup = l->nup;
|
||||
memcpy(up, l->up, sizeof(up));
|
||||
table[slot] = NULL;
|
||||
detach(l);
|
||||
release(wd);
|
||||
let_go(wd, up, nup);
|
||||
}
|
||||
|
||||
static void changed(int wd) {
|
||||
static void changed(int wd, uint32_t mask) {
|
||||
int gone = (mask & GONE) != 0;
|
||||
for (int i = 0; i < MAX_LISTINGS; i++)
|
||||
if (table[i] && (table[i]->wd == wd || wd < 0)) drop(i);
|
||||
if (table[i] && (table[i]->wd == wd || wd < 0 || (gone && under(table[i]->up, table[i]->nup, wd)))) drop(i);
|
||||
for (int i = 0; i < high_water; i++)
|
||||
if (handles[i] && (handles[i]->wd == wd || wd < 0)) __atomic_store_n(&handles[i]->dirty, 1, __ATOMIC_RELAXED);
|
||||
if (handles[i] && (handles[i]->wd == wd || wd < 0 || (gone && under(handles[i]->up, handles[i]->nup, wd))))
|
||||
__atomic_store_n(&handles[i]->dirty, 1, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static void forget_below(const char *path) {
|
||||
if (path == NULL || path[0] != '/') return;
|
||||
size_t n = strlen(path);
|
||||
while (n > 1 && path[n - 1] == '/') n--;
|
||||
for (int i = 0; i < MAX_LISTINGS; i++) {
|
||||
struct listing *l = table[i];
|
||||
if (l == NULL || l->where == NULL) continue;
|
||||
int prefix = l->folds ? strncasecmp(l->where, path, n) == 0 : strncmp(l->where, path, n) == 0;
|
||||
if (prefix && (l->where[n] == '/' || l->where[n] == 0)) drop(i);
|
||||
}
|
||||
}
|
||||
|
||||
static void pump(struct expect *x, int nx) {
|
||||
@@ -194,7 +231,7 @@ static void pump(struct expect *x, int nx) {
|
||||
if (n <= 0) {
|
||||
if (n < 0 && errno != EAGAIN && errno != EINTR) {
|
||||
notify = -2;
|
||||
changed(-1);
|
||||
changed(-1, 0);
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -202,7 +239,7 @@ static void pump(struct expect *x, int nx) {
|
||||
struct inotify_event *ev = (struct inotify_event *)p;
|
||||
p += sizeof(*ev) + ev->len;
|
||||
if (ev->mask & IN_Q_OVERFLOW) {
|
||||
changed(-1);
|
||||
changed(-1, 0);
|
||||
continue;
|
||||
}
|
||||
int k = 0;
|
||||
@@ -212,21 +249,21 @@ static void pump(struct expect *x, int nx) {
|
||||
for (int i = 0; i < high_water; i++)
|
||||
if (handles[i] && handles[i]->wd == ev->wd) __atomic_store_n(&handles[i]->dirty, 1, __ATOMIC_RELAXED);
|
||||
} else {
|
||||
changed(ev->wd);
|
||||
changed(ev->wd, ev->mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
errno = saved;
|
||||
}
|
||||
|
||||
static int watch(const char *path) {
|
||||
static int watch(const char *path, uint32_t mask) {
|
||||
if (notify == -1) notify = inotify_init1(IN_NONBLOCK | IN_CLOEXEC);
|
||||
if (notify < 0) {
|
||||
notify = -2;
|
||||
return -1;
|
||||
}
|
||||
int saved = errno;
|
||||
int wd = inotify_add_watch(notify, path, WATCHED);
|
||||
int wd = inotify_add_watch(notify, path, mask);
|
||||
errno = saved;
|
||||
return wd;
|
||||
}
|
||||
@@ -257,10 +294,11 @@ static void install(struct listing *l) {
|
||||
int slot = find(l->dev, l->ino);
|
||||
if (slot >= 0) {
|
||||
struct listing *old = table[slot];
|
||||
int old_wd = old->wd;
|
||||
int old_wd = old->wd, up[MAX_UP], nup = old->nup;
|
||||
memcpy(up, old->up, sizeof(up));
|
||||
table[slot] = l;
|
||||
detach(old);
|
||||
if (old_wd != l->wd) release(old_wd);
|
||||
let_go(old_wd, up, nup);
|
||||
return;
|
||||
}
|
||||
int victim = -1;
|
||||
@@ -387,11 +425,47 @@ static DIR *hit(const struct stat *st, struct handle *h, long long now) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int above(struct handle *h, const char *path) {
|
||||
char up[PATH_MAX];
|
||||
size_t n = strlen(path);
|
||||
while (n > 1 && path[n - 1] == '/') n--;
|
||||
if (n >= sizeof(up)) return 0;
|
||||
memcpy(up, path, n);
|
||||
up[n] = 0;
|
||||
size_t root = (size_t)(strstr(path, "/steamapps") - path) + 10;
|
||||
for (size_t i = 1; i <= n; i++) {
|
||||
if (up[i] != '/' && up[i] != 0) continue;
|
||||
char at = up[i];
|
||||
up[i] = 0;
|
||||
int wd = i < n ? watch(up, ABOVE) : h->wd;
|
||||
int link = watch(up, LINKED);
|
||||
up[i] = at;
|
||||
if (link < 0) h->blind = 1;
|
||||
else if (link != wd) {
|
||||
h->blind = 1;
|
||||
release(link);
|
||||
}
|
||||
if (i == n) break;
|
||||
if (wd < 0 && i < root) {
|
||||
h->blind = 1;
|
||||
continue;
|
||||
}
|
||||
if (wd < 0) return 0;
|
||||
if (wd == h->wd || under(h->up, h->nup, wd)) continue;
|
||||
if (h->nup == MAX_UP) {
|
||||
release(wd);
|
||||
return 0;
|
||||
}
|
||||
h->up[h->nup++] = wd;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int track(struct handle *h, const char *path) {
|
||||
pthread_mutex_lock(&lock);
|
||||
h->wd = watch(path);
|
||||
DIR *d = h->wd >= 0 ? wrap(h) : NULL;
|
||||
if (d == NULL && h->wd >= 0) release(h->wd);
|
||||
h->wd = watch(path, WATCHED);
|
||||
DIR *d = h->wd >= 0 && above(h, path) ? wrap(h) : NULL;
|
||||
if (d == NULL) let_go(h->wd, h->up, h->nup);
|
||||
pthread_mutex_unlock(&lock);
|
||||
return d != NULL;
|
||||
}
|
||||
@@ -404,6 +478,10 @@ static void filling(struct handle *h, DIR *real, const struct stat *st, long lon
|
||||
h->fill->filled = now;
|
||||
h->fill->used = now;
|
||||
h->fill->wd = h->wd;
|
||||
memcpy(h->fill->up, h->up, sizeof(h->up));
|
||||
h->fill->nup = h->nup;
|
||||
h->fill->blind = h->blind;
|
||||
h->fill->where = strdup(h->path);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -411,7 +489,7 @@ static void untrack(struct handle *h) {
|
||||
pthread_mutex_lock(&lock);
|
||||
int i = registered((DIR *)h);
|
||||
if (i >= 0) __atomic_store_n(&handles[i], NULL, __ATOMIC_RELEASE);
|
||||
release(h->wd);
|
||||
let_go(h->wd, h->up, h->nup);
|
||||
pthread_mutex_unlock(&lock);
|
||||
}
|
||||
|
||||
@@ -454,11 +532,27 @@ static void abandon(struct handle *h) {
|
||||
h->fill = NULL;
|
||||
}
|
||||
|
||||
static int folds(const char *path, const struct stat *st) {
|
||||
char swapped[PATH_MAX];
|
||||
struct stat b;
|
||||
const char *at = strstr(path, "/steamapps");
|
||||
size_t n = strlen(path);
|
||||
if (at == NULL || n >= sizeof(swapped)) return 0;
|
||||
memcpy(swapped, path, n + 1);
|
||||
for (char *c = swapped + (at - path); *c; c++)
|
||||
if ((*c >= 'a' && *c <= 'z') || (*c >= 'A' && *c <= 'Z')) *c ^= 0x20;
|
||||
int saved = errno;
|
||||
int same_dir = stat(swapped, &b) == 0 && b.st_dev == st->st_dev && b.st_ino == st->st_ino;
|
||||
errno = saved;
|
||||
return same_dir;
|
||||
}
|
||||
|
||||
static void finish(struct handle *h) {
|
||||
struct listing *l = h->fill;
|
||||
struct stat st;
|
||||
h->fill = NULL;
|
||||
int current = stat(h->path, &st) == 0 && same(l, &st);
|
||||
if (current) l->folds = folds(h->path, &st);
|
||||
pthread_mutex_lock(&lock);
|
||||
if (!changing) pump(NULL, 0);
|
||||
if (current && !__atomic_load_n(&h->dirty, __ATOMIC_RELAXED)) install(l);
|
||||
@@ -536,7 +630,7 @@ int closedir(DIR *d) {
|
||||
if (h) {
|
||||
__atomic_store_n(&handles[i], NULL, __ATOMIC_RELEASE);
|
||||
if (h->snap) unref(h->snap);
|
||||
release(h->wd);
|
||||
let_go(h->wd, h->up, h->nup);
|
||||
}
|
||||
pthread_mutex_unlock(&lock);
|
||||
if (h == NULL) return next_closedir(d);
|
||||
@@ -658,6 +752,11 @@ static int patch(struct side *s, int what, uint32_t event, int ok, int error, co
|
||||
copy->filled = l->filled;
|
||||
copy->used = l->used;
|
||||
copy->wd = l->wd;
|
||||
memcpy(copy->up, l->up, sizeof(l->up));
|
||||
copy->nup = l->nup;
|
||||
copy->blind = l->blind;
|
||||
copy->where = l->where ? strdup(l->where) : NULL;
|
||||
copy->folds = l->folds;
|
||||
copy->detached = 0;
|
||||
stamp(copy, &s->before);
|
||||
table[slot] = copy;
|
||||
@@ -738,6 +837,7 @@ int bl_dircache_create(int dirfd, const char *path, int flags, mode_t mode,
|
||||
struct expect x; \
|
||||
pthread_mutex_lock(&lock); \
|
||||
int nx = patch(&s, what, event, r == 0, saved, what == ADDED ? path : NULL, &x); \
|
||||
if (r == 0 && what == REMOVED) forget_below(path); \
|
||||
pthread_mutex_unlock(&lock); \
|
||||
end(&x, nx); \
|
||||
pthread_mutex_unlock(&oplock); \
|
||||
@@ -770,6 +870,7 @@ ONE_PATH(remove, REMOVED, IN_DELETE, next_remove(path))
|
||||
if (track_a && strcmp(a.dir, b.dir) == 0) b.known = stat(b.dir, &b.before) == 0; \
|
||||
nx += patch(&b, ADDED, event, r == 0, saved, to, &x[nx]); \
|
||||
} \
|
||||
if (r == 0 && gone) { forget_below(from); forget_below(to); } \
|
||||
pthread_mutex_unlock(&lock); \
|
||||
end(x, nx); \
|
||||
pthread_mutex_unlock(&oplock); \
|
||||
@@ -784,8 +885,62 @@ TWO_PATHS(rename, 1, IN_MOVED_TO, next_rename)
|
||||
TWO_PATHS(link, 0, IN_CREATE, link_or_lock)
|
||||
TWO_PATHS(symlink, 0, IN_CREATE, next_symlink)
|
||||
|
||||
static int named(const struct listing *l, const char *name) {
|
||||
for (size_t i = 0; i < l->n; i++)
|
||||
if (strcasecmp(l->e[i].name, name) == 0) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int folded_match(const char *where, const char *dir) {
|
||||
const char *at = strstr(where, "/steamapps");
|
||||
if (at == NULL) return 0;
|
||||
size_t n = (size_t)(at - where);
|
||||
return strncmp(where, dir, n) == 0 && strcasecmp(where + n, dir + n) == 0;
|
||||
}
|
||||
|
||||
static int same_dir(const struct listing *l, const char *dir) {
|
||||
struct stat st;
|
||||
int saved = errno;
|
||||
int same = stat(dir, &st) == 0 && st.st_dev == l->dev && st.st_ino == l->ino;
|
||||
errno = saved;
|
||||
return same;
|
||||
}
|
||||
|
||||
static int listed_absent(const char *dir, const char *name, long long now) {
|
||||
for (int i = 0; i < MAX_LISTINGS; i++) {
|
||||
struct listing *l = table[i];
|
||||
if (l == NULL || l->where == NULL || l->blind || now - l->filled >= DIR_TTL_NS) continue;
|
||||
if (strcmp(l->where, dir) == 0) return !named(l, name);
|
||||
if (l->folds && folded_match(l->where, dir)) return same_dir(l, dir) && !named(l, name);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int bl_dircache_absent(const char *path) __attribute__((visibility("hidden")));
|
||||
int bl_dircache_absent(const char *path) {
|
||||
struct side s;
|
||||
if (!watching(path) || !split(path, &s)) return 0;
|
||||
for (const char *c = s.name; *c; c++)
|
||||
if ((unsigned char)*c >= 0x80) return 0;
|
||||
long long now = now_ns();
|
||||
pthread_mutex_lock(&lock);
|
||||
int absent = 0;
|
||||
if (!changing) {
|
||||
pump(NULL, 0);
|
||||
absent = listed_absent(s.dir, s.name, now);
|
||||
}
|
||||
pthread_mutex_unlock(&lock);
|
||||
return absent;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int bl_dircache_absent(const char *path) __attribute__((visibility("hidden")));
|
||||
int bl_dircache_absent(const char *path) {
|
||||
(void)path;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int bl_dircache_create(int dirfd, const char *path, int flags, mode_t mode,
|
||||
int (*real)(int, const char *, int, mode_t)) __attribute__((visibility("hidden")));
|
||||
int bl_dircache_create(int dirfd, const char *path, int flags, mode_t mode,
|
||||
|
||||
@@ -111,6 +111,8 @@ char *bl_cached_realpath(const char *path, char *resolved, char *(*real)(const c
|
||||
return strdup(out);
|
||||
}
|
||||
|
||||
int bl_dircache_absent(const char *path) __attribute__((weak, visibility("hidden")));
|
||||
|
||||
static int cached_access(const char *path, int mode, int flags, int (*check)(const char *, int, int)) {
|
||||
unsigned h;
|
||||
int hit;
|
||||
@@ -122,6 +124,10 @@ static int cached_access(const char *path, int mode, int flags, int (*check)(con
|
||||
hit = find(path, KIND_ACCESS, mode, flags, h) != NULL;
|
||||
pthread_mutex_unlock(&lock);
|
||||
if (hit) return 0;
|
||||
if (bl_dircache_absent && bl_dircache_absent(path)) {
|
||||
errno = ENOENT;
|
||||
return -1;
|
||||
}
|
||||
result = check(path, mode, flags);
|
||||
if (result == 0) {
|
||||
pthread_mutex_lock(&lock);
|
||||
|
||||
@@ -3,6 +3,7 @@ from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
PRELOAD = Path(__file__).resolve().parents[1] / "linuxfs/preload"
|
||||
@@ -10,6 +11,7 @@ PRELOAD = Path(__file__).resolve().parents[1] / "linuxfs/preload"
|
||||
FAKE = r"""
|
||||
#define _GNU_SOURCE
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <dlfcn.h>
|
||||
#include <fcntl.h>
|
||||
#include <string.h>
|
||||
@@ -17,9 +19,11 @@ FAKE = r"""
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/statfs.h>
|
||||
#include <sys/inotify.h>
|
||||
#include <unistd.h>
|
||||
|
||||
int real_opendirs;
|
||||
int real_accesses;
|
||||
|
||||
static void coarse(const char *path, struct stat *st) {
|
||||
if (getenv("COARSE") && strstr(path, "/fuse/") && S_ISDIR(st->st_mode)) {
|
||||
@@ -32,6 +36,17 @@ static void coarse(const char *path, struct stat *st) {
|
||||
int stat(const char *path, struct stat *st) {
|
||||
static int (*next)(const char *, struct stat *);
|
||||
if (!next) next = dlsym(RTLD_NEXT, "stat");
|
||||
char folded[4096];
|
||||
const char *fold = getenv("FOLD");
|
||||
const char *upper = fold ? strstr(path, "/STEAMAPPS") : NULL;
|
||||
if (!upper && fold && !strcmp(fold, "all")) upper = strcasestr(path, "/steamapps/");
|
||||
if (upper && strlen(path) < sizeof(folded)) {
|
||||
strcpy(folded, path);
|
||||
memcpy(folded + (upper - path) + 1, "steamapps", 9);
|
||||
if (!strcmp(fold, "all"))
|
||||
for (char *c = folded + (upper - path); *c; c++) if (*c >= 'A' && *c <= 'Z') *c += 'a' - 'A';
|
||||
path = folded;
|
||||
}
|
||||
int r = next(path, st);
|
||||
if (r == 0) coarse(path, st);
|
||||
return r;
|
||||
@@ -61,6 +76,16 @@ int fstatfs(int fd, struct statfs *out) {
|
||||
return r;
|
||||
}
|
||||
|
||||
int inotify_add_watch(int fd, const char *path, uint32_t mask) {
|
||||
static int (*next)(int, const char *, uint32_t);
|
||||
if (!next) next = dlsym(RTLD_NEXT, "inotify_add_watch");
|
||||
if (getenv("NOFOLLOW_FAIL") && (mask & IN_DONT_FOLLOW)) {
|
||||
errno = ENOSPC;
|
||||
return -1;
|
||||
}
|
||||
return next(fd, path, mask);
|
||||
}
|
||||
|
||||
int statfs(const char *path, struct statfs *out) {
|
||||
static int (*next)(const char *, struct statfs *);
|
||||
if (!next) next = dlsym(RTLD_NEXT, "statfs");
|
||||
@@ -82,6 +107,13 @@ DIR *opendir(const char *path) {
|
||||
return next(path);
|
||||
}
|
||||
|
||||
int faccessat(int dirfd, const char *path, int mode, int flags) {
|
||||
static int (*next)(int, const char *, int, int);
|
||||
if (!next) next = dlsym(RTLD_NEXT, "faccessat");
|
||||
if (strstr(path, "/fuse/")) real_accesses++;
|
||||
return next(dirfd, path, mode, flags);
|
||||
}
|
||||
|
||||
DIR *fdopendir(int fd) {
|
||||
static DIR *(*next)(int);
|
||||
if (!next) next = dlsym(RTLD_NEXT, "fdopendir");
|
||||
@@ -94,6 +126,7 @@ PROGRAM = r"""
|
||||
#define _GNU_SOURCE
|
||||
#include <dirent.h>
|
||||
#include <dlfcn.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <pthread.h>
|
||||
#include <stdio.h>
|
||||
@@ -171,6 +204,7 @@ static void stress(const char *dir) {
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
int *opens = dlsym(RTLD_DEFAULT, "real_opendirs");
|
||||
int *accesses = dlsym(RTLD_DEFAULT, "real_accesses");
|
||||
for (int i = 1; i < argc; i++) {
|
||||
const char *op = argv[i], *a = argv[i + 1];
|
||||
if (!strcmp(op, "list")) { list(a); i++; }
|
||||
@@ -178,6 +212,14 @@ int main(int argc, char **argv) {
|
||||
else if (!strcmp(op, "create")) { close(open(a, O_WRONLY | O_CREAT, 0644)); i++; }
|
||||
else if (!strcmp(op, "sneak")) { close((int)syscall(SYS_openat, AT_FDCWD, a, O_WRONLY | O_CREAT, 0644)); i++; }
|
||||
else if (!strcmp(op, "sneak-unlink")) { syscall(SYS_unlinkat, AT_FDCWD, a, 0); i++; }
|
||||
else if (!strcmp(op, "sneak-rename")) { syscall(SYS_renameat, AT_FDCWD, a, AT_FDCWD, argv[i + 2]); i += 2; }
|
||||
else if (!strcmp(op, "sneak-mkdir")) { syscall(SYS_mkdirat, AT_FDCWD, a, 0755); i++; }
|
||||
else if (!strcmp(op, "pause")) {
|
||||
char line[16];
|
||||
printf("ready\n");
|
||||
fflush(stdout);
|
||||
if (!fgets(line, sizeof(line), stdin)) return 1;
|
||||
}
|
||||
else if (!strcmp(op, "unlink")) { unlink(a); i++; }
|
||||
else if (!strcmp(op, "remove")) { remove(a); i++; }
|
||||
else if (!strcmp(op, "mkdir")) { mkdir(a, 0755); i++; }
|
||||
@@ -246,6 +288,12 @@ int main(int argc, char **argv) {
|
||||
waitpid(child, NULL, 0);
|
||||
i += 2;
|
||||
}
|
||||
else if (!strcmp(op, "access")) {
|
||||
int r = access(a, F_OK);
|
||||
printf("%d %d\n", r, r ? errno : 0);
|
||||
i++;
|
||||
}
|
||||
else if (!strcmp(op, "accesses")) printf("%d\n", accesses ? *accesses : -1);
|
||||
else if (!strcmp(op, "opens")) printf("%d\n", opens ? *opens : -1);
|
||||
}
|
||||
return 0;
|
||||
@@ -284,13 +332,39 @@ class DirCachePreloadTest(unittest.TestCase):
|
||||
for name in ("a.bundle", "b.bundle"):
|
||||
(self.staging / name).touch()
|
||||
|
||||
def run_program(self, *args, program=None, env=None):
|
||||
def environment(self, env):
|
||||
environment = dict(os.environ, LD_PRELOAD="%s:%s" % (self.cache, self.fake))
|
||||
environment.update(env or {})
|
||||
result = subprocess.run([str(program or self.steam), *map(str, args)], env=environment,
|
||||
return environment
|
||||
|
||||
def run_program(self, *args, program=None, env=None):
|
||||
result = subprocess.run([str(program or self.steam), *map(str, args)], env=self.environment(env),
|
||||
capture_output=True, text=True, check=True, timeout=60)
|
||||
return result.stdout.splitlines()
|
||||
|
||||
def run_paused(self, *args, between, env=None):
|
||||
probe = subprocess.Popen([str(self.steam), *map(str, args)], env=self.environment(env), text=True,
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
|
||||
timer = threading.Timer(60, probe.kill)
|
||||
timer.start()
|
||||
try:
|
||||
before = []
|
||||
for line in probe.stdout:
|
||||
if line == "ready\n":
|
||||
break
|
||||
before.append(line.rstrip("\n"))
|
||||
else:
|
||||
self.fail("the probe stopped before it was ready")
|
||||
between()
|
||||
after, _ = probe.communicate("go\n", timeout=60)
|
||||
finally:
|
||||
timer.cancel()
|
||||
if probe.poll() is None:
|
||||
probe.kill()
|
||||
probe.communicate()
|
||||
self.assertEqual(0, probe.returncode)
|
||||
return before + after.splitlines()
|
||||
|
||||
def test_repeated_listings_read_the_directory_once(self):
|
||||
out = self.run_program("list", self.staging, "list", self.staging, "list", self.staging, "opens")
|
||||
self.assertEqual(["a.bundle,b.bundle"] * 3 + ["1"], out)
|
||||
@@ -399,6 +473,208 @@ class DirCachePreloadTest(unittest.TestCase):
|
||||
outside.mkdir()
|
||||
self.assertEqual("2", self.run_program("list", outside, "list", outside, "opens")[-1])
|
||||
|
||||
def test_missing_names_are_answered_from_the_listing(self):
|
||||
s = self.staging
|
||||
out = self.run_program("list", s, "access", s / "x.bundle", "access", s / "y/z.bundle",
|
||||
"access", s / "c.bundle", "accesses", "access", s / "a.bundle", "accesses", "opens")
|
||||
self.assertEqual(["-1 2", "-1 2", "-1 2", "1", "0 0", "2", "1"], out[1:])
|
||||
|
||||
def test_a_name_that_differs_only_in_case_is_asked_of_the_filesystem(self):
|
||||
s = self.staging
|
||||
out = self.run_program("list", s, "access", s / "A.BUNDLE", "accesses")
|
||||
self.assertEqual(["-1 2", "1"], out[1:])
|
||||
|
||||
def test_misses_before_the_client_lists_a_directory_reach_the_filesystem(self):
|
||||
s = self.staging
|
||||
out = self.run_program("access", s / "x.bundle", "access", s / "y.bundle", "accesses", "opens")
|
||||
self.assertEqual(["-1 2", "-1 2", "2", "0"], out)
|
||||
|
||||
def test_names_created_after_a_miss_are_found(self):
|
||||
s = self.staging
|
||||
out = self.run_program("list", s, "access", s / "x.bundle", "create", s / "x.bundle", "access", s / "x.bundle",
|
||||
"access", s / "y.bundle", "sneak", s / "y.bundle", "access", s / "y.bundle",
|
||||
"rename", s / "y.bundle", s / "w.bundle", "access", s / "w.bundle",
|
||||
"mkdir", s / "sub", "access", s / "sub",
|
||||
env={"COARSE": "1"})
|
||||
self.assertEqual(["-1 2", "0 0", "-1 2", "0 0", "0 0", "0 0"], out[1:])
|
||||
|
||||
def test_a_differently_cased_directory_reaches_the_filesystem_when_case_matters(self):
|
||||
s = self.staging
|
||||
upper = Path(str(s).replace("downloading", "DOWNLOADING"))
|
||||
out = self.run_program("list", s, "access", upper / "x.bundle", "accesses")
|
||||
self.assertEqual(["-1 2", "1"], out[1:])
|
||||
|
||||
def test_a_differently_cased_directory_uses_the_listing_when_the_filesystem_folds_case(self):
|
||||
s = self.staging
|
||||
upper = Path(str(s).replace("downloading", "DOWNLOADING"))
|
||||
out = self.run_program("list", s, "access", upper / "x.bundle", "access", upper / "A.BUNDLE", "accesses",
|
||||
env={"FOLD": "all"})
|
||||
self.assertEqual(["-1 2", "-1 2", "1"], out[1:])
|
||||
|
||||
def test_a_case_sensitive_folder_below_a_folding_steamapps_reaches_the_filesystem(self):
|
||||
s = self.staging
|
||||
upper = self.lib / "DOWNLOADING" / s.name
|
||||
upper.mkdir(parents=True)
|
||||
(upper / "x.bundle").write_bytes(b"")
|
||||
out = self.run_program("list", s, "access", upper / "x.bundle", "access", upper / "y.bundle", "accesses",
|
||||
env={"FOLD": "1"})
|
||||
self.assertEqual(["0 0", "-1 2", "2"], out[1:])
|
||||
|
||||
def test_a_differently_cased_folder_above_steamapps_reaches_the_filesystem(self):
|
||||
s = self.staging
|
||||
top = self.lib.parent.parent
|
||||
other = top.parent / top.name.swapcase() / s.relative_to(top)
|
||||
self.assertNotEqual(top.name, top.name.swapcase())
|
||||
other.mkdir(parents=True)
|
||||
(other / "x.bundle").write_bytes(b"")
|
||||
out = self.run_program("list", s, "access", other / "x.bundle", "accesses", env={"FOLD": "all"})
|
||||
self.assertEqual(["0 0", "1"], out[1:])
|
||||
|
||||
def test_a_differently_cased_sibling_of_a_folded_listing_reaches_the_filesystem(self):
|
||||
s = self.staging
|
||||
(self.lib.parent / "STEAMAPPS").symlink_to("steamapps")
|
||||
(self.lib / "DOWNLOADING").symlink_to("downloading")
|
||||
other = self.lib / "Downloading" / s.name
|
||||
other.mkdir(parents=True)
|
||||
(other / "x.bundle").touch()
|
||||
out = self.run_program("list", s, "access", other / "x.bundle", "accesses")
|
||||
self.assertEqual("0 0", out[1])
|
||||
|
||||
def test_a_listing_is_dropped_when_a_folder_above_it_is_renamed(self):
|
||||
s = self.staging
|
||||
parent = s.parent
|
||||
moved = parent.parent / "downloading.old"
|
||||
out = self.run_program("list", s, "rename", parent, moved, "mkdir", parent, "mkdir", s,
|
||||
"sneak", s / "x.bundle", "access", s / "x.bundle", "accesses")
|
||||
self.assertEqual(["0 0", "1"], out[1:])
|
||||
|
||||
def test_a_listing_is_dropped_when_another_process_renames_a_folder_above_it(self):
|
||||
s = self.staging
|
||||
parent = s.parent
|
||||
|
||||
def swap():
|
||||
parent.rename(parent.parent / "downloading.old")
|
||||
s.mkdir(parents=True)
|
||||
(s / "x.bundle").touch()
|
||||
|
||||
out = self.run_paused("list", s, "pause", "access", s / "x.bundle", "accesses", between=swap)
|
||||
self.assertEqual(["0 0", "1"], out[1:])
|
||||
|
||||
def test_a_listing_is_dropped_when_another_process_renames_the_library_above_steamapps(self):
|
||||
s = self.staging
|
||||
root = self.lib.parent
|
||||
|
||||
def swap():
|
||||
root.rename(root.parent / "fuse.old")
|
||||
s.mkdir(parents=True)
|
||||
(s / "x.bundle").touch()
|
||||
|
||||
out = self.run_paused("list", s, "pause", "access", s / "x.bundle", "accesses", between=swap)
|
||||
self.assertEqual(["0 0", "1"], out[1:])
|
||||
|
||||
def test_a_repointed_symlink_above_a_listing_does_not_hide_files(self):
|
||||
s = self.staging
|
||||
real = self.lib / "dl-a"
|
||||
s.parent.rename(real)
|
||||
s.parent.symlink_to(real.name)
|
||||
other = self.lib / "dl-b" / s.name
|
||||
other.mkdir(parents=True)
|
||||
(other / "x.bundle").touch()
|
||||
|
||||
def repoint():
|
||||
link = self.lib / "downloading.new"
|
||||
link.symlink_to("dl-b")
|
||||
link.replace(s.parent)
|
||||
|
||||
out = self.run_paused("list", s, "pause", "access", s / "x.bundle", between=repoint)
|
||||
self.assertEqual("0 0", out[1])
|
||||
|
||||
def test_a_symlink_check_that_cannot_be_watched_does_not_hide_files(self):
|
||||
s = self.staging
|
||||
real = self.lib / "dl-a"
|
||||
s.parent.rename(real)
|
||||
s.parent.symlink_to(real.name)
|
||||
other = self.lib / "dl-b" / s.name
|
||||
other.mkdir(parents=True)
|
||||
(other / "x.bundle").touch()
|
||||
|
||||
def repoint():
|
||||
link = self.lib / "downloading.new"
|
||||
link.symlink_to("dl-b")
|
||||
link.replace(s.parent)
|
||||
|
||||
out = self.run_paused("list", s, "pause", "access", s / "x.bundle", between=repoint, env={"NOFOLLOW_FAIL": "1"})
|
||||
self.assertEqual("0 0", out[1])
|
||||
|
||||
def test_an_unreadable_folder_above_steamapps_does_not_hide_files(self):
|
||||
s = self.staging
|
||||
root = self.lib.parent
|
||||
moved = root.parent / "fuse.old"
|
||||
root.chmod(0o311)
|
||||
self.addCleanup(lambda: moved.chmod(0o755) if moved.exists() else root.chmod(0o755))
|
||||
if os.access(root, os.R_OK):
|
||||
self.skipTest("folder permissions are not enforced here")
|
||||
|
||||
def swap():
|
||||
root.rename(moved)
|
||||
s.mkdir(parents=True)
|
||||
(s / "x.bundle").touch()
|
||||
|
||||
out = self.run_paused("list", s, "pause", "access", s / "x.bundle", between=swap)
|
||||
self.assertEqual("0 0", out[1])
|
||||
|
||||
def test_a_listing_is_dropped_when_another_process_removes_a_folder_above_it(self):
|
||||
deep = self.staging / "deep"
|
||||
deep.mkdir()
|
||||
parent = self.staging.parent
|
||||
|
||||
def swap():
|
||||
self.staging.rename(self.lib / "moved")
|
||||
parent.rmdir()
|
||||
deep.mkdir(parents=True)
|
||||
(deep / "x.bundle").touch()
|
||||
|
||||
out = self.run_paused("list", deep, "pause", "access", deep / "x.bundle", "accesses", between=swap)
|
||||
self.assertEqual(["", "0 0", "1"], out)
|
||||
|
||||
def test_a_listing_is_dropped_when_a_folder_above_it_is_renamed_around_the_cache(self):
|
||||
s = self.staging
|
||||
parent = s.parent
|
||||
out = self.run_program("list", s, "sneak-rename", parent, parent.parent / "downloading.old",
|
||||
"sneak-mkdir", parent, "sneak-mkdir", s, "sneak", s / "x.bundle",
|
||||
"access", s / "x.bundle", "accesses")
|
||||
self.assertEqual(["0 0", "1"], out[1:])
|
||||
|
||||
def test_changes_in_a_cached_folder_above_keep_the_listing_below(self):
|
||||
s = self.staging
|
||||
parent = s.parent
|
||||
|
||||
def churn():
|
||||
(parent / "z").touch()
|
||||
(parent / "z").unlink()
|
||||
(parent / "sibling").mkdir()
|
||||
(parent / "sibling").rmdir()
|
||||
|
||||
out = self.run_paused("list", parent, "list", s, "pause", "list", parent, "list", s,
|
||||
"access", s / "x.bundle", "accesses", "opens", between=churn)
|
||||
self.assertEqual([s.name, "a.bundle,b.bundle", s.name, "a.bundle,b.bundle", "-1 2", "0", "3"], out)
|
||||
|
||||
def test_misses_outside_the_client_or_fuse_steamapps_reach_the_filesystem(self):
|
||||
s = self.staging
|
||||
other = self.run_program("list", s, "access", s / "x.bundle", "access", s / "y.bundle", "accesses",
|
||||
program=self.other)
|
||||
self.assertEqual(["-1 2", "-1 2", "2"], other[1:])
|
||||
outside = self.lib.parent / "elsewhere"
|
||||
outside.mkdir()
|
||||
out = self.run_program("list", outside, "access", outside / "x", "access", outside / "y", "accesses")
|
||||
self.assertEqual(["-1 2", "-1 2", "2"], out[1:])
|
||||
|
||||
def test_misses_reach_the_filesystem_when_turned_off(self):
|
||||
s = self.staging
|
||||
out = self.run_program("list", s, "access", s / "x.bundle", "access", s / "y.bundle", "accesses",
|
||||
env={"BL_NO_DIR_CACHE": "1"})
|
||||
self.assertEqual(["-1 2", "-1 2", "2"], out[1:])
|
||||
|
||||
def test_can_be_turned_off(self):
|
||||
out = self.run_program("list", self.staging, "list", self.staging, "opens", env={"BL_NO_DIR_CACHE": "1"})
|
||||
self.assertEqual("2", out[-1])
|
||||
|
||||
Reference in new issue
Block a user