mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-19 12:56:06 +00:00
perf(storage): answer repeated SD library checks from the directory cache (#2881)
This commit is contained in:
1 parent
44ade5eeee
commit
f2d8e9e837
4 files changed
+457
-71
No files matched your search
@@ -451,3 +451,69 @@ synchronous directory updates, so SD installs stay bound by them (about 4 to 5 m
|
||||
2,000 files of reserving plus commit on this card). Removing those costs needs fewer
|
||||
directory operations per file, not faster path translation.
|
||||
|
||||
|
||||
## Repeated round trips in the Steam client (2026-11-19)
|
||||
|
||||
Measured on an AYANEO Pocket FIT Elite with a SanDisk 128 GB U1 card (exFAT, `dirsync`,
|
||||
MediaProvider FUSE) as the second Steam library, by timing every file call the Steam
|
||||
client makes under `/steamapps` during a real uninstall and install.
|
||||
|
||||
Where the time went before, for one install of 3,800 files and directories:
|
||||
|
||||
| Call | Total before | Total after |
|
||||
| --- | --- | --- |
|
||||
| `faccessat` | 31.1 s | 0 s |
|
||||
| `stat` | 29.4 s | 0 s |
|
||||
| `access` | 40.3 s | 13.2 s |
|
||||
| `opendir` | 8.6 s | 3.3 s |
|
||||
| `__xstat64` | 5.4 s | about 0 s |
|
||||
| `rename` | 124.8 s | 91 to 106 s |
|
||||
| file creation | about 22 s | about 22 s |
|
||||
|
||||
Most of the removed calls asked the card a question the directory cache could already
|
||||
answer:
|
||||
|
||||
- A directory the cache holds a fresh listing for is now opened from that listing without
|
||||
a `stat` of the directory first, and its own create, rename and remove calls update the
|
||||
listing from what the call returned instead of re-reading the directory and the new
|
||||
entry.
|
||||
- `access`, `faccessat` and `__xstat64` on a name that a fresh listing shows is missing
|
||||
return `ENOENT` without a round trip, including the differently cased spellings the
|
||||
client probes. An absolute path passed with a directory descriptor takes the same path.
|
||||
- The path cache spreads its slots with the high hash bits, which stops install paths that
|
||||
differ only near the end from evicting each other.
|
||||
|
||||
Results on the card (the first four rows are the 3,800 entry install; the moves are
|
||||
single runs timed to about 3 s):
|
||||
|
||||
| Step | Before | After |
|
||||
| --- | --- | --- |
|
||||
| Reserving space | 68.9 s | 27 to 28 s |
|
||||
| Commit | 150.9 s | 97 to 111 s |
|
||||
| Whole install | 261.0 s | 172 to 195 s |
|
||||
| Uninstall | about 31 s | 28.5 to 29.4 s |
|
||||
| Reserving space, second program (4.4 GB, few files) | 33.9 s | 12.5 s |
|
||||
| Uninstall, second program | 12.3 s | 9.2 s |
|
||||
| Steam move, internal storage to card | 156.8 s | 121.7 s |
|
||||
| Steam move, card to internal storage | 56.5 s | 62.0 s |
|
||||
|
||||
What is left is the card: each rename and each file creation is one synchronous directory
|
||||
update, and the second program's 4.4 GB download and final commit took the same time
|
||||
before and after.
|
||||
|
||||
### Launching from the card
|
||||
|
||||
A program started from the card was traced the same way. Three things were measured:
|
||||
|
||||
- Sequential reads through the mount run at the card's own speed (88 MB/s cold), with or
|
||||
without read-ahead hints. A launch that reads 1 GB spends about 12 s there on this card.
|
||||
- A warm directory listing takes about 0.8 ms. A cold one takes about 77 ms for every
|
||||
128 KB directory cluster, even for a directory with one entry, so listing a 167
|
||||
directory install tree takes 12.9 s cold and 0.37 s warm.
|
||||
- The listings stay warm while the device is idle, but a launch that fills memory pushes
|
||||
them out again in bursts, so a program that lists its tree more than once pays the cold
|
||||
price more than once. Keeping the tree warm from outside for the whole launch moved the
|
||||
traced program's start-up forward by 5.5 to 7 s out of about 89 s.
|
||||
|
||||
The rest of that launch was the program's own launcher and CPU-bound start-up, which did
|
||||
not wait on the card in these traces.
|
||||
@@ -23,10 +23,11 @@
|
||||
#define MAX_ENTRIES 100000
|
||||
#define MAX_HANDLES 64
|
||||
#define MAX_UP 32
|
||||
#define MAX_ALIASES 4
|
||||
#define FUSE_MAGIC 0x65735546
|
||||
#define WATCHED (IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO | IN_DELETE_SELF | IN_MOVE_SELF | IN_ONLYDIR)
|
||||
#define ABOVE (IN_DELETE_SELF | IN_MOVE_SELF | IN_ONLYDIR | IN_MASK_ADD)
|
||||
#define GONE (IN_DELETE_SELF | IN_MOVE_SELF | IN_IGNORED)
|
||||
#define WATCHED (IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO | IN_DELETE_SELF | IN_MOVE_SELF | IN_ATTRIB | IN_ONLYDIR)
|
||||
#define ABOVE (IN_DELETE_SELF | IN_MOVE_SELF | IN_ATTRIB | IN_ONLYDIR | IN_MASK_ADD)
|
||||
#define GONE (IN_DELETE_SELF | IN_MOVE_SELF | IN_ATTRIB | IN_IGNORED)
|
||||
#define LINKED (IN_DELETE_SELF | IN_MOVE_SELF | IN_DONT_FOLLOW | IN_MASK_ADD)
|
||||
|
||||
#if __SIZEOF_POINTER__ == 8
|
||||
@@ -54,6 +55,9 @@ struct listing {
|
||||
size_t len;
|
||||
char *where;
|
||||
int folds;
|
||||
int loose;
|
||||
char *alias[MAX_ALIASES];
|
||||
int aliases;
|
||||
};
|
||||
|
||||
struct handle {
|
||||
@@ -78,6 +82,7 @@ static struct listing *table[MAX_LISTINGS];
|
||||
static struct handle *handles[MAX_HANDLES];
|
||||
static int high_water;
|
||||
static int cached_listings;
|
||||
static int blind_listings;
|
||||
static int changing;
|
||||
static int notify = -1;
|
||||
static int forked;
|
||||
@@ -171,6 +176,7 @@ static void destroy(struct listing *l) {
|
||||
free(l->e);
|
||||
free(l->recs);
|
||||
free(l->where);
|
||||
for (int i = 0; i < MAX_ALIASES; i++) free(l->alias[i]);
|
||||
free(l);
|
||||
}
|
||||
|
||||
@@ -201,6 +207,7 @@ static void let_go(int wd, const int *up, int nup) {
|
||||
|
||||
static void detach(struct listing *l) {
|
||||
__atomic_sub_fetch(&cached_listings, 1, __ATOMIC_RELEASE);
|
||||
if (l->blind || l->where == NULL) blind_listings--;
|
||||
l->detached = 1;
|
||||
if (l->refs == 0) destroy(l);
|
||||
}
|
||||
@@ -255,6 +262,7 @@ static void pump(struct expect *x, int nx) {
|
||||
changed(-1, 0);
|
||||
continue;
|
||||
}
|
||||
if ((ev->mask & IN_ATTRIB) && ev->len) continue;
|
||||
int k = 0;
|
||||
while (k < nx && !(x[k].wd == ev->wd && (ev->mask & x[k].mask) && ev->len && strcmp(ev->name, x[k].name) == 0)) k++;
|
||||
if (k < nx) {
|
||||
@@ -304,6 +312,7 @@ static int push(struct listing *l, ino_t ino, unsigned char type, const char *na
|
||||
static void install(struct listing *l) {
|
||||
l->detached = 0;
|
||||
__atomic_add_fetch(&cached_listings, 1, __ATOMIC_RELEASE);
|
||||
if (l->blind || l->where == NULL) blind_listings++;
|
||||
int slot = find(l->dev, l->ino);
|
||||
if (slot >= 0) {
|
||||
struct listing *old = table[slot];
|
||||
@@ -416,10 +425,104 @@ __attribute__((constructor)) static void resolve(void) {
|
||||
}
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
static int named(const struct listing *l, const char *name) {
|
||||
for (size_t i = 0; i < l->n; i++)
|
||||
if (strcasecmp(l->e[i].name, name) == 0) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int folded_match(const char *where, const char *dir) {
|
||||
const char *at = strstr(where, "/steamapps");
|
||||
if (at == NULL) return 0;
|
||||
size_t n = (size_t)(at - where);
|
||||
return strncmp(where, dir, n) == 0 && strcasecmp(where + n, dir + n) == 0;
|
||||
}
|
||||
|
||||
static int same_dir(const struct listing *l, const char *dir) {
|
||||
struct stat st;
|
||||
int saved = errno;
|
||||
int same = stat(dir, &st) == 0 && st.st_dev == l->dev && st.st_ino == l->ino;
|
||||
errno = saved;
|
||||
return same;
|
||||
}
|
||||
|
||||
static int plain(const char *where, char *dir) {
|
||||
const char *at = strstr(where, "/steamapps");
|
||||
int saved = errno, fine = at != NULL;
|
||||
for (size_t from = fine ? (size_t)(at - where) + 1 : 0; fine && dir[from];) {
|
||||
size_t to = from + strcspn(dir + from, "/");
|
||||
char kept = dir[to];
|
||||
struct stat st;
|
||||
dir[to] = 0;
|
||||
fine = strcmp(dir + from, ".") != 0 && strcmp(dir + from, "..") != 0 && lstat(dir, &st) == 0 && !S_ISLNK(st.st_mode);
|
||||
dir[to] = kept;
|
||||
from = kept ? to + 1 : to;
|
||||
}
|
||||
errno = saved;
|
||||
return fine;
|
||||
}
|
||||
|
||||
static int aliased(struct listing *l, const char *dir) {
|
||||
for (int i = 0; i < MAX_ALIASES; i++)
|
||||
if (l->alias[i] && strcmp(l->alias[i], dir) == 0) return 1;
|
||||
if (!same_dir(l, dir)) return 0;
|
||||
char *copy = strdup(dir);
|
||||
if (copy && !plain(l->where, copy)) {
|
||||
free(copy);
|
||||
copy = NULL;
|
||||
}
|
||||
if (copy) {
|
||||
int at = l->aliases++ % MAX_ALIASES;
|
||||
free(l->alias[at]);
|
||||
l->alias[at] = copy;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static struct listing *listed(const char *dir, long long now) {
|
||||
for (int i = 0; i < MAX_LISTINGS; i++) {
|
||||
struct listing *l = table[i];
|
||||
if (l == NULL || l->where == NULL || l->blind || now - l->filled >= DIR_TTL_NS) continue;
|
||||
if (strcmp(l->where, dir) == 0) return l;
|
||||
if (l->folds && folded_match(l->where, dir)) return aliased(l, dir) ? l : NULL;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int watching(const char *path) {
|
||||
return eligible(path) && __atomic_load_n(&cached_listings, __ATOMIC_ACQUIRE) > 0;
|
||||
}
|
||||
|
||||
static DIR *listed_open(struct handle *h, long long now) {
|
||||
DIR *d = NULL;
|
||||
if (__atomic_load_n(&cached_listings, __ATOMIC_ACQUIRE) == 0) return NULL;
|
||||
take(&lock);
|
||||
if (!changing) {
|
||||
pump(NULL, 0);
|
||||
struct listing *l = listed(h->path, now);
|
||||
if (l && records(l) == 0) {
|
||||
h->snap = l;
|
||||
l->refs++;
|
||||
l->used = now;
|
||||
d = wrap(h);
|
||||
if (d == NULL) {
|
||||
unref(l);
|
||||
h->snap = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
give(&lock);
|
||||
return d;
|
||||
}
|
||||
|
||||
static DIR *hit(const struct stat *st, struct handle *h, long long now) {
|
||||
take(&lock);
|
||||
if (!changing) pump(NULL, 0);
|
||||
int slot = find(st->st_dev, st->st_ino);
|
||||
if (slot >= 0 && table[slot]->loose && now - table[slot]->filled < DIR_TTL_NS) {
|
||||
stamp(table[slot], st);
|
||||
table[slot]->loose = 0;
|
||||
}
|
||||
if (slot >= 0 && same(table[slot], st) && now - table[slot]->filled < DIR_TTL_NS && records(table[slot]) == 0) {
|
||||
h->snap = table[slot];
|
||||
h->snap->refs++;
|
||||
@@ -509,8 +612,7 @@ static void untrack(struct handle *h) {
|
||||
DIR *opendir(const char *path) {
|
||||
NEXT(opendir);
|
||||
struct stat st;
|
||||
if (!eligible(path) || stat(path, &st) != 0 || !S_ISDIR(st.st_mode) || !on_fuse(st.st_dev, path))
|
||||
return next_opendir(path);
|
||||
if (!eligible(path)) return next_opendir(path);
|
||||
struct handle *h = calloc(1, sizeof(*h));
|
||||
char *copy = strdup(path);
|
||||
if (h == NULL || copy == NULL) {
|
||||
@@ -522,7 +624,14 @@ DIR *opendir(const char *path) {
|
||||
h->wd = -1;
|
||||
h->path = copy;
|
||||
long long now = now_ns();
|
||||
DIR *d = hit(&st, h, now);
|
||||
DIR *d = listed_open(h, now);
|
||||
if (d) return d;
|
||||
if (stat(path, &st) != 0 || !S_ISDIR(st.st_mode) || !on_fuse(st.st_dev, path)) {
|
||||
free(h->path);
|
||||
free(h);
|
||||
return next_opendir(path);
|
||||
}
|
||||
d = hit(&st, h, now);
|
||||
if (d) return d;
|
||||
if (!track(h, path)) {
|
||||
free(h->path);
|
||||
@@ -699,7 +808,9 @@ void seekdir(DIR *d, long pos) {
|
||||
}
|
||||
}
|
||||
|
||||
struct side { char dir[PATH_MAX]; const char *name; struct stat before; int known; };
|
||||
struct side { char dir[PATH_MAX]; const char *name; struct stat before; int known; int trusted; };
|
||||
|
||||
struct made { int known; ino_t ino; unsigned char type; };
|
||||
|
||||
static int split(const char *path, struct side *s) {
|
||||
size_t n = strlen(path);
|
||||
@@ -713,10 +824,6 @@ static int split(const char *path, struct side *s) {
|
||||
return strchr(s->name, '/') == NULL && strcmp(s->name, ".") != 0 && strcmp(s->name, "..") != 0;
|
||||
}
|
||||
|
||||
static int watching(const char *path) {
|
||||
return eligible(path) && __atomic_load_n(&cached_listings, __ATOMIC_ACQUIRE) > 0;
|
||||
}
|
||||
|
||||
static void begin(void) {
|
||||
take(&lock);
|
||||
pump(NULL, 0);
|
||||
@@ -725,13 +832,42 @@ static void begin(void) {
|
||||
}
|
||||
|
||||
static void before(struct side *s) {
|
||||
s->known = stat(s->dir, &s->before) == 0;
|
||||
take(&lock);
|
||||
struct listing *l = listed(s->dir, now_ns());
|
||||
int unlisted = l == NULL && blind_listings == 0;
|
||||
s->trusted = l != NULL;
|
||||
if (l) {
|
||||
memset(&s->before, 0, sizeof(s->before));
|
||||
s->before.st_dev = l->dev;
|
||||
s->before.st_ino = l->ino;
|
||||
s->before.st_size = l->size;
|
||||
s->before.st_nlink = l->nlink;
|
||||
s->before.st_mtim = l->mtime;
|
||||
s->before.st_ctim = l->ctime;
|
||||
}
|
||||
give(&lock);
|
||||
s->known = s->trusted || (!unlisted && stat(s->dir, &s->before) == 0);
|
||||
}
|
||||
|
||||
static unsigned char type_of(mode_t mode) {
|
||||
return S_ISDIR(mode) ? DT_DIR : S_ISLNK(mode) ? DT_LNK : S_ISREG(mode) ? DT_REG : DT_UNKNOWN;
|
||||
}
|
||||
|
||||
static int made_at(const struct side *s, const char *target, struct made *m) {
|
||||
struct stat t;
|
||||
if (s->trusted && m && m->known) return 1;
|
||||
if (lstat(target, &t) != 0) return 0;
|
||||
*m = (struct made){1, t.st_ino, type_of(t.st_mode)};
|
||||
return 1;
|
||||
}
|
||||
|
||||
enum { ADDED, REMOVED };
|
||||
|
||||
static int patch(struct side *s, int what, uint32_t event, int ok, int error, const char *target, struct expect *x) {
|
||||
static int patch(struct side *s, int what, uint32_t event, int ok, int error, const char *target, struct made *m,
|
||||
struct expect *x) {
|
||||
struct stat now;
|
||||
struct made local = {0};
|
||||
if (m == NULL) m = &local;
|
||||
if (!s->known) return 0;
|
||||
int slot = find(s->before.st_dev, s->before.st_ino);
|
||||
if (slot < 0) return 0;
|
||||
@@ -750,7 +886,7 @@ static int patch(struct side *s, int what, uint32_t event, int ok, int error, co
|
||||
drop(slot);
|
||||
return 0;
|
||||
}
|
||||
if (stat(s->dir, &now) != 0 || now.st_ino != l->ino) {
|
||||
if (!s->trusted && (stat(s->dir, &now) != 0 || now.st_ino != l->ino)) {
|
||||
drop(slot);
|
||||
return 0;
|
||||
}
|
||||
@@ -771,6 +907,7 @@ static int patch(struct side *s, int what, uint32_t event, int ok, int error, co
|
||||
copy->blind = l->blind;
|
||||
copy->where = l->where ? strdup(l->where) : NULL;
|
||||
copy->folds = l->folds;
|
||||
copy->loose = l->loose;
|
||||
copy->detached = 0;
|
||||
stamp(copy, &s->before);
|
||||
table[slot] = copy;
|
||||
@@ -781,18 +918,17 @@ static int patch(struct side *s, int what, uint32_t event, int ok, int error, co
|
||||
l->recs = NULL;
|
||||
l->len = 0;
|
||||
int fine = 1, expected = 1;
|
||||
struct stat t;
|
||||
if (what == ADDED) {
|
||||
if (exact < l->n) {
|
||||
if (lstat(target, &t) == 0) l->e[exact].ino = t.st_ino;
|
||||
if (event == IN_MOVED_TO && made_at(s, target, m)) l->e[exact].ino = m->ino;
|
||||
expected = event == IN_MOVED_TO;
|
||||
} else if (folded < l->n || lstat(target, &t) != 0) {
|
||||
} else if (folded < l->n || !made_at(s, target, m)) {
|
||||
fine = 0;
|
||||
} else {
|
||||
unsigned char type = S_ISDIR(t.st_mode) ? DT_DIR : S_ISLNK(t.st_mode) ? DT_LNK : S_ISREG(t.st_mode) ? DT_REG : DT_UNKNOWN;
|
||||
fine = l->n < MAX_ENTRIES && push(l, t.st_ino, type, s->name) == 0;
|
||||
fine = l->n < MAX_ENTRIES && push(l, m->ino, m->type, s->name) == 0;
|
||||
}
|
||||
} else if (exact < l->n) {
|
||||
*m = (struct made){1, l->e[exact].ino, l->e[exact].type};
|
||||
free(l->e[exact].name);
|
||||
memmove(&l->e[exact], &l->e[exact + 1], (l->n - exact - 1) * sizeof(l->e[0]));
|
||||
l->n--;
|
||||
@@ -803,7 +939,8 @@ static int patch(struct side *s, int what, uint32_t event, int ok, int error, co
|
||||
drop(slot);
|
||||
return 0;
|
||||
}
|
||||
stamp(l, &now);
|
||||
if (s->trusted) l->loose = 1;
|
||||
else stamp(l, &now);
|
||||
if (!expected) return 0;
|
||||
*x = (struct expect){l->wd, event, s->name};
|
||||
return 1;
|
||||
@@ -829,8 +966,11 @@ int dd_dircache_create(int dirfd, const char *path, int flags, mode_t mode,
|
||||
int fd = real(dirfd, path, flags, mode);
|
||||
int saved = errno;
|
||||
struct expect x;
|
||||
struct stat opened;
|
||||
struct made m = {0};
|
||||
if (fd >= 0 && s.trusted && fstat(fd, &opened) == 0) m = (struct made){1, opened.st_ino, type_of(opened.st_mode)};
|
||||
take(&lock);
|
||||
int nx = patch(&s, ADDED, IN_CREATE, fd >= 0, saved, path, &x);
|
||||
int nx = patch(&s, ADDED, IN_CREATE, fd >= 0, saved, path, &m, &x);
|
||||
give(&lock);
|
||||
end(&x, nx);
|
||||
give(&oplock);
|
||||
@@ -850,7 +990,7 @@ int dd_dircache_create(int dirfd, const char *path, int flags, mode_t mode,
|
||||
int saved = errno; \
|
||||
struct expect x; \
|
||||
take(&lock); \
|
||||
int nx = patch(&s, what, event, r == 0, saved, what == ADDED ? path : NULL, &x); \
|
||||
int nx = patch(&s, what, event, r == 0, saved, what == ADDED ? path : NULL, NULL, &x); \
|
||||
if (r == 0 && what == REMOVED) forget_below(path); \
|
||||
give(&lock); \
|
||||
end(&x, nx); \
|
||||
@@ -885,12 +1025,13 @@ ONE_PATH(remove, REMOVED, IN_DELETE, dd_descriptor_remove(path, next_remove))
|
||||
int r = call(from, to); \
|
||||
int saved = errno; \
|
||||
struct expect x[2]; \
|
||||
struct made m = {0}; \
|
||||
int nx = 0; \
|
||||
take(&lock); \
|
||||
if (track_a) nx += patch(&a, REMOVED, IN_MOVED_FROM, r == 0, saved, NULL, &x[nx]); \
|
||||
if (track_a) nx += patch(&a, REMOVED, IN_MOVED_FROM, r == 0, saved, NULL, &m, &x[nx]); \
|
||||
if (track_b) { \
|
||||
if (track_a && strcmp(a.dir, b.dir) == 0) b.known = stat(b.dir, &b.before) == 0; \
|
||||
nx += patch(&b, ADDED, event, r == 0, saved, to, &x[nx]); \
|
||||
if (track_a && !b.trusted && strcmp(a.dir, b.dir) == 0) b.known = stat(b.dir, &b.before) == 0; \
|
||||
nx += patch(&b, ADDED, event, r == 0, saved, to, &m, &x[nx]); \
|
||||
} \
|
||||
if (r == 0 && gone) { forget_below(from); forget_below(to); } \
|
||||
give(&lock); \
|
||||
@@ -907,37 +1048,6 @@ TWO_PATHS(rename, 1, IN_MOVED_TO, moved)
|
||||
TWO_PATHS(link, 0, IN_CREATE, link_or_lock)
|
||||
TWO_PATHS(symlink, 0, IN_CREATE, next_symlink)
|
||||
|
||||
static int named(const struct listing *l, const char *name) {
|
||||
for (size_t i = 0; i < l->n; i++)
|
||||
if (strcasecmp(l->e[i].name, name) == 0) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int folded_match(const char *where, const char *dir) {
|
||||
const char *at = strstr(where, "/steamapps");
|
||||
if (at == NULL) return 0;
|
||||
size_t n = (size_t)(at - where);
|
||||
return strncmp(where, dir, n) == 0 && strcasecmp(where + n, dir + n) == 0;
|
||||
}
|
||||
|
||||
static int same_dir(const struct listing *l, const char *dir) {
|
||||
struct stat st;
|
||||
int saved = errno;
|
||||
int same = stat(dir, &st) == 0 && st.st_dev == l->dev && st.st_ino == l->ino;
|
||||
errno = saved;
|
||||
return same;
|
||||
}
|
||||
|
||||
static int listed_absent(const char *dir, const char *name, long long now) {
|
||||
for (int i = 0; i < MAX_LISTINGS; i++) {
|
||||
struct listing *l = table[i];
|
||||
if (l == NULL || l->where == NULL || l->blind || now - l->filled >= DIR_TTL_NS) continue;
|
||||
if (strcmp(l->where, dir) == 0) return !named(l, name);
|
||||
if (l->folds && folded_match(l->where, dir)) return same_dir(l, dir) && !named(l, name);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int dd_dircache_absent(const char *path) __attribute__((visibility("hidden")));
|
||||
int dd_dircache_absent(const char *path) {
|
||||
struct side s;
|
||||
@@ -949,7 +1059,8 @@ int dd_dircache_absent(const char *path) {
|
||||
int absent = 0;
|
||||
if (!changing) {
|
||||
pump(NULL, 0);
|
||||
absent = listed_absent(s.dir, s.name, now);
|
||||
struct listing *l = listed(s.dir, now);
|
||||
absent = l != NULL && !named(l, s.name);
|
||||
}
|
||||
give(&lock);
|
||||
return absent;
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include <pthread.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
@@ -55,12 +56,16 @@ static unsigned hash_of(const char *s, int kind, int mode, int flags) {
|
||||
return h;
|
||||
}
|
||||
|
||||
static struct entry *slot(unsigned h) {
|
||||
return &table[(h ^ (h >> 16)) % PATH_CACHE_SLOTS];
|
||||
}
|
||||
|
||||
static int cacheable(const char *path) {
|
||||
return path != NULL && path[0] == '/' && strlen(path) < PATH_CACHE_KEY_MAX && enabled();
|
||||
}
|
||||
|
||||
static struct entry *find(const char *path, int kind, int mode, int flags, unsigned h) {
|
||||
struct entry *e = &table[h % PATH_CACHE_SLOTS];
|
||||
struct entry *e = slot(h);
|
||||
|
||||
if (e->stamp == 0 || e->hash != h || e->kind != kind || e->mode != mode || e->flags != flags) return NULL;
|
||||
if (now_ns() - e->stamp > PATH_CACHE_TTL_NS || strcmp(e->key, path) != 0) return NULL;
|
||||
@@ -68,7 +73,7 @@ static struct entry *find(const char *path, int kind, int mode, int flags, unsig
|
||||
}
|
||||
|
||||
static void store(const char *path, int kind, int mode, int flags, unsigned h, const char *value) {
|
||||
struct entry *e = &table[h % PATH_CACHE_SLOTS];
|
||||
struct entry *e = slot(h);
|
||||
char *copy = value != NULL ? strdup(value) : NULL;
|
||||
|
||||
if (value != NULL && copy == NULL) return;
|
||||
@@ -172,6 +177,10 @@ static int check_cwd(const char *path, int mode, int flags) {
|
||||
return result;
|
||||
}
|
||||
|
||||
static int absolute(const char *path) {
|
||||
return path != NULL && path[0] == '/';
|
||||
}
|
||||
|
||||
int access(const char *path, int mode) {
|
||||
return cached_access(path, mode, 0, check_cwd);
|
||||
}
|
||||
@@ -188,7 +197,7 @@ int faccessat(int dirfd, const char *path, int mode, int flags) {
|
||||
int (*real)(int, const char *, int, int);
|
||||
int err;
|
||||
|
||||
if (dirfd == AT_FDCWD) {
|
||||
if (dirfd == AT_FDCWD || absolute(path)) {
|
||||
return cached_access(path, mode, flags, check_cwd);
|
||||
}
|
||||
real = next_faccessat();
|
||||
@@ -202,3 +211,15 @@ int faccessat(int dirfd, const char *path, int mode, int flags) {
|
||||
errno = err;
|
||||
return -1;
|
||||
}
|
||||
|
||||
int __xstat64(int version, const char *path, struct stat64 *out) {
|
||||
static int (*real)(int, const char *, struct stat64 *);
|
||||
|
||||
if (dd_dircache_absent && dd_dircache_absent(path)) {
|
||||
errno = ENOENT;
|
||||
return -1;
|
||||
}
|
||||
if (real == NULL) real = (int (*)(int, const char *, struct stat64 *))dlsym(RTLD_NEXT, "__xstat64");
|
||||
if (real != NULL) return real(version, path, out);
|
||||
return stat64(path, out);
|
||||
}
|
||||
@@ -24,6 +24,7 @@ FAKE = r"""
|
||||
|
||||
int real_opendirs;
|
||||
int real_accesses;
|
||||
int real_stats;
|
||||
|
||||
static void coarse(const char *path, struct stat *st) {
|
||||
if (getenv("COARSE") && strstr(path, "/fuse/") && S_ISDIR(st->st_mode)) {
|
||||
@@ -33,25 +34,44 @@ static void coarse(const char *path, struct stat *st) {
|
||||
}
|
||||
}
|
||||
|
||||
int stat(const char *path, struct stat *st) {
|
||||
static int (*next)(const char *, struct stat *);
|
||||
if (!next) next = dlsym(RTLD_NEXT, "stat");
|
||||
char folded[4096];
|
||||
static const char *fold_case(const char *path, char *folded, size_t size) {
|
||||
const char *fold = getenv("FOLD");
|
||||
const char *upper = fold ? strstr(path, "/STEAMAPPS") : NULL;
|
||||
if (!upper && fold && !strcmp(fold, "all")) upper = strcasestr(path, "/steamapps/");
|
||||
if (upper && strlen(path) < sizeof(folded)) {
|
||||
strcpy(folded, path);
|
||||
memcpy(folded + (upper - path) + 1, "steamapps", 9);
|
||||
if (!strcmp(fold, "all"))
|
||||
for (char *c = folded + (upper - path); *c; c++) if (*c >= 'A' && *c <= 'Z') *c += 'a' - 'A';
|
||||
path = folded;
|
||||
}
|
||||
if (!upper || strlen(path) >= size) return path;
|
||||
strcpy(folded, path);
|
||||
memcpy(folded + (upper - path) + 1, "steamapps", 9);
|
||||
if (!strcmp(fold, "all"))
|
||||
for (char *c = folded + (upper - path); *c; c++) if (*c >= 'A' && *c <= 'Z') *c += 'a' - 'A';
|
||||
return folded;
|
||||
}
|
||||
|
||||
static int fold_stat(const char *name, const char *path, struct stat *st) {
|
||||
int (*next)(const char *, struct stat *) = dlsym(RTLD_NEXT, name);
|
||||
char folded[4096];
|
||||
if (strstr(path, "/fuse/")) real_stats++;
|
||||
int r = next(path, st);
|
||||
if (r != 0) r = next(fold_case(path, folded, sizeof(folded)), st);
|
||||
return r;
|
||||
}
|
||||
|
||||
int stat(const char *path, struct stat *st) {
|
||||
int r = fold_stat("stat", path, st);
|
||||
if (r == 0) coarse(path, st);
|
||||
return r;
|
||||
}
|
||||
|
||||
int lstat(const char *path, struct stat *st) {
|
||||
return fold_stat("lstat", path, st);
|
||||
}
|
||||
|
||||
int __xstat64(int version, const char *path, struct stat64 *st) {
|
||||
static int (*next)(int, const char *, struct stat64 *);
|
||||
if (!next) next = dlsym(RTLD_NEXT, "__xstat64");
|
||||
if (strstr(path, "/fuse/")) real_stats++;
|
||||
return next ? next(version, path, st) : stat64(path, st);
|
||||
}
|
||||
|
||||
int fstat(int fd, struct stat *st) {
|
||||
static int (*next)(int, struct stat *);
|
||||
if (!next) next = dlsym(RTLD_NEXT, "fstat");
|
||||
@@ -221,6 +241,7 @@ static void stress(const char *dir) {
|
||||
int main(int argc, char **argv) {
|
||||
int *opens = dlsym(RTLD_DEFAULT, "real_opendirs");
|
||||
int *accesses = dlsym(RTLD_DEFAULT, "real_accesses");
|
||||
int *stats = dlsym(RTLD_DEFAULT, "real_stats");
|
||||
for (int i = 1; i < argc; i++) {
|
||||
const char *op = argv[i], *a = argv[i + 1];
|
||||
if (!strcmp(op, "list")) { list(a); i++; }
|
||||
@@ -309,6 +330,27 @@ int main(int argc, char **argv) {
|
||||
printf("%d %d\n", r, r ? errno : 0);
|
||||
i++;
|
||||
}
|
||||
else if (!strcmp(op, "accessat")) {
|
||||
int r = faccessat(STDIN_FILENO, a, F_OK, 0);
|
||||
printf("%d %d\n", r, r ? errno : 0);
|
||||
i++;
|
||||
}
|
||||
else if (!strcmp(op, "xstat")) {
|
||||
int (*old)(int, const char *, struct stat64 *) = dlsym(RTLD_DEFAULT, "__xstat64");
|
||||
struct stat64 st;
|
||||
int r = old ? old(0, a, &st) : -2;
|
||||
printf("%d %d\n", r, r ? errno : 0);
|
||||
i++;
|
||||
}
|
||||
else if (!strcmp(op, "inode")) {
|
||||
DIR *d = opendir(a);
|
||||
struct dirent *e;
|
||||
while (d && (e = readdir(d)))
|
||||
if (!strcmp(e->d_name, argv[i + 2])) printf("%llu\n", (unsigned long long)e->d_ino);
|
||||
if (d) closedir(d);
|
||||
i += 2;
|
||||
}
|
||||
else if (!strcmp(op, "stats")) printf("%d\n", stats ? *stats : -1);
|
||||
else if (!strcmp(op, "accesses")) printf("%d\n", accesses ? *accesses : -1);
|
||||
else if (!strcmp(op, "opens")) printf("%d\n", opens ? *opens : -1);
|
||||
}
|
||||
@@ -675,6 +717,152 @@ class DirCachePreloadTest(unittest.TestCase):
|
||||
"access", s / "x.bundle", "accesses", "opens", between=churn)
|
||||
self.assertEqual([s.name, "a.bundle,b.bundle", s.name, "a.bundle,b.bundle", "-1 2", "0", "3"], out)
|
||||
|
||||
def test_absolute_paths_checked_against_a_descriptor_are_answered_from_the_listing(self):
|
||||
s = self.staging
|
||||
out = self.run_program("list", s, "accessat", s / "x.bundle", "accessat", s / "y.bundle", "accesses",
|
||||
"accessat", s / "a.bundle", "accessat", s / "a.bundle", "accesses",
|
||||
"create", s / "x.bundle", "accessat", s / "x.bundle")
|
||||
self.assertEqual(["-1 2", "-1 2", "0", "0 0", "0 0", "1", "0 0"], out[1:])
|
||||
|
||||
def test_absolute_paths_checked_against_a_descriptor_elsewhere_reach_the_filesystem(self):
|
||||
s = self.staging
|
||||
other = self.run_program("list", s, "accessat", s / "x.bundle", "accessat", s / "x.bundle", "accesses",
|
||||
program=self.other)
|
||||
self.assertEqual(["-1 2", "-1 2", "2"], other[1:])
|
||||
off = self.run_program("list", s, "accessat", s / "x.bundle", "accessat", s / "x.bundle", "accesses",
|
||||
env={"DD_NO_DIR_CACHE": "1", "DD_NO_PATH_CACHE": "1"})
|
||||
self.assertEqual(["-1 2", "-1 2", "2"], off[1:])
|
||||
|
||||
def test_missing_names_asked_through_the_old_stat_entry_are_answered_from_the_listing(self):
|
||||
s = self.staging
|
||||
out = self.run_program("list", s, "stats", "xstat", s / "x.bundle", "stats", "xstat", s / "a.bundle", "stats",
|
||||
"create", s / "x.bundle", "xstat", s / "x.bundle")
|
||||
self.assertEqual(["-1 2", out[1], "0 0", str(int(out[1]) + 1), "0 0"], out[2:])
|
||||
other = self.run_program("list", s, "stats", "xstat", s / "z.bundle", "stats", program=self.other)
|
||||
self.assertEqual(["-1 2", str(int(other[1]) + 1)], other[2:])
|
||||
|
||||
def test_folders_that_differ_only_in_case_stay_cached_together(self):
|
||||
s = self.staging
|
||||
names = ["Content/Data", "content/data", "Content/Characters", "content/characters"]
|
||||
for name in names:
|
||||
(s / name).mkdir(parents=True)
|
||||
checks = [op for name in names for op in ("access", s / name)]
|
||||
out = self.run_program(*checks, "accesses", *checks, *checks, "accesses")
|
||||
self.assertEqual(out[4], out[-1])
|
||||
|
||||
def test_a_cached_listing_is_served_without_asking_the_filesystem(self):
|
||||
s = self.staging
|
||||
out = self.run_program("list", s, "stats", "list", s, "list", s, "access", s / "x.bundle", "stats", "opens")
|
||||
self.assertEqual(out[1], out[-2])
|
||||
self.assertEqual("1", out[-1])
|
||||
|
||||
def test_own_changes_in_a_cached_folder_do_not_ask_the_filesystem(self):
|
||||
s = self.staging
|
||||
out = self.run_program(
|
||||
"list", s, "stats",
|
||||
"create", s / "c.bundle", "rename", s / "c.bundle", s / "d.bundle", "unlink", s / "a.bundle", "stats",
|
||||
"list", s, "raw", s, "opens")
|
||||
self.assertEqual(out[1], out[2])
|
||||
self.assertEqual(["b.bundle,d.bundle", "b.bundle,d.bundle", "1"], out[3:])
|
||||
|
||||
def test_own_changes_in_a_folder_never_listed_do_not_ask_the_filesystem(self):
|
||||
s = self.staging
|
||||
out = self.run_program("stats", "create", s / "c.bundle", "rename", s / "c.bundle", s / "d.bundle",
|
||||
"unlink", s / "d.bundle", "stats", "list", s, "raw", s)
|
||||
self.assertEqual(out[0], out[1])
|
||||
self.assertEqual(["a.bundle,b.bundle", "a.bundle,b.bundle"], out[2:])
|
||||
|
||||
def test_a_differently_cased_folder_is_checked_once(self):
|
||||
s = self.staging
|
||||
upper = Path(str(s).replace("downloading", "DOWNLOADING"))
|
||||
out = self.run_program("list", s, "access", upper / "x.bundle", "stats",
|
||||
"access", upper / "y.bundle", "access", upper / "z.bundle", "stats", "accesses",
|
||||
env={"FOLD": "all"})
|
||||
self.assertEqual(out[2], out[5])
|
||||
self.assertEqual("0", out[-1])
|
||||
|
||||
def test_a_folder_reached_through_a_link_follows_the_link(self):
|
||||
s = self.staging
|
||||
other = self.lib / "elsewhere" / s.name
|
||||
other.mkdir(parents=True)
|
||||
(other / "x.bundle").touch()
|
||||
link = self.lib / "DOWNLOADING"
|
||||
link.symlink_to("downloading")
|
||||
|
||||
def repoint():
|
||||
link.unlink()
|
||||
link.symlink_to("elsewhere")
|
||||
|
||||
out = self.run_paused("list", s, "access", link / s.name / "x.bundle", "pause",
|
||||
"access", link / s.name / "x.bundle", between=repoint, env={"FOLD": "all"})
|
||||
self.assertEqual(["-1 2", "0 0"], out[1:])
|
||||
|
||||
def test_a_linked_last_component_follows_the_link(self):
|
||||
s = self.staging
|
||||
other = self.lib / "elsewhere" / s.name
|
||||
other.mkdir(parents=True)
|
||||
(other / "x.bundle").touch()
|
||||
twin = self.lib / "DOWNLOADING"
|
||||
twin.mkdir()
|
||||
link = twin / s.name
|
||||
link.symlink_to(s)
|
||||
|
||||
def repoint():
|
||||
link.unlink()
|
||||
link.symlink_to(other)
|
||||
|
||||
out = self.run_paused("list", s, "access", link / "x.bundle", "pause",
|
||||
"access", link / "x.bundle", between=repoint, env={"FOLD": "all"})
|
||||
self.assertEqual(["-1 2", "0 0"], out[1:])
|
||||
|
||||
def test_a_folder_reached_through_a_parent_step_is_checked_each_time(self):
|
||||
s = self.staging
|
||||
other = self.lib / "elsewhere"
|
||||
(other / "branch").mkdir(parents=True)
|
||||
(other / s.parent.name / s.name).mkdir(parents=True)
|
||||
(other / s.parent.name / s.name / "x.bundle").touch()
|
||||
(self.lib / "branch").mkdir()
|
||||
twin = self.lib / "BRANCH"
|
||||
twin.mkdir()
|
||||
listed = self.lib / "branch" / ".." / s.parent.name / s.name
|
||||
through = twin / ".." / s.parent.name / s.name
|
||||
|
||||
def repoint():
|
||||
twin.rmdir()
|
||||
twin.symlink_to(other / "branch")
|
||||
|
||||
out = self.run_paused("list", listed, "access", through / "x.bundle", "pause",
|
||||
"access", through / "x.bundle", between=repoint, env={"FOLD": "all"})
|
||||
self.assertEqual(["-1 2", "0 0"], out[1:])
|
||||
|
||||
def test_a_permission_change_on_the_folder_or_above_it_is_noticed(self):
|
||||
s = self.staging
|
||||
for target in (s, s.parent):
|
||||
with self.subTest(target=target.name):
|
||||
out = self.run_paused("list", s, "opens", "pause", "list", s, "opens",
|
||||
between=lambda: target.chmod(0o700))
|
||||
self.assertEqual(int(out[1]) + 1, int(out[3]))
|
||||
|
||||
def test_a_permission_change_on_a_file_keeps_the_listing(self):
|
||||
s = self.staging
|
||||
out = self.run_paused("list", s, "opens", "pause", "list", s, "opens",
|
||||
between=lambda: (s / "a.bundle").chmod(0o600))
|
||||
self.assertEqual(out[1], out[3])
|
||||
|
||||
def test_opening_an_existing_link_keeps_its_entry(self):
|
||||
s = self.staging
|
||||
(s / "l.bundle").symlink_to("a.bundle")
|
||||
out = self.run_program("list", s, "create", s / "l.bundle", "inode", s, "l.bundle", "raw", s)
|
||||
self.assertEqual(str((s / "l.bundle").lstat().st_ino), out[1])
|
||||
|
||||
def test_changes_made_around_the_cache_after_own_changes_are_noticed(self):
|
||||
s = self.staging
|
||||
out = self.run_program("list", s, "create", s / "c.bundle", "sneak", s / "y.bundle", "list", s, "raw", s,
|
||||
"access", s / "y.bundle")
|
||||
self.assertEqual(out[1], out[2])
|
||||
self.assertIn("y.bundle", out[1])
|
||||
self.assertEqual("0 0", out[3])
|
||||
|
||||
def test_misses_outside_the_client_or_fuse_steamapps_reach_the_filesystem(self):
|
||||
s = self.staging
|
||||
other = self.run_program("list", s, "access", s / "x.bundle", "access", s / "y.bundle", "accesses",
|
||||
|
||||
Reference in new issue
Block a user