perf(proot): expand and cache syscall relays

This commit is contained in:
Kurt Himebauch committed 2026-10-28 07:17:53 -04:00
1 parent 785d1bf1da
commit c62b9b5d1b
9 files changed
+719 -81

No files matched your search

+90 -54
View File
@@ -931,29 +931,26 @@ static long statx_call(int dirfd, const char *path, int flags, unsigned mask, st
return 0;
}
int statx(int dirfd, const char *path, int flags, unsigned mask, struct statx *out) {
REAL(statx, int (*)(int, const char *, int, unsigned, struct statx *));
static long fp_statx(int dirfd, const char *path, int flags, unsigned mask, struct statx *out) {
struct res r;
long rs;
if (mask & STATX__RESERVED) { errno = EINVAL; return -1; }
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wnonnull-compare"
if (!out) { errno = EFAULT; return -1; }
if (!path) {
if (!(flags & AT_EMPTY_PATH)) { errno = EFAULT; return -1; }
return real_statx ? real_statx(dirfd, path, flags, mask, out) : ret(sc(SYS_statx, dirfd, (long)path, flags, mask, (long)out));
}
#pragma GCC diagnostic pop
if ((flags & AT_EMPTY_PATH) && !path[0] && fp_ready()) {
hits++;
return ret(statx_call(dirfd, "", flags, mask, out));
}
if (mask & STATX__RESERVED) return -EINVAL;
if (!out) return -EFAULT;
if (!path) return flags & AT_EMPTY_PATH ? FP_SLOW : -EFAULT;
if ((flags & AT_EMPTY_PATH) && !path[0] && fp_ready())
return statx_call(dirfd, "", flags, mask, out);
rs = (flags & ~(AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT | AT_STATX_SYNC_TYPE)) ? FP_SLOW
: resolve(dirfd, path, (flags & AT_SYMLINK_NOFOLLOW ? 0 : 1) | MAY_BE_UNSEEN, &r);
if (rs == 1) rs = -ENOENT;
if (rs == 0 || rs == UNSEEN)
rs = statx_call(AT_FDCWD, r.host, flags | (rs == 0 && !r.magic ? AT_SYMLINK_NOFOLLOW : 0), mask, out);
ANSWER(rs, dirfd, path, real_statx(dirfd, path, flags, mask, out));
return rs;
}
int statx(int dirfd, const char *path, int flags, unsigned mask, struct statx *out) {
REAL(statx, int (*)(int, const char *, int, unsigned, struct statx *));
ANSWER(fp_statx(dirfd, path, flags, mask, out), dirfd, path,
real_statx ? real_statx(dirfd, path, flags, mask, out) : ret(sc(SYS_statx, dirfd, (long)path, flags, mask, (long)out)));
}
@@ -961,8 +958,7 @@ static int effective_is_real(void) {
return !getauxval(AT_SECURE) && getuid() == geteuid() && getgid() == getegid();
}
static int access_common(int dirfd, const char *path, int mode, int flags) {
REAL(faccessat, int (*)(int, const char *, int, int));
static long fp_access(int dirfd, const char *path, int mode, int flags) {
struct res r;
if ((flags & AT_EACCESS) && effective_is_real()) flags &= ~AT_EACCESS;
long rs = (flags & ~AT_SYMLINK_NOFOLLOW) == 0
@@ -970,6 +966,12 @@ static int access_common(int dirfd, const char *path, int mode, int flags) {
if (rs == 1) rs = -ENOENT;
if (rs == 0 && S_ISLNK(r.st.st_mode)) rs = FP_SLOW;
if (rs == 0 || rs == UNSEEN) rs = sc(SYS_faccessat, AT_FDCWD, (long)r.host, mode, 0, 0);
return rs;
}
static int access_common(int dirfd, const char *path, int mode, int flags) {
REAL(faccessat, int (*)(int, const char *, int, int));
long rs = fp_access(dirfd, path, mode, flags);
if (rs != FP_SLOW) { hits++; return ret(rs); }
missed(__func__, dirfd, path);
if (flags == 0) return (int)syscall(SYS_faccessat, dirfd, path, mode);
@@ -997,34 +999,36 @@ static long link_text(const struct res *r, char *text, size_t size, int *changed
return (long)strlen(text);
}
static ssize_t readlink_common(int dirfd, const char *path, char *buf, size_t size) {
REAL(readlinkat, ssize_t (*)(int, const char *, char *, size_t));
static long fp_readlink(int dirfd, const char *path, char *buf, size_t size, size_t *written) {
struct res r;
char text[PATH_MAX];
int changed;
long rs = size ? resolve(dirfd, path, 0, &r) : FP_SLOW;
if (rs == 1) rs = -ENOENT;
if (rs == 0) rs = link_text(&r, text, size, &changed);
if (rs == FP_SLOW) {
missed(__func__, dirfd, path);
return real_readlinkat(dirfd, path, buf, size);
}
hits++;
if (rs < 0) return ret(rs);
if (rs < 0) return rs;
size_t n = rs;
if (!changed) {
n = n < size ? n : size;
memcpy(buf, text, n);
if (written) *written = n;
return n;
}
size_t max = size < PATH_MAX ? size : PATH_MAX;
if (n + 1 < max) {
memcpy(buf, text, n + 1);
if (written) *written = n + 1;
return n;
}
memcpy(buf, text, max);
if (written) *written = max;
return max;
}
static ssize_t readlink_common(int dirfd, const char *path, char *buf, size_t size) {
REAL(readlinkat, ssize_t (*)(int, const char *, char *, size_t));
ANSWER(fp_readlink(dirfd, path, buf, size, NULL), dirfd, path, real_readlinkat(dirfd, path, buf, size));
}
ssize_t readlinkat(int dirfd, const char *path, char *buf, size_t size) { return readlink_common(dirfd, path, buf, size); }
ssize_t readlink(const char *path, char *buf, size_t size) { return readlink_common(AT_FDCWD, path, buf, size); }
@@ -1086,12 +1090,15 @@ char *__getcwd_chk(char *buf, size_t size, size_t room) {
}
int mkdirat(int dirfd, const char *path, mode_t mode) {
REAL(mkdirat, int (*)(int, const char *, mode_t));
static long fp_mkdir(int dirfd, const char *path, mode_t mode) {
struct res r;
long rs = resolve(dirfd, path, MAY_BE_UNSEEN, &r);
if (rs == 0 || rs == 1 || rs == UNSEEN) rs = sc(SYS_mkdirat, AT_FDCWD, (long)r.host, mode, 0, 0);
ANSWER(rs, dirfd, path, real_mkdirat(dirfd, path, mode));
return rs;
}
int mkdirat(int dirfd, const char *path, mode_t mode) {
REAL(mkdirat, int (*)(int, const char *, mode_t));
ANSWER(fp_mkdir(dirfd, path, mode), dirfd, path, real_mkdirat(dirfd, path, mode));
}
int mkdir(const char *path, mode_t mode) { return mkdirat(AT_FDCWD, path, mode); }
@@ -1137,8 +1144,7 @@ int unlinkat(int dirfd, const char *path, int flags) { return unlink_common(dirf
int unlink(const char *path) { return unlink_common(AT_FDCWD, path, 0); }
int rmdir(const char *path) { return unlink_common(AT_FDCWD, path, AT_REMOVEDIR); }
static int rename_common(int olddirfd, const char *old, int newdirfd, const char *new, unsigned flags) {
REAL(renameat2, int (*)(int, const char *, int, const char *, unsigned));
static long fp_rename(int olddirfd, const char *old, int newdirfd, const char *new, unsigned flags) {
struct res from, to;
long rs = flags & ~RENAME_NOREPLACE || trailing_slash(old) || trailing_slash(new) ? FP_SLOW : resolve(olddirfd, old, 0, &from);
if (rs == 1) rs = -ENOENT;
@@ -1153,6 +1159,11 @@ static int rename_common(int olddirfd, const char *old, int newdirfd, const char
if (rs == 0) forget();
}
}
return rs;
}
static int rename_common(int olddirfd, const char *old, int newdirfd, const char *new, unsigned flags) {
REAL(renameat2, int (*)(int, const char *, int, const char *, unsigned));
long rs = fp_rename(olddirfd, old, newdirfd, new, flags);
if (rs != FP_SLOW) { hits++; return ret(rs); }
missed(__func__, olddirfd, old);
if (!flags) return (int)syscall(SYS_renameat, olddirfd, old, newdirfd, new);
@@ -1164,30 +1175,35 @@ int renameat2(int olddirfd, const char *old, int newdirfd, const char *new, unsi
int renameat(int olddirfd, const char *old, int newdirfd, const char *new) { return rename_common(olddirfd, old, newdirfd, new, 0); }
int rename(const char *old, const char *new) { return rename_common(AT_FDCWD, old, AT_FDCWD, new, 0); }
int symlinkat(const char *target, int dirfd, const char *path) {
REAL(symlinkat, int (*)(const char *, int, const char *));
static long fp_symlink(const char *target, int dirfd, const char *path) {
struct res r;
long rs = trailing_slash(path) ? FP_SLOW : resolve(dirfd, path, MAY_BE_UNSEEN, &r);
if (rs == 0 || rs == 1 || rs == UNSEEN) {
rs = sc(SYS_symlinkat, (long)target, AT_FDCWD, (long)r.host, 0, 0);
if (rs == 0) forget();
}
ANSWER(rs, dirfd, path, real_symlinkat(target, dirfd, path));
return rs;
}
int symlinkat(const char *target, int dirfd, const char *path) {
REAL(symlinkat, int (*)(const char *, int, const char *));
ANSWER(fp_symlink(target, dirfd, path), dirfd, path, real_symlinkat(target, dirfd, path));
}
int symlink(const char *target, const char *path) { return symlinkat(target, AT_FDCWD, path); }
int fchmodat(int dirfd, const char *path, mode_t mode, int flags) {
REAL(fchmodat, int (*)(int, const char *, mode_t, int));
static long fp_chmod(int dirfd, const char *path, mode_t mode, int flags) {
struct res r;
long rs = flags ? FP_SLOW : resolve(dirfd, path, 1 | MAY_BE_UNSEEN, &r);
if (rs == 1) rs = -ENOENT;
if (rs == 0 || rs == UNSEEN) rs = sc(SYS_fchmodat, AT_FDCWD, (long)r.host, mode, 0, 0);
ANSWER(rs, dirfd, path, real_fchmodat(dirfd, path, mode, flags));
return rs;
}
int fchmodat(int dirfd, const char *path, mode_t mode, int flags) {
REAL(fchmodat, int (*)(int, const char *, mode_t, int));
ANSWER(fp_chmod(dirfd, path, mode, flags), dirfd, path, real_fchmodat(dirfd, path, mode, flags));
}
int chmod(const char *path, mode_t mode) { return fchmodat(AT_FDCWD, path, mode, 0); }
static int utimens_common(int dirfd, const char *path, const struct timespec times[2], int flags) {
REAL(utimensat, int (*)(int, const char *, const struct timespec[2], int));
static long fp_utimens(int dirfd, const char *path, const struct timespec times[2], int flags) {
struct res r;
long rs;
if (!path && fp_ready()) {
@@ -1197,7 +1213,11 @@ static int utimens_common(int dirfd, const char *path, const struct timespec tim
if (rs == 1) rs = -ENOENT;
if (rs == 0 || rs == UNSEEN) rs = sc(SYS_utimensat, AT_FDCWD, (long)r.host, (long)times, flags, 0);
}
ANSWER(rs, dirfd, path, real_utimensat(dirfd, path, times, flags));
return rs;
}
static int utimens_common(int dirfd, const char *path, const struct timespec times[2], int flags) {
REAL(utimensat, int (*)(int, const char *, const struct timespec[2], int));
ANSWER(fp_utimens(dirfd, path, times, flags), dirfd, path, real_utimensat(dirfd, path, times, flags));
}
int utimensat(int dirfd, const char *path, const struct timespec times[2], int flags) {
return utimens_common(dirfd, path, times, flags);
@@ -1224,23 +1244,29 @@ int utime(const char *path, const struct utimbuf *times) {
return utimens_common(AT_FDCWD, path, ts, 0);
}
int truncate(const char *path, off_t length) {
REAL(truncate, int (*)(const char *, off_t));
static long fp_truncate(const char *path, off_t length) {
struct res r;
long rs = resolve(AT_FDCWD, path, 1 | MAY_BE_UNSEEN, &r);
if (rs == 1) rs = -ENOENT;
if (rs == 0 || rs == UNSEEN) rs = sc(SYS_truncate, (long)r.host, length, 0, 0, 0);
ANSWER(rs, AT_FDCWD, path, real_truncate(path, length));
return rs;
}
int truncate(const char *path, off_t length) {
REAL(truncate, int (*)(const char *, off_t));
ANSWER(fp_truncate(path, length), AT_FDCWD, path, real_truncate(path, length));
}
int truncate64(const char *path, off_t length) __attribute__((alias("truncate")));
static int statfs_common(const char *path, struct statfs *buf, int (*real)(const char *, struct statfs *)) {
static long fp_statfs(const char *path, struct statfs *buf) {
struct res r;
long rs = resolve(AT_FDCWD, path, 1, &r);
if (rs == 1) rs = -ENOENT;
if (rs == 0 && under(r.guest, "/dev/shm", 8)) rs = FP_SLOW;
if (rs == 0) rs = sc(SYS_statfs, (long)r.host, (long)buf, 0, 0, 0);
ANSWER(rs, AT_FDCWD, path, real(path, buf));
return rs;
}
static int statfs_common(const char *path, struct statfs *buf, int (*real)(const char *, struct statfs *)) {
ANSWER(fp_statfs(path, buf), AT_FDCWD, path, real(path, buf));
}
int statfs(const char *path, struct statfs *buf) {
REAL(statfs, int (*)(const char *, struct statfs *));
@@ -1288,12 +1314,16 @@ int statvfs64(const char *path, struct statvfs64 *buf) {
return statvfs_common(path, (struct statvfs *)buf, real_statvfs64);
}
static ssize_t getxattr_common(const char *path, const char *name, void *value, size_t size, int follow,
ssize_t (*real)(const char *, const char *, void *, size_t)) {
static long fp_getxattr(const char *path, const char *name, void *value, size_t size, int follow) {
struct res r;
long rs = resolve(AT_FDCWD, path, follow | MAY_BE_UNSEEN, &r);
if (rs == 1) rs = -ENOENT;
if (rs == 0 || rs == UNSEEN) rs = sc(follow ? SYS_getxattr : SYS_lgetxattr, (long)r.host, (long)name, (long)value, size, 0);
return rs;
}
static ssize_t getxattr_common(const char *path, const char *name, void *value, size_t size, int follow,
ssize_t (*real)(const char *, const char *, void *, size_t)) {
long rs = fp_getxattr(path, name, value, size, follow);
if (rs != FP_SLOW) { hits++; return rs < 0 ? ret(rs) : rs; }
missed(__func__, AT_FDCWD, path);
return real(path, name, value, size);
@@ -1307,12 +1337,16 @@ ssize_t lgetxattr(const char *path, const char *name, void *value, size_t size)
return getxattr_common(path, name, value, size, 0, real_lgetxattr);
}
static ssize_t listxattr_common(const char *path, char *list, size_t size, int follow,
ssize_t (*real)(const char *, char *, size_t)) {
static long fp_listxattr(const char *path, char *list, size_t size, int follow) {
struct res r;
long rs = resolve(AT_FDCWD, path, follow | MAY_BE_UNSEEN, &r);
if (rs == 1) rs = -ENOENT;
if (rs == 0 || rs == UNSEEN) rs = sc(follow ? SYS_listxattr : SYS_llistxattr, (long)r.host, (long)list, size, 0, 0);
return rs;
}
static ssize_t listxattr_common(const char *path, char *list, size_t size, int follow,
ssize_t (*real)(const char *, char *, size_t)) {
long rs = fp_listxattr(path, list, size, follow);
if (rs != FP_SLOW) { hits++; return rs < 0 ? ret(rs) : rs; }
missed(__func__, AT_FDCWD, path);
return real(path, list, size);
@@ -1326,13 +1360,15 @@ ssize_t llistxattr(const char *path, char *list, size_t size) {
return listxattr_common(path, list, size, 0, real_llistxattr);
}
int inotify_add_watch(int fd, const char *path, uint32_t mask) {
REAL(inotify_add_watch, int (*)(int, const char *, uint32_t));
static long fp_inotify(int fd, const char *path, uint32_t mask) {
struct res r;
long rs = resolve(AT_FDCWD, path, (mask & IN_DONT_FOLLOW ? 0 : 1) | MAY_BE_UNSEEN, &r);
if (rs == 1) rs = -ENOENT;
if (rs == 0 || rs == UNSEEN) rs = sc(SYS_inotify_add_watch, fd, (long)r.host, mask, 0, 0);
ANSWER(rs, AT_FDCWD, path, real_inotify_add_watch(fd, path, mask));
if (rs == 0 || rs == 1 || rs == UNSEEN) rs = sc(SYS_inotify_add_watch, fd, (long)r.host, mask, 0, 0);
return rs;
}
int inotify_add_watch(int fd, const char *path, uint32_t mask) {
REAL(inotify_add_watch, int (*)(int, const char *, uint32_t));
ANSWER(fp_inotify(fd, path, mask), AT_FDCWD, path, real_inotify_add_watch(fd, path, mask));
}
+90
View File
@@ -0,0 +1,90 @@
struct svc_cached_sites {
uint64_t magic;
uint32_t api, count;
uint64_t offset[256];
};
static unsigned svc_cache_hits, svc_cache_misses;
static int svc_segment(const struct dl_phdr_info *info, uint64_t at, size_t length, unsigned flags) {
for (int i = 0; i < info->dlpi_phnum; i++) {
const ElfW(Phdr) *p = &info->dlpi_phdr[i];
if (p->p_type == PT_LOAD && (p->p_flags & flags) == flags &&
at >= p->p_vaddr && at - p->p_vaddr <= p->p_memsz &&
length <= p->p_memsz - (at - p->p_vaddr)) return i + 1;
}
return 0;
}
static int svc_cache_name(const struct dl_phdr_info *info, char *path, size_t size) {
const char *enabled = getenv("PROOT_FP_SVC_CACHE");
if (enabled && strcmp(enabled, "1")) return 0;
for (int i = 0; i < info->dlpi_phnum; i++) {
const ElfW(Phdr) *p = &info->dlpi_phdr[i];
if (p->p_type != PT_NOTE || !svc_segment(info, p->p_vaddr, p->p_memsz, PF_R)) continue;
const unsigned char *note = (const unsigned char *)(info->dlpi_addr + p->p_vaddr);
for (size_t at = 0; p->p_memsz - at >= sizeof(ElfW(Nhdr));) {
ElfW(Nhdr) h;
memcpy(&h, note + at, sizeof h);
at += sizeof h;
size_t name = ((size_t)h.n_namesz + 3) & ~(size_t)3;
size_t desc = ((size_t)h.n_descsz + 3) & ~(size_t)3;
if (name > p->p_memsz - at || desc > p->p_memsz - at - name) break;
if (h.n_type == NT_GNU_BUILD_ID && h.n_namesz == 4 && !memcmp(note + at, "GNU", 4) &&
h.n_descsz && h.n_descsz <= 64) {
char id[129];
static const char hex[] = "0123456789abcdef";
for (size_t j = 0; j < h.n_descsz; j++) {
unsigned char b = note[at + name + j];
id[2 * j] = hex[b >> 4];
id[2 * j + 1] = hex[b & 15];
}
id[2 * h.n_descsz] = 0;
int n = snprintf(path, size, "/tmp/.droiddeck-svc-v1-%d-%s", svc_android_api, id);
return n > 0 && (size_t)n < size;
}
at += name + desc;
}
}
return 0;
}
static int svc_cache_read(const struct dl_phdr_info *info, const char *path) {
long fd = fp_open(AT_FDCWD, path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC | O_NONBLOCK, 0);
if (fd < 0) return 0;
struct stat st;
struct svc_cached_sites table;
int valid = sc(SYS_fstat, fd, (long)&st, 0, 0, 0) == 0 && S_ISREG(st.st_mode) &&
st.st_uid == (uid_t)sc(SYS_getuid, 0, 0, 0, 0, 0) && !(st.st_mode & 0077) &&
st.st_size == sizeof table && sc(SYS_read, fd, (long)&table, sizeof table, 0, 0) == sizeof table;
sc(SYS_close, fd, 0, 0, 0, 0);
if (!valid || table.magic != 0x314356534444ULL || table.api != (unsigned)svc_android_api ||
!table.count || table.count > 256 || table.count > 256 - svc_planned) return 0;
for (unsigned i = 0; i < table.count; i++) {
uint64_t offset = table.offset[i];
int segment = svc_segment(info, offset, 4, PF_R | PF_X);
if ((offset & 3) || (i && offset <= table.offset[i - 1]) || !segment) return 0;
const ElfW(Phdr) *p = &info->dlpi_phdr[segment - 1];
const uint32_t *text = (const uint32_t *)(info->dlpi_addr + p->p_vaddr);
size_t at = (offset - p->p_vaddr) / 4;
if (text[at] != 0xd4000001U || !svc_supported(svc_site_number(text, at))) return 0;
}
for (unsigned i = 0; i < table.count; i++)
svc_plan[svc_planned++].site = (uint32_t *)(info->dlpi_addr + table.offset[i]);
return 1;
}
static void svc_cache_write(const struct dl_phdr_info *info, const char *path, size_t first) {
if (svc_planned == first || svc_planned == 256) return;
struct svc_cached_sites table = {.magic = 0x314356534444ULL, .api = svc_android_api, .count = svc_planned - first};
for (unsigned i = 0; i < table.count; i++)
table.offset[i] = (uintptr_t)svc_plan[first + i].site - info->dlpi_addr;
char temporary[256];
int n = snprintf(temporary, sizeof temporary, "%s.part-%d", path, getpid());
if (n <= 0 || n >= (int)sizeof temporary) return;
long fd = fp_open(AT_FDCWD, temporary, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600);
if (fd < 0) return;
long written = sc(SYS_write, fd, (long)&table, sizeof table, 0, 0);
sc(SYS_close, fd, 0, 0, 0, 0);
if (written == sizeof table) fp_rename(AT_FDCWD, temporary, AT_FDCWD, path, 0);
fp_unlink(AT_FDCWD, temporary, 0);
}
+139
View File
@@ -0,0 +1,139 @@
static int svc_android_api;
static __thread struct { long head, tid; } svc_robust __attribute__((tls_model("initial-exec")));
static int svc_host_android_api(void) {
const char *value = getenv("PROOT_FP_ANDROID_API");
if (!value || !*value) return 0;
char *end;
long api = strtol(value, &end, 10);
return !*end && api > 0 && api < 1000 ? (int)api : 0;
}
static long svc_memory(void *local, long remote, size_t size, int write) {
if (!size) return 0;
struct iovec here = {local, size}, there = {(void *)remote, size};
return sc(write ? SYS_process_vm_writev : SYS_process_vm_readv, getpid(),
(long)&here, 1, (long)&there, 1);
}
static int svc_path(long pointer, char *path) {
long n = svc_memory(path, pointer, PATH_MAX, 0);
return n > 0 && memchr(path, 0, (size_t)n) != NULL;
}
static long svc_output(long result, void *value, long out, size_t size) {
if (result < 0) return result;
return svc_memory(value, out, size, 1) == (long)size ? result : FP_SLOW;
}
static int svc_supported(int nr) {
switch (nr) {
case SYS_openat: case SYS_newfstatat: case SYS_statx: case SYS_readlinkat:
case SYS_faccessat: case SYS_mkdirat: case SYS_unlinkat: case SYS_renameat:
case SYS_renameat2: case SYS_symlinkat: case SYS_fchmodat: case SYS_utimensat:
case SYS_truncate: case SYS_statfs: case SYS_getxattr: case SYS_lgetxattr:
case SYS_listxattr: case SYS_llistxattr: case SYS_inotify_add_watch: case SYS_getcwd:
return 1;
case SYS_set_robust_list: case SYS_get_robust_list: case SYS_faccessat2:
return svc_android_api > 0;
case SYS_clone3:
return svc_android_api > 0 && svc_android_api < 35;
default: return 0;
}
}
static long svc_blocked(const long *r) {
if (!svc_android_api) return FP_SLOW;
if (r[8] == SYS_faccessat2 || (r[8] == SYS_clone3 && svc_android_api < 35)) return -ENOSYS;
if (r[8] == SYS_set_robust_list) {
svc_robust.head = r[0];
svc_robust.tid = sc(SYS_gettid, 0, 0, 0, 0, 0);
return -ENOSYS;
}
if (r[8] == SYS_get_robust_list) {
long tid = sc(SYS_gettid, 0, 0, 0, 0, 0);
if ((r[0] && r[0] != tid) || svc_robust.tid != tid || !svc_robust.head) return FP_SLOW;
size_t size = 3 * sizeof(long);
if (svc_memory(&svc_robust.head, r[1], sizeof(long), 1) != sizeof(long) ||
svc_memory(&size, r[2], sizeof size, 1) != sizeof size) return -EFAULT;
return 0;
}
return FP_SLOW;
}
static long svc_path_call(const long *r) {
char path[PATH_MAX], other[PATH_MAX];
struct timespec times[2];
const struct timespec *when = (const struct timespec *)r[2];
int first = 1, second = -1;
switch (r[8]) {
case SYS_getcwd: {
if (own_cwd(path) == FP_SLOW) return FP_SLOW;
size_t n = strlen(path) + 1;
if ((unsigned long)r[1] < n) return -ERANGE;
return svc_output(n, path, r[0], n);
}
case SYS_truncate: case SYS_statfs: case SYS_getxattr: case SYS_lgetxattr:
case SYS_listxattr: case SYS_llistxattr: first = 0; break;
case SYS_renameat: case SYS_renameat2: second = 3; break;
case SYS_symlinkat: first = 0; second = 2; break;
case SYS_utimensat:
if (when) {
if (svc_memory(times, r[2], sizeof times, 0) != sizeof times) return FP_SLOW;
for (int i = 0; i < 2; i++)
if (times[i].tv_nsec != UTIME_NOW && times[i].tv_nsec != UTIME_OMIT &&
(times[i].tv_nsec < 0 || times[i].tv_nsec >= 1000000000L)) return FP_SLOW;
when = times;
}
if (!r[1]) return fp_utimens((int)r[0], NULL, when, (int)r[3]);
break;
case SYS_inotify_add_watch: {
char link[64], target[64];
snprintf(link, sizeof link, "/proc/self/fd/%d", (int)r[0]);
long n = sc(SYS_readlinkat, AT_FDCWD, (long)link, (long)target, sizeof target - 1, 0);
if (n <= 0) return FP_SLOW;
target[n] = 0;
if (strcmp(target, "anon_inode:inotify")) return FP_SLOW;
break;
}
default:
if (!svc_supported((int)r[8])) return FP_SLOW;
}
if (!svc_path(r[first], path) || (second >= 0 && !svc_path(r[second], other))) return FP_SLOW;
switch (r[8]) {
case SYS_openat: return fp_open((int)r[0], path, (int)r[2], (mode_t)r[3]);
case SYS_newfstatat: {
struct stat st;
return svc_output(fp_stat((int)r[0], path, (int)r[3], &st), &st, r[2], sizeof st);
}
case SYS_statx: {
struct statx st;
return svc_output(fp_statx((int)r[0], path, (int)r[2], (unsigned)r[3], &st), &st, r[4], sizeof st);
}
case SYS_readlinkat: {
char text[PATH_MAX];
size_t written = 0;
long result = fp_readlink((int)r[0], path, text, (size_t)r[3], &written);
return svc_output(result, text, r[2], written);
}
case SYS_faccessat: return fp_access((int)r[0], path, (int)r[2], 0);
case SYS_mkdirat: return fp_mkdir((int)r[0], path, (mode_t)r[2]);
case SYS_unlinkat: return fp_unlink((int)r[0], path, (int)r[2]);
case SYS_renameat: case SYS_renameat2:
return fp_rename((int)r[0], path, (int)r[2], other, r[8] == SYS_renameat2 ? (unsigned)r[4] : 0);
case SYS_symlinkat: return fp_symlink(path, (int)r[1], other);
case SYS_fchmodat: return fp_chmod((int)r[0], path, (mode_t)r[2], 0);
case SYS_utimensat: return fp_utimens((int)r[0], path, when, (int)r[3]);
case SYS_truncate: return fp_truncate(path, (off_t)r[1]);
case SYS_statfs: {
struct statfs st;
return svc_output(fp_statfs(path, &st), &st, r[1], sizeof st);
}
case SYS_getxattr: case SYS_lgetxattr:
return fp_getxattr(path, (const char *)r[1], (void *)r[2], (size_t)r[3], r[8] == SYS_getxattr);
case SYS_listxattr: case SYS_llistxattr:
return fp_listxattr(path, (char *)r[1], (size_t)r[2], r[8] == SYS_listxattr);
case SYS_inotify_add_watch: return fp_inotify((int)r[0], path, (uint32_t)r[2]);
default: return FP_SLOW;
}
}
+46 -18
View File
@@ -7,17 +7,23 @@ static inline int svc_number(uint32_t insn) {
static inline int svc_argument_instruction(uint32_t insn) {
if (insn == 0xd503201fU) return 1;
if ((insn & 0xffc003e0U) == 0x290003e0U ||
(insn & 0xffc003e0U) == 0xa90003e0U) return 1;
if ((insn & 0xffc00000U) == 0x29000000U ||
(insn & 0xffc00000U) == 0xa9000000U ||
(insn & 0xffc00000U) == 0xb9000000U ||
(insn & 0xffc00000U) == 0xf9000000U) return 1;
if ((insn & 31) == 8) return 0;
return (insn & 0xffe0ffe0U) == 0xaa0003e0U ||
(insn & 0xffe0ffe0U) == 0x2a0003e0U ||
(insn & 0xfffffc00U) == 0x93407c00U ||
(insn & 0x7f800000U) == 0x52800000U ||
(insn & 0x7f800000U) == 0x12800000U ||
(insn & 0xffc003e0U) == 0xb94003e0U ||
(insn & 0xffc003e0U) == 0xf94003e0U ||
(insn & 0xffc003e0U) == 0xb98003e0U;
(insn & 0xffc00000U) == 0xb9400000U ||
(insn & 0xffc00000U) == 0xf9400000U ||
(insn & 0xffc00000U) == 0xb9800000U ||
(insn & 0xff000000U) == 0x91000000U ||
(insn & 0xff000000U) == 0xd1000000U ||
(insn & 0xff000000U) == 0x11000000U ||
(insn & 0xff000000U) == 0x51000000U;
}
static inline int svc_site_number(const uint32_t *text, size_t at) {
@@ -40,10 +46,14 @@ static inline int svc_branch(uintptr_t from, uintptr_t to, uint32_t *out) {
#if defined(__aarch64__) && !defined(FP_SVC_ENCODINGS_ONLY)
#include <asm/hwcap.h>
#include <link.h>
#include <sys/uio.h>
static __thread int svc_busy __attribute__((tls_model("initial-exec")));
static unsigned svc_sites, svc_pages;
static unsigned long svc_fast, svc_slow;
static unsigned long long svc_scan_ns, svc_patch_ns;
#include "svc-dispatch.h"
static long svc_original(const long *r) {
register long x0 __asm__("x0") = r[0];
@@ -60,20 +70,16 @@ static long svc_original(const long *r) {
__attribute__((used, visibility("hidden")))
long fp_svc_dispatch(const long *r) {
int saved_errno = errno;
long result = FP_SLOW;
if (!svc_busy) {
long result = svc_blocked(r);
if (result == FP_SLOW && !svc_busy) {
svc_busy = 1;
if (r[8] == SYS_openat)
result = fp_open((int)r[0], (const char *)r[1], (int)r[2], (mode_t)r[3]);
else if (r[8] == SYS_newfstatat && r[2])
result = fp_stat((int)r[0], (const char *)r[1], (int)r[3], (struct stat *)r[2]);
if (result != FP_SLOW) __atomic_fetch_add(&svc_fast, 1, __ATOMIC_RELAXED);
result = svc_path_call(r);
svc_busy = 0;
}
if (result == FP_SLOW) {
__atomic_fetch_add(&svc_slow, 1, __ATOMIC_RELAXED);
result = svc_original(r);
}
} else __atomic_fetch_add(&svc_fast, 1, __ATOMIC_RELAXED);
errno = saved_errno;
return result;
}
@@ -141,12 +147,19 @@ struct svc_site { uint32_t *site; uint32_t branch; };
static struct svc_site svc_plan[256];
static size_t svc_planned;
#include "svc-cache.h"
static int svc_object(struct dl_phdr_info *info, size_t size, void *unused) {
(void)size;
(void)unused;
const char *name = strrchr(info->dlpi_name, '/');
name = name ? name + 1 : info->dlpi_name;
if (strcmp(name, "libc.so.6") && strcmp(name, "ld-linux-aarch64.so.1")) return 0;
char cache[224];
int cached = svc_cache_name(info, cache, sizeof cache);
if (cached && svc_cache_read(info, cache)) { svc_cache_hits++; return 0; }
if (cached) svc_cache_misses++;
size_t first = svc_planned;
for (int i = 0; i < info->dlpi_phnum; i++) {
const ElfW(Phdr) *ph = &info->dlpi_phdr[i];
if (ph->p_type != PT_LOAD || !(ph->p_flags & PF_X)) continue;
@@ -154,10 +167,11 @@ static int svc_object(struct dl_phdr_info *info, size_t size, void *unused) {
for (size_t j = 1; j < ph->p_memsz / 4 && svc_planned < 256; j++) {
if (text[j] != 0xd4000001U) continue;
int nr = svc_site_number(text, j);
if (nr != SYS_openat && nr != SYS_newfstatat) continue;
if (!svc_supported(nr)) continue;
svc_plan[svc_planned++].site = &text[j];
}
}
if (cached) svc_cache_write(info, cache, first);
return 0;
}
@@ -199,12 +213,20 @@ done:
}
static void svc_stats(void) {
char line[160];
int n = snprintf(line, sizeof line, "fastpath-svc[%d]: %u sites, %u relay pages, %lu fast, %lu to proot\n",
getpid(), svc_sites, svc_pages, svc_fast, svc_slow);
if (write(2, line, n) < 0) {}
char line[256];
int n = snprintf(line, sizeof line, "fastpath-svc[%d]: %u sites, %u relay pages, %lu fast, %lu to proot; scan %llu us, patch %llu us, android %d, cache %u hit %u miss\n",
getpid(), svc_sites, svc_pages, svc_fast, svc_slow, svc_scan_ns / 1000, svc_patch_ns / 1000, svc_android_api, svc_cache_hits, svc_cache_misses);
if (n > 0 && write(2, line, n < (int)sizeof line ? n : (int)sizeof line - 1) < 0) {}
}
static unsigned long long svc_clock(void) {
struct timespec t;
if (sc(SYS_clock_gettime, CLOCK_MONOTONIC, (long)&t, 0, 0, 0) < 0) return 0;
return (unsigned long long)t.tv_sec * 1000000000ULL + t.tv_nsec;
}
static void svc_child(void) { svc_busy = 0; }
__attribute__((constructor)) static void svc_patch_init(void) {
const char *enabled = getenv("PROOT_FP_PATCH_SVC");
if ((enabled && strcmp(enabled, "1")) || !fp_ready()) return;
@@ -213,11 +235,16 @@ __attribute__((constructor)) static void svc_patch_init(void) {
if (getauxval(AT_HWCAP2) & HWCAP2_SME) return;
#endif
if (!svc_single_thread()) return;
if (pthread_atfork(NULL, NULL, svc_child)) return;
svc_android_api = svc_host_android_api();
size_t page = (size_t)sysconf(_SC_PAGESIZE);
if (page < 4096 || (page & (page - 1))) return;
int mem = (int)sc(SYS_openat, AT_FDCWD, (long)"/proc/self/mem", O_RDWR | O_CLOEXEC, 0, 0);
if (mem < 0) return;
unsigned long long start = svc_clock();
dl_iterate_phdr(svc_object, NULL);
svc_scan_ns = svc_clock() - start;
start = svc_clock();
for (size_t first = 0; first < svc_planned;) {
uint32_t *relay = svc_relay_page((uintptr_t)svc_plan[first].site, page);
if (!relay) { first++; continue; }
@@ -245,6 +272,7 @@ __attribute__((constructor)) static void svc_patch_init(void) {
first = end;
}
sc(SYS_close, mem, 0, 0, 0, 0);
svc_patch_ns = svc_clock() - start;
if (getenv("PROOT_FP_STATS")) atexit(svc_stats);
}
#endif