diff --git a/tools/linuxfs/preload/robust.c b/tools/linuxfs/preload/robust.c index 5de1dc740..81bf5d8cc 100644 --- a/tools/linuxfs/preload/robust.c +++ b/tools/linuxfs/preload/robust.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,11 @@ #ifdef SYS_newfstatat #define BL_SYS_FSTATAT SYS_newfstatat +#define BL_FSTATAT_NAME "fstatat" typedef struct stat bl_stat; #else #define BL_SYS_FSTATAT SYS_fstatat64 +#define BL_FSTATAT_NAME "fstatat64" typedef struct stat64 bl_stat; #endif @@ -43,6 +46,51 @@ struct bl_open_how { static long (*real_syscall)(long, ...); static void *(*real_dlopen)(const char *, int); static int openat2_missing; +static int (*next_fstatat)(int, const char *, bl_stat *, int); +static int (*next_openat)(int, const char *, int, ...); +static ssize_t (*next_readlinkat)(int, const char *, char *, size_t); + +static int find_fastpath(struct dl_phdr_info *info, size_t size, void *name) { + (void)size; + const char *slash = strrchr(info->dlpi_name, '/'); + if (!slash || strcmp(slash + 1, "libblfastpath.so") != 0 || strlen(info->dlpi_name) >= PATH_MAX) { + return 0; + } + strcpy(name, info->dlpi_name); + return 1; +} + +__attribute__((constructor)) static void find_fastpath_calls(void) { + char name[PATH_MAX] = ""; + dl_iterate_phdr(find_fastpath, name); + void *fastpath = *name ? dlopen(name, RTLD_NOLOAD | RTLD_LAZY) : NULL; + if (fastpath) { + next_fstatat = (int (*)(int, const char *, bl_stat *, int))dlsym(fastpath, BL_FSTATAT_NAME); + next_openat = (int (*)(int, const char *, int, ...))dlsym(fastpath, "openat"); + next_readlinkat = (ssize_t (*)(int, const char *, char *, size_t))dlsym(fastpath, "readlinkat"); + } +} + +static long path_stat(int dirfd, const char *path, bl_stat *st, int flags) { + if (!next_fstatat) { + return real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)path, (long)st, (long)flags); + } + return next_fstatat(dirfd, path, st, flags); +} + +static long path_open(int dirfd, const char *path, int flags, long mode) { + if (!next_openat) { + return real_syscall(SYS_openat, (long)dirfd, (long)path, (long)flags, mode); + } + return next_openat(dirfd, path, flags, (mode_t)mode); +} + +static long path_readlink(int dirfd, const char *path, char *buf, size_t size) { + if (!next_readlinkat) { + return real_syscall(SYS_readlinkat, (long)dirfd, (long)path, (long)buf, (long)size); + } + return next_readlinkat(dirfd, path, buf, size); +} void *dlopen(const char *filename, int flags) { if (!real_dlopen) { @@ -93,7 +141,7 @@ static struct robust_list_head *thread_robust_head(void) { // Magic links are procfs links below a process's directory; procfs's root holds only plain ones (self, thread-self, mounts). static int is_magic_link(int dirfd, char *link, size_t parent_len) { - int fd = (int)real_syscall(SYS_openat, (long)dirfd, (long)link, (long)(O_PATH | O_NOFOLLOW | O_CLOEXEC), 0L); + int fd = (int)path_open(dirfd, link, O_PATH | O_NOFOLLOW | O_CLOEXEC, 0L); if (fd < 0) { return 0; } @@ -106,7 +154,7 @@ static int is_magic_link(int dirfd, char *link, size_t parent_len) { char saved = link[parent_len]; link[parent_len] = 0; bl_stat parent; - long result = real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)(parent_len ? link : "."), (long)&parent, 0L); + long result = path_stat(dirfd, (parent_len ? link : "."), &parent, 0); link[parent_len] = saved; return result != 0 || parent.st_ino != BL_PROC_ROOT_INO; } @@ -167,7 +215,7 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) { resolved[candidate_len] = 0; bl_stat st; int follow = !last || trailing_slash || !nofollow; - long stat_result = follow ? real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)resolved, (long)&st, (long)AT_SYMLINK_NOFOLLOW) : -1; + long stat_result = follow ? path_stat(dirfd, resolved, &st, AT_SYMLINK_NOFOLLOW) : -1; if (stat_result == 0 && S_ISLNK(st.st_mode)) { if (++symlinks > BL_MAX_SYMLINKS) { errno = ELOOP; @@ -176,7 +224,7 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) { if (!have_root) { bl_stat root; struct statfs root_fs; - if (real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)".", (long)&root, 0L) != 0) { + if (path_stat(dirfd, ".", &root, 0) != 0) { return -1; } root_dev = root.st_dev; @@ -195,7 +243,7 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) { } char *tail = pending + PATH_MAX - rest_len - 1; memmove(tail, end, rest_len + 1); - long target_len = real_syscall(SYS_readlinkat, (long)dirfd, (long)resolved, (long)pending, (long)(PATH_MAX - rest_len)); + long target_len = path_readlink(dirfd, resolved, pending, PATH_MAX - rest_len); if (target_len <= 0) { if (target_len == 0) { errno = ENOENT; @@ -215,11 +263,11 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) { inside++; } bl_stat probe; - int host = *inside && real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)inside, (long)&probe, (long)AT_SYMLINK_NOFOLLOW) != 0 && - errno == ENOENT && real_syscall(BL_SYS_FSTATAT, (long)AT_FDCWD, (long)pending, (long)&probe, 0L) == 0; + int host = *inside && path_stat(dirfd, inside, &probe, AT_SYMLINK_NOFOLLOW) != 0 && + errno == ENOENT && path_stat(AT_FDCWD, pending, &probe, 0) == 0; pending[target_len] = saved; if (host) { - return real_syscall(SYS_openat, (long)AT_FDCWD, (long)pending, (long)flags, mode); + return path_open(AT_FDCWD, pending, flags, mode); } resolved_len = 0; } @@ -246,7 +294,7 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) { resolved[resolved_len++] = '.'; } resolved[resolved_len] = 0; - return real_syscall(SYS_openat, (long)dirfd, (long)resolved, (long)flags, mode); + return path_open(dirfd, resolved, flags, mode); } static long openat2_via_openat(const long *args) { @@ -270,7 +318,7 @@ static long openat2_via_openat(const long *args) { errno = EXDEV; return -1; } - return real_syscall(SYS_openat, args[0], (long)path, (long)(int)how->flags, (long)how->mode); + return path_open((int)args[0], path, (int)how->flags, (long)how->mode); } static int execveat_is_execve(const long *args) { @@ -316,6 +364,9 @@ long syscall(long number, ...) { } return openat2_via_openat(args); } + if (number == SYS_openat) { + return path_open((int)args[0], (const char *)args[1], (int)args[2], args[3]); + } if (number == SYS_execveat && execveat_is_execve(args)) { return real_syscall(SYS_execve, args[1], args[2], args[3]); } diff --git a/tools/proot/fastpath/fastpath.c b/tools/proot/fastpath/fastpath.c index 9700a1c61..5056ec41e 100644 --- a/tools/proot/fastpath/fastpath.c +++ b/tools/proot/fastpath/fastpath.c @@ -141,12 +141,18 @@ static int traced_by_match(const char *key) { return found; } +static long init_sys(long nr, long a, long b, long c) { + static long (*real_syscall)(long, ...); + if (!real_syscall) real_syscall = (long (*)(long, ...))dlsym(RTLD_NEXT, "syscall"); + return real_syscall(nr, a, b, c); +} + static int program_32bit(const char *prog, int scripts) { char head[256]; - long fd = syscall(SYS_openat, AT_FDCWD, prog, O_RDONLY | O_CLOEXEC); + long fd = init_sys(SYS_openat, AT_FDCWD, (long)prog, O_RDONLY | O_CLOEXEC); if (fd < 0) return 1; - long n = syscall(SYS_read, fd, head, sizeof head - 1); - syscall(SYS_close, fd); + long n = init_sys(SYS_read, fd, (long)head, sizeof head - 1); + init_sys(SYS_close, fd, 0, 0); if (n < 0) return 1; head[n] = 0; if (n >= 2 && head[0] == '#' && head[1] == '!') { @@ -166,10 +172,10 @@ static int fex_runs_32bit(void) { if (strcmp(program_invocation_short_name, "FEX") != 0 && strcmp(program_invocation_short_name, "FEXInterpreter") != 0) return 0; char args[PATH_MAX + 64]; - long fd = syscall(SYS_openat, AT_FDCWD, "/proc/self/cmdline", O_RDONLY | O_CLOEXEC); + long fd = init_sys(SYS_openat, AT_FDCWD, (long)"/proc/self/cmdline", O_RDONLY | O_CLOEXEC); if (fd < 0) return 1; - long n = syscall(SYS_read, fd, args, sizeof args - 1); - syscall(SYS_close, fd); + long n = init_sys(SYS_read, fd, (long)args, sizeof args - 1); + init_sys(SYS_close, fd, 0, 0); if (n <= 0) return 1; args[n] = 0; const char *prog = args + strlen(args) + 1; diff --git a/tools/tests/test_openat2_preload.py b/tools/tests/test_openat2_preload.py index 3791fdd14..a30aa74dd 100644 --- a/tools/tests/test_openat2_preload.py +++ b/tools/tests/test_openat2_preload.py @@ -7,6 +7,7 @@ import tempfile import unittest PRELOAD = Path(__file__).resolve().parents[1] / "linuxfs/preload" +FASTPATH = Path(__file__).resolve().parents[1] / "proot/fastpath/fastpath.c" # Opens each "\t\t" case with openat2(RESOLVE_IN_ROOT) as FEX does: first through the kernel's own # openat2 on argv[1] when it has one, then, with openat2 failing ENOSYS as under proot, through the preload on argv[2]. @@ -84,6 +85,62 @@ int main(int argc, char **argv) { } """ +COUNTER = r""" +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +static int opens, stats, links; + +int openat(int dirfd, const char *path, int flags, ...) { + va_list args; + va_start(args, flags); + mode_t mode = va_arg(args, int); + va_end(args); + opens++; + return ((int (*)(int, const char *, int, ...))dlsym(RTLD_NEXT, "openat"))(dirfd, path, flags, mode); +} + +int fstatat(int dirfd, const char *path, struct stat *st, int flags) { + stats++; + return ((int (*)(int, const char *, struct stat *, int))dlsym(RTLD_NEXT, "fstatat"))(dirfd, path, st, flags); +} + +int fstatat64(int dirfd, const char *path, struct stat64 *st, int flags) { + stats++; + return ((int (*)(int, const char *, struct stat64 *, int))dlsym(RTLD_NEXT, "fstatat64"))(dirfd, path, st, flags); +} + +ssize_t readlinkat(int dirfd, const char *path, char *buf, size_t size) { + links++; + return ((ssize_t (*)(int, const char *, char *, size_t))dlsym(RTLD_NEXT, "readlinkat"))(dirfd, path, buf, size); +} + +__attribute__((destructor)) static void report(void) { + fprintf(stderr, "calls %d %d %d\n", opens, stats, links); +} +""" + +RAW_OPEN = r""" +#define _GNU_SOURCE +#include +#include +#include +#include + +int main(int argc, char **argv) { + char text[16] = ""; + long fd = syscall(SYS_openat, AT_FDCWD, argv[1], O_RDONLY | O_CLOEXEC, 0); + if (fd < 0 || read((int)fd, text, sizeof(text) - 1) < 0) return 1; + printf("%s\n", text); + return syscall(SYS_openat, AT_FDCWD, argv[2], O_RDONLY | O_CLOEXEC, 0) == -1 ? 0 : 1; +} +""" + O_PATH = getattr(os, "O_PATH", 0o10000000) # (dirfd: r = the tree, o = "/"), flags, path, expected: a path in the tree, an errno name, or "proc" for a file outside it. @@ -191,6 +248,37 @@ class Openat2PreloadTest(unittest.TestCase): lines = result.stdout.splitlines() cls.kernel = labels(lines[1:1 + len(CASES)], cls.trees[0], cls.outside) if lines[0] == "kernel" else None cls.preload = labels(lines[-len(every):], cls.trees[1], cls.outside) + counter = cls.tmp / "counter.c" + counter.write_text(COUNTER) + (cls.tmp / "fp").mkdir() + subprocess.run(["cc", "-O2", "-Wall", "-Werror", "-fPIC", "-shared", "-o", str(cls.tmp / "fp/libblfastpath.so"), + str(counter), "-ldl"], check=True) + tree = cls.tmp / "counted" + make_tree(tree, cls.outside) + counted = subprocess.run([str(cls.tmp / "openat2"), str(tree), str(tree)], input=cases, capture_output=True, + text=True, env=dict(os.environ, LD_PRELOAD="%s:%s" % (cls.lib, cls.tmp / "fp/libblfastpath.so")), + timeout=60) + if counted.returncode != 0: + raise AssertionError(counted.stdout + counted.stderr) + cls.counted = labels(counted.stdout.splitlines()[-len(every):], tree, cls.outside) + cls.calls = [int(n) for n in counted.stderr.split("calls ")[-1].split()] + raw = cls.tmp / "raw_open.c" + raw.write_text(RAW_OPEN) + subprocess.run(["cc", "-O2", "-Wall", "-Werror", "-o", str(cls.tmp / "raw_open"), str(raw)], check=True) + cls.raw = subprocess.run([str(cls.tmp / "raw_open"), str(tree / "etc/hostname"), str(tree / "missing")], + capture_output=True, text=True, timeout=60, + env=dict(os.environ, LD_PRELOAD="%s:%s" % (cls.lib, cls.tmp / "fp/libblfastpath.so"))) + (cls.tmp / "real").mkdir() + subprocess.run(["cc", "-O2", "-Wall", "-Werror", "-fPIC", "-shared", "-pthread", "-o", + str(cls.tmp / "real/libblfastpath.so"), str(FASTPATH), "-ldl"], check=True) + shutil.copy(cls.tmp / "raw_open", cls.tmp / "FEX") + try: + cls.fex = subprocess.run([str(cls.tmp / "FEX"), str(tree / "etc/hostname"), str(tree / "missing")], + capture_output=True, text=True, timeout=20, + env=dict(os.environ, PROOT_FP_ROOT="/", PROOT_FP_KEY="test", + LD_PRELOAD="%s:%s" % (cls.lib, cls.tmp / "real/libblfastpath.so"))) + except subprocess.TimeoutExpired: + cls.fex = None @classmethod def tearDownClass(cls): @@ -205,6 +293,20 @@ class Openat2PreloadTest(unittest.TestCase): for case, got in zip(HOST_CASES, self.preload[len(CASES):]): self.assertEqual(got, case[3], case) + def test_the_walk_asks_the_fast_path_when_it_is_loaded(self): + self.assertEqual(self.counted, self.preload) + self.assertTrue(all(n > 0 for n in self.calls), self.calls) + + def test_a_raw_openat_syscall_asks_the_fast_path_too(self): + self.assertEqual(self.raw.returncode, 0, self.raw.stderr) + self.assertEqual(self.raw.stdout, "inside\n") + self.assertEqual(self.raw.stderr.split("calls ")[-1].split(), ["2", "0", "0"]) + + def test_a_first_raw_openat_in_fex_does_not_wait_on_the_fast_path_starting(self): + self.assertIsNotNone(self.fex, "hung") + self.assertEqual(self.fex.returncode, 0, self.fex.stderr) + self.assertEqual(self.fex.stdout, "inside\n") + def test_the_kernel_agrees(self): if self.kernel is None: self.skipTest("this kernel or sandbox has no openat2")