perf: FEX's file lookups and opens go through the fast path instead of proot (#1283)

* perf: FEX's in-root path lookups go through the fast path instead of proot

Android has no openat2, so libblsession walks RESOLVE_IN_ROOT lookups for
FEX one component at a time. Every fstatat, readlinkat and openat in that
walk was a raw syscall that stopped in proot. When libblfastpath is loaded,
the walk now calls its fstatat, readlinkat and openat, which answer without
proot and fall back to the raw syscall themselves. Without it the walk uses
the raw syscalls as before.

One lookup drops from 250-940 us to 26-86 us. A Graveyard Keeper launch
makes 37.5k proot stops instead of 54.0k and reaches its first scene
0.31 s sooner.

* test: count fstatat64 where the preload asks for it

* perf: FEX's own file opens go through the fast path instead of proot

FEX opens host files for the game with syscall(SYS_openat), so every one
stopped in proot even when the fast path could answer it. libblsession's
syscall() now sends SYS_openat to the fast path's openat as well, the same
way the in-root walk does, and makes the raw syscall when the fast path is
not loaded.

A Graveyard Keeper launch makes 28.0k proot stops instead of 37.5k; FEX's
openat stops go from 6,739 to none.

* fix: the fast path's start-up reads don't come back through syscall()

libblsession's syscall() now sends SYS_openat to the fast path. A process
named FEX starts the fast path by reading /proc/self/cmdline with
syscall(SYS_openat), which came straight back into the fast path and
waited on its own pthread_once. Its start-up reads now call libc's
syscall directly.

tools/linuxfs/preload/robust.c

@@ -3,6 +3,7 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
+#include <link.h>
 #include <linux/futex.h>
 #include <linux/magic.h>
 #include <pthread.h>
@@ -28,9 +29,11 @@
 
 #ifdef SYS_newfstatat
 #define BL_SYS_FSTATAT SYS_newfstatat
+#define BL_FSTATAT_NAME "fstatat"
 typedef struct stat bl_stat;
 #else
 #define BL_SYS_FSTATAT SYS_fstatat64
+#define BL_FSTATAT_NAME "fstatat64"
 typedef struct stat64 bl_stat;
 #endif
 
@@ -43,6 +46,51 @@ struct bl_open_how {
 static long (*real_syscall)(long, ...);
 static void *(*real_dlopen)(const char *, int);
 static int openat2_missing;
+static int (*next_fstatat)(int, const char *, bl_stat *, int);
+static int (*next_openat)(int, const char *, int, ...);
+static ssize_t (*next_readlinkat)(int, const char *, char *, size_t);
+
+static int find_fastpath(struct dl_phdr_info *info, size_t size, void *name) {
+  (void)size;
+  const char *slash = strrchr(info->dlpi_name, '/');
+  if (!slash || strcmp(slash + 1, "libblfastpath.so") != 0 || strlen(info->dlpi_name) >= PATH_MAX) {
+    return 0;
+  }
+  strcpy(name, info->dlpi_name);
+  return 1;
+}
+
+__attribute__((constructor)) static void find_fastpath_calls(void) {
+  char name[PATH_MAX] = "";
+  dl_iterate_phdr(find_fastpath, name);
+  void *fastpath = *name ? dlopen(name, RTLD_NOLOAD | RTLD_LAZY) : NULL;
+  if (fastpath) {
+    next_fstatat = (int (*)(int, const char *, bl_stat *, int))dlsym(fastpath, BL_FSTATAT_NAME);
+    next_openat = (int (*)(int, const char *, int, ...))dlsym(fastpath, "openat");
+    next_readlinkat = (ssize_t (*)(int, const char *, char *, size_t))dlsym(fastpath, "readlinkat");
+  }
+}
+
+static long path_stat(int dirfd, const char *path, bl_stat *st, int flags) {
+  if (!next_fstatat) {
+    return real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)path, (long)st, (long)flags);
+  }
+  return next_fstatat(dirfd, path, st, flags);
+}
+
+static long path_open(int dirfd, const char *path, int flags, long mode) {
+  if (!next_openat) {
+    return real_syscall(SYS_openat, (long)dirfd, (long)path, (long)flags, mode);
+  }
+  return next_openat(dirfd, path, flags, (mode_t)mode);
+}
+
+static long path_readlink(int dirfd, const char *path, char *buf, size_t size) {
+  if (!next_readlinkat) {
+    return real_syscall(SYS_readlinkat, (long)dirfd, (long)path, (long)buf, (long)size);
+  }
+  return next_readlinkat(dirfd, path, buf, size);
+}
 
 void *dlopen(const char *filename, int flags) {
   if (!real_dlopen) {
@@ -93,7 +141,7 @@ static struct robust_list_head *thread_robust_head(void) {
 
 // Magic links are procfs links below a process's directory; procfs's root holds only plain ones (self, thread-self, mounts).
 static int is_magic_link(int dirfd, char *link, size_t parent_len) {
-  int fd = (int)real_syscall(SYS_openat, (long)dirfd, (long)link, (long)(O_PATH | O_NOFOLLOW | O_CLOEXEC), 0L);
+  int fd = (int)path_open(dirfd, link, O_PATH | O_NOFOLLOW | O_CLOEXEC, 0L);
   if (fd < 0) {
     return 0;
   }
@@ -106,7 +154,7 @@ static int is_magic_link(int dirfd, char *link, size_t parent_len) {
   char saved = link[parent_len];
   link[parent_len] = 0;
   bl_stat parent;
-  long result = real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)(parent_len ? link : "."), (long)&parent, 0L);
+  long result = path_stat(dirfd, (parent_len ? link : "."), &parent, 0);
   link[parent_len] = saved;
   return result != 0 || parent.st_ino != BL_PROC_ROOT_INO;
 }
@@ -167,7 +215,7 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) {
     resolved[candidate_len] = 0;
     bl_stat st;
     int follow = !last || trailing_slash || !nofollow;
-    long stat_result = follow ? real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)resolved, (long)&st, (long)AT_SYMLINK_NOFOLLOW) : -1;
+    long stat_result = follow ? path_stat(dirfd, resolved, &st, AT_SYMLINK_NOFOLLOW) : -1;
     if (stat_result == 0 && S_ISLNK(st.st_mode)) {
       if (++symlinks > BL_MAX_SYMLINKS) {
         errno = ELOOP;
@@ -176,7 +224,7 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) {
       if (!have_root) {
         bl_stat root;
         struct statfs root_fs;
-        if (real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)".", (long)&root, 0L) != 0) {
+        if (path_stat(dirfd, ".", &root, 0) != 0) {
           return -1;
         }
         root_dev = root.st_dev;
@@ -195,7 +243,7 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) {
       }
       char *tail = pending + PATH_MAX - rest_len - 1;
       memmove(tail, end, rest_len + 1);
-      long target_len = real_syscall(SYS_readlinkat, (long)dirfd, (long)resolved, (long)pending, (long)(PATH_MAX - rest_len));
+      long target_len = path_readlink(dirfd, resolved, pending, PATH_MAX - rest_len);
       if (target_len <= 0) {
         if (target_len == 0) {
           errno = ENOENT;
@@ -215,11 +263,11 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) {
           inside++;
         }
         bl_stat probe;
-        int host = *inside && real_syscall(BL_SYS_FSTATAT, (long)dirfd, (long)inside, (long)&probe, (long)AT_SYMLINK_NOFOLLOW) != 0 &&
-                   errno == ENOENT && real_syscall(BL_SYS_FSTATAT, (long)AT_FDCWD, (long)pending, (long)&probe, 0L) == 0;
+        int host = *inside && path_stat(dirfd, inside, &probe, AT_SYMLINK_NOFOLLOW) != 0 &&
+                   errno == ENOENT && path_stat(AT_FDCWD, pending, &probe, 0) == 0;
         pending[target_len] = saved;
         if (host) {
-          return real_syscall(SYS_openat, (long)AT_FDCWD, (long)pending, (long)flags, mode);
+          return path_open(AT_FDCWD, pending, flags, mode);
         }
         resolved_len = 0;
       }
@@ -246,7 +294,7 @@ static long openat_in_root(int dirfd, const char *path, int flags, long mode) {
     resolved[resolved_len++] = '.';
   }
   resolved[resolved_len] = 0;
-  return real_syscall(SYS_openat, (long)dirfd, (long)resolved, (long)flags, mode);
+  return path_open(dirfd, resolved, flags, mode);
 }
 
 static long openat2_via_openat(const long *args) {
@@ -270,7 +318,7 @@ static long openat2_via_openat(const long *args) {
     errno = EXDEV;
     return -1;
   }
-  return real_syscall(SYS_openat, args[0], (long)path, (long)(int)how->flags, (long)how->mode);
+  return path_open((int)args[0], path, (int)how->flags, (long)how->mode);
 }
 
 static int execveat_is_execve(const long *args) {
@@ -316,6 +364,9 @@ long syscall(long number, ...) {
     }
     return openat2_via_openat(args);
   }
+  if (number == SYS_openat) {
+    return path_open((int)args[0], (const char *)args[1], (int)args[2], args[3]);
+  }
   if (number == SYS_execveat && execveat_is_execve(args)) {
     return real_syscall(SYS_execve, args[1], args[2], args[3]);
   }

tools/proot/fastpath/fastpath.c

@@ -141,12 +141,18 @@ static int traced_by_match(const char *key) {
   return found;
 }
 
+static long init_sys(long nr, long a, long b, long c) {
+  static long (*real_syscall)(long, ...);
+  if (!real_syscall) real_syscall = (long (*)(long, ...))dlsym(RTLD_NEXT, "syscall");
+  return real_syscall(nr, a, b, c);
+}
+
 static int program_32bit(const char *prog, int scripts) {
   char head[256];
-  long fd = syscall(SYS_openat, AT_FDCWD, prog, O_RDONLY | O_CLOEXEC);
+  long fd = init_sys(SYS_openat, AT_FDCWD, (long)prog, O_RDONLY | O_CLOEXEC);
   if (fd < 0) return 1;
-  long n = syscall(SYS_read, fd, head, sizeof head - 1);
-  syscall(SYS_close, fd);
+  long n = init_sys(SYS_read, fd, (long)head, sizeof head - 1);
+  init_sys(SYS_close, fd, 0, 0);
   if (n < 0) return 1;
   head[n] = 0;
   if (n >= 2 && head[0] == '#' && head[1] == '!') {
@@ -166,10 +172,10 @@ static int fex_runs_32bit(void) {
   if (strcmp(program_invocation_short_name, "FEX") != 0 && strcmp(program_invocation_short_name, "FEXInterpreter") != 0)
     return 0;
   char args[PATH_MAX + 64];
-  long fd = syscall(SYS_openat, AT_FDCWD, "/proc/self/cmdline", O_RDONLY | O_CLOEXEC);
+  long fd = init_sys(SYS_openat, AT_FDCWD, (long)"/proc/self/cmdline", O_RDONLY | O_CLOEXEC);
   if (fd < 0) return 1;
-  long n = syscall(SYS_read, fd, args, sizeof args - 1);
-  syscall(SYS_close, fd);
+  long n = init_sys(SYS_read, fd, (long)args, sizeof args - 1);
+  init_sys(SYS_close, fd, 0, 0);
   if (n <= 0) return 1;
   args[n] = 0;
   const char *prog = args + strlen(args) + 1;

tools/tests/test_openat2_preload.py

@@ -7,6 +7,7 @@ import tempfile
 import unittest
 
 PRELOAD = Path(__file__).resolve().parents[1] / "linuxfs/preload"
+FASTPATH = Path(__file__).resolve().parents[1] / "proot/fastpath/fastpath.c"
 
 # Opens each "<dirfd>\t<flags>\t<path>" case with openat2(RESOLVE_IN_ROOT) as FEX does: first through the kernel's own
 # openat2 on argv[1] when it has one, then, with openat2 failing ENOSYS as under proot, through the preload on argv[2].
@@ -84,6 +85,62 @@ int main(int argc, char **argv) {
 }
 """
 
+COUNTER = r"""
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+static int opens, stats, links;
+
+int openat(int dirfd, const char *path, int flags, ...) {
+  va_list args;
+  va_start(args, flags);
+  mode_t mode = va_arg(args, int);
+  va_end(args);
+  opens++;
+  return ((int (*)(int, const char *, int, ...))dlsym(RTLD_NEXT, "openat"))(dirfd, path, flags, mode);
+}
+
+int fstatat(int dirfd, const char *path, struct stat *st, int flags) {
+  stats++;
+  return ((int (*)(int, const char *, struct stat *, int))dlsym(RTLD_NEXT, "fstatat"))(dirfd, path, st, flags);
+}
+
+int fstatat64(int dirfd, const char *path, struct stat64 *st, int flags) {
+  stats++;
+  return ((int (*)(int, const char *, struct stat64 *, int))dlsym(RTLD_NEXT, "fstatat64"))(dirfd, path, st, flags);
+}
+
+ssize_t readlinkat(int dirfd, const char *path, char *buf, size_t size) {
+  links++;
+  return ((ssize_t (*)(int, const char *, char *, size_t))dlsym(RTLD_NEXT, "readlinkat"))(dirfd, path, buf, size);
+}
+
+__attribute__((destructor)) static void report(void) {
+  fprintf(stderr, "calls %d %d %d\n", opens, stats, links);
+}
+"""
+
+RAW_OPEN = r"""
+#define _GNU_SOURCE
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+int main(int argc, char **argv) {
+  char text[16] = "";
+  long fd = syscall(SYS_openat, AT_FDCWD, argv[1], O_RDONLY | O_CLOEXEC, 0);
+  if (fd < 0 || read((int)fd, text, sizeof(text) - 1) < 0) return 1;
+  printf("%s\n", text);
+  return syscall(SYS_openat, AT_FDCWD, argv[2], O_RDONLY | O_CLOEXEC, 0) == -1 ? 0 : 1;
+}
+"""
+
 O_PATH = getattr(os, "O_PATH", 0o10000000)
 
 # (dirfd: r = the tree, o = "/"), flags, path, expected: a path in the tree, an errno name, or "proc" for a file outside it.
@@ -191,6 +248,37 @@ class Openat2PreloadTest(unittest.TestCase):
         lines = result.stdout.splitlines()
         cls.kernel = labels(lines[1:1 + len(CASES)], cls.trees[0], cls.outside) if lines[0] == "kernel" else None
         cls.preload = labels(lines[-len(every):], cls.trees[1], cls.outside)
+        counter = cls.tmp / "counter.c"
+        counter.write_text(COUNTER)
+        (cls.tmp / "fp").mkdir()
+        subprocess.run(["cc", "-O2", "-Wall", "-Werror", "-fPIC", "-shared", "-o", str(cls.tmp / "fp/libblfastpath.so"),
+                        str(counter), "-ldl"], check=True)
+        tree = cls.tmp / "counted"
+        make_tree(tree, cls.outside)
+        counted = subprocess.run([str(cls.tmp / "openat2"), str(tree), str(tree)], input=cases, capture_output=True,
+                                 text=True, env=dict(os.environ, LD_PRELOAD="%s:%s" % (cls.lib, cls.tmp / "fp/libblfastpath.so")),
+                                 timeout=60)
+        if counted.returncode != 0:
+            raise AssertionError(counted.stdout + counted.stderr)
+        cls.counted = labels(counted.stdout.splitlines()[-len(every):], tree, cls.outside)
+        cls.calls = [int(n) for n in counted.stderr.split("calls ")[-1].split()]
+        raw = cls.tmp / "raw_open.c"
+        raw.write_text(RAW_OPEN)
+        subprocess.run(["cc", "-O2", "-Wall", "-Werror", "-o", str(cls.tmp / "raw_open"), str(raw)], check=True)
+        cls.raw = subprocess.run([str(cls.tmp / "raw_open"), str(tree / "etc/hostname"), str(tree / "missing")],
+                                 capture_output=True, text=True, timeout=60,
+                                 env=dict(os.environ, LD_PRELOAD="%s:%s" % (cls.lib, cls.tmp / "fp/libblfastpath.so")))
+        (cls.tmp / "real").mkdir()
+        subprocess.run(["cc", "-O2", "-Wall", "-Werror", "-fPIC", "-shared", "-pthread", "-o",
+                        str(cls.tmp / "real/libblfastpath.so"), str(FASTPATH), "-ldl"], check=True)
+        shutil.copy(cls.tmp / "raw_open", cls.tmp / "FEX")
+        try:
+            cls.fex = subprocess.run([str(cls.tmp / "FEX"), str(tree / "etc/hostname"), str(tree / "missing")],
+                                     capture_output=True, text=True, timeout=20,
+                                     env=dict(os.environ, PROOT_FP_ROOT="/", PROOT_FP_KEY="test",
+                                              LD_PRELOAD="%s:%s" % (cls.lib, cls.tmp / "real/libblfastpath.so")))
+        except subprocess.TimeoutExpired:
+            cls.fex = None
 
     @classmethod
     def tearDownClass(cls):
@@ -205,6 +293,20 @@ class Openat2PreloadTest(unittest.TestCase):
         for case, got in zip(HOST_CASES, self.preload[len(CASES):]):
             self.assertEqual(got, case[3], case)
 
+    def test_the_walk_asks_the_fast_path_when_it_is_loaded(self):
+        self.assertEqual(self.counted, self.preload)
+        self.assertTrue(all(n > 0 for n in self.calls), self.calls)
+
+    def test_a_raw_openat_syscall_asks_the_fast_path_too(self):
+        self.assertEqual(self.raw.returncode, 0, self.raw.stderr)
+        self.assertEqual(self.raw.stdout, "inside\n")
+        self.assertEqual(self.raw.stderr.split("calls ")[-1].split(), ["2", "0", "0"])
+
+    def test_a_first_raw_openat_in_fex_does_not_wait_on_the_fast_path_starting(self):
+        self.assertIsNotNone(self.fex, "hung")
+        self.assertEqual(self.fex.returncode, 0, self.fex.stderr)
+        self.assertEqual(self.fex.stdout, "inside\n")
+
     def test_the_kernel_agrees(self):
         if self.kernel is None:
             self.skipTest("this kernel or sandbox has no openat2")