Start .NET and Godot games under FEX without per-game setup (#168)

app/src/main/java/com/droiddeck/launcher/core/GameEnvironment.kt

@@ -29,6 +29,12 @@ object GameEnvironment {
         "MESA_SHADER_CACHE_DISABLE" to "false",
         "VKD3D_FEATURE_LEVEL" to "12_2",
         "VKD3D_SHADER_MODEL" to "6_9",
+        // .NET 7 and later reserve 256 GB of address space for the GC heap at startup, more than an
+        // Android process has free, and the runtime fails to start. 32 GB is plenty for a game.
+        "DOTNET_GCRegionRange" to "0x800000000",
+        // .NET's W^X writes JIT code through a second mapping that FEX does not watch, so FEX keeps
+        // running the stale translation and the game hangs at startup.
+        "DOTNET_EnableWriteXorExecute" to "0",
     ).apply {
         FexPreset.env(preset).forEach { put(it.substringBefore('='), it.substringAfter('=')) }
     }

app/src/test/java/com/droiddeck/launcher/core/GameEnvironmentTest.kt

@@ -36,6 +36,12 @@ class GameEnvironmentTest {
         assertEquals("6_9", restored["VKD3D_SHADER_MODEL"])
     }
 
+    @Test fun dotnetRuntimeFitsAndroidAddressSpaceForEveryGame() {
+        val values = GameEnvironment.effective(GameEnvironment.Config(), "PERFORMANCE", "43")
+        assertEquals("0x800000000", values["DOTNET_GCRegionRange"])
+        assertEquals("0", values["DOTNET_EnableWriteXorExecute"])
+    }
+
     @Test fun multiSelectionPreservesCustomTokensAndRemovesToggledValues() {
         assertEquals("fps,custom_token,frametimes", GameEnvironmentOptions.toggle("fps,custom_token", "frametimes"))
         assertEquals("custom_token", GameEnvironmentOptions.toggle("fps,custom_token", "fps"))

tools/linuxfs/overlay/usr/local/bin/bannerlator-game-env

@@ -19,11 +19,109 @@ def game_id(env):
     return str(number % 2**32)
 
 
-def apply_config(env, config, app_id):
+# A .NET (CoreCLR) runtime under FEX crashes in the runtime itself unless memory ordering is fully
+# emulated and multiblock is off: the Stability preset.
+DOTNET_FEX = {
+    "FEX_TSOENABLED": "1",
+    "FEX_VECTORTSOENABLED": "1",
+    "FEX_MEMCPYSETTSOENABLED": "1",
+    "FEX_HALFBARRIERTSOENABLED": "1",
+    "FEX_X87REDUCEDPRECISION": "0",
+    "FEX_MULTIBLOCK": "0",
+}
+
+
+def game_files(exe):
+    """Lower-cased names of the files next to the game's executable and one directory below."""
+    names = set()
+    folder = os.path.dirname(exe)
+    try:
+        with os.scandir(folder) as top:
+            for entry in top:
+                names.add(entry.name.lower())
+                if entry.is_dir(follow_symlinks=False):
+                    try:
+                        with os.scandir(entry.path) as inner:
+                            names.update(e.name.lower() for e in inner)
+                    except OSError:
+                        pass
+    except OSError:
+        pass
+    return names
+
+
+class Game:
+    """What a fix gets to look at: the executable, its launch arguments and the files beside it."""
+
+    def __init__(self, exe, args):
+        self.exe = exe
+        self.args = args
+        self.stem = os.path.splitext(os.path.basename(exe))[0].lower()
+        self.files = game_files(exe)
+
+    def has_arg(self, name):
+        return any(a == name or a.startswith(name + "=") for a in self.args)
+
+
+# Each fix looks at a Game and returns (environment, extra arguments, note) for the games it
+# applies to, or None. Adding one is writing a function and decorating it.
+FIXES = []
+
+
+def fix(function):
+    FIXES.append(function)
+    return function
+
+
+def is_godot(game):
+    return game.stem + ".pck" in game.files or "godotsharp.dll" in game.files
+
+
+@fix
+def dotnet_memory_ordering(game):
+    if "coreclr.dll" in game.files:
+        return DOTNET_FEX, [], ".NET runtime: FEX with full TSO and without multiblock"
+
+
+@fix
+def godot_vulkan(game):
+    # Godot 4 on Direct3D 12 through VKD3D-Proton fails to create its swap chain and runs behind a
+    # black window; its own Vulkan renderer draws.
+    if is_godot(game) and not game.has_arg("--rendering-driver"):
+        return {}, ["--rendering-driver", "vulkan"], "Godot: Vulkan renderer"
+
+
+@fix
+def godot_sentry(game):
+    # The Sentry GDExtension's native crash handler takes the .NET runtime down under FEX. Godot
+    # loads it on demand and carries on without it.
+    sentry = sorted(n[:-4] for n in game.files if n.startswith("libsentry.windows.") and n.endswith(".dll"))
+    if is_godot(game) and sentry:
+        return {"WINEDLLOVERRIDES": ";".join(n + "=d" for n in sentry)}, [], "Godot: Sentry crash handler off"
+
+
+def engine_fixes(exe, args):
+    """Environment and extra arguments a game needs because of what it is built with."""
+    game = Game(exe, args)
+    env, extra, notes = {}, [], []
+    for check in FIXES:
+        found = check(game)
+        if found:
+            env.update(found[0])
+            extra += found[1]
+            notes.append(found[2])
+    return env, extra, notes
+
+
+def apply_config(env, config, app_id, fixes=None):
     if config.get("version") != 1:
         raise ValueError("unsupported environment configuration")
     result = env.copy()
     entries = dict(config.get("shared", {}))
+    for name, value in (fixes or {}).items():
+        if name == "WINEDLLOVERRIDES" and entries.get(name, env.get(name)):
+            value = entries.get(name, env.get(name)) + ";" + value
+        entries[name] = value
     entries.update(config.get("games", {}).get(app_id, {}))
     for name, value in entries.items():
         if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
@@ -53,12 +151,16 @@ def main():
     env = os.environ.copy()
     app_id = game_id(env)
     if len(command) > 1 and command[1] == "waitforexitandrun" and app_id:
+        fixes, extra, notes = engine_fixes(command[2], command[3:]) if len(command) > 2 else ({}, [], [])
+        for note in notes:
+            print("bannerlator-game-env: " + note, file=sys.stderr)
+        command += extra
         path = Path.home() / ".config/droiddeck/game-environment.json"
         try:
             with path.open(encoding="utf-8") as source:
-                env = apply_config(env, json.load(source), app_id)
+                env = apply_config(env, json.load(source), app_id, fixes)
         except FileNotFoundError:
-            pass
+            env = apply_config(env, {"version": 1}, app_id, fixes)
         except (OSError, ValueError, TypeError, AttributeError):
             print("bannerlator-game-env: invalid configuration; using inherited environment", file=sys.stderr)
     os.execvpe(command[0], command, env)

tools/tests/test_game_environment.py

@@ -20,6 +20,43 @@ class GameEnvironmentTest(unittest.TestCase):
         self.assertEqual(env["REMOVE"], "inherited")
         self.assertEqual(MODULE["apply_config"](env, config, "43")["CUSTOM"], "shared")
 
+    def test_engine_fixes_follow_the_games_files(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            game = Path(tmp) / "Godot Game"
+            (game / "data_game_windows_x86_64").mkdir(parents=True)
+            for name in ("Game.exe", "Game.pck", "libsentry.windows.release.x86_64.dll", "data_game_windows_x86_64/coreclr.dll"):
+                (game / name).touch()
+            env, extra, notes = MODULE["engine_fixes"](str(game / "Game.exe"), [])
+            self.assertEqual(env["FEX_TSOENABLED"], "1")
+            self.assertEqual(env["FEX_MULTIBLOCK"], "0")
+            self.assertEqual(env["WINEDLLOVERRIDES"], "libsentry.windows.release.x86_64=d")
+            self.assertEqual(extra, ["--rendering-driver", "vulkan"])
+            self.assertEqual(MODULE["engine_fixes"](str(game / "Game.exe"), ["--rendering-driver", "opengl3"])[1], [])
+            other = Path(tmp) / "Other"
+            other.mkdir()
+            (other / "Other.exe").touch()
+            self.assertEqual(MODULE["engine_fixes"](str(other / "Other.exe"), []), ({}, [], []))
+
+    def test_a_new_fix_only_needs_registering(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            (Path(tmp) / "Game.exe").touch()
+            (Path(tmp) / "engine.dll").touch()
+            check = lambda game: ({"ENGINE": "1"}, ["--flag"], "engine") if "engine.dll" in game.files else None
+            MODULE["FIXES"].append(check)
+            try:
+                self.assertEqual(MODULE["engine_fixes"](str(Path(tmp) / "Game.exe"), []), ({"ENGINE": "1"}, ["--flag"], ["engine"]))
+            finally:
+                MODULE["FIXES"].remove(check)
+
+    def test_engine_fixes_sit_between_shared_and_game_profiles(self):
+        env = {"WINEDLLOVERRIDES": "dxgi=n"}
+        fixes = {"FEX_MULTIBLOCK": "0", "WINEDLLOVERRIDES": "libsentry=d"}
+        config = {"version": 1, "shared": {"FEX_MULTIBLOCK": "1"}, "games": {"42": {"FEX_TSOENABLED": "0"}}}
+        result = MODULE["apply_config"](env, config, "42", {**fixes, "FEX_TSOENABLED": "1"})
+        self.assertEqual(result["FEX_MULTIBLOCK"], "0")
+        self.assertEqual(result["FEX_TSOENABLED"], "0")
+        self.assertEqual(result["WINEDLLOVERRIDES"], "dxgi=n;libsentry=d")
+
     def test_invalid_configuration_is_atomic(self):
         env = {"ORIGINAL": "unchanged"}
         for entries in ({"A": "ok", "BAD=KEY": "x"}, {"A": "bad\0value"}, {"A": 1}):