From 8cee1afe5dd9461ebd4397a7d02c26b3ecd6b03b Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 08:48:31 +0300 Subject: [PATCH] The rebuild builds mod plugins in: service run, checks, patch sets and status lines (#845) After the design in #845 by itaskaev2. tools/exefs_plugins.py writes an accepted plugin's patch set (recomp/patches.py plugin lines and hook sites, one ordered set mod_ per plugin, several plugins in one record told apart by their hooks) and a line per plugin of build-cache/plugins.status: built, or refused with the first reason. It also checks the code the hooks reach (direct branches and calls from the stubs, impls, callbacks and targets): no svc but OutputDebugString, imports only from a reviewed list. The init's imports may now include malloc, free, memcpy, memmove and memset, because the init runs again at every start and its heap blocks exist while the game runs. The setup's lift step runs the plugin stage when a mod has exefs/subsdk*: a lift with the plugins, a totk.exe of it in the compile step's folder, the service run, then the lift with the hook sites; a failed stage warns (warn.plugins_skipped) and installs without the plugins. docs/mods.md describes what works, what is refused and why, and the rebuild. Co-Authored-By: Claude Opus 5.5 --- docs/mods.md | 37 ++++- docs/settings.md | 3 +- installer/steps.py | 105 ++++++++++++-- installer/text/en.json | 1 + installer/text/pt-BR.json | 1 + installer/text/ru.json | 1 + installer/text/zh-Hans.json | 1 + tests/exefs_plugins_test.py | 218 +++++++++++++++++++++++++++- tests/installer_test.py | 54 +++++++ tools/exefs_plugins.py | 282 ++++++++++++++++++++++++++++++++++-- tools/make_release.py | 2 +- 11 files changed, 671 insertions(+), 34 deletions(-) diff --git a/docs/mods.md b/docs/mods.md index f369035c..089207b9 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -49,8 +49,8 @@ named ones. ### Mods with code the port cannot run -Some mods bring code the port cannot run: their own Switch code (a subsdk plugin or a replaced main, as UltraCam ships) -or a code patch in a form the port cannot apply. Such a mod stays off as a whole, because its other files may expect +Some mods bring code the port cannot run: their own Switch code (a replaced main, or a subsdk plugin the rebuild +refused, as it refuses UltraCam's) or a code patch in a form the port cannot apply. Such a mod stays off as a whole, because its other files may expect that code. Under its row the Mods tab then shows **Use without its code** (off by default). Switched on, the code the port cannot run is left out and the rest of the mod loads after a restart: its game files, cheats and the code patches that work. The mod's row says which part is left out and why. If the game misbehaves with it, switch it off @@ -96,6 +96,35 @@ A patch file may also change the game's data (constants and tables in its `.roda stored into the loaded game at start, before it runs. When two mods write the same bytes, the higher one wins, as with files. +## Code plugins (exefs subsdk) + +Some mods ship their own Switch code as an exlaunch plugin: a file `exefs\subsdk` (The Second Quest's MSFix, Full +Inventory Expansion's sort fixer, Quick Weapon Durability). On the Switch the plugin starts with the game and +redirects some of the game's instructions into its own code. The port does the same with the plugin's code compiled +ahead of time, after the design in [#845](https://github.com/valentderah/totk-pc-recomp/issues/845) by itaskaev2: + +1. The rebuild (the same one as for code patches above) compiles the plugin with the game, starts the game's code + once in a service mode that runs only the plugin's start-up (`totk.exe --plugin-init`), twice, and records which + instructions it redirects. Each redirect becomes a code site of the mod, built into `totk.exe`. +2. At every start the plugin's start-up runs again, before the game's own start-up code, in the same setting. Its + redirects are compared with the record of the rebuild. When they match, the mod's code sites go on. When they do + not, the plugin stays off for this run, the mod's row says why, and the next start runs the rebuild. +3. The plugin keeps what its start-up made (its memory, its tables), so its code works as on the Switch. The start + costs about 50 ms per plugin; the game runs no differently without a plugin. + +The Mods tab order decides between a plugin and another mod's patch on the same instruction, as for code patches. A +plugin site over one of the port's own patches turns that built-in patch off while the mod is on. + +The rebuild refuses a plugin when its start-up or the code its redirects reach needs what the port does not give it: +system calls other than the few its start-up needs (memory queries and maps, the clock, debug output), library calls +outside a reviewed list (memory and string functions, the heap, text formatting), threads, services or input; a +redirect that is not one of exlaunch's hook shapes (inline hook, replace hook, call-site rewrite); or two runs of its +start-up that differ in code. The row then says **Its plugin ... cannot run in the port** with the reason, and the +mod stays off; **Use without its code** uses the rest of it. UltraCam stays refused: it hooks the graphics API, +reads input and opens sockets. + +`plugin_trace = 1` in `totk.ini` logs the first call into each of a plugin's hooks (`plugins: hook entry ... reached`). + ## Cheats A cheat file is part of a mod: `mods\\cheats\.txt`, switched cheat by cheat on the Cheats tab. Cheats @@ -134,7 +163,9 @@ keeps its textures at a quarter of the size. | This folder has no romfs, exefs or cheats files. | The archive was unpacked one level too deep (`mods\Mod\Mod\romfs`), or it holds no game files. | Move the folders up so `romfs` sits directly in the mod's folder. | | ... files carry another game version in their names. | The files are named for another update (`.Product..`) and the game's own files have no such name. They are still served under those names, as on a console, because the game sometimes asks for one; another update's resource size table is the exception and only lends its entries. | Nothing if the mod works. If it does nothing, download its release for your game update. | | Its code patches (exefs) are made for another game version. | The patch files name another update's build id. | Same: get the release for your update. | -| This mod brings its own game code (a subsdk plugin or a replaced main). | Mods such as UltraCam ship compiled Switch code. The port cannot load it. | The whole mod stays off. **Use without its code** under its row uses the rest of it, which may expect the missing code (see Mods with code the port cannot run). | +| This mod brings its own game code (a subsdk plugin or a replaced main). | The mod replaces the game's main code, which the port cannot load. | The whole mod stays off. **Use without its code** under its row uses the rest of it, which may expect the missing code (see Mods with code the port cannot run). | +| Its plugin ... cannot run in the port: ... | The rebuild refused the mod's code plugin; the reason follows (see Code plugins). | The whole mod stays off. **Use without its code** under its row uses the rest of it. | +| Its plugin stays off this time: it started differently from the build. | The plugin redirected other instructions at this start than in the rebuild, for example after the mod was updated in place. | Nothing: the next start runs the rebuild. | | This mod patches the game code (exefs) in a way the port cannot apply. | The patch changes code in a form the port cannot build, or bytes the port cannot change at start. The reason follows on the row. | The whole mod stays off. **Use without its code** under its row leaves that patch file out and uses the rest of the mod. | | The folder name contains \|. | The mod list uses `\|` between names. | Rename the folder. | diff --git a/docs/settings.md b/docs/settings.md index 6f23cdda..388c47a8 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -24,7 +24,8 @@ Many rows below are developer tools that only act together with another setting | `core_diag` | `1` records why ready guest threads wait for a virtual core and for how long, with a histogram every 60 presents. | Turns on host-call and SVC depth tracking and writes many log lines (one per present over 25 ms); frames get slower. | Caution | | `barrier_merge` (default `0`) | Experimental. `1` keeps a render pass open through the game's Barrier calls that ask only for fragment ordering and texture invalidation, when no draw of the pass can write a storage buffer or image; the next draw that samples a written target gets a by-region barrier in the pass (the feedback layout). In scenes with about a thousand such barriers a frame this cuts the passes from about 1200 to 130 and the GPU time from 24.5 to 7.8 ms. | The game's barrier orders every pixel, the in-pass by-region barrier only the pixel a draw writes: a driver that runs it per screen tile can show flicker or squares at the edges of water, decals or particles. | Caution | | `menu_cursor_log` | `1` logs what the menu cursor code reads at every screen change. | Only adds log lines. | Safe | -| `plugin_tick_offset` (default `0`) | Added to every GetSystemTick in the plugin service run (`totk.exe --plugin-init DIR`), so `tools/exefs_plugins.py` can run a plugin's init twice with other ticks and compare the records. | Has no effect outside `--plugin-init`; the game never asks for GetSystemTick. | Safe | +| `plugin_tick_offset` (default `0`) | Added to every GetSystemTick in the plugin service run (`totk.exe --plugin-init DIR`), so `tools/exefs_plugins.py` can run a plugin's init twice with other ticks and compare the records. | Has no effect outside `--plugin-init`; a plugin's init at the start of the game gets the plain ticks. | Safe | +| `plugin_trace` | `1` logs the first call of each entry of a mod plugin's hooks (its callbacks, replace and call targets) as `plugins: hook entry ... reached`. | Each such call goes through a counting step; nothing changes without a mod plugin. | Safe | | `perf_slow_ms` (default `50`) | Frames at least this many milliseconds long get a `perf: slow frame` log line with the time blocked per cause. | Changes how many frames are logged; a small value fills the log. | Safe | | `replay_check` | `1` reads every texture read-back a second time after a full drain of the replay queue and compares the two. | Stalls the replay queue at every read-back, so frames get much slower; mismatches are only counted and logged. | Caution | | `resolve_check` | `1` compares every cached pool lookup with the locked lookup and stops on a difference. | Slows every pool lookup; the game stops if the two ever differ. | Caution | diff --git a/installer/steps.py b/installer/steps.py index 3c3c1e25..08ff4baa 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -19,7 +19,8 @@ lift cache, gen//lift_cache.json) and rewrites only those whose C changed, those. A change of the mods' code cheats then costs a few seconds of lift and the chunks holding its sites, not a full lift or compile. -Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status and exefs.data (tools/exefs_patches.py) and +Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status and exefs.data (tools/exefs_patches.py), +plugins.status (tools/exefs_plugins.py; plugins.stale is the game's list of plugins whose start differed) and rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild), one argument per line; totk.exe runs it at start when an enabled mod has code cheats the build lacks (runtime/cheats.c). @@ -866,6 +867,43 @@ def exefs_files(ctx): return sorted(out) +def plugin_files(ctx): + """The exlaunch plugins of the mods folder (each mod's exefs/subsdk*, in the first ExeFS place, as + tools/exefs_plugins.py finds them), by mod name, for the lift key and the plugin stage.""" + mods = ctx.dest / 'mods' + if not mods.is_dir(): + return [] + out = [] + for mod in sorted((m for m in mods.iterdir() if m.is_dir()), key=lambda m: m.name): + exefs = next((mod / w for w in EXEFS_PLACES if (mod / w).is_dir()), None) + if exefs: + out += sorted((f for f in exefs.iterdir() if f.is_file() and f.name.lower().startswith('subsdk')), + key=lambda f: f.name.lower()) + return out + + +def plugins_status(ctx): + """The build's plugin status file (tools/exefs_plugins.py --status), which runtime/mods.c reads.""" + return ctx.dest / 'build-cache' / 'plugins.status' + + +def built_plugins(status, files, mods): + """The files of the plugins the status file lists as built (kind `plugin`), in the order of files. The key is + FNV-1a 32 of "/", as tools/exefs_plugins.py and runtime/mods.c compute it.""" + if not status.is_file(): + return [] + built = {line.split('\t')[0] for line in status.read_text(encoding='utf-8').splitlines() + if len(line.split('\t')) >= 3 and line.split('\t')[2] == 'plugin'} + + def key(f): + rel = f.relative_to(mods).as_posix() + h = 0x811C9DC5 + for b in rel.encode(): + h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF + return f'{h:08X}' + return [f for f in files if key(f) in built] + + def exefs_status(ctx): """The build's ExeFS patch status file (tools/exefs_patches.py --status), which runtime/mods.c reads.""" return ctx.dest / 'build-cache' / 'exefs.status' @@ -890,9 +928,12 @@ class Lift(Step): def key(self, ctx): # fingerprint takes a file's name and content only; the paths under mods/ go in too, because tools/cheats.py # and tools/exefs_patches.py name each patch set after its mod folder, so a renamed folder needs a lift - files = cheat_files(ctx) + exefs_files(ctx) + # A mod plugin is built by running its init on a totk.exe of this build (tools/exefs_plugins.py), so the + # runtime is in the key too when there is one. + files = cheat_files(ctx) + exefs_files(ctx) + plugin_files(ctx) + extra = [ctx.layout.tools / 'exefs_plugins.py', runtime_key(ctx)] if plugin_files(ctx) else [] return fingerprint('lift-5', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)), - ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *files, + ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *extra, *files, *(f.relative_to(ctx.dest / 'mods').as_posix() for f in files)) def current(self, ctx, record): @@ -943,12 +984,44 @@ class Lift(Step): else: patches = ctx.temp / out args = ['--exefs', ctx.dest / 'exefs', '--out', gen] + # The mods' exlaunch plugins (tools/exefs_plugins.py, after the design in #845 by itaskaev2): a lift with the + # plugins in the layout, a totk.exe of it in build/ (the compile step's folder, so it recompiles only what + # the hook sites change after), the plugin service run, and the plugins' patch sets for the lift below. A + # refused plugin gets its reason in plugins.status; a failed stage costs the plugins, not the install. + plugins = plugin_files(ctx) + drop([plugins_status(ctx), ctx.dest / 'build-cache' / 'plugins.stale']) + lift_plugins = [] + if plugins: + listed = ['--plugins', ','.join(str(f) for f in plugins)] + code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches] + listed, parse) + if not code: + code, log = run_tool(ctx, 'lift', layout.tools / 'build_game.py', build_args(ctx, gen)) + if not code: + code, log = run_tool(ctx, 'lift', layout.tools / 'exefs_plugins.py', [ + '--exe', ctx.temp / 'build' / ('totk' + EXE), '--out', ctx.temp / 'plugins', '--mods', + ctx.dest / 'mods', '--patches', patches, '--patches-out', ctx.temp / 'patches-plugins.txt', + '--status', plugins_status(ctx)]) + # exit 1 with a status file: refused plugins, their reasons in it; the rest goes on + if plugins_status(ctx).is_file() and (ctx.temp / 'patches-plugins.txt').is_file(): + code = 0 + if code: + drop([plugins_status(ctx)]) + skipped['warn.plugins_skipped'] = log + else: + patches = ctx.temp / 'patches-plugins.txt' + lift_plugins = built_plugins(plugins_status(ctx), plugins, ctx.dest / 'mods') + if lift_plugins: + args += ['--plugins', ','.join(str(f) for f in lift_plugins)] code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches], parse) if code and patches != plain: for tool, out, outputs, inputs, warn in stages: if inputs and warn not in skipped: drop(outputs) skipped[warn] = log + if lift_plugins: + drop([plugins_status(ctx)]) + skipped.setdefault('warn.plugins_skipped', log) + args = ['--exefs', ctx.dest / 'exefs', '--out', gen] code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', plain], parse) if code: raise InstallError('lift.failed', code=code, log=str(log)) @@ -982,6 +1055,21 @@ def runtime_key(ctx): layout.sysroot or 'host', *tools, clang, *[d.name for d in layout.dlls]) +def build_args(ctx, gen): + """tools/build_game.py's arguments for a totk.exe of the lifted C in gen, built in /build.""" + layout, dest = ctx.layout, ctx.dest + cpu = getattr(ctx, 'cpu', 'v3') + lib = layout.runtime_lib_for(cpu) + pgo = layout.pgo(game_version(ctx)) + return ['--gen', gen, '--out', ctx.temp / 'build', '--toolchain', layout.llvm, '--cpu', cpu, + *(['--deps', layout.deps] if layout.deps else []), *(['--sysroot', layout.sysroot] if layout.sysroot else []), + '--runtime', layout.runtime, *(['--runtime-lib', lib] if lib else ()), *(['--pgo', pgo] if pgo else ()), + '--ninja', layout.ninja, + '--version', version_tag(game_version(ctx)), '--expected', layout.expected(game_version(ctx)), + '--exefs', dest / 'exefs', '--romfs', dest / 'romfs', + '--profile', dest / 'profile'] + + class Compile(Step): id = 'compile' @@ -1012,16 +1100,7 @@ class Compile(Step): rep.progress('compile', 'compile', force=True) cpu = getattr(ctx, 'cpu', 'v3') - lib = layout.runtime_lib_for(cpu) - pgo = layout.pgo(game_version(ctx)) - code, log = run_tool(ctx, 'compile', layout.tools / 'build_game.py', [ - '--gen', ctx.temp / 'gen', '--out', build, '--toolchain', layout.llvm, '--cpu', cpu, - *(['--deps', layout.deps] if layout.deps else []), *(['--sysroot', layout.sysroot] if layout.sysroot else []), - '--runtime', layout.runtime, *(['--runtime-lib', lib] if lib else ()), *(['--pgo', pgo] if pgo else ()), - '--ninja', layout.ninja, - '--version', version_tag(game_version(ctx)), '--expected', layout.expected(game_version(ctx)), - '--exefs', dest / 'exefs', '--romfs', dest / 'romfs', - '--profile', dest / 'profile'], parse) + code, log = run_tool(ctx, 'compile', layout.tools / 'build_game.py', build_args(ctx, ctx.temp / 'gen'), parse) exe = build / ('totk' + EXE) if code and link_output_locked(log): raise InstallError('compile.exe_locked', path=str(exe), log=str(log)) diff --git a/installer/text/en.json b/installer/text/en.json index 254a40e7..c40def78 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -108,6 +108,7 @@ "warn.npdm_differs": "main.npdm differs from the reference {version} dump; only the code modules are used.", "warn.cheats_skipped": "The cheats of the mods folder that change the game's code could not be built; the game installs without them. Details: {log}", "warn.exefs_skipped": "The mods' ExeFS patches (.ips, .pchtxt) could not be built; the game installs without them. Details: {log}", + "warn.plugins_skipped": "The mods' code plugins (exefs subsdk) could not be built; the game installs without them. Details: {log}", "warn.xbox_prompts_failed": "The Xbox or PC button prompts could not be prepared (exit code {code}); the game keeps the Switch prompts. Details: {log}", "warn.shaders_skipped": "The graphics driver crashed while building some shaders, so {count} were left out of the preparation. The game builds them when it first needs them, and the driver may crash there too; a driver update may fix it. Details: {log}", "warn.kernel_old": "Linux {version} lacks the write tracking the game relies on (6.7 or later); the game may run very slowly. Updating the system fixes this.", diff --git a/installer/text/pt-BR.json b/installer/text/pt-BR.json index 5c8579b2..3ce41439 100644 --- a/installer/text/pt-BR.json +++ b/installer/text/pt-BR.json @@ -107,6 +107,7 @@ "warn.lang_needs_fonts": "Textos em {lang} precisam das fontes de sistema do Switch, e nenhuma foi informada. O jogo não inicia neste idioma até que sejam adicionadas. Escolha a pasta do firmware (Contents/registered) ou veja \"Switch system fonts\" em docs/installer.md.", "warn.cheats_skipped": "Não foi possível compilar os cheats da pasta de mods que alteram o código do jogo; o jogo é instalado sem eles. Detalhes: {log}", "warn.exefs_skipped": "Não foi possível compilar os patches ExeFS dos mods (.ips, .pchtxt); o jogo é instalado sem eles. Detalhes: {log}", + "warn.plugins_skipped": "Não foi possível compilar os plugins de código dos mods (exefs subsdk); o jogo é instalado sem eles. Detalhes: {log}", "warn.npdm_differs": "O main.npdm difere do dump de referência da versão {version}; só os módulos de código são usados.", "warn.xbox_prompts_failed": "Não foi possível preparar os ícones de botão do Xbox ou do PC (código de saída {code}); o jogo mantém os ícones do Switch. Detalhes: {log}", "warn.shaders_skipped": "O driver gráfico travou ao compilar alguns shaders, então {count} ficaram fora da preparação. O jogo os compila quando precisar deles pela primeira vez, e o driver pode travar ali também; uma atualização do driver pode resolver. Detalhes: {log}", diff --git a/installer/text/ru.json b/installer/text/ru.json index f329d0b2..d20cd076 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -108,6 +108,7 @@ "warn.npdm_differs": "main.npdm отличается от эталонного дампа {version}; используются только модули с кодом.", "warn.cheats_skipped": "Не удалось собрать читы из папки модов, меняющие код игры; игра установится без них. Подробности: {log}", "warn.exefs_skipped": "Не удалось собрать ExeFS-патчи модов (.ips, .pchtxt); игра установится без них. Подробности: {log}", + "warn.plugins_skipped": "Не удалось собрать плагины кода модов (exefs subsdk); игра установится без них. Подробности: {log}", "warn.xbox_prompts_failed": "Не удалось подготовить подсказки кнопок Xbox или PC (код выхода {code}); игра оставит подсказки Switch. Подробности: {log}", "warn.shaders_skipped": "Графический драйвер аварийно завершился при сборке части шейдеров, поэтому {count} из них пропущены при подготовке. Игра соберёт их, когда они впервые понадобятся, и драйвер может упасть и там; обновление драйвера может это исправить. Подробности: {log}", "warn.kernel_old": "В Linux {version} нет отслеживания записи, на которое опирается игра (нужна версия 6.7 или новее); игра может работать очень медленно. Обновление системы это исправит.", diff --git a/installer/text/zh-Hans.json b/installer/text/zh-Hans.json index 1d0e4b8a..1b614894 100644 --- a/installer/text/zh-Hans.json +++ b/installer/text/zh-Hans.json @@ -107,6 +107,7 @@ "warn.lang_needs_fonts": "{lang}文本需要 Switch 系统字体,但未提供。在添加这些字体之前,游戏无法以该语言启动。请选择固件文件夹(Contents/registered),或参阅 docs/installer.md 中的“Switch system fonts”一节。", "warn.cheats_skipped": "无法构建 mods 文件夹中修改游戏代码的金手指;游戏将不含它们进行安装。详情:{log}", "warn.exefs_skipped": "无法构建模组的 ExeFS 补丁(.ips、.pchtxt);游戏将不含它们进行安装。详情:{log}", + "warn.plugins_skipped": "无法构建模组的代码插件(exefs subsdk);游戏将不含它们进行安装。详情:{log}", "warn.npdm_differs": "main.npdm 与参考的 {version} 导出不同;只使用其中的代码模块。", "warn.xbox_prompts_failed": "无法准备 Xbox 或 PC 按键提示(退出代码 {code});游戏将继续使用 Switch 按键提示。详情:{log}", "warn.shaders_skipped": "显卡驱动在编译部分着色器时崩溃,因此有 {count} 个着色器未被预先准备。游戏会在首次需要时编译它们,驱动在那时也可能崩溃;更新驱动可能会解决。详情:{log}", diff --git a/tests/exefs_plugins_test.py b/tests/exefs_plugins_test.py index 6bb23495..7687d6c9 100644 --- a/tests/exefs_plugins_test.py +++ b/tests/exefs_plugins_test.py @@ -1,8 +1,13 @@ """recomp/layout.py with and without mod plugins, and tools/exefs_plugins.py's check of a plugin service run record: -a synthetic exlaunch-shaped record with one inline hook that passes, and one bad record per rule. No game files. +a synthetic exlaunch-shaped record with one inline hook that passes, and one bad record per rule; the check of the +code the hooks reach; the patch lines of the three hook kinds; and what recomp/patches.py and recomp/gen.py make of +those lines (the hook call, the stub's return, the unpatched copy a replace hook's trampoline reaches, the start-up +record in patches.h). The start-up comparison itself runs in the game (runtime/plugin_init.c plugins_start); its +mods.c side is in totk.exe --probe-mods. No game files. Run with .venv (lz4 and numpy, as recomp/): python tests/exefs_plugins_test.py """ +import hashlib import json import struct import sys @@ -14,7 +19,9 @@ ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / 'tools')) sys.path.insert(0, str(ROOT / 'recomp')) import exefs_plugins as xp # noqa: E402 +import gen # noqa: E402 import layout # noqa: E402 +import patches # noqa: E402 from nso import Nso # noqa: E402 @@ -278,5 +285,214 @@ class Diff(unittest.TestCase): self.assertEqual(xp.diff_records(a, record()), []) +# ---- the code the hooks reach +SVC_40, RET_W, NOP = 0xD4000801, 0xD65F03C0, 0xD503201F + + +def code(words, at=0): + img = bytearray(0x200) + for i, w in enumerate(words): + struct.pack_into('x[30] =', head) # no x30 written for the call + self.assertIn(f'void f_{stub:x}(Cpu *c);', c) + h = patches.header_text(t, 'MARK') + self.assertIn('#define RT_PLUGIN_COUNT 1', h) + self.assertIn(f'X(0, "main", {FN:#x}, 0xaa0003f3u, 0x14000000u)', h) + self.assertIn('X("plugin0", 0x4100) X("plugin0", 0x4200)', h) + + def test_stub_returns_instead_of_branching_to_its_trampoline(self): + t = self.table(f'site mod_x main {FN:x} aa0003f3 @call:plugin0+4000 aa0003f3', ['stub plugin0 4000']) + stub = self.bases['plugin0'] + 0x4000 + c = self.emit(t, 'plugin0', stub, stub + 16) + body = c[c.index(f'void f_{stub:x}(Cpu *c) {{'):] + self.assertIn('return;', body) + self.assertNotIn(f'{self.bases["plugin0"] + 0x2000:#x}', body) # the trampoline is not reached + + def test_replace_hook_keeps_an_unpatched_copy(self): + pb = self.bases['plugin0'] + m = self.bases['main'] + FN + new = b(m, pb + 0x4200) + t = self.table(f'site mod_x main {FN:x} aa0003f3 {new:08x}', [f'orig mod_x plugin0 2000 main {FN:x}', + 'seed plugin0 4200']) + self.assertEqual(t.origs_for('main', self.bases['main']), {m}) + self.assertEqual(t.orig_entries('plugin0', self.bases), {pb + 0x2000: m}) + c = self.emit(t, 'main', m, m + 12, origs=[m]) + copy = c[c.index(f'void fo_{m:x}(Cpu *c) {{'):] + self.assertNotIn('rt_patch_on', copy) + self.assertIn('rt_patch_on[0]', c[:c.index(f'void fo_{m:x}')]) + tab = gen.table_text('plugin0', [pb + 0x4200, pb + 0x2000], {pb + 0x2000: gen.orig_name(m)}) + self.assertIn(f'void fo_{m:x}(Cpu *c);', tab) + self.assertIn(f'fo_{m:x}', tab[tab.index('_fn_ptrs'):]) + + def test_image_must_match_its_hash(self): + t = self.table(f'site mod_x main {FN:x} aa0003f3 @call:plugin0+4000 aa0003f3', ['stub plugin0 4000']) + t.plugins['plugin0']['sha256'] = '0' * 64 + with self.assertRaises(patches.PatchError): + gen.open_module('plugin0', self.plugin_file, self.bases['plugin0'], t) + + def test_chunk_key_takes_the_unpatched_copy(self): + # the lift cache (recomp/gen.py chunk_key): a chunk whose function gets an unpatched copy is keyed apart, and a + # chunk without one keeps the key it had before plugins + fns = [(0x100, 0x10c)] + plain = gen.chunk_key('m', 0, fns, {}, [], {}, [], None, None) + self.assertEqual(gen.chunk_key('m', 0, fns, {}, [], {}, [], None, None, {0x200}), plain) + self.assertNotEqual(gen.chunk_key('m', 0, fns, {}, [], {}, [], None, None, {0x100}), plain) + site = {0x100: [(0, [('call', 0x7005204000), 0xAA0003F3])]} + self.assertNotEqual(gen.chunk_key('m', 0, fns, site, [0x100], {}, [], None, None), + gen.chunk_key('m', 0, fns, {0x100: [(0, [('call', 0x7005204010), 0xAA0003F3])]}, [0x100], + {}, [], None, None)) + + def test_no_plugin_lines_no_plugin_header(self): + (self.d / 'q.txt').write_text('set fps60 TOTK_FPS60 0\n') + self.assertNotIn('PLUGIN', patches.header_text(patches.load(self.d / 'q.txt', self.mods), 'MARK')) + + if __name__ == '__main__': unittest.main() diff --git a/tests/installer_test.py b/tests/installer_test.py index d1615661..f890a1c8 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -383,6 +383,60 @@ class LiftRunTest(unittest.TestCase): self.assertEqual(self.outputs(), [False, False]) self.assertIn('warn.exefs_skipped', self.ctx.reporter.out.getvalue()) + def plugin_setup(self, kind): + """A mod with an exlaunch plugin; the fake service run writes its status line of that kind.""" + plugin = self.dest / 'mods' / 'SQ' / 'exefs' / 'subsdk9' + plugin.write_bytes(b'NSO0') + self.ctx.layout = SimpleNamespace(**vars(self.ctx.layout), llvm='llvm', runtime='runtime', ninja='ninja', + deps=None, sysroot=None, runtime_lib_for=lambda cpu: None, + pgo=lambda version: None, expected=lambda version: 'expected.json') + saved = self.run_tool + + def run_tool(ctx, step, script, args, parse=None, cwd=None): + code, log = saved(ctx, step, script, args, parse, cwd) + args = [str(a) for a in args] + if Path(script).name == 'exefs_plugins.py': + Path(args[args.index('--patches-out') + 1]).write_text('patches\n') + status = Path(args[args.index('--status') + 1]) + status.parent.mkdir(parents=True, exist_ok=True) + h = 0x811C9DC5 # FNV-1a 32 of "SQ/exefs/subsdk9", as tools/exefs_plugins.py keys it + for b in b'SQ/exefs/subsdk9': + h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF + status.write_text(f'{h:08X}\t00000000\t{kind}\t-\tnote\n') + return (1 if kind == 'unsupported' else 0), log + return code, log + steps.run_tool = run_tool + return plugin + + def test_plugin_stage(self): + # the lift with the plugin, a totk.exe of it, the service run, then the lift with its hook sites + plugin = self.plugin_setup('plugin') + steps.Lift().run(self.ctx) + self.assertEqual([c[0] for c in self.calls], ['exefs_patches.py', 'gen.py', 'build_game.py', + 'exefs_plugins.py', 'gen.py', 'exefs_patches.py --folded']) + first, last = self.calls[1][1], self.calls[4][1] + self.assertEqual(first[first.index('--plugins') + 1], str(plugin)) + self.assertEqual(last[last.index('--plugins') + 1], str(plugin)) + self.assertEqual(last[last.index('--patches') + 1], str(self.ctx.temp / 'patches-plugins.txt')) + self.assertTrue(steps.plugins_status(self.ctx).is_file()) + + def test_refused_plugin_is_not_lifted(self): + # a refused plugin: its status line stays (the Mods tab says why), the game is lifted without it + self.plugin_setup('unsupported') + steps.Lift().run(self.ctx) + last = self.calls[4][1] + self.assertNotIn('--plugins', last) + self.assertTrue(steps.plugins_status(self.ctx).is_file()) + self.assertNotIn('warn.plugins_skipped', self.ctx.reporter.out.getvalue()) + + def test_failed_plugin_build_warns(self): + self.plugin_setup('plugin') + self.fail = {'build_game.py'} + steps.Lift().run(self.ctx) + self.assertNotIn('--plugins', self.calls[-2][1]) + self.assertFalse(steps.plugins_status(self.ctx).is_file()) + self.assertIn('warn.plugins_skipped', self.ctx.reporter.out.getvalue()) + class CompileLockedTest(unittest.TestCase): """Compile.run when the linker cannot write its totk.exe: another program holds the file (a game started from the diff --git a/tools/exefs_plugins.py b/tools/exefs_plugins.py index 8bdaac44..a0ef82ff 100644 --- a/tools/exefs_plugins.py +++ b/tools/exefs_plugins.py @@ -1,9 +1,24 @@ """Run a mod plugin's init in the plugin service run and check what it did: the build step before the generator emits a plugin's hook sites. After the design in #845 by itaskaev2; the init runs on this port's runtime (totk.exe ---plugin-init, runtime/plugin_init.c) instead of under Unicorn. +--plugin-init, runtime/plugin_init.c) instead of under Unicorn, and it runs again at every start of the game in the +same environment (plugin_init.c plugins_start), so the plugin keeps real state (its heap blocks, its GOT, its alias +addresses). The record says what the build expects that start-up to write. python tools/exefs_plugins.py --exe build/-x/totk.exe --out # two runs, their diff, the check python tools/exefs_plugins.py --check /run1 # the check of one record + python tools/exefs_plugins.py --exe --out --mods --patches + --patches-out --status # and the patch lines for the generator + +With --mods, --patches, --patches-out and --status, an accepted plugin becomes one `ordered` patch set mod_ +(recomp/patches.py: a plugin line, its expected writes, stubs, seeds, orig lines and its hook sites) appended to the +--patches file into --patches-out, and every plugin gets a line of --status (runtime/mods.c reads it as +build-cache/plugins.status), in the format of tools/exefs_patches.py --status: + + + +file hash: FNV-1a 32 of "/"; content hash: the same over its bytes; kind +`plugin` (note: the module and its hook count) or `unsupported` (note: the first reason); built-in sets: the port's +own sets with a site the plugin hooks too ("-" none), which the runtime switches off while the mod is on. The exe must be built with the plugin in its layout (recomp/gen.py --plugins). --out gets run1/ and run2/, one record each; run2 adds a large offset to every GetSystemTick (TOTK_PLUGIN_TICK_OFFSET), so the init picks other alias @@ -24,10 +39,14 @@ The check (check_record) accepts a record only when every rule holds; each broke record lists them), which the generator can emit as a patch site whose replacement is that branch; - a replace hook (Full Inventory Expansion's Sort Fixer): `b` from a function start of main to a function start of the plugin's lift; the Orig trampoline (the original relocated, b site+4) is listed when the pool holds one. -Not checked here yet: a site shared with a built-in patch set or another mod, and svcs in the code the callbacks -reach (both need the generator's view). + - the code the hooks reach (the stubs, exlaunch's hook impls, the callbacks, the replace and call targets, and what + they reach by direct branches and calls, reach()) has no svc but OutputDebugString and calls only the imports + REACH_IMPORTS. An indirect call (blr) is not followed: what it reaches cannot be known before the run. +A site shared with a built-in patch set turns that set off while the mod is on (the status line names it); a site +shared with another mod goes through the run time's pick (the Mods tab order, runtime/patches.c). """ import argparse +import hashlib import json import os import struct @@ -36,18 +55,30 @@ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'recomp')) -from nso import Nso # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from cheats import fnv # noqa: E402 +from exefs_patches import EXEFS_PLACES, builtin_sites # noqa: E402 +from nso import R_AARCH64_JUMP_SLOT, Nso # noqa: E402 ALLOWED_SVCS = {0x06: 'QueryMemory', 0x1E: 'GetSystemTick', 0x27: 'OutputDebugString', 0x29: 'GetInfo', 0x74: 'MapProcessMemory', 0x75: 'UnmapProcessMemory'} ALLOWED_INFO = {2, 3, 4, 5, 12, 13, 14, 15, 65001} -# Reviewed one by one: SNPrintf formats into the caller's buffer; GetDisplayVersion is answered by the service run; -# mersenne_twister_engine::_M_gen_rand (Full Inventory Expansion's alias picker) refills the state of the -# engine object it is given, which lives in the plugin's .bss; memcmp and strlen only read. malloc stays out: Quick -# Weapon Durability's init keeps a heap block's address in its .bss, and that block does not exist at play time. -ALLOWED_IMPORTS = {'_ZN2nn4util8SNPrintfEPcmPKcz', '_ZN2nn2oe17GetDisplayVersionEPNS0_14DisplayVersionE', 'memcmp', 'strlen', - '_ZNSt23mersenne_twister_engineImLm64ELm312ELm156ELm31ELm13043109905998158313ELm29ELm6148914691236517205' - 'ELm17ELm8202884508482404352ELm37ELm18444473444759240704ELm43ELm6364136223846793005EE11_M_gen_randEv'} +# Reviewed one by one: SNPrintf formats into the caller's buffer; GetDisplayVersion is answered by the runtime with +# the build's game version; mersenne_twister_engine::_M_gen_rand (Full Inventory Expansion's alias +# picker) refills the state of the engine object it is given, which lives in the plugin's .bss; memcmp and strlen only +# read; memcpy, memmove and memset write the buffers they are given; malloc and free use the game's heap (sdk's musl +# allocator), and since the init runs again at every start, a block it keeps exists at play time (Quick Weapon +# Durability's init keeps one in its .bss). +SAFE_LIBC = {'malloc', 'free', 'memcpy', 'memmove', 'memset', 'memcmp', 'strlen'} +MT_GEN_RAND = ('_ZNSt23mersenne_twister_engineImLm64ELm312ELm156ELm31ELm13043109905998158313ELm29ELm6148914691236517205' + 'ELm17ELm8202884508482404352ELm37ELm18444473444759240704ELm43ELm6364136223846793005EE11_M_gen_randEv') +SNPRINTF = '_ZN2nn4util8SNPrintfEPcmPKcz' +ALLOWED_IMPORTS = SAFE_LIBC | {SNPRINTF, '_ZN2nn2oe17GetDisplayVersionEPNS0_14DisplayVersionE', MT_GEN_RAND} +# The imports the code the hooks reach may call while the game runs (reach()). Each reviewed for what it touches: the +# libc ones above (the caller's buffers and the game's heap), SNPrintf (the caller's buffer). Nothing that waits, +# talks to a service or changes memory mappings. +REACH_IMPORTS = SAFE_LIBC | {SNPRINTF} +SVC_OUTPUT_DEBUG_STRING = 0x27 HOOK_IMPLS = ('exl_inline_hook_impl', 'exl_inline_float_hook_impl') STUR_X30 = 0xF81F03FE # stur x30, [sp, #-0x10] LDUR_X30 = 0xF85F03FE # ldur x30, [sp, #-0x10] @@ -241,6 +272,203 @@ def check_record(rec, impls): return hooks, reasons +# ---- the code the hooks reach +def plt_imports(nso, img, base, text_end): + """{absolute address of a PLT stub: the import its GOT slot binds} (adrp x16; ldr x17, [x16, #slot]; add x16, x16, + #slot; br x17), from the JUMP_SLOT relocations.""" + slots = {off: nso.syms[sym][0] for off, typ, sym, _ in nso.relocs if typ == R_AARCH64_JUMP_SLOT} + out = {} + for o in range(0, text_end - 16, 4): + w0, w1, _, w3 = struct.unpack_from('<4I', img, o) + if w0 & 0x9F00001F != 0x90000010 or w1 & 0xFFC003FF != 0xF9400211 or w3 != 0xD61F0220: + continue + imm = sext((w0 >> 29 & 3) | (w0 >> 5 & 0x7FFFF) << 2, 21) + slot = ((o & ~0xFFF) + (imm << 12)) + (w1 >> 10 & 0xFFF) * 8 + if slot in slots: + out[base + o] = slots[slot] + return out + + +def reach(img, base, text_end, roots, plt, stubs=()): + """What the code from roots (absolute addresses in the plugin's .text) reaches by falling through, direct branches + and direct calls: (svc immediates, {import: calls}, indirect calls, words outside the .text it branched to). A + stub's b to its trampoline counts as its return (recomp/patches.py `stub`).""" + svcs, imports, indirect, outside = set(), {}, 0, set() + seen, work = set(), list(roots) + stub_rets = {s + 12 for s in stubs} + while work: + a = work.pop() + while True: + if a in seen: + break + if a in plt: + imports[plt[a]] = imports.get(plt[a], 0) + 1 + break + if not base <= a < base + text_end: + outside.add(a) + break + seen.add(a) + w = struct.unpack_from('> 5 & 0xFFFF) + if w & 0x7C000000 == 0x14000000: # b, bl + t = branch_target(w, a) + if w >> 31: + work.append(t) + else: + a = t + continue + elif pc_relative(w, a) and pc_relative(w, a)[0][0] in ('b.cond', 'cbz', 'tbz'): + work.append(pc_relative(w, a)[1]) + elif w & 0xFFFFFC1F == 0xD63F0000: # blr + indirect += 1 + elif is_indirect(w): # br, ret: the path ends + break + elif w == 0 or w & 0xFFE0001F == 0xD4200000: # udf, brk: ends too + break + a += 4 + return svcs, imports, indirect, outside + + +def reach_check(rec, hooks, impls): + """Reasons the code the hooks reach breaks the rules (svcs, imports), and a summary line.""" + reasons, notes = [], [] + for p in (m for m in rec['modules'] if m['plugin'] and m['name'] in rec['images']): + img, base = rec['images'][p['name']], p['base'] + roots, stubs = set(), set() + for h in hooks: + for k in ('stub', 'callback', 'target'): + if h.get(k, '').startswith(p['name'] + '+'): + roots.add(base + int(h[k].split('+')[1], 16)) + if h['kind'] == 'inline' and h['stub'].startswith(p['name'] + '+'): + stubs.add(base + int(h['stub'].split('+')[1], 16)) + roots |= {a for a in impls if base <= a < base + p['text_end']} + nso = Nso(p['file']) if p.get('file') and Path(p['file']).is_file() else None + plt = plt_imports(nso, img, base, p['text_end']) if nso else {} + svcs, imports, indirect, outside = reach(img, base, p['text_end'], sorted(roots), plt, stubs) + for imm in sorted(svcs - {SVC_OUTPUT_DEBUG_STRING}): + reasons.append(f'the code its hooks reach has svc {imm:#x}') + for name in sorted(set(imports) - REACH_IMPORTS): + reasons.append(f'the code its hooks reach calls the import {name}') + notes.append(f'{p["name"]}: hooks reach {len(roots)} roots, imports {", ".join(sorted(imports)) or "none"}, ' + f'{indirect} indirect calls not followed' + (f', {len(outside)} branches out of its .text' + if outside else '')) + return reasons, notes + + +# ---- patch lines and status for the generator and the runtime +def mod_of(nso_path, mods): + """(mod folder name, path of the NSO in the mod) of a plugin file under the mods folder, else None.""" + try: + rel = Path(nso_path).resolve().relative_to(Path(mods).resolve()) + except ValueError: + return None + return rel.parts[0], '/'.join(rel.parts[1:]) + + +def hooks_of(hooks, module): + """The hooks whose stub or target lies in the plugin module: its own, when one record holds several plugins.""" + return [h for h in hooks if (h.get('stub') or h.get('target') or '').split('+')[0] == module] + + +def plugin_lines(rec, hooks, name, record_dir, module): + """The patch lines of the accepted plugin module (recomp/patches.py) for set name: the plugin line, the expected + writes (the words at its hooks' sites), the stubs, the seeds, the orig lines and the hook sites.""" + lines = [] + mods = {m['name']: m for m in rec['modules']} + p = mods[module] + img = rec['images'][p['name']] + text_sha = hashlib.sha256(img[:p['text_end']]).hexdigest() + lines.append(f'plugin {name} {p["name"]} {text_sha} {(Path(record_dir) / p["image"]).resolve().as_posix()}') + own = {int(h['site'].split('+')[1], 16) for h in hooks} + for w in rec['writes']: + if w['module'] == 'main' and w['offset'] in own: + lines.append(f'expect {name} {w["module"]} {w["offset"]:x} {w["old"]:08x} {w["new"]:08x}') + seeds, stubs = {}, {} + + def at(ref): + mod, off = ref.split('+') + return mod, int(off, 16) + + for h in hooks: + site = int(h['site'].split('+')[1], 16) + if h['kind'] == 'inline': + sm, so = at(h['stub']) + stubs.setdefault(sm, set()).add(so) + cm, co = at(h['callback']) + seeds.setdefault(cm, set()).add(co) + # the hook impl the stub calls: its bl, as check_record read it + stub_at = mods[sm]['base'] + so + impl = branch_target(struct.unpack_from('