From b94e8127af18b132a77d77e375e653ba2ddb0e56 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 08:05:18 +0300 Subject: [PATCH 01/21] Mod plugins get their own modules after sdk in the layout and are lifted with the game (#845) recomp/layout.py appends a mod's code modules (exlaunch plugins such as exefs/subsdk9) after sdk as plugin0..N in mod load order, so the game modules keep their bases; layout.json names each plugin's NSO and its SHA-256, and without plugins layout.json and layout.h stay byte-identical. gen.py --plugins lifts them with DT_INIT and the init_array entries as extra seeds, and Module.eh_frame_functions accepts a MOD0 with an empty .eh_frame_hdr range. build_game.py takes the plugin rows from layout.json into modules.c instead of asking the ExeFS hash table for them. No-mod generated C is byte-identical to dev; with MSFix in the layout main, subsdk0 and sdk are too. After the design in #845 by itaskaev2. Co-Authored-By: Claude Opus 5.5 --- recomp/gen.py | 25 +++++++++++++++-------- recomp/layout.py | 48 +++++++++++++++++++++++++++++++++------------ recomp/module.py | 25 +++++++++++++++++++---- recomp/nso.py | 2 +- tools/build_game.py | 22 ++++++++++++++++----- 5 files changed, 91 insertions(+), 31 deletions(-) diff --git a/recomp/gen.py b/recomp/gen.py index a0ddcd56..1a944eda 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -3,7 +3,11 @@ Usage: gen.py --exefs build/extracted/v121/exefs --out generated/v121 [--modules main,subsdk0,sdk] [--jobs N] [--limit N] [--chunk-fns N] [--chunk-bytes B] [--big-fn B] [--patches FILE] [--v-locals K [--v-trust]] - [--no-cache] [--fn-enter-all] + [--no-cache] [--fn-enter-all] [--plugins NSO[,NSO...]] + +--plugins places a mod's code modules (exlaunch plugins) after sdk as plugin0..N (recomp/layout.py) and lifts them with +the game modules; their DT_INIT and init_array entries are seeds too (Module.init_functions). Without it the output is +what it was before plugins existed. Output, under --out: layout.h, layout.json module bases (recomp/layout.py) @@ -63,7 +67,7 @@ import patches # noqa: E402 import regs # noqa: E402 import vlocals # noqa: E402 from check_content import GENERATED_MARKER # noqa: E402 -from layout import layout, write as write_layout # noqa: E402 +from layout import layout, plugin_files, write as write_layout # noqa: E402 from lifter import hx # noqa: E402 from module import Module # noqa: E402 @@ -853,7 +857,10 @@ def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--exefs', type=Path, required=True) ap.add_argument('--out', type=Path, required=True) - ap.add_argument('--modules', default='main,subsdk0,sdk') + ap.add_argument('--modules', default='main,subsdk0,sdk', help='modules to lift (default main,subsdk0,sdk and every ' + '--plugins module)') + ap.add_argument('--plugins', default='', help='comma-separated plugin NSOs (exefs/subsdk* of a mod), in mod load ' + 'order: lifted as plugin0..N after sdk') ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)') ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module') ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most') @@ -887,11 +894,13 @@ def main(): gen_key = generator_key(opts) t0 = time.perf_counter() - mods = layout(args.exefs) - write_layout(mods, args.out) + plugins = [Path(p) for p in args.plugins.split(',') if p] + mods = layout(args.exefs, plugins) + files = plugin_files(plugins) + write_layout(mods, args.out, files) table = patches.load(args.patches, mods) if args.patches else patches.Table() write_if_changed(args.out / 'patches.h', patches.header_text(table, GENERATED_MARKER)) - want = args.modules.split(',') + want = args.modules.split(',') + list(files) unknown = set(want) - {n for n, _, _ in mods} if unknown: raise SystemExit(f'not in {args.exefs}: {", ".join(sorted(unknown))}') @@ -901,8 +910,8 @@ def main(): if name not in want: continue t = time.perf_counter() - path = args.exefs / name - mod = Module(name, path, base, 'c') + path = Path(files[name][0]) if name in files else args.exefs / name + mod = Module(name, path, base, 'c', plugin=name in files) patches.check(table, mod, name) starts, dstats = discover(mod) est = estimate_sizes(mod, starts) diff --git a/recomp/layout.py b/recomp/layout.py index 971d6004..8f2e9a0c 100644 --- a/recomp/layout.py +++ b/recomp/layout.py @@ -6,9 +6,12 @@ calls layout() and writes the result into generated//layout.h and layou build read. Modules are placed in Horizon's load order (rtld is dropped; the host links the modules): -main, subsdk0..9, sdk. Each base is aligned to 2 MiB. Everything must stay below 2^39, the limit of the addresses -IPC buffers can encode. +main, subsdk0..9, sdk. A mod's code modules (exlaunch plugins such as an exefs/subsdk9) follow sdk as plugin0, plugin1, +... in mod load order, so the game modules keep their bases with or without them (after the plugin design in #845 by +itaskaev2). Each base is aligned to 2 MiB. Everything must stay below 2^39, the limit of the addresses IPC buffers can +encode. """ +import hashlib import json from pathlib import Path @@ -24,14 +27,18 @@ def align_up(v, a=ALIGN): return (v + a - 1) & ~(a - 1) -def layout(exefs): - """[(name, base, size)] for the modules present in the ExeFS directory, in load order.""" +def plugin_name(i): + return f'plugin{i}' + + +def layout(exefs, plugins=()): + """[(name, base, size)] for the modules present in the ExeFS directory, then the plugin NSOs (paths, in mod load + order) as plugin0..N, in load order.""" exefs = Path(exefs) + paths = [(name, exefs / name) for name in ORDER if (exefs / name).exists()] + paths += [(plugin_name(i), Path(p)) for i, p in enumerate(plugins)] out, base = [], FIRST_BASE - for name in ORDER: - path = exefs / name - if not path.exists(): - continue + for name, path in paths: size = Nso(str(path)).size out.append((name, base, size)) base = align_up(base + size) @@ -40,12 +47,27 @@ def layout(exefs): return out -def write(mods, outdir): - """layout.json for Python tools and layout.h for the runtime.""" +def plugin_files(plugins): + """{plugin: (absolute path, SHA-256 of the NSO)} for layout.json: the build and the runtime read a plugin from its + mod folder, not from the ExeFS directory, and check that it is the file the code was lifted from.""" + return {plugin_name(i): (Path(p).resolve().as_posix(), hashlib.sha256(Path(p).read_bytes()).hexdigest()) + for i, p in enumerate(plugins)} + + +def write(mods, outdir, files=None): + """layout.json for Python tools and layout.h for the runtime. files: plugin_files() for the plugin entries, which + carry them as "file" and "sha256"; without plugins both files are what they were before plugins existed.""" + files = files or {} outdir = Path(outdir) outdir.mkdir(parents=True, exist_ok=True) - (outdir / 'layout.json').write_text(json.dumps( - [{'name': n, 'base': f'{b:#x}', 'size': f'{s:#x}'} for n, b, s in mods], indent=1) + '\n') + + def entry(n, b, s): + e = {'name': n, 'base': f'{b:#x}', 'size': f'{s:#x}'} + if n in files: + e['file'], e['sha256'] = files[n] + return e + + (outdir / 'layout.json').write_text(json.dumps([entry(n, b, s) for n, b, s in mods], indent=1) + '\n') lines = ['// Module layout chosen by recomp/layout.py. Generated; do not edit.', '#pragma once', ''] for n, b, s in mods: lines.append(f'#define MOD_{n.upper()}_BASE {b:#x}ull') @@ -56,6 +78,6 @@ def write(mods, outdir): def read(outdir): - """[(name, base, size)] from a layout.json written by write().""" + """[(name, base, size)] from a layout.json written by write() (a plugin's file and hash left out).""" data = json.loads((Path(outdir) / 'layout.json').read_text()) return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data] diff --git a/recomp/module.py b/recomp/module.py index c3e4f355..9d4e43e5 100644 --- a/recomp/module.py +++ b/recomp/module.py @@ -10,7 +10,7 @@ from collections import OrderedDict import numpy as np from lifter import Insn, Lifter, Unsupported, hx -from nso import Nso, R_AARCH64_RELATIVE +from nso import DT_INIT, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, Nso, R_AARCH64_RELATIVE LIFT_CACHE = 1 << 16 # lifted instructions kept per module; analyses revisit nearby addresses only @@ -33,10 +33,11 @@ def _read_encoded(img, off, enc, pc_base): class Module: - def __init__(self, name, path, base, simd='c'): + def __init__(self, name, path, base, simd='c', plugin=False): self.name = name self.nso = Nso(str(path)) self.base = base + self.plugin = plugin # a mod's code module (recomp/layout.py plugin): its init functions are seeds too n = self.nso self.image = bytes(n.image) self.lo = base + n.text[0] @@ -83,7 +84,7 @@ class Module: def eh_frame_functions(self): """Function starts from .eh_frame_hdr's binary search table (absolute addresses).""" img, hdr = self.image, self.nso.eh_start - if not hdr or hdr >= len(img): + if not hdr or hdr >= len(img) or self.nso.eh_end <= hdr: # MOD0 with an empty .eh_frame_hdr range (exlaunch) return [] _, ptr_enc, cnt_enc, tbl_enc = img[hdr:hdr + 4] off = hdr + 4 @@ -100,9 +101,25 @@ class Module: starts.append(self.base + loc) return starts + def init_functions(self): + """DT_INIT and the DT_INIT_ARRAY entries (absolute addresses). rtld calls DT_INIT only; the module's DT_INIT + runs the array through blr (SDK and exlaunch modules alike), so the entries are seeds too. An entry holds its + target as the addend of its R_AARCH64_RELATIVE relocation, or as the word itself when no relocation names it.""" + dyn, out = self.nso.dyn, [] + if dyn.get(DT_INIT): + out.append(self.base + dyn[DT_INIT]) + if dyn.get(DT_INIT_ARRAY): + addends = {off: add for off, typ, _, add in self.nso.relocs if typ == R_AARCH64_RELATIVE} + for off in range(dyn[DT_INIT_ARRAY], dyn[DT_INIT_ARRAY] + dyn.get(DT_INIT_ARRAYSZ, 0), 8): + out.append(self.base + addends.get(off, self.nso.u64(off))) + return out + def seeds(self): - """Known function starts: .eh_frame_hdr, exported FUNC symbols, R_AARCH64_RELATIVE targets in .text.""" + """Known function starts: .eh_frame_hdr, exported FUNC symbols, R_AARCH64_RELATIVE targets in .text; for a + plugin also its init functions (exlaunch's DT_INIT starts inside a function discovery would merge it into).""" starts = set(self.eh_frame_functions()) + if self.plugin: + starts.update(self.init_functions()) for _, info, shndx, value, _ in self.nso.syms: if shndx and (info & 0xF) == 2: starts.add(self.base + value) diff --git a/recomp/nso.py b/recomp/nso.py index 03beab28..dfff6122 100644 --- a/recomp/nso.py +++ b/recomp/nso.py @@ -7,7 +7,7 @@ from collections import Counter import lz4.block DT_NULL, DT_NEEDED, DT_PLTRELSZ, DT_HASH, DT_STRTAB, DT_SYMTAB, DT_RELA, DT_RELASZ = 0, 1, 2, 4, 5, 6, 7, 8 -DT_STRSZ, DT_JMPREL, DT_RELACOUNT = 10, 23, 0x6FFFFFF9 +DT_STRSZ, DT_INIT, DT_JMPREL, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, DT_RELACOUNT = 10, 12, 23, 25, 27, 0x6FFFFFF9 DT_RELRSZ, DT_RELR, DT_GNU_HASH = 0x23, 0x24, 0x6FFFFEF5 R_AARCH64_ABS64, R_AARCH64_GLOB_DAT, R_AARCH64_JUMP_SLOT, R_AARCH64_RELATIVE = 257, 1025, 1026, 1027 diff --git a/tools/build_game.py b/tools/build_game.py index 274bd402..84e93682 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -242,6 +242,13 @@ def read_layout(gen): return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data] +def read_plugin_files(gen): + """{plugin: (NSO path, SHA-256)} from layout.json: the mod code modules recomp/layout.py placed after sdk. They + are no ExeFS files, so the runtime reads each from its path and checks it against this hash.""" + data = json.loads((gen / 'layout.json').read_text()) + return {m['name']: (m['file'], m['sha256']) for m in data if 'file' in m} + + def chunk_files(gen, mod): """The C files of one module: the chunks listed in /manifest.json, then _table.c.""" d = gen / mod @@ -366,9 +373,11 @@ def runtime_object(out, src): return out / 'obj' / 'runtime' / rel.with_suffix('.o') -def module_glue(layout, linked, hashes, paths, version): +def module_glue(layout, linked, hashes, paths, version, plugins=None): """C for build//modules.c: the layout table, the linked modules' function tables (dispatch.h), the - expected ExeFS hashes, the game version and the default game paths (host.h).""" + expected ExeFS hashes, the game version and the default game paths (host.h). plugins: read_plugin_files(); a plugin + row of the layout table also names its file and hash.""" + plugins = plugins or {} out = [f'// Module glue written by tools/build_game.py; do not edit. {GENERATED_MARKER}', '#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', ''] for name, base, size in layout: @@ -376,7 +385,9 @@ def module_glue(layout, linked, hashes, paths, version): out.append(f'_Static_assert(MOD_{u}_BASE == {base:#x}ull && MOD_{u}_SIZE == {size:#x}ull, ' f'"layout.h and layout.json disagree on {name}");') out += ['', 'const RtLayoutEntry rt_layout[] = {'] - out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE}},' for n, _, _ in layout] + out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE' + + (f', {c_string(plugins[n][0])}, {c_string(plugins[n][1])}' if n in plugins else '') + '},' + for n, _, _ in layout] out += ['};', f'const int rt_layout_count = {len(layout)};', ''] for name in linked: out += [f'extern const uint64_t {name}_fn_addrs[];', f'extern const GuestFn {name}_fn_ptrs[];', @@ -923,10 +934,11 @@ def main(): cpus = os.cpu_count() or 1 budget = min(MEM_BUDGET, int(total_ram() * 0.8)) jobs = args.jobs or max(1, min(cpus, budget // JOB_MEM)) - missing = [n for n in names if n not in hashes] + plugins = read_plugin_files(gen) + missing = [n for n in names if n not in hashes and n not in plugins] if missing: sys.exit(f'{expected}: no exefs_sha256 for {missing}') - write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version)) + write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version, plugins)) if has_romfs: count, size, dt = romfs_index.write(paths['romfs'], out) print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s') From b488d9e41dee50bfe63cff59e1fe9839da2d6a75 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 08:05:19 +0300 Subject: [PATCH 02/21] totk.exe --plugin-init runs a mod plugin's init on our kernel and records what it did (#845) The plugin service run loads the modules as boot does, runs sdk's DT_INIT and then each plugin's from nn::init::Start's module-init callback (rtld initialises the last loaded module first), writes the record and exits before any game constructor, graphics or audio. The record holds every word the init wrote into a game module, the plugin's image before and after its init, the process-memory maps, and the svcs and imports it called. exlaunch's init needs a Switch-like address space: QueryMemory shows the game's own rtld image below main and the host's ranges as Inaccessible (it takes Code RX runs for modules), and a 64 GiB free pool for its random alias picks. New svcs: GetSystemTick, GetInfo 65001 (current process handle), and Map/UnmapProcessMemory, which only the service run serves: an alias is a copy whose changed words go back into the source when it is unmapped. Any other svc, such as the CreateSession fallback, ends the run with a reason. nn::oe::GetDisplayVersion is answered with the build's game version. The game calls none of these svcs; the SVC trace test is still one mask test per SVC. After the design in #845 by itaskaev2, with the init run natively instead of under Unicorn. Co-Authored-By: Claude Opus 5.5 --- docs/settings.md | 1 + runtime/boot.c | 22 ++- runtime/cfg.c | 2 +- runtime/dispatch.h | 6 +- runtime/host.h | 23 ++- runtime/kernel.h | 6 + runtime/loader.c | 59 +++++- runtime/main.c | 7 +- runtime/memory.c | 4 + runtime/plugin_init.c | 429 ++++++++++++++++++++++++++++++++++++++++++ runtime/svc.c | 22 ++- 11 files changed, 559 insertions(+), 22 deletions(-) create mode 100644 runtime/plugin_init.c diff --git a/docs/settings.md b/docs/settings.md index b887a1af..6f23cdda 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -24,6 +24,7 @@ Many rows below are developer tools that only act together with another setting | `core_diag` | `1` records why ready guest threads wait for a virtual core and for how long, with a histogram every 60 presents. | Turns on host-call and SVC depth tracking and writes many log lines (one per present over 25 ms); frames get slower. | Caution | | `barrier_merge` (default `0`) | Experimental. `1` keeps a render pass open through the game's Barrier calls that ask only for fragment ordering and texture invalidation, when no draw of the pass can write a storage buffer or image; the next draw that samples a written target gets a by-region barrier in the pass (the feedback layout). In scenes with about a thousand such barriers a frame this cuts the passes from about 1200 to 130 and the GPU time from 24.5 to 7.8 ms. | The game's barrier orders every pixel, the in-pass by-region barrier only the pixel a draw writes: a driver that runs it per screen tile can show flicker or squares at the edges of water, decals or particles. | Caution | | `menu_cursor_log` | `1` logs what the menu cursor code reads at every screen change. | Only adds log lines. | Safe | +| `plugin_tick_offset` (default `0`) | Added to every GetSystemTick in the plugin service run (`totk.exe --plugin-init DIR`), so `tools/exefs_plugins.py` can run a plugin's init twice with other ticks and compare the records. | Has no effect outside `--plugin-init`; the game never asks for GetSystemTick. | Safe | | `perf_slow_ms` (default `50`) | Frames at least this many milliseconds long get a `perf: slow frame` log line with the time blocked per cause. | Changes how many frames are logged; a small value fills the log. | Safe | | `replay_check` | `1` reads every texture read-back a second time after a full drain of the replay queue and compares the two. | Stalls the replay queue at every read-back, so frames get much slower; mismatches are only counted and logged. | Caution | | `resolve_check` | `1` compares every cached pool lookup with the locked lookup and stops on a difference. | Slows every pool lookup; the game stops if the two ever differ. | Caution | diff --git a/runtime/boot.c b/runtime/boot.c index bd099b8c..14028825 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -77,6 +77,7 @@ static void cb_loader_ready(Cpu *c) { // afterwards (MacNativeGames loader.c:458-462). static void cb_module_inits(Cpu *c) { if (plat_cas32(&inits_done, 0, 0)) rt_fatal("boot: module constructors requested twice"); + if (rt_config()->plugin_init) plugin_init_run(c, module_named("sdk")); // the plugin service run ends there for (unsigned k = 0; k < sizeof inits_121 / sizeof *inits_121; k++) { const RtLoaded *m = module_named(expected_inits[k].name); uint64_t entries = m->init_array_size / 8; @@ -118,8 +119,8 @@ void rt_boot_event(const char *what, Cpu *c, const char *detail) { rt_fatal("boot: checkpoint %s reached from %s, expected main+%#llx", detail, at, (unsigned long long)CHECKPOINT_RET); if (!plat_cas32(&inits_done, 0, 0)) rt_fatal("boot: checkpoint %s before DT_INIT", detail); - for (int i = 0; i < ldr_count(); i++) - if (ctor_state(ldr_module(i)) != CTOR_DONE) + for (int i = 0; i < ldr_count(); i++) // a plugin's init does not run at play time (plugin_init.c) + if (!ldr_module(i)->plugin && ctor_state(ldr_module(i)) != CTOR_DONE) rt_fatal("boot: checkpoint %s: %s constructor flag reads %u", detail, ldr_module(i)->name, ctor_state(ldr_module(i))); plat_xchg32(&checkpoint_seen, 1); @@ -341,6 +342,23 @@ static void prewarm_main(void *arg) { return; } +// --plugin-init DIR: the modules load and nn::init::Start runs as at boot, and its module-init callback runs sdk's +// DT_INIT and each plugin's instead of the game's constructors, writes the record and exits (plugin_init.c). No game +// constructor, nnMain, shader prewarm, graphics or audio runs. +void rt_boot_plugin_init(void) { + check_npdm(); + plugin_init_setup(); + ldr_load(rt_config()->exefs); + rt_patches_init(); // the generated code reads the patch page + cb_addr[0] = hostfn_register("", cb_loader_ready); + cb_addr[1] = hostfn_register("", cb_module_inits); + cb_addr[2] = hostfn_register("", cb_module_finis); + hostfn_seal(); + rt_guest_start_ns = kern_now_ns(); + kern_run_on_guest_main(guest_main); + rt_fatal("boot: the guest main thread ended"); +} + void rt_boot(void) { check_npdm(); if (!rt_config()->shot) vk_pipeline_async_enable(); // a --shot image must not depend on build timing diff --git a/runtime/cfg.c b/runtime/cfg.c index 38d58207..5dd06739 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -56,7 +56,7 @@ static const char *const known[] = { "MEM_RESERVE_MIB", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_NO_CODE", "MODS_ORDER", "MODS_REBUILD", "MODS_WITHOUT_CODE", "MOTION", "MOUSE_AIM_SCALE", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_ASPECT", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", - "ONE_PER_CORE", "OPERATION_MODE", "OPTREF_GUARD", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCORE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POINTER_IMAGE", + "ONE_PER_CORE", "OPERATION_MODE", "OPTREF_GUARD", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCORE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PLUGIN_TICK_OFFSET", "POINTER_IMAGE", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_BENCH", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_ASYNC", "REC_END", "REC_SPIN_US", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHUNKS", "REPLAY_CHUNK_DRAWS", "REPLAY_SKEW_US", "REPLAY_STATE_CHECK", "REPLAY_WORKERS", diff --git a/runtime/dispatch.h b/runtime/dispatch.h index a46e66e3..81165b17 100644 --- a/runtime/dispatch.h +++ b/runtime/dispatch.h @@ -19,8 +19,10 @@ typedef struct RtModule { } RtModule; // The module glue that tools/build_game.py writes for each build (build//modules.c), from layout.json and -// layout.h: every module of the layout, and the modules linked into this executable. -typedef struct RtLayoutEntry { const char *name; uint64_t base, size; } RtLayoutEntry; +// layout.h: every module of the layout, and the modules linked into this executable. A mod's code module +// (recomp/layout.py plugin, after sdk) also names its NSO file and that file's SHA-256; a game module has NULL there +// and is read from the ExeFS directory. +typedef struct RtLayoutEntry { const char *name; uint64_t base, size; const char *file, *sha256; } RtLayoutEntry; extern const RtLayoutEntry rt_layout[]; extern const int rt_layout_count; int rt_linked_modules(RtModule *out, int max); diff --git a/runtime/host.h b/runtime/host.h index 152e1b82..3d84371c 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -25,7 +25,9 @@ void rt_log_raw(const char *p, size_t n); const char *rt_log_path(void); // the log file, NULL when this run writes none enum { RT_TRACE_SVC = 1, RT_TRACE_HOSTCALL = 2, RT_TRACE_IPC = 4, RT_TRACE_FS = 8, RT_TRACE_GFX = 16, RT_TRACE_PERF = 32, RT_TRACE_SCHED = 64, RT_TRACE_AUDIO = 128, RT_TRACE_PRESENT = 256, - RT_TRACE_HOSTCOUNT = 512 }; // no TOTK_TRACE name of its own: hostfn_count switches it (host.h) + RT_TRACE_HOSTCOUNT = 512, // no TOTK_TRACE name of its own: hostfn_count switches it (host.h) + RT_TRACE_PLUGIN = 1024 }; // no name either: the plugin service run (plugin_init.c) sees every SVC through + // rt_plugin_svc while it runs a plugin's init extern unsigned rt_trace; // from TOTK_TRACE=svc,host,ipc,fs,gfx,perf,sched,audio,present (or all) // prof_win.c: TOTK_PROFILE= sampling profiler; TOTK_TRACE=perf frame timing (waits of the render thread) // PERF_TEX: texture uploads on the render thread (deswizzle, decode, staging), PERF_ASTC included @@ -150,6 +152,7 @@ uint32_t mem_set_attribute(uint64_t a, uint64_t n, uint32_t mask, uint32_t at); uint32_t mem_map(uint64_t dst, uint64_t src, uint64_t n); // 0x04 uint32_t mem_unmap(uint64_t dst, uint64_t src, uint64_t n); // 0x05 uint32_t mem_query(uint64_t a, RtMemoryInfo *out); // 0x06 +extern void (*mem_query_hook)(uint64_t a, RtMemoryInfo *r); // plugin service run only: adjusts mem_query's answer uint32_t mem_map_physical(uint64_t a, uint64_t n); // 0x2C uint32_t mem_unmap_physical(uint64_t a, uint64_t n); // 0x2D int mem_info(uint32_t id, uint64_t *out); // GetInfo memory ids 2-7, 12-17, 21, 22, 28; 0 if not a memory id @@ -199,13 +202,20 @@ int rt_oom_forced_left(int site); // forced failures st typedef struct { const char *name; uint64_t base, size, text_end, data_off; // text_end, data_off: offsets from base uint64_t init, init_array, init_array_size, ctor_flag; // absolute guest addresses; init_array_size in bytes + int plugin; // a mod's code module (layout plugin), not a game module } RtLoaded; void ldr_load(const char *exefs_dir); // load, relocate (overrides applied), seal; once +extern GuestFn ldr_plugin_import; // set before ldr_load (plugin_init.c): every JUMP_SLOT + // import of a plugin binds to this observer int ldr_count(void); const RtLoaded *ldr_module(int i); // order: main, subsdk0, sdk uint64_t ldr_symbol(const char *name); // exported definition, 0 if none uint64_t ldr_plt_stub(const char *module, const char *name); // PLT stub whose GOT slot holds name, 0 if none const char *ldr_describe(uint64_t a, char *buf, int n); // "main+0x1234 (symbol+0x10)", "host ", or the address +uint64_t ldr_nso_size(const char *dir, const char *file); // image size of an ExeFS NSO (SHA-256 checked) +void ldr_map_image(const char *dir, const char *file, uint64_t base, uint64_t *text_end, uint64_t *data_off); + // its segments at base (reserved by the caller), not + // linked: the plugin service run's rtld (plugin_init.c) uint8_t *ldr_read_verified(const char *dir, const char *file, size_t *len); // whole file, SHA-256 checked against // modules.c (fatal on mismatch); free() it @@ -358,13 +368,22 @@ int rt_knob(const char *name, const char *def); void frame_step_present(uint64_t display_ns, int interval); // nvnQueuePresentTexture: the frame's display time and // present interval (60 Hz ticks) // ---- boot (boot.c, main.c) -typedef struct { const char *exefs, *romfs, *profile; int boot; double seconds; const char *shot, *gpu; } RtConfig; +typedef struct { + const char *exefs, *romfs, *profile; + int boot; + double seconds; + const char *shot, *gpu; + const char *plugin_init; // --plugin-init DIR: the plugin service run writes its record there (plugin_init.c) +} RtConfig; // romfs: directory with romfs.bin and index.json; // frames: --frames N (0 = no limit); shot: --shot // PATH; gpu: TOTK_GPU const RtConfig *rt_config(void); void rt_boot_event(const char *what, Cpu *c, const char *detail); // checkpoint log; "nvn" ends --boot void rt_boot(void); // boot.c +void rt_boot_plugin_init(void); // boot.c: --plugin-init, the plugin service run +void plugin_init_setup(void); // plugin_init.c: before ldr_load +void plugin_init_run(Cpu *c, const RtLoaded *sdk); // plugin_init.c: from the module-init callback; exits // Modes of totk.exe. main.c has run mem_init and parsed the paths; each mode calls ldr_load itself if it needs the // modules, registers its host functions, then seals (hostfn_seal, or implicitly at the first rt_call). Return the // process exit code. diff --git a/runtime/kernel.h b/runtime/kernel.h index d719b301..d232f68d 100644 --- a/runtime/kernel.h +++ b/runtime/kernel.h @@ -38,3 +38,9 @@ extern volatile int kern_test_cv_old_order; // WaitProcessWideKeyAtomi // the key (MNG's pre-fix order, the negative control) extern void (*volatile kern_test_cv_hook)(void); // called right after the lock release, holding the kernel lock extern void (*volatile svc_test_exit_hook)(unsigned code); // svc 0x07 calls it instead of ExitProcess (probes) + +// ---- mod plugin service run (plugin_init.c sets these; the game calls none of the SVCs they serve) +extern uint64_t rt_svc_tick_offset; // added to GetSystemTick (0x1E): the service run's second run picks other ticks +extern void (*rt_plugin_svc)(Cpu *c, uint32_t imm, uint64_t pc); // with RT_TRACE_PLUGIN: sees each SVC first +extern uint32_t (*rt_process_memory)(int unmap, uint64_t dst, uint32_t process, uint64_t src, uint64_t size); + // Map/UnmapProcessMemory (0x74/0x75); NULL outside the service run: fatal diff --git a/runtime/loader.c b/runtime/loader.c index 970c5fb7..4e7f833b 100644 --- a/runtime/loader.c +++ b/runtime/loader.c @@ -52,6 +52,7 @@ typedef struct { } Mod; static Mod mods[MOD_COUNT]; +GuestFn ldr_plugin_import; static int nmods; static uint64_t identity_refs, stubs; static int *ovr_used, *obs_used; // per linked table entry, plus probe overrides counted separately @@ -74,6 +75,17 @@ static void hex32(const uint8_t d[32], char out[65]) { for (int i = 0; i < 32; i++) snprintf(out + 2 * i, 3, "%02x", d[i]); } +static uint8_t *read_path_verified(const char *path, const char *want, size_t *len, const char *hint) { + uint8_t *p = read_file(path, len); + if (!p) rt_fatal("loader: cannot read %s (%s)", path, hint); + uint8_t d[32]; + char hex[65]; + rt_sha256(p, *len, d); + hex32(d, hex); + if (strcmp(hex, want)) rt_fatal("loader: %s has SHA-256 %s, expected %s", path, hex, want); + return p; +} + uint8_t *ldr_read_verified(const char *dir, const char *file, size_t *len) { const char *want = NULL; for (int i = 0; i < rt_exefs_sha256_count; i++) @@ -81,14 +93,7 @@ uint8_t *ldr_read_verified(const char *dir, const char *file, size_t *len) { if (!want) rt_fatal("loader: no expected SHA-256 for %s in modules.c", file); char path[1024]; snprintf(path, sizeof path, "%s/%s", dir, file); - uint8_t *p = read_file(path, len); - if (!p) rt_fatal("loader: cannot read %s (extract the game first: tools/extract.py)", path); - uint8_t d[32]; - char hex[65]; - rt_sha256(p, *len, d); - hex32(d, hex); - if (strcmp(hex, want)) rt_fatal("loader: %s has SHA-256 %s, expected %s", path, hex, want); - return p; + return read_path_verified(path, want, len, "extract the game first: tools/extract.py"); } // ---- NSO @@ -141,6 +146,32 @@ static void nso_segment(const uint8_t *raw, const Seg *s, uint8_t *dst, const ch } } +// The plugin service run's stand-in for rtld (plugin_init.c): an ExeFS NSO's image, read and checked like a module but +// neither relocated, linked nor sealed. Nothing runs it; exlaunch only reads it while it walks the address space. +uint64_t ldr_nso_size(const char *dir, const char *file) { + size_t len; + uint8_t *raw = ldr_read_verified(dir, file, &len); + Seg seg[3]; + uint64_t size = nso_segments(raw, len, file, seg); + free(raw); + return size; +} + +void ldr_map_image(const char *dir, const char *file, uint64_t base, uint64_t *text_end, uint64_t *data_off) { + size_t len; + uint8_t *raw = ldr_read_verified(dir, file, &len); + Seg seg[3]; + uint64_t size = nso_segments(raw, len, file, seg); + if (!plat_mem_commit(base, size)) rt_fatal("loader: cannot commit %s at %#llx", file, (unsigned long long)base); + for (int i = 0; i < 3; i++) { + if ((uint64_t)seg[i].mem_off + seg[i].size > size) rt_fatal("loader: %s segment %d exceeds the image", file, i); + nso_segment(raw, &seg[i], (uint8_t *)(uintptr_t)(base + seg[i].mem_off), file, i); + } + free(raw); + *text_end = seg[0].mem_off + seg[0].size; + *data_off = seg[2].mem_off; +} + // ---- symbols static const char *sym_name(const Mod *m, uint32_t i) { return m->strtab + m->syms[i].name; } static int sym_defined(const Mod *m, uint32_t i) { return m->syms[i].shndx != 0; } @@ -273,9 +304,12 @@ static uint64_t decode_ctor_flag(const Mod *m) { return 0; } +// A plugin (lay->file set) comes from its mod folder, checked against the hash the build took when it lifted it. static void load_module(Mod *m, const char *dir, const RtLayoutEntry *lay) { size_t len; - uint8_t *raw = ldr_read_verified(dir, lay->name, &len); + uint8_t *raw = lay->file ? read_path_verified(lay->file, lay->sha256, &len, "the mod's code module the build " + "lifted; rebuild after changing mods") + : ldr_read_verified(dir, lay->name, &len); Seg seg[3]; uint64_t size = nso_segments(raw, len, lay->name, seg); if (size != lay->size) @@ -291,6 +325,7 @@ static void load_module(Mod *m, const char *dir, const RtLayoutEntry *lay) { } free(raw); m->pub.name = lay->name; + m->pub.plugin = lay->file != NULL; m->pub.base = lay->base; m->pub.size = size; m->pub.text_end = seg[0].mem_off + seg[0].size; @@ -328,7 +363,11 @@ static Res resolve(Mod *m, uint32_t si, int jump) { uint64_t guest = sym_defined(m, si) ? m->pub.base + m->syms[si].value : find_symbol(name, NULL); GuestFn f; Res r; - if (jump && (f = hostfn_observer(name))) { + if (jump && m->pub.plugin && ldr_plugin_import) { // the plugin service run counts the plugin's calls by name + uint64_t to = (f = hostfn_override(name)) ? hostfn_register(name, f) : guest; + if (!to) to = hostfn_register(name, unresolved_stub); + r = (Res){hostfn_register_observer(name, ldr_plugin_import, to), RES_OBSERVER}; + } else if (jump && (f = hostfn_observer(name))) { r = (Res){hostfn_register_observer(name, f, guest), RES_OBSERVER}; mark_used(1, name); } else if ((f = hostfn_override(name))) { diff --git a/runtime/main.c b/runtime/main.c index 9ce8517a..d84f2306 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -611,7 +611,7 @@ static int usage(const char *argv0) { "--probe-cfg | --probe-dpi | --probe-mods | --probe-playtime | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " "--probe-vkexec | --probe-astc | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | " - "--prepare-shaders | --warm-shaders | --boot |--seconds S [--shot PNG]]\n", argv0); + "--prepare-shaders | --warm-shaders | --boot | --plugin-init DIR | --seconds S [--shot PNG]]\n", argv0); return 2; } @@ -700,6 +700,10 @@ int main(int argc, char **argv) { } else if (!strcmp(argv[i], "--shot")) { if (++i == argc) return usage(argv[0]); cfg.shot = argv[i]; + } else if (!mode && !strcmp(argv[i], "--plugin-init")) { // the plugin service run (plugin_init.c) + if (++i == argc) return usage(argv[0]); + mode = argv[i - 1]; + cfg.plugin_init = argv[i]; } else if (!mode && !strncmp(argv[i], "--", 2)) { mode = argv[i]; } else { @@ -744,5 +748,6 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--prepare-shaders")) return vk_pipeline_prepare_shaders(); if (!strcmp(mode, "--warm-shaders")) return vk_pipeline_warm_shaders(); if (!strcmp(mode, "--boot")) { cfg.boot = 1; rt_boot(); return 0; } + if (!strcmp(mode, "--plugin-init")) { rt_boot_plugin_init(); return 0; } return usage(argv[0]); } diff --git a/runtime/memory.c b/runtime/memory.c index 078f119a..c452c398 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -562,6 +562,9 @@ static uint64_t run_start(const uint8_t *p, uint64_t i, uint8_t v) { // first return i; } +// The plugin service run (plugin_init.c) shows rtld, its process-memory aliases and its free pool here. +void (*mem_query_hook)(uint64_t a, RtMemoryInfo *r); + uint32_t mem_query(uint64_t a, RtMemoryInfo *out) { RtMemoryInfo r = {.base = a & ~(PAGE - 1), .size = PAGE}; uint64_t low = L.aslr.base; @@ -632,6 +635,7 @@ uint32_t mem_query(uint64_t a, RtMemoryInfo *out) { r.size = hi - lo; } plat_mutex_unlock_shared(&lock); + if (mem_query_hook) mem_query_hook(a, &r); if (out) *out = r; return R_OK; } diff --git a/runtime/plugin_init.c b/runtime/plugin_init.c new file mode 100644 index 00000000..96486584 --- /dev/null +++ b/runtime/plugin_init.c @@ -0,0 +1,429 @@ +// The plugin service run: totk.exe --plugin-init DIR runs the DT_INIT of every mod code module (recomp/layout.py +// plugin, an exlaunch plugin such as a mod's exefs/subsdk9) on this runtime's own kernel and sdk, records what the +// init did and exits before any game module's constructor, nnMain, graphics or audio. The record is the input of +// tools/exefs_plugins.py (checks) and, later, of the generator's hook sites; at play time a plugin's init does not run. +// The design follows #845 by itaskaev2, with the init run natively instead of under Unicorn. +// +// What exlaunch's init needs from the process, and how this run gives it: +// - Its module walk (exl::util::FindModules) reads QueryMemory from 0 up and takes every Code RX, Code R, CodeData +// RW run as a module, the first being rtld and the second main. Our layout has no rtld and reports the host's +// ranges as Code RX (memory.c taken_gap), which would make it take a gap for a module. mem_query_hook shows the +// game's own rtld NSO (exefs/rtld, read, never run) right below main, and the host's ranges as Inaccessible. +// - It picks a random free range in the ASLR region (GetSystemTick seeds it) for its process-memory aliases. Our +// ASLR region is almost all host ranges, so a free pool of POOL_SIZE is reserved after the alias region. +// - GetInfo 65001 (svc.c) gives it the current process handle; MapProcessMemory maps main's .text and its own pages +// at the picked address. Guest addresses are host addresses and the modules are not file-backed, so an alias is a +// copy: its pages hold the source's bytes when mapped, and the words written through it are written into the +// source when it is unmapped and when the init returns. That is how the stubs reach the plugin's image and the +// hook branches reach main's .text. +// - nn::oe::GetDisplayVersion is answered with this build's game version, so the plugin picks its table for the +// installed game without the applet services. +// Only svcs the init needs are allowed (QueryMemory, GetSystemTick, OutputDebugString, GetInfo 2-5, 12-15 and 65001, +// Map/UnmapProcessMemory); any other one, such as CreateSession and ReplyAndReceive of exlaunch's process-handle +// fallback, ends the run with the reason in the record. +// +// The record, in DIR: record.json (modules, the writes into every game module as words, the alias calls, svc and +// import counts, the tick offset, a refusal), plugin.img (the plugin's image after its init, all segments) and +// plugin.pre.img (the same after loading and relocation, before the init). TOTK_PLUGIN_TICK_OFFSET adds to every +// GetSystemTick, so two runs pick different aliases. +#include +#include +#include + +#include "cfg.h" +#include "host.h" +#include "kernel.h" +#include "platform/platform.h" + +#define PAGE 0x1000ull +#define GiB (1ull << 30) +#define POOL_SIZE (64 * GiB) +#define POOL_ALIGN (4 * GiB) +#define RTLD_ALIGN 0x200000ull +#define MAX_ALIASES 256 +#define MAX_IMPORTS 64 +#define CUR_PROCESS 0xFFFF8001u +// Horizon memory states and permissions of QueryMemory, and results. +enum { ST_FREE = 0, ST_CODE = 3, ST_CODEDATA = 4, ST_SHARED_CODE = 0xF, ST_INACCESSIBLE = 0x10 }; +enum { PERM_R = 1, PERM_RW = 3, PERM_RX = 5 }; +#define R_OK 0u +#define R_INVALID_ADDRESS 0xcc01u +#define R_INVALID_STATE 0xd401u +#define R_INVALID_HANDLE 0xe401u +#define R_LIMIT_REACHED 0x10801u +#define SYM_DISPLAY_VERSION "_ZN2nn2oe17GetDisplayVersionEPNS0_14DisplayVersionE" + +typedef struct { + uint64_t dst, src, size; + uint8_t *snap; // the source's bytes when mapped: the words that differ from it were written through the alias + int live, unmapped; +} Alias; +static Alias aliases[MAX_ALIASES]; +static int naliases; +static uint64_t rtld_base, rtld_size, rtld_text_end, rtld_data_off; +static uint64_t pool_base; +static int running; // a plugin's init runs: svcs and imports are counted +static uint64_t svc_count[0x80], info_count[16], info_65001; +static struct { const char *name; uint64_t calls; } imports[MAX_IMPORTS]; +static int nimports; +static uint8_t **pre; // per loaded module: its image before the plugins' inits + +static uint64_t align_up(uint64_t v, uint64_t a) { return (v + a - 1) & ~(a - 1); } + +static void clip(uint64_t a, uint64_t *lo, uint64_t *hi, uint64_t b, uint64_t n) { + if (!n) return; + uint64_t e = b + n; + if (e <= a && e > *lo) *lo = e; + if (b > a && b < *hi) *hi = b; +} + +static void clip_aliases(uint64_t a, uint64_t *lo, uint64_t *hi) { + for (int i = 0; i < naliases; i++) + if (aliases[i].live) clip(a, lo, hi, aliases[i].dst, aliases[i].size); +} + +static int in_module(uint64_t a) { + for (int i = 0; i < ldr_count(); i++) + if (a - ldr_module(i)->base < ldr_module(i)->size) return 1; + return 0; +} + +// ---- QueryMemory as exlaunch expects it (memory.c mem_query_hook) +static void query_hook(uint64_t a, RtMemoryInfo *r) { + for (int i = 0; i < naliases; i++) { + const Alias *al = &aliases[i]; + if (al->live && a - al->dst < al->size) { + *r = (RtMemoryInfo){.base = al->dst, .size = al->size, .state = ST_SHARED_CODE, .perm = PERM_RW}; + return; + } + } + if (a - rtld_base < rtld_size) { + uint64_t off = a - rtld_base, text = align_up(rtld_text_end, PAGE); + if (off < text) *r = (RtMemoryInfo){.base = rtld_base, .size = text, .state = ST_CODE, .perm = PERM_RX}; + else if (off < rtld_data_off) + *r = (RtMemoryInfo){.base = rtld_base + text, .size = rtld_data_off - text, .state = ST_CODE, .perm = PERM_R}; + else + *r = (RtMemoryInfo){.base = rtld_base + rtld_data_off, .size = rtld_size - rtld_data_off, + .state = ST_CODEDATA, .perm = PERM_RW}; + return; + } + uint64_t lo = r->base, hi = r->base + r->size; + if (a - pool_base < POOL_SIZE) { + lo = pool_base, hi = pool_base + POOL_SIZE; + *r = (RtMemoryInfo){.state = ST_FREE}; + } else if (r->state == ST_CODE && !in_module(a)) { // a host range (memory.c taken_gap) + clip(a, &lo, &hi, rtld_base, rtld_size); + clip(a, &lo, &hi, pool_base, POOL_SIZE); + *r = (RtMemoryInfo){.state = ST_INACCESSIBLE}; + } else if (r->state != ST_FREE) { + return; + } + clip_aliases(a, &lo, &hi); + r->base = lo; + r->size = hi - lo; +} + +// ---- Map/UnmapProcessMemory (svc.c rt_process_memory) +static const RtLoaded *module_holding(uint64_t a, uint64_t n) { + for (int i = 0; i < ldr_count(); i++) { + const RtLoaded *m = ldr_module(i); + if (a >= m->base && n <= m->size && a - m->base <= m->size - n) return m; + } + return NULL; +} + +// Writes the words changed through an alias into its source. The source's read-only pages (sealed .text and .rodata) +// are made writable for the copy and sealed again. +static void flush(Alias *al) { + const RtLoaded *m = module_holding(al->src, al->size); + const uint8_t *now = (const uint8_t *)(uintptr_t)al->dst; + for (uint64_t off = 0; off < al->size; off += PAGE) { + if (!memcmp(now + off, al->snap + off, PAGE)) continue; + uint64_t page = al->src + off; + int sealed = page - m->base < m->data_off; + if (sealed && !plat_mem_protect(page, PAGE, PLAT_PROT_RW)) rt_fatal("plugin-init: cannot unseal %#llx", (unsigned long long)page); + for (uint64_t w = 0; w < PAGE; w += 4) + if (memcmp(now + off + w, al->snap + off + w, 4)) memcpy((void *)(uintptr_t)(page + w), now + off + w, 4); + if (sealed && !plat_mem_protect(page, PAGE, PLAT_PROT_READ)) rt_fatal("plugin-init: cannot seal %#llx", (unsigned long long)page); + } + memcpy(al->snap, now, al->size); // flushed: a later flush writes only what changes after this one +} + +static uint32_t process_memory(int unmap, uint64_t dst, uint32_t process, uint64_t src, uint64_t size) { + if (process != CUR_PROCESS) return R_INVALID_HANDLE; + if (!size || ((dst | src | size) & (PAGE - 1))) return R_INVALID_ADDRESS; + if (unmap) { + for (int i = 0; i < naliases; i++) { + Alias *al = &aliases[i]; + if (!al->live || al->dst != dst || al->src != src || al->size != size) continue; + flush(al); + if (!plat_mem_decommit(dst, size)) rt_fatal("plugin-init: cannot unmap the alias at %#llx", (unsigned long long)dst); + free(al->snap); + al->snap = NULL; + al->live = 0; + al->unmapped = 1; + return R_OK; + } + return R_INVALID_STATE; + } + if (!module_holding(src, size)) return R_INVALID_STATE; // only the modules' own memory + RtMemoryInfo q; + mem_query(dst, &q); + if (q.state != ST_FREE || dst + size > q.base + q.size) return R_INVALID_STATE; + if (naliases == MAX_ALIASES) return R_LIMIT_REACHED; + if (!plat_mem_commit(dst, size)) return R_INVALID_STATE; // not inside one of our reservations + Alias *al = &aliases[naliases]; + al->snap = malloc(size); + if (!al->snap) rt_fatal("plugin-init: out of memory for a %#llx byte alias", (unsigned long long)size); + memcpy((void *)(uintptr_t)dst, (const void *)(uintptr_t)src, size); + memcpy(al->snap, (const void *)(uintptr_t)src, size); + *al = (Alias){dst, src, size, al->snap, 1, 0}; + naliases++; + return R_OK; +} + +// ---- svcs and imports +static void write_record(const char *refused); + +static void refuse(const char *why) { + rt_log("plugin-init: refused: %s", why); + write_record(why); + fflush(NULL); + plat_exit(3); +} + +static void plugin_svc(Cpu *c, uint32_t imm, uint64_t pc) { + if (!running) return; + svc_count[imm & 0x7F]++; + char why[256], at[128]; + switch (imm) { + case 0x06: case 0x1E: case 0x27: case 0x74: case 0x75: return; + case 0x29: { + uint32_t id = (uint32_t)c->x[1]; + if ((id >= 2 && id <= 5) || (id >= 12 && id <= 15)) { + info_count[id]++; + return; + } + if (id == 65001) { + info_65001++; + return; + } + snprintf(why, sizeof why, "GetInfo id %u at %s", id, ldr_describe(pc, at, sizeof at)); + refuse(why); + } + } + snprintf(why, sizeof why, "svc %#x at %s (only QueryMemory, GetSystemTick, OutputDebugString, GetInfo 2-5, 12-15 " + "and 65001, Map/UnmapProcessMemory are served)", imm, ldr_describe(pc, at, sizeof at)); + refuse(why); +} + +// Every JUMP_SLOT import of a plugin binds here (loader.c resolve): counted by name, then the guest definition runs, +// except nn::oe::GetDisplayVersion, which gets this build's game version. +static void plugin_import(Cpu *c) { + const char *name = hostfn_name(c->pc); + int i = 0; + while (i < nimports && strcmp(imports[i].name, name)) i++; + if (i == nimports && nimports < MAX_IMPORTS) imports[nimports++].name = name; + if (i < MAX_IMPORTS && running) imports[i].calls++; + if (!strcmp(name, SYM_DISPLAY_VERSION)) { // DisplayVersion { char value[16]; } + char v[16] = {0}; + snprintf(v, sizeof v, "%d.%d.%d", rt_game_version / 100, rt_game_version / 10 % 10, rt_game_version % 10); + memcpy((void *)(uintptr_t)c->x[0], v, sizeof v); + return; + } + rt_original(c); +} + +// ---- the record +static FILE *open_out(const char *file, const char *mode) { + char path[PLAT_PATH_MAX]; + snprintf(path, sizeof path, "%s/%s", rt_config()->plugin_init, file); + FILE *f = fopen(path, mode); + if (!f) rt_fatal("plugin-init: cannot write %s", path); + return f; +} + +static void write_image(const char *file, const void *p, uint64_t n) { + FILE *f = open_out(file, "wb"); + if (fwrite(p, 1, n, f) != n || fclose(f)) rt_fatal("plugin-init: cannot write %s", file); +} + +static void sha256_hex(const void *p, uint64_t n, char out[65]) { + uint8_t d[32]; + rt_sha256(p, n, d); + for (int i = 0; i < 32; i++) snprintf(out + 2 * i, 3, "%02x", d[i]); +} + +static void json_str(FILE *f, const char *s) { + fputc('"', f); + for (; *s; s++) { + if (*s == '"' || *s == '\\') fputc('\\', f); + if ((unsigned char)*s < 0x20) fprintf(f, "\\u%04x", *s); + else fputc(*s, f); + } + fputc('"', f); +} + +// The words of each game module that differ from its image before the plugins' inits: (module, offset, old, new). +static void write_writes(FILE *f) { + int first = 1; + for (int i = 0; i < ldr_count(); i++) { + const RtLoaded *m = ldr_module(i); + if (m->plugin || !pre || !pre[i]) continue; + const uint8_t *now = (const uint8_t *)(uintptr_t)m->base; + for (uint64_t off = 0; off < m->size; off += PAGE) { + if (!memcmp(now + off, pre[i] + off, PAGE)) continue; + for (uint64_t w = off; w < off + PAGE; w += 4) { + uint32_t a, b; + memcpy(&a, pre[i] + w, 4); + memcpy(&b, now + w, 4); + if (a == b) continue; + // fn_start: a lifted function starts at the word (a b there replaces the function) + fprintf(f, "%s\n {\"module\": \"%s\", \"offset\": \"%#llx\", \"old\": \"%#010x\", \"new\": \"%#010x\", " + "\"fn_start\": %s}", first ? "" : ",", m->name, (unsigned long long)w, a, b, + rt_dispatch_lookup(m->base + w) ? "true" : "false"); + first = 0; + } + } + } +} + +static void write_record(const char *refused) { + for (int i = 0; i < naliases; i++) + if (aliases[i].live) flush(&aliases[i]); + FILE *f = open_out("record.json", "w"); + // The marker of generated game-derived files (tools/check_content.py): the record never goes into the repository. + fprintf(f, "{\n\"marker\": \"%s%s\",\n\"game_version\": %d,\n\"tick_offset\": \"%#llx\",\n\"refused\": ", + "TOTK_RECOMP_", "GENERATED_FROM_GAME_CODE", rt_game_version, (unsigned long long)rt_svc_tick_offset); + if (refused) json_str(f, refused); + else fputs("null", f); + fprintf(f, ",\n\"rtld\": {\"base\": \"%#llx\", \"size\": \"%#llx\"},\n\"pool\": {\"base\": \"%#llx\", \"size\": \"%#llx\"},", + (unsigned long long)rtld_base, (unsigned long long)rtld_size, (unsigned long long)pool_base, + (unsigned long long)POOL_SIZE); + fputs("\n\"modules\": [", f); + for (int i = 0; i < ldr_count(); i++) { + const RtLoaded *m = ldr_module(i); + fprintf(f, "%s\n {\"name\": \"%s\", \"base\": \"%#llx\", \"size\": \"%#llx\", \"text_end\": \"%#llx\", " + "\"data_off\": \"%#llx\", \"init\": \"%#llx\", \"plugin\": %s", + i ? "," : "", m->name, (unsigned long long)m->base, (unsigned long long)m->size, + (unsigned long long)m->text_end, (unsigned long long)m->data_off, (unsigned long long)m->init, + m->plugin ? "true" : "false"); + if (m->plugin && !refused) { + char img[64], pre_img[64], hex[65]; + snprintf(img, sizeof img, "%s.img", m->name); + snprintf(pre_img, sizeof pre_img, "%s.pre.img", m->name); + write_image(img, (const void *)(uintptr_t)m->base, m->size); + if (pre && pre[i]) write_image(pre_img, pre[i], m->size); + sha256_hex((const void *)(uintptr_t)m->base, m->size, hex); + fprintf(f, ", \"image\": \"%s\", \"image_sha256\": \"%s\", \"pre_image\": \"%s\"", img, hex, pre_img); + } + if (m->plugin) { // the lifted function starts: a main write may branch straight to one (tools/exefs_plugins.py) + RtModule lm[16]; + int n = rt_linked_modules(lm, 16); + for (int k = 0; k < n; k++) { + if (strcmp(lm[k].name, m->name)) continue; + fputs(", \"functions\": [", f); + for (uint64_t j = 0; j < lm[k].count; j++) + fprintf(f, "%s\"%#llx\"", j ? ", " : "", (unsigned long long)(lm[k].addrs[j] - m->base)); + fputc(']', f); + } + } + for (int k = 0; k < rt_layout_count; k++) + if (rt_layout[k].file && !strcmp(rt_layout[k].name, m->name)) { + fputs(", \"file\": ", f); + json_str(f, rt_layout[k].file); + fprintf(f, ", \"nso_sha256\": \"%s\"", rt_layout[k].sha256); + } + fputc('}', f); + } + fputs("\n],\n\"writes\": [", f); + write_writes(f); + fputs("\n],\n\"aliases\": [", f); + for (int i = 0; i < naliases; i++) + fprintf(f, "%s\n {\"dst\": \"%#llx\", \"src\": \"%#llx\", \"size\": \"%#llx\", \"unmapped\": %s}", i ? "," : "", + (unsigned long long)aliases[i].dst, (unsigned long long)aliases[i].src, + (unsigned long long)aliases[i].size, aliases[i].unmapped ? "true" : "false"); + fputs("\n],\n\"svcs\": {", f); + int first = 1; + for (int i = 0; i < 0x80; i++) + if (svc_count[i]) fprintf(f, "%s\"%#x\": %llu", first ? "" : ", ", i, (unsigned long long)svc_count[i]), first = 0; + fputs("},\n\"get_info\": {", f); + first = 1; + for (int i = 0; i < 16; i++) + if (info_count[i]) fprintf(f, "%s\"%d\": %llu", first ? "" : ", ", i, (unsigned long long)info_count[i]), first = 0; + if (info_65001) fprintf(f, "%s\"65001\": %llu", first ? "" : ", ", (unsigned long long)info_65001); + fputs("},\n\"imports\": {", f); + for (int i = 0; i < nimports; i++) { + fputs(i ? ", " : "", f); + json_str(f, imports[i].name); + fprintf(f, ": %llu", (unsigned long long)imports[i].calls); + } + fputs("}\n}\n", f); + if (fclose(f)) rt_fatal("plugin-init: cannot write record.json"); +} + +// ---- setup and run +// Before ldr_load: the import observer, the svc hooks, the pool and the stand-in for rtld. +void plugin_init_setup(void) { + int plugins = 0; + for (int i = 0; i < rt_layout_count; i++) plugins += rt_layout[i].file != NULL; + if (!plugins) rt_fatal("plugin-init: this build's layout has no plugin module (recomp/gen.py --plugins)"); + plat_file_mkdir_p(rt_config()->plugin_init); + const char *off = cfg_get("TOTK_PLUGIN_TICK_OFFSET"); + rt_svc_tick_offset = off ? strtoull(off, NULL, 0) : 0; + ldr_plugin_import = plugin_import; + rt_process_memory = process_memory; + rt_plugin_svc = plugin_svc; + + const RtGuestLayout *g = mem_layout(); + uint64_t main_base = rt_layout[0].base; + rtld_size = ldr_nso_size(rt_config()->exefs, "rtld"); + rtld_base = main_base - align_up(rtld_size, RTLD_ALIGN); + for (uint64_t at = align_up(g->alias.base + g->alias.size, POOL_ALIGN); at + POOL_SIZE <= rtld_base && !pool_base; + at += POOL_ALIGN) + if (plat_mem_reserve(at, POOL_SIZE, 0)) pool_base = at; + if (!pool_base) rt_fatal("plugin-init: no free %llu GiB below main for the alias pool", POOL_SIZE / GiB); + if (!plat_mem_reserve(rtld_base, align_up(rtld_size, PAGE), 0)) + rt_fatal("plugin-init: %#llx below main is taken; rtld goes there", (unsigned long long)rtld_base); + ldr_map_image(rt_config()->exefs, "rtld", rtld_base, &rtld_text_end, &rtld_data_off); + mem_query_hook = query_hook; + rt_log("plugin-init: rtld at %#llx+%#llx, alias pool %#llx+%llu GiB, tick offset %#llx", (unsigned long long)rtld_base, + (unsigned long long)rtld_size, (unsigned long long)pool_base, POOL_SIZE / GiB, + (unsigned long long)rt_svc_tick_offset); +} + +// From nn::init::Start's module-init callback (boot.c), in place of the game's constructors. rtld initialises the +// modules from the last loaded to the first, so on the Switch sdk's DT_INIT runs before a subsdk plugin's; then each +// plugin's, the last first. Exits with the record written. +void plugin_init_run(Cpu *c, const RtLoaded *sdk) { + rt_log("plugin-init: ENTER %s DT_INIT", sdk->name); + rt_call(c, sdk->init, 0); + pre = calloc((size_t)ldr_count(), sizeof *pre); + if (!pre) rt_fatal("plugin-init: out of memory"); + for (int i = 0; i < ldr_count(); i++) { // the images before the plugins run + const RtLoaded *m = ldr_module(i); + pre[i] = malloc(m->size); + if (!pre[i]) rt_fatal("plugin-init: out of memory for a copy of %s", m->name); + memcpy(pre[i], (const void *)(uintptr_t)m->base, m->size); + } + for (int i = ldr_count() - 1; i >= 0; i--) { + const RtLoaded *m = ldr_module(i); + if (!m->plugin) continue; + if (!m->init) refuse("a plugin without DT_INIT"); + rt_log("plugin-init: ENTER %s DT_INIT at %#llx (%llu init_array entries)", m->name, (unsigned long long)m->init, + (unsigned long long)(m->init_array_size / 8)); + running = 1; + rt_trace |= RT_TRACE_PLUGIN; + rt_call(c, m->init, 0); + rt_trace &= ~(unsigned)RT_TRACE_PLUGIN; + running = 0; + rt_log("plugin-init: %s DT_INIT returned", m->name); + } + write_record(NULL); + int live = 0; + for (int i = 0; i < naliases; i++) live += aliases[i].live; + rt_log("plugin-init: record written to %s (%d process-memory maps, %d still mapped)", rt_config()->plugin_init, + naliases, live); + fflush(NULL); + plat_exit(0); +} diff --git a/runtime/svc.c b/runtime/svc.c index 27cdad19..1f62f62d 100644 --- a/runtime/svc.c +++ b/runtime/svc.c @@ -3,6 +3,8 @@ // The SVC entry of the lifted code: preemption checkpoint, trace, then the thread and // synchronization SVCs (kernel.c), IPC (ipc.c), memory (memory.c), Break, OutputDebugString, GetInfo and ExitProcess. The 21 SVCs // TotK has but never reaches on MNG's boot path stop with a named fatal, the caller and the guest backtrace. +// GetSystemTick, GetInfo 65001 and Map/UnmapProcessMemory are for mod plugins (exlaunch): the game calls none of them, +// and process memory is mapped only in the plugin service run (plugin_init.c). #include #include @@ -17,9 +19,13 @@ #define ST_STACK 0xBu #define PERM_RW 3u #define PROGRAM_ID 0x0100f2c0115b6000ull +#define CUR_PROCESS 0xFFFF8001u // Horizon's pseudo handle of the current process static uint64_t g_svcs; void (*volatile svc_test_exit_hook)(unsigned code); +uint64_t rt_svc_tick_offset; +void (*rt_plugin_svc)(Cpu *c, uint32_t imm, uint64_t pc); +uint32_t (*rt_process_memory)(int unmap, uint64_t dst, uint32_t process, uint64_t src, uint64_t size); RtStats rt_stats(void) { return (RtStats){__atomic_load_n(&g_svcs, __ATOMIC_RELAXED), kern_threads_created(), hostfn_calls()}; @@ -121,6 +127,7 @@ static void get_info(Cpu *c, uint64_t pc) { case 18: v = PROGRAM_ID; break; case 20: v = user_exception_context(); break; // the SDK keeps e.g. the dying message at +0x1C0 case 23: v = 1; break; + case 65001: v = CUR_PROCESS; break; // Mesosphere's current process handle (exlaunch's way to its own memory) default: { char why[64]; snprintf(why, sizeof why, "GetInfo id %u (handle %#x, sub id %#llx)", id, (uint32_t)c->x[2], @@ -151,11 +158,13 @@ void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) { __atomic_fetch_add(&g_svcs, 1, __ATOMIC_RELAXED); - if (rt_trace & RT_TRACE_SVC) { + if (rt_trace & (RT_TRACE_SVC | RT_TRACE_PLUGIN)) { + if ((rt_trace & RT_TRACE_PLUGIN) && rt_plugin_svc) rt_plugin_svc(c, imm, pc); char at[160]; - rt_log("svc %#x at %s x0=%#llx x1=%#llx x2=%#llx x3=%#llx", imm, ldr_describe(pc, at, sizeof at), - (unsigned long long)c->x[0], (unsigned long long)c->x[1], (unsigned long long)c->x[2], - (unsigned long long)c->x[3]); + if (rt_trace & RT_TRACE_SVC) + rt_log("svc %#x at %s x0=%#llx x1=%#llx x2=%#llx x3=%#llx", imm, ldr_describe(pc, at, sizeof at), + (unsigned long long)c->x[0], (unsigned long long)c->x[1], (unsigned long long)c->x[2], + (unsigned long long)c->x[3]); } if (kern_svc(c, imm)) return; uint64_t *x = c->x; @@ -210,6 +219,11 @@ static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) { return; } case 0x29: get_info(c, pc); return; + case 0x1E: x[0] = rt_ticks() + rt_svc_tick_offset; return; // GetSystemTick (19.2 MHz) + case 0x74: case 0x75: // MapProcessMemory / UnmapProcessMemory(x0 dst, w1 process, x2 src, x3 size) + if (!rt_process_memory) svc_fatal(c, imm, pc, "process memory is mapped only in the plugin service run"); + x[0] = rt_process_memory(imm == 0x75, x[0], (uint32_t)x[1], x[2], x[3]); + return; case 0x07: { // ExitProcess, as closing the window (gfx_window_closed): pipeline cache, shader records and // profile saved, logs flushed, exit code 0 char at[160]; From b2b85c05fa5bca5b2c01531b1df52301601a94d5 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 08:05:19 +0300 Subject: [PATCH 03/21] tools/exefs_plugins.py runs a plugin's init twice and checks the record (#845) Two service runs with other GetSystemTick values, the words that differ between them listed with what they point into, and a check that refuses a plugin with a reason per broken rule: a refused run, svcs, GetInfo ids or imports outside the reviewed lists, writes outside main's .text, and writes that are none of exlaunch's inline hooks (b to a stur/bl impl/ldur/b/.quad stub whose trampoline is the original relocated plus b site+4), call-site rewrites (bl to a function start of the plugin's lift) or replace hooks (b from a main function start, Orig trampoline found). tests/exefs_plugins_test.py covers the layout with and without plugins and one bad record per rule. After the design in #845 by itaskaev2. Co-Authored-By: Claude Opus 5.5 --- tests/exefs_plugins_test.py | 282 ++++++++++++++++++++++++++++ tests/run_all.py | 1 + tools/exefs_plugins.py | 358 ++++++++++++++++++++++++++++++++++++ 3 files changed, 641 insertions(+) create mode 100644 tests/exefs_plugins_test.py create mode 100644 tools/exefs_plugins.py diff --git a/tests/exefs_plugins_test.py b/tests/exefs_plugins_test.py new file mode 100644 index 00000000..6bb23495 --- /dev/null +++ b/tests/exefs_plugins_test.py @@ -0,0 +1,282 @@ +"""recomp/layout.py with and without mod plugins, and tools/exefs_plugins.py's check of a plugin service run record: +a synthetic exlaunch-shaped record with one inline hook that passes, and one bad record per rule. No game files. + +Run with .venv (lz4 and numpy, as recomp/): python tests/exefs_plugins_test.py +""" +import json +import struct +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +sys.path.insert(0, str(ROOT / 'recomp')) +import exefs_plugins as xp # noqa: E402 +import layout # noqa: E402 +from nso import Nso # noqa: E402 + + +def nso(size_text, bss=0x1000): + """An uncompressed NSO: .text of size_text bytes (b over MOD0's offset word), .rodata with MOD0 at 0x1000 past + .text, .data with an empty .dynamic.""" + text = bytearray(size_text) + struct.pack_into('> 2) & 0x3FFFFFF + + +def record(original=0x94000010, trampoline_first=None, stub=None, writes=None, **extra): + """A record of one inline hook at main+SITE_OFF whose original is `original` (default bl main+SITE_OFF+0x40).""" + site, stub_at, tramp = MAIN + SITE_OFF, PLUGIN + STUB_OFF, PLUGIN + TRAMP_OFF + img = bytearray(0x14000) + words = stub or [xp.STUR_X30, b(stub_at + 4, PLUGIN + IMPL_OFF, True), xp.LDUR_X30, b(stub_at + 12, tramp)] + struct.pack_into('<4IQ', img, STUB_OFF, *words, PLUGIN + CB_OFF) + if trampoline_first is None: + rel = xp.pc_relative(original, site) + trampoline_first = (b(tramp, rel[1], True) if rel and rel[0][0] == 'b' else original) + struct.pack_into(''], 'compiles its checks with clang'), 'cheats_vm_check': ('.venv', 'cc', ['--cc', ''], 'compiles runtime/cheats.c into a harness with clang'), diff --git a/tools/exefs_plugins.py b/tools/exefs_plugins.py new file mode 100644 index 00000000..8bdaac44 --- /dev/null +++ b/tools/exefs_plugins.py @@ -0,0 +1,358 @@ +"""Run a mod plugin's init in the plugin service run and check what it did: the build step before the generator emits +a plugin's hook sites. After the design in #845 by itaskaev2; the init runs on this port's runtime (totk.exe +--plugin-init, runtime/plugin_init.c) instead of under Unicorn. + + python tools/exefs_plugins.py --exe build/-x/totk.exe --out # two runs, their diff, the check + python tools/exefs_plugins.py --check /run1 # the check of one record + +The exe must be built with the plugin in its layout (recomp/gen.py --plugins). --out gets run1/ and run2/, one record +each; run2 adds a large offset to every GetSystemTick (TOTK_PLUGIN_TICK_OFFSET), so the init picks other alias +addresses. The words that differ between the two records are listed with what they look like, not refused: an init +keeps its alias addresses and the regions GetInfo gave it in its .data, and a plugin's callbacks must not use them +(the generator's check of the code the callbacks reach, still to come). + +The check (check_record) accepts a record only when every rule holds; each broken rule is one reason: + - the run was not refused (an svc outside the served set, such as CreateSession of exlaunch's process-handle + fallback, ends the run with the reason); + - the init called only the svcs ALLOWED_SVCS, GetInfo with ALLOWED_INFO, and the imports ALLOWED_IMPORTS; + - every write into a game module is a word of main's .text whose original is no br, blr or ret, and it is one of: + - an inline hook (MSFix, Full Inventory Expansion): `b ` into a plugin's .text, the stub in exlaunch's + shape (stur x30, [sp, #-0x10]; bl exl_inline_hook_impl or the float one; ldur x30, [sp, #-0x10]; + b ; .quad ), the trampoline the site's original relocated (the + same word, or the same PC-relative target) and `b `; + - a call-site rewrite (Quick Weapon Durability): `bl` straight to a function start of the plugin's lift (the + record lists them), which the generator can emit as a patch site whose replacement is that branch; + - a replace hook (Full Inventory Expansion's Sort Fixer): `b` from a function start of main to a function start of + the plugin's lift; the Orig trampoline (the original relocated, b site+4) is listed when the pool holds one. +Not checked here yet: a site shared with a built-in patch set or another mod, and svcs in the code the callbacks +reach (both need the generator's view). +""" +import argparse +import json +import os +import struct +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'recomp')) +from nso import Nso # noqa: E402 + +ALLOWED_SVCS = {0x06: 'QueryMemory', 0x1E: 'GetSystemTick', 0x27: 'OutputDebugString', 0x29: 'GetInfo', + 0x74: 'MapProcessMemory', 0x75: 'UnmapProcessMemory'} +ALLOWED_INFO = {2, 3, 4, 5, 12, 13, 14, 15, 65001} +# Reviewed one by one: SNPrintf formats into the caller's buffer; GetDisplayVersion is answered by the service run; +# mersenne_twister_engine::_M_gen_rand (Full Inventory Expansion's alias picker) refills the state of the +# engine object it is given, which lives in the plugin's .bss; memcmp and strlen only read. malloc stays out: Quick +# Weapon Durability's init keeps a heap block's address in its .bss, and that block does not exist at play time. +ALLOWED_IMPORTS = {'_ZN2nn4util8SNPrintfEPcmPKcz', '_ZN2nn2oe17GetDisplayVersionEPNS0_14DisplayVersionE', 'memcmp', 'strlen', + '_ZNSt23mersenne_twister_engineImLm64ELm312ELm156ELm31ELm13043109905998158313ELm29ELm6148914691236517205' + 'ELm17ELm8202884508482404352ELm37ELm18444473444759240704ELm43ELm6364136223846793005EE11_M_gen_randEv'} +HOOK_IMPLS = ('exl_inline_hook_impl', 'exl_inline_float_hook_impl') +STUR_X30 = 0xF81F03FE # stur x30, [sp, #-0x10] +LDUR_X30 = 0xF85F03FE # ldur x30, [sp, #-0x10] +STUB_WORDS = 6 # 4 instructions and the callback's .quad +TICK_OFFSET = 0x0123456789AB + + +def u(v): + return int(v, 16) if isinstance(v, str) else v + + +# ---- instructions +def sext(v, bits): + return v - (1 << bits) if v >> (bits - 1) & 1 else v + + +def branch_target(w, pc): + """Target of b / bl at pc, else None.""" + if w & 0x7C000000 == 0x14000000: + return pc + sext(w & 0x3FFFFFF, 26) * 4 + return None + + +def pc_relative(w, pc): + """(kind, absolute value) of a PC-relative instruction at pc, else None: b, bl, b.cond, cbz/cbnz, tbz/tbnz, adr, + adrp, ldr (literal). The kind holds the bits besides the offset, so two encodings match when both agree.""" + if w & 0x7C000000 == 0x14000000: + return ('b', w & 0xFC000000), pc + sext(w & 0x3FFFFFF, 26) * 4 + if w & 0xFF000010 == 0x54000000: + return ('b.cond', w & 0xFF00001F), pc + sext(w >> 5 & 0x7FFFF, 19) * 4 + if w & 0x7E000000 == 0x34000000: + return ('cbz', w & 0xFF00001F), pc + sext(w >> 5 & 0x7FFFF, 19) * 4 + if w & 0x7E000000 == 0x36000000: + return ('tbz', w & 0xFFF8001F), pc + sext(w >> 5 & 0x3FFF, 14) * 4 + if w & 0x9F000000 == 0x10000000: + return ('adr', w & 0x1F), pc + sext((w >> 5 & 0x7FFFF) << 2 | w >> 29 & 3, 21) + if w & 0x9F000000 == 0x90000000: + return ('adrp', w & 0x1F), (pc & ~0xFFF) + (sext((w >> 5 & 0x7FFFF) << 2 | w >> 29 & 3, 21) << 12) + if w & 0x3B000000 == 0x18000000: + return ('ldr', w & 0xFF00001F), pc + sext(w >> 5 & 0x7FFFF, 19) * 4 + return None + + +def is_indirect(w): + """br, blr, ret (and their pointer-authenticated forms): a site exlaunch could hook but the generator cannot.""" + return w & 0xFE1F0000 == 0xD61F0000 and w >> 21 & 0xF in (0, 1, 2) + + +def relocated(orig, site, word, at): + """The trampoline word at `at` does what the original at `site` does.""" + a, b = pc_relative(orig, site), pc_relative(word, at) + if a is None: + return word == orig + return b is not None and a == b + + +def inline_stub(img, o, p): + """The words at image offset o of plugin p have exlaunch's inline-hook entry shape: stur x30; bl; ldur x30; b; + and the .quad after them lies inside the plugin's .text.""" + if o + 4 * STUB_WORDS > p['text_end']: + return False + s = struct.unpack_from('<4I', img, o) + return s[0] == STUR_X30 and s[2] == LDUR_X30 and s[1] >> 26 == 0x25 and s[3] >> 26 == 0x05 + + +def orig_trampoline(img, p, original, site): + """Image offset of a pair in the plugin's .text that runs the site's original instruction relocated, then b site+4 + (exlaunch's Orig trampoline of a replace hook), else None.""" + base = p['base'] + for o in range(0, p['text_end'] - 4, 4): + t1 = struct.unpack_from('> 26 == 0x05 and branch_target(t1, base + o + 4) == site + 4 and \ + relocated(original, site, struct.unpack_from('= main['text_end'] or w['offset'] & 3: + reasons.append(f'a write outside main\'s .text: {where} {w["old"]:#010x} -> {w["new"]:#010x}') + continue + site = main['base'] + w['offset'] + stub = branch_target(w['new'], site) + link = bool(w['new'] >> 31) + if stub is None: + reasons.append(f'{where}: the new word {w["new"]:#010x} is no b or bl') + continue + if is_indirect(w['old']): + reasons.append(f'{where}: the original {w["old"]:#010x} is a br, blr or ret') + p, _ = plugin_word(stub) + if p is None: + reasons.append(f"{where}: {'bl' if link else 'b'} {stub:#x} does not reach into a plugin's .text") + continue + if not (not link and inline_stub(rec['images'][p['name']], stub - p['base'], p)): + # A b or bl straight into the plugin's code (CodePatcher-style call-site rewrite): a site whose replacement + # is that branch, when the target is a function start of the plugin's lift. + # A b replaces the whole main function, so it must sit at a function start of main; exlaunch's replace + # hook keeps the function's first instruction relocated + b site+4 in its pool for the callback's Orig. + o = stub - p['base'] + hook = {'site': f'main+{w["offset"]:#x}', 'original': f'{w["old"]:#010x}', + 'kind': 'call' if link else 'replace', 'target': f'{p["name"]}+{o:#x}'} + orig = orig_trampoline(rec['images'][p['name']], p, w['old'], site) + if orig is not None: + hook['trampoline'] = f'{p["name"]}+{orig:#x}' + hooks.append(hook) + if o not in {u(x) for x in p.get('functions', [])}: + reasons.append(f"{where}: {'bl' if link else 'b'} {p['name']}+{o:#x} is neither an inline-hook stub " + f"nor a function start of {p['name']}'s lift") + if not link and not w.get('fn_start'): + reasons.append(f"{where}: b {p['name']}+{o:#x} from inside a main function (only a function start can " + f"be replaced)") + continue + img, o = rec['images'][p['name']], stub - p['base'] + s = struct.unpack_from('<4IQ', img, o) + impl, tramp, cb = branch_target(s[1], stub + 4), branch_target(s[3], stub + 12), s[4] + hook = {'site': f'main+{w["offset"]:#x}', 'original': f'{w["old"]:#010x}', 'kind': 'inline', + 'stub': f'{p["name"]}+{o:#x}', + 'impl': impls.get(impl, f'{impl:#x}' if impl is not None else None), + 'callback': f'{p["name"]}+{cb - p["base"]:#x}' if p['base'] <= cb < p['base'] + p['text_end'] else f'{cb:#x}', + 'trampoline': f'{p["name"]}+{tramp - p["base"]:#x}' if tramp is not None else None} + hooks.append(hook) + if impl not in impls: + reasons.append(f'{where}: the stub calls {impl:#x}, not {" or ".join(HOOK_IMPLS)}') + if not p['base'] <= cb < p['base'] + p['text_end']: + reasons.append(f'{where}: the stub\'s callback {cb:#x} is outside {p["name"]}\'s .text') + tp, t0 = plugin_word(tramp) + _, t1 = plugin_word(tramp + 4) + if tp is None or t1 is None: + reasons.append(f'{where}: the trampoline {tramp:#x} is outside a plugin\'s .text') + continue + hook['relocated'] = f'{t0:#010x}' + if not relocated(w['old'], site, t0, tramp): + reasons.append(f'{where}: the trampoline {hook["trampoline"]} starts with {t0:#010x}, not the original ' + f'{w["old"]:#010x} relocated') + if branch_target(t1, tramp + 4) != site + 4 or t1 >> 31: + reasons.append(f'{where}: the trampoline\'s second word {t1:#010x} is no b to the site + 4') + return hooks, reasons + + +# ---- two runs +def where_is(v, rec): + """What an address-like value points into, for the diff listing.""" + for m in rec['modules']: + if m['base'] <= v < m['base'] + m['size']: + return f'{m["name"]}+{v - m["base"]:#x}' + for k in ('rtld', 'pool'): + r = rec.get(k) + if r and u(r['base']) <= v < u(r['base']) + u(r['size']): + return f'the {"alias pool" if k == "pool" else k} (+{v - u(r["base"]):#x})' + for a in rec.get('aliases', []): + if u(a['dst']) <= v < u(a['dst']) + u(a['size']): + return f'an alias of {u(a["src"]):#x}' + return None + + +def section(m, off, nso_path): + if off < m['text_end']: + return '.text' + if off < m['data_off']: + return '.rodata' + if nso_path and Path(nso_path).is_file(): + n = Nso(nso_path) + if off < n.data[0] + n.data[1]: + return '.data' + return '.bss' + return '.data/.bss' + + +def diff_records(a, b): + """Lines naming every difference between two records: writes into game modules, plugin image words (8 bytes).""" + out = [] + wa = {(w['module'], w['offset']): w for w in a['writes']} + wb = {(w['module'], w['offset']): w for w in b['writes']} + for k in sorted(set(wa) | set(wb)): + x, y = wa.get(k), wb.get(k) + if not x or not y or x['new'] != y['new']: + out.append(f'write {k[0]}+{k[1]:#x}: run 1 {x and hex(x["new"])}, run 2 {y and hex(y["new"])}') + for m in a['modules']: + if not m['plugin']: + continue + ia, ib = a['images'].get(m['name']), b['images'].get(m['name']) + if ia is None or ib is None or len(ia) != len(ib): + out.append(f'{m["name"]}: image missing or of another size') + continue + for off in range(0, len(ia), 8): + if ia[off:off + 8] == ib[off:off + 8]: + continue + va, vb = struct.unpack_from(' stub {h["stub"]} -> {h["impl"]}, callback ' + f'{h["callback"]}, trampoline {h["trampoline"]} ({h.get("relocated")}; b site+4)') + else: + print(f' {h["kind"]} {h["site"]} (was {h["original"]}) -> {"bl" if h["kind"] == "call" else "b"} ' + f'{h["target"]}' + (f', Orig trampoline {h["trampoline"]}' if h.get('trampoline') else '')) + for r in reasons: + print(f' REFUSED: {r}') + if not reasons: + print(f' accepted: {len(hooks)} hooks') + return reasons + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--exe', type=Path, help='totk.exe built with the plugin in its layout') + ap.add_argument('--out', type=Path, help='directory for run1/ and run2/') + ap.add_argument('--check', type=Path, help='check the record in this directory only') + ap.add_argument('--timeout', type=int, default=300) + args = ap.parse_args() + if args.check: + return 1 if report(load_record(args.check), str(args.check)) else 0 + if not args.exe or not args.out: + ap.error('--exe and --out, or --check') + recs = [] + for i, offset in enumerate((0, TICK_OFFSET), 1): + d = args.out / f'run{i}' + code = run(args.exe, d, offset, args.timeout) + if not (d / 'record.json').is_file(): + print(f'run {i}: no record (exit {code}); see {d / "run.log"}') + return 1 + recs.append(load_record(d)) + print(f'run {i}: exit {code}, tick offset {offset:#x}') + bad = report(recs[0], 'run 1') + diff = diff_records(*recs) + print(f'run 1 against run 2: {len(diff)} differences' + (':' if diff else '')) + for line in diff: + print(f' {line}') + return 1 if bad else 0 + + +if __name__ == '__main__': + sys.exit(main()) From db6d548ba25b7109bf17febcb0633029209db8a7 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 08:48:14 +0300 Subject: [PATCH 04/21] The generator lifts a mod plugin's hook sites: a call into its stub, its replace target and its call-site rewrite (#845) After the design in #845 by itaskaev2. New patch lines carry an exlaunch plugin whose init runs at every start: `plugin` lifts the module from its image after the init (the hook stubs and trampolines the file has as zeros) and names the hash of that .text, `expect` lists the words the init writes into the game, `stub` lifts an inline-hook stub's branch to its trampoline as a return, `seed` adds the callbacks, impls and targets, and `orig` lists an unpatched copy of a hooked main function (fo_
) at the replace hook's trampoline address in the plugin's function table, so the plugin's call of the original does not reach the hook again and rt_dispatch stays as it is. A replacement word @call:+ calls the stub without writing x30, with every register in the Cpu before it and read back after it; the site's own instruction runs after it in place. patches.h gets RT_PLUGINS, RT_PLUGIN_WRITES and RT_PLUGIN_ROOTS for the runtime. Without plugin lines the output is byte-identical to dev (generated C, patches.h and layout of 1.2.1 and 1.4.3). Co-Authored-By: Claude Opus 5.5 --- recomp/gen.py | 112 ++++++++++++++++++++++++++++++++------ recomp/module.py | 18 ++++++- recomp/patches.py | 133 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 240 insertions(+), 23 deletions(-) diff --git a/recomp/gen.py b/recomp/gen.py index 1a944eda..f41b55e1 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -68,7 +68,7 @@ import regs # noqa: E402 import vlocals # noqa: E402 from check_content import GENERATED_MARKER # noqa: E402 from layout import layout, plugin_files, write as write_layout # noqa: E402 -from lifter import hx # noqa: E402 +from lifter import Insn, hx # noqa: E402 from module import Module # noqa: E402 TERMINATORS = ('b', 'br', 'ret', 'trap') @@ -89,6 +89,30 @@ def fname(a): return f'f_{a:x}' +def orig_name(a): + """The unpatched copy of the function at a that a plugin's replace hook reaches as its original (patches.py `orig`).""" + return f'fo_{a:x}' + + +def open_module(name, path, base, table, plugin=False): + """Module name for lifting: a plugin with a `plugin` line of the patch table (recomp/patches.py) is lifted from its + image after the init, its inline-hook stubs returning where they would branch to their trampolines.""" + p = table.plugins.get(name) + if not p: + return Module(name, path, base, 'c', plugin=plugin) + image = Path(p['image']).read_bytes() + mod = Module(name, path, base, 'c', plugin=True, image=image, rets=[o + 12 for o in p['stubs']], + seeds=sorted(p['stubs'] | p['seeds'])) + t0, t1 = mod.nso.text[0], mod.nso.text[0] + mod.nso.text[1] + if hashlib.sha256(image[t0:t1]).hexdigest() != p['sha256']: + raise patches.PatchError(f'{name}: {p["image"]} is not the image whose .text the patch table names') + for o in p['stubs']: + w = int.from_bytes(image[o + 12:o + 16], 'little') + if not mod.in_text(base + o) or w >> 26 != 0x05: + raise patches.PatchError(f'{name}+{o:#x}: no inline-hook stub (its fourth word {w:08x} is no b)') + return mod + + def header(what): return f'// {GENERATED_MARKER}: {what}, generated by recomp/gen.py from game code. Do not edit or commit.\n' @@ -233,12 +257,13 @@ class Emitter: the C (an argument here, or anything else it reads) must go into the lift cache's key: chunk_key.""" def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True, spins=None, - v_locals=vlocals.LOCALS, v_trust=False, hooks=None, fn_enter_all=False, picks=None): + v_locals=vlocals.LOCALS, v_trust=False, hooks=None, fn_enter_all=False, picks=None, origs=None): self.mod = mod self.starts = starts self.patched = patched or {} # {address: [(set index, [replacement words])]} (recomp/patches.py) self.picks = picks or {} # {contested site: (pick index, {ordered set: value})} (recomp/patches.py) self.spins = spins or {} # {spin wait back-edge: its head} (recomp/patches.py `spin`) + self.origs = origs or set() # functions also emitted unpatched, as orig_name() (recomp/patches.py `orig`) self.reg_locals = reg_locals # recomp/regs.py: registers per function in C locals self.kept = regs.CALLEE_SAVED if callee_saved else frozenset() # not reloaded after a call self.v_locals = v_locals # recomp/vlocals.py: V registers per function in C locals (0: all in c->v) @@ -294,9 +319,12 @@ class Emitter: alternative may branch (bl, b, b.cond / cbz / tbz, ret).""" out = [] for _, words in self.patched[a]: - lifted = [self.mod.lifter.lift(w, a) for w in words] + # ('call', target): a plugin's inline hook (recomp/patches.py @call), before the site's own instruction + lifted = [Insn('hook', target=w[1]) if isinstance(w, tuple) else self.mod.lifter.lift(w, a) for w in words] for j, r in enumerate(lifted): last = j == len(lifted) - 1 + if r.kind == 'hook' and not last: + continue if r.kind != 'op' and not (last and r.kind in ('bl', 'b', 'bcond', 'ret')): raise patches.PatchError(f'{a:#x}: replacement word {words[j]:08x} is a {r.kind}') out.append(lifted) @@ -375,6 +403,22 @@ class Emitter: raise patches.PatchError(f'{a:#x}: a patched call that uses flags') return reads, alts[0][1] + def unpatched_copy(self, entry, end, refs): + """C of the function [entry, end) without its patched sites, named orig_name(entry): what a plugin's replace + hook calls as the original (its trampoline, recomp/patches.py `orig`). Its own fall-back fb_ is renamed too.""" + kept = self.patched + self.patched = {a: alts for a, alts in kept.items() if not entry <= a < end} + try: + lines, r = self.function(entry, end) + finally: + self.patched = kept + refs |= r + text = '\n'.join(lines).replace(f'fb_{entry:x}(', f'fbo_{entry:x}(') + head = f'void {fname(entry)}(Cpu *c) {{' + if head not in text: + raise patches.PatchError(f'{entry:#x}: no function head to rename for its unpatched copy') + return text.replace(head, f'void {orig_name(entry)}(Cpu *c) {{', 1) + def function(self, entry, end): """(lines of C, referenced function starts) for the guest function [entry, end).""" mod, starts = self.mod, self.starts @@ -457,6 +501,12 @@ class Emitter: return f'if ({fx(i, r.cond)}) {{ {jump(r.target, i)} }}' if r.kind == 'ret': return f'{spill}return;' + if r.kind == 'hook': + # a call that writes no x30, the whole Cpu current before it and read back after it: the stub saves and + # restores x30 itself, and the callback may change any register through exlaunch's hook context + refs.add(r.target) + self.stats['plugin_hook_calls'] += 1 + return f'{spill}{fname(r.target)}(c);{reload}' raise patches.PatchError(f'{a:#x}: patched instruction is a {r.kind}') def patch(i, ins, a): @@ -552,6 +602,8 @@ def chunk_text(name, number, emitter, fns): lines, r = emitter.function(entry, end) body.append('\n'.join(lines)) refs |= r + if entry in emitter.origs: + body.append(emitter.unpatched_copy(entry, end, refs)) # Every referenced function gets a prototype, including ones defined later in this chunk. protos = ''.join(f'void {fname(r)}(Cpu *c);\n' for r in sorted(refs)) if emitter.uses_patches: @@ -618,7 +670,7 @@ def _emitter(name): e = _W['emitters'].get(name) if e is None: path, base = _W['mods'][name] - mod = Module(name, path, base, 'c') + mod = open_module(name, path, base, _W['patches']) # Module.read is how the jump tables (recomp/jumptables.py _eval) read .text and .rodata at lift time: the # bytes it returns become part of the generated code, so /folded.txt lists them read, reads = mod.read, _W['reads'].setdefault(name, []) @@ -631,7 +683,8 @@ def _emitter(name): _W['patches'].for_module(name, base), spins=_W['patches'].spins_for(name, base), hooks=_W['patches'].hooks_for(name, base), - picks=_W['patches'].picks_for(name, base), **_W['opts']) + picks=_W['patches'].picks_for(name, base), + origs=_W['patches'].origs_for(name, base), **_W['opts']) return e @@ -676,13 +729,19 @@ def _emit_chunk(task): # ------------------------------------------------------------------------------------------------ driver -def table_text(name, addrs): +def table_text(name, addrs, extra=None): + """_table.c: the function starts and their C; extra {address: C function name} adds entries of functions + lifted elsewhere (a plugin's trampoline reaching the unpatched copy of a game function, recomp/patches.py `orig`).""" + extra = extra or {} + names = {a: fname(a) for a in addrs if a not in extra} + names.update(extra) + addrs = sorted(names) lines = [header(f'{name} function table'), '#include "cpu.h"\n\n'] - lines += [f'void {fname(a)}(Cpu *c);\n' for a in addrs] + lines += [f'void {names[a]}(Cpu *c);\n' for a in addrs] lines.append(f'\nconst uint64_t {name}_fn_addrs[] = {{\n') lines.append(',\n'.join(hx(a) for a in addrs)) lines.append(f'\n}};\nconst GuestFn {name}_fn_ptrs[] = {{\n') - lines.append(',\n'.join(fname(a) for a in addrs)) + lines.append(',\n'.join(names[a] for a in addrs)) lines.append(f'\n}};\nconst uint64_t {name}_fn_count = {len(addrs)};\n') return ''.join(lines) @@ -746,7 +805,7 @@ def in_span(addrs, lo, hi): return addrs[bisect.bisect_left(addrs, lo):bisect.bisect_left(addrs, hi)] -def chunk_key(mod_key, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs, picks=None): +def chunk_key(mod_key, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs, picks=None, origs=None): """Key of one chunk: its module, number (in its header) and functions, and the patch sites (set index and replacement words, @page resolved, so the patch page is in them), the pick index and values of its contested sites, spin waits and hooked functions inside its address range; hook_addrs None (FN_ENTER in every function) keys @@ -754,10 +813,15 @@ def chunk_key(mod_key, number, fns, patched, site_addrs, spins, spin_addrs, hook here, with a case in tests/lift_cache_test.py.""" lo, hi = fns[0][0], fns[-1][1] in_chunk = in_span(site_addrs, lo, hi) - sites = [[a, [[k, words] for k, words in patched[a]]] for a in in_chunk] + sites = [[a, [[k, [list(w) if isinstance(w, tuple) else w for w in words]] for k, words in patched[a]]] + for a in in_chunk] hooks = None if hook_addrs is None else in_span(hook_addrs, lo, hi) contested = [[a, picks[a][0], sorted(picks[a][1].items())] for a in in_chunk if a in (picks or {})] - return _key(mod_key, number, fns, sites, [[b, spins[b]] for b in in_span(spin_addrs, lo, hi)], hooks, contested) + parts = [mod_key, number, fns, sites, [[b, spins[b]] for b in in_span(spin_addrs, lo, hi)], hooks, contested] + copies = [e for e, _ in fns if e in (origs or ())] + if copies: # none: the key it had before plugins + parts.append(copies) + return _key(*parts) def _entry_check(file, entry): @@ -911,7 +975,7 @@ def main(): continue t = time.perf_counter() path = Path(files[name][0]) if name in files else args.exefs / name - mod = Module(name, path, base, 'c', plugin=name in files) + mod = open_module(name, path, base, table, plugin=name in files) patches.check(table, mod, name) starts, dstats = discover(mod) est = estimate_sizes(mod, starts) @@ -921,8 +985,25 @@ def main(): chunks = partition(est[:n], args.chunk_fns, args.chunk_bytes, args.big_fn) moddir = args.out / name moddir.mkdir(parents=True, exist_ok=True) + extra = table.orig_entries(name, {n: b for n, b, _ in mods}) + if extra: # a trampoline reaches another function's unpatched copy, not a lift of its own words + keep = ~np.isin(abs_starts, np.array(sorted(extra), dtype=abs_starts.dtype)) + starts, est, abs_starts = starts[keep], est[keep], abs_starts[keep] + n = len(starts) if not args.limit else min(args.limit, len(starts)) + ends = np.append(abs_starts[1:], mod.hi) + chunks = partition(est[:n], args.chunk_fns, args.chunk_bytes, args.big_fn) + missing = sorted(base + o for o in (table.plugins.get(name) or {}).get('seeds', ()) + if base + o not in set(abs_starts.tolist())) + if missing: + raise SystemExit(f'{name}: the seeds {", ".join(hex(a) for a in missing)} are no function starts') mkey = module_key(gen_key, name, path, base, abs_starts) + if table.plugin_inputs(name): + mkey = _key(mkey, table.plugin_inputs(name), _sha256(Path(table.plugins[name]['image']).read_bytes())) patched, spins, picks = table.for_module(name, base), table.spins_for(name, base), table.picks_for(name, base) + origs = table.origs_for(name, base) + bad = sorted(origs - set(abs_starts.tolist())) + if bad: + raise SystemExit(f'{args.patches}: orig {", ".join(hex(a) for a in bad)} is no function start of {name}') site_addrs, spin_addrs = sorted(patched), sorted(spins) hooks = None if args.fn_enter_all else table.hooks_for(name, base) stray = sorted(set(hooks or ()) - set(abs_starts.tolist())) @@ -932,11 +1013,11 @@ def main(): hook_addrs = None if hooks is None else sorted(hooks) for number, (i0, i1) in enumerate(chunks): fns = list(zip(abs_starts[i0:i1].tolist(), ends[i0:i1].tolist())) - key = chunk_key(mkey, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs, picks) + key = chunk_key(mkey, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs, picks, origs) tasks.append((int(est[i0:i1].sum()), (name, number, str(moddir / f'{name}_{number:04d}.c'), fns, key))) paths[name] = (str(path), base) starts_abs[name] = abs_starts - info[name] = {'base': base, 'lo': mod.lo, 'hi': mod.hi, 'emitted': abs_starts[:n].tolist(), + info[name] = {'base': base, 'lo': mod.lo, 'hi': mod.hi, 'emitted': abs_starts[:n].tolist(), 'extra': extra, 'chunks': len(chunks), 'discover': dstats, 'discover_seconds': time.perf_counter() - t, 'moddir': moddir} print(f'{name}: {dstats["functions"]} functions ({dstats["seeds"]} seeds, {dstats["rounds"]} rounds), ' @@ -1001,7 +1082,8 @@ def main(): stats.update(r['stats']) unsupported.update(r['unsupported']) t = time.perf_counter() - table_written = write_if_changed(moddir / f'{name}_table.c', table_text(name, d['emitted'])) + table_written = write_if_changed(moddir / f'{name}_table.c', table_text( + name, d['emitted'], {t: orig_name(f) for t, f in d['extra'].items()})) write_if_changed(moddir / 'folded.txt', folded_text(name, merge_ranges(folded[name]))) removed = remove_stale(moddir, name, d['chunks'], errors) manifest = { diff --git a/recomp/module.py b/recomp/module.py index 9d4e43e5..7155dbfe 100644 --- a/recomp/module.py +++ b/recomp/module.py @@ -12,6 +12,7 @@ import numpy as np from lifter import Insn, Lifter, Unsupported, hx from nso import DT_INIT, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, Nso, R_AARCH64_RELATIVE +RET = 0xD65F03C0 LIFT_CACHE = 1 << 16 # lifted instructions kept per module; analyses revisit nearby addresses only @@ -33,13 +34,27 @@ def _read_encoded(img, off, enc, pc_base): class Module: - def __init__(self, name, path, base, simd='c', plugin=False): + def __init__(self, name, path, base, simd='c', plugin=False, image=None, rets=(), seeds=()): + """image, rets, seeds: a plugin's lift (recomp/patches.py `plugin`): its .text is taken from image (the plugin's + image after its init, holding the hook stubs and trampolines the file has as zeros), the words at the image + offsets rets become `ret` (an inline-hook stub's b to its trampoline), and the image offsets seeds are function + starts too.""" self.name = name self.nso = Nso(str(path)) self.base = base self.plugin = plugin # a mod's code module (recomp/layout.py plugin): its init functions are seeds too n = self.nso self.image = bytes(n.image) + if image is not None: + t0, t1 = n.text[0], n.text[0] + n.text[1] + if len(image) < t1: + raise ValueError(f'{name}: the image after the init is shorter than its .text') + img = bytearray(self.image) + img[t0:t1] = image[t0:t1] + for o in rets: + img[o:o + 4] = RET.to_bytes(4, 'little') + self.image = bytes(img) + self.extra_seeds = [base + o for o in seeds] self.lo = base + n.text[0] self.hi = self.lo + n.text[1] self.words = np.frombuffer(self.image, dtype=' [ ...] + plugin + expect + stub + seed [ ...] + orig +
is a module offset (hex), the instruction word the module has there (checked; a mismatch stops the generator), each an instruction word (hex) or `@page:x`, which is `adrp x, ` encoded for the site's address. The patch page is the first 4 KiB page after main's image (layout.py leaves the rest @@ -44,6 +50,28 @@ and the direct branches back to it (module offsets). Each back-edge (checked: a ) is lifted as `rt_spin_gate(c, ); goto `, which keeps every register and only adds time (runtime/kernel.c kern_spin_gate). +The plugin lines carry a mod's exlaunch plugin (layout.py plugin), whose init runs at every start +(runtime/plugin_init.c) and whose hooks are sites of its set (tools/exefs_plugins.py writes them from the plugin service +run's record; the design follows #845 by itaskaev2): + - `plugin`: belongs to the set ; it is lifted from , the plugin's image after its init (the + service run's plugin.img), whose .text holds the hook stubs and trampolines the NSO file has as zeros. is that .text's hash: the runtime compares it with the .text its own run of the init leaves, and keeps the + set off when they differ. + - `expect`: one word the init writes into a game module (the hook branches into main's .text); the runtime compares + the words its run of the init wrote with these, all or nothing per plugin. + - `stub`: an exlaunch inline-hook entry stub (stur x30; bl impl; ldur x30; b trampoline; .quad callback). Its fourth + word is lifted as a `ret`: the generated site calls the stub and runs its own original instruction after it + (`@call` below), so the trampoline (the original relocated, b site+4) never runs. + - `seed`: function starts of the module the lift must have (the callbacks, the hook impls, branch targets). + - `orig`: the plugin calls its replace hook's original through the trampoline at +; the generator + lifts the function at + a second time without its patched sites + (fo_
) and lists that copy at the trampoline's address in 's function table, so the call reaches + the original code and not the hook again. +A replacement word `@call:+` is a call of the function there that writes no x30: every register in +the Cpu at the call and all read back after it (the callback may change any of them, as exlaunch's hook context +allows); the flags stay as they were. It may stand before the site's last word, which is then the original +instruction. + A `hook` line names guest functions (module offsets of function starts; gen.py stops on any other address) whose entry the runtime watches: their lifted C starts with FN_ENTER (runtime/cpu.h), a load of g_fnprof and a predicted branch to fnprof_hit while the runtime sets it. A table with hook lines puts FN_ENTER into these functions only; a table @@ -73,13 +101,36 @@ class Table: from here (for_module, picks_for, spins_for, hooks_for) is in the lift cache's key (recomp/gen.py chunk_key); a new kind of line the Emitter reads must go there too.""" - def __init__(self, sets=(), sites=None, page=0, spins=None, hooks=None, ordered=()): + def __init__(self, sets=(), sites=None, page=0, spins=None, hooks=None, ordered=(), plugins=None): self.sets = list(sets) self.sites = sites or {} self.page = page self.spins = spins or {} # {module: {back-edge offset: head offset}} self.hooks = hooks or {} # {module: {function offset}}; empty: FN_ENTER in every function self.ordered = set(ordered) + # {module: {'set', 'sha256', 'image', 'expect': [(module, offset, old, new)], 'stubs': {offset}, + # 'seeds': {offset}, 'origs': {trampoline offset: (function module, function offset)}}}; empty without + # plugin lines + self.plugins = plugins or {} + self.bases = {} # {module: base} of the layout load() read + + def plugin_inputs(self, name): + """What the lift of module name takes from its plugin lines (for the lift cache's key), or None.""" + p = self.plugins.get(name) + if not p: + return None + return [p['image'], p['sha256'], sorted(p['stubs']), sorted(p['seeds']), sorted(p['origs'])] + + def origs_for(self, name, base): + """The absolute addresses of module name's functions that a plugin's `orig` line copies.""" + return {base + fo for p in self.plugins.values() for fm, fo in p['origs'].values() if fm == name} + + def orig_entries(self, name, bases): + """{trampoline address in module name: absolute address of the function it reaches} (`orig`).""" + p = self.plugins.get(name) + if not p: + return {} + return {bases[name] + t: bases[fm] + fo for t, (fm, fo) in p['origs'].items()} def picks(self): """The contested sites, by module name and offset: [(module, offset, [ordered set index in file order])]; the @@ -110,19 +161,30 @@ class Table: """{absolute back-edge address: absolute head address}.""" return {base + b: base + h for b, h in self.spins.get(name, {}).items()} - def for_module(self, name, base): - """{absolute address: [(set index, [words])]} with @page resolved.""" + def for_module(self, name, base, bases=None): + """{absolute address: [(set index, [words])]} with @page resolved; an @call word becomes ('call', absolute + address), which needs bases ({module: base}).""" out = {} + + def word(a, w): + if not isinstance(w, tuple): + return w + if w[0] == 'page': + return adrp(a, self.page, w[1]) + known = bases or self.bases + if w[1] not in known: + raise PatchError(f'{name}+{a - base:#x}: @call into {w[1]}, which is not in the layout') + return ('call', known[w[1]] + w[2]) + for off, alts in self.sites.get(name, {}).items(): a = base + off - out[a] = [(k, [adrp(a, self.page, w[1]) if isinstance(w, tuple) else w for w in repl]) - for k, repl, _ in alts] + out[a] = [(k, [word(a, w) for w in repl]) for k, repl, _ in alts] return out def load(path, mods): """Table from a patch file; mods is layout() output [(name, base, size)].""" - sets, index, sites, spins, hooks, ordered = [], {}, {}, {}, {}, set() + sets, index, sites, spins, hooks, ordered, plugins = [], {}, {}, {}, {}, set(), {} bases = {n: (b, s) for n, b, s in mods} for no, raw in enumerate(Path(path).read_text().splitlines(), 1): f = raw.split('#', 1)[0].split() @@ -146,6 +208,11 @@ def load(path, mods): for w in f[5:]: if w.startswith('@page:x'): repl.append(('page', int(w[7:]))) + elif w.startswith('@call:') and '+' in w: + target, at = w[6:].split('+', 1) + if target not in bases: + raise PatchError(f'{where}: @call into {target}, which is not in the layout') + repl.append(('call', target, int(at, 16))) else: repl.append(int(w, 16)) alts = sites.setdefault(f[2], {}).setdefault(off, []) @@ -163,6 +230,15 @@ def load(path, mods): if b in mod: raise PatchError(f'{where}: {f[1]}+{b:#x} is a back-edge twice') mod[b] = head + elif f[0] == 'plugin' and len(f) == 5: + if f[1] not in index or f[2] not in bases: + raise PatchError(f'{where}: unknown set {f[1]} or module {f[2]}') + if f[2] in plugins: + raise PatchError(f'{where}: module {f[2]} has two plugin lines') + plugins[f[2]] = {'set': index[f[1]], 'sha256': f[3], 'image': f[4], 'expect': [], 'stubs': set(), + 'seeds': set(), 'origs': {}} + elif f[0] in ('expect', 'stub', 'seed', 'orig'): + plugin_line(f, where, index, bases, plugins) elif f[0] == 'hook' and len(f) >= 3: if f[1] not in bases: raise PatchError(f'{where}: module {f[1]} not in the layout') @@ -174,7 +250,36 @@ def load(path, mods): nxt = min((b for _, b, _ in mods if b > base), default=None) if nxt is not None and page + PAGE > nxt: raise PatchError(f'no room for the patch page after main ({page:#x})') - return Table(sets, sites, page, spins, hooks, ordered) + table = Table(sets, sites, page, spins, hooks, ordered, plugins) + table.bases = {n: b for n, (b, _) in bases.items()} + return table + + +def plugin_line(f, where, index, bases, plugins): + """An expect, stub, seed or orig line into the plugin entry it belongs to (its `plugin` line comes first).""" + def plugin(name): + if name not in plugins: + raise PatchError(f'{where}: {name} has no plugin line before this one') + return plugins[name] + + if f[0] == 'expect' and len(f) == 6: + if f[1] not in index or f[2] not in bases: + raise PatchError(f'{where}: unknown set {f[1]} or module {f[2]}') + owner = [p for p in plugins.values() if p['set'] == index[f[1]]] + if not owner: + raise PatchError(f'{where}: set {f[1]} owns no plugin') + owner[0]['expect'].append((f[2], int(f[3], 16), int(f[4], 16), int(f[5], 16))) + elif f[0] == 'stub' and len(f) == 3: + plugin(f[1])['stubs'].add(int(f[2], 16)) + elif f[0] == 'seed' and len(f) >= 3: + plugin(f[1])['seeds'].update(int(o, 16) for o in f[2:]) + elif f[0] == 'orig' and len(f) == 6: + p = plugin(f[2]) + if f[1] not in index or p['set'] != index[f[1]] or f[4] not in bases: + raise PatchError(f'{where}: orig of set {f[1]} for {f[2]}, which another set owns, or module {f[4]}') + p['origs'][int(f[3], 16)] = (f[4], int(f[5], 16)) + else: + raise PatchError(f'{where}: cannot parse {" ".join(f)!r}') def check(table, mod, name): @@ -210,6 +315,20 @@ def header_text(table, marker): '// X(index, name, env switch, default)', '#define RT_PATCH_SETS(X) ' + ' '.join(f'X({i}, {n}, "{e}", {d})' for i, (n, e, d) in enumerate(table.sets)), ''] + if table.plugins: # none: the header is what it was before plugins + names = sorted(table.plugins) + writes = [f'X({i}, "{m}", {o:#x}, {old:#010x}u, {new:#010x}u)' for i, n in enumerate(names) + for m, o, old, new in sorted(table.plugins[n]['expect'])] + lines += [f'#define RT_PLUGIN_COUNT {len(names)}', + '// X(index, module, its set, SHA-256 of its .text after the init)', + '#define RT_PLUGINS(X) ' + ' '.join(f'X({i}, "{n}", {table.plugins[n]["set"]}, ' + f'"{table.plugins[n]["sha256"]}")' for i, n in enumerate(names)), + f'#define RT_PLUGIN_WRITE_COUNT {len(writes)}', + '// X(plugin index, module, offset, old word, new word): what the init writes into game modules', + '#define RT_PLUGIN_WRITES(X) ' + ' '.join(writes), + '// X(module, offset): the hook entries (seed lines) TOTK_PLUGIN_TRACE counts (runtime/plugin_init.c)', + '#define RT_PLUGIN_ROOTS(X) ' + ' '.join(f'X("{n}", {o:#x})' for n in names + for o in sorted(table.plugins[n]['seeds'])), ''] picks = table.picks() if picks: # none: the header is what it was before ordered sets lines += [f'#define RT_PATCH_PICK_COUNT {len(picks)}', From d865000929e9d6bbc067728a9459786b598ac0f4 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 08:48:23 +0300 Subject: [PATCH 05/21] A mod plugin's init runs at every start and its hooks go on when it writes what the build recorded (#845) After the design in #845 by itaskaev2, with one change: no image taken after a build-time run. When a plugin's patch set is on, sdk's DT_INIT runs first, then the plugin's, in the environment of the service run (rtld image below main, the QueryMemory view, the alias pool, process-memory aliases), then the game's constructors. That is rtld's order on the Switch (oss-rtld, librtld/source/ld.cpp __rtld_modules_init runs the modules from the last loaded to the first). sdk's first DT_INIT only registers it with its musl and main's DT_INIT finishes it, so its flag stays 1 and it is not called a second time. The plugin keeps its heap blocks, GOT and alias addresses. The words the init wrote into the game modules and the plugin's .text after it are compared with patches.h: equal, the set stays on; different, it goes off for this run, the reason is logged, the Mods tab row says so and build-cache/plugins.stale has the next start run the rebuild. The branches written into main's .text are taken back. mods.c reads build-cache/plugins.status: a built plugin's set joins its mod's sets and the Mods tab order, a refused one keeps its mod off with the reason (the catalog texts in en, ru, pt-BR, zh-Hans), a plugin without a line waits for the rebuild. A plugin file changed after the build is left out by the loader. TOTK_PLUGIN_TRACE=1 logs the first call of each hook entry. Without a plugin in the build nothing of this runs. --probe-mods covers a plugin set against another mod's patch on one site, a refused plugin, one the build lacks and one whose start differs. Co-Authored-By: Claude Opus 5.5 --- runtime/boot.c | 24 +++- runtime/cfg.c | 2 +- runtime/dispatch.c | 15 +++ runtime/dispatch.h | 1 + runtime/host.h | 5 + runtime/kernel.h | 3 +- runtime/loader.c | 30 ++++- runtime/mods.c | 146 ++++++++++++++++++++-- runtime/mods.h | 13 ++ runtime/mods_probe.c | 94 ++++++++++++++ runtime/plugin_init.c | 279 ++++++++++++++++++++++++++++++++++++++++-- runtime/svc.c | 4 +- runtime/ui/menu.cpp | 22 +++- runtime/ui_text.c | 22 ++++ runtime/ui_text.h | 4 + 15 files changed, 631 insertions(+), 33 deletions(-) diff --git a/runtime/boot.c b/runtime/boot.c index 14028825..d14cc301 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -30,6 +30,7 @@ #define CHECKPOINT_RET (RT_V143 ? 0xbdb48cull : RT_V110 ? 0xeb8c14ull : 0xe7a954ull) #define NVN_RET (RT_V143 ? 0xbe0360ull : RT_V110 ? 0xebcdecull : 0xe7eb2cull) #define CTOR_DONE 2 // a module's constructor flag after its DT_INIT ran the init_array +#define CTOR_WAITING 1 // after a DT_INIT that musl deferred (sdk's, called before main's) // The init_array sizes (1.1.0 and 1.2.1: counted in the modules; 1.4.3: DT_INIT_ARRAYSZ); MacNativeGames' boot check // expects the same. The three tables carry the same rows, so cb_module_inits walks them by the size of one. @@ -40,6 +41,7 @@ static const InitCount inits_143[] = {{"main", 10717}, {"subsdk0", 45}, {"sdk", #define expected_inits (RT_V143 ? inits_143 : RT_V110 ? inits_110 : inits_121) static volatile int32_t inits_done, checkpoint_seen; +static int sdk_first; // a mod plugin ran: sdk's DT_INIT came first (cb_module_inits) uint64_t rt_guest_start_ns; static const RtLoaded *module_named(const char *name) { @@ -78,15 +80,27 @@ static void cb_loader_ready(Cpu *c) { static void cb_module_inits(Cpu *c) { if (plat_cas32(&inits_done, 0, 0)) rt_fatal("boot: module constructors requested twice"); if (rt_config()->plugin_init) plugin_init_run(c, module_named("sdk")); // the plugin service run ends there + // A mod plugin's init runs after sdk's and before the game's (plugin_init.c plugins_start): sdk's DT_INIT then + // comes first, as on the Switch; without a plugin the order stays main, subsdk0, sdk. Called first, sdk's DT_INIT + // only registers sdk with its own musl (__nnmusl_init_dso returns 1: deferred), and main's DT_INIT then finishes + // sdk from musl's list of waiting modules (the log shows sdk's RELRO protected and 44 pages of sdk's data written + // during it on 1.4.3): sdk's flag stays 1 and its DT_INIT is not called again (a second call would protect its + // locked RELRO again, and the SDK aborts). + sdk_first = plugins_start(c, module_named("sdk")); for (unsigned k = 0; k < sizeof inits_121 / sizeof *inits_121; k++) { const RtLoaded *m = module_named(expected_inits[k].name); uint64_t entries = m->init_array_size / 8; if (!m->init) rt_fatal("boot: %s has no DT_INIT", m->name); - rt_log("boot: ENTER %s DT_INIT (%llu init_array entries)", m->name, (unsigned long long)entries); - rt_call(c, m->init, 0); + int deferred = sdk_first && m == module_named("sdk"); + if (!deferred) { + rt_log("boot: ENTER %s DT_INIT (%llu init_array entries)", m->name, (unsigned long long)entries); + rt_call(c, m->init, 0); + } uint32_t state = ctor_state(m); rt_logf("boot: DT_INIT %s state=%u (%llu)\n", m->name, state, (unsigned long long)entries); - if (state != CTOR_DONE) rt_fatal("boot: %s DT_INIT left its constructor flag at %u, not 2", m->name, state); + if (state != (deferred ? CTOR_WAITING : CTOR_DONE)) + rt_fatal("boot: %s DT_INIT left its constructor flag at %u, not %d", m->name, state, + deferred ? CTOR_WAITING : CTOR_DONE); if (entries != expected_inits[k].entries) rt_fatal("boot: %s has %llu init_array entries, this game version has %llu", m->name, (unsigned long long)entries, (unsigned long long)expected_inits[k].entries); @@ -120,7 +134,8 @@ void rt_boot_event(const char *what, Cpu *c, const char *detail) { (unsigned long long)CHECKPOINT_RET); if (!plat_cas32(&inits_done, 0, 0)) rt_fatal("boot: checkpoint %s before DT_INIT", detail); for (int i = 0; i < ldr_count(); i++) // a plugin's init does not run at play time (plugin_init.c) - if (!ldr_module(i)->plugin && ctor_state(ldr_module(i)) != CTOR_DONE) + if (!ldr_module(i)->plugin && ctor_state(ldr_module(i)) != CTOR_DONE && + !(sdk_first && !strcmp(ldr_module(i)->name, "sdk") && ctor_state(ldr_module(i)) == CTOR_WAITING)) rt_fatal("boot: checkpoint %s: %s constructor flag reads %u", detail, ldr_module(i)->name, ctor_state(ldr_module(i))); plat_xchg32(&checkpoint_seen, 1); @@ -364,6 +379,7 @@ void rt_boot(void) { if (!rt_config()->shot) vk_pipeline_async_enable(); // a --shot image must not depend on build timing // Shaders and pipelines recorded in earlier runs are made ready while the modules load; the guest waits for them. PlatThread *warm = plat_thread_new(prewarm_main, NULL, "prewarm"); + plugins_setup(); // a mod plugin's environment (plugin_init.c), before its imports bind ldr_load(rt_config()->exefs); rt_patches_init(); mods_exefs_switch(); // after the patch sets' settings: an enabled mod's ExeFS patches on diff --git a/runtime/cfg.c b/runtime/cfg.c index 5dd06739..921af94d 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -56,7 +56,7 @@ static const char *const known[] = { "MEM_RESERVE_MIB", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_NO_CODE", "MODS_ORDER", "MODS_REBUILD", "MODS_WITHOUT_CODE", "MOTION", "MOUSE_AIM_SCALE", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_ASPECT", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", - "ONE_PER_CORE", "OPERATION_MODE", "OPTREF_GUARD", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCORE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PLUGIN_TICK_OFFSET", "POINTER_IMAGE", + "ONE_PER_CORE", "OPERATION_MODE", "OPTREF_GUARD", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCORE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PLUGIN_TICK_OFFSET", "PLUGIN_TRACE", "POINTER_IMAGE", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_BENCH", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_ASYNC", "REC_END", "REC_SPIN_US", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHUNKS", "REPLAY_CHUNK_DRAWS", "REPLAY_SKEW_US", "REPLAY_STATE_CHECK", "REPLAY_WORKERS", diff --git a/runtime/dispatch.c b/runtime/dispatch.c index b284cf6c..931ea9ac 100644 --- a/runtime/dispatch.c +++ b/runtime/dispatch.c @@ -214,6 +214,21 @@ void rt_dispatch(Cpu *c) { __attribute__((musttail)) return f(c); } +// The plugin trace (plugin_init.c, TOTK_PLUGIN_TRACE): the slot of the function start a runs fn from now on. Once per +// traced function, before the game's code runs; the hot path stays as it is. 0 when a is no function start of the +// table or fn lies out of the reach of a slot's 32-bit offset. +int rt_dispatch_reroute(uint64_t a, GuestFn fn) { + uint64_t i = (a - D.lo) >> 2; + if (!g_ready || (a & 3) || a < D.lo || i >= D.slots || !D.slot[i]) return 0; + uintptr_t f = (uintptr_t)fn; + if (f <= D.anchor || f - D.anchor > UINT32_MAX) return 0; + uint64_t page = (uint64_t)(uintptr_t)&D.slot[i] & ~0xFFFull; + if (!plat_mem_protect(page, 0x1000, PLAT_PROT_RW)) return 0; + ((uint32_t *)D.slot)[i] = (uint32_t)(f - D.anchor); // the table is sealed read-only, unsealed above + plat_mem_protect(page, 0x1000, PLAT_PROT_READ); + return 1; +} + RtDispatchStats rt_dispatch_stats(void) { return (RtDispatchStats){D.lo, D.lo + D.slots * 4, D.slots, g_table_bytes, g_functions, g_nhost}; } diff --git a/runtime/dispatch.h b/runtime/dispatch.h index 81165b17..1f224f2e 100644 --- a/runtime/dispatch.h +++ b/runtime/dispatch.h @@ -38,6 +38,7 @@ void rt_dispatch_init(const RtModule *mods, int count); // The host function for a guest address, or NULL when nothing is there. Does not run guest code. GuestFn rt_dispatch_lookup(uint64_t pc); +int rt_dispatch_reroute(uint64_t a, GuestFn fn); // plugin trace only (dispatch.c); 0 when it cannot // "0x7000001234 (main+0x1234)" for a guest address inside a module, else the bare address. Returns buf. const char *rt_guest_name(uint64_t pc, char *buf, int size); diff --git a/runtime/host.h b/runtime/host.h index 3d84371c..d4f8c060 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -384,6 +384,11 @@ void rt_boot(void); // boot.c void rt_boot_plugin_init(void); // boot.c: --plugin-init, the plugin service run void plugin_init_setup(void); // plugin_init.c: before ldr_load void plugin_init_run(Cpu *c, const RtLoaded *sdk); // plugin_init.c: from the module-init callback; exits +void plugins_setup(void); // plugin_init.c: rt_boot, before ldr_load; nothing + // without a plugin in the build +int plugins_start(Cpu *c, const RtLoaded *sdk); // plugin_init.c: from the module-init callback, before + // the game's constructors; 1 when it ran sdk's DT_INIT + // (a plugin's init ran after it) // Modes of totk.exe. main.c has run mem_init and parsed the paths; each mode calls ldr_load itself if it needs the // modules, registers its host functions, then seals (hostfn_seal, or implicitly at the first rt_call). Return the // process exit code. diff --git a/runtime/kernel.h b/runtime/kernel.h index d232f68d..1114758b 100644 --- a/runtime/kernel.h +++ b/runtime/kernel.h @@ -41,6 +41,7 @@ extern void (*volatile svc_test_exit_hook)(unsigned code); // svc 0x07 calls i // ---- mod plugin service run (plugin_init.c sets these; the game calls none of the SVCs they serve) extern uint64_t rt_svc_tick_offset; // added to GetSystemTick (0x1E): the service run's second run picks other ticks -extern void (*rt_plugin_svc)(Cpu *c, uint32_t imm, uint64_t pc); // with RT_TRACE_PLUGIN: sees each SVC first +extern int (*rt_plugin_svc)(Cpu *c, uint32_t imm, uint64_t pc); // with RT_TRACE_PLUGIN: sees each SVC + // first; 1: refused, the result is in x0 and the SVC does not run extern uint32_t (*rt_process_memory)(int unmap, uint64_t dst, uint32_t process, uint64_t src, uint64_t size); // Map/UnmapProcessMemory (0x74/0x75); NULL outside the service run: fatal diff --git a/runtime/loader.c b/runtime/loader.c index 4e7f833b..6c698ffb 100644 --- a/runtime/loader.c +++ b/runtime/loader.c @@ -86,6 +86,26 @@ static uint8_t *read_path_verified(const char *path, const char *want, size_t *l return p; } +// 1 when the plugin file of lay is the one the build lifted; logged when not. +static int plugin_file_ok(const RtLayoutEntry *lay) { + size_t len; + uint8_t *p = read_file(lay->file, &len); + if (!p) { + rt_log("loader: %s: cannot read %s; the plugin is left out", lay->name, lay->file); + return 0; + } + uint8_t d[32]; + char hex[65]; + rt_sha256(p, len, d); + free(p); + hex32(d, hex); + if (strcmp(hex, lay->sha256)) { + rt_log("loader: %s: %s changed since the build lifted it; the plugin is left out", lay->name, lay->file); + return 0; + } + return 1; +} + uint8_t *ldr_read_verified(const char *dir, const char *file, size_t *len) { const char *want = NULL; for (int i = 0; i < rt_exefs_sha256_count; i++) @@ -484,8 +504,14 @@ void ldr_load(const char *exefs_dir) { if (nmods) rt_fatal("ldr_load called twice"); if (!mem_layout()->pool) rt_fatal("ldr_load before mem_init"); if (rt_layout_count > MOD_COUNT) rt_fatal("loader: %d modules in the layout", rt_layout_count); - for (int i = 0; i < rt_layout_count; i++) load_module(&mods[i], exefs_dir, &rt_layout[i]); - nmods = rt_layout_count; + int n = 0; + for (int i = 0; i < rt_layout_count; i++) { + // A mod plugin's file that is gone or changed since the build is left out at play time: its patch set cannot + // run (plugin_init.c plugins_start keeps it off); the plugin service run needs it and stops. + if (rt_layout[i].file && !rt_config()->plugin_init && !plugin_file_ok(&rt_layout[i])) continue; + load_module(&mods[n++], exefs_dir, &rt_layout[i]); + } + nmods = n; ovr_used = calloc(hostfn_table_size(0) + 1, sizeof(int)); obs_used = calloc(hostfn_table_size(1) + 1, sizeof(int)); for (int i = 0; i < nmods; i++) { diff --git a/runtime/mods.c b/runtime/mods.c index 3f99e8dd..5d7667da 100644 --- a/runtime/mods.c +++ b/runtime/mods.c @@ -26,9 +26,14 @@ // .bss) are lines of build-cache/exefs.data, stored into the loaded main before the game runs (mods_exefs_data), as // emulators apply a patch file to the module before its dynamic linker runs. // -// Code modules: an exefs/subsdk* file is a plugin that patches main from its own start-up on the Switch, and a replaced -// exefs/main is another game code; the port cannot load either (ModInfo.code names them): their mod stays off unless -// it is used without its code, which also leaves out the ExeFS patch files the port cannot apply. +// Code modules: an exefs/subsdk* file is a plugin that patches main from its own start-up on the Switch (exlaunch). +// The setup builds it in (tools/exefs_plugins.py, after the design in #845 by itaskaev2) and writes its line into +// build-cache/plugins.status (the format of exefs.status): `plugin`, its patch set mod_ holds its hook sites and +// its init runs at every start (plugin_init.c); `unsupported`, the note says why the build refused it. A plugin without +// a line waits for the rebuild like an ExeFS patch file; one whose start differed from the build is listed in +// build-cache/plugins.stale (mods_plugin_off) and waits for it too. A replaced exefs/main is another game code the port +// cannot load. A code module the port cannot run (ModInfo.code names them) keeps its mod off unless it is used without +// its code, which also leaves out the ExeFS patch files the port cannot apply. #include #include #include @@ -92,6 +97,8 @@ static void code_unrun(ModInfo *m, const char *name) { snprintf(m->code + k, sizeof m->code - k, "%s%s", k ? ", " : "", name); } +static void plugin_file(ModInfo *m, const char *leaf); + // What the ExeFS folder holds: MOD_EXEFS for patches (.ips, .pchtxt), MOD_SUBSDK for code modules (subsdk* plugins, a // replaced main). main.npdm alone is the process's metadata, not code: it counts as neither. static int exefs_cb(const char *name, int kind, uint64_t size, void *ctx) { @@ -100,7 +107,10 @@ static int exefs_cb(const char *name, int kind, uint64_t size, void *ctx) { if (kind != PLAT_FS_FILE) return 1; if (ends_with_ci(name, ".ips") || ends_with_ci(name, ".pchtxt")) { m->kind |= MOD_EXEFS; - } else if (!strncasecmp(name, "subsdk", 6) || !strcasecmp(name, "main")) { + } else if (!strncasecmp(name, "subsdk", 6)) { + m->kind |= MOD_SUBSDK; + plugin_file(m, name); + } else if (!strcasecmp(name, "main")) { m->kind |= MOD_SUBSDK; code_unrun(m, name); } @@ -202,6 +212,7 @@ static int same_build(const char *hex, size_t n) { // build-cache/ next to totk.exe, read once into *text; "" when there is none (mods_probe_cache sets the text). static char *status_text, *data_text; +static int no_stale_file; // --probe-mods: plugins.stale stays in memory static const char *build_cache(const char *name, char **text) { if (*text) return *text; char path[2100]; @@ -217,14 +228,17 @@ static const char *build_cache(const char *name, char **text) { return *text; } +static char *plugin_status_text, *plugin_stale_text; static const char *exefs_status(void) { return build_cache("exefs.status", &status_text); } static const char *exefs_data(void) { return build_cache("exefs.data", &data_text); } +static const char *plugin_status(void) { return build_cache("plugins.status", &plugin_status_text); } +static const char *plugin_stale(void) { return build_cache("plugins.stale", &plugin_stale_text); } -// The status line of key: its kind, built-in sets and note, NUL-terminated copies; 0 when none. -static int status_line(const char *key, const char *content, char *kind, char *over, char *note) { +// The line of key in a status text: its kind, built-in sets and note, NUL-terminated copies; 0 when none. +static int status_in(const char *text, const char *key, const char *content, char *kind, char *over, char *note) { char want[20]; snprintf(want, sizeof want, "%s\t%s\t", key, content); - for (const char *line = exefs_status(); *line;) { + for (const char *line = text; *line;) { const char *end = strchr(line, '\n'); size_t len = end ? (size_t)(end - line) : strlen(line); if (len > 18 && !memcmp(line, want, 18)) { @@ -246,6 +260,69 @@ static int status_line(const char *key, const char *content, char *kind, char *o return 0; } +static int status_line(const char *key, const char *content, char *kind, char *over, char *note) { + return status_in(exefs_status(), key, content, kind, over, note); +} + +// The reason plugins.stale gives for key and content, or NULL. +static const char *stale_reason(const char *key, const char *content, char *out, size_t cap) { + char want[20]; + snprintf(want, sizeof want, "%s\t%s\t", key, content); + for (const char *line = plugin_stale(); *line;) { + const char *end = strchr(line, '\n'); + size_t len = end ? (size_t)(end - line) : strlen(line); + if (len > 18 && !memcmp(line, want, 18)) { + size_t n = len - 18; + if (n && line[len - 1] == '\r') n--; + snprintf(out, cap, "%.*s", (int)(n < cap ? n : cap - 1), line + 18); + return out; + } + line = end ? end + 1 : line + len; + } + return NULL; +} + +// One exefs/subsdk* file of m: a built plugin joins its patch sets, a refused one is code it cannot run, one the build +// lacks (or whose start differed from the build) waits for the rebuild. +static void plugin_file(ModInfo *m, const char *leaf) { + char path[4300], rel[600]; + snprintf(path, sizeof path, "%s/%s", m->exefs, leaf); + size_t root = strlen(dir) + 1 + strlen(m->name) + 1; // m->exefs is // + snprintf(rel, sizeof rel, "%s/%s/%s", m->name, strlen(m->exefs) > root ? m->exefs + root : "exefs", leaf); + uint64_t n = 0; + char *data = (char *)plat_file_read(path, &n); + if (!data) { + rt_log("mods: %s: cannot read %s", m->name, path); + code_unrun(m, leaf); + return; + } + char key[9], content[9], kind[16], over[256], note[256], stale[256]; + snprintf(key, sizeof key, "%08X", fnv(rel, strlen(rel))); + snprintf(content, sizeof content, "%08X", fnv(data, (size_t)n)); + free(data); + if (!status_in(plugin_status(), key, content, kind, over, note) || stale_reason(key, content, stale, sizeof stale)) { + m->pending++; + m->pending_key = (m->pending_key ^ fnv(key, 8)) * 0x01000193u; + m->pending_key = (m->pending_key ^ fnv(content, 8)) * 0x01000193u; + return; + } + if (!strcmp(kind, "plugin") && m->nsets < (int)(sizeof m->sets / sizeof *m->sets) && + m->nplugins < (int)(sizeof m->plugin_keys / sizeof *m->plugin_keys)) { + uint32_t k = (uint32_t)strtoul(key, NULL, 16); + m->sets[m->nsets++] = k; + m->plugin_keys[m->nplugins] = k; + m->plugin_contents[m->nplugins++] = (uint32_t)strtoul(content, NULL, 16); + const char *hooks = strstr(note, ", "); + m->plugin_hooks += hooks ? atoi(hooks + 2) : 0; + size_t at = strlen(m->overrides); + if (*over) snprintf(m->overrides + at, sizeof m->overrides - at, "%s%s", at ? "," : "", over); + } else { + code_unrun(m, leaf); + if (!m->plugin_note[0]) snprintf(m->plugin_note, sizeof m->plugin_note, "%s", note); + } + if (!m->plugin_file[0]) snprintf(m->plugin_file, sizeof m->plugin_file, "%s", leaf); +} + typedef struct { ModInfo *m; const char *folder, *rel; // the folder listed and its path in the mod @@ -592,6 +669,14 @@ void mods_probe_cache(const char *status, const char *data) { data_text = dup_text(data); } +void mods_probe_plugins(const char *status, const char *stale) { + free(plugin_status_text); + free(plugin_stale_text); + plugin_status_text = dup_text(status); + plugin_stale_text = dup_text(stale); + no_stale_file = status != NULL; +} + void mods_rescan(void) { free(mods); mods = NULL; @@ -770,6 +855,53 @@ void mods_exefs_apply(uint64_t base, uint64_t size, uint64_t code_end, code_conflicts(); } +// build-cache/plugins.stale next to totk.exe: one more line (--probe-mods keeps it in memory). +static void stale_append(const char *line) { + size_t had = plugin_stale_text ? strlen(plugin_stale_text) : 0, n = strlen(line); + char *grown = realloc(plugin_stale_text, had + n + 1); + if (grown) { + if (!plugin_stale_text) grown[0] = 0; + memcpy(grown + had, line, n + 1); + plugin_stale_text = grown; + } + char path[2100]; + if (no_stale_file || !plat_exe_path(path, sizeof path)) return; + char *cut = strrchr(path, '/'), *back = strrchr(path, '\\'); + if (back > cut) cut = back; + if (!cut || snprintf(cut + 1, sizeof path - (size_t)(cut + 1 - path), "build-cache/plugins.stale") >= + (int)(sizeof path - (size_t)(cut + 1 - path))) + return; + FILE *f = fopen(path, "ab"); + if (!f) { + rt_log("mods: cannot write %s", path); + return; + } + fputs(line, f); + fclose(f); +} + +void mods_plugin_off(int set, const char *why) { + const ModInfo *owner = set_owner(set); + rt_patch_switch(set, 0); + if (!owner) return; + ModInfo *m = (ModInfo *)owner; + snprintf(m->plugin_off, sizeof m->plugin_off, "%s", why); + const char *name = rt_patch_name(set); + uint32_t key = name ? (uint32_t)strtoul(name + 4, NULL, 16) : 0; + for (int k = 0; k < m->nplugins; k++) { + if (m->plugin_keys[k] != key) continue; + char line[400]; + snprintf(line, sizeof line, "%08X\t%08X\t%s\n", key, m->plugin_contents[k], why); + stale_append(line); + } + rt_log("mods: %s: its plugin stays off this run (%s); it waits for the rebuild at the next start", m->name, why); + int kept = 0; // the code conflicts counted anew without the set + for (int i = 0; i < nconflicts; i++) + if (conflicts[i].part != MOD_PART_CODE) conflicts[kept++] = conflicts[i]; + nconflicts = kept; + code_conflicts(); +} + void mods_exefs_switch(void) { mods_scan(); const RtLoaded *main_mod = NULL; diff --git a/runtime/mods.h b/runtime/mods.h index 466fde80..8fb4f7d6 100644 --- a/runtime/mods.h +++ b/runtime/mods.h @@ -53,6 +53,13 @@ typedef struct { uint32_t data_bytes; // the bytes its usable ExeFS patch files write into main's data, as their // status lines count them ("data ") char overrides[256]; // the port's own patch sets its patches replace, comma separated + // Its exefs/subsdk* plugins (build-cache/plugins.status, tools/exefs_plugins.py): the built ones are patch sets in + // sets[] whose init runs at start (plugin_init.c); a refused one is a code module it cannot run (code). + int nplugins, plugin_hooks; // built plugins and their hooks + uint32_t plugin_keys[4], plugin_contents[4]; // their file and content hashes, for plugins.stale + char plugin_file[64]; // the first plugin's file name + char plugin_note[256]; // why the build refused a plugin ("" none) + char plugin_off[256]; // why its plugin stays off this run: its start differed from the build } ModInfo; // What two mods both change, the higher one winning: files (romfs.c), bytes of main's data and code sites @@ -105,5 +112,11 @@ void mods_rescan(void); // The ExeFS patch files of the enabled mods that the build lacks, their FNV-1a mixed into *key (cheats.c starts the // rebuild for them). int mods_exefs_pending(uint32_t *key); +// From plugin_init.c plugins_start: the plugin patch set `set` stays off this run (why), its mod's row says so, +// build-cache/plugins.stale marks its file for the rebuild at the next start, and the code conflicts are counted anew. +void mods_plugin_off(int set, const char *why); +// --probe-mods: the texts of build-cache/plugins.status and plugins.stale instead of the files; the stale lines +// mods_plugin_off adds stay in memory. +void mods_probe_plugins(const char *status, const char *stale); // The hex of the game main's build id (64 digits), "" when exefs/main cannot be read. const char *mods_main_build_id(void); diff --git a/runtime/mods_probe.c b/runtime/mods_probe.c index cc223bfc..473ea1e5 100644 --- a/runtime/mods_probe.c +++ b/runtime/mods_probe.c @@ -197,6 +197,97 @@ static void probe_code(const char *root) { remove_tree(root); } +// build-cache/plugins.status lines (tools/exefs_plugins.py) that refuse the probe's subsdk9 files of E and H: they stay +// code the port cannot run. +static const char *refused_plugins(void) { + static char text[256]; + snprintf(text, sizeof text, "%08X\t%08X\tunsupported\t-\tthe init called svc 0x40\n" + "%08X\t%08X\tunsupported\t-\tthe init called svc 0x40\n", fnv("E/exefs/subsdk9", 15), fnv("probe", 5), + fnv("H/exefs/subsdk9", 15), fnv("probe", 5)); + return text; +} + +// Mod plugins (build-cache/plugins.status, plugins.stale; mods.c plugin_file, mods_plugin_off) in a folder of their own: +// P RomFS + a built plugin (7 hooks) its set on, ranked by the order; a contested site with Q's patch file +// Q an ExeFS patch file on P's site +// R a plugin the build has no line for waits for the rebuild +// S RomFS + a refused plugin off, the reason kept; used without its code, the rest runs +// Then P's start differs from the build (mods_plugin_off): its set goes off, Q's code runs at the site, the conflict +// goes, a stale line marks P's file, and the next scan has P wait for the rebuild. +static void probe_plugins(const char *root) { + remove_tree(root); + plat_file_mkdir_p(root); + const char *bid = *mods_main_build_id() ? mods_main_build_id() : "0"; + char q[160]; + snprintf(q, sizeof q, "Q/exefs/%s.ips", bid); + static const char *const files[] = {"P/romfs/p.bin", "P/exefs/subsdk9", "R/exefs/subsdk9", "S/romfs/s.bin", + "S/exefs/subsdk9"}; + for (size_t i = 0; i < sizeof files / sizeof *files; i++) put(root, files[i]); + put(root, q); + static char names[3][16]; + static const char *const table[] = {"fps60", names[1], names[2]}; + set_name(names[1], sizeof names[1], "P/exefs/subsdk9"); + set_name(names[2], sizeof names[2], q); + char status[256], plugins[256]; + snprintf(status, sizeof status, "%08X\t%08X\tcode\t-\t\n", fnv(q, strlen(q)), fnv("probe", 5)); + snprintf(plugins, sizeof plugins, "%08X\t%08X\tplugin\tfps60\tplugin0, 7 hooks\n" + "%08X\t%08X\tunsupported\t-\tthe code its hooks reach has svc 0x40\n", fnv("P/exefs/subsdk9", 15), + fnv("probe", 5), fnv("S/exefs/subsdk9", 15), fnv("probe", 5)); + static const int16_t s0[] = {1, 2}; + static const RtPatchPick picks[] = {{"main", 0x100, s0, 2}}; + mods_probe_cache(status, ""); + mods_probe_plugins(plugins, ""); + cfg_set("TOTK_MODS_DIR", root); + cfg_set("TOTK_MODS_DISABLED", ""); + cfg_set("TOTK_MODS_NO_CODE", ""); + for (int round = 0; round < 2; round++) { + cfg_set("TOTK_MODS_ORDER", round ? "Q|P" : "P|Q"); + mods_rescan(); + mods_scan(); + rt_patch_probe_table(table, 3, picks, 1); + rt_patch_switch(0, 1); + mods_exefs_apply(0, 0, 0, NULL); + const ModInfo *pm = by_name("P"), *qm = by_name("Q"), *rm = by_name("R"), *sm = by_name("S"); + CHECK(pm && qm && rm && sm, "plugin scan: four mods"); + if (!pm || !qm || !rm || !sm) break; + CHECK(pm->problem == MOD_OK && pm->nsets == 1 && pm->nplugins == 1 && pm->plugin_hooks == 7 && !pm->code[0] && + EQ(pm->overrides, "fps60"), "a built plugin: its set, its hooks, the built-in set it replaces"); + CHECK(rt_patch_enabled(1) && !rt_patch_enabled(0), "the plugin's set on, the built-in set it replaces off"); + CHECK(rt_patch_pick_winner(0) == (round ? 1 : 0), round ? "Q above P: Q's patch runs at their site" + : "P above Q: P's plugin hook runs at their site"); + CHECK(mods_conflict_count() == 1 && mods_conflict_at(0)->part == MOD_PART_CODE && + mods_conflict_at(0)->winner == (round ? qm : pm)->id, + "the conflict table: a plugin site against a patch file goes by the order"); + CHECK(rm->problem == MOD_EXEFS_LATER && rm->pending == 1, "a plugin the build lacks waits for the rebuild"); + CHECK(sm->problem == MOD_PLUGIN && EQ(sm->code, "subsdk9") && strstr(sm->plugin_note, "svc 0x40"), + "a refused plugin keeps its mod off and says why"); + } + // P above Q, then P's start differs from the build + cfg_set("TOTK_MODS_ORDER", "P|Q"); + mods_rescan(); + mods_scan(); + rt_patch_probe_table(table, 3, picks, 1); + mods_exefs_apply(0, 0, 0, NULL); + mods_plugin_off(1, "probe: 1 of the words the build recorded were not written"); + const ModInfo *pm = by_name("P"); + CHECK(!rt_patch_enabled(1) && rt_patch_pick_winner(0) == 1, "a plugin off at start: the other mod's patch runs"); + CHECK(pm && strstr(pm->plugin_off, "were not written"), "its row says why"); + CHECK(mods_conflict_count() == 0, "its conflict goes with it"); + cfg_set("TOTK_MODS_NO_CODE", "S"); + mods_rescan(); + mods_scan(); + pm = by_name("P"); + const ModInfo *sm = by_name("S"); + CHECK(pm && pm->problem == MOD_EXEFS_LATER && pm->pending == 1, "the next start: its file waits for the rebuild"); + CHECK(sm && sm->problem == MOD_OK && sm->plugin_note[0], "a refused plugin used without its code: the rest runs"); + rt_patch_probe_table(NULL, 0, NULL, 0); + cfg_set("TOTK_MODS_NO_CODE", ""); + cfg_set("TOTK_MODS_ORDER", ""); + mods_probe_cache(NULL, NULL); + mods_probe_plugins(refused_plugins(), ""); + remove_tree(root); +} + // Mods with code the port cannot run (a code module, an ExeFS patch file it cannot apply): used without that code // by their own switch (mods_no_code) or by mods_without_code for every mod. static void probe_without_code(const char *root) { @@ -292,6 +383,7 @@ int probe_mods(void) { if (back > cut) cut = back; if (!cut) return 1; strcpy(cut + 1, "mods-probe"); + mods_probe_plugins(refused_plugins(), ""); // not the build's own plugins.status remove_tree(root); plat_file_mkdir_p(root); static const char *const files[] = { @@ -362,6 +454,8 @@ int probe_mods(void) { probe_data(root); probe_without_code(root); probe_code(root); + probe_plugins(root); + mods_probe_plugins(NULL, NULL); printf("mods: %s\n", fails ? "FAIL" : "ok"); int table = restbl_probe(); return fails || table ? 1 : 0; diff --git a/runtime/plugin_init.c b/runtime/plugin_init.c index 96486584..227f6621 100644 --- a/runtime/plugin_init.c +++ b/runtime/plugin_init.c @@ -1,8 +1,23 @@ -// The plugin service run: totk.exe --plugin-init DIR runs the DT_INIT of every mod code module (recomp/layout.py -// plugin, an exlaunch plugin such as a mod's exefs/subsdk9) on this runtime's own kernel and sdk, records what the -// init did and exits before any game module's constructor, nnMain, graphics or audio. The record is the input of -// tools/exefs_plugins.py (checks) and, later, of the generator's hook sites; at play time a plugin's init does not run. -// The design follows #845 by itaskaev2, with the init run natively instead of under Unicorn. +// Mod plugins' inits (recomp/layout.py plugin, an exlaunch plugin such as a mod's exefs/subsdk9), run natively on +// this runtime's own kernel and sdk. The design follows #845 by itaskaev2, with the init run natively instead of under +// Unicorn, and run again at every start instead of an image taken after one run at build time. +// +// The service run: totk.exe --plugin-init DIR runs every plugin's DT_INIT, records what the init did and exits before +// any game module's constructor, nnMain, graphics or audio. The record is the input of tools/exefs_plugins.py, which +// checks it and writes the plugin's patch set (its hook sites, recomp/patches.py `plugin` lines). +// +// At every start (plugins_start, from the module-init callback in boot.c): when the build has a plugin whose patch set +// its mod switched on, sdk's DT_INIT runs first, then the plugin's, in the same environment as in the service run, and +// then the game's constructors. That is the order of the Switch's rtld, which runs the modules' DT_INITs from the last +// loaded to the first (oss-rtld, librtld/source/ld.cpp __rtld_modules_init walks g_pAutoLoadList backwards; the list is +// filled in address order by __rtld_relocate_modules in librtld/source/relocation.cpp: main, subsdk*, sdk), so sdk +// runs before a subsdk plugin and the plugin before the game's main. The plugin keeps what its init made: heap blocks, +// its GOT, its alias addresses. What the init wrote into the game modules (its hook branches) and the plugin's .text +// after it (its stubs and trampolines) are compared with the record the build used (patches.h RT_PLUGIN_WRITES and +// RT_PLUGINS): equal, the plugin's set stays on and the generated hook sites call into it; different, the set goes off +// for this run, the reason is logged and the mod waits for a rebuild (mods.c mods_plugin_off). The branches written +// into main's .text are taken back afterwards: the lifted code is the code that runs, and main keeps the game's bytes. +// Without a plugin in the build none of this runs (RT_PLUGIN_COUNT 0). // // What exlaunch's init needs from the process, and how this run gives it: // - Its module walk (exl::util::FindModules) reads QueryMemory from 0 up and takes every Code RX, Code R, CodeData @@ -31,10 +46,19 @@ #include #include "cfg.h" +#include "dispatch.h" #include "host.h" #include "kernel.h" +#include "mods.h" #include "platform/platform.h" +#if __has_include("patches.h") +#include "patches.h" +#endif +#ifndef RT_PLUGIN_COUNT +#define RT_PLUGIN_COUNT 0 +#endif + #define PAGE 0x1000ull #define GiB (1ull << 30) #define POOL_SIZE (64 * GiB) @@ -63,6 +87,8 @@ static int naliases; static uint64_t rtld_base, rtld_size, rtld_text_end, rtld_data_off; static uint64_t pool_base; static int running; // a plugin's init runs: svcs and imports are counted +static int play; // plugins_start: a refused SVC fails the plugin instead of ending the process +static char failed[256]; // play: why the running init was refused static uint64_t svc_count[0x80], info_count[16], info_65001; static struct { const char *name; uint64_t calls; } imports[MAX_IMPORTS]; static int nimports; @@ -187,34 +213,44 @@ static void write_record(const char *refused); static void refuse(const char *why) { rt_log("plugin-init: refused: %s", why); + if (play) { // the init goes on with an error result; the plugin stays off (plugins_start) + if (!failed[0]) snprintf(failed, sizeof failed, "%s", why); + return; + } write_record(why); fflush(NULL); plat_exit(3); } -static void plugin_svc(Cpu *c, uint32_t imm, uint64_t pc) { - if (!running) return; +#define R_NOT_IMPLEMENTED 0xF601u + +static int plugin_svc(Cpu *c, uint32_t imm, uint64_t pc) { + if (!running) return 0; svc_count[imm & 0x7F]++; char why[256], at[128]; switch (imm) { - case 0x06: case 0x1E: case 0x27: case 0x74: case 0x75: return; + case 0x06: case 0x1E: case 0x27: case 0x74: case 0x75: return 0; case 0x29: { uint32_t id = (uint32_t)c->x[1]; if ((id >= 2 && id <= 5) || (id >= 12 && id <= 15)) { info_count[id]++; - return; + return 0; } if (id == 65001) { info_65001++; - return; + return 0; } snprintf(why, sizeof why, "GetInfo id %u at %s", id, ldr_describe(pc, at, sizeof at)); refuse(why); + c->x[0] = R_NOT_IMPLEMENTED; + return 1; } } snprintf(why, sizeof why, "svc %#x at %s (only QueryMemory, GetSystemTick, OutputDebugString, GetInfo 2-5, 12-15 " "and 65001, Map/UnmapProcessMemory are served)", imm, ldr_describe(pc, at, sizeof at)); refuse(why); + c->x[0] = R_NOT_IMPLEMENTED; + return 1; } // Every JUMP_SLOT import of a plugin binds here (loader.c resolve): counted by name, then the guest definition runs, @@ -363,14 +399,25 @@ static void write_record(const char *refused) { } // ---- setup and run -// Before ldr_load: the import observer, the svc hooks, the pool and the stand-in for rtld. -void plugin_init_setup(void) { +static int layout_plugins(void) { int plugins = 0; for (int i = 0; i < rt_layout_count; i++) plugins += rt_layout[i].file != NULL; - if (!plugins) rt_fatal("plugin-init: this build's layout has no plugin module (recomp/gen.py --plugins)"); + return plugins; +} + +static void environment(void); + +// Before ldr_load: the import observer, the svc hooks, the pool and the stand-in for rtld. +void plugin_init_setup(void) { + if (!layout_plugins()) rt_fatal("plugin-init: this build's layout has no plugin module (recomp/gen.py --plugins)"); plat_file_mkdir_p(rt_config()->plugin_init); const char *off = cfg_get("TOTK_PLUGIN_TICK_OFFSET"); rt_svc_tick_offset = off ? strtoull(off, NULL, 0) : 0; + environment(); +} + +// The environment of a plugin's init, the same in the service run and at every start. +static void environment(void) { ldr_plugin_import = plugin_import; rt_process_memory = process_memory; rt_plugin_svc = plugin_svc; @@ -392,6 +439,13 @@ void plugin_init_setup(void) { (unsigned long long)rt_svc_tick_offset); } +// rt_boot, before ldr_load: the plugins' environment when the build has one (nothing otherwise). +void plugins_setup(void) { + if (!RT_PLUGIN_COUNT || !layout_plugins()) return; + play = 1; + environment(); +} + // From nn::init::Start's module-init callback (boot.c), in place of the game's constructors. rtld initialises the // modules from the last loaded to the first, so on the Switch sdk's DT_INIT runs before a subsdk plugin's; then each // plugin's, the last first. Exits with the record written. @@ -427,3 +481,202 @@ void plugin_init_run(Cpu *c, const RtLoaded *sdk) { fflush(NULL); plat_exit(0); } + +// ---- at every start +#if RT_PLUGIN_COUNT +static const struct { const char *module; int set; const char *sha256; } plugin_tab[] = { +#define X(i, mod, set, sha) {mod, set, sha}, + RT_PLUGINS(X) +#undef X +}; +static const struct { int plugin; const char *module; uint64_t off; uint32_t old, now; } write_tab[] = { +#define X(p, mod, off, old, now) {p, mod, off, old, now}, + RT_PLUGIN_WRITES(X) +#undef X +}; +#define NWRITES ((int)(sizeof write_tab / sizeof *write_tab)) +#ifdef RT_PLUGIN_ROOTS +static const struct { const char *module; uint64_t off; } root_tab[] = { +#define X(mod, off) {mod, off}, + RT_PLUGIN_ROOTS(X) +#undef X +}; +#define NROOTS ((int)(sizeof root_tab / sizeof *root_tab)) +#else +#define NROOTS 0 +#endif + +static const RtLoaded *loaded(const char *name) { + for (int i = 0; i < ldr_count(); i++) + if (!strcmp(ldr_module(i)->name, name)) return ldr_module(i); + return NULL; +} + +// TOTK_PLUGIN_TRACE = 1: the first call of each hook entry (patches.h RT_PLUGIN_ROOTS: callbacks, replace and call +// targets) is logged with its count so far; the entries run through count_entry, their dispatch slots rerouted once +// at start (dispatch.c rt_dispatch_reroute). The calls inside the plugin are direct and not counted. +static struct { uint64_t addr; GuestFn fn; uint64_t calls; } traced[64]; +static int ntraced; + +static void count_entry(Cpu *c) { + for (int i = 0; i < ntraced; i++) { + if (traced[i].addr != c->pc) continue; + if (__atomic_add_fetch(&traced[i].calls, 1, __ATOMIC_RELAXED) == 1) { + char at[128]; + rt_log("plugins: hook entry %s reached (first call)", ldr_describe(c->pc, at, sizeof at)); + } + __attribute__((musttail)) return traced[i].fn(c); + } + rt_fatal("plugins: traced entry %#llx without its function", (unsigned long long)c->pc); +} + +static void trace_entries(void) { +#ifdef RT_PLUGIN_ROOTS + for (int i = 0; i < NROOTS && ntraced < 64; i++) { + const RtLoaded *m = loaded(root_tab[i].module); + if (!m) continue; + uint64_t a = m->base + root_tab[i].off; + GuestFn f = rt_dispatch_lookup(a); + if (!f) continue; + traced[ntraced] = (typeof(traced[0])){a, f, 0}; + if (rt_dispatch_reroute(a, count_entry)) ntraced++; + else rt_log("plugins: trace: cannot reroute %#llx", (unsigned long long)a); + } +#endif + rt_log("plugins: trace: %d hook entries counted", ntraced); +} + +// Takes back the words the inits changed in the sealed part (.text, .rodata) of a game module: main keeps the game's +// own bytes, the lifted code being what runs. +static void restore_sealed(const RtLoaded *m, const uint8_t *was) { + for (uint64_t off = 0; off < m->data_off; off += PAGE) { + const uint8_t *now = (const uint8_t *)(uintptr_t)(m->base + off); + if (!memcmp(now, was + off, PAGE)) continue; + if (!plat_mem_protect(m->base + off, PAGE, PLAT_PROT_RW)) rt_fatal("plugins: cannot unseal %s+%#llx", m->name, + (unsigned long long)off); + memcpy((void *)(uintptr_t)(m->base + off), was + off, PAGE); + plat_mem_protect(m->base + off, PAGE, PLAT_PROT_READ); + } +} +#endif + +int plugins_start(Cpu *c, const RtLoaded *sdk) { +#if RT_PLUGIN_COUNT + if (!play) return 0; + int run[RT_PLUGIN_COUNT] = {0}, nrun = 0; + char why[RT_PLUGIN_COUNT][256]; + for (int i = 0; i < RT_PLUGIN_COUNT; i++) { + why[i][0] = 0; + const RtLoaded *m = loaded(plugin_tab[i].module); + if (!rt_patch_enabled(plugin_tab[i].set)) continue; + if (!m) { // loader.c left it out (its file is gone or changed): its set cannot run + mods_plugin_off(plugin_tab[i].set, "its file is not the one the build lifted"); + continue; + } + if (!m->init) { + mods_plugin_off(plugin_tab[i].set, "it has no DT_INIT"); + continue; + } + run[i] = 1, nrun++; + } + if (!nrun) return 0; + uint64_t t0 = plat_now_ms(); + rt_log("boot: ENTER %s DT_INIT first: a mod plugin's init runs before the game's constructors", sdk->name); + rt_call(c, sdk->init, 0); + pre = calloc((size_t)ldr_count(), sizeof *pre); + if (!pre) rt_fatal("plugins: out of memory"); + for (int i = 0; i < ldr_count(); i++) { // the game modules before the plugins run + const RtLoaded *m = ldr_module(i); + if (m->plugin) continue; + pre[i] = malloc(m->size); + if (!pre[i]) rt_fatal("plugins: out of memory for a copy of %s", m->name); + memcpy(pre[i], (const void *)(uintptr_t)m->base, m->size); + } + for (int i = RT_PLUGIN_COUNT - 1; i >= 0; i--) { + if (!run[i]) continue; + const RtLoaded *m = loaded(plugin_tab[i].module); + rt_log("plugins: ENTER %s DT_INIT at %#llx", m->name, (unsigned long long)m->init); + failed[0] = 0; + running = 1; + rt_trace |= RT_TRACE_PLUGIN; + rt_call(c, m->init, 0); + rt_trace &= ~(unsigned)RT_TRACE_PLUGIN; + running = 0; + if (failed[0]) snprintf(why[i], sizeof why[i], "its init asked for what the port does not serve: %s", failed); + } + for (int i = 0; i < naliases; i++) + if (aliases[i].live) flush(&aliases[i]); + // What the inits wrote into the game modules, word by word, against the record of the build. + uint8_t seen[NWRITES > 0 ? NWRITES : 1]; + memset(seen, 0, sizeof seen); + int unexpected = 0; + char first[160] = ""; + for (int i = 0; i < ldr_count(); i++) { + const RtLoaded *m = ldr_module(i); + if (!pre[i]) continue; + const uint8_t *now = (const uint8_t *)(uintptr_t)m->base; + for (uint64_t off = 0; off < m->size; off += PAGE) { + if (!memcmp(now + off, pre[i] + off, PAGE)) continue; + for (uint64_t w = off; w < off + PAGE; w += 4) { + uint32_t a, b; + memcpy(&a, pre[i] + w, 4); + memcpy(&b, now + w, 4); + if (a == b) continue; + int k = 0; + while (k < NWRITES && !(write_tab[k].off == w && !strcmp(write_tab[k].module, m->name) && + write_tab[k].old == a && write_tab[k].now == b)) + k++; + if (k < NWRITES) { + seen[k] = 1; + } else if (!unexpected++) { + snprintf(first, sizeof first, "%s+%#llx %08x -> %08x", m->name, (unsigned long long)w, a, b); + } + } + } + restore_sealed(m, pre[i]); + free(pre[i]); + } + free(pre); + pre = NULL; + for (int i = 0; i < RT_PLUGIN_COUNT; i++) { + if (!run[i] || why[i][0]) continue; + const RtLoaded *m = loaded(plugin_tab[i].module); + int missing = 0; + for (int k = 0; k < NWRITES; k++) missing += write_tab[k].plugin == i && !seen[k]; + char hex[65]; + sha256_hex((const void *)(uintptr_t)m->base, m->text_end, hex); + if (unexpected) + snprintf(why[i], sizeof why[i], "the inits wrote %d words the build did not record (first %s)", unexpected, + first); + else if (missing) + snprintf(why[i], sizeof why[i], "%d of the words the build recorded were not written", missing); + else if (strcmp(hex, plugin_tab[i].sha256)) + snprintf(why[i], sizeof why[i], "its code after the init is not the one the build lifted"); + } + mem_query_hook = NULL; // the game's own QueryMemory sees its address space as without plugins + rt_process_memory = NULL; // a callback that maps process memory stops with a named fatal (svc.c) + int on = 0; + for (int i = 0; i < RT_PLUGIN_COUNT; i++) { + if (!run[i]) continue; + if (why[i][0]) { + rt_log("plugins: %s stays off: %s", plugin_tab[i].module, why[i]); + mods_plugin_off(plugin_tab[i].set, why[i]); + continue; + } + int hooks = 0; + for (int k = 0; k < NWRITES; k++) hooks += write_tab[k].plugin == i; + rt_log("plugins: %s: init ran, %d hooks on (set %s)", plugin_tab[i].module, hooks, + rt_patch_name(plugin_tab[i].set)); + on++; + } + int live = 0; + for (int i = 0; i < naliases; i++) live += aliases[i].live; + rt_log("plugins: %d of %d plugins on in %llu ms (%d process-memory maps, %d kept)", on, nrun, + (unsigned long long)(plat_now_ms() - t0), naliases, live); + if (on && cfg_bool("TOTK_PLUGIN_TRACE")) trace_entries(); + return 1; +#else + (void)c, (void)sdk; + return 0; +#endif +} diff --git a/runtime/svc.c b/runtime/svc.c index 1f62f62d..91c92880 100644 --- a/runtime/svc.c +++ b/runtime/svc.c @@ -24,7 +24,7 @@ static uint64_t g_svcs; void (*volatile svc_test_exit_hook)(unsigned code); uint64_t rt_svc_tick_offset; -void (*rt_plugin_svc)(Cpu *c, uint32_t imm, uint64_t pc); +int (*rt_plugin_svc)(Cpu *c, uint32_t imm, uint64_t pc); uint32_t (*rt_process_memory)(int unmap, uint64_t dst, uint32_t process, uint64_t src, uint64_t size); RtStats rt_stats(void) { @@ -159,7 +159,7 @@ void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) { __atomic_fetch_add(&g_svcs, 1, __ATOMIC_RELAXED); if (rt_trace & (RT_TRACE_SVC | RT_TRACE_PLUGIN)) { - if ((rt_trace & RT_TRACE_PLUGIN) && rt_plugin_svc) rt_plugin_svc(c, imm, pc); + if ((rt_trace & RT_TRACE_PLUGIN) && rt_plugin_svc && rt_plugin_svc(c, imm, pc)) return; // refused, x0 set char at[160]; if (rt_trace & RT_TRACE_SVC) rt_log("svc %#x at %s x0=%#llx x1=%#llx x2=%#llx x3=%#llx", imm, ldr_describe(pc, at, sizeof at), diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 921386c4..c7c8e73f 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1270,7 +1270,13 @@ static void tab_mods(float scale) { if (count) snprintf(cheat, sizeof cheat, menu_text(UI_MENU_MODS_CHEATS), count); else snprintf(cheat, sizeof cheat, menu_text(UI_MENU_MODS_CHEATS_FILE), cheats_file()); } - char code[512] = "", problem[512] = "", skipped[512] = "", left[512] = ""; + char code[512] = "", problem[768] = "", skipped[512] = "", left[512] = "", plugin[512] = ""; + if (m->plugin_off[0]) // its plugin's start differed from the build (plugin_init.c) + snprintf(plugin, sizeof plugin, menu_text(UI_MENU_MODS_PLUGIN_OFF), m->plugin_off); + else if (m->plugin_hooks && m->problem == MOD_OK) + snprintf(plugin, sizeof plugin, menu_text(UI_MENU_MODS_PLUGIN_ON), m->plugin_hooks); + else if (m->plugin_note[0] && m->problem != MOD_PLUGIN) // used without its code: the refusal stays visible + snprintf(plugin, sizeof plugin, menu_text(UI_MENU_MODS_PLUGIN_REFUSED), m->plugin_file, m->plugin_note); if (m->code[0] && m->problem != MOD_PLUGIN) // used without its code snprintf(skipped, sizeof skipped, menu_text(UI_MENU_MODS_CODE_SKIPPED), m->code); if (m->unsupported && m->problem != MOD_EXEFS_UNSUPPORTED) @@ -1281,7 +1287,16 @@ static void tab_mods(float scale) { snprintf(code, sizeof code, menu_text(UI_MENU_MODS_EXEFS_ON), m->nsets + m->ndata); switch (m->problem) { case MOD_EMPTY: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EMPTY)); break; - case MOD_PLUGIN: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_SUBSDK)); break; + case MOD_PLUGIN: + if (m->plugin_note[0]) { // the build refused its plugin, and says why + int n = snprintf(problem, sizeof problem, menu_text(UI_MENU_MODS_PLUGIN_REFUSED), m->plugin_file, + m->plugin_note); + if (n >= 0 && n < (int)sizeof problem) + snprintf(problem + n, sizeof problem - n, " %s", menu_text(UI_MENU_MODS_PLUGIN_STAYS_OFF)); + } else { + snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_SUBSDK)); + } + break; case MOD_EXEFS_LATER: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EXEFS_LATER)); break; case MOD_EXEFS_UNSUPPORTED: snprintf(problem, sizeof problem, menu_text(UI_MENU_MODS_EXEFS_UNSUPPORTED), m->note); @@ -1291,7 +1306,8 @@ static void tab_mods(float scale) { default: break; } int k = snprintf(desc, sizeof desc, menu_text(UI_MENU_MODS_ROW_DESC), m->replaced, m->added); - const char *parts[] = {over, other, cheat, code, skipped, left, problem, menu_text(UI_MENU_MODS_MOVE_HINT)}; + const char *parts[] = {over, other, cheat, code, plugin, skipped, left, problem, + menu_text(UI_MENU_MODS_MOVE_HINT)}; for (const char *part : parts) if (*part && k >= 0 && k < (int)sizeof desc) k += snprintf(desc + k, sizeof desc - k, "\n\n%s", part); const char *note = m->enabled && m->problem != MOD_OK ? menu_text(UI_MENU_MODS_UNUSED) : after; diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 17f33965..1c9a5461 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -586,6 +586,12 @@ static const char *const text_en[UI_TEXT_COUNT] = { "stays off while this mod is on.", [UI_MENU_MODS_CODE_SKIPPED] = "Its own game code (%s) does not load: the rest of the mod is used. Switch the mod " "off if the game misbehaves with it.", + [UI_MENU_MODS_PLUGIN_ON] = "Its plugin runs: %d hooks into the game code.", + [UI_MENU_MODS_PLUGIN_REFUSED] = "Its plugin %s cannot run in the port: %s.", + [UI_MENU_MODS_PLUGIN_STAYS_OFF] = "The whole mod stays off. Its switch \"Use ... without its code\" below uses the " + "rest of it.", + [UI_MENU_MODS_PLUGIN_OFF] = "Its plugin stays off this time: it started differently from the build (%s). The game " + "rebuilds it at the next start.", [UI_MENU_MODS_WITHOUT_CODE] = "Use %s without its code", [UI_MENU_MODS_WITHOUT_CODE_DESC] = "The mod above brings code the port cannot run: its own game code (a subsdk " "plugin or a replaced main) or code patches (exefs) the port cannot apply. Off: " @@ -1161,6 +1167,12 @@ static const char *const text_ru[UI_TEXT_COUNT] = { "который выключен, пока этот мод включён.", [UI_MENU_MODS_CODE_SKIPPED] = "Его собственный код игры (%s) не загружается: используется остальная часть мода. " "Выключите мод, если игра с ним ведёт себя странно.", + [UI_MENU_MODS_PLUGIN_ON] = "Его плагин работает: перехватов в коде игры: %d.", + [UI_MENU_MODS_PLUGIN_REFUSED] = "Его плагин %s не может работать в порте: %s.", + [UI_MENU_MODS_PLUGIN_STAYS_OFF] = "Мод выключен целиком. Его переключатель «... без его кода» ниже включает " + "остальную часть мода.", + [UI_MENU_MODS_PLUGIN_OFF] = "Его плагин в этот раз выключен: он запустился не так, как при сборке (%s). Игра " + "пересоберёт его при следующем запуске.", [UI_MENU_MODS_WITHOUT_CODE] = "%s без его кода", [UI_MENU_MODS_WITHOUT_CODE_DESC] = "Мод выше приносит код, который порт не может запустить: свой код игры (плагин " "subsdk или заменённый main) или патчи кода (exefs), которые порт не может " @@ -1770,6 +1782,12 @@ static const char *const text_pt_br[UI_TEXT_COUNT] = { "%s, que fica desligado enquanto este mod está ligado.", [UI_MENU_MODS_CODE_SKIPPED] = "O código próprio do jogo dele (%s) não é carregado: o resto do mod é usado. " "Desligue o mod se o jogo se comportar mal com ele.", + [UI_MENU_MODS_PLUGIN_ON] = "O plugin dele está ativo: %d ganchos no código do jogo.", + [UI_MENU_MODS_PLUGIN_REFUSED] = "O plugin %s dele não pode rodar no port: %s.", + [UI_MENU_MODS_PLUGIN_STAYS_OFF] = "O mod inteiro fica desligado. A chave \"Usar ... sem o código dele\", abaixo, " + "usa o resto dele.", + [UI_MENU_MODS_PLUGIN_OFF] = "O plugin dele fica desligado desta vez: ele iniciou de forma diferente da compilação " + "(%s). O jogo o recompila na próxima inicialização.", [UI_MENU_MODS_WITHOUT_CODE] = "Usar %s sem o código dele", [UI_MENU_MODS_WITHOUT_CODE_DESC] = "O mod acima traz código que o port não consegue rodar: código próprio do jogo " "(um plugin subsdk ou um main substituído) ou patches de código (exefs) que o " @@ -2458,6 +2476,10 @@ static const char *const text_zh_hans[UI_TEXT_COUNT] = { [UI_MENU_MODS_EXEFS_ON] = "正在使用的代码补丁:%d 个文件。", [UI_MENU_MODS_REPLACES_BUILTIN] = "正在使用的代码补丁:%d 个文件。它们取代本移植自带的补丁 %s,此模组开启时该补丁保持关闭。", [UI_MENU_MODS_CODE_SKIPPED] = "它自带的游戏代码(%s)不会加载:模组的其余部分照常使用。如果游戏因此表现异常,请关闭此模组。", + [UI_MENU_MODS_PLUGIN_ON] = "它的插件正在运行:游戏代码中有 %d 处挂钩。", + [UI_MENU_MODS_PLUGIN_REFUSED] = "它的插件 %s 无法在移植版中运行:%s。", + [UI_MENU_MODS_PLUGIN_STAYS_OFF] = "整个模组保持关闭。打开下方它的“不带代码使用 ...”开关即可使用其余部分。", + [UI_MENU_MODS_PLUGIN_OFF] = "它的插件本次保持关闭:启动情况与构建时不同(%s)。游戏会在下次启动时重新构建它。", [UI_MENU_MODS_WITHOUT_CODE] = "不带代码使用 %s", [UI_MENU_MODS_WITHOUT_CODE_DESC] = "上方的模组带有移植版无法运行的代码:它自己的游戏代码(subsdk 插件或替换的 main),或移植版无法应用的代码补丁(exefs)。关闭:整个模组保持关闭。开启:跳过这些代码,使用模组的其余部分:其 RomFS 文件、金手指和可用的代码补丁。它的文件可能依赖缺失的代码;如果游戏表现异常,请再次关闭。下次启动时生效。", [UI_MENU_MODS_WITHOUT_CODE_ALL] = "已对所有模组开启:totk.ini 中 mods_without_code = 1。", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 2423b4dd..96f7114f 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -92,6 +92,10 @@ typedef enum { UI_MENU_MODS_EXEFS_ON, // printf: its ExeFS patch files in use UI_MENU_MODS_REPLACES_BUILTIN, // printf: its ExeFS patch files in use, the port's own patch sets they turn off UI_MENU_MODS_CODE_SKIPPED, // printf: its code modules in exefs, which do not load (used without its code) + UI_MENU_MODS_PLUGIN_ON, // printf: the hooks of its plugin (runtime/plugin_init.c) that run + UI_MENU_MODS_PLUGIN_REFUSED, // printf: its plugin file, why the build refused it (English, from the tool) + UI_MENU_MODS_PLUGIN_STAYS_OFF, // after UI_MENU_MODS_PLUGIN_REFUSED when the whole mod stays off + UI_MENU_MODS_PLUGIN_OFF, // printf: why its plugin's start differed from the build (English) UI_MENU_MODS_WITHOUT_CODE, // printf: the mod's name; its switch "use without its code" (TOTK_MODS_NO_CODE) UI_MENU_MODS_WITHOUT_CODE_DESC, UI_MENU_MODS_WITHOUT_CODE_ALL, // why that switch cannot be changed: TOTK_MODS_WITHOUT_CODE=1 From 8cee1afe5dd9461ebd4397a7d02c26b3ecd6b03b Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 08:48:31 +0300 Subject: [PATCH 06/21] The rebuild builds mod plugins in: service run, checks, patch sets and status lines (#845) After the design in #845 by itaskaev2. tools/exefs_plugins.py writes an accepted plugin's patch set (recomp/patches.py plugin lines and hook sites, one ordered set mod_ per plugin, several plugins in one record told apart by their hooks) and a line per plugin of build-cache/plugins.status: built, or refused with the first reason. It also checks the code the hooks reach (direct branches and calls from the stubs, impls, callbacks and targets): no svc but OutputDebugString, imports only from a reviewed list. The init's imports may now include malloc, free, memcpy, memmove and memset, because the init runs again at every start and its heap blocks exist while the game runs. The setup's lift step runs the plugin stage when a mod has exefs/subsdk*: a lift with the plugins, a totk.exe of it in the compile step's folder, the service run, then the lift with the hook sites; a failed stage warns (warn.plugins_skipped) and installs without the plugins. docs/mods.md describes what works, what is refused and why, and the rebuild. Co-Authored-By: Claude Opus 5.5 --- docs/mods.md | 37 ++++- docs/settings.md | 3 +- installer/steps.py | 105 ++++++++++++-- installer/text/en.json | 1 + installer/text/pt-BR.json | 1 + installer/text/ru.json | 1 + installer/text/zh-Hans.json | 1 + tests/exefs_plugins_test.py | 218 +++++++++++++++++++++++++++- tests/installer_test.py | 54 +++++++ tools/exefs_plugins.py | 282 ++++++++++++++++++++++++++++++++++-- tools/make_release.py | 2 +- 11 files changed, 671 insertions(+), 34 deletions(-) diff --git a/docs/mods.md b/docs/mods.md index f369035c..089207b9 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -49,8 +49,8 @@ named ones. ### Mods with code the port cannot run -Some mods bring code the port cannot run: their own Switch code (a subsdk plugin or a replaced main, as UltraCam ships) -or a code patch in a form the port cannot apply. Such a mod stays off as a whole, because its other files may expect +Some mods bring code the port cannot run: their own Switch code (a replaced main, or a subsdk plugin the rebuild +refused, as it refuses UltraCam's) or a code patch in a form the port cannot apply. Such a mod stays off as a whole, because its other files may expect that code. Under its row the Mods tab then shows **Use without its code** (off by default). Switched on, the code the port cannot run is left out and the rest of the mod loads after a restart: its game files, cheats and the code patches that work. The mod's row says which part is left out and why. If the game misbehaves with it, switch it off @@ -96,6 +96,35 @@ A patch file may also change the game's data (constants and tables in its `.roda stored into the loaded game at start, before it runs. When two mods write the same bytes, the higher one wins, as with files. +## Code plugins (exefs subsdk) + +Some mods ship their own Switch code as an exlaunch plugin: a file `exefs\subsdk` (The Second Quest's MSFix, Full +Inventory Expansion's sort fixer, Quick Weapon Durability). On the Switch the plugin starts with the game and +redirects some of the game's instructions into its own code. The port does the same with the plugin's code compiled +ahead of time, after the design in [#845](https://github.com/valentderah/totk-pc-recomp/issues/845) by itaskaev2: + +1. The rebuild (the same one as for code patches above) compiles the plugin with the game, starts the game's code + once in a service mode that runs only the plugin's start-up (`totk.exe --plugin-init`), twice, and records which + instructions it redirects. Each redirect becomes a code site of the mod, built into `totk.exe`. +2. At every start the plugin's start-up runs again, before the game's own start-up code, in the same setting. Its + redirects are compared with the record of the rebuild. When they match, the mod's code sites go on. When they do + not, the plugin stays off for this run, the mod's row says why, and the next start runs the rebuild. +3. The plugin keeps what its start-up made (its memory, its tables), so its code works as on the Switch. The start + costs about 50 ms per plugin; the game runs no differently without a plugin. + +The Mods tab order decides between a plugin and another mod's patch on the same instruction, as for code patches. A +plugin site over one of the port's own patches turns that built-in patch off while the mod is on. + +The rebuild refuses a plugin when its start-up or the code its redirects reach needs what the port does not give it: +system calls other than the few its start-up needs (memory queries and maps, the clock, debug output), library calls +outside a reviewed list (memory and string functions, the heap, text formatting), threads, services or input; a +redirect that is not one of exlaunch's hook shapes (inline hook, replace hook, call-site rewrite); or two runs of its +start-up that differ in code. The row then says **Its plugin ... cannot run in the port** with the reason, and the +mod stays off; **Use without its code** uses the rest of it. UltraCam stays refused: it hooks the graphics API, +reads input and opens sockets. + +`plugin_trace = 1` in `totk.ini` logs the first call into each of a plugin's hooks (`plugins: hook entry ... reached`). + ## Cheats A cheat file is part of a mod: `mods\\cheats\.txt`, switched cheat by cheat on the Cheats tab. Cheats @@ -134,7 +163,9 @@ keeps its textures at a quarter of the size. | This folder has no romfs, exefs or cheats files. | The archive was unpacked one level too deep (`mods\Mod\Mod\romfs`), or it holds no game files. | Move the folders up so `romfs` sits directly in the mod's folder. | | ... files carry another game version in their names. | The files are named for another update (`.Product..`) and the game's own files have no such name. They are still served under those names, as on a console, because the game sometimes asks for one; another update's resource size table is the exception and only lends its entries. | Nothing if the mod works. If it does nothing, download its release for your game update. | | Its code patches (exefs) are made for another game version. | The patch files name another update's build id. | Same: get the release for your update. | -| This mod brings its own game code (a subsdk plugin or a replaced main). | Mods such as UltraCam ship compiled Switch code. The port cannot load it. | The whole mod stays off. **Use without its code** under its row uses the rest of it, which may expect the missing code (see Mods with code the port cannot run). | +| This mod brings its own game code (a subsdk plugin or a replaced main). | The mod replaces the game's main code, which the port cannot load. | The whole mod stays off. **Use without its code** under its row uses the rest of it, which may expect the missing code (see Mods with code the port cannot run). | +| Its plugin ... cannot run in the port: ... | The rebuild refused the mod's code plugin; the reason follows (see Code plugins). | The whole mod stays off. **Use without its code** under its row uses the rest of it. | +| Its plugin stays off this time: it started differently from the build. | The plugin redirected other instructions at this start than in the rebuild, for example after the mod was updated in place. | Nothing: the next start runs the rebuild. | | This mod patches the game code (exefs) in a way the port cannot apply. | The patch changes code in a form the port cannot build, or bytes the port cannot change at start. The reason follows on the row. | The whole mod stays off. **Use without its code** under its row leaves that patch file out and uses the rest of the mod. | | The folder name contains \|. | The mod list uses `\|` between names. | Rename the folder. | diff --git a/docs/settings.md b/docs/settings.md index 6f23cdda..388c47a8 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -24,7 +24,8 @@ Many rows below are developer tools that only act together with another setting | `core_diag` | `1` records why ready guest threads wait for a virtual core and for how long, with a histogram every 60 presents. | Turns on host-call and SVC depth tracking and writes many log lines (one per present over 25 ms); frames get slower. | Caution | | `barrier_merge` (default `0`) | Experimental. `1` keeps a render pass open through the game's Barrier calls that ask only for fragment ordering and texture invalidation, when no draw of the pass can write a storage buffer or image; the next draw that samples a written target gets a by-region barrier in the pass (the feedback layout). In scenes with about a thousand such barriers a frame this cuts the passes from about 1200 to 130 and the GPU time from 24.5 to 7.8 ms. | The game's barrier orders every pixel, the in-pass by-region barrier only the pixel a draw writes: a driver that runs it per screen tile can show flicker or squares at the edges of water, decals or particles. | Caution | | `menu_cursor_log` | `1` logs what the menu cursor code reads at every screen change. | Only adds log lines. | Safe | -| `plugin_tick_offset` (default `0`) | Added to every GetSystemTick in the plugin service run (`totk.exe --plugin-init DIR`), so `tools/exefs_plugins.py` can run a plugin's init twice with other ticks and compare the records. | Has no effect outside `--plugin-init`; the game never asks for GetSystemTick. | Safe | +| `plugin_tick_offset` (default `0`) | Added to every GetSystemTick in the plugin service run (`totk.exe --plugin-init DIR`), so `tools/exefs_plugins.py` can run a plugin's init twice with other ticks and compare the records. | Has no effect outside `--plugin-init`; a plugin's init at the start of the game gets the plain ticks. | Safe | +| `plugin_trace` | `1` logs the first call of each entry of a mod plugin's hooks (its callbacks, replace and call targets) as `plugins: hook entry ... reached`. | Each such call goes through a counting step; nothing changes without a mod plugin. | Safe | | `perf_slow_ms` (default `50`) | Frames at least this many milliseconds long get a `perf: slow frame` log line with the time blocked per cause. | Changes how many frames are logged; a small value fills the log. | Safe | | `replay_check` | `1` reads every texture read-back a second time after a full drain of the replay queue and compares the two. | Stalls the replay queue at every read-back, so frames get much slower; mismatches are only counted and logged. | Caution | | `resolve_check` | `1` compares every cached pool lookup with the locked lookup and stops on a difference. | Slows every pool lookup; the game stops if the two ever differ. | Caution | diff --git a/installer/steps.py b/installer/steps.py index 3c3c1e25..08ff4baa 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -19,7 +19,8 @@ lift cache, gen//lift_cache.json) and rewrites only those whose C changed, those. A change of the mods' code cheats then costs a few seconds of lift and the chunks holding its sites, not a full lift or compile. -Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status and exefs.data (tools/exefs_patches.py) and +Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status and exefs.data (tools/exefs_patches.py), +plugins.status (tools/exefs_plugins.py; plugins.stale is the game's list of plugins whose start differed) and rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild), one argument per line; totk.exe runs it at start when an enabled mod has code cheats the build lacks (runtime/cheats.c). @@ -866,6 +867,43 @@ def exefs_files(ctx): return sorted(out) +def plugin_files(ctx): + """The exlaunch plugins of the mods folder (each mod's exefs/subsdk*, in the first ExeFS place, as + tools/exefs_plugins.py finds them), by mod name, for the lift key and the plugin stage.""" + mods = ctx.dest / 'mods' + if not mods.is_dir(): + return [] + out = [] + for mod in sorted((m for m in mods.iterdir() if m.is_dir()), key=lambda m: m.name): + exefs = next((mod / w for w in EXEFS_PLACES if (mod / w).is_dir()), None) + if exefs: + out += sorted((f for f in exefs.iterdir() if f.is_file() and f.name.lower().startswith('subsdk')), + key=lambda f: f.name.lower()) + return out + + +def plugins_status(ctx): + """The build's plugin status file (tools/exefs_plugins.py --status), which runtime/mods.c reads.""" + return ctx.dest / 'build-cache' / 'plugins.status' + + +def built_plugins(status, files, mods): + """The files of the plugins the status file lists as built (kind `plugin`), in the order of files. The key is + FNV-1a 32 of "/", as tools/exefs_plugins.py and runtime/mods.c compute it.""" + if not status.is_file(): + return [] + built = {line.split('\t')[0] for line in status.read_text(encoding='utf-8').splitlines() + if len(line.split('\t')) >= 3 and line.split('\t')[2] == 'plugin'} + + def key(f): + rel = f.relative_to(mods).as_posix() + h = 0x811C9DC5 + for b in rel.encode(): + h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF + return f'{h:08X}' + return [f for f in files if key(f) in built] + + def exefs_status(ctx): """The build's ExeFS patch status file (tools/exefs_patches.py --status), which runtime/mods.c reads.""" return ctx.dest / 'build-cache' / 'exefs.status' @@ -890,9 +928,12 @@ class Lift(Step): def key(self, ctx): # fingerprint takes a file's name and content only; the paths under mods/ go in too, because tools/cheats.py # and tools/exefs_patches.py name each patch set after its mod folder, so a renamed folder needs a lift - files = cheat_files(ctx) + exefs_files(ctx) + # A mod plugin is built by running its init on a totk.exe of this build (tools/exefs_plugins.py), so the + # runtime is in the key too when there is one. + files = cheat_files(ctx) + exefs_files(ctx) + plugin_files(ctx) + extra = [ctx.layout.tools / 'exefs_plugins.py', runtime_key(ctx)] if plugin_files(ctx) else [] return fingerprint('lift-5', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)), - ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *files, + ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *extra, *files, *(f.relative_to(ctx.dest / 'mods').as_posix() for f in files)) def current(self, ctx, record): @@ -943,12 +984,44 @@ class Lift(Step): else: patches = ctx.temp / out args = ['--exefs', ctx.dest / 'exefs', '--out', gen] + # The mods' exlaunch plugins (tools/exefs_plugins.py, after the design in #845 by itaskaev2): a lift with the + # plugins in the layout, a totk.exe of it in build/ (the compile step's folder, so it recompiles only what + # the hook sites change after), the plugin service run, and the plugins' patch sets for the lift below. A + # refused plugin gets its reason in plugins.status; a failed stage costs the plugins, not the install. + plugins = plugin_files(ctx) + drop([plugins_status(ctx), ctx.dest / 'build-cache' / 'plugins.stale']) + lift_plugins = [] + if plugins: + listed = ['--plugins', ','.join(str(f) for f in plugins)] + code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches] + listed, parse) + if not code: + code, log = run_tool(ctx, 'lift', layout.tools / 'build_game.py', build_args(ctx, gen)) + if not code: + code, log = run_tool(ctx, 'lift', layout.tools / 'exefs_plugins.py', [ + '--exe', ctx.temp / 'build' / ('totk' + EXE), '--out', ctx.temp / 'plugins', '--mods', + ctx.dest / 'mods', '--patches', patches, '--patches-out', ctx.temp / 'patches-plugins.txt', + '--status', plugins_status(ctx)]) + # exit 1 with a status file: refused plugins, their reasons in it; the rest goes on + if plugins_status(ctx).is_file() and (ctx.temp / 'patches-plugins.txt').is_file(): + code = 0 + if code: + drop([plugins_status(ctx)]) + skipped['warn.plugins_skipped'] = log + else: + patches = ctx.temp / 'patches-plugins.txt' + lift_plugins = built_plugins(plugins_status(ctx), plugins, ctx.dest / 'mods') + if lift_plugins: + args += ['--plugins', ','.join(str(f) for f in lift_plugins)] code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches], parse) if code and patches != plain: for tool, out, outputs, inputs, warn in stages: if inputs and warn not in skipped: drop(outputs) skipped[warn] = log + if lift_plugins: + drop([plugins_status(ctx)]) + skipped.setdefault('warn.plugins_skipped', log) + args = ['--exefs', ctx.dest / 'exefs', '--out', gen] code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', plain], parse) if code: raise InstallError('lift.failed', code=code, log=str(log)) @@ -982,6 +1055,21 @@ def runtime_key(ctx): layout.sysroot or 'host', *tools, clang, *[d.name for d in layout.dlls]) +def build_args(ctx, gen): + """tools/build_game.py's arguments for a totk.exe of the lifted C in gen, built in /build.""" + layout, dest = ctx.layout, ctx.dest + cpu = getattr(ctx, 'cpu', 'v3') + lib = layout.runtime_lib_for(cpu) + pgo = layout.pgo(game_version(ctx)) + return ['--gen', gen, '--out', ctx.temp / 'build', '--toolchain', layout.llvm, '--cpu', cpu, + *(['--deps', layout.deps] if layout.deps else []), *(['--sysroot', layout.sysroot] if layout.sysroot else []), + '--runtime', layout.runtime, *(['--runtime-lib', lib] if lib else ()), *(['--pgo', pgo] if pgo else ()), + '--ninja', layout.ninja, + '--version', version_tag(game_version(ctx)), '--expected', layout.expected(game_version(ctx)), + '--exefs', dest / 'exefs', '--romfs', dest / 'romfs', + '--profile', dest / 'profile'] + + class Compile(Step): id = 'compile' @@ -1012,16 +1100,7 @@ class Compile(Step): rep.progress('compile', 'compile', force=True) cpu = getattr(ctx, 'cpu', 'v3') - lib = layout.runtime_lib_for(cpu) - pgo = layout.pgo(game_version(ctx)) - code, log = run_tool(ctx, 'compile', layout.tools / 'build_game.py', [ - '--gen', ctx.temp / 'gen', '--out', build, '--toolchain', layout.llvm, '--cpu', cpu, - *(['--deps', layout.deps] if layout.deps else []), *(['--sysroot', layout.sysroot] if layout.sysroot else []), - '--runtime', layout.runtime, *(['--runtime-lib', lib] if lib else ()), *(['--pgo', pgo] if pgo else ()), - '--ninja', layout.ninja, - '--version', version_tag(game_version(ctx)), '--expected', layout.expected(game_version(ctx)), - '--exefs', dest / 'exefs', '--romfs', dest / 'romfs', - '--profile', dest / 'profile'], parse) + code, log = run_tool(ctx, 'compile', layout.tools / 'build_game.py', build_args(ctx, ctx.temp / 'gen'), parse) exe = build / ('totk' + EXE) if code and link_output_locked(log): raise InstallError('compile.exe_locked', path=str(exe), log=str(log)) diff --git a/installer/text/en.json b/installer/text/en.json index 254a40e7..c40def78 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -108,6 +108,7 @@ "warn.npdm_differs": "main.npdm differs from the reference {version} dump; only the code modules are used.", "warn.cheats_skipped": "The cheats of the mods folder that change the game's code could not be built; the game installs without them. Details: {log}", "warn.exefs_skipped": "The mods' ExeFS patches (.ips, .pchtxt) could not be built; the game installs without them. Details: {log}", + "warn.plugins_skipped": "The mods' code plugins (exefs subsdk) could not be built; the game installs without them. Details: {log}", "warn.xbox_prompts_failed": "The Xbox or PC button prompts could not be prepared (exit code {code}); the game keeps the Switch prompts. Details: {log}", "warn.shaders_skipped": "The graphics driver crashed while building some shaders, so {count} were left out of the preparation. The game builds them when it first needs them, and the driver may crash there too; a driver update may fix it. Details: {log}", "warn.kernel_old": "Linux {version} lacks the write tracking the game relies on (6.7 or later); the game may run very slowly. Updating the system fixes this.", diff --git a/installer/text/pt-BR.json b/installer/text/pt-BR.json index 5c8579b2..3ce41439 100644 --- a/installer/text/pt-BR.json +++ b/installer/text/pt-BR.json @@ -107,6 +107,7 @@ "warn.lang_needs_fonts": "Textos em {lang} precisam das fontes de sistema do Switch, e nenhuma foi informada. O jogo não inicia neste idioma até que sejam adicionadas. Escolha a pasta do firmware (Contents/registered) ou veja \"Switch system fonts\" em docs/installer.md.", "warn.cheats_skipped": "Não foi possível compilar os cheats da pasta de mods que alteram o código do jogo; o jogo é instalado sem eles. Detalhes: {log}", "warn.exefs_skipped": "Não foi possível compilar os patches ExeFS dos mods (.ips, .pchtxt); o jogo é instalado sem eles. Detalhes: {log}", + "warn.plugins_skipped": "Não foi possível compilar os plugins de código dos mods (exefs subsdk); o jogo é instalado sem eles. Detalhes: {log}", "warn.npdm_differs": "O main.npdm difere do dump de referência da versão {version}; só os módulos de código são usados.", "warn.xbox_prompts_failed": "Não foi possível preparar os ícones de botão do Xbox ou do PC (código de saída {code}); o jogo mantém os ícones do Switch. Detalhes: {log}", "warn.shaders_skipped": "O driver gráfico travou ao compilar alguns shaders, então {count} ficaram fora da preparação. O jogo os compila quando precisar deles pela primeira vez, e o driver pode travar ali também; uma atualização do driver pode resolver. Detalhes: {log}", diff --git a/installer/text/ru.json b/installer/text/ru.json index f329d0b2..d20cd076 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -108,6 +108,7 @@ "warn.npdm_differs": "main.npdm отличается от эталонного дампа {version}; используются только модули с кодом.", "warn.cheats_skipped": "Не удалось собрать читы из папки модов, меняющие код игры; игра установится без них. Подробности: {log}", "warn.exefs_skipped": "Не удалось собрать ExeFS-патчи модов (.ips, .pchtxt); игра установится без них. Подробности: {log}", + "warn.plugins_skipped": "Не удалось собрать плагины кода модов (exefs subsdk); игра установится без них. Подробности: {log}", "warn.xbox_prompts_failed": "Не удалось подготовить подсказки кнопок Xbox или PC (код выхода {code}); игра оставит подсказки Switch. Подробности: {log}", "warn.shaders_skipped": "Графический драйвер аварийно завершился при сборке части шейдеров, поэтому {count} из них пропущены при подготовке. Игра соберёт их, когда они впервые понадобятся, и драйвер может упасть и там; обновление драйвера может это исправить. Подробности: {log}", "warn.kernel_old": "В Linux {version} нет отслеживания записи, на которое опирается игра (нужна версия 6.7 или новее); игра может работать очень медленно. Обновление системы это исправит.", diff --git a/installer/text/zh-Hans.json b/installer/text/zh-Hans.json index 1d0e4b8a..1b614894 100644 --- a/installer/text/zh-Hans.json +++ b/installer/text/zh-Hans.json @@ -107,6 +107,7 @@ "warn.lang_needs_fonts": "{lang}文本需要 Switch 系统字体,但未提供。在添加这些字体之前,游戏无法以该语言启动。请选择固件文件夹(Contents/registered),或参阅 docs/installer.md 中的“Switch system fonts”一节。", "warn.cheats_skipped": "无法构建 mods 文件夹中修改游戏代码的金手指;游戏将不含它们进行安装。详情:{log}", "warn.exefs_skipped": "无法构建模组的 ExeFS 补丁(.ips、.pchtxt);游戏将不含它们进行安装。详情:{log}", + "warn.plugins_skipped": "无法构建模组的代码插件(exefs subsdk);游戏将不含它们进行安装。详情:{log}", "warn.npdm_differs": "main.npdm 与参考的 {version} 导出不同;只使用其中的代码模块。", "warn.xbox_prompts_failed": "无法准备 Xbox 或 PC 按键提示(退出代码 {code});游戏将继续使用 Switch 按键提示。详情:{log}", "warn.shaders_skipped": "显卡驱动在编译部分着色器时崩溃,因此有 {count} 个着色器未被预先准备。游戏会在首次需要时编译它们,驱动在那时也可能崩溃;更新驱动可能会解决。详情:{log}", diff --git a/tests/exefs_plugins_test.py b/tests/exefs_plugins_test.py index 6bb23495..7687d6c9 100644 --- a/tests/exefs_plugins_test.py +++ b/tests/exefs_plugins_test.py @@ -1,8 +1,13 @@ """recomp/layout.py with and without mod plugins, and tools/exefs_plugins.py's check of a plugin service run record: -a synthetic exlaunch-shaped record with one inline hook that passes, and one bad record per rule. No game files. +a synthetic exlaunch-shaped record with one inline hook that passes, and one bad record per rule; the check of the +code the hooks reach; the patch lines of the three hook kinds; and what recomp/patches.py and recomp/gen.py make of +those lines (the hook call, the stub's return, the unpatched copy a replace hook's trampoline reaches, the start-up +record in patches.h). The start-up comparison itself runs in the game (runtime/plugin_init.c plugins_start); its +mods.c side is in totk.exe --probe-mods. No game files. Run with .venv (lz4 and numpy, as recomp/): python tests/exefs_plugins_test.py """ +import hashlib import json import struct import sys @@ -14,7 +19,9 @@ ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / 'tools')) sys.path.insert(0, str(ROOT / 'recomp')) import exefs_plugins as xp # noqa: E402 +import gen # noqa: E402 import layout # noqa: E402 +import patches # noqa: E402 from nso import Nso # noqa: E402 @@ -278,5 +285,214 @@ class Diff(unittest.TestCase): self.assertEqual(xp.diff_records(a, record()), []) +# ---- the code the hooks reach +SVC_40, RET_W, NOP = 0xD4000801, 0xD65F03C0, 0xD503201F + + +def code(words, at=0): + img = bytearray(0x200) + for i, w in enumerate(words): + struct.pack_into('x[30] =', head) # no x30 written for the call + self.assertIn(f'void f_{stub:x}(Cpu *c);', c) + h = patches.header_text(t, 'MARK') + self.assertIn('#define RT_PLUGIN_COUNT 1', h) + self.assertIn(f'X(0, "main", {FN:#x}, 0xaa0003f3u, 0x14000000u)', h) + self.assertIn('X("plugin0", 0x4100) X("plugin0", 0x4200)', h) + + def test_stub_returns_instead_of_branching_to_its_trampoline(self): + t = self.table(f'site mod_x main {FN:x} aa0003f3 @call:plugin0+4000 aa0003f3', ['stub plugin0 4000']) + stub = self.bases['plugin0'] + 0x4000 + c = self.emit(t, 'plugin0', stub, stub + 16) + body = c[c.index(f'void f_{stub:x}(Cpu *c) {{'):] + self.assertIn('return;', body) + self.assertNotIn(f'{self.bases["plugin0"] + 0x2000:#x}', body) # the trampoline is not reached + + def test_replace_hook_keeps_an_unpatched_copy(self): + pb = self.bases['plugin0'] + m = self.bases['main'] + FN + new = b(m, pb + 0x4200) + t = self.table(f'site mod_x main {FN:x} aa0003f3 {new:08x}', [f'orig mod_x plugin0 2000 main {FN:x}', + 'seed plugin0 4200']) + self.assertEqual(t.origs_for('main', self.bases['main']), {m}) + self.assertEqual(t.orig_entries('plugin0', self.bases), {pb + 0x2000: m}) + c = self.emit(t, 'main', m, m + 12, origs=[m]) + copy = c[c.index(f'void fo_{m:x}(Cpu *c) {{'):] + self.assertNotIn('rt_patch_on', copy) + self.assertIn('rt_patch_on[0]', c[:c.index(f'void fo_{m:x}')]) + tab = gen.table_text('plugin0', [pb + 0x4200, pb + 0x2000], {pb + 0x2000: gen.orig_name(m)}) + self.assertIn(f'void fo_{m:x}(Cpu *c);', tab) + self.assertIn(f'fo_{m:x}', tab[tab.index('_fn_ptrs'):]) + + def test_image_must_match_its_hash(self): + t = self.table(f'site mod_x main {FN:x} aa0003f3 @call:plugin0+4000 aa0003f3', ['stub plugin0 4000']) + t.plugins['plugin0']['sha256'] = '0' * 64 + with self.assertRaises(patches.PatchError): + gen.open_module('plugin0', self.plugin_file, self.bases['plugin0'], t) + + def test_chunk_key_takes_the_unpatched_copy(self): + # the lift cache (recomp/gen.py chunk_key): a chunk whose function gets an unpatched copy is keyed apart, and a + # chunk without one keeps the key it had before plugins + fns = [(0x100, 0x10c)] + plain = gen.chunk_key('m', 0, fns, {}, [], {}, [], None, None) + self.assertEqual(gen.chunk_key('m', 0, fns, {}, [], {}, [], None, None, {0x200}), plain) + self.assertNotEqual(gen.chunk_key('m', 0, fns, {}, [], {}, [], None, None, {0x100}), plain) + site = {0x100: [(0, [('call', 0x7005204000), 0xAA0003F3])]} + self.assertNotEqual(gen.chunk_key('m', 0, fns, site, [0x100], {}, [], None, None), + gen.chunk_key('m', 0, fns, {0x100: [(0, [('call', 0x7005204010), 0xAA0003F3])]}, [0x100], + {}, [], None, None)) + + def test_no_plugin_lines_no_plugin_header(self): + (self.d / 'q.txt').write_text('set fps60 TOTK_FPS60 0\n') + self.assertNotIn('PLUGIN', patches.header_text(patches.load(self.d / 'q.txt', self.mods), 'MARK')) + + if __name__ == '__main__': unittest.main() diff --git a/tests/installer_test.py b/tests/installer_test.py index d1615661..f890a1c8 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -383,6 +383,60 @@ class LiftRunTest(unittest.TestCase): self.assertEqual(self.outputs(), [False, False]) self.assertIn('warn.exefs_skipped', self.ctx.reporter.out.getvalue()) + def plugin_setup(self, kind): + """A mod with an exlaunch plugin; the fake service run writes its status line of that kind.""" + plugin = self.dest / 'mods' / 'SQ' / 'exefs' / 'subsdk9' + plugin.write_bytes(b'NSO0') + self.ctx.layout = SimpleNamespace(**vars(self.ctx.layout), llvm='llvm', runtime='runtime', ninja='ninja', + deps=None, sysroot=None, runtime_lib_for=lambda cpu: None, + pgo=lambda version: None, expected=lambda version: 'expected.json') + saved = self.run_tool + + def run_tool(ctx, step, script, args, parse=None, cwd=None): + code, log = saved(ctx, step, script, args, parse, cwd) + args = [str(a) for a in args] + if Path(script).name == 'exefs_plugins.py': + Path(args[args.index('--patches-out') + 1]).write_text('patches\n') + status = Path(args[args.index('--status') + 1]) + status.parent.mkdir(parents=True, exist_ok=True) + h = 0x811C9DC5 # FNV-1a 32 of "SQ/exefs/subsdk9", as tools/exefs_plugins.py keys it + for b in b'SQ/exefs/subsdk9': + h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF + status.write_text(f'{h:08X}\t00000000\t{kind}\t-\tnote\n') + return (1 if kind == 'unsupported' else 0), log + return code, log + steps.run_tool = run_tool + return plugin + + def test_plugin_stage(self): + # the lift with the plugin, a totk.exe of it, the service run, then the lift with its hook sites + plugin = self.plugin_setup('plugin') + steps.Lift().run(self.ctx) + self.assertEqual([c[0] for c in self.calls], ['exefs_patches.py', 'gen.py', 'build_game.py', + 'exefs_plugins.py', 'gen.py', 'exefs_patches.py --folded']) + first, last = self.calls[1][1], self.calls[4][1] + self.assertEqual(first[first.index('--plugins') + 1], str(plugin)) + self.assertEqual(last[last.index('--plugins') + 1], str(plugin)) + self.assertEqual(last[last.index('--patches') + 1], str(self.ctx.temp / 'patches-plugins.txt')) + self.assertTrue(steps.plugins_status(self.ctx).is_file()) + + def test_refused_plugin_is_not_lifted(self): + # a refused plugin: its status line stays (the Mods tab says why), the game is lifted without it + self.plugin_setup('unsupported') + steps.Lift().run(self.ctx) + last = self.calls[4][1] + self.assertNotIn('--plugins', last) + self.assertTrue(steps.plugins_status(self.ctx).is_file()) + self.assertNotIn('warn.plugins_skipped', self.ctx.reporter.out.getvalue()) + + def test_failed_plugin_build_warns(self): + self.plugin_setup('plugin') + self.fail = {'build_game.py'} + steps.Lift().run(self.ctx) + self.assertNotIn('--plugins', self.calls[-2][1]) + self.assertFalse(steps.plugins_status(self.ctx).is_file()) + self.assertIn('warn.plugins_skipped', self.ctx.reporter.out.getvalue()) + class CompileLockedTest(unittest.TestCase): """Compile.run when the linker cannot write its totk.exe: another program holds the file (a game started from the diff --git a/tools/exefs_plugins.py b/tools/exefs_plugins.py index 8bdaac44..a0ef82ff 100644 --- a/tools/exefs_plugins.py +++ b/tools/exefs_plugins.py @@ -1,9 +1,24 @@ """Run a mod plugin's init in the plugin service run and check what it did: the build step before the generator emits a plugin's hook sites. After the design in #845 by itaskaev2; the init runs on this port's runtime (totk.exe ---plugin-init, runtime/plugin_init.c) instead of under Unicorn. +--plugin-init, runtime/plugin_init.c) instead of under Unicorn, and it runs again at every start of the game in the +same environment (plugin_init.c plugins_start), so the plugin keeps real state (its heap blocks, its GOT, its alias +addresses). The record says what the build expects that start-up to write. python tools/exefs_plugins.py --exe build/-x/totk.exe --out # two runs, their diff, the check python tools/exefs_plugins.py --check /run1 # the check of one record + python tools/exefs_plugins.py --exe --out --mods --patches + --patches-out --status # and the patch lines for the generator + +With --mods, --patches, --patches-out and --status, an accepted plugin becomes one `ordered` patch set mod_ +(recomp/patches.py: a plugin line, its expected writes, stubs, seeds, orig lines and its hook sites) appended to the +--patches file into --patches-out, and every plugin gets a line of --status (runtime/mods.c reads it as +build-cache/plugins.status), in the format of tools/exefs_patches.py --status: + + + +file hash: FNV-1a 32 of "/"; content hash: the same over its bytes; kind +`plugin` (note: the module and its hook count) or `unsupported` (note: the first reason); built-in sets: the port's +own sets with a site the plugin hooks too ("-" none), which the runtime switches off while the mod is on. The exe must be built with the plugin in its layout (recomp/gen.py --plugins). --out gets run1/ and run2/, one record each; run2 adds a large offset to every GetSystemTick (TOTK_PLUGIN_TICK_OFFSET), so the init picks other alias @@ -24,10 +39,14 @@ The check (check_record) accepts a record only when every rule holds; each broke record lists them), which the generator can emit as a patch site whose replacement is that branch; - a replace hook (Full Inventory Expansion's Sort Fixer): `b` from a function start of main to a function start of the plugin's lift; the Orig trampoline (the original relocated, b site+4) is listed when the pool holds one. -Not checked here yet: a site shared with a built-in patch set or another mod, and svcs in the code the callbacks -reach (both need the generator's view). + - the code the hooks reach (the stubs, exlaunch's hook impls, the callbacks, the replace and call targets, and what + they reach by direct branches and calls, reach()) has no svc but OutputDebugString and calls only the imports + REACH_IMPORTS. An indirect call (blr) is not followed: what it reaches cannot be known before the run. +A site shared with a built-in patch set turns that set off while the mod is on (the status line names it); a site +shared with another mod goes through the run time's pick (the Mods tab order, runtime/patches.c). """ import argparse +import hashlib import json import os import struct @@ -36,18 +55,30 @@ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'recomp')) -from nso import Nso # noqa: E402 +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from cheats import fnv # noqa: E402 +from exefs_patches import EXEFS_PLACES, builtin_sites # noqa: E402 +from nso import R_AARCH64_JUMP_SLOT, Nso # noqa: E402 ALLOWED_SVCS = {0x06: 'QueryMemory', 0x1E: 'GetSystemTick', 0x27: 'OutputDebugString', 0x29: 'GetInfo', 0x74: 'MapProcessMemory', 0x75: 'UnmapProcessMemory'} ALLOWED_INFO = {2, 3, 4, 5, 12, 13, 14, 15, 65001} -# Reviewed one by one: SNPrintf formats into the caller's buffer; GetDisplayVersion is answered by the service run; -# mersenne_twister_engine::_M_gen_rand (Full Inventory Expansion's alias picker) refills the state of the -# engine object it is given, which lives in the plugin's .bss; memcmp and strlen only read. malloc stays out: Quick -# Weapon Durability's init keeps a heap block's address in its .bss, and that block does not exist at play time. -ALLOWED_IMPORTS = {'_ZN2nn4util8SNPrintfEPcmPKcz', '_ZN2nn2oe17GetDisplayVersionEPNS0_14DisplayVersionE', 'memcmp', 'strlen', - '_ZNSt23mersenne_twister_engineImLm64ELm312ELm156ELm31ELm13043109905998158313ELm29ELm6148914691236517205' - 'ELm17ELm8202884508482404352ELm37ELm18444473444759240704ELm43ELm6364136223846793005EE11_M_gen_randEv'} +# Reviewed one by one: SNPrintf formats into the caller's buffer; GetDisplayVersion is answered by the runtime with +# the build's game version; mersenne_twister_engine::_M_gen_rand (Full Inventory Expansion's alias +# picker) refills the state of the engine object it is given, which lives in the plugin's .bss; memcmp and strlen only +# read; memcpy, memmove and memset write the buffers they are given; malloc and free use the game's heap (sdk's musl +# allocator), and since the init runs again at every start, a block it keeps exists at play time (Quick Weapon +# Durability's init keeps one in its .bss). +SAFE_LIBC = {'malloc', 'free', 'memcpy', 'memmove', 'memset', 'memcmp', 'strlen'} +MT_GEN_RAND = ('_ZNSt23mersenne_twister_engineImLm64ELm312ELm156ELm31ELm13043109905998158313ELm29ELm6148914691236517205' + 'ELm17ELm8202884508482404352ELm37ELm18444473444759240704ELm43ELm6364136223846793005EE11_M_gen_randEv') +SNPRINTF = '_ZN2nn4util8SNPrintfEPcmPKcz' +ALLOWED_IMPORTS = SAFE_LIBC | {SNPRINTF, '_ZN2nn2oe17GetDisplayVersionEPNS0_14DisplayVersionE', MT_GEN_RAND} +# The imports the code the hooks reach may call while the game runs (reach()). Each reviewed for what it touches: the +# libc ones above (the caller's buffers and the game's heap), SNPrintf (the caller's buffer). Nothing that waits, +# talks to a service or changes memory mappings. +REACH_IMPORTS = SAFE_LIBC | {SNPRINTF} +SVC_OUTPUT_DEBUG_STRING = 0x27 HOOK_IMPLS = ('exl_inline_hook_impl', 'exl_inline_float_hook_impl') STUR_X30 = 0xF81F03FE # stur x30, [sp, #-0x10] LDUR_X30 = 0xF85F03FE # ldur x30, [sp, #-0x10] @@ -241,6 +272,203 @@ def check_record(rec, impls): return hooks, reasons +# ---- the code the hooks reach +def plt_imports(nso, img, base, text_end): + """{absolute address of a PLT stub: the import its GOT slot binds} (adrp x16; ldr x17, [x16, #slot]; add x16, x16, + #slot; br x17), from the JUMP_SLOT relocations.""" + slots = {off: nso.syms[sym][0] for off, typ, sym, _ in nso.relocs if typ == R_AARCH64_JUMP_SLOT} + out = {} + for o in range(0, text_end - 16, 4): + w0, w1, _, w3 = struct.unpack_from('<4I', img, o) + if w0 & 0x9F00001F != 0x90000010 or w1 & 0xFFC003FF != 0xF9400211 or w3 != 0xD61F0220: + continue + imm = sext((w0 >> 29 & 3) | (w0 >> 5 & 0x7FFFF) << 2, 21) + slot = ((o & ~0xFFF) + (imm << 12)) + (w1 >> 10 & 0xFFF) * 8 + if slot in slots: + out[base + o] = slots[slot] + return out + + +def reach(img, base, text_end, roots, plt, stubs=()): + """What the code from roots (absolute addresses in the plugin's .text) reaches by falling through, direct branches + and direct calls: (svc immediates, {import: calls}, indirect calls, words outside the .text it branched to). A + stub's b to its trampoline counts as its return (recomp/patches.py `stub`).""" + svcs, imports, indirect, outside = set(), {}, 0, set() + seen, work = set(), list(roots) + stub_rets = {s + 12 for s in stubs} + while work: + a = work.pop() + while True: + if a in seen: + break + if a in plt: + imports[plt[a]] = imports.get(plt[a], 0) + 1 + break + if not base <= a < base + text_end: + outside.add(a) + break + seen.add(a) + w = struct.unpack_from('> 5 & 0xFFFF) + if w & 0x7C000000 == 0x14000000: # b, bl + t = branch_target(w, a) + if w >> 31: + work.append(t) + else: + a = t + continue + elif pc_relative(w, a) and pc_relative(w, a)[0][0] in ('b.cond', 'cbz', 'tbz'): + work.append(pc_relative(w, a)[1]) + elif w & 0xFFFFFC1F == 0xD63F0000: # blr + indirect += 1 + elif is_indirect(w): # br, ret: the path ends + break + elif w == 0 or w & 0xFFE0001F == 0xD4200000: # udf, brk: ends too + break + a += 4 + return svcs, imports, indirect, outside + + +def reach_check(rec, hooks, impls): + """Reasons the code the hooks reach breaks the rules (svcs, imports), and a summary line.""" + reasons, notes = [], [] + for p in (m for m in rec['modules'] if m['plugin'] and m['name'] in rec['images']): + img, base = rec['images'][p['name']], p['base'] + roots, stubs = set(), set() + for h in hooks: + for k in ('stub', 'callback', 'target'): + if h.get(k, '').startswith(p['name'] + '+'): + roots.add(base + int(h[k].split('+')[1], 16)) + if h['kind'] == 'inline' and h['stub'].startswith(p['name'] + '+'): + stubs.add(base + int(h['stub'].split('+')[1], 16)) + roots |= {a for a in impls if base <= a < base + p['text_end']} + nso = Nso(p['file']) if p.get('file') and Path(p['file']).is_file() else None + plt = plt_imports(nso, img, base, p['text_end']) if nso else {} + svcs, imports, indirect, outside = reach(img, base, p['text_end'], sorted(roots), plt, stubs) + for imm in sorted(svcs - {SVC_OUTPUT_DEBUG_STRING}): + reasons.append(f'the code its hooks reach has svc {imm:#x}') + for name in sorted(set(imports) - REACH_IMPORTS): + reasons.append(f'the code its hooks reach calls the import {name}') + notes.append(f'{p["name"]}: hooks reach {len(roots)} roots, imports {", ".join(sorted(imports)) or "none"}, ' + f'{indirect} indirect calls not followed' + (f', {len(outside)} branches out of its .text' + if outside else '')) + return reasons, notes + + +# ---- patch lines and status for the generator and the runtime +def mod_of(nso_path, mods): + """(mod folder name, path of the NSO in the mod) of a plugin file under the mods folder, else None.""" + try: + rel = Path(nso_path).resolve().relative_to(Path(mods).resolve()) + except ValueError: + return None + return rel.parts[0], '/'.join(rel.parts[1:]) + + +def hooks_of(hooks, module): + """The hooks whose stub or target lies in the plugin module: its own, when one record holds several plugins.""" + return [h for h in hooks if (h.get('stub') or h.get('target') or '').split('+')[0] == module] + + +def plugin_lines(rec, hooks, name, record_dir, module): + """The patch lines of the accepted plugin module (recomp/patches.py) for set name: the plugin line, the expected + writes (the words at its hooks' sites), the stubs, the seeds, the orig lines and the hook sites.""" + lines = [] + mods = {m['name']: m for m in rec['modules']} + p = mods[module] + img = rec['images'][p['name']] + text_sha = hashlib.sha256(img[:p['text_end']]).hexdigest() + lines.append(f'plugin {name} {p["name"]} {text_sha} {(Path(record_dir) / p["image"]).resolve().as_posix()}') + own = {int(h['site'].split('+')[1], 16) for h in hooks} + for w in rec['writes']: + if w['module'] == 'main' and w['offset'] in own: + lines.append(f'expect {name} {w["module"]} {w["offset"]:x} {w["old"]:08x} {w["new"]:08x}') + seeds, stubs = {}, {} + + def at(ref): + mod, off = ref.split('+') + return mod, int(off, 16) + + for h in hooks: + site = int(h['site'].split('+')[1], 16) + if h['kind'] == 'inline': + sm, so = at(h['stub']) + stubs.setdefault(sm, set()).add(so) + cm, co = at(h['callback']) + seeds.setdefault(cm, set()).add(co) + # the hook impl the stub calls: its bl, as check_record read it + stub_at = mods[sm]['base'] + so + impl = branch_target(struct.unpack_from(' Date: Tue, 17 Nov 2026 09:25:15 +0300 Subject: [PATCH 07/21] The release compiles plugin_init.c per install: it reads the install's plugin records from patches.h (#845) make_release.py stopped at the runtime library: plugin_init.c includes patches.h, a per-version header, so the library build refused it. Built per install like patches.c, it sees the RT_PLUGINS and RT_PLUGIN_WRITES of the install's own rebuild. Co-Authored-By: Claude Opus 5.5 --- tools/build_game.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/build_game.py b/tools/build_game.py index 84e93682..38b356e8 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -188,8 +188,10 @@ DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw' # Runtime sources compiled per install, not in the library: romfs.c includes romfs_config.h, patches.c and ov_zstd.c # the patches.h of the player's game version (its patch page is the page after that version's main; ov_zstd.c's host # functions sit on it). mouse_direct.c and ov_ability.c also register host callbacks relative to that version's patch -# page. A library source that includes one of PER_VERSION_HEADERS stops the library build (library_sources). -INSTALL_SIDE = {'romfs.c', 'patches.c', 'ov_zstd.c', 'mouse_direct.c', 'ov_ability.c'} +# page. plugin_init.c reads the install's own plugin records from patches.h (RT_PLUGINS, RT_PLUGIN_WRITES): a mod plugin +# the rebuild built in. A library source that includes one of PER_VERSION_HEADERS stops the library build +# (library_sources). +INSTALL_SIDE = {'romfs.c', 'patches.c', 'ov_zstd.c', 'mouse_direct.c', 'ov_ability.c', 'plugin_init.c'} PER_VERSION_HEADERS = ('"patches.h"', '"romfs_config.h"') LIB_NAME = 'libtotkrt.a' AR = UCRT / 'llvm-ar.exe' if WIN else on_path('llvm-ar') From b9b8f3dce1183c8ffa49976f4ad8b22655cd3162 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 09:25:24 +0300 Subject: [PATCH 08/21] Each mod plugin is built or refused on its own: one that is refused, crashes or hangs costs only itself (#845) tools/exefs_plugins.py ran every plugin's init in one service run and refused them all when one was refused. Now a run without inits lists the plugin modules, then each plugin runs in service runs of its own (TOTK_PLUGIN_ONLY, the others stay loaded): a refused svc, a crash (the FATAL line is the reason), a timeout or a failed check refuses that plugin alone, with its reason in plugins.status; the rest are built in. 4 plugins take 2.3 s. At start, plugins_start compares each plugin right after its own init and takes its branches out of main's .text before the next one, so each init sees the game's bytes, as in its service run. Only that init's aliases are flushed: the start stays at 113 ms for Full Inventory Expansion + Quick Weapon Durability (107 ms before). The installer's second lift keeps every plugin of the service runs in the layout when one is built: the records hold that layout's addresses. A refused plugin stays loaded and never starts. A mod with a built plugin and a refused one, used without its code, runs the built one (mods.c; --probe-mods case T). Co-Authored-By: Claude Opus 5.5 --- docs/mods.md | 20 +++-- installer/steps.py | 5 +- runtime/mods.c | 5 +- runtime/mods_probe.c | 40 ++++++--- runtime/plugin_init.c | 147 ++++++++++++++++++--------------- tests/exefs_plugins_test.py | 77 +++++++++++++++++ tests/installer_test.py | 35 ++++++-- tools/exefs_plugins.py | 159 ++++++++++++++++++++++++------------ 8 files changed, 340 insertions(+), 148 deletions(-) diff --git a/docs/mods.md b/docs/mods.md index 089207b9..9e0f1d76 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -103,12 +103,15 @@ Inventory Expansion's sort fixer, Quick Weapon Durability). On the Switch the pl redirects some of the game's instructions into its own code. The port does the same with the plugin's code compiled ahead of time, after the design in [#845](https://github.com/valentderah/totk-pc-recomp/issues/845) by itaskaev2: -1. The rebuild (the same one as for code patches above) compiles the plugin with the game, starts the game's code - once in a service mode that runs only the plugin's start-up (`totk.exe --plugin-init`), twice, and records which - instructions it redirects. Each redirect becomes a code site of the mod, built into `totk.exe`. -2. At every start the plugin's start-up runs again, before the game's own start-up code, in the same setting. Its - redirects are compared with the record of the rebuild. When they match, the mod's code sites go on. When they do - not, the plugin stays off for this run, the mod's row says why, and the next start runs the rebuild. +1. The rebuild (the same one as for code patches above) compiles the plugins with the game, then starts the game's + code in a service mode that runs only one plugin's start-up (`totk.exe --plugin-init`), twice per plugin, and + records which instructions it redirects. Each redirect becomes a code site of the mod, built into `totk.exe`. Each + plugin is judged on its own: one that is refused, or whose start-up crashes or never returns, stays off with its + reason, and the other plugins are built in. +2. At every start each plugin's start-up runs again, before the game's own start-up code, in the same setting: it + sees the game's own instructions, as in its service run. Its redirects are compared with the record of the + rebuild. When they match, the mod's code sites go on. When they do not, the plugin stays off for this run, the + mod's row says why, and the next start runs the rebuild. 3. The plugin keeps what its start-up made (its memory, its tables), so its code works as on the Switch. The start costs about 50 ms per plugin; the game runs no differently without a plugin. @@ -119,8 +122,9 @@ The rebuild refuses a plugin when its start-up or the code its redirects reach n system calls other than the few its start-up needs (memory queries and maps, the clock, debug output), library calls outside a reviewed list (memory and string functions, the heap, text formatting), threads, services or input; a redirect that is not one of exlaunch's hook shapes (inline hook, replace hook, call-site rewrite); or two runs of its -start-up that differ in code. The row then says **Its plugin ... cannot run in the port** with the reason, and the -mod stays off; **Use without its code** uses the rest of it. UltraCam stays refused: it hooks the graphics API, +start-up that differ in code; or a start-up that crashes the service run or does not return within two minutes. The +row then says **Its plugin ... cannot run in the port** with the reason, and the mod stays off; **Use without its +code** uses the rest of it, including its other plugins that were built in. UltraCam stays refused: it hooks the graphics API, reads input and opens sockets. `plugin_trace = 1` in `totk.ini` logs the first call into each of a plugin's hooks (`plugins: hook entry ... reached`). diff --git a/installer/steps.py b/installer/steps.py index 08ff4baa..2510e376 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -1009,7 +1009,10 @@ class Lift(Step): skipped['warn.plugins_skipped'] = log else: patches = ctx.temp / 'patches-plugins.txt' - lift_plugins = built_plugins(plugins_status(ctx), plugins, ctx.dest / 'mods') + # With one plugin built, the layout keeps every plugin of the service runs: the records hold addresses + # of that layout. A refused plugin stays loaded at play, its init never runs (no patch set). + if built_plugins(plugins_status(ctx), plugins, ctx.dest / 'mods'): + lift_plugins = plugins if lift_plugins: args += ['--plugins', ','.join(str(f) for f in lift_plugins)] code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches], parse) diff --git a/runtime/mods.c b/runtime/mods.c index 5d7667da..0d972f51 100644 --- a/runtime/mods.c +++ b/runtime/mods.c @@ -446,11 +446,12 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) { int unsupported = exefs_check(m, path); // A mod whose code part the port cannot use is off as a whole: its RomFS files may expect that code. Used without // its code (its switch, or mods_without_code = 1 for every mod), the code modules (m->code) and the ExeFS patch - // files the port cannot apply are left out and the rest of the mod is used, when it has a rest. + // files the port cannot apply are left out and the rest of the mod is used, when it has a rest. Each plugin of a + // mod is built or refused on its own (tools/exefs_plugins.py), so a built plugin is such a rest too. m->no_code = listed(cfg_get("TOTK_MODS_NO_CODE"), m->name); m->without_code = m->no_code || cfg_bool("TOTK_MODS_WITHOUT_CODE"); int usable = m->nsets || m->ndata || m->kind & (MOD_ROMFS | MOD_CHEATS); - int code_off = m->code[0] && (!m->without_code || !(m->kind & (MOD_ROMFS | MOD_EXEFS | MOD_CHEATS))); + int code_off = m->code[0] && (!m->without_code || !(m->kind & (MOD_ROMFS | MOD_EXEFS | MOD_CHEATS) || m->nplugins)); m->problem = strchr(m->name, '|') ? MOD_BAD_NAME : code_off ? MOD_PLUGIN : unsupported && (!m->without_code || !usable) ? MOD_EXEFS_UNSUPPORTED diff --git a/runtime/mods_probe.c b/runtime/mods_probe.c index 473ea1e5..0d1e22c0 100644 --- a/runtime/mods_probe.c +++ b/runtime/mods_probe.c @@ -212,6 +212,8 @@ static const char *refused_plugins(void) { // Q an ExeFS patch file on P's site // R a plugin the build has no line for waits for the rebuild // S RomFS + a refused plugin off, the reason kept; used without its code, the rest runs +// T a built plugin + a refused one each judged alone: off for the refused one, its reason kept; used +// without its code, the built one's set runs // Then P's start differs from the build (mods_plugin_off): its set goes off, Q's code runs at the site, the conflict // goes, a stale line marks P's file, and the next scan has P wait for the rebuild. static void probe_plugins(const char *root) { @@ -221,18 +223,22 @@ static void probe_plugins(const char *root) { char q[160]; snprintf(q, sizeof q, "Q/exefs/%s.ips", bid); static const char *const files[] = {"P/romfs/p.bin", "P/exefs/subsdk9", "R/exefs/subsdk9", "S/romfs/s.bin", - "S/exefs/subsdk9"}; + "S/exefs/subsdk9", "T/exefs/subsdk8", "T/exefs/subsdk9"}; for (size_t i = 0; i < sizeof files / sizeof *files; i++) put(root, files[i]); put(root, q); - static char names[3][16]; - static const char *const table[] = {"fps60", names[1], names[2]}; + static char names[4][16]; + static const char *const table[] = {"fps60", names[1], names[2], names[3]}; set_name(names[1], sizeof names[1], "P/exefs/subsdk9"); set_name(names[2], sizeof names[2], q); - char status[256], plugins[256]; + set_name(names[3], sizeof names[3], "T/exefs/subsdk9"); + char status[256], plugins[512]; snprintf(status, sizeof status, "%08X\t%08X\tcode\t-\t\n", fnv(q, strlen(q)), fnv("probe", 5)); snprintf(plugins, sizeof plugins, "%08X\t%08X\tplugin\tfps60\tplugin0, 7 hooks\n" - "%08X\t%08X\tunsupported\t-\tthe code its hooks reach has svc 0x40\n", fnv("P/exefs/subsdk9", 15), - fnv("probe", 5), fnv("S/exefs/subsdk9", 15), fnv("probe", 5)); + "%08X\t%08X\tunsupported\t-\tthe code its hooks reach has svc 0x40\n" + "%08X\t%08X\tplugin\t-\tplugin1, 2 hooks\n" + "%08X\t%08X\tunsupported\t-\tthe run was refused: svc 0x1 at plugin2+0x50a0\n", fnv("P/exefs/subsdk9", 15), + fnv("probe", 5), fnv("S/exefs/subsdk9", 15), fnv("probe", 5), fnv("T/exefs/subsdk9", 15), fnv("probe", 5), + fnv("T/exefs/subsdk8", 15), fnv("probe", 5)); static const int16_t s0[] = {1, 2}; static const RtPatchPick picks[] = {{"main", 0x100, s0, 2}}; mods_probe_cache(status, ""); @@ -244,12 +250,13 @@ static void probe_plugins(const char *root) { cfg_set("TOTK_MODS_ORDER", round ? "Q|P" : "P|Q"); mods_rescan(); mods_scan(); - rt_patch_probe_table(table, 3, picks, 1); + rt_patch_probe_table(table, 4, picks, 1); rt_patch_switch(0, 1); mods_exefs_apply(0, 0, 0, NULL); - const ModInfo *pm = by_name("P"), *qm = by_name("Q"), *rm = by_name("R"), *sm = by_name("S"); - CHECK(pm && qm && rm && sm, "plugin scan: four mods"); - if (!pm || !qm || !rm || !sm) break; + const ModInfo *pm = by_name("P"), *qm = by_name("Q"), *rm = by_name("R"), *sm = by_name("S"), + *tm = by_name("T"); + CHECK(pm && qm && rm && sm && tm, "plugin scan: five mods"); + if (!pm || !qm || !rm || !sm || !tm) break; CHECK(pm->problem == MOD_OK && pm->nsets == 1 && pm->nplugins == 1 && pm->plugin_hooks == 7 && !pm->code[0] && EQ(pm->overrides, "fps60"), "a built plugin: its set, its hooks, the built-in set it replaces"); CHECK(rt_patch_enabled(1) && !rt_patch_enabled(0), "the plugin's set on, the built-in set it replaces off"); @@ -261,25 +268,32 @@ static void probe_plugins(const char *root) { CHECK(rm->problem == MOD_EXEFS_LATER && rm->pending == 1, "a plugin the build lacks waits for the rebuild"); CHECK(sm->problem == MOD_PLUGIN && EQ(sm->code, "subsdk9") && strstr(sm->plugin_note, "svc 0x40"), "a refused plugin keeps its mod off and says why"); + CHECK(tm->problem == MOD_PLUGIN && tm->nplugins == 1 && tm->plugin_hooks == 2 && EQ(tm->code, "subsdk8") && + strstr(tm->plugin_note, "svc 0x1") && !rt_patch_enabled(3), + "a built plugin beside a refused one: the mod off, the refused one's reason on its row"); } // P above Q, then P's start differs from the build cfg_set("TOTK_MODS_ORDER", "P|Q"); mods_rescan(); mods_scan(); - rt_patch_probe_table(table, 3, picks, 1); + rt_patch_probe_table(table, 4, picks, 1); mods_exefs_apply(0, 0, 0, NULL); mods_plugin_off(1, "probe: 1 of the words the build recorded were not written"); const ModInfo *pm = by_name("P"); CHECK(!rt_patch_enabled(1) && rt_patch_pick_winner(0) == 1, "a plugin off at start: the other mod's patch runs"); CHECK(pm && strstr(pm->plugin_off, "were not written"), "its row says why"); CHECK(mods_conflict_count() == 0, "its conflict goes with it"); - cfg_set("TOTK_MODS_NO_CODE", "S"); + cfg_set("TOTK_MODS_NO_CODE", "S|T"); mods_rescan(); mods_scan(); + rt_patch_probe_table(table, 4, picks, 1); + mods_exefs_apply(0, 0, 0, NULL); pm = by_name("P"); - const ModInfo *sm = by_name("S"); + const ModInfo *sm = by_name("S"), *tm = by_name("T"); CHECK(pm && pm->problem == MOD_EXEFS_LATER && pm->pending == 1, "the next start: its file waits for the rebuild"); CHECK(sm && sm->problem == MOD_OK && sm->plugin_note[0], "a refused plugin used without its code: the rest runs"); + CHECK(tm && tm->problem == MOD_OK && tm->plugin_note[0] && rt_patch_enabled(3), + "a built plugin beside a refused one, used without its code: the built one's set runs"); rt_patch_probe_table(NULL, 0, NULL, 0); cfg_set("TOTK_MODS_NO_CODE", ""); cfg_set("TOTK_MODS_ORDER", ""); diff --git a/runtime/plugin_init.c b/runtime/plugin_init.c index 227f6621..ffbfcd3a 100644 --- a/runtime/plugin_init.c +++ b/runtime/plugin_init.c @@ -2,9 +2,11 @@ // this runtime's own kernel and sdk. The design follows #845 by itaskaev2, with the init run natively instead of under // Unicorn, and run again at every start instead of an image taken after one run at build time. // -// The service run: totk.exe --plugin-init DIR runs every plugin's DT_INIT, records what the init did and exits before -// any game module's constructor, nnMain, graphics or audio. The record is the input of tools/exefs_plugins.py, which -// checks it and writes the plugin's patch set (its hook sites, recomp/patches.py `plugin` lines). +// The service run: totk.exe --plugin-init DIR runs the plugins' DT_INITs, records what the inits did and exits before +// any game module's constructor, nnMain, graphics or audio. TOTK_PLUGIN_ONLY names the one plugin module whose init +// runs ("-": none); the others stay loaded, as they are at play. tools/exefs_plugins.py runs each plugin on its own, +// so a plugin that is refused, crashes or hangs costs only itself; it checks each record and writes the plugin's patch +// set (its hook sites, recomp/patches.py `plugin` lines). // // At every start (plugins_start, from the module-init callback in boot.c): when the build has a plugin whose patch set // its mod switched on, sdk's DT_INIT runs first, then the plugin's, in the same environment as in the service run, and @@ -15,8 +17,11 @@ // its GOT, its alias addresses. What the init wrote into the game modules (its hook branches) and the plugin's .text // after it (its stubs and trampolines) are compared with the record the build used (patches.h RT_PLUGIN_WRITES and // RT_PLUGINS): equal, the plugin's set stays on and the generated hook sites call into it; different, the set goes off -// for this run, the reason is logged and the mod waits for a rebuild (mods.c mods_plugin_off). The branches written -// into main's .text are taken back afterwards: the lifted code is the code that runs, and main keeps the game's bytes. +// for this run, the reason is logged and the mod waits for a rebuild (mods.c mods_plugin_off). Each plugin is compared +// right after its own init, and the branches it wrote into main's .text are taken back before the next init: the next +// plugin sees the game's own bytes, as in its own service run (two plugins hooking one site each relocate the game's +// instruction; the generated site runs the one the Mods tab order picks). The lifted code is the code that runs, and +// main keeps the game's bytes. // Without a plugin in the build none of this runs (RT_PLUGIN_COUNT 0). // // What exlaunch's init needs from the process, and how this run gives it: @@ -37,10 +42,10 @@ // Map/UnmapProcessMemory); any other one, such as CreateSession and ReplyAndReceive of exlaunch's process-handle // fallback, ends the run with the reason in the record. // -// The record, in DIR: record.json (modules, the writes into every game module as words, the alias calls, svc and -// import counts, the tick offset, a refusal), plugin.img (the plugin's image after its init, all segments) and -// plugin.pre.img (the same after loading and relocation, before the init). TOTK_PLUGIN_TICK_OFFSET adds to every -// GetSystemTick, so two runs pick different aliases. +// The record, in DIR: record.json (modules, which plugins' inits ran, the writes into every game module as words, the +// alias calls, svc and import counts, the tick offset, a refusal), plugin.img (the plugin's image after its init, +// all segments) and plugin.pre.img (the same after loading and relocation, before the init). +// TOTK_PLUGIN_TICK_OFFSET adds to every GetSystemTick, so two runs pick different aliases. #include #include #include @@ -93,6 +98,7 @@ static uint64_t svc_count[0x80], info_count[16], info_65001; static struct { const char *name; uint64_t calls; } imports[MAX_IMPORTS]; static int nimports; static uint8_t **pre; // per loaded module: its image before the plugins' inits +static uint8_t *ran; // service run, per loaded module: its init ran static uint64_t align_up(uint64_t v, uint64_t a) { return (v + a - 1) & ~(a - 1); } @@ -344,6 +350,7 @@ static void write_record(const char *refused) { i ? "," : "", m->name, (unsigned long long)m->base, (unsigned long long)m->size, (unsigned long long)m->text_end, (unsigned long long)m->data_off, (unsigned long long)m->init, m->plugin ? "true" : "false"); + if (m->plugin) fprintf(f, ", \"ran\": %s", ran && ran[i] ? "true" : "false"); if (m->plugin && !refused) { char img[64], pre_img[64], hex[65]; snprintf(img, sizeof img, "%s.img", m->name); @@ -460,9 +467,17 @@ void plugin_init_run(Cpu *c, const RtLoaded *sdk) { if (!pre[i]) rt_fatal("plugin-init: out of memory for a copy of %s", m->name); memcpy(pre[i], (const void *)(uintptr_t)m->base, m->size); } + const char *only = cfg_get("TOTK_PLUGIN_ONLY"); + ran = calloc((size_t)ldr_count(), 1); + if (!ran) rt_fatal("plugin-init: out of memory"); for (int i = ldr_count() - 1; i >= 0; i--) { const RtLoaded *m = ldr_module(i); if (!m->plugin) continue; + if (only && *only && strcmp(only, m->name)) { + rt_log("plugin-init: %s stays loaded, its init does not run (TOTK_PLUGIN_ONLY=%s)", m->name, only); + continue; + } + ran[i] = 1; if (!m->init) refuse("a plugin without DT_INIT"); rt_log("plugin-init: ENTER %s DT_INIT at %#llx (%llu init_array entries)", m->name, (unsigned long long)m->init, (unsigned long long)(m->init_array_size / 8)); @@ -546,17 +561,59 @@ static void trace_entries(void) { rt_log("plugins: trace: %d hook entries counted", ntraced); } -// Takes back the words the inits changed in the sealed part (.text, .rodata) of a game module: main keeps the game's -// own bytes, the lifted code being what runs. -static void restore_sealed(const RtLoaded *m, const uint8_t *was) { - for (uint64_t off = 0; off < m->data_off; off += PAGE) { - const uint8_t *now = (const uint8_t *)(uintptr_t)(m->base + off); - if (!memcmp(now, was + off, PAGE)) continue; - if (!plat_mem_protect(m->base + off, PAGE, PLAT_PROT_RW)) rt_fatal("plugins: cannot unseal %s+%#llx", m->name, - (unsigned long long)off); - memcpy((void *)(uintptr_t)(m->base + off), was + off, PAGE); - plat_mem_protect(m->base + off, PAGE, PLAT_PROT_READ); +// Takes back a page an init changed in the sealed part (.text, .rodata) of a game module: main keeps the game's own +// bytes, the lifted code being what runs. +static void restore_page(const RtLoaded *m, uint64_t off, const uint8_t *was) { + if (!plat_mem_protect(m->base + off, PAGE, PLAT_PROT_RW)) rt_fatal("plugins: cannot unseal %s+%#llx", m->name, + (unsigned long long)off); + memcpy((void *)(uintptr_t)(m->base + off), was, PAGE); + plat_mem_protect(m->base + off, PAGE, PLAT_PROT_READ); +} + +// After plugin i's init: what it wrote into the game modules, word by word, against the record of the build (its +// entries of RT_PLUGIN_WRITES). The first difference goes into why (unless a reason is there already). Then the game +// modules are made what they were before the init: the sealed part (.text, .rodata) gets the game's bytes back, a +// write elsewhere stays and goes into pre, so the next plugin's init is compared only with what it writes itself. +// The aliases this init mapped (from first_alias on) are flushed first; an earlier init's were flushed after it. +static void check_writes(int plugin, int first_alias, char *why, size_t cap) { + for (int i = first_alias; i < naliases; i++) + if (aliases[i].live) flush(&aliases[i]); + uint8_t seen[NWRITES > 0 ? NWRITES : 1]; + memset(seen, 0, sizeof seen); + int unexpected = 0, missing = 0; + char first[160] = ""; + for (int i = 0; i < ldr_count(); i++) { + const RtLoaded *m = ldr_module(i); + if (!pre[i]) continue; + uint8_t *now = (uint8_t *)(uintptr_t)m->base; + for (uint64_t off = 0; off < m->size; off += PAGE) { + if (!memcmp(now + off, pre[i] + off, PAGE)) continue; + for (uint64_t w = off; w < off + PAGE; w += 4) { + uint32_t a, b; + memcpy(&a, pre[i] + w, 4); + memcpy(&b, now + w, 4); + if (a == b) continue; + int k = 0; + while (k < NWRITES && !(write_tab[k].plugin == plugin && write_tab[k].off == w && + !strcmp(write_tab[k].module, m->name) && write_tab[k].old == a && + write_tab[k].now == b)) + k++; + if (k < NWRITES) { + seen[k] = 1; + } else if (!unexpected++) { + snprintf(first, sizeof first, "%s+%#llx %08x -> %08x", m->name, (unsigned long long)w, a, b); + } + } + if (off < m->data_off) restore_page(m, off, pre[i] + off); + else memcpy(pre[i] + off, now + off, PAGE); + } } + for (int k = 0; k < NWRITES; k++) missing += write_tab[k].plugin == plugin && !seen[k]; + if (why[0]) return; + if (unexpected) + snprintf(why, cap, "its init wrote %d words the build did not record (first %s)", unexpected, first); + else if (missing) + snprintf(why, cap, "%d of the words the build recorded were not written", missing); } #endif @@ -597,62 +654,22 @@ int plugins_start(Cpu *c, const RtLoaded *sdk) { const RtLoaded *m = loaded(plugin_tab[i].module); rt_log("plugins: ENTER %s DT_INIT at %#llx", m->name, (unsigned long long)m->init); failed[0] = 0; + int first_alias = naliases; running = 1; rt_trace |= RT_TRACE_PLUGIN; rt_call(c, m->init, 0); rt_trace &= ~(unsigned)RT_TRACE_PLUGIN; running = 0; if (failed[0]) snprintf(why[i], sizeof why[i], "its init asked for what the port does not serve: %s", failed); - } - for (int i = 0; i < naliases; i++) - if (aliases[i].live) flush(&aliases[i]); - // What the inits wrote into the game modules, word by word, against the record of the build. - uint8_t seen[NWRITES > 0 ? NWRITES : 1]; - memset(seen, 0, sizeof seen); - int unexpected = 0; - char first[160] = ""; - for (int i = 0; i < ldr_count(); i++) { - const RtLoaded *m = ldr_module(i); - if (!pre[i]) continue; - const uint8_t *now = (const uint8_t *)(uintptr_t)m->base; - for (uint64_t off = 0; off < m->size; off += PAGE) { - if (!memcmp(now + off, pre[i] + off, PAGE)) continue; - for (uint64_t w = off; w < off + PAGE; w += 4) { - uint32_t a, b; - memcpy(&a, pre[i] + w, 4); - memcpy(&b, now + w, 4); - if (a == b) continue; - int k = 0; - while (k < NWRITES && !(write_tab[k].off == w && !strcmp(write_tab[k].module, m->name) && - write_tab[k].old == a && write_tab[k].now == b)) - k++; - if (k < NWRITES) { - seen[k] = 1; - } else if (!unexpected++) { - snprintf(first, sizeof first, "%s+%#llx %08x -> %08x", m->name, (unsigned long long)w, a, b); - } - } - } - restore_sealed(m, pre[i]); - free(pre[i]); - } - free(pre); - pre = NULL; - for (int i = 0; i < RT_PLUGIN_COUNT; i++) { - if (!run[i] || why[i][0]) continue; - const RtLoaded *m = loaded(plugin_tab[i].module); - int missing = 0; - for (int k = 0; k < NWRITES; k++) missing += write_tab[k].plugin == i && !seen[k]; + check_writes(i, first_alias, why[i], sizeof why[i]); char hex[65]; sha256_hex((const void *)(uintptr_t)m->base, m->text_end, hex); - if (unexpected) - snprintf(why[i], sizeof why[i], "the inits wrote %d words the build did not record (first %s)", unexpected, - first); - else if (missing) - snprintf(why[i], sizeof why[i], "%d of the words the build recorded were not written", missing); - else if (strcmp(hex, plugin_tab[i].sha256)) + if (!why[i][0] && strcmp(hex, plugin_tab[i].sha256)) snprintf(why[i], sizeof why[i], "its code after the init is not the one the build lifted"); } + for (int i = 0; i < ldr_count(); i++) free(pre[i]); + free(pre); + pre = NULL; mem_query_hook = NULL; // the game's own QueryMemory sees its address space as without plugins rt_process_memory = NULL; // a callback that maps process memory stops with a named fatal (svc.c) int on = 0; diff --git a/tests/exefs_plugins_test.py b/tests/exefs_plugins_test.py index 7687d6c9..5243569f 100644 --- a/tests/exefs_plugins_test.py +++ b/tests/exefs_plugins_test.py @@ -379,6 +379,83 @@ class Lines(unittest.TestCase): self.assertFalse(any(x.startswith(('orig', 'stub')) for x in out)) +# ---- one service run per plugin +class PerPlugin(unittest.TestCase): + """main() with three plugins in three mods and a stand-in for the service run: plugin0 hooks its site, plugin1's + init ends the run without a record (a crash), plugin2's run is refused (an svc). Each gets its own status line; + only plugin0's set goes into the patch file.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.d = Path(tmp.name) + self.files = [] + for name in ('A', 'B', 'C'): + f = self.d / 'mods' / name / 'exefs' / 'subsdk9' + f.parent.mkdir(parents=True) + f.write_bytes(nso(PLUGIN_TEXT) + name.encode()) + self.files.append(f) + (self.d / 'patches.txt').write_text('set fps60 TOTK_FPS60 1\n') + self.runs = [] + saved = xp.run, xp.hook_impls + self.addCleanup(lambda: (setattr(xp, 'run', saved[0]), setattr(xp, 'hook_impls', saved[1]))) + xp.run, xp.hook_impls = self.fake_run, lambda rec: IMPLS + + def fake_run(self, exe, out, tick_offset, timeout, only): + self.runs.append(only) + out.mkdir(parents=True, exist_ok=True) + if only == 'plugin1': + return 0xC0000005, 'its init ended the service run without a record (exit 0xc0000005)' + rec = record(writes=[] if only != 'plugin0' else None) + rec['modules'][2]['file'] = str(self.files[0]) + for i in (1, 2): + rec['modules'].append(dict(rec['modules'][2], name=f'plugin{i}', base=PLUGIN + i * 0x100000, + file=str(self.files[i]))) + for m in rec['modules'][2:]: + m['ran'] = m['name'] == only + if only == 'plugin2': + rec['refused'] = 'svc 0x1 at plugin2+0x50a0' + (out / 'plugin0.img').write_bytes(rec.pop('images')['plugin0']) + rec['modules'][2]['image'] = 'plugin0.img' + (out / 'record.json').write_text(json.dumps(rec)) + return 3 if rec['refused'] else 0, None + + def test_each_plugin_alone(self): + out, status, patched = self.d / 'rec', self.d / 'plugins.status', self.d / 'patches-out.txt' + argv = sys.argv + sys.argv = ['exefs_plugins.py', '--exe', 'totk.exe', '--out', str(out), '--mods', str(self.d / 'mods'), + '--patches', str(self.d / 'patches.txt'), '--patches-out', str(patched), '--status', str(status)] + try: + code = xp.main() + finally: + sys.argv = argv + self.assertEqual(code, 1) + self.assertEqual(self.runs, ['-', 'plugin0', 'plugin0', 'plugin1', 'plugin2']) + lines = {line.split('\t')[0]: line.split('\t') for line in status.read_text().splitlines()} + key = {n: xp.fnv(f'{n}/exefs/subsdk9'.encode()) for n in 'ABC'} + self.assertEqual(lines[key['A']][2:], ['plugin', '-', 'plugin0, 1 hooks']) + self.assertEqual(lines[key['B']][2], 'unsupported') + self.assertIn('ended the service run', lines[key['B']][4]) + self.assertEqual(lines[key['C']][2], 'unsupported') + self.assertIn('svc 0x1', lines[key['C']][4]) + text = patched.read_text() + self.assertIn(f'set mod_{key["A"].lower()} ', text) + self.assertNotIn(f'mod_{key["B"].lower()}', text) + self.assertNotIn(f'mod_{key["C"].lower()}', text) + self.assertIn((out / 'plugin0' / 'run1' / 'plugin0.img').resolve().as_posix(), text) + + def test_ran_view_leaves_out_plugins_that_did_not_run(self): + rec = record() + rec['modules'].append(dict(rec['modules'][2], name='plugin1', base=PLUGIN + 0x100000, ran=False)) + rec['modules'][2]['ran'] = True + rec['images']['plugin1'] = rec['images']['plugin0'] + view = xp.ran_view(rec) + self.assertEqual([m['name'] for m in view['modules'] if m['plugin']], ['plugin0']) + self.assertEqual(set(view['images']), {'plugin0'}) + whole = record() # no "ran": every init ran, the record is its own view + self.assertIs(xp.ran_view(whole), whole) + + # ---- the generator on those lines def code_nso(words_at, size_text=0x5000): """An NSO as nso() whose .text holds words_at {offset: word}.""" diff --git a/tests/installer_test.py b/tests/installer_test.py index f890a1c8..57d09a32 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -383,10 +383,14 @@ class LiftRunTest(unittest.TestCase): self.assertEqual(self.outputs(), [False, False]) self.assertIn('warn.exefs_skipped', self.ctx.reporter.out.getvalue()) - def plugin_setup(self, kind): - """A mod with an exlaunch plugin; the fake service run writes its status line of that kind.""" + def plugin_setup(self, kind, more=()): + """A mod with an exlaunch plugin (SQ), and one more mod per (name, kind) of more; the fake service run writes + each plugin's status line of its kind.""" + kinds = {'SQ': kind, **dict(more)} + for name in kinds: + (self.dest / 'mods' / name / 'exefs').mkdir(parents=True, exist_ok=True) + (self.dest / 'mods' / name / 'exefs' / 'subsdk9').write_bytes(b'NSO0' + name.encode()) plugin = self.dest / 'mods' / 'SQ' / 'exefs' / 'subsdk9' - plugin.write_bytes(b'NSO0') self.ctx.layout = SimpleNamespace(**vars(self.ctx.layout), llvm='llvm', runtime='runtime', ninja='ninja', deps=None, sysroot=None, runtime_lib_for=lambda cpu: None, pgo=lambda version: None, expected=lambda version: 'expected.json') @@ -399,11 +403,14 @@ class LiftRunTest(unittest.TestCase): Path(args[args.index('--patches-out') + 1]).write_text('patches\n') status = Path(args[args.index('--status') + 1]) status.parent.mkdir(parents=True, exist_ok=True) - h = 0x811C9DC5 # FNV-1a 32 of "SQ/exefs/subsdk9", as tools/exefs_plugins.py keys it - for b in b'SQ/exefs/subsdk9': - h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF - status.write_text(f'{h:08X}\t00000000\t{kind}\t-\tnote\n') - return (1 if kind == 'unsupported' else 0), log + lines = [] + for name, k in kinds.items(): + h = 0x811C9DC5 # FNV-1a 32 of "/exefs/subsdk9", as tools/exefs_plugins.py keys it + for b in f'{name}/exefs/subsdk9'.encode(): + h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF + lines.append(f'{h:08X}\t00000000\t{k}\t-\tnote\n') + status.write_text(''.join(lines)) + return (1 if 'unsupported' in kinds.values() else 0), log return code, log steps.run_tool = run_tool return plugin @@ -429,6 +436,18 @@ class LiftRunTest(unittest.TestCase): self.assertTrue(steps.plugins_status(self.ctx).is_file()) self.assertNotIn('warn.plugins_skipped', self.ctx.reporter.out.getvalue()) + def test_refused_plugin_stays_in_the_layout_of_a_built_one(self): + # one plugin built, one refused: the lift with the hook sites keeps both in the layout (the records hold the + # addresses of the service runs' layout); the refused one has no patch set, its status line says why + self.plugin_setup('plugin', [('ZZ', 'unsupported')]) + steps.Lift().run(self.ctx) + last = self.calls[4][1] + mods = self.dest / 'mods' + self.assertEqual(last[last.index('--plugins') + 1], + ','.join(str(mods / n / 'exefs' / 'subsdk9') for n in ('SQ', 'ZZ'))) + self.assertIn('unsupported', steps.plugins_status(self.ctx).read_text()) + self.assertNotIn('warn.plugins_skipped', self.ctx.reporter.out.getvalue()) + def test_failed_plugin_build_warns(self): self.plugin_setup('plugin') self.fail = {'build_game.py'} diff --git a/tools/exefs_plugins.py b/tools/exefs_plugins.py index a0ef82ff..73421bdb 100644 --- a/tools/exefs_plugins.py +++ b/tools/exefs_plugins.py @@ -4,8 +4,8 @@ a plugin's hook sites. After the design in #845 by itaskaev2; the init runs on t same environment (plugin_init.c plugins_start), so the plugin keeps real state (its heap blocks, its GOT, its alias addresses). The record says what the build expects that start-up to write. - python tools/exefs_plugins.py --exe build/-x/totk.exe --out # two runs, their diff, the check - python tools/exefs_plugins.py --check /run1 # the check of one record + python tools/exefs_plugins.py --exe build/-x/totk.exe --out # per plugin: two runs, diff, check + python tools/exefs_plugins.py --check //run1 # the check of one record python tools/exefs_plugins.py --exe --out --mods --patches --patches-out --status # and the patch lines for the generator @@ -20,11 +20,16 @@ file hash: FNV-1a 32 of "/"; conten `plugin` (note: the module and its hook count) or `unsupported` (note: the first reason); built-in sets: the port's own sets with a site the plugin hooks too ("-" none), which the runtime switches off while the mod is on. -The exe must be built with the plugin in its layout (recomp/gen.py --plugins). --out gets run1/ and run2/, one record -each; run2 adds a large offset to every GetSystemTick (TOTK_PLUGIN_TICK_OFFSET), so the init picks other alias -addresses. The words that differ between the two records are listed with what they look like, not refused: an init -keeps its alias addresses and the regions GetInfo gave it in its .data, and a plugin's callbacks must not use them -(the generator's check of the code the callbacks reach, still to come). +The exe must be built with the plugins in its layout (recomp/gen.py --plugins). Each plugin runs in service runs of +its own (TOTK_PLUGIN_ONLY): the other plugins stay loaded, their inits do not run. A plugin that is refused, crashes, +hangs or fails the check is refused alone, with its reason; the others are built in. At every start the game runs each +init against the game's own bytes as well (runtime/plugin_init.c plugins_start takes a plugin's writes back before the +next init), so a plugin's own record is what its start must match. --out gets list/ (a run without inits, which names +the plugin modules), and /run1 and /run2 for each plugin; run2 adds a large offset to every +GetSystemTick (TOTK_PLUGIN_TICK_OFFSET), so the init picks other alias addresses. The words that differ between the two +records are listed with what they look like, not refused: an init keeps its alias addresses and the regions GetInfo +gave it in its .data, and a plugin's callbacks must not use them. A difference in code (a write into a game module, +the plugin's .text or .rodata) refuses the plugin: its start could not be compared with the record. The check (check_record) accepts a record only when every rule holds; each broken rule is one reason: - the run was not refused (an svc outside the served set, such as CreateSession of exlaunch's process-handle @@ -90,6 +95,19 @@ def u(v): return int(v, 16) if isinstance(v, str) else v +def ran_view(rec): + """The record as seen from the plugins whose inits ran: a plugin that only stayed loaded (TOTK_PLUGIN_ONLY) counts + as a module that is no plugin, so its exports and code are not checked as this record's. A record without "ran" + (every init ran) comes back as it is.""" + if all(m.get('ran', True) for m in rec['modules'] if m['plugin']): + return rec + view = dict(rec) + view['modules'] = [dict(m, plugin=False) if m['plugin'] and not m.get('ran', True) else m for m in rec['modules']] + names = {m['name'] for m in view['modules'] if m['plugin']} + view['images'] = {k: v for k, v in rec['images'].items() if k in names} + return view + + # ---- instructions def sext(v, bits): return v - (1 << bits) if v >> (bits - 1) & 1 else v @@ -433,35 +451,35 @@ def plugin_files(mods): return out -def emit(rec, hooks, reasons, record_dir, args): - """--patches-out and --status from the record (accepted or not).""" +def emit(results, args): + """--patches-out and --status from the checked plugins. results: one (module, its NSO file, its record or None, + the record's directory, hooks, reasons) per plugin of the layout; no reasons: accepted.""" text = args.patches.read_text() if args.patches else '' builtin = builtin_sites(text) status, sets = [], [] files = {str(f.resolve()): (m, rel, f) for m, rel, f in plugin_files(args.mods)} - plugins = [m for m in rec['modules'] if m['plugin']] - for p in plugins: - hit = files.get(str(Path(p.get('file', '')).resolve())) + for module, file, rec, record_dir, hooks, reasons in results: + hit = files.get(str(Path(file or '').resolve())) if not hit: - print(f'plugins: {p["name"]}: {p.get("file")} is not under {args.mods}') + print(f'plugins: {module}: {file} is not under {args.mods}') continue mod, rel, f = hit key, content = fnv(f'{mod}/{rel}'.encode()), fnv(f.read_bytes()) if reasons: - note = reasons[0].replace('\t', ' ') + note = ' '.join(reasons[0].split()) status.append(f'{key}\t{content}\tunsupported\t-\t{note}') print(f'plugins: unsupported {mod}/{rel} ({note})') continue name = f'mod_{key.lower()}' - own = hooks_of(hooks, p['name']) + own = hooks_of(hooks, module) sites = [int(h['site'].split('+')[1], 16) for h in own] over = sorted({b for a in sites for b in builtin.get(a, [])}) label = f'{mod}/{rel}'.encode('ascii', 'replace').decode() sets.append([f'set {name} TOTK_MODPATCH_{key} 0 ordered # {label}'] + plugin_lines(rec, own, name, record_dir, - p['name'])) - status.append(f'{key}\t{content}\tplugin\t{",".join(over) or "-"}\t{p["name"]}, {len(own)} hooks') - print(f'plugins: plugin {mod}/{rel} as {p["name"]}, {len(own)} hooks' + (f', over {", ".join(over)}' - if over else '')) + module)) + status.append(f'{key}\t{content}\tplugin\t{",".join(over) or "-"}\t{module}, {len(own)} hooks') + print(f'plugins: plugin {mod}/{rel} as {module}, {len(own)} hooks' + (f', over {", ".join(over)}' + if over else '')) if sets: text += '\n# Mod plugins (tools/exefs_plugins.py)\n' + '\n'.join(line for s in sets for line in s) + '\n' args.patches_out.write_text(text) @@ -469,7 +487,7 @@ def emit(rec, hooks, reasons, record_dir, args): args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8') -# ---- two runs +# ---- the service runs def where_is(v, rec): """What an address-like value points into, for the diff listing.""" for m in rec['modules']: @@ -524,17 +542,34 @@ def diff_records(a, b): return out -def run(exe, out, tick_offset, timeout): +def run(exe, out, tick_offset, timeout, only): + """One service run into out, only module `only`'s init running ("-": none). (exit code or None, why the run gave + no record or None); a refused run writes its record too.""" out.mkdir(parents=True, exist_ok=True) - env = dict(os.environ, TOTK_PLUGIN_TICK_OFFSET=f'{tick_offset:#x}', TOTK_BACKGROUND_WINDOW='1', - TOTK_ASSUME_FOCUS='1') - r = subprocess.run([str(exe), '--plugin-init', str(out)], env=env, capture_output=True, text=True, - errors='replace', timeout=timeout) + for f in out.iterdir(): # an earlier run's record must not pass for this one's + if f.is_file() and (f.name in ('record.json', 'run.log') or f.suffix == '.img'): + f.unlink() + env = dict(os.environ, TOTK_PLUGIN_TICK_OFFSET=f'{tick_offset:#x}', TOTK_PLUGIN_ONLY=only, + TOTK_BACKGROUND_WINDOW='1', TOTK_ASSUME_FOCUS='1') + try: + r = subprocess.run([str(exe), '--plugin-init', str(out)], env=env, capture_output=True, text=True, + errors='replace', timeout=timeout) + except subprocess.TimeoutExpired as e: + log = b''.join(x if isinstance(x, bytes) else x.encode() for x in (e.stdout, e.stderr) if x) + (out / 'run.log').write_text(log.decode(errors='replace'), encoding='utf-8') + return None, f'its init did not return within {timeout} s' (out / 'run.log').write_text(r.stdout + r.stderr, encoding='utf-8') - return r.returncode + if not (out / 'record.json').is_file(): + # the runtime's crash report names the fault in a FATAL line (an access violation, an abort) + fatal = next((line.split('FATAL:', 1)[1].strip() for line in (r.stdout + r.stderr).splitlines() + if 'FATAL:' in line), None) + return r.returncode, (f'its init crashed the service run: {fatal}' if fatal else + f'its init ended the service run without a record (exit {r.returncode & 0xFFFFFFFF:#x})') + return r.returncode, None def report(rec, label): + rec = ran_view(rec) impls = hook_impls(rec) hooks, reasons = check_record(rec, impls) more, notes = reach_check(rec, hooks, impls) @@ -559,12 +594,40 @@ def report(rec, label): return hooks, reasons +def plugin_runs(exe, out, module, timeout): + """The two service runs of one plugin (run2 with the tick offset) into out/: (its first record or None, + hooks, reasons).""" + recs = [] + for i, offset in enumerate((0, TICK_OFFSET), 1): + d = out / module / f'run{i}' + code, why = run(exe, d, offset, timeout, module) + if why: + print(f'{module} run {i}: {why}; see {d / "run.log"}') + return None, [], [why] + recs.append(load_record(d)) + print(f'{module} run {i}: exit {code}, tick offset {offset:#x}') + if recs[0].get('refused'): # refused: a second run says nothing more + hooks, bad = report(recs[0], f'{module} run 1') + return recs[0], hooks, bad + hooks, bad = report(recs[0], f'{module} run 1') + diff = diff_records(ran_view(recs[0]), ran_view(recs[1])) + print(f'{module} run 1 against run 2: {len(diff)} differences' + (':' if diff else '')) + for line in diff: + print(f' {line}') + # Words of a game module or of the plugin's .text that differ between the runs: the start-up could not be compared + # with the record. Its .data and .bss may differ (alias addresses). + unstable = [line for line in diff if line.startswith('write ') or '(.text)' in line or '(.rodata)' in line] + if unstable: + bad = bad + [f'two runs of the init differ in code: {unstable[0]}'] + return recs[0], hooks, bad + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - ap.add_argument('--exe', type=Path, help='totk.exe built with the plugin in its layout') - ap.add_argument('--out', type=Path, help='directory for run1/ and run2/') + ap.add_argument('--exe', type=Path, help='totk.exe built with the plugins in its layout') + ap.add_argument('--out', type=Path, help='directory for list/ and /run1, run2') ap.add_argument('--check', type=Path, help='check the record in this directory only') - ap.add_argument('--timeout', type=int, default=300) + ap.add_argument('--timeout', type=int, default=120, help='seconds a service run may take') ap.add_argument('--mods', type=Path, help='the mods folder the plugins come from') ap.add_argument('--patches', type=Path, help='the patch file so far') ap.add_argument('--patches-out', type=Path, help='the patch file with the plugins\' sets added') @@ -578,32 +641,26 @@ def main(): rec = load_record(args.check) hooks, bad = report(rec, str(args.check)) if emitting: - emit(rec, hooks, bad, args.check, args) + emit([(m['name'], m.get('file'), rec, args.check, hooks, bad) for m in ran_view(rec)['modules'] + if m['plugin']], args) return 1 if bad else 0 if not args.exe or not args.out: ap.error('--exe and --out, or --check') - recs = [] - for i, offset in enumerate((0, TICK_OFFSET), 1): - d = args.out / f'run{i}' - code = run(args.exe, d, offset, args.timeout) - if not (d / 'record.json').is_file(): - print(f'run {i}: no record (exit {code}); see {d / "run.log"}') - return 1 - recs.append(load_record(d)) - print(f'run {i}: exit {code}, tick offset {offset:#x}') - hooks, bad = report(recs[0], 'run 1') - diff = diff_records(*recs) - print(f'run 1 against run 2: {len(diff)} differences' + (':' if diff else '')) - for line in diff: - print(f' {line}') - # Words of a game module or of a plugin's .text that differ between the runs: the start-up could not be compared - # with the record. Its .data and .bss may differ (alias addresses). - unstable = [line for line in diff if line.startswith('write ') or '(.text)' in line or '(.rodata)' in line] - if unstable: - bad = bad + [f'two runs of the init differ in code: {unstable[0]}'] + # A run without any init names the plugin modules of the exe's layout and checks that the service run works at all: + # without it no plugin can be built. + code, why = run(args.exe, args.out / 'list', 0, args.timeout, '-') + if why: + print(f'the service run without inits gave no record: {why}; see {args.out / "list" / "run.log"}') + return 2 + listing = load_record(args.out / 'list') + results = [] + for m in (m for m in listing['modules'] if m['plugin']): + rec, hooks, bad = plugin_runs(args.exe, args.out, m['name'], args.timeout) + print(f'{m["name"]}: ' + (f'refused: {bad[0]}' if bad else f'accepted, {len(hooks)} hooks')) + results.append((m['name'], m.get('file'), rec, args.out / m['name'] / 'run1', hooks, bad)) if emitting: - emit(recs[0], hooks, bad, args.out / 'run1', args) - return 1 if bad else 0 + emit(results, args) + return 1 if any(r[5] for r in results) else 0 if __name__ == '__main__': From 77f34a05f5140720e2850d4104d6b1a79ffd45cb Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 09:33:49 +0300 Subject: [PATCH 09/21] A plugin build whose plugins all stay off gives the game its own QueryMemory back (#845) plugins_start returned before clearing mem_query_hook and rt_process_memory when no plugin's set was on (the mod off, removed, or its file changed), so the game ran with the service run's view of memory: host ranges Inaccessible, the rtld stand-in below main. Co-Authored-By: Claude Opus 5.5 --- runtime/plugin_init.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runtime/plugin_init.c b/runtime/plugin_init.c index ffbfcd3a..20204f66 100644 --- a/runtime/plugin_init.c +++ b/runtime/plugin_init.c @@ -636,7 +636,11 @@ int plugins_start(Cpu *c, const RtLoaded *sdk) { } run[i] = 1, nrun++; } - if (!nrun) return 0; + if (!nrun) { // every plugin's mod off or gone: the game sees its address space as without plugins + mem_query_hook = NULL; + rt_process_memory = NULL; + return 0; + } uint64_t t0 = plat_now_ms(); rt_log("boot: ENTER %s DT_INIT first: a mod plugin's init runs before the game's constructors", sdk->name); rt_call(c, sdk->init, 0); From 81c004094db45a76b647adf6b8d5c0b82cfe64fa Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 09:52:37 +0300 Subject: [PATCH 10/21] The installer's plugin service run finds its DLLs: they go beside the totk.exe it builds in temp (#845) On the release path the service run's totk.exe in /build stopped before main with 0xc0000135 (SDL3.dll not found): the DLLs are copied next to totk.exe in the install folder only by the compile step, after the lift. Every plugin was skipped with warn.plugins_skipped. The fake-tool tests did not see it; installer_test now checks the copy. exefs_plugins_test covers run()'s hang and crash reasons. Co-Authored-By: Claude Opus 5.5 --- installer/steps.py | 3 +++ tests/exefs_plugins_test.py | 16 ++++++++++++++++ tests/installer_test.py | 8 +++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/installer/steps.py b/installer/steps.py index 2510e376..70211bf0 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -997,6 +997,9 @@ class Lift(Step): if not code: code, log = run_tool(ctx, 'lift', layout.tools / 'build_game.py', build_args(ctx, gen)) if not code: + # the service run's totk.exe loads its DLLs from its own folder, as the installed one does + for dll in layout.dlls: + replace_file(dll, ctx.temp / 'build' / dll.name) code, log = run_tool(ctx, 'lift', layout.tools / 'exefs_plugins.py', [ '--exe', ctx.temp / 'build' / ('totk' + EXE), '--out', ctx.temp / 'plugins', '--mods', ctx.dest / 'mods', '--patches', patches, '--patches-out', ctx.temp / 'patches-plugins.txt', diff --git a/tests/exefs_plugins_test.py b/tests/exefs_plugins_test.py index 5243569f..071df40b 100644 --- a/tests/exefs_plugins_test.py +++ b/tests/exefs_plugins_test.py @@ -13,6 +13,7 @@ import struct import sys import tempfile import unittest +from types import SimpleNamespace from pathlib import Path ROOT = Path(__file__).resolve().parent.parent @@ -398,6 +399,7 @@ class PerPlugin(unittest.TestCase): (self.d / 'patches.txt').write_text('set fps60 TOTK_FPS60 1\n') self.runs = [] saved = xp.run, xp.hook_impls + self.real_run = xp.run self.addCleanup(lambda: (setattr(xp, 'run', saved[0]), setattr(xp, 'hook_impls', saved[1]))) xp.run, xp.hook_impls = self.fake_run, lambda rec: IMPLS @@ -444,6 +446,20 @@ class PerPlugin(unittest.TestCase): self.assertNotIn(f'mod_{key["C"].lower()}', text) self.assertIn((out / 'plugin0' / 'run1' / 'plugin0.img').resolve().as_posix(), text) + def test_run_reasons(self): + # the real run(): a hang becomes its own reason, a crash names the FATAL line + saved = xp.subprocess.run + self.addCleanup(setattr, xp.subprocess, 'run', saved) + + def hang(cmd, **kw): + raise xp.subprocess.TimeoutExpired(cmd, kw['timeout'], output=b'started', stderr=None) + xp.subprocess.run = hang + self.assertEqual(self.real_run(None, self.d / 'h', 0, 7, 'plugin0'), (None, 'its init did not return within 7 s')) + xp.subprocess.run = lambda cmd, **kw: SimpleNamespace(returncode=0xC0000005, stdout='x\nFATAL: access violation ' + '(0xc0000005) at host 1\n', stderr='') + code, why = self.real_run(None, self.d / 'c', 0, 7, 'plugin0') + self.assertEqual(why, 'its init crashed the service run: access violation (0xc0000005) at host 1') + def test_ran_view_leaves_out_plugins_that_did_not_run(self): rec = record() rec['modules'].append(dict(rec['modules'][2], name='plugin1', base=PLUGIN + 0x100000, ran=False)) diff --git a/tests/installer_test.py b/tests/installer_test.py index 57d09a32..4f8c8941 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -391,7 +391,11 @@ class LiftRunTest(unittest.TestCase): (self.dest / 'mods' / name / 'exefs').mkdir(parents=True, exist_ok=True) (self.dest / 'mods' / name / 'exefs' / 'subsdk9').write_bytes(b'NSO0' + name.encode()) plugin = self.dest / 'mods' / 'SQ' / 'exefs' / 'subsdk9' - self.ctx.layout = SimpleNamespace(**vars(self.ctx.layout), llvm='llvm', runtime='runtime', ninja='ninja', + dll = self.dest.parent / 'release-bin' / 'SDL3.dll' + dll.parent.mkdir(exist_ok=True) + dll.write_bytes(b'MZ') + (self.ctx.temp / 'build').mkdir(parents=True, exist_ok=True) + self.ctx.layout = SimpleNamespace(**vars(self.ctx.layout), llvm='llvm', runtime='runtime', ninja='ninja', dlls=[dll], deps=None, sysroot=None, runtime_lib_for=lambda cpu: None, pgo=lambda version: None, expected=lambda version: 'expected.json') saved = self.run_tool @@ -426,6 +430,8 @@ class LiftRunTest(unittest.TestCase): self.assertEqual(last[last.index('--plugins') + 1], str(plugin)) self.assertEqual(last[last.index('--patches') + 1], str(self.ctx.temp / 'patches-plugins.txt')) self.assertTrue(steps.plugins_status(self.ctx).is_file()) + # the service run's totk.exe finds its DLLs beside it + self.assertTrue((self.ctx.temp / 'build' / 'SDL3.dll').is_file()) def test_refused_plugin_is_not_lifted(self): # a refused plugin: its status line stays (the Mods tab says why), the game is lifted without it From c97ceb64511fe84ac6eaf78886ba36cc27e4f7bc Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 10:35:32 +0300 Subject: [PATCH 11/21] A plugin stage that failed runs again once a release changes the installer: steps.py is in the lift key with plugins (#845) After warn.plugins_skipped the lift key still matched its inputs, so the next setup run (with the fixed release) found lift and compile current and never built the plugin. installer/steps.py runs that stage, so it joins the key when the mods hold a plugin. Co-Authored-By: Claude Opus 5.5 --- installer/steps.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/installer/steps.py b/installer/steps.py index 70211bf0..44387eb4 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -929,9 +929,10 @@ class Lift(Step): # fingerprint takes a file's name and content only; the paths under mods/ go in too, because tools/cheats.py # and tools/exefs_patches.py name each patch set after its mod folder, so a renamed folder needs a lift # A mod plugin is built by running its init on a totk.exe of this build (tools/exefs_plugins.py), so the - # runtime is in the key too when there is one. + # runtime is in the key too when there is one, and this file, which runs that stage: a plugin stage that failed + # (warn.plugins_skipped, the game built without the plugins) runs again once a release changes it. files = cheat_files(ctx) + exefs_files(ctx) + plugin_files(ctx) - extra = [ctx.layout.tools / 'exefs_plugins.py', runtime_key(ctx)] if plugin_files(ctx) else [] + extra = [ctx.layout.tools / 'exefs_plugins.py', Path(__file__), runtime_key(ctx)] if plugin_files(ctx) else [] return fingerprint('lift-5', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)), ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *extra, *files, *(f.relative_to(ctx.dest / 'mods').as_posix() for f in files)) From c477b67a0993421eabdb87819df498bcc5320107 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 10:35:49 +0300 Subject: [PATCH 12/21] The runtime library's module tables hold 32 modules, not the release's MOD_COUNT: an install's plugins fit (#845) loader.c and memory.c sized their tables by MOD_COUNT of the layout.h make_release.py built libtotkrt.a against (3: main, subsdk0, sdk). An install whose rebuild adds a plugin has 4, and its service run stopped at once with "loader: 4 modules in the layout"; dev builds compile those files against their own layout and never saw it. RT_MAX_MODULES (host.h) is the bound now; the loader still refuses a layout above it. Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 3 +++ runtime/loader.c | 8 ++++---- runtime/memory.c | 4 ++-- tools/make_release.py | 3 ++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index d4f8c060..8bb2804a 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -207,6 +207,9 @@ typedef struct { void ldr_load(const char *exefs_dir); // load, relocate (overrides applied), seal; once extern GuestFn ldr_plugin_import; // set before ldr_load (plugin_init.c): every JUMP_SLOT // import of a plugin binds to this observer +// The module tables of the runtime library: it is built once per release (tools/build_game.py --make-runtime-lib), +// before an install's layout (recomp/layout.py) adds its mod plugins after sdk, so they do not use that MOD_COUNT. +#define RT_MAX_MODULES 32 int ldr_count(void); const RtLoaded *ldr_module(int i); // order: main, subsdk0, sdk uint64_t ldr_symbol(const char *name); // exported definition, 0 if none diff --git a/runtime/loader.c b/runtime/loader.c index 6c698ffb..b1911fe7 100644 --- a/runtime/loader.c +++ b/runtime/loader.c @@ -47,11 +47,11 @@ typedef struct { uint64_t *plt_slot, *plt_stub, nplt; // PLT stubs sorted by GOT slot (ldr_plt_stub), built on first use // statistics uint64_t kinds[K_COUNT], sym_import, sym_own; - uint32_t imports, import_to[MOD_COUNT], weak_unresolved, strong_unresolved; + uint32_t imports, import_to[RT_MAX_MODULES], weak_unresolved, strong_unresolved; uint32_t init_off; } Mod; -static Mod mods[MOD_COUNT]; +static Mod mods[RT_MAX_MODULES]; GuestFn ldr_plugin_import; static int nmods; static uint64_t identity_refs, stubs; @@ -503,7 +503,7 @@ static void seal(Mod *m) { void ldr_load(const char *exefs_dir) { if (nmods) rt_fatal("ldr_load called twice"); if (!mem_layout()->pool) rt_fatal("ldr_load before mem_init"); - if (rt_layout_count > MOD_COUNT) rt_fatal("loader: %d modules in the layout", rt_layout_count); + if (rt_layout_count > RT_MAX_MODULES) rt_fatal("loader: %d modules in the layout", rt_layout_count); int n = 0; for (int i = 0; i < rt_layout_count; i++) { // A mod plugin's file that is gone or changed since the build is left out at play time: its patch set cannot @@ -622,7 +622,7 @@ uint64_t ldr_plt_stub(const char *module, const char *name) { typedef struct { const char *name; uint64_t kinds[K_COUNT], total, sym_import, sym_own; - uint32_t imports, to[MOD_COUNT], weak, init_off, init_array; + uint32_t imports, to[RT_MAX_MODULES], weak, init_off, init_array; } Expect; static const Expect expect_121[] = { {"main", {508816, 1415, 32, 641}, 510904, 2088, 0, 675, {0, 16, 647}, 12, 0x30, 10673}, diff --git a/runtime/memory.c b/runtime/memory.c index c452c398..5df53916 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -57,7 +57,7 @@ static uint8_t *alias_pages; // 1 per mapped 4 KiB page of the alias regi typedef struct { uint64_t dst, src, size; } StackMap; static StackMap stack_maps[MAX_STACK_MAPS]; static unsigned nstack_maps; -static uint8_t *module_perm[MOD_COUNT]; // per data page of each module: perm | ATTR_PERMISSION_LOCKED +static uint8_t *module_perm[RT_MAX_MODULES]; // per data page of each module: perm | ATTR_PERMISSION_LOCKED static uint64_t align_up(uint64_t v, uint64_t a) { return (v + a - 1) & ~(a - 1); } @@ -530,7 +530,7 @@ uint32_t mem_unmap_physical(uint64_t a, uint64_t n) { return alias_map(1, a, n); // as taken (Code, RX), so the SDK never picks it. static void taken_gap(uint64_t a, uint64_t *base, uint64_t *size) { uint64_t lo = 0, hi = RT_ADDRESS_LIMIT; - RtRange known[3 + MOD_COUNT + 2]; + RtRange known[3 + RT_MAX_MODULES + 2]; int n = 0; known[n++] = (RtRange){L.aslr.base, low_size}; known[n++] = L.alias; diff --git a/tools/make_release.py b/tools/make_release.py index 2122ccd2..5cc9ede6 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -25,7 +25,8 @@ runtime/ and bin/ (PATH holds the Windows system directory and those), then runs Both need one game version extracted and lifted first (tools/extract.py, recomp/gen.py): generated/v121 or, without it, generated/v143, and for --prove its build/extracted/ (they are read, nothing is written to them). The -runtime library reads only what both versions' layout.h share (MOD_COUNT, MOD_MAIN_BASE), so either serves. +runtime library reads only what both versions' layout.h share (MOD_MAIN_BASE; its module tables hold RT_MAX_MODULES, so an +install's layout may add mod plugins), so either serves. A release also needs build/xlate built from the current sources (tools/build_xlate.py); the script stops first if not. From aa184f9fd27f5246ce38211f0af15809533a9cf3 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 10:35:49 +0300 Subject: [PATCH 13/21] A mod plugin the build has whose file is gone starts the rebuild, which goes back to a build without it (#845) Removing a plugin mod left its module in the layout: the loader skipped the missing file and the game ran, but the rebuild that drops the plugin never came. The loader counts layout plugins whose file no longer exists (ldr_plugins_gone) and rebuild_if_needed treats them like files waiting for the rebuild. A changed file was already pending through its mod's status line. On the 1.4.3 install: The Second Quest removed, the start asked for the rebuild, the setup lifted and compiled the no-plugin build in 30.5 s, the game reached gameplay. Co-Authored-By: Claude Opus 5.5 --- runtime/cheats.c | 13 +++++++------ runtime/host.h | 1 + runtime/loader.c | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/runtime/cheats.c b/runtime/cheats.c index 11991dc3..b96962c8 100644 --- a/runtime/cheats.c +++ b/runtime/cheats.c @@ -517,8 +517,9 @@ static void plain_name(char *s) { // ---- rebuild: code cheats and ExeFS patches the build lacks // The setup writes build-cache\rebuild.txt (installer/steps.py write_rebuild_command), the command that starts its // window in rebuild mode on this install folder, one argument per line. When an enabled mod has code cheats this build -// lacks (CHEAT_REBUILD) or ExeFS patch files it lacks (mods_exefs_pending), the game runs it and exits; the setup lifts -// and compiles them in and starts the game again. +// lacks (CHEAT_REBUILD) or ExeFS patch files it lacks (mods_exefs_pending), or the build has a mod plugin whose file is +// gone (ldr_plugins_gone: its mod was removed), the game runs it and exits; the setup lifts and compiles them in (or the +// plugin out) and starts the game again. // build-cache\rebuild.skip holds the key of the cheats a rebuild was started for: a rebuild that failed or was // cancelled leaves them unbuilt, and the same set does not start it again. TOTK_MODS_REBUILD=off never starts it. // Scripted runs (--seconds, --boot) never start it. @@ -532,7 +533,7 @@ static void rebuild_if_needed(const char *dir) { key = (key ^ fnv(c->info.key, strlen(c->info.key))) * 0x01000193u; key = (key ^ c->content) * 0x01000193u; } - int patches = mods_exefs_pending(&key); + int patches = mods_exefs_pending(&key) + ldr_plugins_gone(&key); if (!need && !patches) return; char cmd_path[PLAT_PATH_MAX + 64], skip_path[PLAT_PATH_MAX + 64], skip[16] = "", want[16]; snprintf(cmd_path, sizeof cmd_path, "%sbuild-cache" SEP "rebuild.txt", dir); @@ -548,7 +549,7 @@ static void rebuild_if_needed(const char *dir) { char *cmd = why ? NULL : read_text(cmd_path); if (!why && !cmd) why = "no build-cache/rebuild.txt (a build without the setup)"; if (why) { - rt_log("cheats: %d code cheats and %d ExeFS patch files are not built into the game; no rebuild: %s", need, + rt_log("cheats: %d code cheats and %d ExeFS patch files or plugins differ from the build; no rebuild: %s", need, patches, why); free(cmd); return; @@ -565,12 +566,12 @@ static void rebuild_if_needed(const char *dir) { } argv[argc] = NULL; if (argc < 2 || !plat_file_write(skip_path, want, strlen(want))) { - rt_log("cheats: %d code cheats and %d ExeFS patch files are not built into the game; cannot start the " + rt_log("cheats: %d code cheats and %d ExeFS patch files or plugins differ from the build; cannot start the " "rebuild (%s)", need, patches, cmd_path); free(cmd); return; } - rt_log("cheats: %d code cheats and %d ExeFS patch files are not built into the game; starting the rebuild: %s", + rt_log("cheats: %d code cheats and %d ExeFS patch files or plugins differ from the build; starting the rebuild: %s", need, patches, argv[0]); int ok = plat_spawn(argv, dir); free(cmd); diff --git a/runtime/host.h b/runtime/host.h index 8bb2804a..401ba3ec 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -211,6 +211,7 @@ extern GuestFn ldr_plugin_import; // set before ldr_loa // before an install's layout (recomp/layout.py) adds its mod plugins after sdk, so they do not use that MOD_COUNT. #define RT_MAX_MODULES 32 int ldr_count(void); +int ldr_plugins_gone(uint32_t *key); // layout plugins whose file is gone; mixes their paths into key const RtLoaded *ldr_module(int i); // order: main, subsdk0, sdk uint64_t ldr_symbol(const char *name); // exported definition, 0 if none uint64_t ldr_plt_stub(const char *module, const char *name); // PLT stub whose GOT slot holds name, 0 if none diff --git a/runtime/loader.c b/runtime/loader.c index b1911fe7..635d4313 100644 --- a/runtime/loader.c +++ b/runtime/loader.c @@ -86,12 +86,26 @@ static uint8_t *read_path_verified(const char *path, const char *want, size_t *l return p; } +// Mod plugins of the layout whose file is gone (their mod was removed): a rebuild without them is due (cheats.c +// rebuild_if_needed), keyed by their paths. +static int plugins_gone; +static uint32_t plugins_gone_key = 0x811C9DC5u; + +int ldr_plugins_gone(uint32_t *key) { + if (plugins_gone) *key = (*key ^ plugins_gone_key) * 0x01000193u; + return plugins_gone; +} + // 1 when the plugin file of lay is the one the build lifted; logged when not. static int plugin_file_ok(const RtLayoutEntry *lay) { size_t len; uint8_t *p = read_file(lay->file, &len); if (!p) { rt_log("loader: %s: cannot read %s; the plugin is left out", lay->name, lay->file); + if (!plat_file_exists(lay->file)) { + plugins_gone++; + for (const char *c = lay->file; *c; c++) plugins_gone_key = (plugins_gone_key ^ (uint8_t)*c) * 0x01000193u; + } return 0; } uint8_t d[32]; From 791e077c549d75d5d691d90a95f5fe5e05e0d86d Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 10:36:10 +0300 Subject: [PATCH 14/21] docs/mods.md: removing a plugin mod starts the rebuild without it (#845) Co-Authored-By: Claude Opus 5.5 --- docs/mods.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/mods.md b/docs/mods.md index 9e0f1d76..c29503ec 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -114,6 +114,8 @@ ahead of time, after the design in [#845](https://github.com/valentderah/totk-pc mod's row says why, and the next start runs the rebuild. 3. The plugin keeps what its start-up made (its memory, its tables), so its code works as on the Switch. The start costs about 50 ms per plugin; the game runs no differently without a plugin. +4. Removing a plugin mod from the mods folder starts the rebuild at the next start, which builds the game without + it (half a minute on 12 cores with the lift cache). A plugin mod switched off on the Mods tab only stays off. The Mods tab order decides between a plugin and another mod's patch on the same instruction, as for code patches. A plugin site over one of the port's own patches turns that built-in patch off while the mod is on. From 501f0d3e6f03d02cb0d6d83643a26149627003d8 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 10:54:17 +0300 Subject: [PATCH 15/21] The setup installs a build kit into the game folder, and the game's rebuild for code mods runs from it (#845) The rebuild for ExeFS patches, code cheats and mod plugins ran the setup window from the setup folder, whose absolute path build-cache/rebuild.txt held, with the generated C and objects in that folder's temp/. A moved or deleted setup folder left code mods without a rebuild. - A new Kit step after the check copies the release folder without TotK-Setup.exe into /build-kit (installer/layout.py KIT_ITEMS), and every later step runs its tools from there. A newer setup replaces the kit, the kit's own rebuild keeps it. - The release's temp folder is /build-cache/work, so the lift cache and the objects stay with the game. - rebuild.txt names the kit relative to the game folder; runtime/cheats.c and runtime/romfs.c resolve relative entries against the exe's folder. - runtime_key drops the lines of clang --version that name its folder, so a rebuild from the kit or a moved game folder keeps the compile and prepare steps the setup recorded. - An update over an install of an older setup moves that setup's temp folder into build-cache/work when both are on one disk. Co-Authored-By: Claude Opus 5.5 --- docs/installer.md | 60 ++++++++++----- docs/mods.md | 8 +- installer/cli.py | 24 ++---- installer/gui.py | 4 +- installer/guilogic.py | 2 +- installer/layout.py | 50 ++++++++++-- installer/runner.py | 6 ++ installer/steps.py | 146 ++++++++++++++++++++++++++++++++---- installer/text/en.json | 4 + installer/text/pt-BR.json | 4 + installer/text/ru.json | 4 + installer/text/zh-Hans.json | 4 + runtime/cheats.c | 10 ++- runtime/platform/platform.h | 11 +++ runtime/romfs.c | 11 ++- tests/installer_test.py | 126 ++++++++++++++++++++++++++++++- tools/make_release.py | 1 + 17 files changed, 410 insertions(+), 65 deletions(-) diff --git a/docs/installer.md b/docs/installer.md index 347e9434..24b3a22d 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -1,8 +1,9 @@ # Installer A public release that turns the player's own game files into a playable TotK folder. The release carries no game -data and no code derived from the game: the installer builds the game on the player's machine. The installed folder -holds only the game; the installer and its tools stay outside it. +data and no code derived from the game: the installer builds the game on the player's machine. On Windows the +installed folder also holds the build kit, the installer's tools the game rebuilds itself with for code mods, so the +setup folder can be deleted after the install and the install folder moved. ## What the player does @@ -16,7 +17,8 @@ holds only the game; the installer and its tools stay outside it. needed for Chinese, Korean and Traditional Chinese text; - the game language and the install folder. 3. Waits for the setup, one time. The window shows each step and its progress. -4. Plays `totk.exe` from the install folder. The installer and the source NSP or XCI (or NSZ, XCZ) can be deleted. +4. Plays `totk.exe` from the install folder. The setup folder and the source NSP or XCI (or NSZ, XCZ) can be deleted, + and the install folder can be moved: code mods rebuild the game with the build kit inside it. The install folder (`main.c` `portable_paths()` reads this layout; it stays portable): @@ -32,14 +34,22 @@ The install folder (`main.c` `portable_paths()` reads this layout; it stays port logs/ totk.log of the last run, totk.1.log to totk.4.log of the runs before it shader and pipeline caches prepared by the setup; shader-cache/shader-cache.pack holds the translations totk-install.json what built this folder: release version, input hashes, finished steps + build-kit/ Windows: the release folder without TotK-Setup.exe (about 0.65 GB); the game's rebuild + for code mods runs the setup window from it + build-cache/ what the rebuild built in (cheats.status, exefs.status, plugins.status), rebuild.txt, + and work/: the generated C, the lift cache and the objects (about 1.6 GB, Windows) ``` ## Updating the port The player runs a newer installer on the same folder. It reads `totk-install.json`, keeps `romfs/`, `exefs/`, -`shared-fonts/`, `profile/` and `totk.ini` after checking their hashes, rebuilds `totk.exe` and re-prepares the -shaders. A newer release that changes nothing game-derived skips the rebuild. The window starts with the last install folder -and the base, update, keys and firmware paths it ran with (paths only, never the keys file's contents), kept in +`shared-fonts/`, `profile/` and `totk.ini` after checking their hashes, replaces `build-kit/` with its own files, +rebuilds `totk.exe` and re-prepares the shaders. The setup is an installer and updater: nothing after the install +needs its folder. An install made by a setup before the build kit kept its temp folder in that setup's folder +(`/temp/-`, the setup `build-cache/rebuild.txt` named); the first newer setup moves it into +`build-cache/work` when both are on one disk (on another it starts empty) and writes the new `rebuild.txt`. Until then +the old `rebuild.txt` keeps using the old setup folder. A newer release that changes nothing game-derived skips the +rebuild. The window starts with the last install folder and the base, update, keys and firmware paths it ran with (paths only, never the keys file's contents), kept in `%LOCALAPPDATA%\TotK Recomp\setup.json` (`$XDG_STATE_HOME/totk-recomp/setup.json` elsewhere). Entries that no longer exist stay empty. A reinstall or a switch to another game version starts filled in. @@ -53,21 +63,27 @@ loads the saves 1.4.3 wrote. `tools/extract.py`), the keys are new enough for the update, enough space is free on the target disk (about 36 GiB at the peak of a first install, 20 GiB after it), the CPU is at least x86-64-v2 (SSE4.2; with AVX2/FMA the faster `v3` build is used, otherwise the `v2` one), a Vulkan 1.3 GPU exists. Each failure is one clear message. -2. **Fonts** with Chinese, Korean or Traditional Chinese text and a firmware folder: extract the Switch system fonts +2. **Kit** (Windows): copy the release folder without `TotK-Setup.exe` into `build-kit/` in the install folder + (`installer/steps.py` `Kit`, `installer/layout.py` `KIT_ITEMS`); every step after it runs its tools from there, so + the commands the build records (the compiler's path in the Ninja file) are the ones the game's rebuild runs later. + A newer setup replaces the kit; the kit's own rebuild keeps it. The steps' keys fingerprint file contents, not + paths (`runtime_key` leaves out the lines of `clang --version` that name its folder), so a rebuild from the kit + or from a moved install folder keeps every step the setup recorded. +3. **Fonts** with Chinese, Korean or Traditional Chinese text and a firmware folder: extract the Switch system fonts from the firmware into `shared-fonts/` (MacNativeGames' `extract_shared_fonts.py`, which verifies every NCA layer and BFTTF header). Without a firmware folder the setup warns and goes on; the game then lacks those glyphs. -3. **Extract** ExeFS and RomFS into the install folder and verify the hashes. -4. **Lift** the ExeFS to C (`recomp/gen.py`, `recomp/patches-v121.txt` or `patches-v143.txt`) into a temp folder next - to the installer. The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets first +4. **Extract** ExeFS and RomFS into the install folder and verify the hashes. +5. **Lift** the ExeFS to C (`recomp/gen.py`, `recomp/patches-v121.txt` or `patches-v143.txt`) into the temp folder, + `build-cache/work` in the install folder (`.setup-temp` on Linux, `--temp DIR` elsewhere). The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets first (`tools/cheats.py`, `tools/exefs_patches.py`, status in `build-cache/cheats.status` and `exefs.status`); a file the tools refuse costs those cheats or patches, not the install. The patch files' writes into main's data go to `build-cache/exefs.data`, which the game stores into main at start; after the lift they are checked against the bytes the lifted code read as constants (`/main/folded.txt`, its jump tables). -5. **Compile** the generated C, `modules.c`, `romfs.c`, `patches.c` and `ov_zstd.c` (they bake the per-install layout, RomFS +6. **Compile** the generated C, `modules.c`, `romfs.c`, `patches.c` and `ov_zstd.c` (they bake the per-install layout, RomFS hashes and the game version's patch page) and link them with the prebuilt runtime library for the host's level (`libtotkrt.a` or `libtotkrt-v2.a`) into `totk.exe` (`build_game.py --runtime-lib`; about 3 min on 24 cores). -6. **Prepare** (only when the runtime, the translator or the game version changed, not for a new lift) the RomFS +7. **Prepare** (only when the runtime, the translator or the game version changed, not for a new lift) the RomFS patches and the shaders (`tools/make_romfs_patches.py`, `tools/prepare_shaders.py`, `totk.exe --prepare-shaders`) in the install folder, then fold the shader cache's loose files (four per shader, about 117,000) into `shader-cache/shader-cache.pack` (`tools/pack_shader_cache.py`). totk.exe reads the pack first and @@ -82,7 +98,7 @@ loads the saves 1.4.3 wrote. When the graphics driver crashes while building the shaders, `prepare_shaders.py` runs that part again without the variants it was building ([building.md](building.md), step 6), and the setup finishes with a warning that names how many were left out. -7. **Shapes**: the UI laid out for each screen shape this player needs, `romfs-patches-x` +8. **Shapes**: the UI laid out for each screen shape this player needs, `romfs-patches-x` (`--aspect`), and with Xbox or PC prompts in `totk.ini` that layout with those prompts too, `romfs-patches-