From b94e8127af18b132a77d77e375e653ba2ddb0e56 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 17 Nov 2026 08:05:18 +0300 Subject: [PATCH] Mod plugins get their own modules after sdk in the layout and are lifted with the game (#845) recomp/layout.py appends a mod's code modules (exlaunch plugins such as exefs/subsdk9) after sdk as plugin0..N in mod load order, so the game modules keep their bases; layout.json names each plugin's NSO and its SHA-256, and without plugins layout.json and layout.h stay byte-identical. gen.py --plugins lifts them with DT_INIT and the init_array entries as extra seeds, and Module.eh_frame_functions accepts a MOD0 with an empty .eh_frame_hdr range. build_game.py takes the plugin rows from layout.json into modules.c instead of asking the ExeFS hash table for them. No-mod generated C is byte-identical to dev; with MSFix in the layout main, subsdk0 and sdk are too. After the design in #845 by itaskaev2. Co-Authored-By: Claude Opus 5.5 --- recomp/gen.py | 25 +++++++++++++++-------- recomp/layout.py | 48 +++++++++++++++++++++++++++++++++------------ recomp/module.py | 25 +++++++++++++++++++---- recomp/nso.py | 2 +- tools/build_game.py | 22 ++++++++++++++++----- 5 files changed, 91 insertions(+), 31 deletions(-) diff --git a/recomp/gen.py b/recomp/gen.py index a0ddcd56..1a944eda 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -3,7 +3,11 @@ Usage: gen.py --exefs build/extracted/v121/exefs --out generated/v121 [--modules main,subsdk0,sdk] [--jobs N] [--limit N] [--chunk-fns N] [--chunk-bytes B] [--big-fn B] [--patches FILE] [--v-locals K [--v-trust]] - [--no-cache] [--fn-enter-all] + [--no-cache] [--fn-enter-all] [--plugins NSO[,NSO...]] + +--plugins places a mod's code modules (exlaunch plugins) after sdk as plugin0..N (recomp/layout.py) and lifts them with +the game modules; their DT_INIT and init_array entries are seeds too (Module.init_functions). Without it the output is +what it was before plugins existed. Output, under --out: layout.h, layout.json module bases (recomp/layout.py) @@ -63,7 +67,7 @@ import patches # noqa: E402 import regs # noqa: E402 import vlocals # noqa: E402 from check_content import GENERATED_MARKER # noqa: E402 -from layout import layout, write as write_layout # noqa: E402 +from layout import layout, plugin_files, write as write_layout # noqa: E402 from lifter import hx # noqa: E402 from module import Module # noqa: E402 @@ -853,7 +857,10 @@ def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--exefs', type=Path, required=True) ap.add_argument('--out', type=Path, required=True) - ap.add_argument('--modules', default='main,subsdk0,sdk') + ap.add_argument('--modules', default='main,subsdk0,sdk', help='modules to lift (default main,subsdk0,sdk and every ' + '--plugins module)') + ap.add_argument('--plugins', default='', help='comma-separated plugin NSOs (exefs/subsdk* of a mod), in mod load ' + 'order: lifted as plugin0..N after sdk') ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)') ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module') ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most') @@ -887,11 +894,13 @@ def main(): gen_key = generator_key(opts) t0 = time.perf_counter() - mods = layout(args.exefs) - write_layout(mods, args.out) + plugins = [Path(p) for p in args.plugins.split(',') if p] + mods = layout(args.exefs, plugins) + files = plugin_files(plugins) + write_layout(mods, args.out, files) table = patches.load(args.patches, mods) if args.patches else patches.Table() write_if_changed(args.out / 'patches.h', patches.header_text(table, GENERATED_MARKER)) - want = args.modules.split(',') + want = args.modules.split(',') + list(files) unknown = set(want) - {n for n, _, _ in mods} if unknown: raise SystemExit(f'not in {args.exefs}: {", ".join(sorted(unknown))}') @@ -901,8 +910,8 @@ def main(): if name not in want: continue t = time.perf_counter() - path = args.exefs / name - mod = Module(name, path, base, 'c') + path = Path(files[name][0]) if name in files else args.exefs / name + mod = Module(name, path, base, 'c', plugin=name in files) patches.check(table, mod, name) starts, dstats = discover(mod) est = estimate_sizes(mod, starts) diff --git a/recomp/layout.py b/recomp/layout.py index 971d6004..8f2e9a0c 100644 --- a/recomp/layout.py +++ b/recomp/layout.py @@ -6,9 +6,12 @@ calls layout() and writes the result into generated//layout.h and layou build read. Modules are placed in Horizon's load order (rtld is dropped; the host links the modules): -main, subsdk0..9, sdk. Each base is aligned to 2 MiB. Everything must stay below 2^39, the limit of the addresses -IPC buffers can encode. +main, subsdk0..9, sdk. A mod's code modules (exlaunch plugins such as an exefs/subsdk9) follow sdk as plugin0, plugin1, +... in mod load order, so the game modules keep their bases with or without them (after the plugin design in #845 by +itaskaev2). Each base is aligned to 2 MiB. Everything must stay below 2^39, the limit of the addresses IPC buffers can +encode. """ +import hashlib import json from pathlib import Path @@ -24,14 +27,18 @@ def align_up(v, a=ALIGN): return (v + a - 1) & ~(a - 1) -def layout(exefs): - """[(name, base, size)] for the modules present in the ExeFS directory, in load order.""" +def plugin_name(i): + return f'plugin{i}' + + +def layout(exefs, plugins=()): + """[(name, base, size)] for the modules present in the ExeFS directory, then the plugin NSOs (paths, in mod load + order) as plugin0..N, in load order.""" exefs = Path(exefs) + paths = [(name, exefs / name) for name in ORDER if (exefs / name).exists()] + paths += [(plugin_name(i), Path(p)) for i, p in enumerate(plugins)] out, base = [], FIRST_BASE - for name in ORDER: - path = exefs / name - if not path.exists(): - continue + for name, path in paths: size = Nso(str(path)).size out.append((name, base, size)) base = align_up(base + size) @@ -40,12 +47,27 @@ def layout(exefs): return out -def write(mods, outdir): - """layout.json for Python tools and layout.h for the runtime.""" +def plugin_files(plugins): + """{plugin: (absolute path, SHA-256 of the NSO)} for layout.json: the build and the runtime read a plugin from its + mod folder, not from the ExeFS directory, and check that it is the file the code was lifted from.""" + return {plugin_name(i): (Path(p).resolve().as_posix(), hashlib.sha256(Path(p).read_bytes()).hexdigest()) + for i, p in enumerate(plugins)} + + +def write(mods, outdir, files=None): + """layout.json for Python tools and layout.h for the runtime. files: plugin_files() for the plugin entries, which + carry them as "file" and "sha256"; without plugins both files are what they were before plugins existed.""" + files = files or {} outdir = Path(outdir) outdir.mkdir(parents=True, exist_ok=True) - (outdir / 'layout.json').write_text(json.dumps( - [{'name': n, 'base': f'{b:#x}', 'size': f'{s:#x}'} for n, b, s in mods], indent=1) + '\n') + + def entry(n, b, s): + e = {'name': n, 'base': f'{b:#x}', 'size': f'{s:#x}'} + if n in files: + e['file'], e['sha256'] = files[n] + return e + + (outdir / 'layout.json').write_text(json.dumps([entry(n, b, s) for n, b, s in mods], indent=1) + '\n') lines = ['// Module layout chosen by recomp/layout.py. Generated; do not edit.', '#pragma once', ''] for n, b, s in mods: lines.append(f'#define MOD_{n.upper()}_BASE {b:#x}ull') @@ -56,6 +78,6 @@ def write(mods, outdir): def read(outdir): - """[(name, base, size)] from a layout.json written by write().""" + """[(name, base, size)] from a layout.json written by write() (a plugin's file and hash left out).""" data = json.loads((Path(outdir) / 'layout.json').read_text()) return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data] diff --git a/recomp/module.py b/recomp/module.py index c3e4f355..9d4e43e5 100644 --- a/recomp/module.py +++ b/recomp/module.py @@ -10,7 +10,7 @@ from collections import OrderedDict import numpy as np from lifter import Insn, Lifter, Unsupported, hx -from nso import Nso, R_AARCH64_RELATIVE +from nso import DT_INIT, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, Nso, R_AARCH64_RELATIVE LIFT_CACHE = 1 << 16 # lifted instructions kept per module; analyses revisit nearby addresses only @@ -33,10 +33,11 @@ def _read_encoded(img, off, enc, pc_base): class Module: - def __init__(self, name, path, base, simd='c'): + def __init__(self, name, path, base, simd='c', plugin=False): self.name = name self.nso = Nso(str(path)) self.base = base + self.plugin = plugin # a mod's code module (recomp/layout.py plugin): its init functions are seeds too n = self.nso self.image = bytes(n.image) self.lo = base + n.text[0] @@ -83,7 +84,7 @@ class Module: def eh_frame_functions(self): """Function starts from .eh_frame_hdr's binary search table (absolute addresses).""" img, hdr = self.image, self.nso.eh_start - if not hdr or hdr >= len(img): + if not hdr or hdr >= len(img) or self.nso.eh_end <= hdr: # MOD0 with an empty .eh_frame_hdr range (exlaunch) return [] _, ptr_enc, cnt_enc, tbl_enc = img[hdr:hdr + 4] off = hdr + 4 @@ -100,9 +101,25 @@ class Module: starts.append(self.base + loc) return starts + def init_functions(self): + """DT_INIT and the DT_INIT_ARRAY entries (absolute addresses). rtld calls DT_INIT only; the module's DT_INIT + runs the array through blr (SDK and exlaunch modules alike), so the entries are seeds too. An entry holds its + target as the addend of its R_AARCH64_RELATIVE relocation, or as the word itself when no relocation names it.""" + dyn, out = self.nso.dyn, [] + if dyn.get(DT_INIT): + out.append(self.base + dyn[DT_INIT]) + if dyn.get(DT_INIT_ARRAY): + addends = {off: add for off, typ, _, add in self.nso.relocs if typ == R_AARCH64_RELATIVE} + for off in range(dyn[DT_INIT_ARRAY], dyn[DT_INIT_ARRAY] + dyn.get(DT_INIT_ARRAYSZ, 0), 8): + out.append(self.base + addends.get(off, self.nso.u64(off))) + return out + def seeds(self): - """Known function starts: .eh_frame_hdr, exported FUNC symbols, R_AARCH64_RELATIVE targets in .text.""" + """Known function starts: .eh_frame_hdr, exported FUNC symbols, R_AARCH64_RELATIVE targets in .text; for a + plugin also its init functions (exlaunch's DT_INIT starts inside a function discovery would merge it into).""" starts = set(self.eh_frame_functions()) + if self.plugin: + starts.update(self.init_functions()) for _, info, shndx, value, _ in self.nso.syms: if shndx and (info & 0xF) == 2: starts.add(self.base + value) diff --git a/recomp/nso.py b/recomp/nso.py index 03beab28..dfff6122 100644 --- a/recomp/nso.py +++ b/recomp/nso.py @@ -7,7 +7,7 @@ from collections import Counter import lz4.block DT_NULL, DT_NEEDED, DT_PLTRELSZ, DT_HASH, DT_STRTAB, DT_SYMTAB, DT_RELA, DT_RELASZ = 0, 1, 2, 4, 5, 6, 7, 8 -DT_STRSZ, DT_JMPREL, DT_RELACOUNT = 10, 23, 0x6FFFFFF9 +DT_STRSZ, DT_INIT, DT_JMPREL, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, DT_RELACOUNT = 10, 12, 23, 25, 27, 0x6FFFFFF9 DT_RELRSZ, DT_RELR, DT_GNU_HASH = 0x23, 0x24, 0x6FFFFEF5 R_AARCH64_ABS64, R_AARCH64_GLOB_DAT, R_AARCH64_JUMP_SLOT, R_AARCH64_RELATIVE = 257, 1025, 1026, 1027 diff --git a/tools/build_game.py b/tools/build_game.py index 274bd402..84e93682 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -242,6 +242,13 @@ def read_layout(gen): return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data] +def read_plugin_files(gen): + """{plugin: (NSO path, SHA-256)} from layout.json: the mod code modules recomp/layout.py placed after sdk. They + are no ExeFS files, so the runtime reads each from its path and checks it against this hash.""" + data = json.loads((gen / 'layout.json').read_text()) + return {m['name']: (m['file'], m['sha256']) for m in data if 'file' in m} + + def chunk_files(gen, mod): """The C files of one module: the chunks listed in /manifest.json, then _table.c.""" d = gen / mod @@ -366,9 +373,11 @@ def runtime_object(out, src): return out / 'obj' / 'runtime' / rel.with_suffix('.o') -def module_glue(layout, linked, hashes, paths, version): +def module_glue(layout, linked, hashes, paths, version, plugins=None): """C for build//modules.c: the layout table, the linked modules' function tables (dispatch.h), the - expected ExeFS hashes, the game version and the default game paths (host.h).""" + expected ExeFS hashes, the game version and the default game paths (host.h). plugins: read_plugin_files(); a plugin + row of the layout table also names its file and hash.""" + plugins = plugins or {} out = [f'// Module glue written by tools/build_game.py; do not edit. {GENERATED_MARKER}', '#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', ''] for name, base, size in layout: @@ -376,7 +385,9 @@ def module_glue(layout, linked, hashes, paths, version): out.append(f'_Static_assert(MOD_{u}_BASE == {base:#x}ull && MOD_{u}_SIZE == {size:#x}ull, ' f'"layout.h and layout.json disagree on {name}");') out += ['', 'const RtLayoutEntry rt_layout[] = {'] - out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE}},' for n, _, _ in layout] + out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE' + + (f', {c_string(plugins[n][0])}, {c_string(plugins[n][1])}' if n in plugins else '') + '},' + for n, _, _ in layout] out += ['};', f'const int rt_layout_count = {len(layout)};', ''] for name in linked: out += [f'extern const uint64_t {name}_fn_addrs[];', f'extern const GuestFn {name}_fn_ptrs[];', @@ -923,10 +934,11 @@ def main(): cpus = os.cpu_count() or 1 budget = min(MEM_BUDGET, int(total_ram() * 0.8)) jobs = args.jobs or max(1, min(cpus, budget // JOB_MEM)) - missing = [n for n in names if n not in hashes] + plugins = read_plugin_files(gen) + missing = [n for n in names if n not in hashes and n not in plugins] if missing: sys.exit(f'{expected}: no exefs_sha256 for {missing}') - write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version)) + write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version, plugins)) if has_romfs: count, size, dt = romfs_index.write(paths['romfs'], out) print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s')