Mod plugins get their own modules after sdk in the layout and are lifted with the game (#845)

recomp/layout.py appends a mod's code modules (exlaunch plugins such as exefs/subsdk9) after sdk as plugin0..N in
mod load order, so the game modules keep their bases; layout.json names each plugin's NSO and its SHA-256, and without
plugins layout.json and layout.h stay byte-identical. gen.py --plugins lifts them with DT_INIT and the init_array
entries as extra seeds, and Module.eh_frame_functions accepts a MOD0 with an empty .eh_frame_hdr range. build_game.py
takes the plugin rows from layout.json into modules.c instead of asking the ExeFS hash table for them.

No-mod generated C is byte-identical to dev; with MSFix in the layout main, subsdk0 and sdk are too.

After the design in #845 by itaskaev2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-17 08:05:18 +03:00
1 parent f85058070e
commit b94e8127af
5 files changed
+91 -31

No files matched your search

+17 -8
View File
@@ -3,7 +3,11 @@
Usage: gen.py --exefs build/extracted/v121/exefs --out generated/v121 [--modules main,subsdk0,sdk] [--jobs N]
[--limit N] [--chunk-fns N] [--chunk-bytes B] [--big-fn B] [--patches FILE] [--v-locals K [--v-trust]]
[--no-cache] [--fn-enter-all]
[--no-cache] [--fn-enter-all] [--plugins NSO[,NSO...]]
--plugins places a mod's code modules (exlaunch plugins) after sdk as plugin0..N (recomp/layout.py) and lifts them with
the game modules; their DT_INIT and init_array entries are seeds too (Module.init_functions). Without it the output is
what it was before plugins existed.
Output, under --out:
layout.h, layout.json module bases (recomp/layout.py)
@@ -63,7 +67,7 @@ import patches # noqa: E402
import regs # noqa: E402
import vlocals # noqa: E402
from check_content import GENERATED_MARKER # noqa: E402
from layout import layout, write as write_layout # noqa: E402
from layout import layout, plugin_files, write as write_layout # noqa: E402
from lifter import hx # noqa: E402
from module import Module # noqa: E402
@@ -853,7 +857,10 @@ def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--exefs', type=Path, required=True)
ap.add_argument('--out', type=Path, required=True)
ap.add_argument('--modules', default='main,subsdk0,sdk')
ap.add_argument('--modules', default='main,subsdk0,sdk', help='modules to lift (default main,subsdk0,sdk and every '
'--plugins module)')
ap.add_argument('--plugins', default='', help='comma-separated plugin NSOs (exefs/subsdk* of a mod), in mod load '
'order: lifted as plugin0..N after sdk')
ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)')
ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module')
ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most')
@@ -887,11 +894,13 @@ def main():
gen_key = generator_key(opts)
t0 = time.perf_counter()
mods = layout(args.exefs)
write_layout(mods, args.out)
plugins = [Path(p) for p in args.plugins.split(',') if p]
mods = layout(args.exefs, plugins)
files = plugin_files(plugins)
write_layout(mods, args.out, files)
table = patches.load(args.patches, mods) if args.patches else patches.Table()
write_if_changed(args.out / 'patches.h', patches.header_text(table, GENERATED_MARKER))
want = args.modules.split(',')
want = args.modules.split(',') + list(files)
unknown = set(want) - {n for n, _, _ in mods}
if unknown:
raise SystemExit(f'not in {args.exefs}: {", ".join(sorted(unknown))}')
@@ -901,8 +910,8 @@ def main():
if name not in want:
continue
t = time.perf_counter()
path = args.exefs / name
mod = Module(name, path, base, 'c')
path = Path(files[name][0]) if name in files else args.exefs / name
mod = Module(name, path, base, 'c', plugin=name in files)
patches.check(table, mod, name)
starts, dstats = discover(mod)
est = estimate_sizes(mod, starts)
+35 -13
View File
@@ -6,9 +6,12 @@ calls layout() and writes the result into generated/<version>/layout.h and layou
build read.
Modules are placed in Horizon's load order (rtld is dropped; the host links the modules):
main, subsdk0..9, sdk. Each base is aligned to 2 MiB. Everything must stay below 2^39, the limit of the addresses
IPC buffers can encode.
main, subsdk0..9, sdk. A mod's code modules (exlaunch plugins such as an exefs/subsdk9) follow sdk as plugin0, plugin1,
... in mod load order, so the game modules keep their bases with or without them (after the plugin design in #845 by
itaskaev2). Each base is aligned to 2 MiB. Everything must stay below 2^39, the limit of the addresses IPC buffers can
encode.
"""
import hashlib
import json
from pathlib import Path
@@ -24,14 +27,18 @@ def align_up(v, a=ALIGN):
return (v + a - 1) & ~(a - 1)
def layout(exefs):
"""[(name, base, size)] for the modules present in the ExeFS directory, in load order."""
def plugin_name(i):
return f'plugin{i}'
def layout(exefs, plugins=()):
"""[(name, base, size)] for the modules present in the ExeFS directory, then the plugin NSOs (paths, in mod load
order) as plugin0..N, in load order."""
exefs = Path(exefs)
paths = [(name, exefs / name) for name in ORDER if (exefs / name).exists()]
paths += [(plugin_name(i), Path(p)) for i, p in enumerate(plugins)]
out, base = [], FIRST_BASE
for name in ORDER:
path = exefs / name
if not path.exists():
continue
for name, path in paths:
size = Nso(str(path)).size
out.append((name, base, size))
base = align_up(base + size)
@@ -40,12 +47,27 @@ def layout(exefs):
return out
def write(mods, outdir):
"""layout.json for Python tools and layout.h for the runtime."""
def plugin_files(plugins):
"""{plugin<N>: (absolute path, SHA-256 of the NSO)} for layout.json: the build and the runtime read a plugin from its
mod folder, not from the ExeFS directory, and check that it is the file the code was lifted from."""
return {plugin_name(i): (Path(p).resolve().as_posix(), hashlib.sha256(Path(p).read_bytes()).hexdigest())
for i, p in enumerate(plugins)}
def write(mods, outdir, files=None):
"""layout.json for Python tools and layout.h for the runtime. files: plugin_files() for the plugin entries, which
carry them as "file" and "sha256"; without plugins both files are what they were before plugins existed."""
files = files or {}
outdir = Path(outdir)
outdir.mkdir(parents=True, exist_ok=True)
(outdir / 'layout.json').write_text(json.dumps(
[{'name': n, 'base': f'{b:#x}', 'size': f'{s:#x}'} for n, b, s in mods], indent=1) + '\n')
def entry(n, b, s):
e = {'name': n, 'base': f'{b:#x}', 'size': f'{s:#x}'}
if n in files:
e['file'], e['sha256'] = files[n]
return e
(outdir / 'layout.json').write_text(json.dumps([entry(n, b, s) for n, b, s in mods], indent=1) + '\n')
lines = ['// Module layout chosen by recomp/layout.py. Generated; do not edit.', '#pragma once', '']
for n, b, s in mods:
lines.append(f'#define MOD_{n.upper()}_BASE {b:#x}ull')
@@ -56,6 +78,6 @@ def write(mods, outdir):
def read(outdir):
"""[(name, base, size)] from a layout.json written by write()."""
"""[(name, base, size)] from a layout.json written by write() (a plugin's file and hash left out)."""
data = json.loads((Path(outdir) / 'layout.json').read_text())
return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data]
+21 -4
View File
@@ -10,7 +10,7 @@ from collections import OrderedDict
import numpy as np
from lifter import Insn, Lifter, Unsupported, hx
from nso import Nso, R_AARCH64_RELATIVE
from nso import DT_INIT, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, Nso, R_AARCH64_RELATIVE
LIFT_CACHE = 1 << 16 # lifted instructions kept per module; analyses revisit nearby addresses only
@@ -33,10 +33,11 @@ def _read_encoded(img, off, enc, pc_base):
class Module:
def __init__(self, name, path, base, simd='c'):
def __init__(self, name, path, base, simd='c', plugin=False):
self.name = name
self.nso = Nso(str(path))
self.base = base
self.plugin = plugin # a mod's code module (recomp/layout.py plugin<N>): its init functions are seeds too
n = self.nso
self.image = bytes(n.image)
self.lo = base + n.text[0]
@@ -83,7 +84,7 @@ class Module:
def eh_frame_functions(self):
"""Function starts from .eh_frame_hdr's binary search table (absolute addresses)."""
img, hdr = self.image, self.nso.eh_start
if not hdr or hdr >= len(img):
if not hdr or hdr >= len(img) or self.nso.eh_end <= hdr: # MOD0 with an empty .eh_frame_hdr range (exlaunch)
return []
_, ptr_enc, cnt_enc, tbl_enc = img[hdr:hdr + 4]
off = hdr + 4
@@ -100,9 +101,25 @@ class Module:
starts.append(self.base + loc)
return starts
def init_functions(self):
"""DT_INIT and the DT_INIT_ARRAY entries (absolute addresses). rtld calls DT_INIT only; the module's DT_INIT
runs the array through blr (SDK and exlaunch modules alike), so the entries are seeds too. An entry holds its
target as the addend of its R_AARCH64_RELATIVE relocation, or as the word itself when no relocation names it."""
dyn, out = self.nso.dyn, []
if dyn.get(DT_INIT):
out.append(self.base + dyn[DT_INIT])
if dyn.get(DT_INIT_ARRAY):
addends = {off: add for off, typ, _, add in self.nso.relocs if typ == R_AARCH64_RELATIVE}
for off in range(dyn[DT_INIT_ARRAY], dyn[DT_INIT_ARRAY] + dyn.get(DT_INIT_ARRAYSZ, 0), 8):
out.append(self.base + addends.get(off, self.nso.u64(off)))
return out
def seeds(self):
"""Known function starts: .eh_frame_hdr, exported FUNC symbols, R_AARCH64_RELATIVE targets in .text."""
"""Known function starts: .eh_frame_hdr, exported FUNC symbols, R_AARCH64_RELATIVE targets in .text; for a
plugin also its init functions (exlaunch's DT_INIT starts inside a function discovery would merge it into)."""
starts = set(self.eh_frame_functions())
if self.plugin:
starts.update(self.init_functions())
for _, info, shndx, value, _ in self.nso.syms:
if shndx and (info & 0xF) == 2:
starts.add(self.base + value)
+1 -1
View File
@@ -7,7 +7,7 @@ from collections import Counter
import lz4.block
DT_NULL, DT_NEEDED, DT_PLTRELSZ, DT_HASH, DT_STRTAB, DT_SYMTAB, DT_RELA, DT_RELASZ = 0, 1, 2, 4, 5, 6, 7, 8
DT_STRSZ, DT_JMPREL, DT_RELACOUNT = 10, 23, 0x6FFFFFF9
DT_STRSZ, DT_INIT, DT_JMPREL, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, DT_RELACOUNT = 10, 12, 23, 25, 27, 0x6FFFFFF9
DT_RELRSZ, DT_RELR, DT_GNU_HASH = 0x23, 0x24, 0x6FFFFEF5
R_AARCH64_ABS64, R_AARCH64_GLOB_DAT, R_AARCH64_JUMP_SLOT, R_AARCH64_RELATIVE = 257, 1025, 1026, 1027
+17 -5
View File
@@ -242,6 +242,13 @@ def read_layout(gen):
return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data]
def read_plugin_files(gen):
"""{plugin<N>: (NSO path, SHA-256)} from layout.json: the mod code modules recomp/layout.py placed after sdk. They
are no ExeFS files, so the runtime reads each from its path and checks it against this hash."""
data = json.loads((gen / 'layout.json').read_text())
return {m['name']: (m['file'], m['sha256']) for m in data if 'file' in m}
def chunk_files(gen, mod):
"""The C files of one module: the chunks listed in <mod>/manifest.json, then <mod>_table.c."""
d = gen / mod
@@ -366,9 +373,11 @@ def runtime_object(out, src):
return out / 'obj' / 'runtime' / rel.with_suffix('.o')
def module_glue(layout, linked, hashes, paths, version):
def module_glue(layout, linked, hashes, paths, version, plugins=None):
"""C for build/<version>/modules.c: the layout table, the linked modules' function tables (dispatch.h), the
expected ExeFS hashes, the game version and the default game paths (host.h)."""
expected ExeFS hashes, the game version and the default game paths (host.h). plugins: read_plugin_files(); a plugin
row of the layout table also names its file and hash."""
plugins = plugins or {}
out = [f'// Module glue written by tools/build_game.py; do not edit. {GENERATED_MARKER}',
'#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', '']
for name, base, size in layout:
@@ -376,7 +385,9 @@ def module_glue(layout, linked, hashes, paths, version):
out.append(f'_Static_assert(MOD_{u}_BASE == {base:#x}ull && MOD_{u}_SIZE == {size:#x}ull, '
f'"layout.h and layout.json disagree on {name}");')
out += ['', 'const RtLayoutEntry rt_layout[] = {']
out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE}},' for n, _, _ in layout]
out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE' +
(f', {c_string(plugins[n][0])}, {c_string(plugins[n][1])}' if n in plugins else '') + '},'
for n, _, _ in layout]
out += ['};', f'const int rt_layout_count = {len(layout)};', '']
for name in linked:
out += [f'extern const uint64_t {name}_fn_addrs[];', f'extern const GuestFn {name}_fn_ptrs[];',
@@ -923,10 +934,11 @@ def main():
cpus = os.cpu_count() or 1
budget = min(MEM_BUDGET, int(total_ram() * 0.8))
jobs = args.jobs or max(1, min(cpus, budget // JOB_MEM))
missing = [n for n in names if n not in hashes]
plugins = read_plugin_files(gen)
missing = [n for n in names if n not in hashes and n not in plugins]
if missing:
sys.exit(f'{expected}: no exefs_sha256 for {missing}')
write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version))
write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version, plugins))
if has_romfs:
count, size, dt = romfs_index.write(paths['romfs'], out)
print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s')