mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 04:15:57 +00:00
Mod plugins get their own modules after sdk in the layout and are lifted with the game (#845)
recomp/layout.py appends a mod's code modules (exlaunch plugins such as exefs/subsdk9) after sdk as plugin0..N in mod load order, so the game modules keep their bases; layout.json names each plugin's NSO and its SHA-256, and without plugins layout.json and layout.h stay byte-identical. gen.py --plugins lifts them with DT_INIT and the init_array entries as extra seeds, and Module.eh_frame_functions accepts a MOD0 with an empty .eh_frame_hdr range. build_game.py takes the plugin rows from layout.json into modules.c instead of asking the ExeFS hash table for them. No-mod generated C is byte-identical to dev; with MSFix in the layout main, subsdk0 and sdk are too. After the design in #845 by itaskaev2. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f85058070e
commit
b94e8127af
5 files changed
+91
-31
No files matched your search
+17
-8
@@ -3,7 +3,11 @@
|
||||
|
||||
Usage: gen.py --exefs build/extracted/v121/exefs --out generated/v121 [--modules main,subsdk0,sdk] [--jobs N]
|
||||
[--limit N] [--chunk-fns N] [--chunk-bytes B] [--big-fn B] [--patches FILE] [--v-locals K [--v-trust]]
|
||||
[--no-cache] [--fn-enter-all]
|
||||
[--no-cache] [--fn-enter-all] [--plugins NSO[,NSO...]]
|
||||
|
||||
--plugins places a mod's code modules (exlaunch plugins) after sdk as plugin0..N (recomp/layout.py) and lifts them with
|
||||
the game modules; their DT_INIT and init_array entries are seeds too (Module.init_functions). Without it the output is
|
||||
what it was before plugins existed.
|
||||
|
||||
Output, under --out:
|
||||
layout.h, layout.json module bases (recomp/layout.py)
|
||||
@@ -63,7 +67,7 @@ import patches # noqa: E402
|
||||
import regs # noqa: E402
|
||||
import vlocals # noqa: E402
|
||||
from check_content import GENERATED_MARKER # noqa: E402
|
||||
from layout import layout, write as write_layout # noqa: E402
|
||||
from layout import layout, plugin_files, write as write_layout # noqa: E402
|
||||
from lifter import hx # noqa: E402
|
||||
from module import Module # noqa: E402
|
||||
|
||||
@@ -853,7 +857,10 @@ def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, required=True)
|
||||
ap.add_argument('--out', type=Path, required=True)
|
||||
ap.add_argument('--modules', default='main,subsdk0,sdk')
|
||||
ap.add_argument('--modules', default='main,subsdk0,sdk', help='modules to lift (default main,subsdk0,sdk and every '
|
||||
'--plugins module)')
|
||||
ap.add_argument('--plugins', default='', help='comma-separated plugin NSOs (exefs/subsdk* of a mod), in mod load '
|
||||
'order: lifted as plugin0..N after sdk')
|
||||
ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)')
|
||||
ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module')
|
||||
ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most')
|
||||
@@ -887,11 +894,13 @@ def main():
|
||||
gen_key = generator_key(opts)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
mods = layout(args.exefs)
|
||||
write_layout(mods, args.out)
|
||||
plugins = [Path(p) for p in args.plugins.split(',') if p]
|
||||
mods = layout(args.exefs, plugins)
|
||||
files = plugin_files(plugins)
|
||||
write_layout(mods, args.out, files)
|
||||
table = patches.load(args.patches, mods) if args.patches else patches.Table()
|
||||
write_if_changed(args.out / 'patches.h', patches.header_text(table, GENERATED_MARKER))
|
||||
want = args.modules.split(',')
|
||||
want = args.modules.split(',') + list(files)
|
||||
unknown = set(want) - {n for n, _, _ in mods}
|
||||
if unknown:
|
||||
raise SystemExit(f'not in {args.exefs}: {", ".join(sorted(unknown))}')
|
||||
@@ -901,8 +910,8 @@ def main():
|
||||
if name not in want:
|
||||
continue
|
||||
t = time.perf_counter()
|
||||
path = args.exefs / name
|
||||
mod = Module(name, path, base, 'c')
|
||||
path = Path(files[name][0]) if name in files else args.exefs / name
|
||||
mod = Module(name, path, base, 'c', plugin=name in files)
|
||||
patches.check(table, mod, name)
|
||||
starts, dstats = discover(mod)
|
||||
est = estimate_sizes(mod, starts)
|
||||
|
||||
+35
-13
@@ -6,9 +6,12 @@ calls layout() and writes the result into generated/<version>/layout.h and layou
|
||||
build read.
|
||||
|
||||
Modules are placed in Horizon's load order (rtld is dropped; the host links the modules):
|
||||
main, subsdk0..9, sdk. Each base is aligned to 2 MiB. Everything must stay below 2^39, the limit of the addresses
|
||||
IPC buffers can encode.
|
||||
main, subsdk0..9, sdk. A mod's code modules (exlaunch plugins such as an exefs/subsdk9) follow sdk as plugin0, plugin1,
|
||||
... in mod load order, so the game modules keep their bases with or without them (after the plugin design in #845 by
|
||||
itaskaev2). Each base is aligned to 2 MiB. Everything must stay below 2^39, the limit of the addresses IPC buffers can
|
||||
encode.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
@@ -24,14 +27,18 @@ def align_up(v, a=ALIGN):
|
||||
return (v + a - 1) & ~(a - 1)
|
||||
|
||||
|
||||
def layout(exefs):
|
||||
"""[(name, base, size)] for the modules present in the ExeFS directory, in load order."""
|
||||
def plugin_name(i):
|
||||
return f'plugin{i}'
|
||||
|
||||
|
||||
def layout(exefs, plugins=()):
|
||||
"""[(name, base, size)] for the modules present in the ExeFS directory, then the plugin NSOs (paths, in mod load
|
||||
order) as plugin0..N, in load order."""
|
||||
exefs = Path(exefs)
|
||||
paths = [(name, exefs / name) for name in ORDER if (exefs / name).exists()]
|
||||
paths += [(plugin_name(i), Path(p)) for i, p in enumerate(plugins)]
|
||||
out, base = [], FIRST_BASE
|
||||
for name in ORDER:
|
||||
path = exefs / name
|
||||
if not path.exists():
|
||||
continue
|
||||
for name, path in paths:
|
||||
size = Nso(str(path)).size
|
||||
out.append((name, base, size))
|
||||
base = align_up(base + size)
|
||||
@@ -40,12 +47,27 @@ def layout(exefs):
|
||||
return out
|
||||
|
||||
|
||||
def write(mods, outdir):
|
||||
"""layout.json for Python tools and layout.h for the runtime."""
|
||||
def plugin_files(plugins):
|
||||
"""{plugin<N>: (absolute path, SHA-256 of the NSO)} for layout.json: the build and the runtime read a plugin from its
|
||||
mod folder, not from the ExeFS directory, and check that it is the file the code was lifted from."""
|
||||
return {plugin_name(i): (Path(p).resolve().as_posix(), hashlib.sha256(Path(p).read_bytes()).hexdigest())
|
||||
for i, p in enumerate(plugins)}
|
||||
|
||||
|
||||
def write(mods, outdir, files=None):
|
||||
"""layout.json for Python tools and layout.h for the runtime. files: plugin_files() for the plugin entries, which
|
||||
carry them as "file" and "sha256"; without plugins both files are what they were before plugins existed."""
|
||||
files = files or {}
|
||||
outdir = Path(outdir)
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
(outdir / 'layout.json').write_text(json.dumps(
|
||||
[{'name': n, 'base': f'{b:#x}', 'size': f'{s:#x}'} for n, b, s in mods], indent=1) + '\n')
|
||||
|
||||
def entry(n, b, s):
|
||||
e = {'name': n, 'base': f'{b:#x}', 'size': f'{s:#x}'}
|
||||
if n in files:
|
||||
e['file'], e['sha256'] = files[n]
|
||||
return e
|
||||
|
||||
(outdir / 'layout.json').write_text(json.dumps([entry(n, b, s) for n, b, s in mods], indent=1) + '\n')
|
||||
lines = ['// Module layout chosen by recomp/layout.py. Generated; do not edit.', '#pragma once', '']
|
||||
for n, b, s in mods:
|
||||
lines.append(f'#define MOD_{n.upper()}_BASE {b:#x}ull')
|
||||
@@ -56,6 +78,6 @@ def write(mods, outdir):
|
||||
|
||||
|
||||
def read(outdir):
|
||||
"""[(name, base, size)] from a layout.json written by write()."""
|
||||
"""[(name, base, size)] from a layout.json written by write() (a plugin's file and hash left out)."""
|
||||
data = json.loads((Path(outdir) / 'layout.json').read_text())
|
||||
return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data]
|
||||
+21
-4
@@ -10,7 +10,7 @@ from collections import OrderedDict
|
||||
import numpy as np
|
||||
|
||||
from lifter import Insn, Lifter, Unsupported, hx
|
||||
from nso import Nso, R_AARCH64_RELATIVE
|
||||
from nso import DT_INIT, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, Nso, R_AARCH64_RELATIVE
|
||||
|
||||
LIFT_CACHE = 1 << 16 # lifted instructions kept per module; analyses revisit nearby addresses only
|
||||
|
||||
@@ -33,10 +33,11 @@ def _read_encoded(img, off, enc, pc_base):
|
||||
|
||||
|
||||
class Module:
|
||||
def __init__(self, name, path, base, simd='c'):
|
||||
def __init__(self, name, path, base, simd='c', plugin=False):
|
||||
self.name = name
|
||||
self.nso = Nso(str(path))
|
||||
self.base = base
|
||||
self.plugin = plugin # a mod's code module (recomp/layout.py plugin<N>): its init functions are seeds too
|
||||
n = self.nso
|
||||
self.image = bytes(n.image)
|
||||
self.lo = base + n.text[0]
|
||||
@@ -83,7 +84,7 @@ class Module:
|
||||
def eh_frame_functions(self):
|
||||
"""Function starts from .eh_frame_hdr's binary search table (absolute addresses)."""
|
||||
img, hdr = self.image, self.nso.eh_start
|
||||
if not hdr or hdr >= len(img):
|
||||
if not hdr or hdr >= len(img) or self.nso.eh_end <= hdr: # MOD0 with an empty .eh_frame_hdr range (exlaunch)
|
||||
return []
|
||||
_, ptr_enc, cnt_enc, tbl_enc = img[hdr:hdr + 4]
|
||||
off = hdr + 4
|
||||
@@ -100,9 +101,25 @@ class Module:
|
||||
starts.append(self.base + loc)
|
||||
return starts
|
||||
|
||||
def init_functions(self):
|
||||
"""DT_INIT and the DT_INIT_ARRAY entries (absolute addresses). rtld calls DT_INIT only; the module's DT_INIT
|
||||
runs the array through blr (SDK and exlaunch modules alike), so the entries are seeds too. An entry holds its
|
||||
target as the addend of its R_AARCH64_RELATIVE relocation, or as the word itself when no relocation names it."""
|
||||
dyn, out = self.nso.dyn, []
|
||||
if dyn.get(DT_INIT):
|
||||
out.append(self.base + dyn[DT_INIT])
|
||||
if dyn.get(DT_INIT_ARRAY):
|
||||
addends = {off: add for off, typ, _, add in self.nso.relocs if typ == R_AARCH64_RELATIVE}
|
||||
for off in range(dyn[DT_INIT_ARRAY], dyn[DT_INIT_ARRAY] + dyn.get(DT_INIT_ARRAYSZ, 0), 8):
|
||||
out.append(self.base + addends.get(off, self.nso.u64(off)))
|
||||
return out
|
||||
|
||||
def seeds(self):
|
||||
"""Known function starts: .eh_frame_hdr, exported FUNC symbols, R_AARCH64_RELATIVE targets in .text."""
|
||||
"""Known function starts: .eh_frame_hdr, exported FUNC symbols, R_AARCH64_RELATIVE targets in .text; for a
|
||||
plugin also its init functions (exlaunch's DT_INIT starts inside a function discovery would merge it into)."""
|
||||
starts = set(self.eh_frame_functions())
|
||||
if self.plugin:
|
||||
starts.update(self.init_functions())
|
||||
for _, info, shndx, value, _ in self.nso.syms:
|
||||
if shndx and (info & 0xF) == 2:
|
||||
starts.add(self.base + value)
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ from collections import Counter
|
||||
import lz4.block
|
||||
|
||||
DT_NULL, DT_NEEDED, DT_PLTRELSZ, DT_HASH, DT_STRTAB, DT_SYMTAB, DT_RELA, DT_RELASZ = 0, 1, 2, 4, 5, 6, 7, 8
|
||||
DT_STRSZ, DT_JMPREL, DT_RELACOUNT = 10, 23, 0x6FFFFFF9
|
||||
DT_STRSZ, DT_INIT, DT_JMPREL, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, DT_RELACOUNT = 10, 12, 23, 25, 27, 0x6FFFFFF9
|
||||
DT_RELRSZ, DT_RELR, DT_GNU_HASH = 0x23, 0x24, 0x6FFFFEF5
|
||||
R_AARCH64_ABS64, R_AARCH64_GLOB_DAT, R_AARCH64_JUMP_SLOT, R_AARCH64_RELATIVE = 257, 1025, 1026, 1027
|
||||
|
||||
|
||||
+17
-5
@@ -242,6 +242,13 @@ def read_layout(gen):
|
||||
return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data]
|
||||
|
||||
|
||||
def read_plugin_files(gen):
|
||||
"""{plugin<N>: (NSO path, SHA-256)} from layout.json: the mod code modules recomp/layout.py placed after sdk. They
|
||||
are no ExeFS files, so the runtime reads each from its path and checks it against this hash."""
|
||||
data = json.loads((gen / 'layout.json').read_text())
|
||||
return {m['name']: (m['file'], m['sha256']) for m in data if 'file' in m}
|
||||
|
||||
|
||||
def chunk_files(gen, mod):
|
||||
"""The C files of one module: the chunks listed in <mod>/manifest.json, then <mod>_table.c."""
|
||||
d = gen / mod
|
||||
@@ -366,9 +373,11 @@ def runtime_object(out, src):
|
||||
return out / 'obj' / 'runtime' / rel.with_suffix('.o')
|
||||
|
||||
|
||||
def module_glue(layout, linked, hashes, paths, version):
|
||||
def module_glue(layout, linked, hashes, paths, version, plugins=None):
|
||||
"""C for build/<version>/modules.c: the layout table, the linked modules' function tables (dispatch.h), the
|
||||
expected ExeFS hashes, the game version and the default game paths (host.h)."""
|
||||
expected ExeFS hashes, the game version and the default game paths (host.h). plugins: read_plugin_files(); a plugin
|
||||
row of the layout table also names its file and hash."""
|
||||
plugins = plugins or {}
|
||||
out = [f'// Module glue written by tools/build_game.py; do not edit. {GENERATED_MARKER}',
|
||||
'#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', '']
|
||||
for name, base, size in layout:
|
||||
@@ -376,7 +385,9 @@ def module_glue(layout, linked, hashes, paths, version):
|
||||
out.append(f'_Static_assert(MOD_{u}_BASE == {base:#x}ull && MOD_{u}_SIZE == {size:#x}ull, '
|
||||
f'"layout.h and layout.json disagree on {name}");')
|
||||
out += ['', 'const RtLayoutEntry rt_layout[] = {']
|
||||
out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE}},' for n, _, _ in layout]
|
||||
out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE' +
|
||||
(f', {c_string(plugins[n][0])}, {c_string(plugins[n][1])}' if n in plugins else '') + '},'
|
||||
for n, _, _ in layout]
|
||||
out += ['};', f'const int rt_layout_count = {len(layout)};', '']
|
||||
for name in linked:
|
||||
out += [f'extern const uint64_t {name}_fn_addrs[];', f'extern const GuestFn {name}_fn_ptrs[];',
|
||||
@@ -923,10 +934,11 @@ def main():
|
||||
cpus = os.cpu_count() or 1
|
||||
budget = min(MEM_BUDGET, int(total_ram() * 0.8))
|
||||
jobs = args.jobs or max(1, min(cpus, budget // JOB_MEM))
|
||||
missing = [n for n in names if n not in hashes]
|
||||
plugins = read_plugin_files(gen)
|
||||
missing = [n for n in names if n not in hashes and n not in plugins]
|
||||
if missing:
|
||||
sys.exit(f'{expected}: no exefs_sha256 for {missing}')
|
||||
write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version))
|
||||
write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths, args.version, plugins))
|
||||
if has_romfs:
|
||||
count, size, dt = romfs_index.write(paths['romfs'], out)
|
||||
print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s')
|
||||
|
||||
Reference in new issue
Block a user