mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 04:15:57 +00:00
Merge branch 'pr835' into task/exefsdata
This commit is contained in:
commit
257f6201a3
13 files changed
+1001
-102
No files matched your search
+4
-1
@@ -231,7 +231,10 @@ Before a commit:
|
||||
A change to the button prompts (`tools/make_romfs_patches.py --prompts`, `tools/button_icons.py`) passes
|
||||
`.venv-py314\Scripts\python tests\romfs_prompts_test.py`, which runs the tool on a RomFS it makes itself (no game
|
||||
files); a change to the settings passes `totk.exe --probe-cfg` and `python tests\settings_doc_check.py`, which fails
|
||||
when a setting is described neither in `totk.ini.example` nor in [settings.md](settings.md).
|
||||
when a setting is described neither in `totk.ini.example` nor in [settings.md](settings.md). A change to the mods' ExeFS patches
|
||||
(`tools/exefs_patches.py`, `runtime/mods.c`, the `folded.txt` of `recomp/gen.py`) passes
|
||||
`.venv\Scripts\python tests\exefs_patches_test.py`, which makes its own modules and runs `gen.py` on one with a jump
|
||||
table (no game files), and `totk.exe --probe-mods`.
|
||||
|
||||
The lifter (`recomp/`) and the CPU headers (`runtime/cpu.h`, `runtime/simd.h`) are checked against Unicorn. Every lifter
|
||||
change passes these, all run with `.venv`:
|
||||
|
||||
+3
-1
@@ -60,7 +60,9 @@ loads the saves 1.4.3 wrote.
|
||||
4. **Lift** the ExeFS to C (`recomp/gen.py`, `recomp/patches-v121.txt` or `patches-v143.txt`) into a temp folder next
|
||||
to the installer. The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets first
|
||||
(`tools/cheats.py`, `tools/exefs_patches.py`, status in `build-cache/cheats.status` and `exefs.status`); a file the
|
||||
tools refuse costs those cheats or patches, not the install.
|
||||
tools refuse costs those cheats or patches, not the install. The patch files' writes into main's data go to
|
||||
`build-cache/exefs.data`, which the game stores into main at start; after the lift they are checked against the
|
||||
bytes the lifted code read as constants (`<gen>/main/folded.txt`, its jump tables).
|
||||
5. **Compile** the generated C, `modules.c`, `romfs.c`, `patches.c` and `ov_zstd.c` (they bake the per-install layout, RomFS
|
||||
hashes and the game version's patch page) and link
|
||||
them with the prebuilt runtime library for the host's level (`libtotkrt.a` or `libtotkrt-v2.a`) into `totk.exe`
|
||||
|
||||
@@ -71,6 +71,10 @@ The rebuild needs the setup folder the game was installed from. When that folder
|
||||
A code patch over one of the port's own patches (the 60 FPS patch, the resolution patch and the other entries under
|
||||
**Built into the port**) turns that built-in patch off while the mod is on. The mod's row names it.
|
||||
|
||||
A patch file may also change the game's data (constants and tables in its `.rodata` and `.data`). Those bytes are
|
||||
stored into the loaded game at start, before it runs. When two mods write the same bytes, the higher one wins,
|
||||
as with files.
|
||||
|
||||
## Cheats
|
||||
|
||||
A cheat file is part of a mod: `mods\<mod>\cheats\<build id>.txt`, switched cheat by cheat on the Cheats tab. Cheats
|
||||
|
||||
+41
-16
@@ -16,9 +16,10 @@ logs/: the next run lifts into the same gen/, where recomp/gen.py rewrites only
|
||||
in build/ recompiles only those. A change of the mods' code cheats then costs the lift (about a minute) and the chunks
|
||||
it touches, not a full compile.
|
||||
|
||||
Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status (tools/exefs_patches.py) and rebuild.txt, the command that starts this setup in
|
||||
its rebuild mode on the install folder (installer/gui.py --rebuild), one argument per line; totk.exe runs it at start
|
||||
when an enabled mod has code cheats the build lacks (runtime/cheats.c).
|
||||
Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status and exefs.data (tools/exefs_patches.py) and
|
||||
rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild),
|
||||
one argument per line; totk.exe runs it at start when an enabled mod has code cheats the build lacks
|
||||
(runtime/cheats.c).
|
||||
"""
|
||||
import hashlib
|
||||
import http.client
|
||||
@@ -855,6 +856,12 @@ def exefs_status(ctx):
|
||||
return ctx.dest / 'build-cache' / 'exefs.status'
|
||||
|
||||
|
||||
def exefs_data(ctx):
|
||||
"""The data writes of the mods' ExeFS patch files (tools/exefs_patches.py --data), which runtime/mods.c stores into
|
||||
main before the game runs."""
|
||||
return ctx.dest / 'build-cache' / 'exefs.data'
|
||||
|
||||
|
||||
def cheats_status(ctx):
|
||||
"""The build's cheat status file (tools/cheats.py --status), which runtime/cheats.c reads: in build-cache, not in a
|
||||
mod's own cheats folder, which the player (or TKMM's merge) may empty."""
|
||||
@@ -869,7 +876,7 @@ class Lift(Step):
|
||||
# fingerprint takes a file's name and content only; the paths under mods/ go in too, because tools/cheats.py
|
||||
# and tools/exefs_patches.py name each patch set after its mod folder, so a renamed folder needs a lift
|
||||
files = cheat_files(ctx) + exefs_files(ctx)
|
||||
return fingerprint('lift-4', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)),
|
||||
return fingerprint('lift-5', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)),
|
||||
ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *files,
|
||||
*(f.relative_to(ctx.dest / 'mods').as_posix() for f in files))
|
||||
|
||||
@@ -890,36 +897,54 @@ class Lift(Step):
|
||||
if m:
|
||||
rep.progress('lift', 'emit', int(m.group(1)), int(m.group(2)), 'chunks')
|
||||
|
||||
# The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets (tools/cheats.py,
|
||||
# tools/exefs_patches.py); their status files tell the runtime what the build has. Both are the player's
|
||||
# downloads, so files the tools refuse cost those cheats or patches, not the install: the game is built
|
||||
# without them and the setup warns.
|
||||
# The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets and data writes
|
||||
# (tools/cheats.py, tools/exefs_patches.py); their status files tell the runtime what the build has. Both are
|
||||
# the player's downloads, so files the tools refuse cost those cheats or patches, not the install: the game is
|
||||
# built without them and the setup warns.
|
||||
plain = layout.patches(game_version(ctx))
|
||||
stages = [('cheats.py', 'patches-cheats.txt', cheats_status(ctx), cheat_files(ctx), 'warn.cheats_skipped'),
|
||||
('exefs_patches.py', 'patches-exefs.txt', exefs_status(ctx), exefs_files(ctx), 'warn.exefs_skipped')]
|
||||
# (tool, patch file it writes, its outputs for the runtime: the status first, then the data file, the files it
|
||||
# reads, the warning when it fails)
|
||||
stages = [('cheats.py', 'patches-cheats.txt', [cheats_status(ctx)], cheat_files(ctx), 'warn.cheats_skipped'),
|
||||
('exefs_patches.py', 'patches-exefs.txt', [exefs_status(ctx), exefs_data(ctx)], exefs_files(ctx),
|
||||
'warn.exefs_skipped')]
|
||||
patches, skipped = plain, {}
|
||||
for tool, out, status, inputs, warn in stages:
|
||||
status.unlink(missing_ok=True) # no status: the runtime treats every such file as not built
|
||||
|
||||
def drop(outputs): # no status: the runtime treats every such file as not built
|
||||
for f in outputs:
|
||||
f.unlink(missing_ok=True)
|
||||
|
||||
for tool, out, outputs, inputs, warn in stages:
|
||||
drop(outputs)
|
||||
if not inputs:
|
||||
continue
|
||||
code, log = run_tool(ctx, 'lift', layout.tools / tool,
|
||||
['--exefs', ctx.dest / 'exefs', '--mods', ctx.dest / 'mods', '--patches', patches,
|
||||
'--out', ctx.temp / out, '--status', status])
|
||||
'--out', ctx.temp / out, '--status', outputs[0],
|
||||
*(['--data', outputs[1]] if len(outputs) > 1 else [])])
|
||||
if code:
|
||||
status.unlink(missing_ok=True)
|
||||
drop(outputs)
|
||||
skipped[warn] = log
|
||||
else:
|
||||
patches = ctx.temp / out
|
||||
args = ['--exefs', ctx.dest / 'exefs', '--out', gen]
|
||||
code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches], parse)
|
||||
if code and patches != plain:
|
||||
for tool, out, status, inputs, warn in stages:
|
||||
for tool, out, outputs, inputs, warn in stages:
|
||||
if inputs and warn not in skipped:
|
||||
status.unlink(missing_ok=True)
|
||||
drop(outputs)
|
||||
skipped[warn] = log
|
||||
code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', plain], parse)
|
||||
if code:
|
||||
raise InstallError('lift.failed', code=code, log=str(log))
|
||||
# The mods' data writes must stay off the bytes the lifted code reads as constants (its jump tables), which only
|
||||
# the lift knows: gen.py lists them in folded.txt, and a file that writes them becomes unsupported
|
||||
if exefs_data(ctx).is_file():
|
||||
code, log = run_tool(ctx, 'lift', layout.tools / 'exefs_patches.py',
|
||||
['--exefs', ctx.dest / 'exefs', '--status', exefs_status(ctx), '--data',
|
||||
exefs_data(ctx), '--folded', gen / 'main' / 'folded.txt'])
|
||||
if code:
|
||||
drop([exefs_status(ctx), exefs_data(ctx)])
|
||||
skipped.setdefault('warn.exefs_skipped', log)
|
||||
for warn, log in skipped.items():
|
||||
rep.warning(warn, log=str(log))
|
||||
(gen / STAMP).write_text(ctx.keys['lift'] + '\n')
|
||||
|
||||
+41
-2
@@ -9,6 +9,9 @@ Output, under --out:
|
||||
<mod>/<mod>_NNNN.c one `void f_<addr>(Cpu *c)` per guest function, about --chunk-bytes of C per file
|
||||
<mod>/<mod>_table.c sorted <mod>_fn_addrs[] / <mod>_fn_ptrs[] / <mod>_fn_count for rt_dispatch
|
||||
<mod>/manifest.json chunk list, function count, statistics and timings
|
||||
<mod>/folded.txt the module's bytes the generated code took as constants: what the jump tables read at
|
||||
lift time, as module offsets [start, end); tools/exefs_patches.py keeps a mod's data
|
||||
writes off them
|
||||
patches.h code patch sets for the runtime (recomp/patches.py; no sets without --patches)
|
||||
|
||||
Function discovery runs in this process, vectorised over the module's .text words. Function starts are the module
|
||||
@@ -555,7 +558,8 @@ def _retry(fn, *args):
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ workers
|
||||
|
||||
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter}
|
||||
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter},
|
||||
# 'reads' {name: [(start, end)]} module bytes read since the last chunk
|
||||
|
||||
|
||||
def _worker_init(mods, starts, table, opts, jobs):
|
||||
@@ -565,6 +569,7 @@ def _worker_init(mods, starts, table, opts, jobs):
|
||||
_W['patches'] = table
|
||||
_W['opts'] = opts # Emitter keyword arguments
|
||||
_W['emitters'] = {}
|
||||
_W['reads'] = {}
|
||||
|
||||
|
||||
def _emitter(name):
|
||||
@@ -572,12 +577,31 @@ def _emitter(name):
|
||||
if e is None:
|
||||
path, base = _W['mods'][name]
|
||||
mod = Module(name, path, base, 'c')
|
||||
# Module.read is how the jump tables (recomp/jumptables.py _eval) read .text and .rodata at lift time: the
|
||||
# bytes it returns become part of the generated code, so <mod>/folded.txt lists them
|
||||
read, reads = mod.read, _W['reads'].setdefault(name, [])
|
||||
|
||||
def recorded(a, n, signed=False):
|
||||
reads.append((a - base, a - base + n))
|
||||
return read(a, n, signed=signed)
|
||||
mod.read = recorded
|
||||
e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()),
|
||||
_W['patches'].for_module(name, base),
|
||||
spins=_W['patches'].spins_for(name, base), **_W['opts'])
|
||||
return e
|
||||
|
||||
|
||||
def merge_ranges(ranges):
|
||||
"""Sorted [start, end) ranges with overlapping and touching ones joined."""
|
||||
out = []
|
||||
for s, e in sorted(ranges):
|
||||
if out and s <= out[-1][1]:
|
||||
out[-1][1] = max(out[-1][1], e)
|
||||
else:
|
||||
out.append([s, e])
|
||||
return out
|
||||
|
||||
|
||||
def _emit_chunk(task):
|
||||
"""Worker: write one chunk. Returns its statistics."""
|
||||
name, number, path, fns = task
|
||||
@@ -595,9 +619,12 @@ def _emit_chunk(task):
|
||||
written = write_if_changed(Path(path), text)
|
||||
except OSError as ex:
|
||||
written, error = False, f'{path}: {ex}'
|
||||
reads = _W['reads'].get(name, [])
|
||||
folded = merge_ranges(reads)
|
||||
reads.clear()
|
||||
return {'module': name, 'number': number, 'file': Path(path).name, 'functions': len(fns), 'bytes': len(text),
|
||||
'written': written, 'error': error, 'seconds': time.perf_counter() - t0, 'stats': dict(e.stats),
|
||||
'unsupported': dict(e.unsupported)}
|
||||
'unsupported': dict(e.unsupported), 'folded': folded}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ driver
|
||||
@@ -613,6 +640,14 @@ def table_text(name, addrs):
|
||||
return ''.join(lines)
|
||||
|
||||
|
||||
def folded_text(name, ranges):
|
||||
"""<mod>/folded.txt: the module offsets [start, end) of the bytes the jump tables read at lift time, in hex."""
|
||||
lines = [f'# {GENERATED_MARKER}: {name} bytes the generated code took as constants (jump tables), module offsets '
|
||||
f'[start, end), generated by recomp/gen.py from game code. Do not edit or commit.']
|
||||
lines += [f'{s:x} {e:x}' for s, e in ranges]
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def remove_stale(moddir, name, nchunks, errors):
|
||||
"""Delete chunks an earlier run wrote beyond this run's count, and temporary files an interrupted run left."""
|
||||
pat = re.compile(re.escape(name) + r'_(\d{4})\.c')
|
||||
@@ -714,6 +749,7 @@ def main():
|
||||
# Largest chunks first, so the pool does not end on a long tail.
|
||||
tasks.sort(key=lambda t: -t[0])
|
||||
results = {name: [] for name in info}
|
||||
folded = {name: [] for name in info}
|
||||
done_at, errors = {}, []
|
||||
t_emit = time.perf_counter()
|
||||
ctx = mp.get_context('spawn')
|
||||
@@ -724,9 +760,11 @@ def main():
|
||||
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
|
||||
try:
|
||||
# Chunk writes are atomic and idempotent, so a chunk lost with a failed pool is simply emitted again.
|
||||
# Its folded bytes come with its result: a chunk emitted again reads them again in its new worker.
|
||||
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts, jobs)) as pool:
|
||||
for r in pool.imap_unordered(_emit_chunk, pending, chunksize=1):
|
||||
results[r['module']].append(r)
|
||||
folded[r['module']] += r['folded']
|
||||
if r['error']:
|
||||
errors.append(r['error'])
|
||||
done_at[r['module']] = time.perf_counter() - t_emit
|
||||
@@ -753,6 +791,7 @@ def main():
|
||||
unsupported.update(r['unsupported'])
|
||||
t = time.perf_counter()
|
||||
table_written = write_if_changed(moddir / f'{name}_table.c', table_text(name, d['emitted']))
|
||||
write_if_changed(moddir / 'folded.txt', folded_text(name, merge_ranges(folded[name])))
|
||||
removed = remove_stale(moddir, name, d['chunks'], errors)
|
||||
manifest = {
|
||||
'module': name,
|
||||
|
||||
+209
-33
@@ -15,7 +15,9 @@
|
||||
// when its build id is main's (an IPS file's name, a pchtxt file's @nsobid line); others are another version's and
|
||||
// counted. The setup builds every mod's files into patch sets and writes build-cache/exefs.status next to totk.exe
|
||||
// (tools/exefs_patches.py has the format): a file whose line is missing or has another content hash waits for the
|
||||
// rebuild at start (cheats.c), and its mod stays off until then.
|
||||
// rebuild at start (cheats.c), and its mod stays off until then. A file's writes into main's data (.rodata, .data,
|
||||
// .bss) are lines of build-cache/exefs.data, stored into the loaded main before the game runs (mods_exefs_data), as
|
||||
// emulators apply a patch file to the module before its dynamic linker runs.
|
||||
#include <ctype.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
@@ -170,22 +172,26 @@ static int same_build(const char *hex, size_t n) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// build-cache/exefs.status next to totk.exe, read once; "" when there is none.
|
||||
static const char *exefs_status(void) {
|
||||
static char *text;
|
||||
if (text) return text;
|
||||
// build-cache/<name> next to totk.exe, read once into *text; "" when there is none (mods_probe_cache sets the text).
|
||||
static char *status_text, *data_text;
|
||||
static const char *build_cache(const char *name, char **text) {
|
||||
if (*text) return *text;
|
||||
char path[2100];
|
||||
uint64_t n = 0;
|
||||
if (plat_exe_path(path, sizeof path)) {
|
||||
char *cut = strrchr(path, '/'), *back = strrchr(path, '\\');
|
||||
if (back > cut) cut = back;
|
||||
if (cut) snprintf(cut + 1, sizeof path - (size_t)(cut + 1 - path), "build-cache/exefs.status");
|
||||
text = (char *)plat_file_read(path, &n);
|
||||
if (cut && snprintf(cut + 1, sizeof path - (size_t)(cut + 1 - path), "build-cache/%s", name) <
|
||||
(int)(sizeof path - (size_t)(cut + 1 - path)))
|
||||
*text = (char *)plat_file_read(path, &n);
|
||||
}
|
||||
if (!text) text = calloc(1, 1);
|
||||
return text;
|
||||
if (!*text) *text = calloc(1, 1);
|
||||
return *text;
|
||||
}
|
||||
|
||||
static const char *exefs_status(void) { return build_cache("exefs.status", &status_text); }
|
||||
static const char *exefs_data(void) { return build_cache("exefs.data", &data_text); }
|
||||
|
||||
// The status line of key: its kind, built-in sets and note, NUL-terminated copies; 0 when none.
|
||||
static int status_line(const char *key, const char *content, char *kind, char *over, char *note) {
|
||||
char want[20];
|
||||
@@ -194,8 +200,9 @@ static int status_line(const char *key, const char *content, char *kind, char *o
|
||||
const char *end = strchr(line, '\n');
|
||||
size_t len = end ? (size_t)(end - line) : strlen(line);
|
||||
if (len > 18 && !memcmp(line, want, 18)) {
|
||||
char copy[600];
|
||||
snprintf(copy, sizeof copy, "%.*s", (int)(len < sizeof copy ? len : sizeof copy - 1), line + 18);
|
||||
char copy[600]; // the line after the two hashes, without the next line
|
||||
size_t rest = len - 18;
|
||||
snprintf(copy, sizeof copy, "%.*s", (int)(rest < sizeof copy ? rest : sizeof copy - 1), line + 18);
|
||||
char *t1 = strchr(copy, '\t'), *t2 = t1 ? strchr(t1 + 1, '\t') : NULL;
|
||||
if (!t1 || !t2) return 0;
|
||||
*t1 = *t2 = 0;
|
||||
@@ -269,9 +276,14 @@ static int exefs_file_cb(const char *leaf, int kind, uint64_t size, void *ctx) {
|
||||
m->sets[m->nsets++] = (uint32_t)strtoul(key, NULL, 16);
|
||||
size_t k = strlen(m->overrides);
|
||||
if (*over) snprintf(m->overrides + k, sizeof m->overrides - k, "%s%s", k ? "," : "", over);
|
||||
} else if (!strcmp(kind_s, "data") && m->ndata < (int)(sizeof m->data / sizeof *m->data)) {
|
||||
m->data[m->ndata++] = (uint32_t)strtoul(key, NULL, 16);
|
||||
} else if (!strcmp(kind_s, "other")) {
|
||||
m->other_version++;
|
||||
}
|
||||
unsigned long bytes; // a code or data file's data writes: exefs.data must hold them all (mods_exefs_data)
|
||||
if ((!strcmp(kind_s, "code") || !strcmp(kind_s, "data")) && sscanf(note, "data %lu", &bytes) == 1)
|
||||
m->data_bytes += (uint32_t)bytes;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -326,13 +338,13 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) {
|
||||
if (m->cheats[0]) plat_fs_list(m->cheats, cheats_cb, &m->kind);
|
||||
int unsupported = exefs_check(m, path);
|
||||
// a mod whose code part the port cannot use is off as a whole: its RomFS files may expect that code
|
||||
m->problem = strchr(m->name, '|') ? MOD_BAD_NAME
|
||||
: m->kind & MOD_SUBSDK ? MOD_PLUGIN
|
||||
: unsupported ? MOD_EXEFS_UNSUPPORTED
|
||||
: m->pending ? MOD_EXEFS_LATER
|
||||
: m->nsets || m->kind & (MOD_ROMFS | MOD_CHEATS) ? MOD_OK
|
||||
: m->kind & MOD_EXEFS ? MOD_EXEFS_OTHER
|
||||
: MOD_EMPTY;
|
||||
m->problem = strchr(m->name, '|') ? MOD_BAD_NAME
|
||||
: m->kind & MOD_SUBSDK ? MOD_PLUGIN
|
||||
: unsupported ? MOD_EXEFS_UNSUPPORTED
|
||||
: m->pending ? MOD_EXEFS_LATER
|
||||
: m->nsets || m->ndata || m->kind & (MOD_ROMFS | MOD_CHEATS) ? MOD_OK
|
||||
: m->kind & MOD_EXEFS ? MOD_EXEFS_OTHER
|
||||
: MOD_EMPTY;
|
||||
m->enabled = !listed(disabled, m->name);
|
||||
nmods++;
|
||||
return 1;
|
||||
@@ -376,10 +388,11 @@ void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx),
|
||||
plat_fs_list(folder, peek_cb, &pk);
|
||||
}
|
||||
|
||||
static int scanned;
|
||||
|
||||
void mods_scan(void) {
|
||||
static int done;
|
||||
if (done) return;
|
||||
done = 1;
|
||||
if (scanned) return;
|
||||
scanned = 1;
|
||||
if (!mods_folder(dir, sizeof dir)) return;
|
||||
if (!plat_dir_exists(dir)) {
|
||||
rt_log("mods: no folder %s", dir);
|
||||
@@ -403,18 +416,171 @@ void mods_scan(void) {
|
||||
rt_log("mods: %d folders in %s", nmods, dir);
|
||||
}
|
||||
|
||||
void mods_exefs_switch(void) {
|
||||
mods_scan();
|
||||
for (int i = 0; i < nmods; i++) {
|
||||
const ModInfo *m = &mods[i];
|
||||
if (!start_enabled[i] || m->problem != MOD_OK || !m->nsets) continue;
|
||||
for (int k = 0; k < m->nsets; k++) {
|
||||
char name[32];
|
||||
snprintf(name, sizeof name, "mod_%08x", m->sets[k]);
|
||||
int set = rt_patch_set(name);
|
||||
if (set >= 0) rt_patch_switch(set, 1);
|
||||
else rt_log("mods: %s: patch set %s is not in this build", m->name, name);
|
||||
// ---- ExeFS data writes (build-cache/exefs.data; tools/exefs_patches.py data_line has the format)
|
||||
|
||||
// 1 when key names one of m's usable ExeFS patch files: one with a patch set, or one with data writes only.
|
||||
static int owns(const ModInfo *m, uint32_t key) {
|
||||
for (int k = 0; k < m->nsets; k++)
|
||||
if (m->sets[k] == key) return 1;
|
||||
for (int k = 0; k < m->ndata; k++)
|
||||
if (m->data[k] == key) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hex_digit(char c) {
|
||||
return c >= '0' && c <= '9' ? c - '0'
|
||||
: c >= 'a' && c <= 'f' ? c - 'a' + 10
|
||||
: c >= 'A' && c <= 'F' ? c - 'A' + 10
|
||||
: -1;
|
||||
}
|
||||
|
||||
// One line: "<file hash>\t<content hash>\t<offset in main>\t<bytes>[\tbase]", all hex. 0 when malformed.
|
||||
typedef struct { char key[9], content[9]; uint64_t off; uint8_t bytes[256]; int n, base; } DataRun;
|
||||
static int parse_run(const char *line, size_t len, DataRun *r) {
|
||||
char copy[1100], *f[6];
|
||||
if (len >= sizeof copy) return 0;
|
||||
memcpy(copy, line, len);
|
||||
copy[len] = 0;
|
||||
int nf = 0;
|
||||
for (char *p = copy; p && nf < 6; nf++) {
|
||||
f[nf] = p;
|
||||
p = strchr(p, '\t');
|
||||
if (p) *p++ = 0;
|
||||
}
|
||||
if (nf < 4 || nf > 5 || strlen(f[0]) != 8 || strlen(f[1]) != 8) return 0;
|
||||
memcpy(r->key, f[0], 9);
|
||||
memcpy(r->content, f[1], 9);
|
||||
char *end;
|
||||
r->off = strtoull(f[2], &end, 16);
|
||||
size_t hex = strlen(f[3]);
|
||||
if (!*f[2] || *end || !hex || hex % 2 || hex / 2 > sizeof r->bytes) return 0;
|
||||
r->n = (int)(hex / 2);
|
||||
for (int i = 0; i < r->n; i++) {
|
||||
int hi = hex_digit(f[3][2 * i]), lo = hex_digit(f[3][2 * i + 1]);
|
||||
if (hi < 0 || lo < 0) return 0;
|
||||
r->bytes[i] = (uint8_t)(hi << 4 | lo);
|
||||
}
|
||||
r->base = nf == 5;
|
||||
return nf == 4 || (!strcmp(f[4], "base") && r->n == 8);
|
||||
}
|
||||
|
||||
// Stores n bytes into the loaded main at a: a read-only page (.rodata, loader.c seal) is writable for its part of the
|
||||
// copy only, as cheats.c wr_main does. 0 when a page is neither readable nor writable.
|
||||
static int store_main(uint64_t a, const uint8_t *from, int n) {
|
||||
while (n > 0) {
|
||||
uint64_t page = a & ~0xFFFull, end;
|
||||
int k = (int)(page + 0x1000 - a);
|
||||
if (k > n) k = n;
|
||||
int prot = plat_mem_state(page, page + 0x1000, &end);
|
||||
if (prot != PLAT_PROT_READ && prot != PLAT_PROT_RW) return 0;
|
||||
if (prot == PLAT_PROT_READ && !plat_mem_protect(page, 0x1000, PLAT_PROT_RW)) return 0;
|
||||
memcpy((void *)(uintptr_t)a, from, (size_t)k);
|
||||
if (prot == PLAT_PROT_READ) plat_mem_protect(page, 0x1000, PLAT_PROT_READ);
|
||||
a += (uint64_t)k;
|
||||
from += k;
|
||||
n -= k;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
long mods_exefs_data(const ModInfo *m, uint64_t base, uint64_t size, uint64_t code_end,
|
||||
int (*store)(uint64_t addr, const uint8_t *bytes, int n)) {
|
||||
if (!store) store = store_main;
|
||||
long total = 0, found = 0;
|
||||
for (int pass = 0; pass < 2; pass++) { // pass 0 checks every line, pass 1 stores
|
||||
// exefs.data must hold every byte the status lines count: one missing or cut short would leave the code
|
||||
// patches on without their data
|
||||
if (pass && found != (long)m->data_bytes) {
|
||||
rt_log("mods: %s: build-cache/exefs.data holds %ld of the %u bytes its ExeFS patch files write", m->name,
|
||||
found, m->data_bytes);
|
||||
return -1;
|
||||
}
|
||||
for (const char *line = exefs_data(), *next; *line; line = next) {
|
||||
const char *nl = strchr(line, '\n');
|
||||
size_t len = nl ? (size_t)(nl - line) : strlen(line);
|
||||
next = nl ? nl + 1 : line + len;
|
||||
if (len && line[len - 1] == '\r') len--;
|
||||
char key[9];
|
||||
snprintf(key, sizeof key, "%.*s", (int)(len < 8 ? len : 8), line);
|
||||
if (!len || *line == '#' || !owns(m, (uint32_t)strtoul(key, NULL, 16))) continue;
|
||||
DataRun r;
|
||||
char kind[16], over[256], note[256];
|
||||
// a line whose file the status does not list as built (another content) or that leaves main's data: the
|
||||
// two build-cache files and this build disagree
|
||||
if (!parse_run(line, len, &r) || !status_line(r.key, r.content, kind, over, note) ||
|
||||
(strcmp(kind, "code") && strcmp(kind, "data")) || r.off < code_end || r.off > size ||
|
||||
(uint64_t)r.n > size - r.off) {
|
||||
rt_log("mods: %s: a line of build-cache/exefs.data does not fit this build: %.*s", m->name, (int)len,
|
||||
line);
|
||||
return -1;
|
||||
}
|
||||
if (!pass) {
|
||||
found += r.n;
|
||||
continue;
|
||||
}
|
||||
if (r.base) { // a DT_RELR word holds its addend; the dynamic linker adds the module's base
|
||||
uint64_t w;
|
||||
memcpy(&w, r.bytes, 8);
|
||||
w += base;
|
||||
memcpy(r.bytes, &w, 8);
|
||||
}
|
||||
if (!store(base + r.off, r.bytes, r.n)) {
|
||||
rt_log("mods: %s: cannot write main+%#llx", m->name, (unsigned long long)r.off);
|
||||
return -1;
|
||||
}
|
||||
total += r.n;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
static char *dup_text(const char *s) {
|
||||
if (!s) return NULL;
|
||||
size_t n = strlen(s) + 1;
|
||||
char *d = malloc(n);
|
||||
if (d) memcpy(d, s, n);
|
||||
return d;
|
||||
}
|
||||
|
||||
void mods_probe_cache(const char *status, const char *data) {
|
||||
free(status_text);
|
||||
free(data_text);
|
||||
status_text = dup_text(status);
|
||||
data_text = dup_text(data);
|
||||
}
|
||||
|
||||
void mods_rescan(void) {
|
||||
free(mods);
|
||||
mods = NULL;
|
||||
nmods = 0;
|
||||
scanned = 0;
|
||||
}
|
||||
|
||||
// Switches the patch set mod_<key> on; 0 when this build lacks it (logged).
|
||||
static int set_on(const ModInfo *m, uint32_t key) {
|
||||
char name[32];
|
||||
snprintf(name, sizeof name, "mod_%08x", key);
|
||||
int set = rt_patch_set(name);
|
||||
if (set >= 0) rt_patch_switch(set, 1);
|
||||
else rt_log("mods: %s: patch set %s is not in this build", m->name, name);
|
||||
return set >= 0;
|
||||
}
|
||||
|
||||
void mods_exefs_apply(uint64_t base, uint64_t size, uint64_t code_end,
|
||||
int (*store)(uint64_t addr, const uint8_t *bytes, int n)) {
|
||||
// bottom first: the top mod's data writes land last and win, as its RomFS files do. Patch sets are only switches
|
||||
// (rt_patch_switch), so their order does not matter.
|
||||
for (int i = nmods - 1; i >= 0; i--) {
|
||||
const ModInfo *m = &mods[i];
|
||||
if (!start_enabled[i] || m->problem != MOD_OK || !(m->nsets || m->ndata)) continue;
|
||||
// the data first: a mod whose data writes cannot all be stored keeps its code patches off too, as half of a
|
||||
// patch file is worse than none
|
||||
long stored = size ? mods_exefs_data(m, base, size, code_end, store) : 0;
|
||||
if (stored < 0) {
|
||||
rt_log("mods: %s: its ExeFS patch files stay off: their data writes could not all be stored", m->name);
|
||||
continue;
|
||||
}
|
||||
for (int k = 0; k < m->nsets; k++) set_on(m, m->sets[k]);
|
||||
char over[256];
|
||||
snprintf(over, sizeof over, "%s", m->overrides);
|
||||
for (char *s = strtok(over, ","); s; s = strtok(NULL, ",")) {
|
||||
@@ -422,10 +588,20 @@ void mods_exefs_switch(void) {
|
||||
if (set >= 0) rt_patch_switch(set, 0);
|
||||
rt_log("mods: %s replaces the port's patch set %s; it stays off", m->name, s);
|
||||
}
|
||||
rt_log("mods: %s: %d ExeFS patch files on", m->name, m->nsets);
|
||||
rt_log("mods: %s: %d ExeFS patch files on, %ld bytes of main's data stored", m->name, m->nsets + m->ndata,
|
||||
stored);
|
||||
}
|
||||
}
|
||||
|
||||
void mods_exefs_switch(void) {
|
||||
mods_scan();
|
||||
const RtLoaded *main_mod = NULL;
|
||||
for (int i = 0; ldr_module(i); i++)
|
||||
if (!strcmp(ldr_module(i)->name, "main")) main_mod = ldr_module(i);
|
||||
if (main_mod) mods_exefs_apply(main_mod->base, main_mod->size, main_mod->text_end, NULL);
|
||||
else mods_exefs_apply(0, 0, 0, NULL);
|
||||
}
|
||||
|
||||
int mods_exefs_pending(uint32_t *key) {
|
||||
int n = 0;
|
||||
for (int i = 0; i < nmods; i++) {
|
||||
|
||||
+24
-2
@@ -2,7 +2,8 @@
|
||||
// them: <mod>/romfs, <mod>/exefs and <mod>/cheats, the Atmosphere layout (atmosphere/contents/<title id>/romfs,
|
||||
// atmosphere/exefs_patches/<name>/*.ips) or TKMM's merged output (romfs or romfslite, exefs, cheats). ExeFS patch
|
||||
// files (.ips, .pchtxt) for the game's main are patch sets the setup builds into the game (tools/exefs_patches.py),
|
||||
// switched with their mod (mods_exefs_switch). The list is scanned once at start (romfs.c overlay_init);
|
||||
// switched with their mod (mods_exefs_switch), and data writes stored into main before the game runs (mods_exefs_data).
|
||||
// The list is scanned once at start (romfs.c overlay_init);
|
||||
// its order and switches live in totk.ini [mods] (TOTK_MODS_ORDER: names top first, separated by '|';
|
||||
// TOTK_MODS_DISABLED: names switched off). The top mod wins a file two mods replace.
|
||||
#pragma once
|
||||
@@ -36,6 +37,10 @@ typedef struct {
|
||||
uint32_t pending, pending_key; // ExeFS patch files the build lacks; FNV-1a of their names and contents
|
||||
int nsets;
|
||||
uint32_t sets[16]; // the built patch sets mod_<hash> of its ExeFS patch files
|
||||
int ndata;
|
||||
uint32_t data[16]; // the hashes of its ExeFS patch files with data writes only (no set)
|
||||
uint32_t data_bytes; // the bytes its usable ExeFS patch files write into main's data, as their
|
||||
// status lines count them ("data <bytes>")
|
||||
char overrides[256]; // the port's own patch sets its patches replace, comma separated
|
||||
} ModInfo;
|
||||
|
||||
@@ -51,8 +56,25 @@ ModInfo *mods_at(int i); // 0 = top of the order; NULL out of range
|
||||
void mods_set_enabled(int i, int on);
|
||||
void mods_move(int i, int delta); // -1 up, +1 down; nothing at the ends
|
||||
int mods_changed(void); // 1 when the order or a switch differs from the start
|
||||
// After rt_patches_init: the patch sets of the enabled mods' ExeFS patches on, the port's own sets they replace off.
|
||||
// After ldr_load and rt_patches_init, before any game code: the patch sets of the enabled mods' ExeFS patches on, the
|
||||
// port's own sets they replace off, and their data writes stored into main (mods_exefs_data).
|
||||
void mods_exefs_switch(void);
|
||||
// What mods_exefs_switch does with main's image at base (size bytes, code below code_end; size 0: no data stored), for
|
||||
// every enabled mod from the bottom one up, so that the top mod's data writes win where two mods write the same bytes.
|
||||
// store as for mods_exefs_data.
|
||||
void mods_exefs_apply(uint64_t base, uint64_t size, uint64_t code_end,
|
||||
int (*store)(uint64_t addr, const uint8_t *bytes, int n));
|
||||
// Stores the data writes of mod m's ExeFS patch files (build-cache/exefs.data) into main's image at base: size bytes,
|
||||
// code below code_end. A DT_RELR word gets base added, as the dynamic linker adds it. Every line is checked before the
|
||||
// first store: one that falls outside [code_end, size) or does not belong to a usable file of m keeps them all out.
|
||||
// store writes n bytes at a guest address, 1 on success; NULL: into the loaded main, whose .rodata is read-only.
|
||||
// Returns the bytes stored, -1 when a line kept them out or a store failed. --probe-mods passes a buffer and memcpy.
|
||||
long mods_exefs_data(const ModInfo *m, uint64_t base, uint64_t size, uint64_t code_end,
|
||||
int (*store)(uint64_t addr, const uint8_t *bytes, int n));
|
||||
// --probe-mods: the texts of build-cache/exefs.status and exefs.data instead of the files (NULL: read the file), and a
|
||||
// fresh scan at the next mods_scan.
|
||||
void mods_probe_cache(const char *status, const char *data);
|
||||
void mods_rescan(void);
|
||||
// The ExeFS patch files of the enabled mods that the build lacks, their FNV-1a mixed into *key (cheats.c starts the
|
||||
// rebuild for them).
|
||||
int mods_exefs_pending(uint32_t *key);
|
||||
|
||||
+88
-1
@@ -39,6 +39,93 @@ static int remove_cb(const char *name, int kind, uint64_t size, void *ctx) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
static uint32_t fnv(const char *p, size_t n) {
|
||||
uint32_t h = 0x811C9DC5u;
|
||||
for (size_t i = 0; i < n; i++) h = (h ^ (uint8_t)p[i]) * 0x01000193u;
|
||||
return h;
|
||||
}
|
||||
|
||||
static ModInfo *by_name(const char *name) {
|
||||
for (int i = 0; mods_at(i); i++)
|
||||
if (!strcmp(mods_at(i)->name, name)) return mods_at(i);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// mods_exefs_data's store into a buffer standing in for main
|
||||
static int store_buffer(uint64_t a, const uint8_t *p, int n) {
|
||||
memcpy((void *)(uintptr_t)a, p, (size_t)n);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// A second scan of a folder of its own: data writes of ExeFS patch files (build-cache/exefs.data, given as text).
|
||||
static void probe_data(const char *root) {
|
||||
remove_tree(root);
|
||||
plat_file_mkdir_p(root);
|
||||
const char *bid = *mods_main_build_id() ? mods_main_build_id() : "0";
|
||||
char n_ips[160], p_ips[160], q_ips[160], x_ips[160], y_ips[160];
|
||||
snprintf(n_ips, sizeof n_ips, "N/exefs/%s.ips", bid);
|
||||
snprintf(p_ips, sizeof p_ips, "P/exefs/%s.ips", bid);
|
||||
snprintf(q_ips, sizeof q_ips, "Q/exefs/%s.ips", bid);
|
||||
snprintf(x_ips, sizeof x_ips, "X/exefs/%s.ips", bid);
|
||||
snprintf(y_ips, sizeof y_ips, "Y/exefs/%s.ips", bid);
|
||||
const char *files[] = {n_ips, p_ips, q_ips, x_ips, y_ips};
|
||||
for (size_t i = 0; i < sizeof files / sizeof *files; i++) put(root, files[i]);
|
||||
uint32_t kn = fnv(n_ips, strlen(n_ips)), kp = fnv(p_ips, strlen(p_ips)), kq = fnv(q_ips, strlen(q_ips));
|
||||
uint32_t kx = fnv(x_ips, strlen(x_ips)), ky = fnv(y_ips, strlen(y_ips));
|
||||
uint32_t c = fnv("probe", 5);
|
||||
char status[512], data[768];
|
||||
snprintf(status, sizeof status,
|
||||
"%08X\t%08X\tdata\t-\tdata 11\n%08X\t%08X\tdata\t-\tdata 1\n%08X\t%08X\tdata\t-\tdata 4\n"
|
||||
"%08X\t%08X\tdata\t-\tdata 1\n%08X\t%08X\tdata\t-\tdata 1\n",
|
||||
kn, c, kp, c, kq, c, kx, c, ky, c);
|
||||
snprintf(data, sizeof data,
|
||||
"# file hash, content hash, offset in main, bytes[, base]\n"
|
||||
"%08X\t%08X\t10\t414243\n" // N: three bytes of data
|
||||
"%08X\t%08X\t18\t0800000000000000\tbase\r\n" // N: a DT_RELR word, 8 plus main's base (a CRLF line)
|
||||
"DEADBEEF\t%08X\t20\t99\n" // another file's: not N's
|
||||
"%08X\t%08X\t4\t77\n" // P: below main's data
|
||||
"%08X\t%08X\t30\t66\n" // Q: 1 of the 4 bytes its status counts
|
||||
"%08X\t%08X\t28\tBB\n" // X: the byte Y writes too
|
||||
"%08X\t%08X\t28\tAA\n", // Y, above X in the order: its byte stays
|
||||
kn, c, kn, c, c, kp, c, kq, c, kx, c, ky, c);
|
||||
mods_probe_cache(status, data);
|
||||
cfg_set("TOTK_MODS_DIR", root);
|
||||
cfg_set("TOTK_MODS_ORDER", "Y"); // Y on top, the rest by name: Y N P Q X
|
||||
cfg_set("TOTK_MODS_DISABLED", "");
|
||||
mods_rescan();
|
||||
mods_scan();
|
||||
ModInfo *n = by_name("N"), *p = by_name("P"), *q = by_name("Q");
|
||||
CHECK(n && p && q && by_name("X") && by_name("Y"), "second scan: five mods");
|
||||
if (n && p && q) {
|
||||
CHECK(n->kind == MOD_EXEFS && n->problem == MOD_OK && n->ndata == 1 && !n->nsets,
|
||||
"an ExeFS patch file with data writes only makes a usable mod");
|
||||
uint8_t img[0x40];
|
||||
memset(img, 0, sizeof img);
|
||||
uint64_t base = (uint64_t)(uintptr_t)img, word;
|
||||
long stored = mods_exefs_data(n, base, sizeof img, 8, store_buffer);
|
||||
memcpy(&word, img + 0x18, 8);
|
||||
CHECK(stored == 11 && !memcmp(img + 0x10, "ABC", 3) && word == base + 8 && !img[0x20],
|
||||
"data writes: bytes as they are, a DT_RELR word rebased, other files' lines left");
|
||||
memset(img, 0x5A, sizeof img);
|
||||
int untouched = 1;
|
||||
CHECK(mods_exefs_data(p, base, sizeof img, 8, store_buffer) == -1, "a line below main's data keeps all out");
|
||||
CHECK(n->data_bytes == 11 && q->data_bytes == 4, "the status lines count the data bytes");
|
||||
CHECK(mods_exefs_data(q, base, sizeof img, 8, store_buffer) == -1,
|
||||
"a data file holding fewer bytes than the status counts keeps all out");
|
||||
for (size_t i = 0; i < sizeof img; i++) untouched &= img[i] == 0x5A;
|
||||
CHECK(untouched, "nothing stored when a line does not fit or bytes are missing");
|
||||
// every enabled mod as at start: X and Y write the same byte, and Y, the higher one, wins as its RomFS files
|
||||
// would; P and Q stay out as above
|
||||
memset(img, 0, sizeof img);
|
||||
CHECK(EQ(mods_at(0)->name, "Y") && EQ(mods_at(4)->name, "X"), "mods_order puts Y on top and X at the bottom");
|
||||
mods_exefs_apply(base, sizeof img, 8, store_buffer);
|
||||
CHECK(img[0x28] == 0xAA, "two mods write the same byte: the top mod's byte stays");
|
||||
CHECK(!memcmp(img + 0x10, "ABC", 3) && !img[0x30], "the other mods' data as each one stores it alone");
|
||||
}
|
||||
mods_probe_cache(NULL, NULL);
|
||||
remove_tree(root);
|
||||
}
|
||||
|
||||
// s ends in tail, either path separator matching either.
|
||||
static int ends_with(const char *s, const char *tail) {
|
||||
size_t n = strlen(s), t = strlen(tail);
|
||||
@@ -126,7 +213,7 @@ int probe_mods(void) {
|
||||
CHECK(EQ(cfg_get("TOTK_MODS_DISABLED"), "A"), "switching A off writes it");
|
||||
CHECK(mods_changed(), "changes seen");
|
||||
}
|
||||
remove_tree(root);
|
||||
probe_data(root);
|
||||
printf("mods: %s\n", fails ? "FAIL" : "ok");
|
||||
int table = restbl_probe();
|
||||
return fails || table ? 1 : 0;
|
||||
|
||||
+2
-2
@@ -1109,8 +1109,8 @@ static void tab_mods(float scale) {
|
||||
char code[512] = "", problem[512] = "";
|
||||
if (m->nsets && m->overrides[0])
|
||||
snprintf(code, sizeof code, menu_text(UI_MENU_MODS_REPLACES_BUILTIN), m->nsets, m->overrides);
|
||||
else if (m->nsets)
|
||||
snprintf(code, sizeof code, menu_text(UI_MENU_MODS_EXEFS_ON), m->nsets);
|
||||
else if (m->nsets || m->ndata)
|
||||
snprintf(code, sizeof code, menu_text(UI_MENU_MODS_EXEFS_ON), m->nsets + m->ndata);
|
||||
switch (m->problem) {
|
||||
case MOD_EMPTY: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EMPTY)); break;
|
||||
case MOD_PLUGIN: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_SUBSDK)); break;
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""tools/exefs_patches.py on modules it makes itself: code writes become sites, data writes become runs of the data file
|
||||
as emulators apply a patch file before the dynamic linker (plain bytes, DT_RELR words rebased, writes into words an
|
||||
explicit-addend relocation fills dropped), the writes it refuses (outside the image, into the linker's tables or a
|
||||
lazily bound pointer), the status and data files, and the check after the lift against the bytes recomp/gen.py lists in
|
||||
folded.txt, with gen.py run on a module with one jump table. No game files.
|
||||
|
||||
Run with .venv (lz4 and numpy, as recomp/): python tests/exefs_patches_test.py
|
||||
"""
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import exefs_patches as ep # noqa: E402
|
||||
|
||||
BUILD_ID = bytes.fromhex('0123456789abcdef') + bytes(24)
|
||||
BID_NAME = '0123456789ABCDEF' # an IPS file name may leave the build id's trailing zeros out
|
||||
MOV_W8_100, MOV_W8_200, RET = 0x52802008, 0x52804008, 0xD65F03C0
|
||||
R_RELATIVE, R_GLOB_DAT, R_JUMP_SLOT = 1027, 1025, 1026
|
||||
|
||||
|
||||
def nso(text, ro, data, bss=0x1000):
|
||||
"""An uncompressed NSO with .text at 0, .rodata at 0x1000 and .data at 0x2000."""
|
||||
segs = [(0x0, text), (0x1000, ro), (0x2000, data)]
|
||||
hdr = bytearray(0x100)
|
||||
hdr[0:4] = b'NSO0'
|
||||
off = 0x100
|
||||
for i, (mem, d) in enumerate(segs):
|
||||
struct.pack_into('<IIII', hdr, 0x10 + 0x10 * i, off, mem, len(d), bss if i == 2 else 0)
|
||||
struct.pack_into('<I', hdr, 0x60 + 4 * i, len(d))
|
||||
off += len(d)
|
||||
hdr[0x40:0x60] = BUILD_ID
|
||||
return bytes(hdr) + b''.join(d for _, d in segs)
|
||||
|
||||
|
||||
def put(buf, off, data):
|
||||
buf[off:off + len(data)] = data
|
||||
|
||||
|
||||
def reloc_module():
|
||||
"""fn at 0x8 (mov w8, #0x100; ret); .rodata: constants at 0x1000, MOD0 at 0x1100 and the relocation tables at
|
||||
0x1200 (RELA: a RELATIVE word at 0x2100, a GLOB_DAT word at 0x2108), 0x1300 (JMPREL: a JUMP_SLOT word at 0x2110)
|
||||
and 0x1400 (RELR: the word at 0x2118, addend 0x30); .data: .dynamic at 0x2000, plain bytes at 0x2200."""
|
||||
text = struct.pack('<4I', 0x14000002, 0x1100, MOV_W8_100, RET)
|
||||
ro = bytearray(0x500)
|
||||
put(ro, 0x000, b'constant bytes!!')
|
||||
put(ro, 0x100, b'MOD0' + struct.pack('<6i', 0x2000 - 0x1100, 0, 0, -0x1100, -0x1100, 0))
|
||||
put(ro, 0x200, struct.pack('<QQq', 0x2100, R_RELATIVE, 0x8) + struct.pack('<QQq', 0x2108, R_GLOB_DAT, 0))
|
||||
put(ro, 0x300, struct.pack('<QQq', 0x2110, R_JUMP_SLOT, 0))
|
||||
put(ro, 0x400, struct.pack('<Q', 0x2118))
|
||||
data = bytearray(0x300)
|
||||
dyn = [(7, 0x1200), (8, 48), (23, 0x1300), (2, 24), (0x24, 0x1400), (0x23, 8), (0, 0)]
|
||||
put(data, 0x000, b''.join(struct.pack('<QQ', t, v) for t, v in dyn))
|
||||
put(data, 0x118, struct.pack('<Q', 0x30))
|
||||
put(data, 0x200, b'ABCDEFGH')
|
||||
return nso(text, bytes(ro), bytes(data))
|
||||
|
||||
|
||||
# A module with one jump table (clang's compressed form), assembled from:
|
||||
# b fn ; 0x0
|
||||
# .word mod0 ; 0x4: the MOD0 offset
|
||||
# fn: cmp w0, #3 ; 0x8
|
||||
# b.hi Ldefault
|
||||
# adrp x8, table ; table = 0x1020: .byte 0, 2, 4, 6
|
||||
# add x8, x8, #0x20
|
||||
# adr x9, L0
|
||||
# ldrb w10, [x8, w0, uxtw]
|
||||
# add x9, x9, x10, lsl #2
|
||||
# br x9
|
||||
# L0..L3: mov w0, #10/20/30/40; ret ; Ldefault: mov w0, #0; ret
|
||||
# .rodata: 32 bytes of text before the table, 21 after it, MOD0 at 0x1040 and a RELATIVE relocation to fn (the
|
||||
# lifter's function seed); .data: .dynamic (DT_RELA) and the relocated word.
|
||||
JT_TEXT = bytes.fromhex(
|
||||
'02000014401000001f0c0071e8010054080000b008810091890000100a49603829090a8b20011fd640018052c0035fd680028052'
|
||||
'c0035fd6c0038052c0035fd600058052c0035fd600008052c0035fd6')
|
||||
JT_RO = bytes.fromhex(
|
||||
'756e746f7563686564206279746573206265666f726520746865207461626c6500020406627974657320616674657220746865'
|
||||
'207461626c65000000000000004d4f4430c00f00000000000000000000c0efffffc0efffff00000000000000003020000000000000'
|
||||
'03040000000000000800000000000000')
|
||||
JT_DATA = struct.pack('<7Q', 7, 0x1060, 8, 24, 0, 0, 0) # DT_RELA, DT_RELASZ, DT_NULL; the word the RELATIVE fills
|
||||
|
||||
|
||||
def ips32(writes):
|
||||
"""An IPS32 file of {image offset: bytes} (offsets count the NSO header, as emulators read them)."""
|
||||
body = b''.join((off + ep.NSO_HEADER).to_bytes(4, 'big') + len(d).to_bytes(2, 'big') + d
|
||||
for off, d in sorted(writes.items()))
|
||||
return b'IPS32' + body + b'EEOF'
|
||||
|
||||
|
||||
class Classify(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
path = Path(cls.tmp.name) / 'main'
|
||||
path.write_bytes(reloc_module())
|
||||
cls.m = ep.Main(path)
|
||||
cls.md = ep.MainData(cls.m)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def run_file(self, writes, md=True):
|
||||
return ep.classify(self.m, self.md if md else None, self.m.nso.build_id, Path(f'{BID_NAME}.ips'),
|
||||
ips32(writes))
|
||||
|
||||
def test_relocated_words(self):
|
||||
self.assertEqual(self.md.rela, {0x2100: R_RELATIVE, 0x2108: R_GLOB_DAT, 0x2110: R_JUMP_SLOT})
|
||||
self.assertEqual(self.md.relr, {0x2118})
|
||||
|
||||
def test_code_only(self):
|
||||
kind, note, sites, runs, dropped = self.run_file({0x8: struct.pack('<I', MOV_W8_200)})
|
||||
self.assertEqual((kind, note, runs, dropped), ('code', '', [], 0))
|
||||
self.assertEqual(sites, {0x8: [MOV_W8_200]})
|
||||
|
||||
def test_data_only(self):
|
||||
kind, note, sites, runs, dropped = self.run_file({0x1000: b'\xAA\xBB', 0x2200: b'XY'})
|
||||
self.assertEqual((kind, sites, dropped), ('data', {}, 0))
|
||||
self.assertEqual(runs, [(0x1000, b'\xAA\xBB', False), (0x2200, b'XY', False)])
|
||||
|
||||
def test_code_and_data(self):
|
||||
kind, _, sites, runs, _ = self.run_file({0x8: struct.pack('<I', MOV_W8_200), 0x1004: b'\x01'})
|
||||
self.assertEqual(kind, 'code')
|
||||
self.assertEqual(sites, {0x8: [MOV_W8_200]})
|
||||
self.assertEqual(runs, [(0x1004, b'\x01', False)])
|
||||
|
||||
def test_same_bytes_are_nothing(self):
|
||||
kind, _, _, runs, _ = self.run_file({0x1000: b'constant'})
|
||||
self.assertEqual((kind, runs), ('empty', []))
|
||||
|
||||
def test_explicit_addend_words_change_nothing(self):
|
||||
kind, _, _, runs, dropped = self.run_file({0x2100: b'\x11\x22\x33\x44', 0x2108: b'\x55'})
|
||||
self.assertEqual((kind, runs, dropped), ('empty', [], 5))
|
||||
|
||||
def test_relr_word_is_rebased(self):
|
||||
kind, _, _, runs, _ = self.run_file({0x2118: b'\x31'})
|
||||
self.assertEqual(kind, 'data')
|
||||
self.assertEqual(runs, [(0x2118, struct.pack('<Q', 0x31), True)])
|
||||
|
||||
def test_long_runs_are_split(self):
|
||||
_, _, _, runs, _ = self.run_file({0x2200: bytes(i % 255 + 1 for i in range(ep.RUN_MAX)) + b'\xFF'})
|
||||
self.assertEqual([(o, len(d)) for o, d, _ in runs], [(0x2200, ep.RUN_MAX), (0x2200 + ep.RUN_MAX, 1)])
|
||||
|
||||
def test_refused_writes(self):
|
||||
for writes, why in (({0x2110: b'\x01'}, 'binds lazily'), ({0x2000: b'\x01'}, 'linker\'s tables'),
|
||||
({0x1208: b'\x01'}, 'linker\'s tables'), ({0x4000: b'\x01'}, 'outside main\'s image'),
|
||||
({-0x80: b'\x01'}, 'outside main\'s image')):
|
||||
kind, note, sites, runs, _ = self.run_file({0x8: struct.pack('<I', MOV_W8_200), **writes})
|
||||
self.assertEqual((kind, sites, runs), ('unsupported', {}, []), note)
|
||||
self.assertIn(why, note)
|
||||
|
||||
def test_data_without_a_data_file(self):
|
||||
kind, note, _, _, _ = self.run_file({0x1000: b'\xAA'}, md=False)
|
||||
self.assertEqual(kind, 'unsupported')
|
||||
self.assertIn('only code patches are supported', note)
|
||||
|
||||
def test_pchtxt(self):
|
||||
text = (f'@nsobid-{BID_NAME}\n@flag offset_shift 0x100\n@enabled\n'
|
||||
f'00001000 AABB\n00002200 "XY"\n@disabled\n00002204 CC\n')
|
||||
kind, _, _, runs, _ = ep.classify(self.m, self.md, self.m.nso.build_id, Path('x.pchtxt'), text.encode())
|
||||
self.assertEqual(kind, 'data')
|
||||
self.assertEqual(runs, [(0x1000, b'\xAA\xBB', False), (0x2200, b'XY', False)])
|
||||
|
||||
|
||||
class Tool(unittest.TestCase):
|
||||
"""The command line: build, then the check after the lift."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.d = Path(self.tmp.name)
|
||||
(self.d / 'exefs').mkdir()
|
||||
(self.d / 'patches.txt').write_text('# no sets of the port\n')
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def mod_file(self, mod, writes):
|
||||
f = self.d / 'mods' / mod / 'exefs' / f'{BID_NAME}.ips'
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
f.write_bytes(ips32(writes))
|
||||
|
||||
def tool(self, *args):
|
||||
r = subprocess.run([sys.executable, str(ROOT / 'tools' / 'exefs_patches.py'), '--exefs', str(self.d / 'exefs'),
|
||||
*map(str, args)], capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
||||
return r.stdout
|
||||
|
||||
def build(self):
|
||||
self.tool('--mods', self.d / 'mods', '--patches', self.d / 'patches.txt', '--out', self.d / 'out.txt',
|
||||
'--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data')
|
||||
status = {f[0]: f for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
|
||||
data = [line.split('\t') for line in (self.d / 'exefs.data').read_text().splitlines()
|
||||
if not line.startswith('#')]
|
||||
return status, data
|
||||
|
||||
def key(self, mod):
|
||||
return ep.fnv(f'{mod}/exefs/{BID_NAME}.ips'.encode())
|
||||
|
||||
def test_build_and_check(self):
|
||||
(self.d / 'exefs' / 'main').write_bytes(reloc_module())
|
||||
self.mod_file('Code', {0x8: struct.pack('<I', MOV_W8_200), 0x2200: b'Q'})
|
||||
self.mod_file('Data', {0x1000: b'\xAA', 0x2118: b'\x31'})
|
||||
self.mod_file('Bad', {0x2110: b'\x01'})
|
||||
status, data = self.build()
|
||||
code, dat, bad = self.key('Code'), self.key('Data'), self.key('Bad')
|
||||
self.assertEqual([status[k][2] for k in (code, dat, bad)], ['code', 'data', 'unsupported'])
|
||||
self.assertEqual([status[k][4] for k in (code, dat)], ['data 1', 'data 9']) # bytes its data lines hold
|
||||
self.assertIn(f'set mod_{code.lower()} TOTK_MODPATCH_{code} 0', (self.d / 'out.txt').read_text())
|
||||
self.assertEqual(sorted((f[0], f[2], f[3], f[4:]) for f in data),
|
||||
sorted([(code, '2200', '51', []), (dat, '1000', 'aa', []),
|
||||
(dat, '2118', '3100000000000000', ['base'])]))
|
||||
# after the lift: .rodata bytes the lifted code read make the Data file unsupported; the Code file's data is
|
||||
# in .data, which the lifter never reads
|
||||
folded = self.d / 'folded.txt'
|
||||
folded.write_text('# test\n1000 1001\n')
|
||||
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', folded)
|
||||
status2 = {f[0]: f for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
|
||||
self.assertEqual(status2[code][2], 'code')
|
||||
self.assertEqual(status2[dat][2:4], ['unsupported', '-'])
|
||||
self.assertIn('jump table', status2[dat][4])
|
||||
left = [line.split('\t')[0] for line in (self.d / 'exefs.data').read_text().splitlines()
|
||||
if not line.startswith('#')]
|
||||
self.assertEqual(left, [code])
|
||||
|
||||
def test_check_without_the_lift_list(self):
|
||||
(self.d / 'exefs' / 'main').write_bytes(reloc_module())
|
||||
self.mod_file('Ro', {0x1000: b'\xAA'})
|
||||
self.mod_file('Rw', {0x2200: b'Q'})
|
||||
self.build()
|
||||
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded',
|
||||
self.d / 'missing.txt')
|
||||
status = {f[0]: f[2] for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
|
||||
self.assertEqual((status[self.key('Ro')], status[self.key('Rw')]), ('unsupported', 'data'))
|
||||
|
||||
def test_jump_table_through_the_lift(self):
|
||||
"""recomp/gen.py lists the jump table's bytes in folded.txt; a write into them is refused, one beside them
|
||||
is kept."""
|
||||
(self.d / 'exefs' / 'main').write_bytes(nso(JT_TEXT, JT_RO, JT_DATA))
|
||||
r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'),
|
||||
'--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1'],
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
folded = self.d / 'gen' / 'main' / 'folded.txt'
|
||||
self.assertEqual(ep.read_ranges(folded), [(0x1020, 0x1024)])
|
||||
self.mod_file('Table', {0x1022: b'\x00'})
|
||||
self.mod_file('Beside', {0x1024: b'B'})
|
||||
self.build()
|
||||
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', folded)
|
||||
status = {f[0]: f[2] for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
|
||||
self.assertEqual((status[self.key('Table')], status[self.key('Beside')]), ('unsupported', 'data'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -315,6 +315,75 @@ class LiftKeyTest(unittest.TestCase):
|
||||
self.assertEqual(self.key(), before)
|
||||
|
||||
|
||||
class LiftRunTest(unittest.TestCase):
|
||||
"""Lift.run with fake tools: the ExeFS patch stage writes the status and data files, the lift lists the bytes it
|
||||
read, and the data writes are checked against that list after it; a failed lift or check drops both files."""
|
||||
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
base = Path(tmp.name)
|
||||
self.dest, temp = base / 'install', base / 'temp'
|
||||
self.dest.mkdir()
|
||||
temp.mkdir()
|
||||
layout = SimpleNamespace(recomp=ROOT / 'recomp', tools=ROOT / 'tools',
|
||||
patches=lambda version: base / 'patches-v143.txt')
|
||||
self.ctx = SimpleNamespace(dest=self.dest, temp=temp, layout=layout, keys={'lift': 'k'},
|
||||
state={'version': '1.4.3'}, reporter=Reporter(out=io.StringIO()))
|
||||
self.addCleanup(self.ctx.reporter.close)
|
||||
mod = self.dest / 'mods' / 'SQ' / 'exefs'
|
||||
mod.mkdir(parents=True)
|
||||
(mod / '277178B7DBA1B6D4.ips').write_bytes(b'IPS32EEOF')
|
||||
self.calls, self.fail = [], set()
|
||||
saved = steps.run_tool
|
||||
self.addCleanup(lambda: setattr(steps, 'run_tool', saved))
|
||||
steps.run_tool = self.run_tool
|
||||
|
||||
def run_tool(self, ctx, step, script, args, parse=None, cwd=None):
|
||||
args = [str(a) for a in args]
|
||||
name = Path(script).name + (' --folded' if '--folded' in args else '')
|
||||
self.calls.append((name, args))
|
||||
log = ctx.temp / 'tool.log'
|
||||
log.write_text('')
|
||||
if name in self.fail:
|
||||
return 1, log
|
||||
if name == 'exefs_patches.py':
|
||||
for flag in ('--status', '--data', '--out'):
|
||||
p = Path(args[args.index(flag) + 1])
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(flag)
|
||||
elif name == 'gen.py':
|
||||
out = Path(args[args.index('--out') + 1]) / 'main'
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / 'folded.txt').write_text('# test\n')
|
||||
return 0, log
|
||||
|
||||
def outputs(self):
|
||||
return [p.is_file() for p in (steps.exefs_status(self.ctx), steps.exefs_data(self.ctx))]
|
||||
|
||||
def test_data_file_and_the_check_after_the_lift(self):
|
||||
steps.Lift().run(self.ctx)
|
||||
self.assertEqual([c[0] for c in self.calls], ['exefs_patches.py', 'gen.py', 'exefs_patches.py --folded'])
|
||||
build = self.calls[0][1]
|
||||
self.assertEqual(build[build.index('--data') + 1], str(steps.exefs_data(self.ctx)))
|
||||
check = self.calls[2][1]
|
||||
self.assertEqual(check[check.index('--folded') + 1], str(self.ctx.temp / 'gen' / 'main' / 'folded.txt'))
|
||||
self.assertEqual(check[check.index('--status') + 1], str(steps.exefs_status(self.ctx)))
|
||||
self.assertEqual(self.outputs(), [True, True])
|
||||
|
||||
def test_failed_lift_drops_the_data_file(self):
|
||||
self.fail = {'gen.py'}
|
||||
with self.assertRaises(steps.InstallError):
|
||||
steps.Lift().run(self.ctx)
|
||||
self.assertEqual(self.outputs(), [False, False])
|
||||
|
||||
def test_failed_check_drops_both_files(self):
|
||||
self.fail = {'exefs_patches.py --folded'}
|
||||
steps.Lift().run(self.ctx)
|
||||
self.assertEqual(self.outputs(), [False, False])
|
||||
self.assertIn('warn.exefs_skipped', self.ctx.reporter.out.getvalue())
|
||||
|
||||
|
||||
class IniTest(unittest.TestCase):
|
||||
EXAMPLE = '[display]\n# vk_present = mailbox\n\n[language]\n# locale = en-US\n# lang = en-US\n# x = 1\n'
|
||||
|
||||
|
||||
@@ -164,6 +164,8 @@ class Main:
|
||||
def __init__(self, path):
|
||||
n = Nso(str(path))
|
||||
self.image = bytes(n.image)
|
||||
n.image = self.image # one copy of the image; tools/exefs_patches.py reads nso's dynamic info
|
||||
self.nso = n
|
||||
self.text = n.text # (offset, size)
|
||||
self.code_end = n.rodata[0] # .text and its padding up to .rodata, where code caves go
|
||||
self.lifter = Lifter(BASE, 'c')
|
||||
|
||||
+256
-44
@@ -1,11 +1,16 @@
|
||||
"""ExeFS patches of the mods folder to code patch sets: IPS, IPS32 and IPSwitch (.pchtxt) files for the game's main.
|
||||
"""ExeFS patches of the mods folder to code patch sets and data writes: IPS, IPS32 and IPSwitch (.pchtxt) files for
|
||||
the game's main.
|
||||
|
||||
Lifted code cannot be byte-patched when the game runs, so a patch file's writes into main's code become `site` lines
|
||||
(recomp/patches.py), one patch set per file, which the runtime switches on while its mod is on (runtime/mods.c). The
|
||||
writes go through the same rules as the mods' code cheats (tools/cheats.py convert): an instruction replaced in place,
|
||||
a branch into a code cave the file writes into main's zero padding inlined at the branch.
|
||||
a branch into a code cave the file writes into main's zero padding inlined at the branch. Writes into main's data
|
||||
(.rodata, .data, .bss) need no lifting: the runtime stores them into the loaded module before the game runs, as
|
||||
emulators and Atmosphere apply a patch file to the module before its dynamic linker runs (data_runs).
|
||||
|
||||
python tools/exefs_patches.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file> --status <file>
|
||||
python tools/exefs_patches.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file>
|
||||
--status <file> --data <file>
|
||||
python tools/exefs_patches.py --exefs <dir> --status <file> --data <file> --folded <gen>/main/folded.txt
|
||||
|
||||
The files are <mod>/exefs/*.ips and *.pchtxt (or the Atmosphere places of the exefs folder, as runtime/mods.c finds
|
||||
them) and <mod>/atmosphere/exefs_patches/<name>/*.ips. A file is for main when its build id matches main's: an IPS
|
||||
@@ -13,19 +18,37 @@ file's name, a pchtxt file's @nsobid line (trailing zeros may be left out). Offs
|
||||
offsets count the NSO header (0x100 bytes) before main's image; a pchtxt offset adds its `@flag offset_shift` first.
|
||||
A pchtxt patch line applies between @enabled and @disabled; @stop ends the file.
|
||||
|
||||
--out gets the --patches file followed by one set per usable file (`mod_<hash>`, switch TOTK_MODPATCH_<HASH>, default
|
||||
off) and its sites. --status gets one line per file, mods by name, tab separated:
|
||||
--out gets the --patches file followed by one set per file with sites (`mod_<hash>`, switch TOTK_MODPATCH_<HASH>,
|
||||
default off) and its sites. --status gets one line per file, mods by name, tab separated:
|
||||
|
||||
<file hash> <content hash> <kind> <built-in sets> <note>
|
||||
|
||||
file hash: FNV-1a 32 of "<mod folder name>/<path in the mod>" (forward slashes); content hash: the same over the file's
|
||||
bytes, so the runtime sees a file that changed after the build. kind:
|
||||
code a set of sites
|
||||
code a set of sites, and data writes when the file has some
|
||||
data data writes only
|
||||
other for another game version or module: not used
|
||||
empty no write into main for this version (or only the game's own words)
|
||||
unsupported <note> says why: a write into main's data, a write the site rules cannot express
|
||||
empty no write into main for this version (or only the game's own bytes)
|
||||
unsupported <note> says why: a write the site rules cannot express, a write outside main's image, into the dynamic
|
||||
linker's tables or into a pointer it binds lazily, or (after the lift) into bytes the lifted code reads
|
||||
as constants
|
||||
built-in sets: the port's own sets (the --patches file's sets that are no cheats) with a site this file patches too,
|
||||
comma separated, or "-": the runtime switches them off while the mod is on, so the mod's code runs.
|
||||
comma separated, or "-": the runtime switches them off while the mod is on, so the mod's code runs. The note of a code
|
||||
or data file with data writes is `data <bytes>`, the bytes its data lines hold: the runtime keeps the mod's patches off
|
||||
when the data file does not hold them all.
|
||||
|
||||
--data gets one line per run of data bytes of the code and data files, tab separated (runtime/mods.c reads it):
|
||||
|
||||
<file hash> <content hash> <offset in main, hex> <bytes, hex> [base]
|
||||
|
||||
`base` marks a DT_RELR word: 8 bytes, little-endian, the word before relocation; the runtime adds main's base, as the
|
||||
dynamic linker does. Without --data a data write makes its file unsupported.
|
||||
|
||||
The lifted code reads some of main's .text and .rodata at lift time instead of at run time: the jump tables
|
||||
(recomp/jumptables.py), whose targets become labels of the generated C. A data write over such bytes would not reach
|
||||
the lifted code. recomp/gen.py lists the ranges it read in <gen>/<module>/folded.txt; the second form above, run after
|
||||
the lift, makes every file with a data write into those ranges unsupported (its status line says so and its data lines
|
||||
go). Without that list, every data write into .rodata counts as one into read bytes.
|
||||
|
||||
A file the rules cannot express is unsupported as a whole: half of a patch is worse than none.
|
||||
|
||||
@@ -39,9 +62,12 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from cheats import TITLE, Main, convert, fnv # noqa: E402
|
||||
from nso import (DT_JMPREL, DT_NULL, DT_PLTRELSZ, DT_RELA, DT_RELASZ, DT_RELR, DT_RELRSZ, DT_STRSZ, # noqa: E402
|
||||
DT_STRTAB, DT_SYMTAB, R_AARCH64_JUMP_SLOT)
|
||||
|
||||
NSO_HEADER = 0x100
|
||||
EXEFS_PLACES = ('exefs', f'atmosphere/contents/{TITLE}/exefs', f'contents/{TITLE}/exefs', f'{TITLE}/exefs')
|
||||
RUN_MAX = 256 # bytes per line of the data file (runtime/mods.c reads lines of at most 1024 characters)
|
||||
|
||||
|
||||
def patch_files(mods):
|
||||
@@ -159,6 +185,99 @@ def word_writes(main, writes):
|
||||
return words
|
||||
|
||||
|
||||
# ---- writes into data
|
||||
|
||||
def relocated(n):
|
||||
"""({offset: type} of the words main's explicit-addend relocations (DT_RELA, DT_JMPREL) fill, {offset} of its
|
||||
DT_RELR words): the 8-byte words the dynamic linker writes at start (runtime/loader.c relocate_table,
|
||||
relocate_relr)."""
|
||||
rela = {}
|
||||
for table, size in ((DT_RELA, DT_RELASZ), (DT_JMPREL, DT_PLTRELSZ)):
|
||||
if table in n.dyn:
|
||||
for i in range(n.dyn[size] // 24):
|
||||
off, info, _ = struct.unpack_from('<QQq', n.image, n.dyn[table] + 24 * i)
|
||||
rela[off] = info & 0xFFFFFFFF
|
||||
relr, where = set(), 0
|
||||
for i in range(n.dyn.get(DT_RELRSZ, 0) // 8):
|
||||
e = struct.unpack_from('<Q', n.image, n.dyn[DT_RELR] + 8 * i)[0]
|
||||
if not e & 1: # an address; the bitmap after it covers the 63 words that follow
|
||||
relr.add(e)
|
||||
where = e + 8
|
||||
else:
|
||||
relr.update(where + 8 * k for k in range(63) if e >> (k + 1) & 1)
|
||||
where += 63 * 8
|
||||
return rela, relr
|
||||
|
||||
|
||||
def linker_tables(n):
|
||||
"""[(start, end)] of main's .dynamic, relocation, symbol and string tables: the dynamic linker reads them at start,
|
||||
so a write into them would change what it does, which no store into the loaded module can follow."""
|
||||
end = n.dynamic
|
||||
while struct.unpack_from('<Q', n.image, end)[0] != DT_NULL:
|
||||
end += 16
|
||||
out = [(n.dynamic, end + 16)]
|
||||
for table, size in ((DT_RELA, DT_RELASZ), (DT_JMPREL, DT_PLTRELSZ), (DT_RELR, DT_RELRSZ), (DT_STRTAB, DT_STRSZ)):
|
||||
if table in n.dyn and size in n.dyn:
|
||||
out.append((n.dyn[table], n.dyn[table] + n.dyn[size]))
|
||||
if DT_SYMTAB in n.dyn:
|
||||
out.append((n.dyn[DT_SYMTAB], n.dyn[DT_SYMTAB] + 24 * n.symbol_count()))
|
||||
return out
|
||||
|
||||
|
||||
class MainData:
|
||||
"""What a write into main's data meets: main's image from .rodata to the end of .bss, the words the dynamic linker
|
||||
fills and the tables it reads."""
|
||||
|
||||
def __init__(self, main):
|
||||
n = main.nso
|
||||
self.image, self.lo, self.size, self.data = main.image, main.code_end, n.size, n.data[0]
|
||||
self.rela, self.relr = relocated(n)
|
||||
self.linker = linker_tables(n)
|
||||
|
||||
|
||||
def data_runs(md, writes):
|
||||
"""The data part of one file ({offset: byte} outside main's code), applied as emulators and Atmosphere apply a patch
|
||||
file: to the module before its dynamic linker runs. Returns ([(offset, bytes, rebase)], dropped):
|
||||
|
||||
- a byte no relocation covers is stored as it is (runs of consecutive bytes, at most RUN_MAX long);
|
||||
- a DT_RELR word holds its addend: the word with the file's bytes in it, rebase set (main's base gets added);
|
||||
- the linker overwrites a word an explicit-addend relocation fills whatever the file wrote there: such bytes
|
||||
change nothing and only count in dropped.
|
||||
|
||||
Bytes equal to main's are left out. ValueError for a write outside main's image (below .rodata: the NSO header),
|
||||
into the dynamic linker's tables, or into a function pointer it binds lazily (JUMP_SLOT): on the Switch it holds
|
||||
the file's value plus main's base until the first call binds it, which no store before the start can reproduce."""
|
||||
plain, words, dropped = {}, {}, 0
|
||||
for off in sorted(writes):
|
||||
if not md.lo <= off < md.size:
|
||||
raise ValueError(f'it writes outside main\'s image at main+{off:X}')
|
||||
if any(a <= off < b for a, b in md.linker):
|
||||
raise ValueError(f'it writes the dynamic linker\'s tables at main+{off:X}')
|
||||
w = next((w for w in range(off - 7, off + 1) if w in md.rela or w in md.relr), None)
|
||||
if w is None:
|
||||
plain[off] = writes[off]
|
||||
elif w in md.relr:
|
||||
words.setdefault(w, bytearray(md.image[w:w + 8]))[off - w] = writes[off]
|
||||
elif md.rela[w] == R_AARCH64_JUMP_SLOT:
|
||||
raise ValueError(f'it writes a function pointer the dynamic linker binds lazily at main+{w:X}')
|
||||
else:
|
||||
dropped += 1
|
||||
runs = []
|
||||
for off in sorted(o for o, b in plain.items() if b != md.image[o]):
|
||||
last = runs[-1] if runs else None
|
||||
if last and last[0] + len(last[1]) == off and len(last[1]) < RUN_MAX:
|
||||
runs[-1] = (last[0], last[1] + bytes([plain[off]]), False)
|
||||
else:
|
||||
runs.append((off, bytes([plain[off]]), False))
|
||||
runs += [(w, bytes(b), True) for w, b in words.items() if bytes(b) != md.image[w:w + 8]]
|
||||
return sorted(runs), dropped
|
||||
|
||||
|
||||
def data_line(key, content, run):
|
||||
off, data, rebase = run
|
||||
return f'{key}\t{content}\t{off:x}\t{data.hex()}' + ('\tbase' if rebase else '')
|
||||
|
||||
|
||||
def builtin_sites(text):
|
||||
"""{offset in main: [set name]} of the patch file's sets that are no cheats or mod patches."""
|
||||
out = {}
|
||||
@@ -169,41 +288,45 @@ def builtin_sites(text):
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, required=True)
|
||||
ap.add_argument('--mods', type=Path, required=True, help='the mods folder next to totk.exe')
|
||||
ap.add_argument('--patches', type=Path, required=True, help='the patch file so far')
|
||||
ap.add_argument('--out', type=Path, required=True, help='the patch file with the mods\' ExeFS patches added')
|
||||
ap.add_argument('--status', type=Path, required=True)
|
||||
args = ap.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace')
|
||||
def classify(m, md, bid, path, raw):
|
||||
"""(kind, note, sites, runs, dropped) of one patch file: path gives its name and suffix, raw its bytes, bid main's
|
||||
build id; md None: no data file, so a write into main's data makes the file unsupported."""
|
||||
try:
|
||||
if path.suffix.lower() == '.ips':
|
||||
for_main = same_build(path.stem, bid)
|
||||
writes = parse_ips(raw) if for_main else {}
|
||||
else:
|
||||
pid, writes = parse_pchtxt(raw.decode('utf-8-sig', errors='replace'))
|
||||
for_main = pid is not None and same_build(pid, bid)
|
||||
if not for_main:
|
||||
return 'other', '', {}, [], 0
|
||||
code = {o: b for o, b in writes.items() if m.in_code(o - o % 4)}
|
||||
data = {o: b for o, b in writes.items() if o not in code}
|
||||
if data and md is None:
|
||||
raise ValueError(f'it writes main\'s data at main+{min(data):X} (only code patches are supported)')
|
||||
sites = convert(m, word_writes(m, code)) if code else {}
|
||||
runs, dropped = data_runs(md, data) if data else ([], 0)
|
||||
return 'code' if sites else 'data' if runs else 'empty', '', sites, runs, dropped
|
||||
except ValueError as e:
|
||||
return 'unsupported', str(e), {}, [], 0
|
||||
|
||||
|
||||
DATA_HEADER = '# file hash, content hash, offset in main, bytes[, base] (tools/exefs_patches.py)'
|
||||
|
||||
|
||||
def build(args):
|
||||
m = Main(args.exefs / 'main')
|
||||
raw = (args.exefs / 'main').read_bytes()
|
||||
bid = raw[0x40:0x60].hex()
|
||||
md = MainData(m) if args.data else None
|
||||
bid = m.nso.build_id
|
||||
text = args.patches.read_text()
|
||||
builtin = builtin_sites(text)
|
||||
files = patch_files(args.mods)
|
||||
sets, sites, status, kinds = [], [], [], {}
|
||||
sets, sites, status, data_lines, kinds = [], [], [], [], {}
|
||||
for mod, rel, f in files:
|
||||
data = f.read_bytes()
|
||||
key, content = fnv(f'{mod}/{rel}'.encode()), fnv(data)
|
||||
kind, note, over, file_sites = 'code', '', [], {}
|
||||
try:
|
||||
if f.suffix.lower() == '.ips':
|
||||
for_main = same_build(f.stem, bid)
|
||||
writes = parse_ips(data) if for_main else {}
|
||||
else:
|
||||
pid, writes = parse_pchtxt(data.decode('utf-8-sig', errors='replace'))
|
||||
for_main = pid is not None and same_build(pid, bid)
|
||||
if not for_main:
|
||||
kind = 'other'
|
||||
else:
|
||||
file_sites = convert(m, word_writes(m, writes))
|
||||
if not file_sites:
|
||||
kind = 'empty'
|
||||
except ValueError as e:
|
||||
kind, note, file_sites = 'unsupported', str(e), {}
|
||||
raw = f.read_bytes()
|
||||
key, content = fnv(f'{mod}/{rel}'.encode()), fnv(raw)
|
||||
kind, note, file_sites, runs, dropped = classify(m, md, bid, f, raw)
|
||||
over = []
|
||||
if file_sites:
|
||||
name = f'mod_{key.lower()}'
|
||||
label = f'{mod}/{rel}'.encode('ascii', 'replace').decode()
|
||||
@@ -211,19 +334,108 @@ def main():
|
||||
for a, repl in sorted(file_sites.items()):
|
||||
sites.append(f'site {name} main {a:x} {m.word(a):08x} {" ".join(f"{w:08x}" for w in repl)}')
|
||||
over += [s for s in builtin.get(a, []) if s not in over]
|
||||
data_lines += [data_line(key, content, r) for r in runs]
|
||||
data_bytes = sum(len(r[1]) for r in runs)
|
||||
kinds[kind] = kinds.get(kind, 0) + 1
|
||||
status.append(f'{key}\t{content}\t{kind}\t{",".join(over) or "-"}\t{note.replace(chr(9), " ")}')
|
||||
print(f'exefs: {kind:11} {mod}/{rel}' + (f' ({note})' if note else '')
|
||||
+ (f' ({len(file_sites)} sites, over {", ".join(over)})' if over else
|
||||
f' ({len(file_sites)} sites)' if file_sites else ''))
|
||||
status_note = f'data {data_bytes}' if runs else note.replace(chr(9), ' ')
|
||||
status.append(f'{key}\t{content}\t{kind}\t{",".join(over) or "-"}\t{status_note}')
|
||||
parts = [note] if note else []
|
||||
if file_sites:
|
||||
parts.append(f'{len(file_sites)} sites' + (f', over {", ".join(over)}' if over else ''))
|
||||
if runs:
|
||||
parts.append(f'{data_bytes} bytes of main\'s data')
|
||||
if dropped:
|
||||
parts.append(f'{dropped} bytes into words the dynamic linker fills, which change nothing')
|
||||
print(f'exefs: {kind:11} {mod}/{rel}' + (f' ({"; ".join(parts)})' if parts else ''))
|
||||
if sets:
|
||||
text += ('\n# ExeFS patches of the mods folder (tools/exefs_patches.py)\n' + '\n'.join(sets) + '\n'
|
||||
+ '\n'.join(sites) + '\n')
|
||||
args.out.write_text(text)
|
||||
args.status.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8')
|
||||
if args.data:
|
||||
args.data.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.data.write_text('\n'.join([DATA_HEADER] + data_lines) + '\n', encoding='utf-8')
|
||||
print(f'exefs: {len(files)} patch files: ' + ', '.join(f'{n} {k}' for k, n in sorted(kinds.items()))
|
||||
+ f'; {len(sites)} sites')
|
||||
+ f'; {len(sites)} sites, {len(data_lines)} data runs')
|
||||
|
||||
|
||||
def read_ranges(path):
|
||||
"""[(start, end)] of a folded.txt (recomp/gen.py): hex module offsets, one range per line, '#' comments."""
|
||||
out = []
|
||||
for line in path.read_text().splitlines():
|
||||
f = line.split('#', 1)[0].split()
|
||||
if len(f) == 2:
|
||||
out.append((int(f[0], 16), int(f[1], 16)))
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def data_offset(main_path):
|
||||
"""The offset of main's .data segment in its image, from the NSO header (no decompression)."""
|
||||
with open(main_path, 'rb') as f:
|
||||
head = f.read(0x40)
|
||||
return struct.unpack_from('<III', head, 0x30)[1]
|
||||
|
||||
|
||||
def recheck(args):
|
||||
"""After the lift: every file with a data write into bytes the lifted code read at lift time becomes unsupported."""
|
||||
if not args.status.is_file() or not args.data.is_file():
|
||||
print('exefs: no status or data file: nothing to check')
|
||||
return
|
||||
folded = read_ranges(args.folded) if args.folded.is_file() else None
|
||||
data_off = data_offset(args.exefs / 'main')
|
||||
lines = args.data.read_text(encoding='utf-8').splitlines()
|
||||
bad = {}
|
||||
for line in lines:
|
||||
f = line.split('\t')
|
||||
if line.startswith('#') or len(f) < 4 or f[0] in bad:
|
||||
continue
|
||||
a = int(f[2], 16)
|
||||
b = a + len(f[3]) // 2
|
||||
if folded is None and a < data_off:
|
||||
bad[f[0]] = (f'it writes main+{a:X} in .rodata, and the lift left no list of the bytes it reads as '
|
||||
f'constants')
|
||||
elif folded is not None:
|
||||
hit = next((s for s, e in folded if s < b and a < e), None)
|
||||
if hit is not None:
|
||||
bad[f[0]] = (f'it writes main+{max(a, hit):X}, which the lifted code reads when the game is built '
|
||||
f'(a jump table): the change would not reach it')
|
||||
if not bad:
|
||||
print(f'exefs: data writes checked against {"the bytes the lift read" if folded is not None else ".rodata"}: '
|
||||
f'none meets them')
|
||||
return
|
||||
status = []
|
||||
for line in args.status.read_text(encoding='utf-8').splitlines():
|
||||
f = line.split('\t')
|
||||
if len(f) >= 5 and f[0] in bad:
|
||||
line = '\t'.join([f[0], f[1], 'unsupported', '-', bad[f[0]]])
|
||||
print(f'exefs: unsupported file {f[0]} ({bad[f[0]]})')
|
||||
status.append(line)
|
||||
args.status.write_text('\n'.join(status) + '\n', encoding='utf-8')
|
||||
kept = [line for line in lines if line.split('\t')[0] not in bad]
|
||||
args.data.write_text('\n'.join(kept) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, required=True)
|
||||
ap.add_argument('--mods', type=Path, help='the mods folder next to totk.exe')
|
||||
ap.add_argument('--patches', type=Path, help='the patch file so far')
|
||||
ap.add_argument('--out', type=Path, help='the patch file with the mods\' ExeFS patches added')
|
||||
ap.add_argument('--status', type=Path, required=True)
|
||||
ap.add_argument('--data', type=Path, help='the data writes of the mods\' patch files (runtime/mods.c reads them)')
|
||||
ap.add_argument('--folded', type=Path, help='after the lift: <gen>/main/folded.txt; checks --status and --data '
|
||||
'against it instead of building')
|
||||
args = ap.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace')
|
||||
if args.folded:
|
||||
if not args.data:
|
||||
ap.error('--folded needs --data')
|
||||
recheck(args)
|
||||
else:
|
||||
if not (args.mods and args.patches and args.out):
|
||||
ap.error('--mods, --patches and --out are needed to build')
|
||||
build(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in new issue
Block a user