diff --git a/docs/building.md b/docs/building.md index 37c05e9f..5be975bc 100644 --- a/docs/building.md +++ b/docs/building.md @@ -231,7 +231,10 @@ Before a commit: A change to the button prompts (`tools/make_romfs_patches.py --prompts`, `tools/button_icons.py`) passes `.venv-py314\Scripts\python tests\romfs_prompts_test.py`, which runs the tool on a RomFS it makes itself (no game files); a change to the settings passes `totk.exe --probe-cfg` and `python tests\settings_doc_check.py`, which fails -when a setting is described neither in `totk.ini.example` nor in [settings.md](settings.md). +when a setting is described neither in `totk.ini.example` nor in [settings.md](settings.md). A change to the mods' ExeFS patches +(`tools/exefs_patches.py`, `runtime/mods.c`, the `folded.txt` of `recomp/gen.py`) passes +`.venv\Scripts\python tests\exefs_patches_test.py`, which makes its own modules and runs `gen.py` on one with a jump +table (no game files), and `totk.exe --probe-mods`. The lifter (`recomp/`) and the CPU headers (`runtime/cpu.h`, `runtime/simd.h`) are checked against Unicorn. Every lifter change passes these, all run with `.venv`: diff --git a/docs/installer.md b/docs/installer.md index 790faa88..46d8c5c9 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -60,7 +60,9 @@ loads the saves 1.4.3 wrote. 4. **Lift** the ExeFS to C (`recomp/gen.py`, `recomp/patches-v121.txt` or `patches-v143.txt`) into a temp folder next to the installer. The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets first (`tools/cheats.py`, `tools/exefs_patches.py`, status in `build-cache/cheats.status` and `exefs.status`); a file the - tools refuse costs those cheats or patches, not the install. + tools refuse costs those cheats or patches, not the install. The patch files' writes into main's data go to + `build-cache/exefs.data`, which the game stores into main at start; after the lift they are checked against the + bytes the lifted code read as constants (`/main/folded.txt`, its jump tables). 5. **Compile** the generated C, `modules.c`, `romfs.c`, `patches.c` and `ov_zstd.c` (they bake the per-install layout, RomFS hashes and the game version's patch page) and link them with the prebuilt runtime library for the host's level (`libtotkrt.a` or `libtotkrt-v2.a`) into `totk.exe` diff --git a/docs/mods.md b/docs/mods.md index 0734f382..c384095f 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -71,6 +71,10 @@ The rebuild needs the setup folder the game was installed from. When that folder A code patch over one of the port's own patches (the 60 FPS patch, the resolution patch and the other entries under **Built into the port**) turns that built-in patch off while the mod is on. The mod's row names it. +A patch file may also change the game's data (constants and tables in its `.rodata` and `.data`). Those bytes are +stored into the loaded game at start, before it runs. When two mods write the same bytes, the higher one wins, +as with files. + ## Cheats A cheat file is part of a mod: `mods\\cheats\.txt`, switched cheat by cheat on the Cheats tab. Cheats diff --git a/installer/steps.py b/installer/steps.py index d9b1303f..4db8a2b5 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -16,9 +16,10 @@ logs/: the next run lifts into the same gen/, where recomp/gen.py rewrites only in build/ recompiles only those. A change of the mods' code cheats then costs the lift (about a minute) and the chunks it touches, not a full compile. -Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status (tools/exefs_patches.py) and rebuild.txt, the command that starts this setup in -its rebuild mode on the install folder (installer/gui.py --rebuild), one argument per line; totk.exe runs it at start -when an enabled mod has code cheats the build lacks (runtime/cheats.c). +Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status and exefs.data (tools/exefs_patches.py) and +rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild), +one argument per line; totk.exe runs it at start when an enabled mod has code cheats the build lacks +(runtime/cheats.c). """ import hashlib import http.client @@ -855,6 +856,12 @@ def exefs_status(ctx): return ctx.dest / 'build-cache' / 'exefs.status' +def exefs_data(ctx): + """The data writes of the mods' ExeFS patch files (tools/exefs_patches.py --data), which runtime/mods.c stores into + main before the game runs.""" + return ctx.dest / 'build-cache' / 'exefs.data' + + def cheats_status(ctx): """The build's cheat status file (tools/cheats.py --status), which runtime/cheats.c reads: in build-cache, not in a mod's own cheats folder, which the player (or TKMM's merge) may empty.""" @@ -869,7 +876,7 @@ class Lift(Step): # fingerprint takes a file's name and content only; the paths under mods/ go in too, because tools/cheats.py # and tools/exefs_patches.py name each patch set after its mod folder, so a renamed folder needs a lift files = cheat_files(ctx) + exefs_files(ctx) - return fingerprint('lift-4', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)), + return fingerprint('lift-5', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)), ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *files, *(f.relative_to(ctx.dest / 'mods').as_posix() for f in files)) @@ -890,36 +897,54 @@ class Lift(Step): if m: rep.progress('lift', 'emit', int(m.group(1)), int(m.group(2)), 'chunks') - # The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets (tools/cheats.py, - # tools/exefs_patches.py); their status files tell the runtime what the build has. Both are the player's - # downloads, so files the tools refuse cost those cheats or patches, not the install: the game is built - # without them and the setup warns. + # The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets and data writes + # (tools/cheats.py, tools/exefs_patches.py); their status files tell the runtime what the build has. Both are + # the player's downloads, so files the tools refuse cost those cheats or patches, not the install: the game is + # built without them and the setup warns. plain = layout.patches(game_version(ctx)) - stages = [('cheats.py', 'patches-cheats.txt', cheats_status(ctx), cheat_files(ctx), 'warn.cheats_skipped'), - ('exefs_patches.py', 'patches-exefs.txt', exefs_status(ctx), exefs_files(ctx), 'warn.exefs_skipped')] + # (tool, patch file it writes, its outputs for the runtime: the status first, then the data file, the files it + # reads, the warning when it fails) + stages = [('cheats.py', 'patches-cheats.txt', [cheats_status(ctx)], cheat_files(ctx), 'warn.cheats_skipped'), + ('exefs_patches.py', 'patches-exefs.txt', [exefs_status(ctx), exefs_data(ctx)], exefs_files(ctx), + 'warn.exefs_skipped')] patches, skipped = plain, {} - for tool, out, status, inputs, warn in stages: - status.unlink(missing_ok=True) # no status: the runtime treats every such file as not built + + def drop(outputs): # no status: the runtime treats every such file as not built + for f in outputs: + f.unlink(missing_ok=True) + + for tool, out, outputs, inputs, warn in stages: + drop(outputs) if not inputs: continue code, log = run_tool(ctx, 'lift', layout.tools / tool, ['--exefs', ctx.dest / 'exefs', '--mods', ctx.dest / 'mods', '--patches', patches, - '--out', ctx.temp / out, '--status', status]) + '--out', ctx.temp / out, '--status', outputs[0], + *(['--data', outputs[1]] if len(outputs) > 1 else [])]) if code: - status.unlink(missing_ok=True) + drop(outputs) skipped[warn] = log else: patches = ctx.temp / out args = ['--exefs', ctx.dest / 'exefs', '--out', gen] code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches], parse) if code and patches != plain: - for tool, out, status, inputs, warn in stages: + for tool, out, outputs, inputs, warn in stages: if inputs and warn not in skipped: - status.unlink(missing_ok=True) + drop(outputs) skipped[warn] = log code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', plain], parse) if code: raise InstallError('lift.failed', code=code, log=str(log)) + # The mods' data writes must stay off the bytes the lifted code reads as constants (its jump tables), which only + # the lift knows: gen.py lists them in folded.txt, and a file that writes them becomes unsupported + if exefs_data(ctx).is_file(): + code, log = run_tool(ctx, 'lift', layout.tools / 'exefs_patches.py', + ['--exefs', ctx.dest / 'exefs', '--status', exefs_status(ctx), '--data', + exefs_data(ctx), '--folded', gen / 'main' / 'folded.txt']) + if code: + drop([exefs_status(ctx), exefs_data(ctx)]) + skipped.setdefault('warn.exefs_skipped', log) for warn, log in skipped.items(): rep.warning(warn, log=str(log)) (gen / STAMP).write_text(ctx.keys['lift'] + '\n') diff --git a/recomp/gen.py b/recomp/gen.py index bd582b68..d50c81f5 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -9,6 +9,9 @@ Output, under --out: /_NNNN.c one `void f_(Cpu *c)` per guest function, about --chunk-bytes of C per file /_table.c sorted _fn_addrs[] / _fn_ptrs[] / _fn_count for rt_dispatch /manifest.json chunk list, function count, statistics and timings + /folded.txt the module's bytes the generated code took as constants: what the jump tables read at + lift time, as module offsets [start, end); tools/exefs_patches.py keeps a mod's data + writes off them patches.h code patch sets for the runtime (recomp/patches.py; no sets without --patches) Function discovery runs in this process, vectorised over the module's .text words. Function starts are the module @@ -555,7 +558,8 @@ def _retry(fn, *args): # ------------------------------------------------------------------------------------------------ workers -_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter} +_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter}, + # 'reads' {name: [(start, end)]} module bytes read since the last chunk def _worker_init(mods, starts, table, opts, jobs): @@ -565,6 +569,7 @@ def _worker_init(mods, starts, table, opts, jobs): _W['patches'] = table _W['opts'] = opts # Emitter keyword arguments _W['emitters'] = {} + _W['reads'] = {} def _emitter(name): @@ -572,12 +577,31 @@ def _emitter(name): if e is None: path, base = _W['mods'][name] mod = Module(name, path, base, 'c') + # Module.read is how the jump tables (recomp/jumptables.py _eval) read .text and .rodata at lift time: the + # bytes it returns become part of the generated code, so /folded.txt lists them + read, reads = mod.read, _W['reads'].setdefault(name, []) + + def recorded(a, n, signed=False): + reads.append((a - base, a - base + n)) + return read(a, n, signed=signed) + mod.read = recorded e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()), _W['patches'].for_module(name, base), spins=_W['patches'].spins_for(name, base), **_W['opts']) return e +def merge_ranges(ranges): + """Sorted [start, end) ranges with overlapping and touching ones joined.""" + out = [] + for s, e in sorted(ranges): + if out and s <= out[-1][1]: + out[-1][1] = max(out[-1][1], e) + else: + out.append([s, e]) + return out + + def _emit_chunk(task): """Worker: write one chunk. Returns its statistics.""" name, number, path, fns = task @@ -595,9 +619,12 @@ def _emit_chunk(task): written = write_if_changed(Path(path), text) except OSError as ex: written, error = False, f'{path}: {ex}' + reads = _W['reads'].get(name, []) + folded = merge_ranges(reads) + reads.clear() return {'module': name, 'number': number, 'file': Path(path).name, 'functions': len(fns), 'bytes': len(text), 'written': written, 'error': error, 'seconds': time.perf_counter() - t0, 'stats': dict(e.stats), - 'unsupported': dict(e.unsupported)} + 'unsupported': dict(e.unsupported), 'folded': folded} # ------------------------------------------------------------------------------------------------ driver @@ -613,6 +640,14 @@ def table_text(name, addrs): return ''.join(lines) +def folded_text(name, ranges): + """/folded.txt: the module offsets [start, end) of the bytes the jump tables read at lift time, in hex.""" + lines = [f'# {GENERATED_MARKER}: {name} bytes the generated code took as constants (jump tables), module offsets ' + f'[start, end), generated by recomp/gen.py from game code. Do not edit or commit.'] + lines += [f'{s:x} {e:x}' for s, e in ranges] + return '\n'.join(lines) + '\n' + + def remove_stale(moddir, name, nchunks, errors): """Delete chunks an earlier run wrote beyond this run's count, and temporary files an interrupted run left.""" pat = re.compile(re.escape(name) + r'_(\d{4})\.c') @@ -714,6 +749,7 @@ def main(): # Largest chunks first, so the pool does not end on a long tail. tasks.sort(key=lambda t: -t[0]) results = {name: [] for name in info} + folded = {name: [] for name in info} done_at, errors = {}, [] t_emit = time.perf_counter() ctx = mp.get_context('spawn') @@ -724,9 +760,11 @@ def main(): print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True) try: # Chunk writes are atomic and idempotent, so a chunk lost with a failed pool is simply emitted again. + # Its folded bytes come with its result: a chunk emitted again reads them again in its new worker. with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts, jobs)) as pool: for r in pool.imap_unordered(_emit_chunk, pending, chunksize=1): results[r['module']].append(r) + folded[r['module']] += r['folded'] if r['error']: errors.append(r['error']) done_at[r['module']] = time.perf_counter() - t_emit @@ -753,6 +791,7 @@ def main(): unsupported.update(r['unsupported']) t = time.perf_counter() table_written = write_if_changed(moddir / f'{name}_table.c', table_text(name, d['emitted'])) + write_if_changed(moddir / 'folded.txt', folded_text(name, merge_ranges(folded[name]))) removed = remove_stale(moddir, name, d['chunks'], errors) manifest = { 'module': name, diff --git a/runtime/mods.c b/runtime/mods.c index d960a407..ba763632 100644 --- a/runtime/mods.c +++ b/runtime/mods.c @@ -15,7 +15,9 @@ // when its build id is main's (an IPS file's name, a pchtxt file's @nsobid line); others are another version's and // counted. The setup builds every mod's files into patch sets and writes build-cache/exefs.status next to totk.exe // (tools/exefs_patches.py has the format): a file whose line is missing or has another content hash waits for the -// rebuild at start (cheats.c), and its mod stays off until then. +// rebuild at start (cheats.c), and its mod stays off until then. A file's writes into main's data (.rodata, .data, +// .bss) are lines of build-cache/exefs.data, stored into the loaded main before the game runs (mods_exefs_data), as +// emulators apply a patch file to the module before its dynamic linker runs. #include #include #include @@ -170,22 +172,26 @@ static int same_build(const char *hex, size_t n) { return 1; } -// build-cache/exefs.status next to totk.exe, read once; "" when there is none. -static const char *exefs_status(void) { - static char *text; - if (text) return text; +// build-cache/ next to totk.exe, read once into *text; "" when there is none (mods_probe_cache sets the text). +static char *status_text, *data_text; +static const char *build_cache(const char *name, char **text) { + if (*text) return *text; char path[2100]; uint64_t n = 0; if (plat_exe_path(path, sizeof path)) { char *cut = strrchr(path, '/'), *back = strrchr(path, '\\'); if (back > cut) cut = back; - if (cut) snprintf(cut + 1, sizeof path - (size_t)(cut + 1 - path), "build-cache/exefs.status"); - text = (char *)plat_file_read(path, &n); + if (cut && snprintf(cut + 1, sizeof path - (size_t)(cut + 1 - path), "build-cache/%s", name) < + (int)(sizeof path - (size_t)(cut + 1 - path))) + *text = (char *)plat_file_read(path, &n); } - if (!text) text = calloc(1, 1); - return text; + if (!*text) *text = calloc(1, 1); + return *text; } +static const char *exefs_status(void) { return build_cache("exefs.status", &status_text); } +static const char *exefs_data(void) { return build_cache("exefs.data", &data_text); } + // The status line of key: its kind, built-in sets and note, NUL-terminated copies; 0 when none. static int status_line(const char *key, const char *content, char *kind, char *over, char *note) { char want[20]; @@ -194,8 +200,9 @@ static int status_line(const char *key, const char *content, char *kind, char *o const char *end = strchr(line, '\n'); size_t len = end ? (size_t)(end - line) : strlen(line); if (len > 18 && !memcmp(line, want, 18)) { - char copy[600]; - snprintf(copy, sizeof copy, "%.*s", (int)(len < sizeof copy ? len : sizeof copy - 1), line + 18); + char copy[600]; // the line after the two hashes, without the next line + size_t rest = len - 18; + snprintf(copy, sizeof copy, "%.*s", (int)(rest < sizeof copy ? rest : sizeof copy - 1), line + 18); char *t1 = strchr(copy, '\t'), *t2 = t1 ? strchr(t1 + 1, '\t') : NULL; if (!t1 || !t2) return 0; *t1 = *t2 = 0; @@ -269,9 +276,14 @@ static int exefs_file_cb(const char *leaf, int kind, uint64_t size, void *ctx) { m->sets[m->nsets++] = (uint32_t)strtoul(key, NULL, 16); size_t k = strlen(m->overrides); if (*over) snprintf(m->overrides + k, sizeof m->overrides - k, "%s%s", k ? "," : "", over); + } else if (!strcmp(kind_s, "data") && m->ndata < (int)(sizeof m->data / sizeof *m->data)) { + m->data[m->ndata++] = (uint32_t)strtoul(key, NULL, 16); } else if (!strcmp(kind_s, "other")) { m->other_version++; } + unsigned long bytes; // a code or data file's data writes: exefs.data must hold them all (mods_exefs_data) + if ((!strcmp(kind_s, "code") || !strcmp(kind_s, "data")) && sscanf(note, "data %lu", &bytes) == 1) + m->data_bytes += (uint32_t)bytes; return 1; } @@ -326,13 +338,13 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) { if (m->cheats[0]) plat_fs_list(m->cheats, cheats_cb, &m->kind); int unsupported = exefs_check(m, path); // a mod whose code part the port cannot use is off as a whole: its RomFS files may expect that code - m->problem = strchr(m->name, '|') ? MOD_BAD_NAME - : m->kind & MOD_SUBSDK ? MOD_PLUGIN - : unsupported ? MOD_EXEFS_UNSUPPORTED - : m->pending ? MOD_EXEFS_LATER - : m->nsets || m->kind & (MOD_ROMFS | MOD_CHEATS) ? MOD_OK - : m->kind & MOD_EXEFS ? MOD_EXEFS_OTHER - : MOD_EMPTY; + m->problem = strchr(m->name, '|') ? MOD_BAD_NAME + : m->kind & MOD_SUBSDK ? MOD_PLUGIN + : unsupported ? MOD_EXEFS_UNSUPPORTED + : m->pending ? MOD_EXEFS_LATER + : m->nsets || m->ndata || m->kind & (MOD_ROMFS | MOD_CHEATS) ? MOD_OK + : m->kind & MOD_EXEFS ? MOD_EXEFS_OTHER + : MOD_EMPTY; m->enabled = !listed(disabled, m->name); nmods++; return 1; @@ -376,10 +388,11 @@ void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), plat_fs_list(folder, peek_cb, &pk); } +static int scanned; + void mods_scan(void) { - static int done; - if (done) return; - done = 1; + if (scanned) return; + scanned = 1; if (!mods_folder(dir, sizeof dir)) return; if (!plat_dir_exists(dir)) { rt_log("mods: no folder %s", dir); @@ -403,18 +416,171 @@ void mods_scan(void) { rt_log("mods: %d folders in %s", nmods, dir); } -void mods_exefs_switch(void) { - mods_scan(); - for (int i = 0; i < nmods; i++) { - const ModInfo *m = &mods[i]; - if (!start_enabled[i] || m->problem != MOD_OK || !m->nsets) continue; - for (int k = 0; k < m->nsets; k++) { - char name[32]; - snprintf(name, sizeof name, "mod_%08x", m->sets[k]); - int set = rt_patch_set(name); - if (set >= 0) rt_patch_switch(set, 1); - else rt_log("mods: %s: patch set %s is not in this build", m->name, name); +// ---- ExeFS data writes (build-cache/exefs.data; tools/exefs_patches.py data_line has the format) + +// 1 when key names one of m's usable ExeFS patch files: one with a patch set, or one with data writes only. +static int owns(const ModInfo *m, uint32_t key) { + for (int k = 0; k < m->nsets; k++) + if (m->sets[k] == key) return 1; + for (int k = 0; k < m->ndata; k++) + if (m->data[k] == key) return 1; + return 0; +} + +static int hex_digit(char c) { + return c >= '0' && c <= '9' ? c - '0' + : c >= 'a' && c <= 'f' ? c - 'a' + 10 + : c >= 'A' && c <= 'F' ? c - 'A' + 10 + : -1; +} + +// One line: "\t\t\t[\tbase]", all hex. 0 when malformed. +typedef struct { char key[9], content[9]; uint64_t off; uint8_t bytes[256]; int n, base; } DataRun; +static int parse_run(const char *line, size_t len, DataRun *r) { + char copy[1100], *f[6]; + if (len >= sizeof copy) return 0; + memcpy(copy, line, len); + copy[len] = 0; + int nf = 0; + for (char *p = copy; p && nf < 6; nf++) { + f[nf] = p; + p = strchr(p, '\t'); + if (p) *p++ = 0; + } + if (nf < 4 || nf > 5 || strlen(f[0]) != 8 || strlen(f[1]) != 8) return 0; + memcpy(r->key, f[0], 9); + memcpy(r->content, f[1], 9); + char *end; + r->off = strtoull(f[2], &end, 16); + size_t hex = strlen(f[3]); + if (!*f[2] || *end || !hex || hex % 2 || hex / 2 > sizeof r->bytes) return 0; + r->n = (int)(hex / 2); + for (int i = 0; i < r->n; i++) { + int hi = hex_digit(f[3][2 * i]), lo = hex_digit(f[3][2 * i + 1]); + if (hi < 0 || lo < 0) return 0; + r->bytes[i] = (uint8_t)(hi << 4 | lo); + } + r->base = nf == 5; + return nf == 4 || (!strcmp(f[4], "base") && r->n == 8); +} + +// Stores n bytes into the loaded main at a: a read-only page (.rodata, loader.c seal) is writable for its part of the +// copy only, as cheats.c wr_main does. 0 when a page is neither readable nor writable. +static int store_main(uint64_t a, const uint8_t *from, int n) { + while (n > 0) { + uint64_t page = a & ~0xFFFull, end; + int k = (int)(page + 0x1000 - a); + if (k > n) k = n; + int prot = plat_mem_state(page, page + 0x1000, &end); + if (prot != PLAT_PROT_READ && prot != PLAT_PROT_RW) return 0; + if (prot == PLAT_PROT_READ && !plat_mem_protect(page, 0x1000, PLAT_PROT_RW)) return 0; + memcpy((void *)(uintptr_t)a, from, (size_t)k); + if (prot == PLAT_PROT_READ) plat_mem_protect(page, 0x1000, PLAT_PROT_READ); + a += (uint64_t)k; + from += k; + n -= k; + } + return 1; +} + +long mods_exefs_data(const ModInfo *m, uint64_t base, uint64_t size, uint64_t code_end, + int (*store)(uint64_t addr, const uint8_t *bytes, int n)) { + if (!store) store = store_main; + long total = 0, found = 0; + for (int pass = 0; pass < 2; pass++) { // pass 0 checks every line, pass 1 stores + // exefs.data must hold every byte the status lines count: one missing or cut short would leave the code + // patches on without their data + if (pass && found != (long)m->data_bytes) { + rt_log("mods: %s: build-cache/exefs.data holds %ld of the %u bytes its ExeFS patch files write", m->name, + found, m->data_bytes); + return -1; } + for (const char *line = exefs_data(), *next; *line; line = next) { + const char *nl = strchr(line, '\n'); + size_t len = nl ? (size_t)(nl - line) : strlen(line); + next = nl ? nl + 1 : line + len; + if (len && line[len - 1] == '\r') len--; + char key[9]; + snprintf(key, sizeof key, "%.*s", (int)(len < 8 ? len : 8), line); + if (!len || *line == '#' || !owns(m, (uint32_t)strtoul(key, NULL, 16))) continue; + DataRun r; + char kind[16], over[256], note[256]; + // a line whose file the status does not list as built (another content) or that leaves main's data: the + // two build-cache files and this build disagree + if (!parse_run(line, len, &r) || !status_line(r.key, r.content, kind, over, note) || + (strcmp(kind, "code") && strcmp(kind, "data")) || r.off < code_end || r.off > size || + (uint64_t)r.n > size - r.off) { + rt_log("mods: %s: a line of build-cache/exefs.data does not fit this build: %.*s", m->name, (int)len, + line); + return -1; + } + if (!pass) { + found += r.n; + continue; + } + if (r.base) { // a DT_RELR word holds its addend; the dynamic linker adds the module's base + uint64_t w; + memcpy(&w, r.bytes, 8); + w += base; + memcpy(r.bytes, &w, 8); + } + if (!store(base + r.off, r.bytes, r.n)) { + rt_log("mods: %s: cannot write main+%#llx", m->name, (unsigned long long)r.off); + return -1; + } + total += r.n; + } + } + return total; +} + +static char *dup_text(const char *s) { + if (!s) return NULL; + size_t n = strlen(s) + 1; + char *d = malloc(n); + if (d) memcpy(d, s, n); + return d; +} + +void mods_probe_cache(const char *status, const char *data) { + free(status_text); + free(data_text); + status_text = dup_text(status); + data_text = dup_text(data); +} + +void mods_rescan(void) { + free(mods); + mods = NULL; + nmods = 0; + scanned = 0; +} + +// Switches the patch set mod_ on; 0 when this build lacks it (logged). +static int set_on(const ModInfo *m, uint32_t key) { + char name[32]; + snprintf(name, sizeof name, "mod_%08x", key); + int set = rt_patch_set(name); + if (set >= 0) rt_patch_switch(set, 1); + else rt_log("mods: %s: patch set %s is not in this build", m->name, name); + return set >= 0; +} + +void mods_exefs_apply(uint64_t base, uint64_t size, uint64_t code_end, + int (*store)(uint64_t addr, const uint8_t *bytes, int n)) { + // bottom first: the top mod's data writes land last and win, as its RomFS files do. Patch sets are only switches + // (rt_patch_switch), so their order does not matter. + for (int i = nmods - 1; i >= 0; i--) { + const ModInfo *m = &mods[i]; + if (!start_enabled[i] || m->problem != MOD_OK || !(m->nsets || m->ndata)) continue; + // the data first: a mod whose data writes cannot all be stored keeps its code patches off too, as half of a + // patch file is worse than none + long stored = size ? mods_exefs_data(m, base, size, code_end, store) : 0; + if (stored < 0) { + rt_log("mods: %s: its ExeFS patch files stay off: their data writes could not all be stored", m->name); + continue; + } + for (int k = 0; k < m->nsets; k++) set_on(m, m->sets[k]); char over[256]; snprintf(over, sizeof over, "%s", m->overrides); for (char *s = strtok(over, ","); s; s = strtok(NULL, ",")) { @@ -422,10 +588,20 @@ void mods_exefs_switch(void) { if (set >= 0) rt_patch_switch(set, 0); rt_log("mods: %s replaces the port's patch set %s; it stays off", m->name, s); } - rt_log("mods: %s: %d ExeFS patch files on", m->name, m->nsets); + rt_log("mods: %s: %d ExeFS patch files on, %ld bytes of main's data stored", m->name, m->nsets + m->ndata, + stored); } } +void mods_exefs_switch(void) { + mods_scan(); + const RtLoaded *main_mod = NULL; + for (int i = 0; ldr_module(i); i++) + if (!strcmp(ldr_module(i)->name, "main")) main_mod = ldr_module(i); + if (main_mod) mods_exefs_apply(main_mod->base, main_mod->size, main_mod->text_end, NULL); + else mods_exefs_apply(0, 0, 0, NULL); +} + int mods_exefs_pending(uint32_t *key) { int n = 0; for (int i = 0; i < nmods; i++) { diff --git a/runtime/mods.h b/runtime/mods.h index 318a9632..345fcc2b 100644 --- a/runtime/mods.h +++ b/runtime/mods.h @@ -2,7 +2,8 @@ // them: /romfs, /exefs and /cheats, the Atmosphere layout (atmosphere/contents//romfs, // atmosphere/exefs_patches/<name>/*.ips) or TKMM's merged output (romfs or romfslite, exefs, cheats). ExeFS patch // files (.ips, .pchtxt) for the game's main are patch sets the setup builds into the game (tools/exefs_patches.py), -// switched with their mod (mods_exefs_switch). The list is scanned once at start (romfs.c overlay_init); +// switched with their mod (mods_exefs_switch), and data writes stored into main before the game runs (mods_exefs_data). +// The list is scanned once at start (romfs.c overlay_init); // its order and switches live in totk.ini [mods] (TOTK_MODS_ORDER: names top first, separated by '|'; // TOTK_MODS_DISABLED: names switched off). The top mod wins a file two mods replace. #pragma once @@ -36,6 +37,10 @@ typedef struct { uint32_t pending, pending_key; // ExeFS patch files the build lacks; FNV-1a of their names and contents int nsets; uint32_t sets[16]; // the built patch sets mod_<hash> of its ExeFS patch files + int ndata; + uint32_t data[16]; // the hashes of its ExeFS patch files with data writes only (no set) + uint32_t data_bytes; // the bytes its usable ExeFS patch files write into main's data, as their + // status lines count them ("data <bytes>") char overrides[256]; // the port's own patch sets its patches replace, comma separated } ModInfo; @@ -51,8 +56,25 @@ ModInfo *mods_at(int i); // 0 = top of the order; NULL out of range void mods_set_enabled(int i, int on); void mods_move(int i, int delta); // -1 up, +1 down; nothing at the ends int mods_changed(void); // 1 when the order or a switch differs from the start -// After rt_patches_init: the patch sets of the enabled mods' ExeFS patches on, the port's own sets they replace off. +// After ldr_load and rt_patches_init, before any game code: the patch sets of the enabled mods' ExeFS patches on, the +// port's own sets they replace off, and their data writes stored into main (mods_exefs_data). void mods_exefs_switch(void); +// What mods_exefs_switch does with main's image at base (size bytes, code below code_end; size 0: no data stored), for +// every enabled mod from the bottom one up, so that the top mod's data writes win where two mods write the same bytes. +// store as for mods_exefs_data. +void mods_exefs_apply(uint64_t base, uint64_t size, uint64_t code_end, + int (*store)(uint64_t addr, const uint8_t *bytes, int n)); +// Stores the data writes of mod m's ExeFS patch files (build-cache/exefs.data) into main's image at base: size bytes, +// code below code_end. A DT_RELR word gets base added, as the dynamic linker adds it. Every line is checked before the +// first store: one that falls outside [code_end, size) or does not belong to a usable file of m keeps them all out. +// store writes n bytes at a guest address, 1 on success; NULL: into the loaded main, whose .rodata is read-only. +// Returns the bytes stored, -1 when a line kept them out or a store failed. --probe-mods passes a buffer and memcpy. +long mods_exefs_data(const ModInfo *m, uint64_t base, uint64_t size, uint64_t code_end, + int (*store)(uint64_t addr, const uint8_t *bytes, int n)); +// --probe-mods: the texts of build-cache/exefs.status and exefs.data instead of the files (NULL: read the file), and a +// fresh scan at the next mods_scan. +void mods_probe_cache(const char *status, const char *data); +void mods_rescan(void); // The ExeFS patch files of the enabled mods that the build lacks, their FNV-1a mixed into *key (cheats.c starts the // rebuild for them). int mods_exefs_pending(uint32_t *key); diff --git a/runtime/mods_probe.c b/runtime/mods_probe.c index e576ee84..a348fc69 100644 --- a/runtime/mods_probe.c +++ b/runtime/mods_probe.c @@ -39,6 +39,93 @@ static int remove_cb(const char *name, int kind, uint64_t size, void *ctx) { return 1; } +static uint32_t fnv(const char *p, size_t n) { + uint32_t h = 0x811C9DC5u; + for (size_t i = 0; i < n; i++) h = (h ^ (uint8_t)p[i]) * 0x01000193u; + return h; +} + +static ModInfo *by_name(const char *name) { + for (int i = 0; mods_at(i); i++) + if (!strcmp(mods_at(i)->name, name)) return mods_at(i); + return NULL; +} + +// mods_exefs_data's store into a buffer standing in for main +static int store_buffer(uint64_t a, const uint8_t *p, int n) { + memcpy((void *)(uintptr_t)a, p, (size_t)n); + return 1; +} + +// A second scan of a folder of its own: data writes of ExeFS patch files (build-cache/exefs.data, given as text). +static void probe_data(const char *root) { + remove_tree(root); + plat_file_mkdir_p(root); + const char *bid = *mods_main_build_id() ? mods_main_build_id() : "0"; + char n_ips[160], p_ips[160], q_ips[160], x_ips[160], y_ips[160]; + snprintf(n_ips, sizeof n_ips, "N/exefs/%s.ips", bid); + snprintf(p_ips, sizeof p_ips, "P/exefs/%s.ips", bid); + snprintf(q_ips, sizeof q_ips, "Q/exefs/%s.ips", bid); + snprintf(x_ips, sizeof x_ips, "X/exefs/%s.ips", bid); + snprintf(y_ips, sizeof y_ips, "Y/exefs/%s.ips", bid); + const char *files[] = {n_ips, p_ips, q_ips, x_ips, y_ips}; + for (size_t i = 0; i < sizeof files / sizeof *files; i++) put(root, files[i]); + uint32_t kn = fnv(n_ips, strlen(n_ips)), kp = fnv(p_ips, strlen(p_ips)), kq = fnv(q_ips, strlen(q_ips)); + uint32_t kx = fnv(x_ips, strlen(x_ips)), ky = fnv(y_ips, strlen(y_ips)); + uint32_t c = fnv("probe", 5); + char status[512], data[768]; + snprintf(status, sizeof status, + "%08X\t%08X\tdata\t-\tdata 11\n%08X\t%08X\tdata\t-\tdata 1\n%08X\t%08X\tdata\t-\tdata 4\n" + "%08X\t%08X\tdata\t-\tdata 1\n%08X\t%08X\tdata\t-\tdata 1\n", + kn, c, kp, c, kq, c, kx, c, ky, c); + snprintf(data, sizeof data, + "# file hash, content hash, offset in main, bytes[, base]\n" + "%08X\t%08X\t10\t414243\n" // N: three bytes of data + "%08X\t%08X\t18\t0800000000000000\tbase\r\n" // N: a DT_RELR word, 8 plus main's base (a CRLF line) + "DEADBEEF\t%08X\t20\t99\n" // another file's: not N's + "%08X\t%08X\t4\t77\n" // P: below main's data + "%08X\t%08X\t30\t66\n" // Q: 1 of the 4 bytes its status counts + "%08X\t%08X\t28\tBB\n" // X: the byte Y writes too + "%08X\t%08X\t28\tAA\n", // Y, above X in the order: its byte stays + kn, c, kn, c, c, kp, c, kq, c, kx, c, ky, c); + mods_probe_cache(status, data); + cfg_set("TOTK_MODS_DIR", root); + cfg_set("TOTK_MODS_ORDER", "Y"); // Y on top, the rest by name: Y N P Q X + cfg_set("TOTK_MODS_DISABLED", ""); + mods_rescan(); + mods_scan(); + ModInfo *n = by_name("N"), *p = by_name("P"), *q = by_name("Q"); + CHECK(n && p && q && by_name("X") && by_name("Y"), "second scan: five mods"); + if (n && p && q) { + CHECK(n->kind == MOD_EXEFS && n->problem == MOD_OK && n->ndata == 1 && !n->nsets, + "an ExeFS patch file with data writes only makes a usable mod"); + uint8_t img[0x40]; + memset(img, 0, sizeof img); + uint64_t base = (uint64_t)(uintptr_t)img, word; + long stored = mods_exefs_data(n, base, sizeof img, 8, store_buffer); + memcpy(&word, img + 0x18, 8); + CHECK(stored == 11 && !memcmp(img + 0x10, "ABC", 3) && word == base + 8 && !img[0x20], + "data writes: bytes as they are, a DT_RELR word rebased, other files' lines left"); + memset(img, 0x5A, sizeof img); + int untouched = 1; + CHECK(mods_exefs_data(p, base, sizeof img, 8, store_buffer) == -1, "a line below main's data keeps all out"); + CHECK(n->data_bytes == 11 && q->data_bytes == 4, "the status lines count the data bytes"); + CHECK(mods_exefs_data(q, base, sizeof img, 8, store_buffer) == -1, + "a data file holding fewer bytes than the status counts keeps all out"); + for (size_t i = 0; i < sizeof img; i++) untouched &= img[i] == 0x5A; + CHECK(untouched, "nothing stored when a line does not fit or bytes are missing"); + // every enabled mod as at start: X and Y write the same byte, and Y, the higher one, wins as its RomFS files + // would; P and Q stay out as above + memset(img, 0, sizeof img); + CHECK(EQ(mods_at(0)->name, "Y") && EQ(mods_at(4)->name, "X"), "mods_order puts Y on top and X at the bottom"); + mods_exefs_apply(base, sizeof img, 8, store_buffer); + CHECK(img[0x28] == 0xAA, "two mods write the same byte: the top mod's byte stays"); + CHECK(!memcmp(img + 0x10, "ABC", 3) && !img[0x30], "the other mods' data as each one stores it alone"); + } + mods_probe_cache(NULL, NULL); + remove_tree(root); +} + // s ends in tail, either path separator matching either. static int ends_with(const char *s, const char *tail) { size_t n = strlen(s), t = strlen(tail); @@ -126,7 +213,7 @@ int probe_mods(void) { CHECK(EQ(cfg_get("TOTK_MODS_DISABLED"), "A"), "switching A off writes it"); CHECK(mods_changed(), "changes seen"); } - remove_tree(root); + probe_data(root); printf("mods: %s\n", fails ? "FAIL" : "ok"); int table = restbl_probe(); return fails || table ? 1 : 0; diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index bd500fa3..aa845156 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1109,8 +1109,8 @@ static void tab_mods(float scale) { char code[512] = "", problem[512] = ""; if (m->nsets && m->overrides[0]) snprintf(code, sizeof code, menu_text(UI_MENU_MODS_REPLACES_BUILTIN), m->nsets, m->overrides); - else if (m->nsets) - snprintf(code, sizeof code, menu_text(UI_MENU_MODS_EXEFS_ON), m->nsets); + else if (m->nsets || m->ndata) + snprintf(code, sizeof code, menu_text(UI_MENU_MODS_EXEFS_ON), m->nsets + m->ndata); switch (m->problem) { case MOD_EMPTY: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EMPTY)); break; case MOD_PLUGIN: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_SUBSDK)); break; diff --git a/tests/exefs_patches_test.py b/tests/exefs_patches_test.py new file mode 100644 index 00000000..3a6f51e7 --- /dev/null +++ b/tests/exefs_patches_test.py @@ -0,0 +1,258 @@ +"""tools/exefs_patches.py on modules it makes itself: code writes become sites, data writes become runs of the data file +as emulators apply a patch file before the dynamic linker (plain bytes, DT_RELR words rebased, writes into words an +explicit-addend relocation fills dropped), the writes it refuses (outside the image, into the linker's tables or a +lazily bound pointer), the status and data files, and the check after the lift against the bytes recomp/gen.py lists in +folded.txt, with gen.py run on a module with one jump table. No game files. + +Run with .venv (lz4 and numpy, as recomp/): python tests/exefs_patches_test.py +""" +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +import exefs_patches as ep # noqa: E402 + +BUILD_ID = bytes.fromhex('0123456789abcdef') + bytes(24) +BID_NAME = '0123456789ABCDEF' # an IPS file name may leave the build id's trailing zeros out +MOV_W8_100, MOV_W8_200, RET = 0x52802008, 0x52804008, 0xD65F03C0 +R_RELATIVE, R_GLOB_DAT, R_JUMP_SLOT = 1027, 1025, 1026 + + +def nso(text, ro, data, bss=0x1000): + """An uncompressed NSO with .text at 0, .rodata at 0x1000 and .data at 0x2000.""" + segs = [(0x0, text), (0x1000, ro), (0x2000, data)] + hdr = bytearray(0x100) + hdr[0:4] = b'NSO0' + off = 0x100 + for i, (mem, d) in enumerate(segs): + struct.pack_into('<IIII', hdr, 0x10 + 0x10 * i, off, mem, len(d), bss if i == 2 else 0) + struct.pack_into('<I', hdr, 0x60 + 4 * i, len(d)) + off += len(d) + hdr[0x40:0x60] = BUILD_ID + return bytes(hdr) + b''.join(d for _, d in segs) + + +def put(buf, off, data): + buf[off:off + len(data)] = data + + +def reloc_module(): + """fn at 0x8 (mov w8, #0x100; ret); .rodata: constants at 0x1000, MOD0 at 0x1100 and the relocation tables at + 0x1200 (RELA: a RELATIVE word at 0x2100, a GLOB_DAT word at 0x2108), 0x1300 (JMPREL: a JUMP_SLOT word at 0x2110) + and 0x1400 (RELR: the word at 0x2118, addend 0x30); .data: .dynamic at 0x2000, plain bytes at 0x2200.""" + text = struct.pack('<4I', 0x14000002, 0x1100, MOV_W8_100, RET) + ro = bytearray(0x500) + put(ro, 0x000, b'constant bytes!!') + put(ro, 0x100, b'MOD0' + struct.pack('<6i', 0x2000 - 0x1100, 0, 0, -0x1100, -0x1100, 0)) + put(ro, 0x200, struct.pack('<QQq', 0x2100, R_RELATIVE, 0x8) + struct.pack('<QQq', 0x2108, R_GLOB_DAT, 0)) + put(ro, 0x300, struct.pack('<QQq', 0x2110, R_JUMP_SLOT, 0)) + put(ro, 0x400, struct.pack('<Q', 0x2118)) + data = bytearray(0x300) + dyn = [(7, 0x1200), (8, 48), (23, 0x1300), (2, 24), (0x24, 0x1400), (0x23, 8), (0, 0)] + put(data, 0x000, b''.join(struct.pack('<QQ', t, v) for t, v in dyn)) + put(data, 0x118, struct.pack('<Q', 0x30)) + put(data, 0x200, b'ABCDEFGH') + return nso(text, bytes(ro), bytes(data)) + + +# A module with one jump table (clang's compressed form), assembled from: +# b fn ; 0x0 +# .word mod0 ; 0x4: the MOD0 offset +# fn: cmp w0, #3 ; 0x8 +# b.hi Ldefault +# adrp x8, table ; table = 0x1020: .byte 0, 2, 4, 6 +# add x8, x8, #0x20 +# adr x9, L0 +# ldrb w10, [x8, w0, uxtw] +# add x9, x9, x10, lsl #2 +# br x9 +# L0..L3: mov w0, #10/20/30/40; ret ; Ldefault: mov w0, #0; ret +# .rodata: 32 bytes of text before the table, 21 after it, MOD0 at 0x1040 and a RELATIVE relocation to fn (the +# lifter's function seed); .data: .dynamic (DT_RELA) and the relocated word. +JT_TEXT = bytes.fromhex( + '02000014401000001f0c0071e8010054080000b008810091890000100a49603829090a8b20011fd640018052c0035fd680028052' + 'c0035fd6c0038052c0035fd600058052c0035fd600008052c0035fd6') +JT_RO = bytes.fromhex( + '756e746f7563686564206279746573206265666f726520746865207461626c6500020406627974657320616674657220746865' + '207461626c65000000000000004d4f4430c00f00000000000000000000c0efffffc0efffff00000000000000003020000000000000' + '03040000000000000800000000000000') +JT_DATA = struct.pack('<7Q', 7, 0x1060, 8, 24, 0, 0, 0) # DT_RELA, DT_RELASZ, DT_NULL; the word the RELATIVE fills + + +def ips32(writes): + """An IPS32 file of {image offset: bytes} (offsets count the NSO header, as emulators read them).""" + body = b''.join((off + ep.NSO_HEADER).to_bytes(4, 'big') + len(d).to_bytes(2, 'big') + d + for off, d in sorted(writes.items())) + return b'IPS32' + body + b'EEOF' + + +class Classify(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + path = Path(cls.tmp.name) / 'main' + path.write_bytes(reloc_module()) + cls.m = ep.Main(path) + cls.md = ep.MainData(cls.m) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def run_file(self, writes, md=True): + return ep.classify(self.m, self.md if md else None, self.m.nso.build_id, Path(f'{BID_NAME}.ips'), + ips32(writes)) + + def test_relocated_words(self): + self.assertEqual(self.md.rela, {0x2100: R_RELATIVE, 0x2108: R_GLOB_DAT, 0x2110: R_JUMP_SLOT}) + self.assertEqual(self.md.relr, {0x2118}) + + def test_code_only(self): + kind, note, sites, runs, dropped = self.run_file({0x8: struct.pack('<I', MOV_W8_200)}) + self.assertEqual((kind, note, runs, dropped), ('code', '', [], 0)) + self.assertEqual(sites, {0x8: [MOV_W8_200]}) + + def test_data_only(self): + kind, note, sites, runs, dropped = self.run_file({0x1000: b'\xAA\xBB', 0x2200: b'XY'}) + self.assertEqual((kind, sites, dropped), ('data', {}, 0)) + self.assertEqual(runs, [(0x1000, b'\xAA\xBB', False), (0x2200, b'XY', False)]) + + def test_code_and_data(self): + kind, _, sites, runs, _ = self.run_file({0x8: struct.pack('<I', MOV_W8_200), 0x1004: b'\x01'}) + self.assertEqual(kind, 'code') + self.assertEqual(sites, {0x8: [MOV_W8_200]}) + self.assertEqual(runs, [(0x1004, b'\x01', False)]) + + def test_same_bytes_are_nothing(self): + kind, _, _, runs, _ = self.run_file({0x1000: b'constant'}) + self.assertEqual((kind, runs), ('empty', [])) + + def test_explicit_addend_words_change_nothing(self): + kind, _, _, runs, dropped = self.run_file({0x2100: b'\x11\x22\x33\x44', 0x2108: b'\x55'}) + self.assertEqual((kind, runs, dropped), ('empty', [], 5)) + + def test_relr_word_is_rebased(self): + kind, _, _, runs, _ = self.run_file({0x2118: b'\x31'}) + self.assertEqual(kind, 'data') + self.assertEqual(runs, [(0x2118, struct.pack('<Q', 0x31), True)]) + + def test_long_runs_are_split(self): + _, _, _, runs, _ = self.run_file({0x2200: bytes(i % 255 + 1 for i in range(ep.RUN_MAX)) + b'\xFF'}) + self.assertEqual([(o, len(d)) for o, d, _ in runs], [(0x2200, ep.RUN_MAX), (0x2200 + ep.RUN_MAX, 1)]) + + def test_refused_writes(self): + for writes, why in (({0x2110: b'\x01'}, 'binds lazily'), ({0x2000: b'\x01'}, 'linker\'s tables'), + ({0x1208: b'\x01'}, 'linker\'s tables'), ({0x4000: b'\x01'}, 'outside main\'s image'), + ({-0x80: b'\x01'}, 'outside main\'s image')): + kind, note, sites, runs, _ = self.run_file({0x8: struct.pack('<I', MOV_W8_200), **writes}) + self.assertEqual((kind, sites, runs), ('unsupported', {}, []), note) + self.assertIn(why, note) + + def test_data_without_a_data_file(self): + kind, note, _, _, _ = self.run_file({0x1000: b'\xAA'}, md=False) + self.assertEqual(kind, 'unsupported') + self.assertIn('only code patches are supported', note) + + def test_pchtxt(self): + text = (f'@nsobid-{BID_NAME}\n@flag offset_shift 0x100\n@enabled\n' + f'00001000 AABB\n00002200 "XY"\n@disabled\n00002204 CC\n') + kind, _, _, runs, _ = ep.classify(self.m, self.md, self.m.nso.build_id, Path('x.pchtxt'), text.encode()) + self.assertEqual(kind, 'data') + self.assertEqual(runs, [(0x1000, b'\xAA\xBB', False), (0x2200, b'XY', False)]) + + +class Tool(unittest.TestCase): + """The command line: build, then the check after the lift.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.d = Path(self.tmp.name) + (self.d / 'exefs').mkdir() + (self.d / 'patches.txt').write_text('# no sets of the port\n') + + def tearDown(self): + self.tmp.cleanup() + + def mod_file(self, mod, writes): + f = self.d / 'mods' / mod / 'exefs' / f'{BID_NAME}.ips' + f.parent.mkdir(parents=True, exist_ok=True) + f.write_bytes(ips32(writes)) + + def tool(self, *args): + r = subprocess.run([sys.executable, str(ROOT / 'tools' / 'exefs_patches.py'), '--exefs', str(self.d / 'exefs'), + *map(str, args)], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + return r.stdout + + def build(self): + self.tool('--mods', self.d / 'mods', '--patches', self.d / 'patches.txt', '--out', self.d / 'out.txt', + '--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data') + status = {f[0]: f for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())} + data = [line.split('\t') for line in (self.d / 'exefs.data').read_text().splitlines() + if not line.startswith('#')] + return status, data + + def key(self, mod): + return ep.fnv(f'{mod}/exefs/{BID_NAME}.ips'.encode()) + + def test_build_and_check(self): + (self.d / 'exefs' / 'main').write_bytes(reloc_module()) + self.mod_file('Code', {0x8: struct.pack('<I', MOV_W8_200), 0x2200: b'Q'}) + self.mod_file('Data', {0x1000: b'\xAA', 0x2118: b'\x31'}) + self.mod_file('Bad', {0x2110: b'\x01'}) + status, data = self.build() + code, dat, bad = self.key('Code'), self.key('Data'), self.key('Bad') + self.assertEqual([status[k][2] for k in (code, dat, bad)], ['code', 'data', 'unsupported']) + self.assertEqual([status[k][4] for k in (code, dat)], ['data 1', 'data 9']) # bytes its data lines hold + self.assertIn(f'set mod_{code.lower()} TOTK_MODPATCH_{code} 0', (self.d / 'out.txt').read_text()) + self.assertEqual(sorted((f[0], f[2], f[3], f[4:]) for f in data), + sorted([(code, '2200', '51', []), (dat, '1000', 'aa', []), + (dat, '2118', '3100000000000000', ['base'])])) + # after the lift: .rodata bytes the lifted code read make the Data file unsupported; the Code file's data is + # in .data, which the lifter never reads + folded = self.d / 'folded.txt' + folded.write_text('# test\n1000 1001\n') + self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', folded) + status2 = {f[0]: f for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())} + self.assertEqual(status2[code][2], 'code') + self.assertEqual(status2[dat][2:4], ['unsupported', '-']) + self.assertIn('jump table', status2[dat][4]) + left = [line.split('\t')[0] for line in (self.d / 'exefs.data').read_text().splitlines() + if not line.startswith('#')] + self.assertEqual(left, [code]) + + def test_check_without_the_lift_list(self): + (self.d / 'exefs' / 'main').write_bytes(reloc_module()) + self.mod_file('Ro', {0x1000: b'\xAA'}) + self.mod_file('Rw', {0x2200: b'Q'}) + self.build() + self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', + self.d / 'missing.txt') + status = {f[0]: f[2] for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())} + self.assertEqual((status[self.key('Ro')], status[self.key('Rw')]), ('unsupported', 'data')) + + def test_jump_table_through_the_lift(self): + """recomp/gen.py lists the jump table's bytes in folded.txt; a write into them is refused, one beside them + is kept.""" + (self.d / 'exefs' / 'main').write_bytes(nso(JT_TEXT, JT_RO, JT_DATA)) + r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'), + '--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1'], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + folded = self.d / 'gen' / 'main' / 'folded.txt' + self.assertEqual(ep.read_ranges(folded), [(0x1020, 0x1024)]) + self.mod_file('Table', {0x1022: b'\x00'}) + self.mod_file('Beside', {0x1024: b'B'}) + self.build() + self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', folded) + status = {f[0]: f[2] for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())} + self.assertEqual((status[self.key('Table')], status[self.key('Beside')]), ('unsupported', 'data')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/installer_test.py b/tests/installer_test.py index 0e1d67ff..b57ccf41 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -315,6 +315,75 @@ class LiftKeyTest(unittest.TestCase): self.assertEqual(self.key(), before) +class LiftRunTest(unittest.TestCase): + """Lift.run with fake tools: the ExeFS patch stage writes the status and data files, the lift lists the bytes it + read, and the data writes are checked against that list after it; a failed lift or check drops both files.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + base = Path(tmp.name) + self.dest, temp = base / 'install', base / 'temp' + self.dest.mkdir() + temp.mkdir() + layout = SimpleNamespace(recomp=ROOT / 'recomp', tools=ROOT / 'tools', + patches=lambda version: base / 'patches-v143.txt') + self.ctx = SimpleNamespace(dest=self.dest, temp=temp, layout=layout, keys={'lift': 'k'}, + state={'version': '1.4.3'}, reporter=Reporter(out=io.StringIO())) + self.addCleanup(self.ctx.reporter.close) + mod = self.dest / 'mods' / 'SQ' / 'exefs' + mod.mkdir(parents=True) + (mod / '277178B7DBA1B6D4.ips').write_bytes(b'IPS32EEOF') + self.calls, self.fail = [], set() + saved = steps.run_tool + self.addCleanup(lambda: setattr(steps, 'run_tool', saved)) + steps.run_tool = self.run_tool + + def run_tool(self, ctx, step, script, args, parse=None, cwd=None): + args = [str(a) for a in args] + name = Path(script).name + (' --folded' if '--folded' in args else '') + self.calls.append((name, args)) + log = ctx.temp / 'tool.log' + log.write_text('') + if name in self.fail: + return 1, log + if name == 'exefs_patches.py': + for flag in ('--status', '--data', '--out'): + p = Path(args[args.index(flag) + 1]) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(flag) + elif name == 'gen.py': + out = Path(args[args.index('--out') + 1]) / 'main' + out.mkdir(parents=True, exist_ok=True) + (out / 'folded.txt').write_text('# test\n') + return 0, log + + def outputs(self): + return [p.is_file() for p in (steps.exefs_status(self.ctx), steps.exefs_data(self.ctx))] + + def test_data_file_and_the_check_after_the_lift(self): + steps.Lift().run(self.ctx) + self.assertEqual([c[0] for c in self.calls], ['exefs_patches.py', 'gen.py', 'exefs_patches.py --folded']) + build = self.calls[0][1] + self.assertEqual(build[build.index('--data') + 1], str(steps.exefs_data(self.ctx))) + check = self.calls[2][1] + self.assertEqual(check[check.index('--folded') + 1], str(self.ctx.temp / 'gen' / 'main' / 'folded.txt')) + self.assertEqual(check[check.index('--status') + 1], str(steps.exefs_status(self.ctx))) + self.assertEqual(self.outputs(), [True, True]) + + def test_failed_lift_drops_the_data_file(self): + self.fail = {'gen.py'} + with self.assertRaises(steps.InstallError): + steps.Lift().run(self.ctx) + self.assertEqual(self.outputs(), [False, False]) + + def test_failed_check_drops_both_files(self): + self.fail = {'exefs_patches.py --folded'} + steps.Lift().run(self.ctx) + self.assertEqual(self.outputs(), [False, False]) + self.assertIn('warn.exefs_skipped', self.ctx.reporter.out.getvalue()) + + class IniTest(unittest.TestCase): EXAMPLE = '[display]\n# vk_present = mailbox\n\n[language]\n# locale = en-US\n# lang = en-US\n# x = 1\n' diff --git a/tools/cheats.py b/tools/cheats.py index 5406fe4d..30e0bb68 100644 --- a/tools/cheats.py +++ b/tools/cheats.py @@ -164,6 +164,8 @@ class Main: def __init__(self, path): n = Nso(str(path)) self.image = bytes(n.image) + n.image = self.image # one copy of the image; tools/exefs_patches.py reads nso's dynamic info + self.nso = n self.text = n.text # (offset, size) self.code_end = n.rodata[0] # .text and its padding up to .rodata, where code caves go self.lifter = Lifter(BASE, 'c') diff --git a/tools/exefs_patches.py b/tools/exefs_patches.py index 7f221a46..74cc12ee 100644 --- a/tools/exefs_patches.py +++ b/tools/exefs_patches.py @@ -1,11 +1,16 @@ -"""ExeFS patches of the mods folder to code patch sets: IPS, IPS32 and IPSwitch (.pchtxt) files for the game's main. +"""ExeFS patches of the mods folder to code patch sets and data writes: IPS, IPS32 and IPSwitch (.pchtxt) files for +the game's main. Lifted code cannot be byte-patched when the game runs, so a patch file's writes into main's code become `site` lines (recomp/patches.py), one patch set per file, which the runtime switches on while its mod is on (runtime/mods.c). The writes go through the same rules as the mods' code cheats (tools/cheats.py convert): an instruction replaced in place, -a branch into a code cave the file writes into main's zero padding inlined at the branch. +a branch into a code cave the file writes into main's zero padding inlined at the branch. Writes into main's data +(.rodata, .data, .bss) need no lifting: the runtime stores them into the loaded module before the game runs, as +emulators and Atmosphere apply a patch file to the module before its dynamic linker runs (data_runs). - python tools/exefs_patches.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file> --status <file> + python tools/exefs_patches.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file> + --status <file> --data <file> + python tools/exefs_patches.py --exefs <dir> --status <file> --data <file> --folded <gen>/main/folded.txt The files are <mod>/exefs/*.ips and *.pchtxt (or the Atmosphere places of the exefs folder, as runtime/mods.c finds them) and <mod>/atmosphere/exefs_patches/<name>/*.ips. A file is for main when its build id matches main's: an IPS @@ -13,19 +18,37 @@ file's name, a pchtxt file's @nsobid line (trailing zeros may be left out). Offs offsets count the NSO header (0x100 bytes) before main's image; a pchtxt offset adds its `@flag offset_shift` first. A pchtxt patch line applies between @enabled and @disabled; @stop ends the file. ---out gets the --patches file followed by one set per usable file (`mod_<hash>`, switch TOTK_MODPATCH_<HASH>, default -off) and its sites. --status gets one line per file, mods by name, tab separated: +--out gets the --patches file followed by one set per file with sites (`mod_<hash>`, switch TOTK_MODPATCH_<HASH>, +default off) and its sites. --status gets one line per file, mods by name, tab separated: <file hash> <content hash> <kind> <built-in sets> <note> file hash: FNV-1a 32 of "<mod folder name>/<path in the mod>" (forward slashes); content hash: the same over the file's bytes, so the runtime sees a file that changed after the build. kind: - code a set of sites + code a set of sites, and data writes when the file has some + data data writes only other for another game version or module: not used - empty no write into main for this version (or only the game's own words) - unsupported <note> says why: a write into main's data, a write the site rules cannot express + empty no write into main for this version (or only the game's own bytes) + unsupported <note> says why: a write the site rules cannot express, a write outside main's image, into the dynamic + linker's tables or into a pointer it binds lazily, or (after the lift) into bytes the lifted code reads + as constants built-in sets: the port's own sets (the --patches file's sets that are no cheats) with a site this file patches too, -comma separated, or "-": the runtime switches them off while the mod is on, so the mod's code runs. +comma separated, or "-": the runtime switches them off while the mod is on, so the mod's code runs. The note of a code +or data file with data writes is `data <bytes>`, the bytes its data lines hold: the runtime keeps the mod's patches off +when the data file does not hold them all. + +--data gets one line per run of data bytes of the code and data files, tab separated (runtime/mods.c reads it): + + <file hash> <content hash> <offset in main, hex> <bytes, hex> [base] + +`base` marks a DT_RELR word: 8 bytes, little-endian, the word before relocation; the runtime adds main's base, as the +dynamic linker does. Without --data a data write makes its file unsupported. + +The lifted code reads some of main's .text and .rodata at lift time instead of at run time: the jump tables +(recomp/jumptables.py), whose targets become labels of the generated C. A data write over such bytes would not reach +the lifted code. recomp/gen.py lists the ranges it read in <gen>/<module>/folded.txt; the second form above, run after +the lift, makes every file with a data write into those ranges unsupported (its status line says so and its data lines +go). Without that list, every data write into .rodata counts as one into read bytes. A file the rules cannot express is unsupported as a whole: half of a patch is worse than none. @@ -39,9 +62,12 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from cheats import TITLE, Main, convert, fnv # noqa: E402 +from nso import (DT_JMPREL, DT_NULL, DT_PLTRELSZ, DT_RELA, DT_RELASZ, DT_RELR, DT_RELRSZ, DT_STRSZ, # noqa: E402 + DT_STRTAB, DT_SYMTAB, R_AARCH64_JUMP_SLOT) NSO_HEADER = 0x100 EXEFS_PLACES = ('exefs', f'atmosphere/contents/{TITLE}/exefs', f'contents/{TITLE}/exefs', f'{TITLE}/exefs') +RUN_MAX = 256 # bytes per line of the data file (runtime/mods.c reads lines of at most 1024 characters) def patch_files(mods): @@ -159,6 +185,99 @@ def word_writes(main, writes): return words +# ---- writes into data + +def relocated(n): + """({offset: type} of the words main's explicit-addend relocations (DT_RELA, DT_JMPREL) fill, {offset} of its + DT_RELR words): the 8-byte words the dynamic linker writes at start (runtime/loader.c relocate_table, + relocate_relr).""" + rela = {} + for table, size in ((DT_RELA, DT_RELASZ), (DT_JMPREL, DT_PLTRELSZ)): + if table in n.dyn: + for i in range(n.dyn[size] // 24): + off, info, _ = struct.unpack_from('<QQq', n.image, n.dyn[table] + 24 * i) + rela[off] = info & 0xFFFFFFFF + relr, where = set(), 0 + for i in range(n.dyn.get(DT_RELRSZ, 0) // 8): + e = struct.unpack_from('<Q', n.image, n.dyn[DT_RELR] + 8 * i)[0] + if not e & 1: # an address; the bitmap after it covers the 63 words that follow + relr.add(e) + where = e + 8 + else: + relr.update(where + 8 * k for k in range(63) if e >> (k + 1) & 1) + where += 63 * 8 + return rela, relr + + +def linker_tables(n): + """[(start, end)] of main's .dynamic, relocation, symbol and string tables: the dynamic linker reads them at start, + so a write into them would change what it does, which no store into the loaded module can follow.""" + end = n.dynamic + while struct.unpack_from('<Q', n.image, end)[0] != DT_NULL: + end += 16 + out = [(n.dynamic, end + 16)] + for table, size in ((DT_RELA, DT_RELASZ), (DT_JMPREL, DT_PLTRELSZ), (DT_RELR, DT_RELRSZ), (DT_STRTAB, DT_STRSZ)): + if table in n.dyn and size in n.dyn: + out.append((n.dyn[table], n.dyn[table] + n.dyn[size])) + if DT_SYMTAB in n.dyn: + out.append((n.dyn[DT_SYMTAB], n.dyn[DT_SYMTAB] + 24 * n.symbol_count())) + return out + + +class MainData: + """What a write into main's data meets: main's image from .rodata to the end of .bss, the words the dynamic linker + fills and the tables it reads.""" + + def __init__(self, main): + n = main.nso + self.image, self.lo, self.size, self.data = main.image, main.code_end, n.size, n.data[0] + self.rela, self.relr = relocated(n) + self.linker = linker_tables(n) + + +def data_runs(md, writes): + """The data part of one file ({offset: byte} outside main's code), applied as emulators and Atmosphere apply a patch + file: to the module before its dynamic linker runs. Returns ([(offset, bytes, rebase)], dropped): + + - a byte no relocation covers is stored as it is (runs of consecutive bytes, at most RUN_MAX long); + - a DT_RELR word holds its addend: the word with the file's bytes in it, rebase set (main's base gets added); + - the linker overwrites a word an explicit-addend relocation fills whatever the file wrote there: such bytes + change nothing and only count in dropped. + + Bytes equal to main's are left out. ValueError for a write outside main's image (below .rodata: the NSO header), + into the dynamic linker's tables, or into a function pointer it binds lazily (JUMP_SLOT): on the Switch it holds + the file's value plus main's base until the first call binds it, which no store before the start can reproduce.""" + plain, words, dropped = {}, {}, 0 + for off in sorted(writes): + if not md.lo <= off < md.size: + raise ValueError(f'it writes outside main\'s image at main+{off:X}') + if any(a <= off < b for a, b in md.linker): + raise ValueError(f'it writes the dynamic linker\'s tables at main+{off:X}') + w = next((w for w in range(off - 7, off + 1) if w in md.rela or w in md.relr), None) + if w is None: + plain[off] = writes[off] + elif w in md.relr: + words.setdefault(w, bytearray(md.image[w:w + 8]))[off - w] = writes[off] + elif md.rela[w] == R_AARCH64_JUMP_SLOT: + raise ValueError(f'it writes a function pointer the dynamic linker binds lazily at main+{w:X}') + else: + dropped += 1 + runs = [] + for off in sorted(o for o, b in plain.items() if b != md.image[o]): + last = runs[-1] if runs else None + if last and last[0] + len(last[1]) == off and len(last[1]) < RUN_MAX: + runs[-1] = (last[0], last[1] + bytes([plain[off]]), False) + else: + runs.append((off, bytes([plain[off]]), False)) + runs += [(w, bytes(b), True) for w, b in words.items() if bytes(b) != md.image[w:w + 8]] + return sorted(runs), dropped + + +def data_line(key, content, run): + off, data, rebase = run + return f'{key}\t{content}\t{off:x}\t{data.hex()}' + ('\tbase' if rebase else '') + + def builtin_sites(text): """{offset in main: [set name]} of the patch file's sets that are no cheats or mod patches.""" out = {} @@ -169,41 +288,45 @@ def builtin_sites(text): return out -def main(): - ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - ap.add_argument('--exefs', type=Path, required=True) - ap.add_argument('--mods', type=Path, required=True, help='the mods folder next to totk.exe') - ap.add_argument('--patches', type=Path, required=True, help='the patch file so far') - ap.add_argument('--out', type=Path, required=True, help='the patch file with the mods\' ExeFS patches added') - ap.add_argument('--status', type=Path, required=True) - args = ap.parse_args() - sys.stdout.reconfigure(errors='replace') +def classify(m, md, bid, path, raw): + """(kind, note, sites, runs, dropped) of one patch file: path gives its name and suffix, raw its bytes, bid main's + build id; md None: no data file, so a write into main's data makes the file unsupported.""" + try: + if path.suffix.lower() == '.ips': + for_main = same_build(path.stem, bid) + writes = parse_ips(raw) if for_main else {} + else: + pid, writes = parse_pchtxt(raw.decode('utf-8-sig', errors='replace')) + for_main = pid is not None and same_build(pid, bid) + if not for_main: + return 'other', '', {}, [], 0 + code = {o: b for o, b in writes.items() if m.in_code(o - o % 4)} + data = {o: b for o, b in writes.items() if o not in code} + if data and md is None: + raise ValueError(f'it writes main\'s data at main+{min(data):X} (only code patches are supported)') + sites = convert(m, word_writes(m, code)) if code else {} + runs, dropped = data_runs(md, data) if data else ([], 0) + return 'code' if sites else 'data' if runs else 'empty', '', sites, runs, dropped + except ValueError as e: + return 'unsupported', str(e), {}, [], 0 + + +DATA_HEADER = '# file hash, content hash, offset in main, bytes[, base] (tools/exefs_patches.py)' + + +def build(args): m = Main(args.exefs / 'main') - raw = (args.exefs / 'main').read_bytes() - bid = raw[0x40:0x60].hex() + md = MainData(m) if args.data else None + bid = m.nso.build_id text = args.patches.read_text() builtin = builtin_sites(text) files = patch_files(args.mods) - sets, sites, status, kinds = [], [], [], {} + sets, sites, status, data_lines, kinds = [], [], [], [], {} for mod, rel, f in files: - data = f.read_bytes() - key, content = fnv(f'{mod}/{rel}'.encode()), fnv(data) - kind, note, over, file_sites = 'code', '', [], {} - try: - if f.suffix.lower() == '.ips': - for_main = same_build(f.stem, bid) - writes = parse_ips(data) if for_main else {} - else: - pid, writes = parse_pchtxt(data.decode('utf-8-sig', errors='replace')) - for_main = pid is not None and same_build(pid, bid) - if not for_main: - kind = 'other' - else: - file_sites = convert(m, word_writes(m, writes)) - if not file_sites: - kind = 'empty' - except ValueError as e: - kind, note, file_sites = 'unsupported', str(e), {} + raw = f.read_bytes() + key, content = fnv(f'{mod}/{rel}'.encode()), fnv(raw) + kind, note, file_sites, runs, dropped = classify(m, md, bid, f, raw) + over = [] if file_sites: name = f'mod_{key.lower()}' label = f'{mod}/{rel}'.encode('ascii', 'replace').decode() @@ -211,19 +334,108 @@ def main(): for a, repl in sorted(file_sites.items()): sites.append(f'site {name} main {a:x} {m.word(a):08x} {" ".join(f"{w:08x}" for w in repl)}') over += [s for s in builtin.get(a, []) if s not in over] + data_lines += [data_line(key, content, r) for r in runs] + data_bytes = sum(len(r[1]) for r in runs) kinds[kind] = kinds.get(kind, 0) + 1 - status.append(f'{key}\t{content}\t{kind}\t{",".join(over) or "-"}\t{note.replace(chr(9), " ")}') - print(f'exefs: {kind:11} {mod}/{rel}' + (f' ({note})' if note else '') - + (f' ({len(file_sites)} sites, over {", ".join(over)})' if over else - f' ({len(file_sites)} sites)' if file_sites else '')) + status_note = f'data {data_bytes}' if runs else note.replace(chr(9), ' ') + status.append(f'{key}\t{content}\t{kind}\t{",".join(over) or "-"}\t{status_note}') + parts = [note] if note else [] + if file_sites: + parts.append(f'{len(file_sites)} sites' + (f', over {", ".join(over)}' if over else '')) + if runs: + parts.append(f'{data_bytes} bytes of main\'s data') + if dropped: + parts.append(f'{dropped} bytes into words the dynamic linker fills, which change nothing') + print(f'exefs: {kind:11} {mod}/{rel}' + (f' ({"; ".join(parts)})' if parts else '')) if sets: text += ('\n# ExeFS patches of the mods folder (tools/exefs_patches.py)\n' + '\n'.join(sets) + '\n' + '\n'.join(sites) + '\n') args.out.write_text(text) args.status.parent.mkdir(parents=True, exist_ok=True) args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8') + if args.data: + args.data.parent.mkdir(parents=True, exist_ok=True) + args.data.write_text('\n'.join([DATA_HEADER] + data_lines) + '\n', encoding='utf-8') print(f'exefs: {len(files)} patch files: ' + ', '.join(f'{n} {k}' for k, n in sorted(kinds.items())) - + f'; {len(sites)} sites') + + f'; {len(sites)} sites, {len(data_lines)} data runs') + + +def read_ranges(path): + """[(start, end)] of a folded.txt (recomp/gen.py): hex module offsets, one range per line, '#' comments.""" + out = [] + for line in path.read_text().splitlines(): + f = line.split('#', 1)[0].split() + if len(f) == 2: + out.append((int(f[0], 16), int(f[1], 16))) + return sorted(out) + + +def data_offset(main_path): + """The offset of main's .data segment in its image, from the NSO header (no decompression).""" + with open(main_path, 'rb') as f: + head = f.read(0x40) + return struct.unpack_from('<III', head, 0x30)[1] + + +def recheck(args): + """After the lift: every file with a data write into bytes the lifted code read at lift time becomes unsupported.""" + if not args.status.is_file() or not args.data.is_file(): + print('exefs: no status or data file: nothing to check') + return + folded = read_ranges(args.folded) if args.folded.is_file() else None + data_off = data_offset(args.exefs / 'main') + lines = args.data.read_text(encoding='utf-8').splitlines() + bad = {} + for line in lines: + f = line.split('\t') + if line.startswith('#') or len(f) < 4 or f[0] in bad: + continue + a = int(f[2], 16) + b = a + len(f[3]) // 2 + if folded is None and a < data_off: + bad[f[0]] = (f'it writes main+{a:X} in .rodata, and the lift left no list of the bytes it reads as ' + f'constants') + elif folded is not None: + hit = next((s for s, e in folded if s < b and a < e), None) + if hit is not None: + bad[f[0]] = (f'it writes main+{max(a, hit):X}, which the lifted code reads when the game is built ' + f'(a jump table): the change would not reach it') + if not bad: + print(f'exefs: data writes checked against {"the bytes the lift read" if folded is not None else ".rodata"}: ' + f'none meets them') + return + status = [] + for line in args.status.read_text(encoding='utf-8').splitlines(): + f = line.split('\t') + if len(f) >= 5 and f[0] in bad: + line = '\t'.join([f[0], f[1], 'unsupported', '-', bad[f[0]]]) + print(f'exefs: unsupported file {f[0]} ({bad[f[0]]})') + status.append(line) + args.status.write_text('\n'.join(status) + '\n', encoding='utf-8') + kept = [line for line in lines if line.split('\t')[0] not in bad] + args.data.write_text('\n'.join(kept) + '\n', encoding='utf-8') + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--exefs', type=Path, required=True) + ap.add_argument('--mods', type=Path, help='the mods folder next to totk.exe') + ap.add_argument('--patches', type=Path, help='the patch file so far') + ap.add_argument('--out', type=Path, help='the patch file with the mods\' ExeFS patches added') + ap.add_argument('--status', type=Path, required=True) + ap.add_argument('--data', type=Path, help='the data writes of the mods\' patch files (runtime/mods.c reads them)') + ap.add_argument('--folded', type=Path, help='after the lift: <gen>/main/folded.txt; checks --status and --data ' + 'against it instead of building') + args = ap.parse_args() + sys.stdout.reconfigure(errors='replace') + if args.folded: + if not args.data: + ap.error('--folded needs --data') + recheck(args) + else: + if not (args.mods and args.patches and args.out): + ap.error('--mods, --patches and --out are needed to build') + build(args) if __name__ == '__main__':