From 59c8d20fe3cc3148f31f102a5c488194230cf3b3 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 1 Nov 2026 13:14:37 +0300 Subject: [PATCH] Serve the mods folder as RomFS overlays: the higher mod wins a file, unusable mods are skipped Player mod files are checked by identity, not hashed, and a path several overlays give is served by one: a higher mod beats a lower one, any player mod beats the romfs_mods manifest and the port's own patches. A mod path that differs from a RomFS path only in case is served under the RomFS spelling. Files named for another game version (.Product.., missing from this RomFS) are counted and left out. A player mod that replaces a file of the Xbox prompts folder sends the game back to the Switch prompts, as a romfs_mods manifest already did. Co-Authored-By: Claude Opus 5.5 --- runtime/fs.h | 1 + runtime/romfs.c | 201 +++++++++++++++++++++++++++++++++++--- tests/mods_romfs_check.py | 95 ++++++++++++++++++ 3 files changed, 281 insertions(+), 16 deletions(-) create mode 100644 tests/mods_romfs_check.py diff --git a/runtime/fs.h b/runtime/fs.h index 5c857e24..21d9f8cf 100644 --- a/runtime/fs.h +++ b/runtime/fs.h @@ -38,6 +38,7 @@ uint64_t romfs_file_count(void); // files in the index const char *romfs_entry(uint64_t i, uint64_t *offset, uint64_t *size); // path, image offset and size of index row i const char *romfs_image_path(void); // the romfs.bin path fs_init opened size_t romfs_overlay_count(void); +int romfs_patches_loaded(void); // a romfs-patches manifest gave at least one file // The button prompts (input_prompts_folder): the folder loaded at start, NULL for the game's own prompts; whether a // folder had a manifest at start; and why the folder the settings asked for at start was not loaded (ROMFS_PROMPTS_OK // for any other folder: it was not asked for). diff --git a/runtime/romfs.c b/runtime/romfs.c index a389593f..2e673b4f 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -17,14 +17,21 @@ // TOTK_ROMFS_MODS; unset or "0" = none. A missing path, a size or hash mismatch or a changed original stops the start. // An overlay file is opened when the game opens it; its identity (volume, file id, size, last write) is kept from the // start check, and an open that finds another file there stops the game. +// +// Player mods (mods.c): the files under each enabled mod's RomFS root are overlays too, checked by identity only (no +// hash, no manifest). A path several overlays give is served by one: a player mod higher in the order beats a lower +// one, any player mod beats the TOTK_ROMFS_MODS manifest, which beats romfs-patches (overlay_contest). A mod path that +// matches a RomFS path only without case is served under the RomFS spelling. #include #include #include +#include #include "cfg.h" #include "fs.h" #include "host.h" #include "input_map.h" +#include "mods.h" #include "romfs_config.h" #include "romfs_path.h" #include "platform/platform.h" @@ -39,7 +46,9 @@ typedef struct { uint64_t offset, size; uint32_t name, len; } Entry; _Static_assert(sizeof(Entry) == 24, "index entry layout"); typedef struct { uint32_t volume, pad; uint64_t file_id, size, write_time, create_time; } Identity; // no padding -typedef struct { uint64_t index, size; char *path, *name; Identity id; } Overlay; +// prio: which overlay serves a path two of them give (the higher); romfs-patches 0, the TOTK_ROMFS_MODS manifest 1, a +// player mod 2 + its distance from the bottom of the order. mod: its index in mods_at + 1, 0 for any other overlay. +typedef struct { uint64_t index, size; char *path, *name; Identity id; int prio, mod; } Overlay; static uint8_t *index_bytes; static size_t index_size; @@ -57,6 +66,7 @@ static Overlay *overlays; // sorted by RomFS index after overlay_init static size_t noverlay, overlay_cap; static char **added; // RomFS paths of the added files, sorted; added[k] is index file_count + k static size_t nadded; +static int patches_loaded; // a romfs-patches manifest gave a file (romfs_patches_loaded) static void need(int c, const char *m) { if (!c) rt_fatal("RomFS: %s", m); @@ -205,8 +215,8 @@ static uint32_t overlay_slot(uint64_t i) { // ---- overlays // Loads and checks one overlay folder. A failed check stops the game, except with optional set (the Xbox prompts // folder): the folder is dropped, the reason logged, and 0 returned. Returns 1 when the folder is loaded or has no -// manifest. -static int overlay_load(const char *dir, int optional) { +// manifest. prio: the Overlay field (0 for the port's own folders, 1 for the TOTK_ROMFS_MODS manifest). +static int overlay_load(const char *dir, int optional, int prio) { #define BAD(...) do { if (!optional) rt_fatal(__VA_ARGS__); rt_log(__VA_ARGS__); goto bad; } while (0) char path[2100]; need(snprintf(path, sizeof path, "%s/manifest.tsv", dir) < (int)sizeof path, "overlay path"); @@ -263,7 +273,8 @@ static int overlay_load(const char *dir, int optional) { } char *copy = strdup(file), *rel = strdup(name); need(copy && rel, "overlay path copy"); - overlays[noverlay++] = (Overlay){i, size, copy, rel, id}; + overlays[noverlay++] = (Overlay){i, size, copy, rel, id, prio, 0}; + if (!prio) patches_loaded = 1; if (noverlay - first <= 16) rt_log("RomFS overlay: %s (%llu bytes) %s", name, size, add ? "is added" : "replaces the original"); } @@ -284,6 +295,161 @@ bad: #undef BAD } +// ---- player mods (mods.c): every file under an enabled mod's RomFS root, by identity (no hash: a texture pack is +// gigabytes and the player changes it at will; romfs_open still stops on a file that changed after the start) +static uint32_t *folded; // RomFS indices sorted by name ignoring ASCII case; built on the first mod path that misses + +static int folded_order(const void *a, const void *b) { + return strcasecmp(entry_name(entry(*(const uint32_t *)a)), entry_name(entry(*(const uint32_t *)b))); +} + +// The RomFS index of name, or of the RomFS path equal to it but for case (a mod made on Windows, where an emulator's +// folder lookup ignores case); UINT64_MAX for a path the RomFS lacks. +static uint64_t index_any_case(const char *name) { + uint64_t i = lower(name); + if (i < file_count && !strcmp(entry_name(entry(i)), name)) return i; + if (!folded) { + folded = malloc(file_count * sizeof *folded); + need(folded != NULL, "case-folded index"); + for (uint64_t k = 0; k < file_count; k++) folded[k] = (uint32_t)k; + qsort(folded, file_count, sizeof *folded, folded_order); + } + uint64_t lo = 0, hi = file_count; + while (lo < hi) { + uint64_t m = lo + (hi - lo) / 2; + if (strcasecmp(entry_name(entry(folded[m])), name) < 0) lo = m + 1; + else hi = m; + } + return lo < file_count && !strcasecmp(entry_name(entry(folded[lo])), name) ? folded[lo] : UINT64_MAX; +} + +static void overlay_push(Overlay o) { + if (noverlay == overlay_cap) { + overlay_cap = overlay_cap ? overlay_cap * 2 : 16; + overlays = realloc(overlays, overlay_cap * sizeof *overlays); + need(overlays != NULL, "overlay table"); + } + overlays[noverlay++] = o; +} + +// 1 when path names a file of one game version only (Product. in a name: the resource size table, the actor +// database and other version-bound files); the RomFS lacking it means it is another version's. +static int other_version(const char *path) { + for (const char *p = strstr(path, ".Product."); p; p = strstr(p + 1, ".Product.")) { + const char *d = p + strlen(".Product."); + if (*d >= '0' && *d <= '9') return 1; + } + return 0; +} + +typedef struct { int k, prio; const char *dir, *rel; } ModWalk; + +static void mod_walk(int k, int prio, const char *dir, const char *rel); + +// One entry of a mod's RomFS folder (mod_walk). +static int mod_walk_cb(const char *leaf, int kind, uint64_t size, void *ctx) { + (void)size; + const ModWalk *w = ctx; + ModInfo *m = mods_at(w->k); + char file[4200], path[4200]; + if (snprintf(file, sizeof file, "%s/%s", w->dir, leaf) >= (int)sizeof file || + snprintf(path, sizeof path, "%s%s%s", w->rel, *w->rel ? "/" : "", leaf) >= (int)sizeof path) { + rt_log("mods: %s: path too long under %s; left out", m->name, w->dir); + return 1; + } + if (kind == PLAT_FS_DIR) { + mod_walk(w->k, w->prio, file, path); + return 1; + } + if (kind != PLAT_FS_FILE) return 1; + if (!romfs_path_valid_relative(path)) { + rt_log("mods: %s: %s is not a RomFS path; left out", m->name, path); + return 1; + } + uint64_t i = index_any_case(path); + if (i == UINT64_MAX && other_version(path)) { + m->other_version++; + return 1; + } + PlatFile *f = open_read(file); + if (!f) { + rt_log("mods: %s: cannot open %s; left out", m->name, file); + return 1; + } + Identity id = identity_of(f); + plat_file_close(f); + char *copy = strdup(file), *rp = strdup(i == UINT64_MAX ? path : entry_name(entry(i))); + need(copy && rp, "overlay path copy"); + overlay_push((Overlay){i == UINT64_MAX ? ADDED : i, id.size, copy, rp, id, w->prio, w->k + 1}); + return 1; +} + +// The files under dir (a mod's RomFS root; rel: the RomFS path of dir, "" at the root) as overlays of mod k. +static void mod_walk(int k, int prio, const char *dir, const char *rel) { + ModWalk w = {k, prio, dir, rel}; + plat_fs_list(dir, mod_walk_cb, &w); +} + +// What overlays compete on: the RomFS index, an added file's path; the higher prio first. +static int contest_order(const void *a, const void *b) { + const Overlay *x = a, *y = b; + if (x->index != y->index) return (x->index > y->index) - (x->index < y->index); + int c = x->index == ADDED ? strcmp(x->name, y->name) : 0; + return c ? c : y->prio - x->prio; +} + +static const char *owner_name(const Overlay *o) { + return o->mod ? mods_at(o->mod - 1)->name : o->prio ? "romfs_mods" : "the port's patches"; +} + +// One overlay per path: the highest prio serves it; the others are dropped and counted on their mod. +static void overlay_contest(void) { + if (!noverlay) return; + qsort(overlays, noverlay, sizeof *overlays, contest_order); + size_t kept = 0; + for (size_t k = 0; k < noverlay; k++) { + Overlay *o = &overlays[k], *w = kept ? &overlays[kept - 1] : NULL; + if (w && w->index == o->index && (o->index != ADDED || !strcmp(w->name, o->name))) { + if (o->mod) { + ModInfo *m = mods_at(o->mod - 1); + m->overridden++; + if (!m->overridden_by[0]) snprintf(m->overridden_by, sizeof m->overridden_by, "%s", owner_name(w)); + } else if (w->mod) { + rt_log("RomFS overlay: %s from mod %s replaces the one of %s", o->name, owner_name(w), owner_name(o)); + } + free(o->path); + free(o->name); + continue; + } + if (o->mod) { + if (o->index == ADDED) mods_at(o->mod - 1)->added++; + else mods_at(o->mod - 1)->replaced++; + } + overlays[kept++] = *o; + } + noverlay = kept; +} + +static void mods_overlay_init(void) { + mods_scan(); + int n = mods_count(); + for (int k = 0; k < n; k++) { + const ModInfo *m = mods_at(k); + // a mod with ExeFS patches waits for the rebuild that applies them: half of a mod is worse than none + if (m->enabled && (m->kind & MOD_ROMFS) && m->problem == MOD_OK) mod_walk(k, 2 + (n - k), m->romfs, ""); + } +} + +static void mods_overlay_log(void) { + for (int k = 0; k < mods_count(); k++) { + const ModInfo *m = mods_at(k); + if (!m->enabled || !(m->kind & MOD_ROMFS) || m->problem != MOD_OK) continue; + rt_log("mods: %s: %u files served (%u replaced, %u added), %u overridden%s%s, %u for another game version", + m->name, m->replaced + m->added, m->replaced, m->added, m->overridden, m->overridden ? " by " : "", + m->overridden_by, m->other_version); + } +} + static int overlay_order(const void *a, const void *b) { uint64_t x = ((const Overlay *)a)->index, y = ((const Overlay *)b)->index; return (x > y) - (x < y); @@ -293,7 +459,7 @@ static int added_order(const void *a, const void *b) { return strcmp((*(Overlay *const *)a)->name, (*(Overlay *const *)b)->name); } -// Number the added files in path order after the RomFS's own (file_count + k); two overlays adding one path stop. +// Number the added files in path order after the RomFS's own (file_count + k); overlay_contest left one per path. static void added_init(void) { Overlay **list = NULL; for (size_t k = 0; k < noverlay; k++) @@ -307,8 +473,6 @@ static void added_init(void) { added = malloc(nadded * sizeof *added); need(added != NULL, "added files"); for (size_t k = 0; k < nadded; k++) { - if (k && !strcmp(list[k]->name, list[k - 1]->name)) - rt_fatal("RomFS overlay: %s and %s both add %s", list[k - 1]->path, list[k]->path, list[k]->name); list[k]->index = file_count + k; added[k] = list[k]->name; } @@ -459,8 +623,9 @@ static void overlay_init(void) { // patches with Xbox button icons (make_romfs_patches.py --prompts). The game reads the archives once at boot, so // the prompts stay until restart even though the layout itself changes live. Each folder replaces romfs-patches as // a whole and the setup builds no folder of another shape with Xbox prompts, so a game resolution that is not 16:9 - // keeps the Switch prompts. So does a RomFS mod that replaces one of the prompts' files (the fonts, the resource - // size table): romfs-patches loads in their place, where two overlays replacing one file would stop the game. + // keeps the Switch prompts. So does a RomFS mod (the TOTK_ROMFS_MODS manifest or a player mod) that replaces one of + // the prompts' files (the fonts, the resource size table): romfs-patches loads in their place, so the mod's file + // is not served beside prompts files made for the game's own. char def[2100], path[2100]; const char *patches = cfg_get("TOTK_ROMFS_PATCHES"), *mods = cfg_get("TOTK_ROMFS_MODS"); int wide = 0; @@ -493,7 +658,7 @@ static void overlay_init(void) { size_t first = noverlay; if (prompts_wanted && prompts_off == ROMFS_PROMPTS_OK) { snprintf(path, sizeof path, "%s/%s", index_dir, prompts_wanted); // fits: its manifest path did - if (overlay_load(path, 1)) { + if (overlay_load(path, 1, 0)) { prompts_loaded = prompts_wanted; rt_log("RomFS overlay: Xbox button prompts (%s)", path); } else { @@ -512,23 +677,25 @@ static void overlay_init(void) { plat_file_exists(plain); if (plain_ok) plain[strlen(plain) - strlen("/manifest.tsv")] = 0; if (!patches && !prompts_loaded && plain_ok) patches = plain; - if (patches && *patches && strcmp(patches, "0")) overlay_load(patches, 0); + if (patches && *patches && strcmp(patches, "0")) overlay_load(patches, 0, 0); size_t mods_first = noverlay; - if (mods && *mods && strcmp(mods, "0")) overlay_load(mods, 0); + if (mods && *mods && strcmp(mods, "0")) overlay_load(mods, 0, 1); + mods_overlay_init(); const char *clash = prompts_loaded ? overlay_clash(first, prompts_end, mods_first, noverlay) : NULL; if (clash) { - rt_log("RomFS overlay: the RomFS mods folder replaces %s, which the Xbox prompts change too; Switch prompts stay", - clash); + rt_log("RomFS overlay: a RomFS mod replaces %s, which the Xbox prompts change too; Switch prompts stay", clash); overlay_drop(first, prompts_end); prompts_loaded = NULL; prompts_off = ROMFS_PROMPTS_MODS; - if (plain_ok) overlay_load(plain, 0); + if (plain_ok) overlay_load(plain, 0, 0); } + overlay_contest(); + mods_overlay_log(); heapdef_overlay(); camera_overlay(); added_init(); if (noverlay) qsort(overlays, noverlay, sizeof *overlays, overlay_order); - for (size_t k = 1; k < noverlay; k++) + for (size_t k = 1; k < noverlay; k++) // overlay_contest left one per file; the generated ones replace in place if (overlays[k].index == overlays[k - 1].index) rt_fatal("RomFS overlay: %s and %s replace the same file %s", overlays[k - 1].path, overlays[k].path, overlays[k].name); @@ -650,6 +817,8 @@ const char *romfs_image_path(void) { return image_file; } size_t romfs_overlay_count(void) { return noverlay; } +int romfs_patches_loaded(void) { return patches_loaded; } + const char *romfs_overlay(size_t k, const char **file, uint64_t *size, int *is_added) { if (k >= noverlay) return NULL; *file = overlays[k].path; diff --git a/tests/mods_romfs_check.py b/tests/mods_romfs_check.py new file mode 100644 index 00000000..bbd5efd9 --- /dev/null +++ b/tests/mods_romfs_check.py @@ -0,0 +1,95 @@ +"""The mods folder served as RomFS overlays (runtime/romfs.c, runtime/mods.c): load order, added files, case, Unicode. + + .venv\\Scripts\\python tests\\mods_romfs_check.py --exe build\\v121-\\totk.exe --version v121 + +Builds a mods folder in a temp directory with files of its own bytes at real RomFS paths (from +build/extracted//romfs/index.json), runs `totk.exe --probe-fs` on it and reads the `mods:` lines of the log. +--probe-fs compares the bytes it reads through the RomFS with every overlay file, so a wrong winner fails it too. +Needs the extracted RomFS of the version. +""" +import argparse +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +UNI = 'Ж мод' + + +def pick_paths(version): + index = json.loads((ROOT / 'build/extracted' / version / 'romfs/index.json').read_text()) + paths = [f['path'] for f in index['files'] if 1024 <= f['size'] <= 65536] + have = set(f['path'] for f in index['files']) + p1 = next(p for p in paths if any(c.isalpha() for c in p.rsplit('/', 1)[-1])) + p2 = next(p for p in paths if p != p1) + i = next(k for k, c in enumerate(p1) if c.isalpha() and k > p1.rfind('/')) + p1case = p1[:i] + p1[i].swapcase() + p1[i + 1:] + assert p1case not in have + return p1, p2, p1case + + +def put(root, mod, rel, data, part='romfs'): + f = Path(root) / mod / part / rel + f.parent.mkdir(parents=True, exist_ok=True) + f.write_bytes(data * 64) + + +def run(exe, mods, order, disabled=''): + env = dict(os.environ, TOTK_MODS_DIR=str(mods), TOTK_MODS_ORDER=order, TOTK_MODS_DISABLED=disabled) + r = subprocess.run([str(exe), '--probe-fs'], env=env, capture_output=True, timeout=600) + log = (r.stdout + r.stderr).decode('utf-8', 'replace') + return r.returncode, log + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--exe', type=Path, required=True) + ap.add_argument('--version', choices=('v121', 'v143'), required=True) + args = ap.parse_args() + p1, p2, p1case = pick_paths(args.version) + fails = 0 + + def check(ok, what, log=None): + nonlocal fails + print(('PASS ' if ok else 'FAIL ') + what) + if not ok: + fails += 1 + if log: + print('\n'.join(l for l in log.splitlines() if 'mods' in l or 'overlay' in l.lower())[-3000:]) + + with tempfile.TemporaryDirectory() as tmp: + mods = Path(tmp) / 'mods' + put(mods, 'Top', p1, b'mod-A') + put(mods, 'Low', p1, b'mod-B') + put(mods, 'Low', p2, b'mod-C') + put(mods, 'Top', 'New/added.bin', b'mod-D') + put(mods, 'Low', 'New/added.bin', b'mod-E') + put(mods, 'Case', p1case, b'mod-F') + put(mods, UNI, p2, b'mod-G') + # a file of another game version's (no RomFS has a Product.999 table): counted, never served + put(mods, 'Low', 'System/Resource/ResourceSizeTable.Product.999.rsizetable.zs', b'mod-H') + put(mods, 'Lite', 'Lite/added.bin', b'mod-I', part='romfslite') # TKMM's romfslite folder + + code, log = run(args.exe, mods, f'Top|Low|{UNI}|Case') + check(code == 0, 'probe-fs passes with five mods (served bytes equal the winners\' files)', log) + for line in ('mods: Top: 2 files served (1 replaced, 1 added), 0 overridden', + 'mods: Low: 1 files served (1 replaced, 0 added), 2 overridden by Top, 1 for another game version', + 'mods: Case: 0 files served (0 replaced, 0 added), 1 overridden by Top', + f'mods: {UNI}: 0 files served (0 replaced, 0 added), 1 overridden by Low', + 'mods: Lite: 1 files served (0 replaced, 1 added), 0 overridden'): + check(line in log, line, log) + check(p1case not in log, 'the case-flipped path is served as the RomFS path, not added', log) + + code, log = run(args.exe, mods, f'Top|Low|{UNI}|Case', disabled='Top') + check(code == 0, 'probe-fs passes with Top switched off', log) + check('mods: Low: 3 files served (2 replaced, 1 added), 0 overridden' in log, 'Low serves P1 and the added file', log) + check('mods: Top: ' not in log.replace('mods: Top: off', ''), 'a switched-off mod serves nothing', log) + print('mods_romfs_check:', 'FAIL' if fails else 'ok') + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main())