// Adapted from MacNativeGames games/totk/runtime/romfs.c @ b75be8cf (MIT, Copyright (c) 2026 MacNativeGames // contributors). // Read-only game RomFS backed by the player's extracted image. // // Input: /romfs.bin (rt_config()->romfs), whose six IVFC levels and SHA-256 tools/extract.py verified, and // romfs-index.bin next to totk.exe (TOTK_ROMFS_INDEX overrides), which tools/romfs_index.py wrote together with // romfs_config.h. fs_init checks the index's SHA-256 and layout and the image size. The full 17 GB hash runs only with // TOTK_ROMFS_VERIFY=full; a proof file (.romfs-verified-v1 next to the index, or in TOTK_ROMFS_PROOF_DIR) keyed by // the file's volume, file id, size and times skips it on later starts, TOTK_ROMFS_VERIFY=force ignores the proof. // // Reads are positional (platform.h plat_file_pread), so threads read the image in parallel without a file-pointer // lock. TOTK_ROMFS_MMAP=1 maps the image instead. // // Overlays: files listed in a directory's manifest.tsv are served in place of the originals. Each line names a RomFS // path, the overlay's size and SHA-256 and the original's SHA-256 ("-" adds a file the RomFS lacks; it is numbered // after the RomFS's own files, file_count + k in path order). The directories, in this order: TOTK_ROMFS_PATCHES and // TOTK_ROMFS_MODS; unset or "0" = none. A missing path, a size or hash mismatch or a changed original stops the start. // An overlay file is opened when the game opens it; its identity (volume, file id, size, last write) is kept from the // start check, and an open that finds another file there stops the game. // // Player mods (mods.c): the files under each enabled mod's RomFS root are overlays too, checked by identity only (no // hash, no manifest). A path several overlays give is served by one: a player mod higher in the order beats a lower // one, any player mod beats the TOTK_ROMFS_MODS manifest, which beats romfs-patches (overlay_contest). A mod path that // matches a RomFS path only without case is served under the RomFS spelling. // // The resource size table: with a player mod enabled, the game's table is rebuilt at start with the tables the mods // ship merged in and the entries of the mods' files raised to what they need (restbl_overlay), the members of their // packs too (restbl_packs), and served from /mods/ in place of the game's or a mod's table. #include #include #include #include #include "cfg.h" #include "fs.h" #include "host.h" #include "input_map.h" #include "mods.h" #include "romfs_config.h" #include "romfs_path.h" #include "platform/platform.h" #include "restbl.h" #include "sha256.h" #include #define NH 65536 // RomFS handle slots #define READ_CHUNK (64u << 20) // largest single read #define HASH_CHUNK (8u << 20) #define ADDED UINT64_MAX // an added overlay file's index until overlay_init numbers it typedef struct { uint64_t offset, size; uint32_t name, len; } Entry; _Static_assert(sizeof(Entry) == 24, "index entry layout"); typedef struct { uint32_t volume, pad; uint64_t file_id, size, write_time, create_time; } Identity; // no padding // prio: which overlay serves a path two of them give (the higher); romfs-patches 0, the TOTK_ROMFS_MODS manifest 1, a // player mod 2 + its distance from the bottom of the order. mod: its index in mods_at + 1, 0 for any other overlay. typedef struct { uint64_t index, size; char *path, *name; Identity id; int prio, mod; } Overlay; static uint8_t *index_bytes; static size_t index_size; static uint64_t file_count, strings_offset; static PlatFile *image; static const uint8_t *image_map; static char image_file[2048]; static char index_dir[2048]; // the directory of romfs-index.bin: the proof file's default place static char mount_name[32]; static PlatMutex lock = PLAT_MUTEX_INIT; static uint64_t handles[NH]; // file index + 1; 0 is free static uint32_t handle_ovl[NH]; // overlay slot + 1 of an open handle, 0 = original static PlatFile *handle_file[NH]; // the handle's open overlay file, when handle_ovl is set static Overlay *overlays; // sorted by RomFS index after overlay_init static size_t noverlay, overlay_cap; static char **added; // RomFS paths of the added files, sorted; added[k] is index file_count + k static size_t nadded; static int patches_loaded; // a romfs-patches manifest gave a file (romfs_patches_loaded) static void need(int c, const char *m) { if (!c) rt_fatal("RomFS: %s", m); } static Entry entry(uint64_t i) { Entry e; memcpy(&e, index_bytes + 24 + i * 24, 24); return e; } static const char *entry_name(Entry e) { return (const char *)index_bytes + strings_offset + e.name; } static double now_s(void) { return (double)plat_now_ns() / 1e9; } // ---- positional reads // Bytes read at off (0 at or past the end of the file), -1 on an I/O error. static int64_t read_at(PlatFile *f, void *buf, uint32_t n, uint64_t off) { mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting int64_t got = plat_file_pread(f, buf, n, off); if (got == -2) // a read-only page in the buffer: a frozen region missed its thaw rt_log("RomFS: read of %u bytes into %p refused: the buffer is not writable", n, buf); return got < 0 ? -1 : got; } static PlatFile *open_read(const char *path) { return plat_file_open(path); } static Identity identity_of(PlatFile *f) { PlatFileId i; need(plat_file_id(f, &i), "file information"); return (Identity){(uint32_t)i.volume, 0, i.file_id, i.size, i.write_time, i.create_time}; } // ---- SHA-256 over a file range, streaming static void hex32(const uint8_t d[32], char out[65]) { for (int i = 0; i < 32; i++) snprintf(out + 2 * i, 3, "%02x", d[i]); } static void sha_range(PlatFile *f, uint64_t offset, uint64_t size, char hex[65], int progress) { Sha256 h; sha256_init(&h); uint8_t *buf = malloc(HASH_CHUNK); need(buf != NULL, "hash buffer"); double t0 = now_s(); for (uint64_t done = 0; done < size;) { uint32_t want = size - done > HASH_CHUNK ? HASH_CHUNK : (uint32_t)(size - done); int64_t n = read_at(f, buf, want, offset + done); need(n > 0, "read of hash input"); sha256_update(&h, buf, (size_t)n); done += (uint64_t)n; if (progress && (done % (2ull << 30) < (uint64_t)n || done == size)) rt_log("RomFS: hashed %.1f of %.1f GiB (%.0f MiB/s)", done / 1073741824.0, size / 1073741824.0, done / 1048576.0 / (now_s() - t0 + 1e-9)); } uint8_t d[32]; sha256_final(&h, d); free(buf); hex32(d, hex); } // ---- full image verification with a proof file typedef struct { char magic[16], sha256[65]; uint8_t pad[7]; Identity id; } RomfsProof; static void verify_image(const char *dir) { const char *mode = cfg_get("TOTK_ROMFS_VERIFY"); if (!mode || (strcmp(mode, "full") && strcmp(mode, "force"))) return; const char *pdir = cfg_get("TOTK_ROMFS_PROOF_DIR"); if (pdir && *pdir) dir = pdir; RomfsProof want = {0}, saved; memcpy(want.magic, "TOTKRFS-PROOF-1", 16); memcpy(want.sha256, ROMFS_IMAGE_SHA256, sizeof ROMFS_IMAGE_SHA256 < 65 ? sizeof ROMFS_IMAGE_SHA256 : 65); want.id = identity_of(image); char path[2100]; need(snprintf(path, sizeof path, "%s/.romfs-verified-v1", dir) < (int)sizeof path, "proof path"); if (strcmp(mode, "force")) { PlatFile *p = open_read(path); if (p) { int ok = plat_file_pread(p, &saved, sizeof saved, 0) == sizeof saved; plat_file_close(p); if (ok && !memcmp(&saved, &want, sizeof want)) { rt_log("RomFS unchanged since its full SHA-256 check (%s)", path); return; } } } rt_log("RomFS: verifying the SHA-256 of %s (%llu bytes)", image_file, (unsigned long long)want.id.size); char hex[65]; sha_range(image, 0, want.id.size, hex, 1); if (strcmp(hex, ROMFS_IMAGE_SHA256)) rt_fatal("RomFS image %s has SHA-256 %s, the extraction recorded %s (extract it again)", image_file, hex, ROMFS_IMAGE_SHA256); Identity after = identity_of(image); need(!memcmp(&after, &want.id, sizeof after), "image changed during verification"); plat_file_write_atomic(path, &want, sizeof want); rt_log("RomFS: image SHA-256 ok"); } // ---- index lookup static uint64_t lower(const char *name) { uint64_t lo = 0, hi = file_count; while (lo < hi) { uint64_t m = lo + (hi - lo) / 2; if (strcmp(entry_name(entry(m)), name) < 0) lo = m + 1; else hi = m; } return lo; } static size_t added_lower(const char *p) { size_t lo = 0, hi = nadded; while (lo < hi) { size_t m = lo + (hi - lo) / 2; if (strcmp(added[m], p) < 0) lo = m + 1; else hi = m; } return lo; } // The index of RomFS path p (an added file: file_count + k), or UINT64_MAX. static uint64_t file_index(const char *p) { uint64_t i = lower(p); if (i < file_count && !strcmp(p, entry_name(entry(i)))) return i; size_t k = added_lower(p); return k < nadded && !strcmp(added[k], p) ? file_count + k : UINT64_MAX; } // 1 when a RomFS file or an added one lies under folder (a path ending in '/'). static int folder_exists(const char *folder) { size_t n = strlen(folder); uint64_t i = lower(folder); if (i < file_count && !strncmp(entry_name(entry(i)), folder, n)) return 1; size_t k = added_lower(folder); return k < nadded && !strncmp(added[k], folder, n); } // The overlay slot + 1 that replaces RomFS file i, or 0. static uint32_t overlay_slot(uint64_t i) { size_t lo = 0, hi = noverlay; while (lo < hi) { size_t m = lo + (hi - lo) / 2; if (overlays[m].index < i) lo = m + 1; else hi = m; } return lo < noverlay && overlays[lo].index == i ? (uint32_t)(lo + 1) : 0; } // ---- overlays // Loads and checks one overlay folder. A failed check stops the game, except with optional set (the Xbox prompts // folder): the folder is dropped, the reason logged, and 0 returned. Returns 1 when the folder is loaded or has no // manifest. prio: the Overlay field (0 for the port's own folders, 1 for the TOTK_ROMFS_MODS manifest). static int overlay_load(const char *dir, int optional, int prio) { #define BAD(...) do { if (!optional) rt_fatal(__VA_ARGS__); rt_log(__VA_ARGS__); goto bad; } while (0) char path[2100]; need(snprintf(path, sizeof path, "%s/manifest.tsv", dir) < (int)sizeof path, "overlay path"); FILE *m = fopen(path, "rb"); if (!m) { rt_log("RomFS overlay: none (%s)", path); return 1; } char line[8192]; size_t first = noverlay; uint64_t hashed = 0; double t0 = now_s(); while (fgets(line, sizeof line, m)) { if (line[0] == '#' || line[0] == '\n' || line[0] == '\r') continue; char name[4096], sha[65], orig[65], hex[65]; unsigned long long size = 0; need(sscanf(line, "%4095[^\t]\t%llu\t%64[0-9a-f]\t%64[0-9a-f-]", name, &size, sha, orig) == 4 && strlen(sha) == 64 && (strlen(orig) == 64 || !strcmp(orig, "-")), "overlay manifest line"); need(romfs_path_valid_relative(name), "overlay path"); uint64_t i = lower(name); int known = i < file_count && !strcmp(entry_name(entry(i)), name), add = !strcmp(orig, "-"); if (!known && !add) BAD("RomFS overlay: %s/manifest.tsv names %s, which the RomFS lacks; a new file has \"-\" as its " "original's SHA-256", dir, name); if (known && add) BAD("RomFS overlay: %s/manifest.tsv adds %s, which the RomFS already has; give its original's SHA-256", dir, name); Entry e = {0}; if (known) { e = entry(i); sha_range(image, e.offset, e.size, hex, 0); if (strcmp(hex, orig)) BAD("RomFS overlay: the original of %s has SHA-256 %s, %s/manifest.tsv says %s", name, hex, dir, orig); } else { i = ADDED; } char file[8200]; need(snprintf(file, sizeof file, "%s/%s", dir, name) < (int)sizeof file, "overlay file path"); PlatFile *f = open_read(file); if (!f) BAD("RomFS overlay: %s missing", file); Identity id = identity_of(f); if (id.size != size) { plat_file_close(f); BAD("RomFS overlay: %s has %llu bytes, its manifest says %llu", file, (unsigned long long)id.size, size); } sha_range(f, 0, size, hex, 0); plat_file_close(f); if (strcmp(hex, sha)) BAD("RomFS overlay: %s has SHA-256 %s, its manifest says %s", file, hex, sha); hashed += e.size + size; if (noverlay == overlay_cap) { overlay_cap = overlay_cap ? overlay_cap * 2 : 16; overlays = realloc(overlays, overlay_cap * sizeof *overlays); need(overlays != NULL, "overlay table"); } char *copy = strdup(file), *rel = strdup(name); need(copy && rel, "overlay path copy"); overlays[noverlay++] = (Overlay){i, size, copy, rel, id, prio, 0}; if (!prio) patches_loaded = 1; if (noverlay - first <= 16) rt_log("RomFS overlay: %s (%llu bytes) %s", name, size, add ? "is added" : "replaces the original"); } fclose(m); if (noverlay - first > 16) rt_log("RomFS overlay: %zu files from %s (%zu not listed)", noverlay - first, dir, noverlay - first - 16); if (noverlay > first) rt_log("RomFS overlay: %s checked, %zu files, %.1f MiB hashed in %.2f s", dir, noverlay - first, hashed / 1048576.0, now_s() - t0); return 1; bad: fclose(m); for (size_t k = first; k < noverlay; k++) { free((void *)overlays[k].path); free((void *)overlays[k].name); } noverlay = first; return 0; #undef BAD } // ---- player mods (mods.c): every file under an enabled mod's RomFS root, by identity (no hash: a texture pack is // gigabytes and the player changes it at will; romfs_open still stops on a file that changed after the start) static uint32_t *folded; // RomFS indices sorted by name ignoring ASCII case; built on the first mod path that misses static int folded_order(const void *a, const void *b) { return strcasecmp(entry_name(entry(*(const uint32_t *)a)), entry_name(entry(*(const uint32_t *)b))); } // The RomFS index of name, or of the RomFS path equal to it but for case (a mod made on Windows, where an emulator's // folder lookup ignores case); UINT64_MAX for a path the RomFS lacks. static uint64_t index_any_case(const char *name) { uint64_t i = lower(name); if (i < file_count && !strcmp(entry_name(entry(i)), name)) return i; if (!folded) { folded = malloc(file_count * sizeof *folded); need(folded != NULL, "case-folded index"); for (uint64_t k = 0; k < file_count; k++) folded[k] = (uint32_t)k; qsort(folded, file_count, sizeof *folded, folded_order); } uint64_t lo = 0, hi = file_count; while (lo < hi) { uint64_t m = lo + (hi - lo) / 2; if (strcasecmp(entry_name(entry(folded[m])), name) < 0) lo = m + 1; else hi = m; } return lo < file_count && !strcasecmp(entry_name(entry(folded[lo])), name) ? folded[lo] : UINT64_MAX; } static void overlay_push(Overlay o) { if (noverlay == overlay_cap) { overlay_cap = overlay_cap ? overlay_cap * 2 : 16; overlays = realloc(overlays, overlay_cap * sizeof *overlays); need(overlays != NULL, "overlay table"); } overlays[noverlay++] = o; } // 1 when path names a file of one game version only (Product. in a name: the resource size table, the actor // database and other version-bound files); the RomFS lacking it means it is another version's, or a name this version // still asks for (mod_walk_cb). static int other_version(const char *path) { for (const char *p = strstr(path, ".Product."); p; p = strstr(p + 1, ".Product.")) { const char *d = p + strlen(".Product."); if (*d >= '0' && *d <= '9') return 1; } return 0; } // ---- the resource size table's inputs: the tables the overlays carry, before overlay_contest keeps one of them static char restbl_rel[256]; // the RomFS path of the game's table, "" when it has none typedef struct { char *file; int mod, other; // mod: ModInfo index + 1, 0 for the port's folders; other: made for another game version } Shipped; static Shipped *shipped; static size_t nshipped; static int ends_with(const char *s, const char *tail) { size_t n = strlen(s), k = strlen(tail); return n >= k && !strcmp(s + n - k, tail); } // 1 when rel is a resource size table of any game version. static int is_restbl(const char *rel) { return !strncmp(rel, "System/Resource/ResourceSizeTable.", 34) && ends_with(rel, ".rsizetable.zs"); } static void shipped_add(const char *file, int mod, int other) { Shipped *grown = realloc(shipped, (nshipped + 1) * sizeof *shipped); char *copy = strdup(file); need(grown && copy, "resource size table list"); shipped = grown; shipped[nshipped++] = (Shipped){copy, mod, other}; } // The game's table: the one System/Resource/ResourceSizeTable.*.rsizetable.zs of the RomFS. static void restbl_find(void) { for (uint64_t i = lower("System/Resource/ResourceSizeTable."); i < file_count; i++) { const char *n = entry_name(entry(i)); if (strncmp(n, "System/Resource/ResourceSizeTable.", 34)) break; if (is_restbl(n) && strlen(n) < sizeof restbl_rel) { snprintf(restbl_rel, sizeof restbl_rel, "%s", n); return; } } } // The tables the overlays carry for this game version, every one of them (overlay_contest keeps one). static void restbl_collect(void) { for (size_t k = 0; k < noverlay; k++) if (restbl_rel[0] && !strcmp(overlays[k].name, restbl_rel)) shipped_add(overlays[k].path, overlays[k].mod, 0); } typedef struct { int k, prio; const char *dir, *rel; } ModWalk; static void mod_walk(int k, int prio, const char *dir, const char *rel); // One entry of a mod's RomFS folder (mod_walk). static int mod_walk_cb(const char *leaf, int kind, uint64_t size, void *ctx) { (void)size; const ModWalk *w = ctx; ModInfo *m = mods_at(w->k); char file[4200], path[4200]; if (snprintf(file, sizeof file, "%s/%s", w->dir, leaf) >= (int)sizeof file || snprintf(path, sizeof path, "%s%s%s", w->rel, *w->rel ? "/" : "", leaf) >= (int)sizeof path) { rt_log("mods: %s: path too long under %s; left out", m->name, w->dir); return 1; } if (kind == PLAT_FS_DIR) { mod_walk(w->k, w->prio, file, path); return 1; } if (kind != PLAT_FS_FILE) return 1; if (!romfs_path_valid_relative(path)) { rt_log("mods: %s: %s is not a RomFS path; left out", m->name, path); return 1; } uint64_t i = index_any_case(path); if (i == UINT64_MAX && other_version(path)) { m->other_version++; // Another version's resource size table is no file this game opens; its entries for this mod's files still // apply. if (is_restbl(path)) { shipped_add(file, w->k + 1, 1); return 1; } // Any other file is served under its own name below, as an added file, the way LayeredFS shows a console every // file of a mod: the game does ask for names its RomFS lacks. Measured on 1.4.3 with mods on (trace = fs): it // opened Sound/Resource/Pack_MasterSword.Product.110.bars.zs, where its RomFS holds Product.140 and the // unversioned file only. A sound mod that ships its pack under all those names stayed silent while this file // was left out. } PlatFile *f = open_read(file); if (!f) { rt_log("mods: %s: cannot open %s; left out", m->name, file); return 1; } Identity id = identity_of(f); plat_file_close(f); char *copy = strdup(file), *rp = strdup(i == UINT64_MAX ? path : entry_name(entry(i))); need(copy && rp, "overlay path copy"); overlay_push((Overlay){i == UINT64_MAX ? ADDED : i, id.size, copy, rp, id, w->prio, w->k + 1}); return 1; } // The files under dir (a mod's RomFS root; rel: the RomFS path of dir, "" at the root) as overlays of mod k. static void mod_walk(int k, int prio, const char *dir, const char *rel) { ModWalk w = {k, prio, dir, rel}; plat_fs_list(dir, mod_walk_cb, &w); } // What overlays compete on: the RomFS index, an added file's path; the higher prio first. static int contest_order(const void *a, const void *b) { const Overlay *x = a, *y = b; if (x->index != y->index) return (x->index > y->index) - (x->index < y->index); int c = x->index == ADDED ? strcmp(x->name, y->name) : 0; return c ? c : y->prio - x->prio; } static const char *owner_name(const Overlay *o) { return o->mod ? mods_at(o->mod - 1)->name : o->prio ? "romfs_mods" : "the port's patches"; } // One overlay per path: the highest prio serves it; the others are dropped and counted on their mod. static void overlay_contest(void) { if (!noverlay) return; qsort(overlays, noverlay, sizeof *overlays, contest_order); size_t kept = 0; for (size_t k = 0; k < noverlay; k++) { Overlay *o = &overlays[k], *w = kept ? &overlays[kept - 1] : NULL; if (w && w->index == o->index && (o->index != ADDED || !strcmp(w->name, o->name))) { if (o->mod) { ModInfo *m = mods_at(o->mod - 1); m->overridden++; if (!m->overridden_by[0]) snprintf(m->overridden_by, sizeof m->overridden_by, "%s", owner_name(w)); } else if (w->mod) { rt_log("RomFS overlay: %s from mod %s replaces the one of %s", o->name, owner_name(w), owner_name(o)); } free(o->path); free(o->name); continue; } if (o->mod) { if (o->index == ADDED) mods_at(o->mod - 1)->added++; else mods_at(o->mod - 1)->replaced++; } overlays[kept++] = *o; } noverlay = kept; } static void mods_overlay_init(void) { mods_scan(); int n = mods_count(); for (int k = 0; k < n; k++) { const ModInfo *m = mods_at(k); // a mod with ExeFS patches waits for the rebuild that applies them: half of a mod is worse than none if (m->enabled && (m->kind & MOD_ROMFS) && m->problem == MOD_OK) mod_walk(k, 2 + (n - k), m->romfs, ""); } } static void mods_overlay_log(void) { for (int k = 0; k < mods_count(); k++) { const ModInfo *m = mods_at(k); if (!m->enabled || !(m->kind & MOD_ROMFS) || m->problem != MOD_OK) continue; rt_log("mods: %s: %u files served (%u replaced, %u added), %u overridden%s%s, %u for another game version", m->name, m->replaced + m->added, m->replaced, m->added, m->overridden, m->overridden ? " by " : "", m->overridden_by, m->other_version); } } static int overlay_order(const void *a, const void *b) { uint64_t x = ((const Overlay *)a)->index, y = ((const Overlay *)b)->index; return (x > y) - (x < y); } static int added_order(const void *a, const void *b) { return strcmp((*(Overlay *const *)a)->name, (*(Overlay *const *)b)->name); } // Number the added files in path order after the RomFS's own (file_count + k); overlay_contest left one per path. static void added_init(void) { Overlay **list = NULL; for (size_t k = 0; k < noverlay; k++) if (overlays[k].index == ADDED) { list = realloc(list, (nadded + 1) * sizeof *list); need(list != NULL, "added files"); list[nadded++] = &overlays[k]; } if (!nadded) return; qsort(list, nadded, sizeof *list, added_order); added = malloc(nadded * sizeof *added); need(added != NULL, "added files"); for (size_t k = 0; k < nadded; k++) { list[k]->index = file_count + k; added[k] = list[k]->name; } free(list); rt_log("RomFS overlay: %zu files added to the RomFS", nadded); } // RomFS file `rel` as the image holds it, whatever overlays replace it. malloc'd; NULL when it is missing, unreadable // or 16 MiB or more (logged with `keeps`, what the game does without it). static void *read_original(const char *rel, size_t *n, const char *keeps) { uint64_t i = lower(rel); if (i >= file_count || strcmp(entry_name(entry(i)), rel)) return NULL; uint64_t size = entry(i).size; void *in = size && size < (16u << 20) ? malloc((size_t)size) : NULL; if (!in || read_at(image, in, (uint32_t)size, entry(i).offset) != (int64_t)size) { free(in); rt_log("RomFS: cannot read %s; %s", rel, keeps); return NULL; } *n = (size_t)size; return in; } // RomFS file `rel` as the game would read it: from an overlay that replaces it (*base), else from the image. malloc'd; // NULL when it is missing, unreadable or 16 MiB or more (logged with `keeps`, what the game does without it). static void *read_served(const char *rel, Overlay **base, size_t *n, const char *keeps) { uint64_t i = lower(rel); *base = NULL; if (i >= file_count || strcmp(entry_name(entry(i)), rel)) { rt_log("RomFS: %s missing; %s", rel, keeps); return NULL; } for (size_t k = 0; k < noverlay; k++) if (overlays[k].index == i) *base = &overlays[k]; PlatFile *f = *base ? open_read((*base)->path) : image; uint64_t off = *base ? 0 : entry(i).offset, size = *base ? (*base)->size : entry(i).size; void *in = size && size < (16u << 20) && f ? malloc((size_t)size) : NULL; int ok = in && read_at(f, in, (uint32_t)size, off) == (int64_t)size; if (*base && f) plat_file_close(f); if (!ok) { free(in); rt_log("RomFS: cannot read %s; %s", rel, keeps); return NULL; } *n = (size_t)size; return in; } // Serves out[0..out_n) (malloc'd, freed here) in place of RomFS file `rel` and of an overlay that replaces it (base): // written to //. `what` names the change in the log. 0 when it cannot be written. static int serve_generated(const char *rel, Overlay *base, void *out, size_t out_n, const char *sub, const char *what, const char *keeps) { char dir[2100], file[2200]; const char *leaf = strrchr(rel, '/') ? strrchr(rel, '/') + 1 : rel; need(snprintf(dir, sizeof dir, "%s/%s", rt_config()->profile, sub) < (int)sizeof dir, "generated overlay path"); need(snprintf(file, sizeof file, "%s/%s", dir, leaf) < (int)sizeof file, "generated overlay path"); plat_file_mkdir_p(dir); int ok = plat_file_write(file, out, out_n); free(out); PlatFile *r = ok ? open_read(file) : NULL; if (!r) { rt_log("RomFS: cannot write %s; %s", file, keeps); return 0; } Identity id = identity_of(r); plat_file_close(r); char *copy = strdup(file); need(copy != NULL, "overlay path copy"); if (base) { free(base->path); base->path = copy; base->size = out_n; base->id = id; } else { if (noverlay == overlay_cap) { overlay_cap = overlay_cap ? overlay_cap * 2 : 16; overlays = realloc(overlays, overlay_cap * sizeof *overlays); need(overlays != NULL, "overlay table"); } char *name = strdup(rel); need(name != NULL, "overlay path copy"); overlays[noverlay++] = (Overlay){lower(rel), out_n, copy, name, id}; } rt_log("RomFS overlay: %s (%zu bytes, %s) replaces %s", file, out_n, what, base ? "the mod's file" : "the original"); return 1; } // ---- the resource size table (restbl.c) // The decompressed size of a file's bytes as the game unpacks them: the content size its zstd frame records (.zs; // the game's files and zstd's own tools write it), four times the bytes for a frame without one, else its size. static uint64_t content_size(PlatFile *f, uint64_t off, uint64_t size, const char *rel) { if (!ends_with(rel, ".zs") || !f) return size; uint8_t head[18]; uint32_t n = size < sizeof head ? (uint32_t)size : (uint32_t)sizeof head; if (read_at(f, head, n, off) != n) return size; unsigned long long c = ZSTD_getFrameContentSize(head, n); return c == ZSTD_CONTENTSIZE_ERROR ? size : c == ZSTD_CONTENTSIZE_UNKNOWN ? size * 4 : c; } // A table from its zstd bytes; NULL when they are not one. static Restbl *restbl_unpack(const void *z, size_t n) { unsigned long long raw = ZSTD_getFrameContentSize(z, n); if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > (64u << 20)) return NULL; uint8_t *d = malloc((size_t)raw ? (size_t)raw : 1); size_t got = d ? ZSTD_decompress(d, (size_t)raw, z, n) : 0; Restbl *t = d && !ZSTD_isError(got) && got == raw ? restbl_parse(d, (size_t)raw) : NULL; free(d); return t; } // ---- the members of the mods' packs (restbl_pack_raise). The game loads each member of a pack (Pack/**/*.pack.zs, a // SARC archive compressed with a dictionary of Pack/ZsDic.pack.zs) as a resource of its own, in a heap sized by the // member's own entry, so a mod that changes a member and ships no table for it (or an older one) leaves the game's // entry. A member that outgrew its entry's slack loads, then its heap runs out: TotK 1.4.3 writes through a null // pointer at main+0x21794c when the 0x68-byte object an .ainb member builds after its file no longer fits. #define PACK_MAX (256u << 20) // the largest pack taken, decompressed (AI.Global.Product.140.pack: 24 MiB) typedef struct { ZSTD_DDict *dict[4]; unsigned id[4]; int n; ZSTD_DCtx *dctx; } Packs; // The bytes of zstd frame z[0..n), with the dictionary of ps its header names; malloc'd. NULL when it is not such a // frame with a recorded size of at most PACK_MAX. static uint8_t *pack_unpack(Packs *ps, const void *z, size_t n, size_t *out_n) { unsigned long long raw = ZSTD_getFrameContentSize(z, n); unsigned id = ZSTD_getDictID_fromFrame(z, n); ZSTD_DDict *dd = NULL; for (int i = 0; i < ps->n; i++) if (ps->id[i] == id) dd = ps->dict[i]; if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > PACK_MAX || (id && !dd) || !ps->dctx) return NULL; uint8_t *d = malloc(raw ? (size_t)raw : 1); size_t got = !d ? 0 : dd ? ZSTD_decompress_usingDDict(ps->dctx, d, (size_t)raw, z, n, dd) : ZSTD_decompressDCtx(ps->dctx, d, (size_t)raw, z, n); if (!d || ZSTD_isError(got) || got != raw) { free(d); return NULL; } *out_n = (size_t)raw; return d; } static void pack_dict_add(const char *name, const uint8_t *data, size_t size, void *ctx) { Packs *ps = ctx; ZSTD_DDict *dd = ps->n < 4 && ends_with(name, ".zsdic") ? ZSTD_createDDict(data, size) : NULL; // a copy if (!dd) return; ps->dict[ps->n] = dd; ps->id[ps->n++] = ZSTD_getDictID_fromDDict(dd); } // The dictionaries of Pack/ZsDic.pack.zs as served (zs.zsdic, pack.zsdic, bcett.byml.zsdic): a SARC compressed // without one. 0 when it holds none. static int packs_open(Packs *ps) { Overlay *base; size_t n = 0, rn = 0; ps->dctx = ZSTD_createDCtx(); void *z = read_served("Pack/ZsDic.pack.zs", &base, &n, "the members of the mods' packs keep their entries"); uint8_t *raw = z ? pack_unpack(ps, z, n, &rn) : NULL; int ok = raw && restbl_sarc_each(raw, rn, pack_dict_add, ps) && ps->n; free(z); free(raw); return ok; } static void packs_close(Packs *ps) { for (int i = 0; i < ps->n; i++) ZSTD_freeDDict(ps->dict[i]); ZSTD_freeDCtx(ps->dctx); } // Overlay o (a pack) and the game's copy of it, decompressed; NULL when o cannot be read or unpacked. *orig stays NULL // for a pack the game lacks or whose copy cannot be read. static uint8_t *pack_pair(Packs *ps, const Overlay *o, size_t *n, uint8_t **orig, size_t *orig_n) { uint64_t zn = 0; void *z = plat_file_read(o->path, &zn); uint8_t *mod = z ? pack_unpack(ps, z, (size_t)zn, n) : NULL; free(z); *orig = NULL; if (!mod || o->index == ADDED) return mod; Entry e = entry(o->index); void *g = e.size < READ_CHUNK ? malloc(e.size ? (size_t)e.size : 1) : NULL; if (g && read_at(image, g, (uint32_t)e.size, e.offset) == (int64_t)e.size) *orig = pack_unpack(ps, g, (size_t)e.size, orig_n); free(g); return mod; } typedef struct { const char *pack, *mod; int lines; } PackNote; static void pack_note(const char *key, uint32_t was, uint32_t now, void *ctx) { PackNote *pn = ctx; if (pn->lines++ < 16) rt_log("RomFS: resource size table: %s in %s (%s): entry %u -> %u", key, pn->pack, pn->mod, was, now); } // The entries of the members of every pack a mod serves, raised in out against the game's table base; the number // raised, *packs the number of packs with one. static int restbl_packs(Restbl *out, const Restbl *base, int *packs, int *lines) { Packs ps = {0}; int raised = 0, opened = 0, bad = 0; *packs = 0; for (size_t k = 0; k < noverlay; k++) { const Overlay *o = &overlays[k]; if ((!o->mod && o->prio != 1) || !ends_with(o->name, ".pack.zs")) continue; if (!opened++ && !packs_open(&ps)) { rt_log("RomFS: no zstd dictionary in Pack/ZsDic.pack.zs; the members of mods' packs keep their entries"); break; } // A pack whose game copy cannot be read counts as new: each member gets an estimate, which only raises entries. size_t n = 0, orig_n = 0; uint8_t *orig, *mod = pack_pair(&ps, o, &n, &orig, &orig_n); PackNote pn = {o->name, owner_name(o), *lines}; int r = mod ? restbl_pack_raise(out, base, mod, n, orig, orig_n, pack_note, &pn) : -1; if (r < 0 && bad++ < 4) rt_log("RomFS: %s (%s) is not a pack this build reads; its members keep their entries", o->name, owner_name(o)); *lines = pn.lines; raised += r > 0 ? r : 0; *packs += r > 0; free(mod); free(orig); } packs_close(&ps); return raised; } // With a player mod (or the TOTK_ROMFS_MODS manifest) serving files: the game's table with every shipped table of this // version merged in (an entry larger than the game's wins), the entries a mod's table for another version gives for // that mod's files, and each served mod file's entry raised to what its size needs (restbl_grow for a file the game // has, restbl_estimate for an added one), the members of the packs mods serve too (restbl_packs). Never lowers the // game's entries. static void restbl_overlay(void) { int player = 0; for (size_t k = 0; k < noverlay; k++) player |= overlays[k].mod || overlays[k].prio == 1; if (!player || !restbl_rel[0]) return; static const char keeps[] = "mod files that grew may fail to load"; double t0 = now_s(); uint64_t gi = lower(restbl_rel); Entry ge = entry(gi); void *gz = ge.size < (64u << 20) ? malloc((size_t)ge.size) : NULL; Restbl *base = NULL, *out = NULL; if (gz && read_at(image, gz, (uint32_t)ge.size, ge.offset) == (int64_t)ge.size) { base = restbl_unpack(gz, (size_t)ge.size); out = restbl_unpack(gz, (size_t)ge.size); } free(gz); if (!base || !out) { rt_log("RomFS: cannot read the resource size table %s; %s", restbl_rel, keeps); restbl_free(base); restbl_free(out); return; } int tables = 0, merged = 0, raised = 0, estimated = 0; for (size_t s = 0; s < nshipped; s++) { uint64_t n = 0; void *z = plat_file_read(shipped[s].file, &n); Restbl *t = z ? restbl_unpack(z, (size_t)n) : NULL; free(z); if (!t) { rt_log("RomFS: %s is not a resource size table; left out", shipped[s].file); continue; } tables++; if (!shipped[s].other) { merged += restbl_merge(out, base, t); } else { // another version's table: only the entries of the files its mod serves for (size_t k = 0; k < noverlay; k++) { char key[4200]; const char *kp = restbl_key(overlays[k].name, key, sizeof key); if (overlays[k].mod == shipped[s].mod) merged += restbl_take(out, t, &kp, 1); } } restbl_free(t); } Overlay *served = NULL; for (size_t k = 0; k < noverlay; k++) { Overlay *o = &overlays[k]; if (o->index == gi) served = o; if ((!o->mod && o->prio != 1) || o->index == gi) continue; char key[4200]; restbl_key(o->name, key, sizeof key); int mc = ends_with(o->name, ".mc"); PlatFile *f = open_read(o->path); uint64_t now = mc ? o->size : content_size(f, 0, o->size, o->name); if (f) plat_file_close(f); uint32_t entry_size, want; if (o->index != ADDED) { if (!restbl_get(base, key, &entry_size)) continue; // the game sizes it without the table (streamed) Entry e = entry(o->index); uint64_t was = mc ? e.size : content_size(image, e.offset, e.size, o->name); want = restbl_grow(key, entry_size, was, now, mc); } else { want = restbl_estimate(key, mc ? now * 8 : now); // a mesh codec file unpacks to several times its size estimated += want != 0; } raised += want && restbl_raise(out, key, want); } int packs = 0, lines = 0, members = restbl_packs(out, base, &packs, &lines); if (lines > 16) rt_log("RomFS: resource size table: %d more entries of pack members raised", lines - 16); restbl_free(base); if (!merged && !raised && !members && !served) { restbl_free(out); rt_log("RomFS: resource size table: the mods' files fit the game's entries (%.0f ms)", (now_s() - t0) * 1000); return; } size_t rn = 0, zn = 0; uint8_t *raw = restbl_write(out, &rn); restbl_free(out); size_t cap = raw ? ZSTD_compressBound(rn) : 0; void *z = raw ? malloc(cap) : NULL; if (z) zn = ZSTD_compress(z, cap, raw, rn, 3); free(raw); if (!z || ZSTD_isError(zn)) { free(z); rt_log("RomFS: cannot pack the merged resource size table; %s", keeps); return; } if (serve_generated(restbl_rel, served, z, zn, "mods", "merged resource sizes", keeps)) rt_log("RomFS: resource size table: %d tables merged (%d entries), %d entries raised for mod files (%d " "added files estimated), %d for members of %d mod packs, %.0f ms", tables, merged, raised, estimated, members, packs, (now_s() - t0) * 1000); } // The HeapDef table a bigger game resolution or TOTK_TEX_HEAP_EXTRA_MB needs (resolution.c): its texture heaps and // their parents grow. The base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS // original; the grown table is written to /resolution/ and served in the base's place. A served mod table // whose own TextureHeap is grown (an HD texture pack's, resolution.c res_heapdef_tex_grown) is not the base: its // growth is already in the amount (resolution.c mod_tex_heap_kib). In its place the table of the highest switched-on // mod below it that changes other heaps only grows by the amount (plain_mod_heapdef: with the pack above The Second // Quest, the default order for HDPacks.com's "4kTextures", the game's own sizes for RSDB and UI were too small for // that mod's files and the game crashed loading a save), else the RomFS original; either is served in the pack's // place. A mod table that changes other heaps only and is served itself stays the base. #define HEAPDEF "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" // The HeapDef table of the highest switched-on mod that changes other heaps only (malloc'd, *n, *who its name); NULL // when no mod has one. Mods above the served table ship none, so the first found is the highest below it. static void *plain_mod_heapdef(size_t *n, const char **who) { for (int i = 0; i < mods_count(); i++) { const ModInfo *m = mods_at(i); char path[2300]; if (!m->enabled || !m->romfs[0] || snprintf(path, sizeof path, "%s/%s", m->romfs, HEAPDEF) >= (int)sizeof path) continue; uint64_t fn = 0; uint8_t *f = plat_file_read(path, &fn); if (!f) continue; if (fn < (16u << 20) && res_heapdef_plain(f, (size_t)fn)) { *n = (size_t)fn; *who = m->name; return f; } free(f); } return NULL; } static void heapdef_overlay(void) { if (!res_heap_extra_kib() && !res_tex_heap_extra_bytes() && !res_grass_heap_extra_kib()) return; static const char keeps[] = "the game keeps its own resolution"; Overlay *base; size_t n; void *in = read_served(HEAPDEF, &base, &n, keeps), *out = NULL; size_t out_n = 0; if (in && base && base->mod && res_heapdef_tex_grown(in, n)) { free(in); const char *who = NULL; if ((in = plain_mod_heapdef(&n, &who))) rt_log("RomFS: %s: the HeapDef of %s with the texture heap growth in place of the mod's table", mods_at(base->mod - 1)->name, who); else if ((in = read_original(HEAPDEF, &n, keeps))) rt_log("RomFS: %s: the game's own HeapDef with the mod's texture heap growth in place of the mod's table", mods_at(base->mod - 1)->name); } int ok = in && res_heapdef_build(in, n, &out, &out_n); free(in); if (ok && serve_generated(HEAPDEF, base, out, out_n, "resolution", "grown heaps", keeps)) res_heapdef_served(); } // The player camera's pack with its stick speeds multiplied by TOTK_STICK_SENS (camera_speed.c). The base is an // overlay that replaces the pack (a mod's camera stays), else the RomFS original; the dictionary is the RomFS's own. #define CAMPACK "Pack/Actor/PlayerCamera.pack.zs" static void camera_overlay(void) { if (cam_speed_wanted() == 1.0f) return; static const char keeps[] = "the camera keeps its own speed"; Overlay *base, *dic_base; size_t n, dn; void *in = read_served(CAMPACK, &base, &n, keeps), *out = NULL; void *dic = in ? read_served("Pack/ZsDic.pack.zs", &dic_base, &dn, keeps) : NULL; size_t out_n = 0; int ok = in && dic && cam_pack_build(in, n, dic, dn, &out, &out_n); free(in); free(dic); if (ok && serve_generated(CAMPACK, base, out, out_n, "camera", "camera speed", keeps)) cam_pack_served(); } // The button prompts' folders (input_prompts_folder) and whether each had a manifest next to romfs-index.bin at start; // the one the settings asked for at start, the one overlay_init loaded (NULL: the game's own prompts) and why the first // did not load (ROMFS_PROMPTS_*). The menu compares them with the settings in effect (ui/menu.cpp prompts_note). // Each folder keeps its name, but at a game resolution of another shape than 16:9 its files come from dir, // -x (overlay_init), and ready is that one's. static struct { const char *name; int ready; char dir[64]; } prompts_folders[3]; static const char *prompts_wanted, *prompts_loaded; static int prompts_off; const char *romfs_prompts(void) { return prompts_loaded; } int romfs_prompts_ready(const char *folder) { for (int i = 0; i < 3; i++) if (folder && prompts_folders[i].name && !strcmp(prompts_folders[i].name, folder)) return prompts_folders[i].ready; return 0; } int romfs_prompts_off(const char *folder) { return folder && prompts_wanted && !strcmp(folder, prompts_wanted) ? prompts_off : ROMFS_PROMPTS_OK; } // The first RomFS file that the overlays [a, b) and [c, d) both replace, or NULL. static const char *overlay_clash(size_t a, size_t b, size_t c, size_t d) { for (size_t i = a; i < b; i++) for (size_t j = c; j < d; j++) if (overlays[i].index != ADDED && overlays[i].index == overlays[j].index && strcmp(overlays[i].name, restbl_rel)) // restbl_overlay merges both tables return overlays[i].name; return NULL; } // Drops the overlays [a, b), the ones after them moving down. static void overlay_drop(size_t a, size_t b) { for (size_t k = a; k < b; k++) { free((void *)overlays[k].path); free((void *)overlays[k].name); } memmove(overlays + a, overlays + b, (noverlay - b) * sizeof *overlays); noverlay -= b - a; } static void overlay_init(void) { // TOTK_ROMFS_PATCHES unset: romfs-patches next to romfs-index.bin if it has a manifest (tools/make_romfs_patches.py). // A game resolution that is not 16:9 takes romfs-patches-x instead when it is there: the same overlay with // the UI laid out for that shape (make_romfs_patches.py --aspect), when the game version has the camera aspect patch // the layout goes with (res_wants_wide_overlay). // With prompts = xbox, the Xbox prompts' folder for the pad layout in its place (input_prompts_folder): the same // patches with Xbox button icons (make_romfs_patches.py --prompts); prompts = pc takes romfs-patches-pc whatever // the pad layout. The game reads the archives once at boot, so the prompts stay until restart even though the // layout itself changes live. Each folder replaces romfs-patches as a whole. At a game resolution of another shape // the prompts come from -x (make_romfs_patches.py --prompts