diff --git a/docs/tester-guide.md b/docs/tester-guide.md index 72405204..a0289283 100644 --- a/docs/tester-guide.md +++ b/docs/tester-guide.md @@ -64,6 +64,10 @@ it up or down. When two mods change the same file, the higher one wins. export location to an empty folder inside `mods/` (for example `mods/TKMM`). That folder is then one mod here. - Files made for another game version (names with `.Product..` that this version does not have) are left out; the Mods tab counts them. Look for the mod's release for your game version. +- The resource size table (`System/Resource/ResourceSizeTable.Product....rsizetable.zs`), which tells the game how + much memory each file needs, is rebuilt at every start with mods enabled: the tables the mods ship are merged and + the entries of files a mod made larger or added are raised. The log line `RomFS: resource size table: ...` says + how many. A mod's table for another game version still counts for that mod's own files. - Mods that patch the game code (`exefs/*.ips`, `exefs/*.pchtxt`) are listed but not applied yet; a mod with both kinds of files stays off as a whole until they are. - Mods with their own code (`exefs/subsdk*` or a replaced `exefs/main`, such as UltraCam) never load, their RomFS files diff --git a/runtime/restbl.c b/runtime/restbl.c new file mode 100644 index 00000000..7436ddd1 --- /dev/null +++ b/runtime/restbl.c @@ -0,0 +1,236 @@ +// The resource size table (restbl.h). Layout of a decompressed RESTBL, little-endian: +// +// "RESTBL", u32 version (1), u32 name length (160), u32 hashed count, u32 named count +// hashed entries {u32 crc32 of the key, u32 size}, sorted by the hash +// named entries {char key[name length], u32 size}: the keys whose hash collides with another key's +// +// A key is a RomFS path without .zs or .mc. The estimates come from the 1.4.3 table against its own files: most types +// need the decompressed size plus at most 8 KiB (headers, alignment); a few need a multiple of it (bfres 1.5, bkres +// 2.05, bgyml 3.85, asb 1.4, bstar 1.2, bphsc 1.1). A type the table does not list takes 1.5 times its size plus 8 KiB. +#include +#include + +#include "restbl.h" + +typedef struct { uint32_t hash, size; } Hashed; + +struct Restbl { + uint32_t version, name_len; + Hashed *hashed; + size_t nhashed, hashed_cap; + uint8_t *named; // named entries as in the file: key (name_len bytes, NUL padded) then u32 size + size_t nnamed; +}; + +static uint32_t rd32(const uint8_t *p) { return p[0] | p[1] << 8 | p[2] << 16 | (uint32_t)p[3] << 24; } +static void wr32(uint8_t *p, uint32_t v) { + for (int i = 0; i < 4; i++) p[i] = (uint8_t)(v >> 8 * i); +} + +static uint32_t crc32_of(const char *s) { + uint32_t c = ~0u; + for (; *s; s++) { + c ^= (uint8_t)*s; + for (int k = 0; k < 8; k++) c = c >> 1 ^ (0xEDB88320u & (0u - (c & 1))); + } + return ~c; +} + +Restbl *restbl_parse(const uint8_t *data, size_t n) { + if (n < 22 || memcmp(data, "RESTBL", 6)) return NULL; + uint32_t version = rd32(data + 6), name_len = rd32(data + 10), nh = rd32(data + 14), nn = rd32(data + 18); + if (!name_len || name_len > 4096 || (uint64_t)nh * 8 + (uint64_t)nn * (name_len + 4) != n - 22) return NULL; + Restbl *t = calloc(1, sizeof *t); + if (!t) return NULL; + t->version = version, t->name_len = name_len; + t->hashed = malloc(((size_t)nh + 1) * sizeof *t->hashed); + t->named = malloc((size_t)nn * (name_len + 4) + 1); + if (!t->hashed || !t->named) { + restbl_free(t); + return NULL; + } + for (uint32_t i = 0; i < nh; i++) t->hashed[i] = (Hashed){rd32(data + 22 + i * 8), rd32(data + 26 + i * 8)}; + t->nhashed = t->hashed_cap = nh; + if (!t->hashed_cap) t->hashed_cap = 1; + memcpy(t->named, data + 22 + (size_t)nh * 8, (size_t)nn * (name_len + 4)); + t->nnamed = nn; + return t; +} + +void restbl_free(Restbl *t) { + if (!t) return; + free(t->hashed); + free(t->named); + free(t); +} + +const char *restbl_key(const char *path, char *buf, size_t n) { + size_t len = strlen(path); + if (len >= 3 && (!strcmp(path + len - 3, ".zs") || !strcmp(path + len - 3, ".mc"))) len -= 3; + if (len >= n) len = n - 1; + memcpy(buf, path, len); + buf[len] = 0; + return buf; +} + +// The named entry of key, or NULL. +static uint8_t *named_of(const Restbl *t, const char *key) { + size_t len = strlen(key); + if (len >= t->name_len) return NULL; + for (size_t i = 0; i < t->nnamed; i++) { + uint8_t *e = t->named + i * (t->name_len + 4); + if (!memcmp(e, key, len + 1)) return e; + } + return NULL; +} + +// The position of hash in the hashed entries, or where it would go. +static size_t hashed_at(const Restbl *t, uint32_t hash) { + size_t lo = 0, hi = t->nhashed; + while (lo < hi) { + size_t m = lo + (hi - lo) / 2; + if (t->hashed[m].hash < hash) lo = m + 1; + else hi = m; + } + return lo; +} + +int restbl_get(const Restbl *t, const char *key, uint32_t *size) { + const uint8_t *e = named_of(t, key); + if (e) { + *size = rd32(e + t->name_len); + return 1; + } + uint32_t h = crc32_of(key); + size_t i = hashed_at(t, h); + if (i < t->nhashed && t->hashed[i].hash == h) { + *size = t->hashed[i].size; + return 1; + } + return 0; +} + +// The hashed entry of hash raised to size (added when missing); 1 when the table changed. +static int raise_hash(Restbl *t, uint32_t h, uint32_t size) { + size_t i = hashed_at(t, h); + if (i < t->nhashed && t->hashed[i].hash == h) { + if (t->hashed[i].size >= size) return 0; + t->hashed[i].size = size; + return 1; + } + if (t->nhashed == t->hashed_cap) { + size_t cap = t->hashed_cap * 2; + Hashed *grown = realloc(t->hashed, cap * sizeof *grown); + if (!grown) return 0; + t->hashed = grown, t->hashed_cap = cap; + } + memmove(t->hashed + i + 1, t->hashed + i, (t->nhashed - i) * sizeof *t->hashed); + t->hashed[i] = (Hashed){h, size}; + t->nhashed++; + return 1; +} + +int restbl_raise(Restbl *t, const char *key, uint32_t size) { + uint8_t *e = named_of(t, key); + if (!e) return raise_hash(t, crc32_of(key), size); + if (rd32(e + t->name_len) >= size) return 0; + wr32(e + t->name_len, size); + return 1; +} + +int restbl_merge(Restbl *t, const Restbl *base, const Restbl *shipped) { + int raised = 0; + for (size_t i = 0; i < shipped->nhashed; i++) { + Hashed s = shipped->hashed[i]; + size_t k = hashed_at(base, s.hash); + if (k < base->nhashed && base->hashed[k].hash == s.hash && base->hashed[k].size >= s.size) continue; + raised += raise_hash(t, s.hash, s.size); + } + if (shipped->name_len != t->name_len) return raised; + for (size_t i = 0; i < shipped->nnamed; i++) { + const uint8_t *e = shipped->named + i * (shipped->name_len + 4); + char key[4097]; + memcpy(key, e, shipped->name_len); + key[shipped->name_len] = 0; + uint32_t size = rd32(e + shipped->name_len), have; + if (restbl_get(base, key, &have) && have >= size) continue; + uint8_t *mine = named_of(t, key); + if (mine) { + if (rd32(mine + t->name_len) < size) wr32(mine + t->name_len, size), raised++; + continue; + } + uint8_t *grown = realloc(t->named, (t->nnamed + 1) * (t->name_len + 4)); + if (!grown) return raised; + t->named = grown; + memcpy(t->named + t->nnamed * (t->name_len + 4), e, t->name_len + 4); + t->nnamed++; + raised++; + } + return raised; +} + +int restbl_take(Restbl *t, const Restbl *from, const char *const *keys, size_t n) { + int raised = 0; + for (size_t i = 0; i < n; i++) { + uint32_t size; + if (restbl_get(from, keys[i], &size)) raised += restbl_raise(t, keys[i], size); + } + return raised; +} + +uint8_t *restbl_write(const Restbl *t, size_t *n) { + size_t size = 22 + t->nhashed * 8 + t->nnamed * (t->name_len + 4); + uint8_t *out = malloc(size); + if (!out) return NULL; + memcpy(out, "RESTBL", 6); + wr32(out + 6, t->version); + wr32(out + 10, t->name_len); + wr32(out + 14, (uint32_t)t->nhashed); + wr32(out + 18, (uint32_t)t->nnamed); + for (size_t i = 0; i < t->nhashed; i++) { + wr32(out + 22 + i * 8, t->hashed[i].hash); + wr32(out + 26 + i * 8, t->hashed[i].size); + } + memcpy(out + 22 + t->nhashed * 8, t->named, t->nnamed * (t->name_len + 4)); + *n = size; + return out; +} + +// ---- estimates + +// The factor a type's entry needs over its decompressed size (1000 = 1.0); 0 for a type the table never lists. +static unsigned type_factor(const char *key) { + static const struct { const char *ext; unsigned f; } types[] = { + {"bfres", 1500}, {"bkres", 2050}, {"bgyml", 3850}, {"asb", 1400}, {"bstar", 1200}, {"bphsc", 1100}, + {"byml", 1000}, {"txtg", 1000}, {"pack", 1000}, {"bntx", 1000}, {"bphsh", 1000}, {"vsts", 1000}, + {"ainb", 1000}, {"bphnm", 1000}, {"bars", 1000}, {"chunk", 1000}, {"bfevfl", 1000}, {"quad", 1000}, + {"baev", 1000}, {"bcul", 1000}, {"crbin", 1000}, {"wbr", 1000}, {"bhtmp", 1000}, {"dpi", 1000}, + {"cai", 1000}, {"png", 1000}, {"cutinfo", 1000}, {"blwp", 1000}, {"sarc", 1000}, {"tscb", 1000}, + {"bfsha", 1000}, {"jpg", 1000}, {"beco", 1000}, {"bfarc", 1000}, {"blarc", 1000}, {"belnk", 1000}, + {"genvb", 1000}, {"bnsh", 1000}, {"bslnk", 1000}, {"baatarc", 1000}, {"bagst", 1000}, + {"bwav", 0}, {"webm", 0}, + }; + const char *leaf = strrchr(key, '/') ? strrchr(key, '/') + 1 : key, *dot = strrchr(leaf, '.'); + if (!dot) return 1500; + for (size_t i = 0; i < sizeof types / sizeof *types; i++) + if (!strcmp(dot + 1, types[i].ext)) return types[i].f; + return 1500; +} + +static uint32_t clamp32(uint64_t v) { return v > 0xFFFFFFF0u ? 0xFFFFFFF0u : (uint32_t)v; } + +uint32_t restbl_estimate(const char *key, uint64_t size) { + unsigned f = type_factor(key); + if (!f) return 0; + uint64_t e = (size * f + 999) / 1000 + 8192; + return clamp32((e + 4095) & ~(uint64_t)4095); +} + +uint32_t restbl_grow(const char *key, uint32_t entry, uint64_t old, uint64_t new_size, int scaled) { + if (new_size <= old) return entry; + if (scaled) return clamp32(old ? ((uint64_t)entry * new_size + old - 1) / old : restbl_estimate(key, new_size * 8)); + unsigned f = type_factor(key); + if (!f) return entry; + uint64_t e = entry + ((new_size - old) * f + 999) / 1000; + return clamp32((e + 31) & ~(uint64_t)31); +} diff --git a/runtime/restbl.h b/runtime/restbl.h new file mode 100644 index 00000000..430480f5 --- /dev/null +++ b/runtime/restbl.h @@ -0,0 +1,33 @@ +// The game's resource size table (restbl.c): System/Resource/ResourceSizeTable.Product..Nin_NX_NVN.rsizetable.zs, a +// zstd-compressed RESTBL. The game sizes the buffer it loads a file into from the file's entry, so a mod file larger +// than the original needs a larger entry, and a file a mod adds needs one of its own. romfs.c merges the tables the +// mods ship and raises the entries of the files they serve (romfs.c restbl_overlay). +#pragma once +#include +#include + +typedef struct Restbl Restbl; + +// The table of a decompressed RESTBL (copied); NULL when it is not one. +Restbl *restbl_parse(const uint8_t *data, size_t n); +void restbl_free(Restbl *t); +// The table's key of a RomFS path: the path without a trailing .zs or .mc, into buf. +const char *restbl_key(const char *path, char *buf, size_t n); +// 1 with *size when key has an entry. +int restbl_get(const Restbl *t, const char *key, uint32_t *size); +// The entry of key raised to size (added when missing); 1 when the table changed. +int restbl_raise(Restbl *t, const char *key, uint32_t size); +// Every entry of shipped larger than base's (or missing from base) raised in t; the number of entries raised. +int restbl_merge(Restbl *t, const Restbl *base, const Restbl *shipped); +// Each entry of from whose key one of keys[0..n) names, raised in t; the number raised. +int restbl_take(Restbl *t, const Restbl *from, const char *const *keys, size_t n); +// The table as a decompressed RESTBL (malloc'd); NULL when out of memory. +uint8_t *restbl_write(const Restbl *t, size_t *n); + +// The entry a file of the key's type and decompressed size needs when the game has none for it; 0 for a type the +// table never lists (streamed sound and video). +uint32_t restbl_estimate(const char *key, uint64_t size); +// The entry of a file the game lists with `entry` for its own copy of old bytes, now new bytes: raised by the growth +// times the type's factor, never lowered. scaled: the sizes are compressed ones (a mesh codec .mc file), whose entry +// grows in proportion. +uint32_t restbl_grow(const char *key, uint32_t entry, uint64_t old, uint64_t new_size, int scaled); diff --git a/runtime/romfs.c b/runtime/romfs.c index 2e673b4f..1f214f91 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -22,6 +22,10 @@ // hash, no manifest). A path several overlays give is served by one: a player mod higher in the order beats a lower // one, any player mod beats the TOTK_ROMFS_MODS manifest, which beats romfs-patches (overlay_contest). A mod path that // matches a RomFS path only without case is served under the RomFS spelling. +// +// The resource size table: with a player mod enabled, the game's table is rebuilt at start with the tables the mods +// ship merged in and the entries of the mods' files raised to what they need (restbl_overlay), and served from +// /mods/ in place of the game's or a mod's table. #include #include #include @@ -35,8 +39,11 @@ #include "romfs_config.h" #include "romfs_path.h" #include "platform/platform.h" +#include "restbl.h" #include "sha256.h" +#include + #define NH 65536 // RomFS handle slots #define READ_CHUNK (64u << 20) // largest single read #define HASH_CHUNK (8u << 20) @@ -342,6 +349,51 @@ static int other_version(const char *path) { return 0; } +// ---- the resource size table's inputs: the tables the overlays carry, before overlay_contest keeps one of them +static char restbl_rel[256]; // the RomFS path of the game's table, "" when it has none +typedef struct { + char *file; + int mod, other; // mod: ModInfo index + 1, 0 for the port's folders; other: made for another game version +} Shipped; +static Shipped *shipped; +static size_t nshipped; + +static int ends_with(const char *s, const char *tail) { + size_t n = strlen(s), k = strlen(tail); + return n >= k && !strcmp(s + n - k, tail); +} + +// 1 when rel is a resource size table of any game version. +static int is_restbl(const char *rel) { + return !strncmp(rel, "System/Resource/ResourceSizeTable.", 34) && ends_with(rel, ".rsizetable.zs"); +} + +static void shipped_add(const char *file, int mod, int other) { + Shipped *grown = realloc(shipped, (nshipped + 1) * sizeof *shipped); + char *copy = strdup(file); + need(grown && copy, "resource size table list"); + shipped = grown; + shipped[nshipped++] = (Shipped){copy, mod, other}; +} + +// The game's table: the one System/Resource/ResourceSizeTable.*.rsizetable.zs of the RomFS. +static void restbl_find(void) { + for (uint64_t i = lower("System/Resource/ResourceSizeTable."); i < file_count; i++) { + const char *n = entry_name(entry(i)); + if (strncmp(n, "System/Resource/ResourceSizeTable.", 34)) break; + if (is_restbl(n) && strlen(n) < sizeof restbl_rel) { + snprintf(restbl_rel, sizeof restbl_rel, "%s", n); + return; + } + } +} + +// The tables the overlays carry for this game version, every one of them (overlay_contest keeps one). +static void restbl_collect(void) { + for (size_t k = 0; k < noverlay; k++) + if (restbl_rel[0] && !strcmp(overlays[k].name, restbl_rel)) shipped_add(overlays[k].path, overlays[k].mod, 0); +} + typedef struct { int k, prio; const char *dir, *rel; } ModWalk; static void mod_walk(int k, int prio, const char *dir, const char *rel); @@ -368,6 +420,7 @@ static int mod_walk_cb(const char *leaf, int kind, uint64_t size, void *ctx) { } uint64_t i = index_any_case(path); if (i == UINT64_MAX && other_version(path)) { + if (is_restbl(path)) shipped_add(file, w->k + 1, 1); // its entries for this mod's files still apply m->other_version++; return 1; } @@ -544,6 +597,123 @@ static int serve_generated(const char *rel, Overlay *base, void *out, size_t out return 1; } +// ---- the resource size table (restbl.c) + +// The decompressed size of a file's bytes as the game unpacks them: the content size its zstd frame records (.zs; +// the game's files and zstd's own tools write it), four times the bytes for a frame without one, else its size. +static uint64_t content_size(PlatFile *f, uint64_t off, uint64_t size, const char *rel) { + if (!ends_with(rel, ".zs") || !f) return size; + uint8_t head[18]; + uint32_t n = size < sizeof head ? (uint32_t)size : (uint32_t)sizeof head; + if (read_at(f, head, n, off) != n) return size; + unsigned long long c = ZSTD_getFrameContentSize(head, n); + return c == ZSTD_CONTENTSIZE_ERROR ? size : c == ZSTD_CONTENTSIZE_UNKNOWN ? size * 4 : c; +} + +// A table from its zstd bytes; NULL when they are not one. +static Restbl *restbl_unpack(const void *z, size_t n) { + unsigned long long raw = ZSTD_getFrameContentSize(z, n); + if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > (64u << 20)) return NULL; + uint8_t *d = malloc((size_t)raw ? (size_t)raw : 1); + size_t got = d ? ZSTD_decompress(d, (size_t)raw, z, n) : 0; + Restbl *t = d && !ZSTD_isError(got) && got == raw ? restbl_parse(d, (size_t)raw) : NULL; + free(d); + return t; +} + +// With a player mod (or the TOTK_ROMFS_MODS manifest) serving files: the game's table with every shipped table of this +// version merged in (an entry larger than the game's wins), the entries a mod's table for another version gives for +// that mod's files, and each served mod file's entry raised to what its size needs (restbl_grow for a file the game +// has, restbl_estimate for an added one). Never lowers the game's entries. +static void restbl_overlay(void) { + int player = 0; + for (size_t k = 0; k < noverlay; k++) player |= overlays[k].mod || overlays[k].prio == 1; + if (!player || !restbl_rel[0]) return; + static const char keeps[] = "mod files that grew may fail to load"; + double t0 = now_s(); + uint64_t gi = lower(restbl_rel); + Entry ge = entry(gi); + void *gz = ge.size < (64u << 20) ? malloc((size_t)ge.size) : NULL; + Restbl *base = NULL, *out = NULL; + if (gz && read_at(image, gz, (uint32_t)ge.size, ge.offset) == (int64_t)ge.size) { + base = restbl_unpack(gz, (size_t)ge.size); + out = restbl_unpack(gz, (size_t)ge.size); + } + free(gz); + if (!base || !out) { + rt_log("RomFS: cannot read the resource size table %s; %s", restbl_rel, keeps); + restbl_free(base); + restbl_free(out); + return; + } + int tables = 0, merged = 0, raised = 0, estimated = 0; + for (size_t s = 0; s < nshipped; s++) { + uint64_t n = 0; + void *z = plat_file_read(shipped[s].file, &n); + Restbl *t = z ? restbl_unpack(z, (size_t)n) : NULL; + free(z); + if (!t) { + rt_log("RomFS: %s is not a resource size table; left out", shipped[s].file); + continue; + } + tables++; + if (!shipped[s].other) { + merged += restbl_merge(out, base, t); + } else { // another version's table: only the entries of the files its mod serves + for (size_t k = 0; k < noverlay; k++) { + char key[4200]; + const char *kp = restbl_key(overlays[k].name, key, sizeof key); + if (overlays[k].mod == shipped[s].mod) merged += restbl_take(out, t, &kp, 1); + } + } + restbl_free(t); + } + Overlay *served = NULL; + for (size_t k = 0; k < noverlay; k++) { + Overlay *o = &overlays[k]; + if (o->index == gi) served = o; + if ((!o->mod && o->prio != 1) || o->index == gi) continue; + char key[4200]; + restbl_key(o->name, key, sizeof key); + int mc = ends_with(o->name, ".mc"); + PlatFile *f = open_read(o->path); + uint64_t now = mc ? o->size : content_size(f, 0, o->size, o->name); + if (f) plat_file_close(f); + uint32_t entry_size, want; + if (o->index != ADDED) { + if (!restbl_get(base, key, &entry_size)) continue; // the game sizes it without the table (streamed) + Entry e = entry(o->index); + uint64_t was = mc ? e.size : content_size(image, e.offset, e.size, o->name); + want = restbl_grow(key, entry_size, was, now, mc); + } else { + want = restbl_estimate(key, mc ? now * 8 : now); // a mesh codec file unpacks to several times its size + estimated += want != 0; + } + raised += want && restbl_raise(out, key, want); + } + restbl_free(base); + if (!merged && !raised && !served) { + restbl_free(out); + rt_log("RomFS: resource size table: the mods' files fit the game's entries (%.0f ms)", (now_s() - t0) * 1000); + return; + } + size_t rn = 0, zn = 0; + uint8_t *raw = restbl_write(out, &rn); + restbl_free(out); + size_t cap = raw ? ZSTD_compressBound(rn) : 0; + void *z = raw ? malloc(cap) : NULL; + if (z) zn = ZSTD_compress(z, cap, raw, rn, 3); + free(raw); + if (!z || ZSTD_isError(zn)) { + free(z); + rt_log("RomFS: cannot pack the merged resource size table; %s", keeps); + return; + } + if (serve_generated(restbl_rel, served, z, zn, "mods", "merged resource sizes", keeps)) + rt_log("RomFS: resource size table: %d tables merged (%d entries), %d entries raised for mod files (%d " + "added files estimated), %.0f ms", tables, merged, raised, estimated, (now_s() - t0) * 1000); +} + // The HeapDef table a bigger game resolution needs (resolution.c): its texture heap and that heap's parent grow. The // base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS original; the grown table is // written to /resolution/ and served in the base's place. @@ -600,7 +770,9 @@ int romfs_prompts_off(const char *folder) { static const char *overlay_clash(size_t a, size_t b, size_t c, size_t d) { for (size_t i = a; i < b; i++) for (size_t j = c; j < d; j++) - if (overlays[i].index != ADDED && overlays[i].index == overlays[j].index) return overlays[i].name; + if (overlays[i].index != ADDED && overlays[i].index == overlays[j].index && + strcmp(overlays[i].name, restbl_rel)) // restbl_overlay merges both tables + return overlays[i].name; return NULL; } @@ -624,8 +796,8 @@ static void overlay_init(void) { // the prompts stay until restart even though the layout itself changes live. Each folder replaces romfs-patches as // a whole and the setup builds no folder of another shape with Xbox prompts, so a game resolution that is not 16:9 // keeps the Switch prompts. So does a RomFS mod (the TOTK_ROMFS_MODS manifest or a player mod) that replaces one of - // the prompts' files (the fonts, the resource size table): romfs-patches loads in their place, so the mod's file - // is not served beside prompts files made for the game's own. + // the prompts' fonts: romfs-patches loads in their place, so the mod's file is not served beside prompts files made + // for the game's own. A mod's resource size table is no clash: restbl_overlay merges it with the prompts' one. char def[2100], path[2100]; const char *patches = cfg_get("TOTK_ROMFS_PATCHES"), *mods = cfg_get("TOTK_ROMFS_MODS"); int wide = 0; @@ -655,6 +827,7 @@ static void overlay_init(void) { : patches ? ROMFS_PROMPTS_PATCHES : !romfs_prompts_ready(prompts_wanted) ? ROMFS_PROMPTS_MISSING : ROMFS_PROMPTS_OK; + restbl_find(); size_t first = noverlay; if (prompts_wanted && prompts_off == ROMFS_PROMPTS_OK) { snprintf(path, sizeof path, "%s/%s", index_dir, prompts_wanted); // fits: its manifest path did @@ -689,8 +862,10 @@ static void overlay_init(void) { prompts_off = ROMFS_PROMPTS_MODS; if (plain_ok) overlay_load(plain, 0, 0); } + restbl_collect(); overlay_contest(); mods_overlay_log(); + restbl_overlay(); heapdef_overlay(); camera_overlay(); added_init(); diff --git a/tests/mods_romfs_check.py b/tests/mods_romfs_check.py index bbd5efd9..c3eb5b1b 100644 --- a/tests/mods_romfs_check.py +++ b/tests/mods_romfs_check.py @@ -5,16 +5,22 @@ Builds a mods folder in a temp directory with files of its own bytes at real RomFS paths (from build/extracted//romfs/index.json), runs `totk.exe --probe-fs` on it and reads the `mods:` lines of the log. --probe-fs compares the bytes it reads through the RomFS with every overlay file, so a wrong winner fails it too. +A second folder checks the merged resource size table the start writes to the profile (runtime/romfs.c +restbl_overlay): a grown file's entry, an added file's, a shipped table's and another version's table's. Needs the extracted RomFS of the version. """ import argparse import json import os +import struct import subprocess import sys import tempfile +import zlib from pathlib import Path +import zstandard + ROOT = Path(__file__).resolve().parents[1] UNI = 'Ж мод' @@ -31,19 +37,94 @@ def pick_paths(version): return p1, p2, p1case -def put(root, mod, rel, data, part='romfs'): +def put(root, mod, rel, data, part='romfs', times=64): f = Path(root) / mod / part / rel f.parent.mkdir(parents=True, exist_ok=True) - f.write_bytes(data * 64) + f.write_bytes(data * times) def run(exe, mods, order, disabled=''): + """--probe-fs with a profile of its own beside mods (the merged resource size table is written there).""" env = dict(os.environ, TOTK_MODS_DIR=str(mods), TOTK_MODS_ORDER=order, TOTK_MODS_DISABLED=disabled) - r = subprocess.run([str(exe), '--probe-fs'], env=env, capture_output=True, timeout=600) + profile = Path(mods).parent / 'profile' + r = subprocess.run([str(exe), '--profile', str(profile), '--probe-fs'], env=env, capture_output=True, timeout=600) log = (r.stdout + r.stderr).decode('utf-8', 'replace') return r.returncode, log +# ---- the resource size table (RESTBL: "RESTBL", u32 version, u32 name length, u32 hashed count, u32 named count, +# {u32 crc32, u32 size} sorted, then {char[name length], u32 size}) +def restbl_read(data): + raw = zstandard.ZstdDecompressor().decompress(data) + assert raw[:6] == b'RESTBL' + _, name_len, hashed, named = struct.unpack_from('<4I', raw, 6) + out = dict(struct.unpack_from(' size, existing hashes only).""" + out = bytearray(raw) + hashed = struct.unpack_from('= g + 100000, f'a file grown by 100000 bytes: entry {g} -> {merged[key(grown)]}') + added = merged.get(key('Pack/Actor/ModTest_Added.pack.zs'), 0) + check(added >= 50000 + 256, f'an added 50000-byte pack gets an entry ({added})') + check(merged[key(shipped_key)] == game[key(shipped_key)] + 12345, 'a shipped table\'s larger entry is merged') + check(merged[key(old_key)] == game[key(old_key)] + 7777, 'another version\'s table raises its own mod\'s file') + check(merged[key(old_other)] == game[key(old_other)], 'another version\'s table leaves other files alone') + lower = [h for h, s in game.items() if merged.get(h, 0) < s] + check(not lower, f'no entry of the game\'s table is lowered ({len(lower)})') + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--exe', type=Path, required=True) @@ -87,6 +168,8 @@ def main(): check(code == 0, 'probe-fs passes with Top switched off', log) check('mods: Low: 3 files served (2 replaced, 1 added), 0 overridden' in log, 'Low serves P1 and the added file', log) check('mods: Top: ' not in log.replace('mods: Top: off', ''), 'a switched-off mod serves nothing', log) + + restbl_case(args.exe, args.version, check, tmp) print('mods_romfs_check:', 'FAIL' if fails else 'ok') return 1 if fails else 0