From b672c6498fb84f42beaa731f0b2c5ee6d1272ad9 Mon Sep 17 00:00:00 2001 From: Eric Dallo Date: Thu, 5 Nov 2026 10:49:55 -0300 Subject: [PATCH] Raise the size table entries of the files inside the mods' packs, whose heaps ran out when the game loaded them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The game loads each member of a pack (Pack/**/*.pack.zs) as a resource of its own, in a heap sized by that member's own entry in the resource size table. The port raised the entries of the mods' loose files only, so a mod that changed a file inside a pack and shipped no table for it left the game's entry. A crash when heavy snow started in Hebra was that: an .ainb member of about 13 KB loaded into a heap sized for the game's smaller copy, 0x48 bytes stayed free after the file, the frame heap gave null for the 0x68-byte object the resource builds next, and main+0x21794c stored that object's vtable at address 0. restbl_overlay now unpacks each pack a mod serves and the game's copy of it with the dictionaries of Pack/ZsDic.pack.zs, and raises each member's entry as it does a loose file's: by the growth for a member the game's copy has, by the estimate of its type for one it lacks. It never lowers an entry, a mod's own table included, and the log names the first members it raised. --probe-mods checks restbl_pack_raise on archives of its own, and docs/mods.md says that the files inside packs count too. 🤖 Generated with [ECA](https://eca.dev) (nubank-anthropic/opus-5.5 - max) Co-Authored-By: eca-agent --- docs/mods.md | 5 +- runtime/main.c | 3 +- runtime/mods_probe.c | 7 ++- runtime/restbl.c | 99 ++++++++++++++++++++++++++++++ runtime/restbl.h | 17 ++++++ runtime/restbl_probe.c | 133 +++++++++++++++++++++++++++++++++++++++++ runtime/romfs.c | 125 ++++++++++++++++++++++++++++++++++++-- 7 files changed, 380 insertions(+), 9 deletions(-) create mode 100644 runtime/restbl_probe.c diff --git a/docs/mods.md b/docs/mods.md index 72f9eca6..d26553db 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -48,7 +48,10 @@ Two mods that edit the same game file do not combine: the higher one wins the wh (`.tkcl` files) go the same way. The port merges the resource size table on its own: at every start with mods on, it combines the tables the mods ship -and raises the entries for files a mod made larger or added. A separate RESTBL mod is not needed. +and raises the entries for files a mod made larger or added, the files inside a mod's packs (`Pack\**\*.pack.zs`) +included. A separate RESTBL mod is not needed. The table tells the game how much memory each file needs: a file that +outgrew its entry crashes the game when it loads, which can be long after the start, when the game first needs it. The +log line `RomFS: resource size table: ...` says how many entries were raised and names the first files inside packs. ## Code patches (ExeFS) diff --git a/runtime/main.c b/runtime/main.c index c23fc49c..720896f8 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -2,7 +2,8 @@ // // totk.exe [--exefs DIR] [--romfs DIR] [--profile DIR] // --selftest layout, dispatch table and host-function registry; needs no game files -// --probe-mods the mods folder scan, order and switches (mods.c); needs no game files +// --probe-mods the mods folder scan, order and switches (mods.c), the size table entries of the members of +// mods' packs (restbl_probe.c); needs no game files // --probe-loader load and relocate the modules, check them against the measured 1.2.1 values // --probe-kernel, --probe-ipc, --probe-services, --probe-fs host probes // --probe-discord the Rich Presence IPC exchange; skips without a Discord client; no game files diff --git a/runtime/mods_probe.c b/runtime/mods_probe.c index 221cb2d7..e576ee84 100644 --- a/runtime/mods_probe.c +++ b/runtime/mods_probe.c @@ -1,5 +1,6 @@ // --probe-mods: the mods folder scan, its order and switches (mods.c). Builds a mods folder of its own next to totk.exe, -// scans it and removes it. Needs no game files. +// scans it and removes it. Also the resource size table's entries of the members of the mods' packs (restbl_probe.c). +// Needs no game files. #include #include @@ -7,6 +8,7 @@ #include "host.h" #include "mods.h" #include "platform/platform.h" +#include "restbl.h" static int fails; #define CHECK(c, what) do { if (!(c)) { fails++; fprintf(stderr, "FAIL mods: %s\n", what); } } while (0) @@ -126,5 +128,6 @@ int probe_mods(void) { } remove_tree(root); printf("mods: %s\n", fails ? "FAIL" : "ok"); - return fails ? 1 : 0; + int table = restbl_probe(); + return fails || table ? 1 : 0; } diff --git a/runtime/restbl.c b/runtime/restbl.c index 7436ddd1..3530f1d7 100644 --- a/runtime/restbl.c +++ b/runtime/restbl.c @@ -234,3 +234,102 @@ uint32_t restbl_grow(const char *key, uint32_t entry, uint64_t old, uint64_t new uint64_t e = entry + ((new_size - old) * f + 999) / 1000; return clamp32((e + 31) & ~(uint64_t)31); } + +// ---- packs +// SARC: "SARC", u16 header size, u16 byte order mark 0xfeff, u32 archive size, u32 data offset, u16 version, u16 pad; +// SFAT: "SFAT", u16 header size, u16 node count, u32 hash key, then per node {u32 name hash, u32 attributes (bit 24: it +// has a name, bits 0-23: the name's offset in SFNT / 4), u32 data start, u32 data end} with the data offsets relative +// to the archive's data offset; SFNT: "SFNT", u16 header size, u16 pad, then the NUL-terminated names. +static uint32_t rd16(const uint8_t *p) { return p[0] | p[1] << 8; } + +int restbl_sarc_each(const uint8_t *p, size_t n, RestblMember *each, void *ctx) { + if (n < 0x14 || memcmp(p, "SARC", 4) || rd16(p + 6) != 0xfeff) return 0; + size_t head = rd16(p + 4), data = rd32(p + 12); + if (head + 12 > n || memcmp(p + head, "SFAT", 4) || data > n) return 0; + size_t count = rd16(p + head + 6), nodes = head + rd16(p + head + 4), names = nodes + count * 16; + if (names + 8 > n || memcmp(p + names, "SFNT", 4) || rd16(p + names + 4) < 8) return 0; + names += rd16(p + names + 4); + for (size_t i = 0; i < count; i++) { + const uint8_t *e = p + nodes + i * 16; + uint32_t attr = rd32(e + 4), lo = rd32(e + 8), hi = rd32(e + 12); + if (!(attr >> 24)) continue; // without a name: nothing the game looks up by path + size_t s = names + (size_t)(attr & 0xffffff) * 4; + if (s >= n || !memchr(p + s, 0, n - s) || lo > hi || hi > n - data) return 0; + each((const char *)p + s, p + data + lo, hi - lo, ctx); + } + return 1; +} + +// The decompressed size a zstd frame p[0..n) records (Frame_Content_Size), or 4 times n when it records none, as +// romfs.c content_size takes a loose file's. +static uint64_t frame_content_size(const uint8_t *p, size_t n) { + static const uint8_t did_bytes[4] = {0, 1, 2, 4}; + if (n < 6 || rd32(p) != 0xFD2FB528u) return n; + uint8_t fhd = p[4]; + int single = fhd >> 5 & 1, fcs = fhd >> 6; + size_t at = 5 + !single + did_bytes[fhd & 3], len = fcs ? 1u << fcs : single; + if (!len || at + len > n) return (uint64_t)n * 4; + uint64_t v = 0; + for (size_t i = 0; i < len; i++) v |= (uint64_t)p[at + i] << 8 * i; + return len == 2 ? v + 256 : v; +} + +typedef struct { const char *name; uint64_t size; } Member; +typedef struct { Member *m; size_t n, cap; int full; } Members; + +static int ends_with(const char *s, const char *tail) { + size_t n = strlen(s), k = strlen(tail); + return n >= k && !strcmp(s + n - k, tail); +} + +// A member's size as the game unpacks it: a .zs member's content size, else its bytes (a .mc member's compressed ones). +static void member_add(const char *name, const uint8_t *data, size_t size, void *ctx) { + Members *ms = ctx; + if (ms->n == ms->cap) { + size_t cap = ms->cap ? ms->cap * 2 : 64; + Member *grown = realloc(ms->m, cap * sizeof *grown); + if (!grown) { + ms->full = 1; + return; + } + ms->m = grown, ms->cap = cap; + } + ms->m[ms->n++] = (Member){name, ends_with(name, ".zs") ? frame_content_size(data, size) : size}; +} + +static int by_name(const void *a, const void *b) { + return strcmp(((const Member *)a)->name, ((const Member *)b)->name); +} + +int restbl_pack_raise(Restbl *t, const Restbl *base, const uint8_t *mod, size_t mod_n, const uint8_t *orig, + size_t orig_n, RestblNote *note, void *ctx) { + Members m = {0}, o = {0}; + int raised = -1; + if (!restbl_sarc_each(mod, mod_n, member_add, &m) || m.full || + (orig && (!restbl_sarc_each(orig, orig_n, member_add, &o) || o.full))) + goto done; + if (o.n) qsort(o.m, o.n, sizeof *o.m, by_name); + raised = 0; + for (size_t i = 0; i < m.n; i++) { + char key[1024]; + uint32_t entry, before = 0, want; + restbl_key(m.m[i].name, key, sizeof key); + int mc = ends_with(m.m[i].name, ".mc"); + const Member *was = o.n ? bsearch(&m.m[i], o.m, o.n, sizeof *o.m, by_name) : NULL; + if (was) { + if (!restbl_get(base, key, &entry)) continue; // the game sizes it without the table (streamed) + want = restbl_grow(key, entry, was->size, m.m[i].size, mc); + } else { // new to this pack: no copy of the game's to grow from + want = restbl_estimate(key, mc ? m.m[i].size * 8 : m.m[i].size); + } + restbl_get(t, key, &before); + if (want && restbl_raise(t, key, want)) { + raised++; + if (note) note(key, before, want, ctx); + } + } +done: + free(m.m); + free(o.m); + return raised; +} diff --git a/runtime/restbl.h b/runtime/restbl.h index 430480f5..359d1713 100644 --- a/runtime/restbl.h +++ b/runtime/restbl.h @@ -31,3 +31,20 @@ uint32_t restbl_estimate(const char *key, uint64_t size); // times the type's factor, never lowered. scaled: the sizes are compressed ones (a mesh codec .mc file), whose entry // grows in proportion. uint32_t restbl_grow(const char *key, uint32_t entry, uint64_t old, uint64_t new_size, int scaled); + +// The members of the SARC archive p[0..n): each(name, data, size, ctx) for every named member, in the archive's order. +// 0 when p is not a SARC archive or a member lies outside it. +typedef void RestblMember(const char *name, const uint8_t *data, size_t size, void *ctx); +int restbl_sarc_each(const uint8_t *p, size_t n, RestblMember *each, void *ctx); + +// A pack (Pack/**/*.pack.zs, decompressed: a SARC archive) whose members the game loads as resources of their own, each +// sized by its own entry. The entries of a mod's pack `mod` raised in t against the game's table `base` and the game's +// copy of the pack `orig` (NULL for a pack the game lacks): a member that grew by restbl_grow, a member the game's copy +// lacks by restbl_estimate, the others left as they are. note(key, entry before, entry after, ctx) for each raised +// entry. The number raised; -1 when either pack is not a SARC archive. +typedef void RestblNote(const char *key, uint32_t was, uint32_t now, void *ctx); +int restbl_pack_raise(Restbl *t, const Restbl *base, const uint8_t *mod, size_t mod_n, const uint8_t *orig, + size_t orig_n, RestblNote *note, void *ctx); + +// restbl_probe.c, part of --probe-mods: the number of failed checks. +int restbl_probe(void); diff --git a/runtime/restbl_probe.c b/runtime/restbl_probe.c new file mode 100644 index 00000000..3fdd9b17 --- /dev/null +++ b/runtime/restbl_probe.c @@ -0,0 +1,133 @@ +// The entries of the members of the mods' packs (restbl.c restbl_pack_raise), on archives and tables of its own; part +// of --probe-mods (mods_probe.c). Needs no game files. +#include +#include +#include + +#include "restbl.h" + +static int fails; +#define CHECK(c, what) do { if (!(c)) { fails++; fprintf(stderr, "FAIL restbl: %s\n", what); } } while (0) + +// A member of a probe archive: its name and size; a .zs member is a zstd frame header recording `content` bytes: +// fcs 4, a frame with a window descriptor and a 4-byte field; 2, a single-segment frame with a 2-byte field holding +// content - 256. +typedef struct { const char *name; size_t size, content; int fcs; } Member; + +static void wr16(uint8_t *p, unsigned v) { p[0] = (uint8_t)v, p[1] = (uint8_t)(v >> 8); } +static void wr32(uint8_t *p, uint32_t v) { wr16(p, v & 0xffff), wr16(p + 2, v >> 16); } + +// A SARC archive of the members m[0..n); unnamed: the index of a member without a name, -1 for none. malloc'd. +static uint8_t *sarc(const Member *m, int n, int unnamed, size_t *out_n) { + size_t names = 0, data = 0; + for (int i = 0; i < n; i++) names += (strlen(m[i].name) + 4) & ~(size_t)3, data += (m[i].size + 7) & ~(size_t)7; + size_t sfat = 0x14, sfnt = sfat + 12 + 16 * (size_t)n, start = (sfnt + 8 + names + 15) & ~(size_t)15; + size_t total = start + data; + uint8_t *p = calloc(1, total); + if (!p) return NULL; + memcpy(p, "SARC", 4), wr16(p + 4, 0x14), wr16(p + 6, 0xfeff); + wr32(p + 8, (uint32_t)total), wr32(p + 12, (uint32_t)start); + memcpy(p + sfat, "SFAT", 4), wr16(p + sfat + 4, 12), wr16(p + sfat + 6, (unsigned)n), wr32(p + sfat + 8, 0x65); + memcpy(p + sfnt, "SFNT", 4), wr16(p + sfnt + 4, 8); + size_t name_at = 0, data_at = 0; + for (int i = 0; i < n; i++) { + uint8_t *node = p + sfat + 12 + 16 * (size_t)i, *d = p + start + data_at; + wr32(node + 4, i == unnamed ? 0 : 0x01000000u | (uint32_t)(name_at / 4)); + wr32(node + 8, (uint32_t)data_at), wr32(node + 12, (uint32_t)(data_at + m[i].size)); + memcpy(p + sfnt + 8 + name_at, m[i].name, strlen(m[i].name)); + memset(d, 0x5a, m[i].size); + if (m[i].fcs == 4) wr32(d, 0xFD2FB528u), d[4] = 0x80, d[5] = 0x58, wr32(d + 6, (uint32_t)m[i].content); + if (m[i].fcs == 2) wr32(d, 0xFD2FB528u), d[4] = 0x60, wr16(d + 5, (unsigned)(m[i].content - 256)); + name_at += (strlen(m[i].name) + 4) & ~(size_t)3; + data_at += (m[i].size + 7) & ~(size_t)7; + } + *out_n = total; + return p; +} + +// An empty table with the entries keys[i] = sizes[i]. +static Restbl *table(const char *const *keys, const uint32_t *sizes, int n) { + uint8_t head[22] = "RESTBL"; + wr32(head + 6, 1), wr32(head + 10, 160), wr32(head + 14, 0), wr32(head + 18, 0); + Restbl *t = restbl_parse(head, sizeof head); + for (int i = 0; t && i < n; i++) restbl_raise(t, keys[i], sizes[i]); + return t; +} + +static uint32_t get(const Restbl *t, const char *key) { + uint32_t v = 0; + return restbl_get(t, key, &v) ? v : 0; +} + +static int visited; +static void count(const char *name, const uint8_t *data, size_t size, void *ctx) { + (void)name, (void)data, (void)size, (void)ctx; + visited++; +} + +static int notes; +static void note(const char *key, uint32_t was, uint32_t now, void *ctx) { + (void)key, (void)ctx; + notes += now > was; +} + +int restbl_probe(void) { + fails = 0; + static const char *const keys[] = {"AI/Grow.root.ainb", "AI/Same.root.ainb", "AI/Shrink.root.ainb", + "Component/Shared.bgyml", "Model/Mesh.bfres", "AI/Zs.ainb", "AI/Zs2.ainb"}; + static const uint32_t sizes[] = {13512, 5000, 6000, 9000, 30000, 4400, 3500}; + // The game's copy: an .ainb member a mod grows (by 288 bytes, past its entry's slack), one it keeps, one it + // shrinks, one the table does not size, a mesh codec member and two compressed ones. + static const Member orig[] = {{"AI/Grow.root.ainb", 13088, 0, 0}, {"AI/Same.root.ainb", 4600, 0, 0}, + {"AI/Shrink.root.ainb", 5600, 0, 0}, {"AI/NotSized.ainb", 100, 0, 0}, + {"Model/Mesh.bfres.mc", 1000, 0, 0}, {"AI/Zs.ainb.zs", 64, 4000, 4}, + {"AI/Zs2.ainb.zs", 64, 3900, 2}}; + // The mod's copy: those, plus members new to the pack (one the game sizes elsewhere, one it lacks, a streamed one). + static const Member mod[] = {{"AI/Grow.root.ainb", 13376, 0, 0}, {"AI/Same.root.ainb", 4600, 0, 0}, + {"AI/Shrink.root.ainb", 5000, 0, 0}, {"AI/NotSized.ainb", 300, 0, 0}, + {"Component/New/X.bgyml", 1000, 0, 0}, {"Component/Shared.bgyml", 2000, 0, 0}, + {"Sound/Stream.bwav", 500, 0, 0}, {"Model/Mesh.bfres.mc", 1500, 0, 0}, + {"AI/Zs.ainb.zs", 64, 4500, 4}, {"AI/Zs2.ainb.zs", 64, 4200, 2}}; + size_t on = 0, mn = 0; + uint8_t *op = sarc(orig, 7, -1, &on), *mp = sarc(mod, 10, 4, &mn); // the mod's New/X has no name: left alone + Restbl *base = table(keys, sizes, 7), *t = table(keys, sizes, 7); + CHECK(op && mp && base && t, "probe archives and tables"); + if (op && mp && base && t) { + visited = 0; + CHECK(restbl_sarc_each(mp, mn, count, NULL) && visited == 9, "SARC members: every named one"); + int r = restbl_pack_raise(t, base, mp, mn, op, on, note, NULL); + CHECK(r == 5 && notes == 5, "five entries raised, each noted"); + CHECK(get(t, "AI/Grow.root.ainb") == 13824, "a member that grew: its entry plus the growth, 32-byte steps"); + CHECK(get(t, "AI/Same.root.ainb") == 5000 && get(t, "AI/Shrink.root.ainb") == 6000, + "members that kept or lost bytes keep their entries"); + CHECK(!get(t, "AI/NotSized.ainb"), "a member the game sizes without the table gets no entry"); + CHECK(!get(t, "Component/New/X.bgyml"), "a member without a name is not looked up"); + CHECK(get(t, "Component/Shared.bgyml") == 16384, "a member new to the pack: the estimate of its type"); + CHECK(!get(t, "Sound/Stream.bwav"), "a streamed member new to the pack gets no entry"); + CHECK(get(t, "Model/Mesh.bfres") == 45000, "a mesh codec member's entry grows in proportion"); + CHECK(get(t, "AI/Zs.ainb") == 4928 && get(t, "AI/Zs2.ainb") == 3808, + "a compressed member: the growth of the size its frame records (4- and 2-byte fields)"); + notes = 0; + CHECK(restbl_pack_raise(t, base, mp, mn, op, on, note, NULL) == 0 && !notes, "a second pass raises nothing"); + Restbl *shipped = table(keys, (const uint32_t[]){20000, 5000, 6000, 9000, 30000, 4400, 3500}, 7); + CHECK(shipped && restbl_pack_raise(shipped, base, mp, mn, op, on, NULL, NULL) == 4 && + get(shipped, "AI/Grow.root.ainb") == 20000, + "an entry a mod's own table raised further stays"); + restbl_free(shipped); + Restbl *added = table(keys, sizes, 7); + CHECK(added && restbl_pack_raise(added, base, mp, mn, NULL, 0, NULL, NULL) == 7 && + get(added, "AI/Grow.root.ainb") == 24576 && get(added, "AI/NotSized.ainb") == 12288 && + get(added, "AI/Zs2.ainb") == 16384 && get(added, "Model/Mesh.bfres") == 30000, + "a pack the game lacks: every member gets the estimate of its type, an entry above it stays"); + restbl_free(added); + CHECK(restbl_pack_raise(t, base, (const uint8_t *)"SARC", 4, NULL, 0, NULL, NULL) == -1 && + restbl_pack_raise(t, base, mp, mn, mp, 0x20, NULL, NULL) == -1, + "an archive that is not one, or whose game copy is cut short, raises nothing"); + } + free(op); + free(mp); + restbl_free(base); + restbl_free(t); + printf("restbl: %s\n", fails ? "FAIL" : "ok"); + return fails; +} diff --git a/runtime/romfs.c b/runtime/romfs.c index 1a0d5b71..c062c4b2 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -24,8 +24,8 @@ // matches a RomFS path only without case is served under the RomFS spelling. // // The resource size table: with a player mod enabled, the game's table is rebuilt at start with the tables the mods -// ship merged in and the entries of the mods' files raised to what they need (restbl_overlay), and served from -// /mods/ in place of the game's or a mod's table. +// ship merged in and the entries of the mods' files raised to what they need (restbl_overlay), the members of their +// packs too (restbl_packs), and served from /mods/ in place of the game's or a mod's table. #include #include #include @@ -621,10 +621,122 @@ static Restbl *restbl_unpack(const void *z, size_t n) { return t; } +// ---- the members of the mods' packs (restbl_pack_raise). The game loads each member of a pack (Pack/**/*.pack.zs, a +// SARC archive compressed with a dictionary of Pack/ZsDic.pack.zs) as a resource of its own, in a heap sized by the +// member's own entry, so a mod that changes a member and ships no table for it (or an older one) leaves the game's +// entry. A member that outgrew its entry's slack loads, then its heap runs out: TotK 1.4.3 writes through a null +// pointer at main+0x21794c when the 0x68-byte object an .ainb member builds after its file no longer fits. +#define PACK_MAX (256u << 20) // the largest pack taken, decompressed (AI.Global.Product.140.pack: 24 MiB) +typedef struct { ZSTD_DDict *dict[4]; unsigned id[4]; int n; ZSTD_DCtx *dctx; } Packs; + +// The bytes of zstd frame z[0..n), with the dictionary of ps its header names; malloc'd. NULL when it is not such a +// frame with a recorded size of at most PACK_MAX. +static uint8_t *pack_unpack(Packs *ps, const void *z, size_t n, size_t *out_n) { + unsigned long long raw = ZSTD_getFrameContentSize(z, n); + unsigned id = ZSTD_getDictID_fromFrame(z, n); + ZSTD_DDict *dd = NULL; + for (int i = 0; i < ps->n; i++) + if (ps->id[i] == id) dd = ps->dict[i]; + if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > PACK_MAX || (id && !dd) || !ps->dctx) + return NULL; + uint8_t *d = malloc(raw ? (size_t)raw : 1); + size_t got = !d ? 0 : dd ? ZSTD_decompress_usingDDict(ps->dctx, d, (size_t)raw, z, n, dd) + : ZSTD_decompressDCtx(ps->dctx, d, (size_t)raw, z, n); + if (!d || ZSTD_isError(got) || got != raw) { + free(d); + return NULL; + } + *out_n = (size_t)raw; + return d; +} + +static void pack_dict_add(const char *name, const uint8_t *data, size_t size, void *ctx) { + Packs *ps = ctx; + ZSTD_DDict *dd = ps->n < 4 && ends_with(name, ".zsdic") ? ZSTD_createDDict(data, size) : NULL; // a copy + if (!dd) return; + ps->dict[ps->n] = dd; + ps->id[ps->n++] = ZSTD_getDictID_fromDDict(dd); +} + +// The dictionaries of Pack/ZsDic.pack.zs as served (zs.zsdic, pack.zsdic, bcett.byml.zsdic): a SARC compressed +// without one. 0 when it holds none. +static int packs_open(Packs *ps) { + Overlay *base; + size_t n = 0, rn = 0; + ps->dctx = ZSTD_createDCtx(); + void *z = read_served("Pack/ZsDic.pack.zs", &base, &n, "the members of the mods' packs keep their entries"); + uint8_t *raw = z ? pack_unpack(ps, z, n, &rn) : NULL; + int ok = raw && restbl_sarc_each(raw, rn, pack_dict_add, ps) && ps->n; + free(z); + free(raw); + return ok; +} + +static void packs_close(Packs *ps) { + for (int i = 0; i < ps->n; i++) ZSTD_freeDDict(ps->dict[i]); + ZSTD_freeDCtx(ps->dctx); +} + +// Overlay o (a pack) and the game's copy of it, decompressed; NULL when o cannot be read or unpacked. *orig stays NULL +// for a pack the game lacks or whose copy cannot be read. +static uint8_t *pack_pair(Packs *ps, const Overlay *o, size_t *n, uint8_t **orig, size_t *orig_n) { + uint64_t zn = 0; + void *z = plat_file_read(o->path, &zn); + uint8_t *mod = z ? pack_unpack(ps, z, (size_t)zn, n) : NULL; + free(z); + *orig = NULL; + if (!mod || o->index == ADDED) return mod; + Entry e = entry(o->index); + void *g = e.size < READ_CHUNK ? malloc(e.size ? (size_t)e.size : 1) : NULL; + if (g && read_at(image, g, (uint32_t)e.size, e.offset) == (int64_t)e.size) + *orig = pack_unpack(ps, g, (size_t)e.size, orig_n); + free(g); + return mod; +} + +typedef struct { const char *pack, *mod; int lines; } PackNote; + +static void pack_note(const char *key, uint32_t was, uint32_t now, void *ctx) { + PackNote *pn = ctx; + if (pn->lines++ < 16) + rt_log("RomFS: resource size table: %s in %s (%s): entry %u -> %u", key, pn->pack, pn->mod, was, now); +} + +// The entries of the members of every pack a mod serves, raised in out against the game's table base; the number +// raised, *packs the number of packs with one. +static int restbl_packs(Restbl *out, const Restbl *base, int *packs, int *lines) { + Packs ps = {0}; + int raised = 0, opened = 0, bad = 0; + *packs = 0; + for (size_t k = 0; k < noverlay; k++) { + const Overlay *o = &overlays[k]; + if ((!o->mod && o->prio != 1) || !ends_with(o->name, ".pack.zs")) continue; + if (!opened++ && !packs_open(&ps)) { + rt_log("RomFS: no zstd dictionary in Pack/ZsDic.pack.zs; the members of mods' packs keep their entries"); + break; + } + // A pack whose game copy cannot be read counts as new: each member gets an estimate, which only raises entries. + size_t n = 0, orig_n = 0; + uint8_t *orig, *mod = pack_pair(&ps, o, &n, &orig, &orig_n); + PackNote pn = {o->name, owner_name(o), *lines}; + int r = mod ? restbl_pack_raise(out, base, mod, n, orig, orig_n, pack_note, &pn) : -1; + if (r < 0 && bad++ < 4) rt_log("RomFS: %s (%s) is not a pack this build reads; its members keep their entries", + o->name, owner_name(o)); + *lines = pn.lines; + raised += r > 0 ? r : 0; + *packs += r > 0; + free(mod); + free(orig); + } + packs_close(&ps); + return raised; +} + // With a player mod (or the TOTK_ROMFS_MODS manifest) serving files: the game's table with every shipped table of this // version merged in (an entry larger than the game's wins), the entries a mod's table for another version gives for // that mod's files, and each served mod file's entry raised to what its size needs (restbl_grow for a file the game -// has, restbl_estimate for an added one). Never lowers the game's entries. +// has, restbl_estimate for an added one), the members of the packs mods serve too (restbl_packs). Never lowers the +// game's entries. static void restbl_overlay(void) { int player = 0; for (size_t k = 0; k < noverlay; k++) player |= overlays[k].mod || overlays[k].prio == 1; @@ -691,8 +803,10 @@ static void restbl_overlay(void) { } raised += want && restbl_raise(out, key, want); } + int packs = 0, lines = 0, members = restbl_packs(out, base, &packs, &lines); + if (lines > 16) rt_log("RomFS: resource size table: %d more entries of pack members raised", lines - 16); restbl_free(base); - if (!merged && !raised && !served) { + if (!merged && !raised && !members && !served) { restbl_free(out); rt_log("RomFS: resource size table: the mods' files fit the game's entries (%.0f ms)", (now_s() - t0) * 1000); return; @@ -711,7 +825,8 @@ static void restbl_overlay(void) { } if (serve_generated(restbl_rel, served, z, zn, "mods", "merged resource sizes", keeps)) rt_log("RomFS: resource size table: %d tables merged (%d entries), %d entries raised for mod files (%d " - "added files estimated), %.0f ms", tables, merged, raised, estimated, (now_s() - t0) * 1000); + "added files estimated), %d for members of %d mod packs, %.0f ms", tables, merged, raised, estimated, + members, packs, (now_s() - t0) * 1000); } // The HeapDef table a bigger game resolution needs (resolution.c): its texture heap and that heap's parent grow. The