mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 04:35:57 +00:00
Raise the size table entries of the files inside the mods' packs, whose heaps ran out when the game loaded them
The game loads each member of a pack (Pack/**/*.pack.zs) as a resource of its own, in a heap sized by that member's own entry in the resource size table. The port raised the entries of the mods' loose files only, so a mod that changed a file inside a pack and shipped no table for it left the game's entry. A crash when heavy snow started in Hebra was that: an .ainb member of about 13 KB loaded into a heap sized for the game's smaller copy, 0x48 bytes stayed free after the file, the frame heap gave null for the 0x68-byte object the resource builds next, and main+0x21794c stored that object's vtable at address 0. restbl_overlay now unpacks each pack a mod serves and the game's copy of it with the dictionaries of Pack/ZsDic.pack.zs, and raises each member's entry as it does a loose file's: by the growth for a member the game's copy has, by the estimate of its type for one it lacks. It never lowers an entry, a mod's own table included, and the log names the first members it raised. --probe-mods checks restbl_pack_raise on archives of its own, and docs/mods.md says that the files inside packs count too. 🤖 Generated with [ECA](https://eca.dev) (nubank-anthropic/opus-5.5 - max) Co-Authored-By: eca-agent <git@eca.dev>
This commit is contained in:
7 files changed
+380
-9
No files matched your search
+4
-1
@@ -48,7 +48,10 @@ Two mods that edit the same game file do not combine: the higher one wins the wh
|
||||
(`.tkcl` files) go the same way.
|
||||
|
||||
The port merges the resource size table on its own: at every start with mods on, it combines the tables the mods ship
|
||||
and raises the entries for files a mod made larger or added. A separate RESTBL mod is not needed.
|
||||
and raises the entries for files a mod made larger or added, the files inside a mod's packs (`Pack\**\*.pack.zs`)
|
||||
included. A separate RESTBL mod is not needed. The table tells the game how much memory each file needs: a file that
|
||||
outgrew its entry crashes the game when it loads, which can be long after the start, when the game first needs it. The
|
||||
log line `RomFS: resource size table: ...` says how many entries were raised and names the first files inside packs.
|
||||
|
||||
## Code patches (ExeFS)
|
||||
|
||||
|
||||
+2
-1
@@ -2,7 +2,8 @@
|
||||
//
|
||||
// totk.exe [--exefs DIR] [--romfs DIR] [--profile DIR] <mode>
|
||||
// --selftest layout, dispatch table and host-function registry; needs no game files
|
||||
// --probe-mods the mods folder scan, order and switches (mods.c); needs no game files
|
||||
// --probe-mods the mods folder scan, order and switches (mods.c), the size table entries of the members of
|
||||
// mods' packs (restbl_probe.c); needs no game files
|
||||
// --probe-loader load and relocate the modules, check them against the measured 1.2.1 values
|
||||
// --probe-kernel, --probe-ipc, --probe-services, --probe-fs host probes
|
||||
// --probe-discord the Rich Presence IPC exchange; skips without a Discord client; no game files
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// --probe-mods: the mods folder scan, its order and switches (mods.c). Builds a mods folder of its own next to totk.exe,
|
||||
// scans it and removes it. Needs no game files.
|
||||
// scans it and removes it. Also the resource size table's entries of the members of the mods' packs (restbl_probe.c).
|
||||
// Needs no game files.
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
@@ -7,6 +8,7 @@
|
||||
#include "host.h"
|
||||
#include "mods.h"
|
||||
#include "platform/platform.h"
|
||||
#include "restbl.h"
|
||||
|
||||
static int fails;
|
||||
#define CHECK(c, what) do { if (!(c)) { fails++; fprintf(stderr, "FAIL mods: %s\n", what); } } while (0)
|
||||
@@ -126,5 +128,6 @@ int probe_mods(void) {
|
||||
}
|
||||
remove_tree(root);
|
||||
printf("mods: %s\n", fails ? "FAIL" : "ok");
|
||||
return fails ? 1 : 0;
|
||||
int table = restbl_probe();
|
||||
return fails || table ? 1 : 0;
|
||||
}
|
||||
@@ -234,3 +234,102 @@ uint32_t restbl_grow(const char *key, uint32_t entry, uint64_t old, uint64_t new
|
||||
uint64_t e = entry + ((new_size - old) * f + 999) / 1000;
|
||||
return clamp32((e + 31) & ~(uint64_t)31);
|
||||
}
|
||||
|
||||
// ---- packs
|
||||
// SARC: "SARC", u16 header size, u16 byte order mark 0xfeff, u32 archive size, u32 data offset, u16 version, u16 pad;
|
||||
// SFAT: "SFAT", u16 header size, u16 node count, u32 hash key, then per node {u32 name hash, u32 attributes (bit 24: it
|
||||
// has a name, bits 0-23: the name's offset in SFNT / 4), u32 data start, u32 data end} with the data offsets relative
|
||||
// to the archive's data offset; SFNT: "SFNT", u16 header size, u16 pad, then the NUL-terminated names.
|
||||
static uint32_t rd16(const uint8_t *p) { return p[0] | p[1] << 8; }
|
||||
|
||||
int restbl_sarc_each(const uint8_t *p, size_t n, RestblMember *each, void *ctx) {
|
||||
if (n < 0x14 || memcmp(p, "SARC", 4) || rd16(p + 6) != 0xfeff) return 0;
|
||||
size_t head = rd16(p + 4), data = rd32(p + 12);
|
||||
if (head + 12 > n || memcmp(p + head, "SFAT", 4) || data > n) return 0;
|
||||
size_t count = rd16(p + head + 6), nodes = head + rd16(p + head + 4), names = nodes + count * 16;
|
||||
if (names + 8 > n || memcmp(p + names, "SFNT", 4) || rd16(p + names + 4) < 8) return 0;
|
||||
names += rd16(p + names + 4);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
const uint8_t *e = p + nodes + i * 16;
|
||||
uint32_t attr = rd32(e + 4), lo = rd32(e + 8), hi = rd32(e + 12);
|
||||
if (!(attr >> 24)) continue; // without a name: nothing the game looks up by path
|
||||
size_t s = names + (size_t)(attr & 0xffffff) * 4;
|
||||
if (s >= n || !memchr(p + s, 0, n - s) || lo > hi || hi > n - data) return 0;
|
||||
each((const char *)p + s, p + data + lo, hi - lo, ctx);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
// The decompressed size a zstd frame p[0..n) records (Frame_Content_Size), or 4 times n when it records none, as
|
||||
// romfs.c content_size takes a loose file's.
|
||||
static uint64_t frame_content_size(const uint8_t *p, size_t n) {
|
||||
static const uint8_t did_bytes[4] = {0, 1, 2, 4};
|
||||
if (n < 6 || rd32(p) != 0xFD2FB528u) return n;
|
||||
uint8_t fhd = p[4];
|
||||
int single = fhd >> 5 & 1, fcs = fhd >> 6;
|
||||
size_t at = 5 + !single + did_bytes[fhd & 3], len = fcs ? 1u << fcs : single;
|
||||
if (!len || at + len > n) return (uint64_t)n * 4;
|
||||
uint64_t v = 0;
|
||||
for (size_t i = 0; i < len; i++) v |= (uint64_t)p[at + i] << 8 * i;
|
||||
return len == 2 ? v + 256 : v;
|
||||
}
|
||||
|
||||
typedef struct { const char *name; uint64_t size; } Member;
|
||||
typedef struct { Member *m; size_t n, cap; int full; } Members;
|
||||
|
||||
static int ends_with(const char *s, const char *tail) {
|
||||
size_t n = strlen(s), k = strlen(tail);
|
||||
return n >= k && !strcmp(s + n - k, tail);
|
||||
}
|
||||
|
||||
// A member's size as the game unpacks it: a .zs member's content size, else its bytes (a .mc member's compressed ones).
|
||||
static void member_add(const char *name, const uint8_t *data, size_t size, void *ctx) {
|
||||
Members *ms = ctx;
|
||||
if (ms->n == ms->cap) {
|
||||
size_t cap = ms->cap ? ms->cap * 2 : 64;
|
||||
Member *grown = realloc(ms->m, cap * sizeof *grown);
|
||||
if (!grown) {
|
||||
ms->full = 1;
|
||||
return;
|
||||
}
|
||||
ms->m = grown, ms->cap = cap;
|
||||
}
|
||||
ms->m[ms->n++] = (Member){name, ends_with(name, ".zs") ? frame_content_size(data, size) : size};
|
||||
}
|
||||
|
||||
static int by_name(const void *a, const void *b) {
|
||||
return strcmp(((const Member *)a)->name, ((const Member *)b)->name);
|
||||
}
|
||||
|
||||
int restbl_pack_raise(Restbl *t, const Restbl *base, const uint8_t *mod, size_t mod_n, const uint8_t *orig,
|
||||
size_t orig_n, RestblNote *note, void *ctx) {
|
||||
Members m = {0}, o = {0};
|
||||
int raised = -1;
|
||||
if (!restbl_sarc_each(mod, mod_n, member_add, &m) || m.full ||
|
||||
(orig && (!restbl_sarc_each(orig, orig_n, member_add, &o) || o.full)))
|
||||
goto done;
|
||||
if (o.n) qsort(o.m, o.n, sizeof *o.m, by_name);
|
||||
raised = 0;
|
||||
for (size_t i = 0; i < m.n; i++) {
|
||||
char key[1024];
|
||||
uint32_t entry, before = 0, want;
|
||||
restbl_key(m.m[i].name, key, sizeof key);
|
||||
int mc = ends_with(m.m[i].name, ".mc");
|
||||
const Member *was = o.n ? bsearch(&m.m[i], o.m, o.n, sizeof *o.m, by_name) : NULL;
|
||||
if (was) {
|
||||
if (!restbl_get(base, key, &entry)) continue; // the game sizes it without the table (streamed)
|
||||
want = restbl_grow(key, entry, was->size, m.m[i].size, mc);
|
||||
} else { // new to this pack: no copy of the game's to grow from
|
||||
want = restbl_estimate(key, mc ? m.m[i].size * 8 : m.m[i].size);
|
||||
}
|
||||
restbl_get(t, key, &before);
|
||||
if (want && restbl_raise(t, key, want)) {
|
||||
raised++;
|
||||
if (note) note(key, before, want, ctx);
|
||||
}
|
||||
}
|
||||
done:
|
||||
free(m.m);
|
||||
free(o.m);
|
||||
return raised;
|
||||
}
|
||||
@@ -31,3 +31,20 @@ uint32_t restbl_estimate(const char *key, uint64_t size);
|
||||
// times the type's factor, never lowered. scaled: the sizes are compressed ones (a mesh codec .mc file), whose entry
|
||||
// grows in proportion.
|
||||
uint32_t restbl_grow(const char *key, uint32_t entry, uint64_t old, uint64_t new_size, int scaled);
|
||||
|
||||
// The members of the SARC archive p[0..n): each(name, data, size, ctx) for every named member, in the archive's order.
|
||||
// 0 when p is not a SARC archive or a member lies outside it.
|
||||
typedef void RestblMember(const char *name, const uint8_t *data, size_t size, void *ctx);
|
||||
int restbl_sarc_each(const uint8_t *p, size_t n, RestblMember *each, void *ctx);
|
||||
|
||||
// A pack (Pack/**/*.pack.zs, decompressed: a SARC archive) whose members the game loads as resources of their own, each
|
||||
// sized by its own entry. The entries of a mod's pack `mod` raised in t against the game's table `base` and the game's
|
||||
// copy of the pack `orig` (NULL for a pack the game lacks): a member that grew by restbl_grow, a member the game's copy
|
||||
// lacks by restbl_estimate, the others left as they are. note(key, entry before, entry after, ctx) for each raised
|
||||
// entry. The number raised; -1 when either pack is not a SARC archive.
|
||||
typedef void RestblNote(const char *key, uint32_t was, uint32_t now, void *ctx);
|
||||
int restbl_pack_raise(Restbl *t, const Restbl *base, const uint8_t *mod, size_t mod_n, const uint8_t *orig,
|
||||
size_t orig_n, RestblNote *note, void *ctx);
|
||||
|
||||
// restbl_probe.c, part of --probe-mods: the number of failed checks.
|
||||
int restbl_probe(void);
|
||||
@@ -0,0 +1,133 @@
|
||||
// The entries of the members of the mods' packs (restbl.c restbl_pack_raise), on archives and tables of its own; part
|
||||
// of --probe-mods (mods_probe.c). Needs no game files.
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "restbl.h"
|
||||
|
||||
static int fails;
|
||||
#define CHECK(c, what) do { if (!(c)) { fails++; fprintf(stderr, "FAIL restbl: %s\n", what); } } while (0)
|
||||
|
||||
// A member of a probe archive: its name and size; a .zs member is a zstd frame header recording `content` bytes:
|
||||
// fcs 4, a frame with a window descriptor and a 4-byte field; 2, a single-segment frame with a 2-byte field holding
|
||||
// content - 256.
|
||||
typedef struct { const char *name; size_t size, content; int fcs; } Member;
|
||||
|
||||
static void wr16(uint8_t *p, unsigned v) { p[0] = (uint8_t)v, p[1] = (uint8_t)(v >> 8); }
|
||||
static void wr32(uint8_t *p, uint32_t v) { wr16(p, v & 0xffff), wr16(p + 2, v >> 16); }
|
||||
|
||||
// A SARC archive of the members m[0..n); unnamed: the index of a member without a name, -1 for none. malloc'd.
|
||||
static uint8_t *sarc(const Member *m, int n, int unnamed, size_t *out_n) {
|
||||
size_t names = 0, data = 0;
|
||||
for (int i = 0; i < n; i++) names += (strlen(m[i].name) + 4) & ~(size_t)3, data += (m[i].size + 7) & ~(size_t)7;
|
||||
size_t sfat = 0x14, sfnt = sfat + 12 + 16 * (size_t)n, start = (sfnt + 8 + names + 15) & ~(size_t)15;
|
||||
size_t total = start + data;
|
||||
uint8_t *p = calloc(1, total);
|
||||
if (!p) return NULL;
|
||||
memcpy(p, "SARC", 4), wr16(p + 4, 0x14), wr16(p + 6, 0xfeff);
|
||||
wr32(p + 8, (uint32_t)total), wr32(p + 12, (uint32_t)start);
|
||||
memcpy(p + sfat, "SFAT", 4), wr16(p + sfat + 4, 12), wr16(p + sfat + 6, (unsigned)n), wr32(p + sfat + 8, 0x65);
|
||||
memcpy(p + sfnt, "SFNT", 4), wr16(p + sfnt + 4, 8);
|
||||
size_t name_at = 0, data_at = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
uint8_t *node = p + sfat + 12 + 16 * (size_t)i, *d = p + start + data_at;
|
||||
wr32(node + 4, i == unnamed ? 0 : 0x01000000u | (uint32_t)(name_at / 4));
|
||||
wr32(node + 8, (uint32_t)data_at), wr32(node + 12, (uint32_t)(data_at + m[i].size));
|
||||
memcpy(p + sfnt + 8 + name_at, m[i].name, strlen(m[i].name));
|
||||
memset(d, 0x5a, m[i].size);
|
||||
if (m[i].fcs == 4) wr32(d, 0xFD2FB528u), d[4] = 0x80, d[5] = 0x58, wr32(d + 6, (uint32_t)m[i].content);
|
||||
if (m[i].fcs == 2) wr32(d, 0xFD2FB528u), d[4] = 0x60, wr16(d + 5, (unsigned)(m[i].content - 256));
|
||||
name_at += (strlen(m[i].name) + 4) & ~(size_t)3;
|
||||
data_at += (m[i].size + 7) & ~(size_t)7;
|
||||
}
|
||||
*out_n = total;
|
||||
return p;
|
||||
}
|
||||
|
||||
// An empty table with the entries keys[i] = sizes[i].
|
||||
static Restbl *table(const char *const *keys, const uint32_t *sizes, int n) {
|
||||
uint8_t head[22] = "RESTBL";
|
||||
wr32(head + 6, 1), wr32(head + 10, 160), wr32(head + 14, 0), wr32(head + 18, 0);
|
||||
Restbl *t = restbl_parse(head, sizeof head);
|
||||
for (int i = 0; t && i < n; i++) restbl_raise(t, keys[i], sizes[i]);
|
||||
return t;
|
||||
}
|
||||
|
||||
static uint32_t get(const Restbl *t, const char *key) {
|
||||
uint32_t v = 0;
|
||||
return restbl_get(t, key, &v) ? v : 0;
|
||||
}
|
||||
|
||||
static int visited;
|
||||
static void count(const char *name, const uint8_t *data, size_t size, void *ctx) {
|
||||
(void)name, (void)data, (void)size, (void)ctx;
|
||||
visited++;
|
||||
}
|
||||
|
||||
static int notes;
|
||||
static void note(const char *key, uint32_t was, uint32_t now, void *ctx) {
|
||||
(void)key, (void)ctx;
|
||||
notes += now > was;
|
||||
}
|
||||
|
||||
int restbl_probe(void) {
|
||||
fails = 0;
|
||||
static const char *const keys[] = {"AI/Grow.root.ainb", "AI/Same.root.ainb", "AI/Shrink.root.ainb",
|
||||
"Component/Shared.bgyml", "Model/Mesh.bfres", "AI/Zs.ainb", "AI/Zs2.ainb"};
|
||||
static const uint32_t sizes[] = {13512, 5000, 6000, 9000, 30000, 4400, 3500};
|
||||
// The game's copy: an .ainb member a mod grows (by 288 bytes, past its entry's slack), one it keeps, one it
|
||||
// shrinks, one the table does not size, a mesh codec member and two compressed ones.
|
||||
static const Member orig[] = {{"AI/Grow.root.ainb", 13088, 0, 0}, {"AI/Same.root.ainb", 4600, 0, 0},
|
||||
{"AI/Shrink.root.ainb", 5600, 0, 0}, {"AI/NotSized.ainb", 100, 0, 0},
|
||||
{"Model/Mesh.bfres.mc", 1000, 0, 0}, {"AI/Zs.ainb.zs", 64, 4000, 4},
|
||||
{"AI/Zs2.ainb.zs", 64, 3900, 2}};
|
||||
// The mod's copy: those, plus members new to the pack (one the game sizes elsewhere, one it lacks, a streamed one).
|
||||
static const Member mod[] = {{"AI/Grow.root.ainb", 13376, 0, 0}, {"AI/Same.root.ainb", 4600, 0, 0},
|
||||
{"AI/Shrink.root.ainb", 5000, 0, 0}, {"AI/NotSized.ainb", 300, 0, 0},
|
||||
{"Component/New/X.bgyml", 1000, 0, 0}, {"Component/Shared.bgyml", 2000, 0, 0},
|
||||
{"Sound/Stream.bwav", 500, 0, 0}, {"Model/Mesh.bfres.mc", 1500, 0, 0},
|
||||
{"AI/Zs.ainb.zs", 64, 4500, 4}, {"AI/Zs2.ainb.zs", 64, 4200, 2}};
|
||||
size_t on = 0, mn = 0;
|
||||
uint8_t *op = sarc(orig, 7, -1, &on), *mp = sarc(mod, 10, 4, &mn); // the mod's New/X has no name: left alone
|
||||
Restbl *base = table(keys, sizes, 7), *t = table(keys, sizes, 7);
|
||||
CHECK(op && mp && base && t, "probe archives and tables");
|
||||
if (op && mp && base && t) {
|
||||
visited = 0;
|
||||
CHECK(restbl_sarc_each(mp, mn, count, NULL) && visited == 9, "SARC members: every named one");
|
||||
int r = restbl_pack_raise(t, base, mp, mn, op, on, note, NULL);
|
||||
CHECK(r == 5 && notes == 5, "five entries raised, each noted");
|
||||
CHECK(get(t, "AI/Grow.root.ainb") == 13824, "a member that grew: its entry plus the growth, 32-byte steps");
|
||||
CHECK(get(t, "AI/Same.root.ainb") == 5000 && get(t, "AI/Shrink.root.ainb") == 6000,
|
||||
"members that kept or lost bytes keep their entries");
|
||||
CHECK(!get(t, "AI/NotSized.ainb"), "a member the game sizes without the table gets no entry");
|
||||
CHECK(!get(t, "Component/New/X.bgyml"), "a member without a name is not looked up");
|
||||
CHECK(get(t, "Component/Shared.bgyml") == 16384, "a member new to the pack: the estimate of its type");
|
||||
CHECK(!get(t, "Sound/Stream.bwav"), "a streamed member new to the pack gets no entry");
|
||||
CHECK(get(t, "Model/Mesh.bfres") == 45000, "a mesh codec member's entry grows in proportion");
|
||||
CHECK(get(t, "AI/Zs.ainb") == 4928 && get(t, "AI/Zs2.ainb") == 3808,
|
||||
"a compressed member: the growth of the size its frame records (4- and 2-byte fields)");
|
||||
notes = 0;
|
||||
CHECK(restbl_pack_raise(t, base, mp, mn, op, on, note, NULL) == 0 && !notes, "a second pass raises nothing");
|
||||
Restbl *shipped = table(keys, (const uint32_t[]){20000, 5000, 6000, 9000, 30000, 4400, 3500}, 7);
|
||||
CHECK(shipped && restbl_pack_raise(shipped, base, mp, mn, op, on, NULL, NULL) == 4 &&
|
||||
get(shipped, "AI/Grow.root.ainb") == 20000,
|
||||
"an entry a mod's own table raised further stays");
|
||||
restbl_free(shipped);
|
||||
Restbl *added = table(keys, sizes, 7);
|
||||
CHECK(added && restbl_pack_raise(added, base, mp, mn, NULL, 0, NULL, NULL) == 7 &&
|
||||
get(added, "AI/Grow.root.ainb") == 24576 && get(added, "AI/NotSized.ainb") == 12288 &&
|
||||
get(added, "AI/Zs2.ainb") == 16384 && get(added, "Model/Mesh.bfres") == 30000,
|
||||
"a pack the game lacks: every member gets the estimate of its type, an entry above it stays");
|
||||
restbl_free(added);
|
||||
CHECK(restbl_pack_raise(t, base, (const uint8_t *)"SARC", 4, NULL, 0, NULL, NULL) == -1 &&
|
||||
restbl_pack_raise(t, base, mp, mn, mp, 0x20, NULL, NULL) == -1,
|
||||
"an archive that is not one, or whose game copy is cut short, raises nothing");
|
||||
}
|
||||
free(op);
|
||||
free(mp);
|
||||
restbl_free(base);
|
||||
restbl_free(t);
|
||||
printf("restbl: %s\n", fails ? "FAIL" : "ok");
|
||||
return fails;
|
||||
}
|
||||
+120
-5
@@ -24,8 +24,8 @@
|
||||
// matches a RomFS path only without case is served under the RomFS spelling.
|
||||
//
|
||||
// The resource size table: with a player mod enabled, the game's table is rebuilt at start with the tables the mods
|
||||
// ship merged in and the entries of the mods' files raised to what they need (restbl_overlay), and served from
|
||||
// <profile>/mods/ in place of the game's or a mod's table.
|
||||
// ship merged in and the entries of the mods' files raised to what they need (restbl_overlay), the members of their
|
||||
// packs too (restbl_packs), and served from <profile>/mods/ in place of the game's or a mod's table.
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -621,10 +621,122 @@ static Restbl *restbl_unpack(const void *z, size_t n) {
|
||||
return t;
|
||||
}
|
||||
|
||||
// ---- the members of the mods' packs (restbl_pack_raise). The game loads each member of a pack (Pack/**/*.pack.zs, a
|
||||
// SARC archive compressed with a dictionary of Pack/ZsDic.pack.zs) as a resource of its own, in a heap sized by the
|
||||
// member's own entry, so a mod that changes a member and ships no table for it (or an older one) leaves the game's
|
||||
// entry. A member that outgrew its entry's slack loads, then its heap runs out: TotK 1.4.3 writes through a null
|
||||
// pointer at main+0x21794c when the 0x68-byte object an .ainb member builds after its file no longer fits.
|
||||
#define PACK_MAX (256u << 20) // the largest pack taken, decompressed (AI.Global.Product.140.pack: 24 MiB)
|
||||
typedef struct { ZSTD_DDict *dict[4]; unsigned id[4]; int n; ZSTD_DCtx *dctx; } Packs;
|
||||
|
||||
// The bytes of zstd frame z[0..n), with the dictionary of ps its header names; malloc'd. NULL when it is not such a
|
||||
// frame with a recorded size of at most PACK_MAX.
|
||||
static uint8_t *pack_unpack(Packs *ps, const void *z, size_t n, size_t *out_n) {
|
||||
unsigned long long raw = ZSTD_getFrameContentSize(z, n);
|
||||
unsigned id = ZSTD_getDictID_fromFrame(z, n);
|
||||
ZSTD_DDict *dd = NULL;
|
||||
for (int i = 0; i < ps->n; i++)
|
||||
if (ps->id[i] == id) dd = ps->dict[i];
|
||||
if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > PACK_MAX || (id && !dd) || !ps->dctx)
|
||||
return NULL;
|
||||
uint8_t *d = malloc(raw ? (size_t)raw : 1);
|
||||
size_t got = !d ? 0 : dd ? ZSTD_decompress_usingDDict(ps->dctx, d, (size_t)raw, z, n, dd)
|
||||
: ZSTD_decompressDCtx(ps->dctx, d, (size_t)raw, z, n);
|
||||
if (!d || ZSTD_isError(got) || got != raw) {
|
||||
free(d);
|
||||
return NULL;
|
||||
}
|
||||
*out_n = (size_t)raw;
|
||||
return d;
|
||||
}
|
||||
|
||||
static void pack_dict_add(const char *name, const uint8_t *data, size_t size, void *ctx) {
|
||||
Packs *ps = ctx;
|
||||
ZSTD_DDict *dd = ps->n < 4 && ends_with(name, ".zsdic") ? ZSTD_createDDict(data, size) : NULL; // a copy
|
||||
if (!dd) return;
|
||||
ps->dict[ps->n] = dd;
|
||||
ps->id[ps->n++] = ZSTD_getDictID_fromDDict(dd);
|
||||
}
|
||||
|
||||
// The dictionaries of Pack/ZsDic.pack.zs as served (zs.zsdic, pack.zsdic, bcett.byml.zsdic): a SARC compressed
|
||||
// without one. 0 when it holds none.
|
||||
static int packs_open(Packs *ps) {
|
||||
Overlay *base;
|
||||
size_t n = 0, rn = 0;
|
||||
ps->dctx = ZSTD_createDCtx();
|
||||
void *z = read_served("Pack/ZsDic.pack.zs", &base, &n, "the members of the mods' packs keep their entries");
|
||||
uint8_t *raw = z ? pack_unpack(ps, z, n, &rn) : NULL;
|
||||
int ok = raw && restbl_sarc_each(raw, rn, pack_dict_add, ps) && ps->n;
|
||||
free(z);
|
||||
free(raw);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static void packs_close(Packs *ps) {
|
||||
for (int i = 0; i < ps->n; i++) ZSTD_freeDDict(ps->dict[i]);
|
||||
ZSTD_freeDCtx(ps->dctx);
|
||||
}
|
||||
|
||||
// Overlay o (a pack) and the game's copy of it, decompressed; NULL when o cannot be read or unpacked. *orig stays NULL
|
||||
// for a pack the game lacks or whose copy cannot be read.
|
||||
static uint8_t *pack_pair(Packs *ps, const Overlay *o, size_t *n, uint8_t **orig, size_t *orig_n) {
|
||||
uint64_t zn = 0;
|
||||
void *z = plat_file_read(o->path, &zn);
|
||||
uint8_t *mod = z ? pack_unpack(ps, z, (size_t)zn, n) : NULL;
|
||||
free(z);
|
||||
*orig = NULL;
|
||||
if (!mod || o->index == ADDED) return mod;
|
||||
Entry e = entry(o->index);
|
||||
void *g = e.size < READ_CHUNK ? malloc(e.size ? (size_t)e.size : 1) : NULL;
|
||||
if (g && read_at(image, g, (uint32_t)e.size, e.offset) == (int64_t)e.size)
|
||||
*orig = pack_unpack(ps, g, (size_t)e.size, orig_n);
|
||||
free(g);
|
||||
return mod;
|
||||
}
|
||||
|
||||
typedef struct { const char *pack, *mod; int lines; } PackNote;
|
||||
|
||||
static void pack_note(const char *key, uint32_t was, uint32_t now, void *ctx) {
|
||||
PackNote *pn = ctx;
|
||||
if (pn->lines++ < 16)
|
||||
rt_log("RomFS: resource size table: %s in %s (%s): entry %u -> %u", key, pn->pack, pn->mod, was, now);
|
||||
}
|
||||
|
||||
// The entries of the members of every pack a mod serves, raised in out against the game's table base; the number
|
||||
// raised, *packs the number of packs with one.
|
||||
static int restbl_packs(Restbl *out, const Restbl *base, int *packs, int *lines) {
|
||||
Packs ps = {0};
|
||||
int raised = 0, opened = 0, bad = 0;
|
||||
*packs = 0;
|
||||
for (size_t k = 0; k < noverlay; k++) {
|
||||
const Overlay *o = &overlays[k];
|
||||
if ((!o->mod && o->prio != 1) || !ends_with(o->name, ".pack.zs")) continue;
|
||||
if (!opened++ && !packs_open(&ps)) {
|
||||
rt_log("RomFS: no zstd dictionary in Pack/ZsDic.pack.zs; the members of mods' packs keep their entries");
|
||||
break;
|
||||
}
|
||||
// A pack whose game copy cannot be read counts as new: each member gets an estimate, which only raises entries.
|
||||
size_t n = 0, orig_n = 0;
|
||||
uint8_t *orig, *mod = pack_pair(&ps, o, &n, &orig, &orig_n);
|
||||
PackNote pn = {o->name, owner_name(o), *lines};
|
||||
int r = mod ? restbl_pack_raise(out, base, mod, n, orig, orig_n, pack_note, &pn) : -1;
|
||||
if (r < 0 && bad++ < 4) rt_log("RomFS: %s (%s) is not a pack this build reads; its members keep their entries",
|
||||
o->name, owner_name(o));
|
||||
*lines = pn.lines;
|
||||
raised += r > 0 ? r : 0;
|
||||
*packs += r > 0;
|
||||
free(mod);
|
||||
free(orig);
|
||||
}
|
||||
packs_close(&ps);
|
||||
return raised;
|
||||
}
|
||||
|
||||
// With a player mod (or the TOTK_ROMFS_MODS manifest) serving files: the game's table with every shipped table of this
|
||||
// version merged in (an entry larger than the game's wins), the entries a mod's table for another version gives for
|
||||
// that mod's files, and each served mod file's entry raised to what its size needs (restbl_grow for a file the game
|
||||
// has, restbl_estimate for an added one). Never lowers the game's entries.
|
||||
// has, restbl_estimate for an added one), the members of the packs mods serve too (restbl_packs). Never lowers the
|
||||
// game's entries.
|
||||
static void restbl_overlay(void) {
|
||||
int player = 0;
|
||||
for (size_t k = 0; k < noverlay; k++) player |= overlays[k].mod || overlays[k].prio == 1;
|
||||
@@ -691,8 +803,10 @@ static void restbl_overlay(void) {
|
||||
}
|
||||
raised += want && restbl_raise(out, key, want);
|
||||
}
|
||||
int packs = 0, lines = 0, members = restbl_packs(out, base, &packs, &lines);
|
||||
if (lines > 16) rt_log("RomFS: resource size table: %d more entries of pack members raised", lines - 16);
|
||||
restbl_free(base);
|
||||
if (!merged && !raised && !served) {
|
||||
if (!merged && !raised && !members && !served) {
|
||||
restbl_free(out);
|
||||
rt_log("RomFS: resource size table: the mods' files fit the game's entries (%.0f ms)", (now_s() - t0) * 1000);
|
||||
return;
|
||||
@@ -711,7 +825,8 @@ static void restbl_overlay(void) {
|
||||
}
|
||||
if (serve_generated(restbl_rel, served, z, zn, "mods", "merged resource sizes", keeps))
|
||||
rt_log("RomFS: resource size table: %d tables merged (%d entries), %d entries raised for mod files (%d "
|
||||
"added files estimated), %.0f ms", tables, merged, raised, estimated, (now_s() - t0) * 1000);
|
||||
"added files estimated), %d for members of %d mod packs, %.0f ms", tables, merged, raised, estimated,
|
||||
members, packs, (now_s() - t0) * 1000);
|
||||
}
|
||||
|
||||
// The HeapDef table a bigger game resolution needs (resolution.c): its texture heap and that heap's parent grow. The
|
||||
|
||||
Reference in new issue
Block a user