mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 12:25:56 +00:00
Merge the resource size table of the enabled mods at start
The game sizes the buffer for a file from its entry in System/Resource/ResourceSizeTable.*.rsizetable.zs, so a mod file that grew could fail to load, and a file a mod added had no entry. With a player mod enabled, romfs.c restbl_overlay now rebuilds the table at start from: - the game's table; - every table of this version that a mod (or the Xbox prompts folder) ships, where the larger entry wins; - a mod's table for another version, for that mod's own files only; - each served mod file: a grown file's entry rises by the growth times its type's factor, and an added file gets an estimate. The factors come from the 1.4.3 table measured against its own files (most types need the size plus at most 8 KiB; bfres 1.5x, bkres 2x, bgyml 3.85x). The result is served from <profile>/mods/ and takes about 90 ms. A mod's table no longer counts as a clash with the Xbox prompts. Game entries are never lowered. tests/mods_romfs_check.py checks the merged table and now passes probe-fs a profile of its own. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ac69f39529
commit
3dd7f846b8
5 files changed
+537
-6
No files matched your search
@@ -64,6 +64,10 @@ it up or down. When two mods change the same file, the higher one wins.
|
||||
export location to an empty folder inside `mods/` (for example `mods/TKMM`). That folder is then one mod here.
|
||||
- Files made for another game version (names with `.Product.<number>.` that this version does not have) are left out;
|
||||
the Mods tab counts them. Look for the mod's release for your game version.
|
||||
- The resource size table (`System/Resource/ResourceSizeTable.Product.<n>...rsizetable.zs`), which tells the game how
|
||||
much memory each file needs, is rebuilt at every start with mods enabled: the tables the mods ship are merged and
|
||||
the entries of files a mod made larger or added are raised. The log line `RomFS: resource size table: ...` says
|
||||
how many. A mod's table for another game version still counts for that mod's own files.
|
||||
- Mods that patch the game code (`exefs/*.ips`, `exefs/*.pchtxt`) are listed but not applied yet; a mod with both kinds
|
||||
of files stays off as a whole until they are.
|
||||
- Mods with their own code (`exefs/subsdk*` or a replaced `exefs/main`, such as UltraCam) never load, their RomFS files
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
// The resource size table (restbl.h). Layout of a decompressed RESTBL, little-endian:
|
||||
//
|
||||
// "RESTBL", u32 version (1), u32 name length (160), u32 hashed count, u32 named count
|
||||
// hashed entries {u32 crc32 of the key, u32 size}, sorted by the hash
|
||||
// named entries {char key[name length], u32 size}: the keys whose hash collides with another key's
|
||||
//
|
||||
// A key is a RomFS path without .zs or .mc. The estimates come from the 1.4.3 table against its own files: most types
|
||||
// need the decompressed size plus at most 8 KiB (headers, alignment); a few need a multiple of it (bfres 1.5, bkres
|
||||
// 2.05, bgyml 3.85, asb 1.4, bstar 1.2, bphsc 1.1). A type the table does not list takes 1.5 times its size plus 8 KiB.
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "restbl.h"
|
||||
|
||||
typedef struct { uint32_t hash, size; } Hashed;
|
||||
|
||||
struct Restbl {
|
||||
uint32_t version, name_len;
|
||||
Hashed *hashed;
|
||||
size_t nhashed, hashed_cap;
|
||||
uint8_t *named; // named entries as in the file: key (name_len bytes, NUL padded) then u32 size
|
||||
size_t nnamed;
|
||||
};
|
||||
|
||||
static uint32_t rd32(const uint8_t *p) { return p[0] | p[1] << 8 | p[2] << 16 | (uint32_t)p[3] << 24; }
|
||||
static void wr32(uint8_t *p, uint32_t v) {
|
||||
for (int i = 0; i < 4; i++) p[i] = (uint8_t)(v >> 8 * i);
|
||||
}
|
||||
|
||||
static uint32_t crc32_of(const char *s) {
|
||||
uint32_t c = ~0u;
|
||||
for (; *s; s++) {
|
||||
c ^= (uint8_t)*s;
|
||||
for (int k = 0; k < 8; k++) c = c >> 1 ^ (0xEDB88320u & (0u - (c & 1)));
|
||||
}
|
||||
return ~c;
|
||||
}
|
||||
|
||||
Restbl *restbl_parse(const uint8_t *data, size_t n) {
|
||||
if (n < 22 || memcmp(data, "RESTBL", 6)) return NULL;
|
||||
uint32_t version = rd32(data + 6), name_len = rd32(data + 10), nh = rd32(data + 14), nn = rd32(data + 18);
|
||||
if (!name_len || name_len > 4096 || (uint64_t)nh * 8 + (uint64_t)nn * (name_len + 4) != n - 22) return NULL;
|
||||
Restbl *t = calloc(1, sizeof *t);
|
||||
if (!t) return NULL;
|
||||
t->version = version, t->name_len = name_len;
|
||||
t->hashed = malloc(((size_t)nh + 1) * sizeof *t->hashed);
|
||||
t->named = malloc((size_t)nn * (name_len + 4) + 1);
|
||||
if (!t->hashed || !t->named) {
|
||||
restbl_free(t);
|
||||
return NULL;
|
||||
}
|
||||
for (uint32_t i = 0; i < nh; i++) t->hashed[i] = (Hashed){rd32(data + 22 + i * 8), rd32(data + 26 + i * 8)};
|
||||
t->nhashed = t->hashed_cap = nh;
|
||||
if (!t->hashed_cap) t->hashed_cap = 1;
|
||||
memcpy(t->named, data + 22 + (size_t)nh * 8, (size_t)nn * (name_len + 4));
|
||||
t->nnamed = nn;
|
||||
return t;
|
||||
}
|
||||
|
||||
void restbl_free(Restbl *t) {
|
||||
if (!t) return;
|
||||
free(t->hashed);
|
||||
free(t->named);
|
||||
free(t);
|
||||
}
|
||||
|
||||
const char *restbl_key(const char *path, char *buf, size_t n) {
|
||||
size_t len = strlen(path);
|
||||
if (len >= 3 && (!strcmp(path + len - 3, ".zs") || !strcmp(path + len - 3, ".mc"))) len -= 3;
|
||||
if (len >= n) len = n - 1;
|
||||
memcpy(buf, path, len);
|
||||
buf[len] = 0;
|
||||
return buf;
|
||||
}
|
||||
|
||||
// The named entry of key, or NULL.
|
||||
static uint8_t *named_of(const Restbl *t, const char *key) {
|
||||
size_t len = strlen(key);
|
||||
if (len >= t->name_len) return NULL;
|
||||
for (size_t i = 0; i < t->nnamed; i++) {
|
||||
uint8_t *e = t->named + i * (t->name_len + 4);
|
||||
if (!memcmp(e, key, len + 1)) return e;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// The position of hash in the hashed entries, or where it would go.
|
||||
static size_t hashed_at(const Restbl *t, uint32_t hash) {
|
||||
size_t lo = 0, hi = t->nhashed;
|
||||
while (lo < hi) {
|
||||
size_t m = lo + (hi - lo) / 2;
|
||||
if (t->hashed[m].hash < hash) lo = m + 1;
|
||||
else hi = m;
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
|
||||
int restbl_get(const Restbl *t, const char *key, uint32_t *size) {
|
||||
const uint8_t *e = named_of(t, key);
|
||||
if (e) {
|
||||
*size = rd32(e + t->name_len);
|
||||
return 1;
|
||||
}
|
||||
uint32_t h = crc32_of(key);
|
||||
size_t i = hashed_at(t, h);
|
||||
if (i < t->nhashed && t->hashed[i].hash == h) {
|
||||
*size = t->hashed[i].size;
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// The hashed entry of hash raised to size (added when missing); 1 when the table changed.
|
||||
static int raise_hash(Restbl *t, uint32_t h, uint32_t size) {
|
||||
size_t i = hashed_at(t, h);
|
||||
if (i < t->nhashed && t->hashed[i].hash == h) {
|
||||
if (t->hashed[i].size >= size) return 0;
|
||||
t->hashed[i].size = size;
|
||||
return 1;
|
||||
}
|
||||
if (t->nhashed == t->hashed_cap) {
|
||||
size_t cap = t->hashed_cap * 2;
|
||||
Hashed *grown = realloc(t->hashed, cap * sizeof *grown);
|
||||
if (!grown) return 0;
|
||||
t->hashed = grown, t->hashed_cap = cap;
|
||||
}
|
||||
memmove(t->hashed + i + 1, t->hashed + i, (t->nhashed - i) * sizeof *t->hashed);
|
||||
t->hashed[i] = (Hashed){h, size};
|
||||
t->nhashed++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int restbl_raise(Restbl *t, const char *key, uint32_t size) {
|
||||
uint8_t *e = named_of(t, key);
|
||||
if (!e) return raise_hash(t, crc32_of(key), size);
|
||||
if (rd32(e + t->name_len) >= size) return 0;
|
||||
wr32(e + t->name_len, size);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int restbl_merge(Restbl *t, const Restbl *base, const Restbl *shipped) {
|
||||
int raised = 0;
|
||||
for (size_t i = 0; i < shipped->nhashed; i++) {
|
||||
Hashed s = shipped->hashed[i];
|
||||
size_t k = hashed_at(base, s.hash);
|
||||
if (k < base->nhashed && base->hashed[k].hash == s.hash && base->hashed[k].size >= s.size) continue;
|
||||
raised += raise_hash(t, s.hash, s.size);
|
||||
}
|
||||
if (shipped->name_len != t->name_len) return raised;
|
||||
for (size_t i = 0; i < shipped->nnamed; i++) {
|
||||
const uint8_t *e = shipped->named + i * (shipped->name_len + 4);
|
||||
char key[4097];
|
||||
memcpy(key, e, shipped->name_len);
|
||||
key[shipped->name_len] = 0;
|
||||
uint32_t size = rd32(e + shipped->name_len), have;
|
||||
if (restbl_get(base, key, &have) && have >= size) continue;
|
||||
uint8_t *mine = named_of(t, key);
|
||||
if (mine) {
|
||||
if (rd32(mine + t->name_len) < size) wr32(mine + t->name_len, size), raised++;
|
||||
continue;
|
||||
}
|
||||
uint8_t *grown = realloc(t->named, (t->nnamed + 1) * (t->name_len + 4));
|
||||
if (!grown) return raised;
|
||||
t->named = grown;
|
||||
memcpy(t->named + t->nnamed * (t->name_len + 4), e, t->name_len + 4);
|
||||
t->nnamed++;
|
||||
raised++;
|
||||
}
|
||||
return raised;
|
||||
}
|
||||
|
||||
int restbl_take(Restbl *t, const Restbl *from, const char *const *keys, size_t n) {
|
||||
int raised = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
uint32_t size;
|
||||
if (restbl_get(from, keys[i], &size)) raised += restbl_raise(t, keys[i], size);
|
||||
}
|
||||
return raised;
|
||||
}
|
||||
|
||||
uint8_t *restbl_write(const Restbl *t, size_t *n) {
|
||||
size_t size = 22 + t->nhashed * 8 + t->nnamed * (t->name_len + 4);
|
||||
uint8_t *out = malloc(size);
|
||||
if (!out) return NULL;
|
||||
memcpy(out, "RESTBL", 6);
|
||||
wr32(out + 6, t->version);
|
||||
wr32(out + 10, t->name_len);
|
||||
wr32(out + 14, (uint32_t)t->nhashed);
|
||||
wr32(out + 18, (uint32_t)t->nnamed);
|
||||
for (size_t i = 0; i < t->nhashed; i++) {
|
||||
wr32(out + 22 + i * 8, t->hashed[i].hash);
|
||||
wr32(out + 26 + i * 8, t->hashed[i].size);
|
||||
}
|
||||
memcpy(out + 22 + t->nhashed * 8, t->named, t->nnamed * (t->name_len + 4));
|
||||
*n = size;
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- estimates
|
||||
|
||||
// The factor a type's entry needs over its decompressed size (1000 = 1.0); 0 for a type the table never lists.
|
||||
static unsigned type_factor(const char *key) {
|
||||
static const struct { const char *ext; unsigned f; } types[] = {
|
||||
{"bfres", 1500}, {"bkres", 2050}, {"bgyml", 3850}, {"asb", 1400}, {"bstar", 1200}, {"bphsc", 1100},
|
||||
{"byml", 1000}, {"txtg", 1000}, {"pack", 1000}, {"bntx", 1000}, {"bphsh", 1000}, {"vsts", 1000},
|
||||
{"ainb", 1000}, {"bphnm", 1000}, {"bars", 1000}, {"chunk", 1000}, {"bfevfl", 1000}, {"quad", 1000},
|
||||
{"baev", 1000}, {"bcul", 1000}, {"crbin", 1000}, {"wbr", 1000}, {"bhtmp", 1000}, {"dpi", 1000},
|
||||
{"cai", 1000}, {"png", 1000}, {"cutinfo", 1000}, {"blwp", 1000}, {"sarc", 1000}, {"tscb", 1000},
|
||||
{"bfsha", 1000}, {"jpg", 1000}, {"beco", 1000}, {"bfarc", 1000}, {"blarc", 1000}, {"belnk", 1000},
|
||||
{"genvb", 1000}, {"bnsh", 1000}, {"bslnk", 1000}, {"baatarc", 1000}, {"bagst", 1000},
|
||||
{"bwav", 0}, {"webm", 0},
|
||||
};
|
||||
const char *leaf = strrchr(key, '/') ? strrchr(key, '/') + 1 : key, *dot = strrchr(leaf, '.');
|
||||
if (!dot) return 1500;
|
||||
for (size_t i = 0; i < sizeof types / sizeof *types; i++)
|
||||
if (!strcmp(dot + 1, types[i].ext)) return types[i].f;
|
||||
return 1500;
|
||||
}
|
||||
|
||||
static uint32_t clamp32(uint64_t v) { return v > 0xFFFFFFF0u ? 0xFFFFFFF0u : (uint32_t)v; }
|
||||
|
||||
uint32_t restbl_estimate(const char *key, uint64_t size) {
|
||||
unsigned f = type_factor(key);
|
||||
if (!f) return 0;
|
||||
uint64_t e = (size * f + 999) / 1000 + 8192;
|
||||
return clamp32((e + 4095) & ~(uint64_t)4095);
|
||||
}
|
||||
|
||||
uint32_t restbl_grow(const char *key, uint32_t entry, uint64_t old, uint64_t new_size, int scaled) {
|
||||
if (new_size <= old) return entry;
|
||||
if (scaled) return clamp32(old ? ((uint64_t)entry * new_size + old - 1) / old : restbl_estimate(key, new_size * 8));
|
||||
unsigned f = type_factor(key);
|
||||
if (!f) return entry;
|
||||
uint64_t e = entry + ((new_size - old) * f + 999) / 1000;
|
||||
return clamp32((e + 31) & ~(uint64_t)31);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// The game's resource size table (restbl.c): System/Resource/ResourceSizeTable.Product.<n>.Nin_NX_NVN.rsizetable.zs, a
|
||||
// zstd-compressed RESTBL. The game sizes the buffer it loads a file into from the file's entry, so a mod file larger
|
||||
// than the original needs a larger entry, and a file a mod adds needs one of its own. romfs.c merges the tables the
|
||||
// mods ship and raises the entries of the files they serve (romfs.c restbl_overlay).
|
||||
#pragma once
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
typedef struct Restbl Restbl;
|
||||
|
||||
// The table of a decompressed RESTBL (copied); NULL when it is not one.
|
||||
Restbl *restbl_parse(const uint8_t *data, size_t n);
|
||||
void restbl_free(Restbl *t);
|
||||
// The table's key of a RomFS path: the path without a trailing .zs or .mc, into buf.
|
||||
const char *restbl_key(const char *path, char *buf, size_t n);
|
||||
// 1 with *size when key has an entry.
|
||||
int restbl_get(const Restbl *t, const char *key, uint32_t *size);
|
||||
// The entry of key raised to size (added when missing); 1 when the table changed.
|
||||
int restbl_raise(Restbl *t, const char *key, uint32_t size);
|
||||
// Every entry of shipped larger than base's (or missing from base) raised in t; the number of entries raised.
|
||||
int restbl_merge(Restbl *t, const Restbl *base, const Restbl *shipped);
|
||||
// Each entry of from whose key one of keys[0..n) names, raised in t; the number raised.
|
||||
int restbl_take(Restbl *t, const Restbl *from, const char *const *keys, size_t n);
|
||||
// The table as a decompressed RESTBL (malloc'd); NULL when out of memory.
|
||||
uint8_t *restbl_write(const Restbl *t, size_t *n);
|
||||
|
||||
// The entry a file of the key's type and decompressed size needs when the game has none for it; 0 for a type the
|
||||
// table never lists (streamed sound and video).
|
||||
uint32_t restbl_estimate(const char *key, uint64_t size);
|
||||
// The entry of a file the game lists with `entry` for its own copy of old bytes, now new bytes: raised by the growth
|
||||
// times the type's factor, never lowered. scaled: the sizes are compressed ones (a mesh codec .mc file), whose entry
|
||||
// grows in proportion.
|
||||
uint32_t restbl_grow(const char *key, uint32_t entry, uint64_t old, uint64_t new_size, int scaled);
|
||||
+178
-3
@@ -22,6 +22,10 @@
|
||||
// hash, no manifest). A path several overlays give is served by one: a player mod higher in the order beats a lower
|
||||
// one, any player mod beats the TOTK_ROMFS_MODS manifest, which beats romfs-patches (overlay_contest). A mod path that
|
||||
// matches a RomFS path only without case is served under the RomFS spelling.
|
||||
//
|
||||
// The resource size table: with a player mod enabled, the game's table is rebuilt at start with the tables the mods
|
||||
// ship merged in and the entries of the mods' files raised to what they need (restbl_overlay), and served from
|
||||
// <profile>/mods/ in place of the game's or a mod's table.
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -35,8 +39,11 @@
|
||||
#include "romfs_config.h"
|
||||
#include "romfs_path.h"
|
||||
#include "platform/platform.h"
|
||||
#include "restbl.h"
|
||||
#include "sha256.h"
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
#define NH 65536 // RomFS handle slots
|
||||
#define READ_CHUNK (64u << 20) // largest single read
|
||||
#define HASH_CHUNK (8u << 20)
|
||||
@@ -342,6 +349,51 @@ static int other_version(const char *path) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---- the resource size table's inputs: the tables the overlays carry, before overlay_contest keeps one of them
|
||||
static char restbl_rel[256]; // the RomFS path of the game's table, "" when it has none
|
||||
typedef struct {
|
||||
char *file;
|
||||
int mod, other; // mod: ModInfo index + 1, 0 for the port's folders; other: made for another game version
|
||||
} Shipped;
|
||||
static Shipped *shipped;
|
||||
static size_t nshipped;
|
||||
|
||||
static int ends_with(const char *s, const char *tail) {
|
||||
size_t n = strlen(s), k = strlen(tail);
|
||||
return n >= k && !strcmp(s + n - k, tail);
|
||||
}
|
||||
|
||||
// 1 when rel is a resource size table of any game version.
|
||||
static int is_restbl(const char *rel) {
|
||||
return !strncmp(rel, "System/Resource/ResourceSizeTable.", 34) && ends_with(rel, ".rsizetable.zs");
|
||||
}
|
||||
|
||||
static void shipped_add(const char *file, int mod, int other) {
|
||||
Shipped *grown = realloc(shipped, (nshipped + 1) * sizeof *shipped);
|
||||
char *copy = strdup(file);
|
||||
need(grown && copy, "resource size table list");
|
||||
shipped = grown;
|
||||
shipped[nshipped++] = (Shipped){copy, mod, other};
|
||||
}
|
||||
|
||||
// The game's table: the one System/Resource/ResourceSizeTable.*.rsizetable.zs of the RomFS.
|
||||
static void restbl_find(void) {
|
||||
for (uint64_t i = lower("System/Resource/ResourceSizeTable."); i < file_count; i++) {
|
||||
const char *n = entry_name(entry(i));
|
||||
if (strncmp(n, "System/Resource/ResourceSizeTable.", 34)) break;
|
||||
if (is_restbl(n) && strlen(n) < sizeof restbl_rel) {
|
||||
snprintf(restbl_rel, sizeof restbl_rel, "%s", n);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The tables the overlays carry for this game version, every one of them (overlay_contest keeps one).
|
||||
static void restbl_collect(void) {
|
||||
for (size_t k = 0; k < noverlay; k++)
|
||||
if (restbl_rel[0] && !strcmp(overlays[k].name, restbl_rel)) shipped_add(overlays[k].path, overlays[k].mod, 0);
|
||||
}
|
||||
|
||||
typedef struct { int k, prio; const char *dir, *rel; } ModWalk;
|
||||
|
||||
static void mod_walk(int k, int prio, const char *dir, const char *rel);
|
||||
@@ -368,6 +420,7 @@ static int mod_walk_cb(const char *leaf, int kind, uint64_t size, void *ctx) {
|
||||
}
|
||||
uint64_t i = index_any_case(path);
|
||||
if (i == UINT64_MAX && other_version(path)) {
|
||||
if (is_restbl(path)) shipped_add(file, w->k + 1, 1); // its entries for this mod's files still apply
|
||||
m->other_version++;
|
||||
return 1;
|
||||
}
|
||||
@@ -544,6 +597,123 @@ static int serve_generated(const char *rel, Overlay *base, void *out, size_t out
|
||||
return 1;
|
||||
}
|
||||
|
||||
// ---- the resource size table (restbl.c)
|
||||
|
||||
// The decompressed size of a file's bytes as the game unpacks them: the content size its zstd frame records (.zs;
|
||||
// the game's files and zstd's own tools write it), four times the bytes for a frame without one, else its size.
|
||||
static uint64_t content_size(PlatFile *f, uint64_t off, uint64_t size, const char *rel) {
|
||||
if (!ends_with(rel, ".zs") || !f) return size;
|
||||
uint8_t head[18];
|
||||
uint32_t n = size < sizeof head ? (uint32_t)size : (uint32_t)sizeof head;
|
||||
if (read_at(f, head, n, off) != n) return size;
|
||||
unsigned long long c = ZSTD_getFrameContentSize(head, n);
|
||||
return c == ZSTD_CONTENTSIZE_ERROR ? size : c == ZSTD_CONTENTSIZE_UNKNOWN ? size * 4 : c;
|
||||
}
|
||||
|
||||
// A table from its zstd bytes; NULL when they are not one.
|
||||
static Restbl *restbl_unpack(const void *z, size_t n) {
|
||||
unsigned long long raw = ZSTD_getFrameContentSize(z, n);
|
||||
if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > (64u << 20)) return NULL;
|
||||
uint8_t *d = malloc((size_t)raw ? (size_t)raw : 1);
|
||||
size_t got = d ? ZSTD_decompress(d, (size_t)raw, z, n) : 0;
|
||||
Restbl *t = d && !ZSTD_isError(got) && got == raw ? restbl_parse(d, (size_t)raw) : NULL;
|
||||
free(d);
|
||||
return t;
|
||||
}
|
||||
|
||||
// With a player mod (or the TOTK_ROMFS_MODS manifest) serving files: the game's table with every shipped table of this
|
||||
// version merged in (an entry larger than the game's wins), the entries a mod's table for another version gives for
|
||||
// that mod's files, and each served mod file's entry raised to what its size needs (restbl_grow for a file the game
|
||||
// has, restbl_estimate for an added one). Never lowers the game's entries.
|
||||
static void restbl_overlay(void) {
|
||||
int player = 0;
|
||||
for (size_t k = 0; k < noverlay; k++) player |= overlays[k].mod || overlays[k].prio == 1;
|
||||
if (!player || !restbl_rel[0]) return;
|
||||
static const char keeps[] = "mod files that grew may fail to load";
|
||||
double t0 = now_s();
|
||||
uint64_t gi = lower(restbl_rel);
|
||||
Entry ge = entry(gi);
|
||||
void *gz = ge.size < (64u << 20) ? malloc((size_t)ge.size) : NULL;
|
||||
Restbl *base = NULL, *out = NULL;
|
||||
if (gz && read_at(image, gz, (uint32_t)ge.size, ge.offset) == (int64_t)ge.size) {
|
||||
base = restbl_unpack(gz, (size_t)ge.size);
|
||||
out = restbl_unpack(gz, (size_t)ge.size);
|
||||
}
|
||||
free(gz);
|
||||
if (!base || !out) {
|
||||
rt_log("RomFS: cannot read the resource size table %s; %s", restbl_rel, keeps);
|
||||
restbl_free(base);
|
||||
restbl_free(out);
|
||||
return;
|
||||
}
|
||||
int tables = 0, merged = 0, raised = 0, estimated = 0;
|
||||
for (size_t s = 0; s < nshipped; s++) {
|
||||
uint64_t n = 0;
|
||||
void *z = plat_file_read(shipped[s].file, &n);
|
||||
Restbl *t = z ? restbl_unpack(z, (size_t)n) : NULL;
|
||||
free(z);
|
||||
if (!t) {
|
||||
rt_log("RomFS: %s is not a resource size table; left out", shipped[s].file);
|
||||
continue;
|
||||
}
|
||||
tables++;
|
||||
if (!shipped[s].other) {
|
||||
merged += restbl_merge(out, base, t);
|
||||
} else { // another version's table: only the entries of the files its mod serves
|
||||
for (size_t k = 0; k < noverlay; k++) {
|
||||
char key[4200];
|
||||
const char *kp = restbl_key(overlays[k].name, key, sizeof key);
|
||||
if (overlays[k].mod == shipped[s].mod) merged += restbl_take(out, t, &kp, 1);
|
||||
}
|
||||
}
|
||||
restbl_free(t);
|
||||
}
|
||||
Overlay *served = NULL;
|
||||
for (size_t k = 0; k < noverlay; k++) {
|
||||
Overlay *o = &overlays[k];
|
||||
if (o->index == gi) served = o;
|
||||
if ((!o->mod && o->prio != 1) || o->index == gi) continue;
|
||||
char key[4200];
|
||||
restbl_key(o->name, key, sizeof key);
|
||||
int mc = ends_with(o->name, ".mc");
|
||||
PlatFile *f = open_read(o->path);
|
||||
uint64_t now = mc ? o->size : content_size(f, 0, o->size, o->name);
|
||||
if (f) plat_file_close(f);
|
||||
uint32_t entry_size, want;
|
||||
if (o->index != ADDED) {
|
||||
if (!restbl_get(base, key, &entry_size)) continue; // the game sizes it without the table (streamed)
|
||||
Entry e = entry(o->index);
|
||||
uint64_t was = mc ? e.size : content_size(image, e.offset, e.size, o->name);
|
||||
want = restbl_grow(key, entry_size, was, now, mc);
|
||||
} else {
|
||||
want = restbl_estimate(key, mc ? now * 8 : now); // a mesh codec file unpacks to several times its size
|
||||
estimated += want != 0;
|
||||
}
|
||||
raised += want && restbl_raise(out, key, want);
|
||||
}
|
||||
restbl_free(base);
|
||||
if (!merged && !raised && !served) {
|
||||
restbl_free(out);
|
||||
rt_log("RomFS: resource size table: the mods' files fit the game's entries (%.0f ms)", (now_s() - t0) * 1000);
|
||||
return;
|
||||
}
|
||||
size_t rn = 0, zn = 0;
|
||||
uint8_t *raw = restbl_write(out, &rn);
|
||||
restbl_free(out);
|
||||
size_t cap = raw ? ZSTD_compressBound(rn) : 0;
|
||||
void *z = raw ? malloc(cap) : NULL;
|
||||
if (z) zn = ZSTD_compress(z, cap, raw, rn, 3);
|
||||
free(raw);
|
||||
if (!z || ZSTD_isError(zn)) {
|
||||
free(z);
|
||||
rt_log("RomFS: cannot pack the merged resource size table; %s", keeps);
|
||||
return;
|
||||
}
|
||||
if (serve_generated(restbl_rel, served, z, zn, "mods", "merged resource sizes", keeps))
|
||||
rt_log("RomFS: resource size table: %d tables merged (%d entries), %d entries raised for mod files (%d "
|
||||
"added files estimated), %.0f ms", tables, merged, raised, estimated, (now_s() - t0) * 1000);
|
||||
}
|
||||
|
||||
// The HeapDef table a bigger game resolution needs (resolution.c): its texture heap and that heap's parent grow. The
|
||||
// base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS original; the grown table is
|
||||
// written to <profile>/resolution/ and served in the base's place.
|
||||
@@ -600,7 +770,9 @@ int romfs_prompts_off(const char *folder) {
|
||||
static const char *overlay_clash(size_t a, size_t b, size_t c, size_t d) {
|
||||
for (size_t i = a; i < b; i++)
|
||||
for (size_t j = c; j < d; j++)
|
||||
if (overlays[i].index != ADDED && overlays[i].index == overlays[j].index) return overlays[i].name;
|
||||
if (overlays[i].index != ADDED && overlays[i].index == overlays[j].index &&
|
||||
strcmp(overlays[i].name, restbl_rel)) // restbl_overlay merges both tables
|
||||
return overlays[i].name;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -624,8 +796,8 @@ static void overlay_init(void) {
|
||||
// the prompts stay until restart even though the layout itself changes live. Each folder replaces romfs-patches as
|
||||
// a whole and the setup builds no folder of another shape with Xbox prompts, so a game resolution that is not 16:9
|
||||
// keeps the Switch prompts. So does a RomFS mod (the TOTK_ROMFS_MODS manifest or a player mod) that replaces one of
|
||||
// the prompts' files (the fonts, the resource size table): romfs-patches loads in their place, so the mod's file
|
||||
// is not served beside prompts files made for the game's own.
|
||||
// the prompts' fonts: romfs-patches loads in their place, so the mod's file is not served beside prompts files made
|
||||
// for the game's own. A mod's resource size table is no clash: restbl_overlay merges it with the prompts' one.
|
||||
char def[2100], path[2100];
|
||||
const char *patches = cfg_get("TOTK_ROMFS_PATCHES"), *mods = cfg_get("TOTK_ROMFS_MODS");
|
||||
int wide = 0;
|
||||
@@ -655,6 +827,7 @@ static void overlay_init(void) {
|
||||
: patches ? ROMFS_PROMPTS_PATCHES
|
||||
: !romfs_prompts_ready(prompts_wanted) ? ROMFS_PROMPTS_MISSING
|
||||
: ROMFS_PROMPTS_OK;
|
||||
restbl_find();
|
||||
size_t first = noverlay;
|
||||
if (prompts_wanted && prompts_off == ROMFS_PROMPTS_OK) {
|
||||
snprintf(path, sizeof path, "%s/%s", index_dir, prompts_wanted); // fits: its manifest path did
|
||||
@@ -689,8 +862,10 @@ static void overlay_init(void) {
|
||||
prompts_off = ROMFS_PROMPTS_MODS;
|
||||
if (plain_ok) overlay_load(plain, 0, 0);
|
||||
}
|
||||
restbl_collect();
|
||||
overlay_contest();
|
||||
mods_overlay_log();
|
||||
restbl_overlay();
|
||||
heapdef_overlay();
|
||||
camera_overlay();
|
||||
added_init();
|
||||
|
||||
@@ -5,16 +5,22 @@
|
||||
Builds a mods folder in a temp directory with files of its own bytes at real RomFS paths (from
|
||||
build/extracted/<version>/romfs/index.json), runs `totk.exe --probe-fs` on it and reads the `mods:` lines of the log.
|
||||
--probe-fs compares the bytes it reads through the RomFS with every overlay file, so a wrong winner fails it too.
|
||||
A second folder checks the merged resource size table the start writes to the profile (runtime/romfs.c
|
||||
restbl_overlay): a grown file's entry, an added file's, a shipped table's and another version's table's.
|
||||
Needs the extracted RomFS of the version.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
import zstandard
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
UNI = 'Ж мод'
|
||||
|
||||
@@ -31,19 +37,94 @@ def pick_paths(version):
|
||||
return p1, p2, p1case
|
||||
|
||||
|
||||
def put(root, mod, rel, data, part='romfs'):
|
||||
def put(root, mod, rel, data, part='romfs', times=64):
|
||||
f = Path(root) / mod / part / rel
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
f.write_bytes(data * 64)
|
||||
f.write_bytes(data * times)
|
||||
|
||||
|
||||
def run(exe, mods, order, disabled=''):
|
||||
"""--probe-fs with a profile of its own beside mods (the merged resource size table is written there)."""
|
||||
env = dict(os.environ, TOTK_MODS_DIR=str(mods), TOTK_MODS_ORDER=order, TOTK_MODS_DISABLED=disabled)
|
||||
r = subprocess.run([str(exe), '--probe-fs'], env=env, capture_output=True, timeout=600)
|
||||
profile = Path(mods).parent / 'profile'
|
||||
r = subprocess.run([str(exe), '--profile', str(profile), '--probe-fs'], env=env, capture_output=True, timeout=600)
|
||||
log = (r.stdout + r.stderr).decode('utf-8', 'replace')
|
||||
return r.returncode, log
|
||||
|
||||
|
||||
# ---- the resource size table (RESTBL: "RESTBL", u32 version, u32 name length, u32 hashed count, u32 named count,
|
||||
# {u32 crc32, u32 size} sorted, then {char[name length], u32 size})
|
||||
def restbl_read(data):
|
||||
raw = zstandard.ZstdDecompressor().decompress(data)
|
||||
assert raw[:6] == b'RESTBL'
|
||||
_, name_len, hashed, named = struct.unpack_from('<4I', raw, 6)
|
||||
out = dict(struct.unpack_from('<II', raw, 22 + i * 8) for i in range(hashed))
|
||||
hashes = [struct.unpack_from('<I', raw, 22 + i * 8)[0] for i in range(hashed)]
|
||||
assert hashes == sorted(hashes), 'hashed entries out of order'
|
||||
at = 22 + hashed * 8
|
||||
for _ in range(named):
|
||||
out[raw[at:at + name_len].split(b'\0')[0].decode()] = struct.unpack_from('<I', raw, at + name_len)[0]
|
||||
at += name_len + 4
|
||||
return out, raw
|
||||
|
||||
|
||||
def restbl_pack(entries, raw):
|
||||
"""raw (a table) with the sizes of its hashed entries set from entries (crc32 -> size, existing hashes only)."""
|
||||
out = bytearray(raw)
|
||||
hashed = struct.unpack_from('<I', raw, 14)[0]
|
||||
for i in range(hashed):
|
||||
h = struct.unpack_from('<I', raw, 22 + i * 8)[0]
|
||||
if h in entries:
|
||||
struct.pack_into('<I', out, 26 + i * 8, entries[h])
|
||||
return zstandard.ZstdCompressor(write_content_size=True).compress(bytes(out))
|
||||
|
||||
|
||||
def key(path):
|
||||
return zlib.crc32((path[:-3] if path.endswith(('.zs', '.mc')) else path).encode())
|
||||
|
||||
|
||||
def restbl_case(exe, version, check, tmp):
|
||||
romfs = ROOT / 'build/extracted' / version / 'romfs'
|
||||
index = json.loads((romfs / 'index.json').read_text())
|
||||
files = {f['path']: f for f in index['files']}
|
||||
table_path = next(p for p in files if p.startswith('System/Resource/ResourceSizeTable.') and
|
||||
p.endswith('.rsizetable.zs'))
|
||||
with (romfs / 'romfs.bin').open('rb') as image:
|
||||
def read(path, n=None):
|
||||
image.seek(files[path]['offset'])
|
||||
return image.read(n or files[path]['size'])
|
||||
game, game_raw = restbl_read(read(table_path))
|
||||
packs = [p for p in files if p.startswith('Pack/Actor/') and p.endswith('.pack.zs') and key(p) in game]
|
||||
grown, shipped_key, old_key, old_other = packs[0], packs[1], packs[2], packs[3]
|
||||
was = zstandard.get_frame_parameters(read(grown, 18)).content_size
|
||||
zs = zstandard.ZstdCompressor(write_content_size=True)
|
||||
mods = Path(tmp) / 'restbl' / 'mods'
|
||||
put(mods, 'Grow', grown, zs.compress(bytes(was + 100000)), times=1)
|
||||
put(mods, 'Grow', 'Pack/Actor/ModTest_Added.pack.zs', zs.compress(bytes(50000)), times=1)
|
||||
put(mods, 'Ships', table_path, restbl_pack({key(shipped_key): game[key(shipped_key)] + 12345}, game_raw), times=1)
|
||||
put(mods, 'Old', old_key, b'old', times=1) # smaller than the original: only its table raises it
|
||||
put(mods, 'Old', table_path.replace('.Product.', '.Product.999', 1) if '.Product.' in table_path else
|
||||
'System/Resource/ResourceSizeTable.Product.999.Nin_NX_NVN.rsizetable.zs',
|
||||
restbl_pack({key(old_key): game[key(old_key)] + 7777, key(old_other): game[key(old_other)] + 5555},
|
||||
game_raw), times=1)
|
||||
code, log = run(exe, mods, 'Grow|Ships|Old')
|
||||
check(code == 0, 'probe-fs passes with the resource size table mods', log)
|
||||
merged_file = mods.parent / 'profile' / 'mods' / table_path.rsplit('/', 1)[1]
|
||||
check(merged_file.is_file(), f'the merged table is written to the profile ({merged_file.name})', log)
|
||||
if not merged_file.is_file():
|
||||
return
|
||||
merged, _ = restbl_read(merged_file.read_bytes())
|
||||
g = game[key(grown)]
|
||||
check(merged[key(grown)] >= g + 100000, f'a file grown by 100000 bytes: entry {g} -> {merged[key(grown)]}')
|
||||
added = merged.get(key('Pack/Actor/ModTest_Added.pack.zs'), 0)
|
||||
check(added >= 50000 + 256, f'an added 50000-byte pack gets an entry ({added})')
|
||||
check(merged[key(shipped_key)] == game[key(shipped_key)] + 12345, 'a shipped table\'s larger entry is merged')
|
||||
check(merged[key(old_key)] == game[key(old_key)] + 7777, 'another version\'s table raises its own mod\'s file')
|
||||
check(merged[key(old_other)] == game[key(old_other)], 'another version\'s table leaves other files alone')
|
||||
lower = [h for h, s in game.items() if merged.get(h, 0) < s]
|
||||
check(not lower, f'no entry of the game\'s table is lowered ({len(lower)})')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exe', type=Path, required=True)
|
||||
@@ -87,6 +168,8 @@ def main():
|
||||
check(code == 0, 'probe-fs passes with Top switched off', log)
|
||||
check('mods: Low: 3 files served (2 replaced, 1 added), 0 overridden' in log, 'Low serves P1 and the added file', log)
|
||||
check('mods: Top: ' not in log.replace('mods: Top: off', ''), 'a switched-off mod serves nothing', log)
|
||||
|
||||
restbl_case(args.exe, args.version, check, tmp)
|
||||
print('mods_romfs_check:', 'FAIL' if fails else 'ok')
|
||||
return 1 if fails else 0
|
||||
|
||||
|
||||
Reference in new issue
Block a user