Grow the texture heap a mod's HeapDef asks for instead of serving that table

HD texture packs made for the emulators ship their own HeapDef with TextureHeap and its parent grown for the
emulators' 8 GB memory option (HDPacks.com's 4K pack: by 3,419,000 KiB). Served as it is, the game asked for room the
application pool does not have and crashed at start (#673); without the file it looped the panic Blood Moon.

Before mem_init sizes the pool, resolution.c reads the HeapDef of every switched-on mod (mods.c mods_peek_romfs, which
lists the mods' RomFS roots without the rest of the scan: mods_scan needs paths main sets later) and counts the largest
TextureHeap growth over the shipped 506,880 KiB like TOTK_TEX_HEAP_EXTRA_MB (the larger of the two, at most 4 GiB). The
pool and the heap cap grow by it, and romfs.c heapdef_overlay serves the game's own table grown by that amount in the
mod's place, so the other heaps keep the sizes of the game version that runs.

Tested with the HDPacks.com 4K Texture Pack v1.0 as players install it, on 1.4.3 and 1.2.1 (RTX 4080 SUPER): the game
loads a save and runs at 60 fps with 0 validation errors, and holds 4.5 GB of game memory pools against 1.2 GB without
the pack. docs/mods.md, README.md and totk.ini.example say so; other packs come with no guarantee.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -35,7 +35,8 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md).
 - Shaders prepared at install instead of during play
 - Mods made for emulators: unpack a mod into the `mods` folder next to `totk.exe`; switch and order them in the menu. RomFS
   mods, TKMM's merged output and ExeFS code patches (`.ips`, `.pchtxt`) work; the game rebuilds itself for code patches
-  ([docs/mods.md](docs/mods.md))
+  ([docs/mods.md](docs/mods.md)). HD texture packs work: the port grows the texture heap a pack asks for by itself. Tested
+  with the HDPacks.com 4K Texture Pack v1.0 on 1.4.3 and 1.2.1; other packs come with no guarantee
 - Atmosphère cheats: put a cheat file into a mod and switch its cheats in the menu ([docs/cheats.md](docs/cheats.md))
 
 ## Planned

docs/mods.md

@@ -87,6 +87,21 @@ The port keys a mod by its folder name. A renamed folder counts as a new mod:
 
 Rename mod folders before you set them up in the menu, not after.
 
+## HD texture packs
+
+Texture packs made for the emulators replace the game's textures with larger ones, so the heap the world's textures
+stream into must grow. Such a pack grows it in its own `System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs`, sized for
+the emulators' 8 GB memory option. The port reads that file at start: the texture heap and the game's memory pool grow
+by the pack's amount, and the game gets the port's own heap table with that growth (the log says
+`resolution: mod <name> ships a HeapDef with TextureHeap ...`). Nothing needs to be set. For a pack without such a file
+that still loops the panic Blood Moon, set `tex_heap_extra_mb` under `[mods]` in `totk.ini` (0 to 4096 MiB).
+
+Tested: **HDPacks.com Tears of the Kingdom 4K Texture Pack v1.0** (`4kTextures`, 10,407 textures, 6.5 GB) on game 1.4.3
+and 1.2.1, RTX 4080 SUPER: it loads into the game from a save and runs at the same frame rate, with about 3.3 GB more of
+textures in memory (4.5 GB of game memory pools against 1.2 GB without it). Other packs are not tested and come with no
+guarantee. Such a pack needs a lot of video memory; on a GPU with less than 8 GB, `astc_bc = 1` under `[performance]`
+keeps its textures at a quarter of the size.
+
 ## What does not work
 
 | On the Mods tab | Why | What to do |

runtime/host.h

@@ -262,6 +262,7 @@ uint64_t res_heap_extra_bytes(void);                       // added to the appli
 uint64_t res_heap_extra_kib(void);                         // added to the HeapDef texture heap and its parent; 0 = none
 uint64_t res_tex_heap_extra_bytes(void);                   // TOTK_TEX_HEAP_EXTRA_MB: added to the application pool and
                                                            // the heap cap (mem_init), 2 MiB steps; 0 = none
+int res_tex_heap_from_mod(void);                           // a switched-on mod's HeapDef grows the texture heap
 int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n);   // the HeapDef table (.byml.zs) with
                                                            // the grown heaps, malloc'd; 0 = cannot (logged)
 void res_heapdef_served(void);                             // romfs.c serves the grown table

runtime/mods.c

@@ -338,22 +338,49 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) {
     return 1;
 }
 
+// The mods folder into out: TOTK_MODS_DIR, else <dir of totk.exe>/mods. 0 when the exe's path is unknown.
+static int mods_folder(char *out, size_t cap) {
+    const char *d = cfg_get("TOTK_MODS_DIR");
+    if (d && *d) return snprintf(out, cap, "%s", d) < (int)cap;
+    char exe[2048];
+    if (!plat_exe_path(exe, sizeof exe)) return 0;
+    char *cut = strrchr(exe, '/'), *back = strrchr(exe, '\\');
+    if (back > cut) cut = back;
+    if (!cut) return 0;
+    *cut = 0;
+    return snprintf(out, cap, "%s/mods", exe) < (int)cap;
+}
+
+typedef struct {
+    const char *folder, *disabled;
+    void (*cb)(const char *name, const char *romfs, void *ctx);
+    void *ctx;
+} Peek;
+
+static int peek_cb(const char *name, int kind, uint64_t size, void *ctx) {
+    (void)size;
+    const Peek *pk = ctx;
+    if (kind != PLAT_FS_DIR || listed(pk->disabled, name)) return 1;
+    char path[2100], romfs[2048];
+    if (snprintf(path, sizeof path, "%s/%s", pk->folder, name) >= (int)sizeof path) return 1;
+    find_root(path, "romfs", romfs, sizeof romfs);
+    if (!romfs[0]) find_root(path, "romfslite", romfs, sizeof romfs);
+    if (romfs[0]) pk->cb(name, romfs, pk->ctx);
+    return 1;
+}
+
+void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), void *ctx) {
+    char folder[2048];
+    if (!mods_folder(folder, sizeof folder) || !plat_dir_exists(folder)) return;
+    Peek pk = {folder, cfg_get("TOTK_MODS_DISABLED"), cb, ctx};
+    plat_fs_list(folder, peek_cb, &pk);
+}
+
 void mods_scan(void) {
     static int done;
     if (done) return;
     done = 1;
-    const char *d = cfg_get("TOTK_MODS_DIR");
-    if (d && *d) {
-        snprintf(dir, sizeof dir, "%s", d);
-    } else {
-        char exe[2048];
-        if (!plat_exe_path(exe, sizeof exe)) return;
-        char *cut = strrchr(exe, '/'), *back = strrchr(exe, '\\');
-        if (back > cut) cut = back;
-        if (!cut) return;
-        *cut = 0;
-        snprintf(dir, sizeof dir, "%s/mods", exe);
-    }
+    if (!mods_folder(dir, sizeof dir)) return;
     if (!plat_dir_exists(dir)) {
         rt_log("mods: no folder %s", dir);
         return;

runtime/mods.h

@@ -40,6 +40,9 @@ typedef struct {
 } ModInfo;
 
 void mods_scan(void);              // once; later calls return at once
+// Every switched-on mod folder's RomFS root, without the rest of the scan (no ExeFS checks, no order): for mem_init,
+// which runs before the paths mods_scan needs are set (resolution.c mod_tex_heap_kib). cb gets the name and root.
+void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), void *ctx);
 const char *mods_dir(void);        // the folder scanned (default: <dir of totk.exe>/mods)
 int mods_count(void);
 ModInfo *mods_at(int i);           // 0 = top of the order; NULL out of range

runtime/resolution.c

@@ -39,6 +39,7 @@
 
 #include "cfg.h"
 #include "host.h"
+#include "mods.h"
 #include "platform/platform.h"
 
 #define BASE_W 1920
@@ -46,6 +47,7 @@
 #define SCENE_W 1600
 #define SCENE_H 900
 #define DTA_KIB 113664u      // DynamicTextureAllocatorMem, shipped
+#define TEX_HEAP_KIB 506880u // TextureHeap, shipped (1.2.1 and 1.4.3): what a mod's own HeapDef is measured against
 #define SCENE_MAX 4096       // per side, as MacNativeGames' TOTK_RENDER_RES
 #define SHADOW_BASE 960      // the sun shadow map's base size (its cascades: a D16 array of 4 layers)
 
@@ -193,7 +195,9 @@ uint64_t res_heap_extra_bytes(void) { return ((res_heap_extra_kib() << 10) + 0x1
 // world's textures stream into. HD texture packs made for emulators with 8 GB of DRAM grow it by about 3.3 GiB in their
 // own HeapDef; without the room the game runs out and starts its panic Blood Moon over and over. The pool and the heap
 // cap grow by the same amount (memory.c mem_init).
-static uint64_t tex_heap_extra_kib(void) {
+static uint64_t mod_tex_heap_kib(void);
+
+static uint64_t cfg_tex_heap_kib(void) {
     const char *v = cfg_get("TOTK_TEX_HEAP_EXTRA_MB");
     if (!v || !*v) return 0;
     char *end;
@@ -206,6 +210,12 @@ static uint64_t tex_heap_extra_kib(void) {
     return (uint64_t)mb << 10;
 }
 
+// The larger of TOTK_TEX_HEAP_EXTRA_MB and what an enabled mod's own HeapDef adds to the texture heap (mod_tex_heap_kib).
+static uint64_t tex_heap_extra_kib(void) {
+    uint64_t c = cfg_tex_heap_kib(), m = mod_tex_heap_kib();
+    return c > m ? c : m;
+}
+
 uint64_t res_tex_heap_extra_bytes(void) { return ((tex_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; }
 
 // ---- the HeapDef table. BYML version 2..7, little endian ("YB"): a hash node is {u8 0xC1, u24 count} and count
@@ -245,28 +255,82 @@ static size_t heap_size_field(const Byml *b, uint32_t heaps, const char *heap) {
     return at && u32_at(b, at, &node) ? field(b, node, "Size", 0xd1) : 0;
 }
 
-int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) {
-    uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib();
-    *out = NULL;
-    if (!extra && !tex) return 0;
+// A HeapDef in b's place (zstd frame in[0..n) with its content size, no dictionary): b.p malloc'd and *heaps its
+// "Heaps" node; 0 when it is not one (b.p freed). why: what the log line says the caller keeps then.
+static int heapdef_open(const void *in, size_t n, Byml *b, uint32_t *heaps, const char *why) {
+    b->p = NULL;
     if (ZSTD_getDictID_fromFrame(in, n)) {
-        rt_log("resolution: the HeapDef table needs a zstd dictionary; the game keeps its own resolution");
+        rt_log("resolution: the HeapDef table needs a zstd dictionary; %s", why);
         return 0;
     }
     unsigned long long raw = ZSTD_getFrameContentSize(in, n);
     if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > (64u << 20)) {
-        rt_log("resolution: the HeapDef table is not a zstd frame with a size; the game keeps its own resolution");
+        rt_log("resolution: the HeapDef table is not a zstd frame with a size; %s", why);
         return 0;
     }
-    Byml b = {malloc((size_t)raw), (size_t)raw, 0};
-    if (!b.p) return 0;
-    size_t got = ZSTD_decompress(b.p, b.n, in, n);
-    uint32_t root, heaps = 0;
+    *b = (Byml){malloc((size_t)raw), (size_t)raw, 0};
+    if (!b->p) return 0;
+    size_t got = ZSTD_decompress(b->p, b->n, in, n);
+    uint32_t root;
     uint16_t ver = 0;
-    if (!ZSTD_isError(got) && got == b.n && b.n >= 16) memcpy(&ver, b.p + 2, 2);
-    size_t heaps_at = 0, dta_at = 0, gfx_at = 0;
-    if (ver >= 2 && ver <= 7 && !memcmp(b.p, "YB", 2) && u32_at(&b, 4, &b.keys) && u32_at(&b, 12, &root) &&
-        (heaps_at = field(&b, root, "Heaps", 0xc1)) && u32_at(&b, heaps_at, &heaps)) {
+    if (!ZSTD_isError(got) && got == b->n && b->n >= 16) memcpy(&ver, b->p + 2, 2);
+    size_t heaps_at = 0;
+    if (ver >= 2 && ver <= 7 && !memcmp(b->p, "YB", 2) && u32_at(b, 4, &b->keys) && u32_at(b, 12, &root) &&
+        (heaps_at = field(b, root, "Heaps", 0xc1)) && u32_at(b, heaps_at, heaps))
+        return 1;
+    free(b->p);
+    b->p = NULL;
+    return 0;
+}
+
+// HD texture packs made for the emulators ship their own HeapDef with "TextureHeap" and its parent grown (HDPacks.com's
+// 4K pack for TotK: by 3,419,000 KiB, 3.3 GiB, for the emulators' 8 GB memory option). Served as it is, the game asks
+// for that room and the application pool has none: it crashed at start (issue #673). So before mem_init sizes the pool,
+// every switched-on mod's HeapDef is read (mods.c mods_peek_romfs: mods_scan needs paths main sets later) and the
+// largest TextureHeap growth over the shipped size counts as TOTK_TEX_HEAP_EXTRA_MB (the larger of the two, at most
+// 4 GiB). romfs.c heapdef_overlay then serves the game's own table grown by that amount in the mod's place, so the
+// other heaps keep the sizes of the game version that runs.
+#define HEAPDEF_REL "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs"
+static struct { int done; uint64_t kib; } MT;
+
+static void mod_heapdef(const char *name, const char *romfs, void *ctx) {
+    (void)ctx;
+    char path[2300];
+    if (snprintf(path, sizeof path, "%s/%s", romfs, HEAPDEF_REL) >= (int)sizeof path) return;
+    uint64_t n = 0;
+    uint8_t *in = plat_file_read(path, &n);
+    if (!in) return;
+    Byml b;
+    uint32_t heaps;
+    size_t th_at = 0;
+    int32_t th = 0;
+    if (heapdef_open(in, (size_t)n, &b, &heaps, "its texture heap is left out") &&
+        (th_at = heap_size_field(&b, heaps, "TextureHeap")))
+        memcpy(&th, b.p + th_at, 4);
+    free(b.p);
+    free(in);
+    int64_t grow = (int64_t)th - (int64_t)TEX_HEAP_KIB;
+    if (grow > (int64_t)(4096u << 10)) grow = 4096u << 10;
+    if (grow > 0 && (uint64_t)grow > MT.kib) MT.kib = (uint64_t)grow;
+    rt_log("resolution: mod %s ships a HeapDef with TextureHeap %d KiB (the game's %u KiB)%s", name, th, TEX_HEAP_KIB,
+           grow > 0 ? ": the port grows the texture heap and the pool by the difference" : "");
+}
+
+static uint64_t mod_tex_heap_kib(void) {
+    if (!MT.done) MT.done = 1, mods_peek_romfs(mod_heapdef, NULL);
+    return MT.kib;
+}
+
+int res_tex_heap_from_mod(void) { return mod_tex_heap_kib() > 0; }
+
+int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) {
+    uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib();
+    *out = NULL;
+    if (!extra && !tex) return 0;
+    Byml b;
+    uint32_t heaps = 0;
+    size_t dta_at = 0, gfx_at = 0;
+    if (heapdef_open(in, n, &b, &heaps, "the game keeps its own resolution")) {
         dta_at = heap_size_field(&b, heaps, "DynamicTextureAllocatorMem");
         gfx_at = heap_size_field(&b, heaps, "Graphics");
     }
@@ -316,8 +380,8 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) {
         rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx,
                gfx2);
     if (tex)
-        rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB)", th,
-               th2, gg, gg2);
+        rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB or a "
+               "mod's HeapDef)", th, th2, gg, gg2);
     return 1;
 }
 

runtime/romfs.c

@@ -535,6 +535,21 @@ static void added_init(void) {
 
 // RomFS file `rel` as the game would read it: from an overlay that replaces it (*base), else from the image. malloc'd;
 // NULL when it is missing, unreadable or 16 MiB or more (logged with `keeps`, what the game does without it).
+// RomFS file rel as the RomFS holds it, whatever overlays replace it; NULL (logged) when it cannot be read.
+static void *read_original(const char *rel, size_t *n, const char *keeps) {
+    uint64_t i = lower(rel);
+    if (i >= file_count || strcmp(entry_name(entry(i)), rel)) return NULL;
+    uint64_t size = entry(i).size;
+    void *in = size && size < (16u << 20) ? malloc((size_t)size) : NULL;
+    if (!in || read_at(image, in, (uint32_t)size, entry(i).offset) != (int64_t)size) {
+        free(in);
+        rt_log("RomFS: cannot read %s; %s", rel, keeps);
+        return NULL;
+    }
+    *n = (size_t)size;
+    return in;
+}
+
 static void *read_served(const char *rel, Overlay **base, size_t *n, const char *keeps) {
     uint64_t i = lower(rel);
     *base = NULL;
@@ -831,7 +846,9 @@ static void restbl_overlay(void) {
 
 // The HeapDef table a bigger game resolution or TOTK_TEX_HEAP_EXTRA_MB needs (resolution.c): its texture heaps and
 // their parents grow. The base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS
-// original; the grown table is written to <profile>/resolution/ and served in the base's place.
+// original; the grown table is written to <profile>/resolution/ and served in the base's place. A mod's table whose
+// texture heap is grown (an HD texture pack's, resolution.c mod_tex_heap_kib) is not the base: that growth is already
+// in the amount, so the RomFS original grows by it and is served in the mod's place.
 #define HEAPDEF "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs"
 static void heapdef_overlay(void) {
     if (!res_heap_extra_kib() && !res_tex_heap_extra_bytes()) return;
@@ -840,6 +857,12 @@ static void heapdef_overlay(void) {
     size_t n;
     void *in = read_served(HEAPDEF, &base, &n, keeps), *out = NULL;
     size_t out_n = 0;
+    if (in && base && base->mod && res_tex_heap_from_mod()) {
+        free(in);
+        in = read_original(HEAPDEF, &n, keeps);
+        rt_log("RomFS: %s: the game's own HeapDef with the mod's texture heap growth in place of the mod's table",
+               mods_at(base->mod - 1)->name);
+    }
     int ok = in && res_heapdef_build(in, n, &out, &out_n);
     free(in);
     if (ok && serve_generated(HEAPDEF, base, out, out_n, "resolution", "grown heaps", keeps)) res_heapdef_served();

totk.ini.example

@@ -367,9 +367,10 @@
 # A rebuild that failed or was cancelled is not started again for the same cheats. Default: auto.
 # mods_rebuild = auto
 # Extra room in MiB (0 to 4096) for the heap the world's textures stream into, for HD texture packs. Packs made for
-# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs),
-# which replaces the port's own: take that file out of the pack and set this instead (about 3400 for the HDPacks 4K
-# pack). Without the room the game keeps starting its panic Blood Moon. The memory pool grows by the same amount.
+# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs):
+# the port reads that growth at start and applies it itself (about 3339 MiB for the HDPacks 4K pack), so such a pack
+# needs nothing here. Set it for a pack that ships textures without a HeapDef. Without the room the game keeps starting
+# its panic Blood Moon. The memory pool grows by the same amount; the larger of this and a mod's growth counts.
 # Read at start. Default: 0. Such a pack also needs much more video memory: astc_bc = 1 ([performance]) keeps its
 # textures at a quarter of it.
 # tex_heap_extra_mb = 0