From a57ea119face0564f1fdae52fe5119475029b980 Mon Sep 17 00:00:00 2001 From: Herbert Costa <246700963+kurerb@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:57:54 -0300 Subject: [PATCH 1/4] Grow the world texture heap for HD texture packs with tex_heap_extra_mb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HD texture packs made for emulators with 8 GB of DRAM, such as the HDPacks 4K pack, grow "TextureHeap" in their own HeapDef table. Served as a mod, that table replaces the one the port grows for the resolution and the shadow map, and the game crashes at boot; without it the pack's textures do not fit, and the game loads and then starts its panic Blood Moon over and over. tex_heap_extra_mb ([mods], 0 to 4096 MiB, read at start) grows TextureHeap and its parent ゲームグラフィックス in the port's HeapDef, and the application pool and the heap cap by the same amount. LOW_SIZE goes from 6 to 10 GiB so the stack and heap block holds the largest heap. At 0 the pool, the heap cap and the HeapDef are what they were. The selftest's "beyond the pool" MapPhysicalMemory asked for 4 GiB, which a grown pool holds; it now asks for just past the pool. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 3 ++- runtime/host.h | 2 ++ runtime/main.c | 4 +++- runtime/memory.c | 12 +++++------ runtime/resolution.c | 50 +++++++++++++++++++++++++++++++++++++++++--- totk.ini.example | 7 +++++++ 6 files changed, 67 insertions(+), 11 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index ec395f97..a9efd386 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -59,7 +59,7 @@ static const char *const known[] = { "SHADER_MEMO", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -82,6 +82,7 @@ static const struct { const char *key, *section; } sections[] = { {"SHADOW_SCALE", "display"}, {"SHADOW_RANGE", "display"}, {"CONTACT_SHADOWS", "display"}, {"PCSS", "display"}, {"MODS_DIR", "mods"}, {"MODS_ORDER", "mods"}, {"MODS_DISABLED", "mods"}, {"MODS_REBUILD", "mods"}, + {"TEX_HEAP_EXTRA_MB", "mods"}, {"MENU_LANG", "language"}, {"LANG", "language"}, {"OPERATION_MODE", "language"}, {"VOLUME", "audio"}, {"AUDIO_CHANNELS", "audio"}, {"MOTION", "input"}, {"RUMBLE_STRENGTH", "input"}, {"RUMBLE_CURVE", "input"}, {"RUMBLE_TRIGGERS", "input"}, diff --git a/runtime/host.h b/runtime/host.h index 21d0534f..5381c3dd 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -260,6 +260,8 @@ void cam_pack_served(void); // romfs.c serves the // ---- game resolution (resolution.c): TOTK_OUTPUT_RES, TOTK_RENDER_SCALE uint64_t res_heap_extra_bytes(void); // added to the application pool (mem_init), 2 MiB steps uint64_t res_heap_extra_kib(void); // added to the HeapDef texture heap and its parent; 0 = none +uint64_t res_tex_heap_extra_bytes(void); // TOTK_TEX_HEAP_EXTRA_MB: added to the application pool and + // the heap cap (mem_init), 2 MiB steps; 0 = none int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n); // the HeapDef table (.byml.zs) with // the grown heaps, malloc'd; 0 = cannot (logged) void res_heapdef_served(void); // romfs.c serves the grown table diff --git a/runtime/main.c b/runtime/main.c index 49d227a0..d2c68575 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -397,7 +397,9 @@ static void check_memory(void) { CHECK(q.state == 0 && q.base == g->alias.base && q.size == g->alias.size, "QueryMemory on the free alias region"); CHECK(mem_map_physical(a, 0x1000) == 0 && LD64(a + 0x10) == 0 && mem_unmap_physical(a, 0x1000) == 0, "remapped physical memory is not zeroed"); - CHECK(mem_map_physical(a, 0x100000000ull) == 0x10801, "MapPhysicalMemory beyond the pool is not LimitReached"); + // past the pool whatever its size (TOTK_APP_POOL, TOTK_TEX_HEAP_EXTRA_MB): 4 GiB fits a grown one + CHECK(mem_map_physical(a, (g->pool + 0x100000) & ~0xfffffull) == 0x10801, + "MapPhysicalMemory beyond the pool is not LimitReached"); mem_query(RT_HOSTFN_BASE + 0x40, &q); CHECK(q.state == 0x10 && q.base == RT_HOSTFN_BASE, "QueryMemory on the host-function window"); mem_query(g->alias.base + g->alias.size + 0x1000, &q); diff --git a/runtime/memory.c b/runtime/memory.c index 4dcc064c..79ccf70c 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -20,11 +20,11 @@ #define PAGE 0x1000ull #define GiB (1ull << 30) #define LOW_SEARCH_FROM 0x100000000ull -#define LOW_SIZE (6 * GiB) +#define LOW_SIZE (10 * GiB) // the heap at its largest: HEAP_CAP plus TOTK_TEX_HEAP_EXTRA_MB's 4 GiB #define STACK_OFF 0x10000000ull #define STACK_SIZE 0x10000000ull #define HEAP_OFF (STACK_OFF + STACK_SIZE + 0x10000000ull) -#define HEAP_CAP (4 * GiB) +#define HEAP_CAP (4 * GiB) // without TOTK_TEX_HEAP_EXTRA_MB, which raises it by its own size (mem_init) #define ALIAS_SIZE (64 * GiB) #define ALIAS_ALIGN (4 * GiB) #define ALIAS_PAGES (ALIAS_SIZE / PAGE) @@ -223,7 +223,7 @@ void mem_init(void) { uint64_t low = 0, alias = 0; for (uint64_t at = LOW_SEARCH_FROM; at + LOW_SIZE <= mods_lo && !low; at += GiB) if (reserve_at(at, LOW_SIZE, 1)) low = at; - if (!low) reexec_or_fail("a 6 GiB stack and heap block below 2^39"); + if (!low) reexec_or_fail("a 10 GiB stack and heap block below 2^39"); for (uint64_t at = align_up(low + LOW_SIZE, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) if (reserve_at(at, ALIAS_SIZE, 1)) alias = at; if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39"); @@ -238,11 +238,11 @@ void mem_init(void) { L = (RtGuestLayout){ .aslr = {low, RT_ADDRESS_LIMIT - low}, .stack = {low + STACK_OFF, STACK_SIZE}, - .heap = {low + HEAP_OFF, HEAP_CAP}, + .heap = {low + HEAP_OFF, HEAP_CAP + res_tex_heap_extra_bytes()}, .alias = {alias, ALIAS_SIZE}, .kernel = {RT_KERNEL_BASE, RT_KERNEL_SIZE}, .hostfn = {RT_HOSTFN_BASE, RT_HOSTFN_SIZE}, - .pool = pool ? strtoull(pool, NULL, 0) : DEFAULT_POOL + res_heap_extra_bytes(), // resolution.c + .pool = pool ? strtoull(pool, NULL, 0) : DEFAULT_POOL + res_heap_extra_bytes() + res_tex_heap_extra_bytes(), }; if (!L.pool) rt_fatal("memory: TOTK_APP_POOL is 0"); } @@ -276,7 +276,7 @@ static int inside(uint64_t a, uint64_t n, RtRange r) { return a >= r.base && n < // ---- 0x01 SetHeapSize uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { plat_mutex_lock(&lock); - if (size > HEAP_CAP || size % 0x200000 || size + image_bytes() > L.pool) { + if (size > L.heap.size || size % 0x200000 || size + image_bytes() > L.pool) { plat_mutex_unlock(&lock); return R_INVALID_STATE; } diff --git a/runtime/resolution.c b/runtime/resolution.c index 3e70fefa..157e2b00 100644 --- a/runtime/resolution.c +++ b/runtime/resolution.c @@ -189,6 +189,25 @@ uint64_t res_heap_extra_kib(void) { uint64_t res_heap_extra_bytes(void) { return ((res_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; } +// TOTK_TEX_HEAP_EXTRA_MB (0 to 4096): more room in "TextureHeap" and its parent "ゲームグラフィックス", the heap the +// world's textures stream into. HD texture packs made for emulators with 8 GB of DRAM grow it by about 3.3 GiB in their +// own HeapDef; without the room the game runs out and starts its panic Blood Moon over and over. The pool and the heap +// cap grow by the same amount (memory.c mem_init). +static uint64_t tex_heap_extra_kib(void) { + const char *v = cfg_get("TOTK_TEX_HEAP_EXTRA_MB"); + if (!v || !*v) return 0; + char *end; + long mb = strtol(v, &end, 10); + if (end == v || *end || mb < 0 || mb > 4096) { + static int logged; + if (!logged++) rt_log("resolution: TOTK_TEX_HEAP_EXTRA_MB=%s is not 0..4096; 0", v); + return 0; + } + return (uint64_t)mb << 10; +} + +uint64_t res_tex_heap_extra_bytes(void) { return ((tex_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; } + // ---- the HeapDef table. BYML version 2..7, little endian ("YB"): a hash node is {u8 0xC1, u24 count} and count // entries {u24 key index, u8 type, u32 value}; an int (type 0xD1) is stored in the entry, so a size changes in place. typedef struct { uint8_t *p; size_t n; uint32_t keys; } Byml; @@ -227,9 +246,9 @@ static size_t heap_size_field(const Byml *b, uint32_t heaps, const char *heap) { } int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { - uint64_t extra = res_heap_extra_kib(); + uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib(); *out = NULL; - if (!extra) return 0; + if (!extra && !tex) return 0; if (ZSTD_getDictID_fromFrame(in, n)) { rt_log("resolution: the HeapDef table needs a zstd dictionary; the game keeps its own resolution"); return 0; @@ -263,6 +282,26 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { int32_t dta2 = dta + (int32_t)extra, gfx2 = gfx + (int32_t)extra; memcpy(b.p + dta_at, &dta2, 4); memcpy(b.p + gfx_at, &gfx2, 4); + int32_t th = 0, th2 = 0, gg = 0, gg2 = 0; + if (tex) { // TOTK_TEX_HEAP_EXTRA_MB: the world texture heap and its parent grow by the same amount + static const char parent[] = "\xe3\x82\xb2\xe3\x83\xbc\xe3\x83\xa0\xe3\x82\xb0\xe3\x83\xa9\xe3\x83\x95" + "\xe3\x82\xa3\xe3\x83\x83\xe3\x82\xaf\xe3\x82\xb9"; // ゲームグラフィックス + size_t th_at = heap_size_field(&b, heaps, "TextureHeap"), gg_at = heap_size_field(&b, heaps, parent); + if (th_at && gg_at) { + memcpy(&th, b.p + th_at, 4); + memcpy(&gg, b.p + gg_at, 4); + th2 = th + (int32_t)tex, gg2 = gg + (int32_t)tex; + memcpy(b.p + th_at, &th2, 4); + memcpy(b.p + gg_at, &gg2, 4); + } else { + rt_log("resolution: the HeapDef table has no TextureHeap sizes; TOTK_TEX_HEAP_EXTRA_MB left out"); + tex = 0; + if (!extra) { + free(b.p); + return 0; + } + } + } size_t cap = ZSTD_compressBound(b.n); void *z = malloc(cap); size_t zn = z ? ZSTD_compress(z, cap, b.p, b.n, 9) : 0; @@ -273,7 +312,12 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { } *out = z; *out_n = zn; - rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx, gfx2); + if (extra) + rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx, + gfx2); + if (tex) + rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB)", th, + th2, gg, gg2); return 1; } diff --git a/totk.ini.example b/totk.ini.example index 3ef2537b..c3f9f743 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -366,6 +366,13 @@ # the setup's rebuild at start (a few minutes) and starts again after it. off: the game starts with the code it has. # A rebuild that failed or was cancelled is not started again for the same cheats. Default: auto. # mods_rebuild = auto +# Extra room in MiB (0 to 4096) for the heap the world's textures stream into, for HD texture packs. Packs made for +# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs), +# which replaces the port's own: take that file out of the pack and set this instead (about 3400 for the HDPacks 4K +# pack). Without the room the game keeps starting its panic Blood Moon. The memory pool grows by the same amount. +# Read at start. Default: 0. Such a pack also needs much more video memory: astc_bc = 1 ([performance]) keeps its +# textures at a quarter of it. +# tex_heap_extra_mb = 0 [files] # Developer overlay: a folder with RomFS files and a manifest.tsv (tools/make_romfs_patches.py format). Players put From 2a70683007ad0e90397907c8eab1bde6194d682c Mon Sep 17 00:00:00 2001 From: Herbert Costa <246700963+kurerb@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:57:54 -0300 Subject: [PATCH 2/4] Grow the HeapDef for tex_heap_extra_mb at the game's own resolution too heapdef_overlay returned at once when res_heap_extra_kib() was 0, which it is at 1920x1080 with the game's scene size and shadow map, so tex_heap_extra_mb did nothing there and a 1080p player with an HD pack still hit the Blood Moon loop. It now also builds the table when res_tex_heap_extra_bytes() is set. Found by itaskaev2 in review. Co-Authored-By: Claude Opus 5.5 --- runtime/romfs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/romfs.c b/runtime/romfs.c index c062c4b2..c14b9aa6 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -829,12 +829,12 @@ static void restbl_overlay(void) { members, packs, (now_s() - t0) * 1000); } -// The HeapDef table a bigger game resolution needs (resolution.c): its texture heap and that heap's parent grow. The -// base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS original; the grown table is -// written to /resolution/ and served in the base's place. +// The HeapDef table a bigger game resolution or TOTK_TEX_HEAP_EXTRA_MB needs (resolution.c): its texture heaps and +// their parents grow. The base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS +// original; the grown table is written to /resolution/ and served in the base's place. #define HEAPDEF "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" static void heapdef_overlay(void) { - if (!res_heap_extra_kib()) return; + if (!res_heap_extra_kib() && !res_tex_heap_extra_bytes()) return; static const char keeps[] = "the game keeps its own resolution"; Overlay *base; size_t n; From 8ac207f3537add01f41bd564f0ae10cfa93318a1 Mon Sep 17 00:00:00 2001 From: itaskaev2 <165683571+itaskaev2@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:28:25 +0100 Subject: [PATCH 3/4] Grow the texture heap a mod's HeapDef asks for instead of serving that table HD texture packs made for the emulators ship their own HeapDef with TextureHeap and its parent grown for the emulators' 8 GB memory option (HDPacks.com's 4K pack: by 3,419,000 KiB). Served as it is, the game asked for room the application pool does not have and crashed at start (#673); without the file it looped the panic Blood Moon. Before mem_init sizes the pool, resolution.c reads the HeapDef of every switched-on mod (mods.c mods_peek_romfs, which lists the mods' RomFS roots without the rest of the scan: mods_scan needs paths main sets later) and counts the largest TextureHeap growth over the shipped 506,880 KiB like TOTK_TEX_HEAP_EXTRA_MB (the larger of the two, at most 4 GiB). The pool and the heap cap grow by it, and romfs.c heapdef_overlay serves the game's own table grown by that amount in the mod's place, so the other heaps keep the sizes of the game version that runs. Tested with the HDPacks.com 4K Texture Pack v1.0 as players install it, on 1.4.3 and 1.2.1 (RTX 4080 SUPER): the game loads a save and runs at 60 fps with 0 validation errors, and holds 4.5 GB of game memory pools against 1.2 GB without the pack. docs/mods.md, README.md and totk.ini.example say so; other packs come with no guarantee. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- docs/mods.md | 15 +++++++ runtime/host.h | 1 + runtime/mods.c | 51 +++++++++++++++++------ runtime/mods.h | 3 ++ runtime/resolution.c | 98 ++++++++++++++++++++++++++++++++++++-------- runtime/romfs.c | 25 ++++++++++- totk.ini.example | 7 ++-- 8 files changed, 169 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index c0ffc147..38780dce 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,8 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). - Shaders prepared at install instead of during play - Mods made for emulators: unpack a mod into the `mods` folder next to `totk.exe`; switch and order them in the menu. RomFS mods, TKMM's merged output and ExeFS code patches (`.ips`, `.pchtxt`) work; the game rebuilds itself for code patches - ([docs/mods.md](docs/mods.md)) + ([docs/mods.md](docs/mods.md)). HD texture packs work: the port grows the texture heap a pack asks for by itself. Tested + with the HDPacks.com 4K Texture Pack v1.0 on 1.4.3 and 1.2.1; other packs come with no guarantee - Atmosphère cheats: put a cheat file into a mod and switch its cheats in the menu ([docs/cheats.md](docs/cheats.md)) ## Planned diff --git a/docs/mods.md b/docs/mods.md index d26553db..0734f382 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -87,6 +87,21 @@ The port keys a mod by its folder name. A renamed folder counts as a new mod: Rename mod folders before you set them up in the menu, not after. +## HD texture packs + +Texture packs made for the emulators replace the game's textures with larger ones, so the heap the world's textures +stream into must grow. Such a pack grows it in its own `System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs`, sized for +the emulators' 8 GB memory option. The port reads that file at start: the texture heap and the game's memory pool grow +by the pack's amount, and the game gets the port's own heap table with that growth (the log says +`resolution: mod ships a HeapDef with TextureHeap ...`). Nothing needs to be set. For a pack without such a file +that still loops the panic Blood Moon, set `tex_heap_extra_mb` under `[mods]` in `totk.ini` (0 to 4096 MiB). + +Tested: **HDPacks.com Tears of the Kingdom 4K Texture Pack v1.0** (`4kTextures`, 10,407 textures, 6.5 GB) on game 1.4.3 +and 1.2.1, RTX 4080 SUPER: it loads into the game from a save and runs at the same frame rate, with about 3.3 GB more of +textures in memory (4.5 GB of game memory pools against 1.2 GB without it). Other packs are not tested and come with no +guarantee. Such a pack needs a lot of video memory; on a GPU with less than 8 GB, `astc_bc = 1` under `[performance]` +keeps its textures at a quarter of the size. + ## What does not work | On the Mods tab | Why | What to do | diff --git a/runtime/host.h b/runtime/host.h index 5381c3dd..d46336b9 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -262,6 +262,7 @@ uint64_t res_heap_extra_bytes(void); // added to the appli uint64_t res_heap_extra_kib(void); // added to the HeapDef texture heap and its parent; 0 = none uint64_t res_tex_heap_extra_bytes(void); // TOTK_TEX_HEAP_EXTRA_MB: added to the application pool and // the heap cap (mem_init), 2 MiB steps; 0 = none +int res_tex_heap_from_mod(void); // a switched-on mod's HeapDef grows the texture heap int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n); // the HeapDef table (.byml.zs) with // the grown heaps, malloc'd; 0 = cannot (logged) void res_heapdef_served(void); // romfs.c serves the grown table diff --git a/runtime/mods.c b/runtime/mods.c index 4f3ad08c..d960a407 100644 --- a/runtime/mods.c +++ b/runtime/mods.c @@ -338,22 +338,49 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) { return 1; } +// The mods folder into out: TOTK_MODS_DIR, else /mods. 0 when the exe's path is unknown. +static int mods_folder(char *out, size_t cap) { + const char *d = cfg_get("TOTK_MODS_DIR"); + if (d && *d) return snprintf(out, cap, "%s", d) < (int)cap; + char exe[2048]; + if (!plat_exe_path(exe, sizeof exe)) return 0; + char *cut = strrchr(exe, '/'), *back = strrchr(exe, '\\'); + if (back > cut) cut = back; + if (!cut) return 0; + *cut = 0; + return snprintf(out, cap, "%s/mods", exe) < (int)cap; +} + +typedef struct { + const char *folder, *disabled; + void (*cb)(const char *name, const char *romfs, void *ctx); + void *ctx; +} Peek; + +static int peek_cb(const char *name, int kind, uint64_t size, void *ctx) { + (void)size; + const Peek *pk = ctx; + if (kind != PLAT_FS_DIR || listed(pk->disabled, name)) return 1; + char path[2100], romfs[2048]; + if (snprintf(path, sizeof path, "%s/%s", pk->folder, name) >= (int)sizeof path) return 1; + find_root(path, "romfs", romfs, sizeof romfs); + if (!romfs[0]) find_root(path, "romfslite", romfs, sizeof romfs); + if (romfs[0]) pk->cb(name, romfs, pk->ctx); + return 1; +} + +void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), void *ctx) { + char folder[2048]; + if (!mods_folder(folder, sizeof folder) || !plat_dir_exists(folder)) return; + Peek pk = {folder, cfg_get("TOTK_MODS_DISABLED"), cb, ctx}; + plat_fs_list(folder, peek_cb, &pk); +} + void mods_scan(void) { static int done; if (done) return; done = 1; - const char *d = cfg_get("TOTK_MODS_DIR"); - if (d && *d) { - snprintf(dir, sizeof dir, "%s", d); - } else { - char exe[2048]; - if (!plat_exe_path(exe, sizeof exe)) return; - char *cut = strrchr(exe, '/'), *back = strrchr(exe, '\\'); - if (back > cut) cut = back; - if (!cut) return; - *cut = 0; - snprintf(dir, sizeof dir, "%s/mods", exe); - } + if (!mods_folder(dir, sizeof dir)) return; if (!plat_dir_exists(dir)) { rt_log("mods: no folder %s", dir); return; diff --git a/runtime/mods.h b/runtime/mods.h index 98a513db..318a9632 100644 --- a/runtime/mods.h +++ b/runtime/mods.h @@ -40,6 +40,9 @@ typedef struct { } ModInfo; void mods_scan(void); // once; later calls return at once +// Every switched-on mod folder's RomFS root, without the rest of the scan (no ExeFS checks, no order): for mem_init, +// which runs before the paths mods_scan needs are set (resolution.c mod_tex_heap_kib). cb gets the name and root. +void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), void *ctx); const char *mods_dir(void); // the folder scanned (default: /mods) int mods_count(void); ModInfo *mods_at(int i); // 0 = top of the order; NULL out of range diff --git a/runtime/resolution.c b/runtime/resolution.c index 157e2b00..ac4fa75c 100644 --- a/runtime/resolution.c +++ b/runtime/resolution.c @@ -39,6 +39,7 @@ #include "cfg.h" #include "host.h" +#include "mods.h" #include "platform/platform.h" #define BASE_W 1920 @@ -46,6 +47,7 @@ #define SCENE_W 1600 #define SCENE_H 900 #define DTA_KIB 113664u // DynamicTextureAllocatorMem, shipped +#define TEX_HEAP_KIB 506880u // TextureHeap, shipped (1.2.1 and 1.4.3): what a mod's own HeapDef is measured against #define SCENE_MAX 4096 // per side, as MacNativeGames' TOTK_RENDER_RES #define SHADOW_BASE 960 // the sun shadow map's base size (its cascades: a D16 array of 4 layers) @@ -193,7 +195,9 @@ uint64_t res_heap_extra_bytes(void) { return ((res_heap_extra_kib() << 10) + 0x1 // world's textures stream into. HD texture packs made for emulators with 8 GB of DRAM grow it by about 3.3 GiB in their // own HeapDef; without the room the game runs out and starts its panic Blood Moon over and over. The pool and the heap // cap grow by the same amount (memory.c mem_init). -static uint64_t tex_heap_extra_kib(void) { +static uint64_t mod_tex_heap_kib(void); + +static uint64_t cfg_tex_heap_kib(void) { const char *v = cfg_get("TOTK_TEX_HEAP_EXTRA_MB"); if (!v || !*v) return 0; char *end; @@ -206,6 +210,12 @@ static uint64_t tex_heap_extra_kib(void) { return (uint64_t)mb << 10; } +// The larger of TOTK_TEX_HEAP_EXTRA_MB and what an enabled mod's own HeapDef adds to the texture heap (mod_tex_heap_kib). +static uint64_t tex_heap_extra_kib(void) { + uint64_t c = cfg_tex_heap_kib(), m = mod_tex_heap_kib(); + return c > m ? c : m; +} + uint64_t res_tex_heap_extra_bytes(void) { return ((tex_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; } // ---- the HeapDef table. BYML version 2..7, little endian ("YB"): a hash node is {u8 0xC1, u24 count} and count @@ -245,28 +255,82 @@ static size_t heap_size_field(const Byml *b, uint32_t heaps, const char *heap) { return at && u32_at(b, at, &node) ? field(b, node, "Size", 0xd1) : 0; } -int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { - uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib(); - *out = NULL; - if (!extra && !tex) return 0; +// A HeapDef in b's place (zstd frame in[0..n) with its content size, no dictionary): b.p malloc'd and *heaps its +// "Heaps" node; 0 when it is not one (b.p freed). why: what the log line says the caller keeps then. +static int heapdef_open(const void *in, size_t n, Byml *b, uint32_t *heaps, const char *why) { + b->p = NULL; if (ZSTD_getDictID_fromFrame(in, n)) { - rt_log("resolution: the HeapDef table needs a zstd dictionary; the game keeps its own resolution"); + rt_log("resolution: the HeapDef table needs a zstd dictionary; %s", why); return 0; } unsigned long long raw = ZSTD_getFrameContentSize(in, n); if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > (64u << 20)) { - rt_log("resolution: the HeapDef table is not a zstd frame with a size; the game keeps its own resolution"); + rt_log("resolution: the HeapDef table is not a zstd frame with a size; %s", why); return 0; } - Byml b = {malloc((size_t)raw), (size_t)raw, 0}; - if (!b.p) return 0; - size_t got = ZSTD_decompress(b.p, b.n, in, n); - uint32_t root, heaps = 0; + *b = (Byml){malloc((size_t)raw), (size_t)raw, 0}; + if (!b->p) return 0; + size_t got = ZSTD_decompress(b->p, b->n, in, n); + uint32_t root; uint16_t ver = 0; - if (!ZSTD_isError(got) && got == b.n && b.n >= 16) memcpy(&ver, b.p + 2, 2); - size_t heaps_at = 0, dta_at = 0, gfx_at = 0; - if (ver >= 2 && ver <= 7 && !memcmp(b.p, "YB", 2) && u32_at(&b, 4, &b.keys) && u32_at(&b, 12, &root) && - (heaps_at = field(&b, root, "Heaps", 0xc1)) && u32_at(&b, heaps_at, &heaps)) { + if (!ZSTD_isError(got) && got == b->n && b->n >= 16) memcpy(&ver, b->p + 2, 2); + size_t heaps_at = 0; + if (ver >= 2 && ver <= 7 && !memcmp(b->p, "YB", 2) && u32_at(b, 4, &b->keys) && u32_at(b, 12, &root) && + (heaps_at = field(b, root, "Heaps", 0xc1)) && u32_at(b, heaps_at, heaps)) + return 1; + free(b->p); + b->p = NULL; + return 0; +} + +// HD texture packs made for the emulators ship their own HeapDef with "TextureHeap" and its parent grown (HDPacks.com's +// 4K pack for TotK: by 3,419,000 KiB, 3.3 GiB, for the emulators' 8 GB memory option). Served as it is, the game asks +// for that room and the application pool has none: it crashed at start (issue #673). So before mem_init sizes the pool, +// every switched-on mod's HeapDef is read (mods.c mods_peek_romfs: mods_scan needs paths main sets later) and the +// largest TextureHeap growth over the shipped size counts as TOTK_TEX_HEAP_EXTRA_MB (the larger of the two, at most +// 4 GiB). romfs.c heapdef_overlay then serves the game's own table grown by that amount in the mod's place, so the +// other heaps keep the sizes of the game version that runs. +#define HEAPDEF_REL "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" +static struct { int done; uint64_t kib; } MT; + +static void mod_heapdef(const char *name, const char *romfs, void *ctx) { + (void)ctx; + char path[2300]; + if (snprintf(path, sizeof path, "%s/%s", romfs, HEAPDEF_REL) >= (int)sizeof path) return; + uint64_t n = 0; + uint8_t *in = plat_file_read(path, &n); + if (!in) return; + Byml b; + uint32_t heaps; + size_t th_at = 0; + int32_t th = 0; + if (heapdef_open(in, (size_t)n, &b, &heaps, "its texture heap is left out") && + (th_at = heap_size_field(&b, heaps, "TextureHeap"))) + memcpy(&th, b.p + th_at, 4); + free(b.p); + free(in); + int64_t grow = (int64_t)th - (int64_t)TEX_HEAP_KIB; + if (grow > (int64_t)(4096u << 10)) grow = 4096u << 10; + if (grow > 0 && (uint64_t)grow > MT.kib) MT.kib = (uint64_t)grow; + rt_log("resolution: mod %s ships a HeapDef with TextureHeap %d KiB (the game's %u KiB)%s", name, th, TEX_HEAP_KIB, + grow > 0 ? ": the port grows the texture heap and the pool by the difference" : ""); +} + +static uint64_t mod_tex_heap_kib(void) { + if (!MT.done) MT.done = 1, mods_peek_romfs(mod_heapdef, NULL); + return MT.kib; +} + +int res_tex_heap_from_mod(void) { return mod_tex_heap_kib() > 0; } + +int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { + uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib(); + *out = NULL; + if (!extra && !tex) return 0; + Byml b; + uint32_t heaps = 0; + size_t dta_at = 0, gfx_at = 0; + if (heapdef_open(in, n, &b, &heaps, "the game keeps its own resolution")) { dta_at = heap_size_field(&b, heaps, "DynamicTextureAllocatorMem"); gfx_at = heap_size_field(&b, heaps, "Graphics"); } @@ -316,8 +380,8 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx, gfx2); if (tex) - rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB)", th, - th2, gg, gg2); + rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB or a " + "mod's HeapDef)", th, th2, gg, gg2); return 1; } diff --git a/runtime/romfs.c b/runtime/romfs.c index c14b9aa6..38de9ab2 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -535,6 +535,21 @@ static void added_init(void) { // RomFS file `rel` as the game would read it: from an overlay that replaces it (*base), else from the image. malloc'd; // NULL when it is missing, unreadable or 16 MiB or more (logged with `keeps`, what the game does without it). +// RomFS file rel as the RomFS holds it, whatever overlays replace it; NULL (logged) when it cannot be read. +static void *read_original(const char *rel, size_t *n, const char *keeps) { + uint64_t i = lower(rel); + if (i >= file_count || strcmp(entry_name(entry(i)), rel)) return NULL; + uint64_t size = entry(i).size; + void *in = size && size < (16u << 20) ? malloc((size_t)size) : NULL; + if (!in || read_at(image, in, (uint32_t)size, entry(i).offset) != (int64_t)size) { + free(in); + rt_log("RomFS: cannot read %s; %s", rel, keeps); + return NULL; + } + *n = (size_t)size; + return in; +} + static void *read_served(const char *rel, Overlay **base, size_t *n, const char *keeps) { uint64_t i = lower(rel); *base = NULL; @@ -831,7 +846,9 @@ static void restbl_overlay(void) { // The HeapDef table a bigger game resolution or TOTK_TEX_HEAP_EXTRA_MB needs (resolution.c): its texture heaps and // their parents grow. The base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS -// original; the grown table is written to /resolution/ and served in the base's place. +// original; the grown table is written to /resolution/ and served in the base's place. A mod's table whose +// texture heap is grown (an HD texture pack's, resolution.c mod_tex_heap_kib) is not the base: that growth is already +// in the amount, so the RomFS original grows by it and is served in the mod's place. #define HEAPDEF "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" static void heapdef_overlay(void) { if (!res_heap_extra_kib() && !res_tex_heap_extra_bytes()) return; @@ -840,6 +857,12 @@ static void heapdef_overlay(void) { size_t n; void *in = read_served(HEAPDEF, &base, &n, keeps), *out = NULL; size_t out_n = 0; + if (in && base && base->mod && res_tex_heap_from_mod()) { + free(in); + in = read_original(HEAPDEF, &n, keeps); + rt_log("RomFS: %s: the game's own HeapDef with the mod's texture heap growth in place of the mod's table", + mods_at(base->mod - 1)->name); + } int ok = in && res_heapdef_build(in, n, &out, &out_n); free(in); if (ok && serve_generated(HEAPDEF, base, out, out_n, "resolution", "grown heaps", keeps)) res_heapdef_served(); diff --git a/totk.ini.example b/totk.ini.example index c3f9f743..4a09a50c 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -367,9 +367,10 @@ # A rebuild that failed or was cancelled is not started again for the same cheats. Default: auto. # mods_rebuild = auto # Extra room in MiB (0 to 4096) for the heap the world's textures stream into, for HD texture packs. Packs made for -# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs), -# which replaces the port's own: take that file out of the pack and set this instead (about 3400 for the HDPacks 4K -# pack). Without the room the game keeps starting its panic Blood Moon. The memory pool grows by the same amount. +# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs): +# the port reads that growth at start and applies it itself (about 3339 MiB for the HDPacks 4K pack), so such a pack +# needs nothing here. Set it for a pack that ships textures without a HeapDef. Without the room the game keeps starting +# its panic Blood Moon. The memory pool grows by the same amount; the larger of this and a mod's growth counts. # Read at start. Default: 0. Such a pack also needs much more video memory: astc_bc = 1 ([performance]) keeps its # textures at a quarter of it. # tex_heap_extra_mb = 0 From 2ad82557fd73777de3e1071c1dbc09f40ef95cb1 Mon Sep 17 00:00:00 2001 From: itaskaev2 <165683571+itaskaev2@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:43:01 +0100 Subject: [PATCH 4/4] Reserve the larger stack and heap block only for a texture heap extra The block below the module window stays 6 GiB, as on dev, and grows by the texture heap extra (tex_heap_extra_mb or a mod's HeapDef) rounded up to a GiB. A player without an HD pack reserves nothing more. Co-Authored-By: Claude Opus 5.5 --- runtime/memory.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/runtime/memory.c b/runtime/memory.c index 79ccf70c..019c0fab 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -20,7 +20,7 @@ #define PAGE 0x1000ull #define GiB (1ull << 30) #define LOW_SEARCH_FROM 0x100000000ull -#define LOW_SIZE (10 * GiB) // the heap at its largest: HEAP_CAP plus TOTK_TEX_HEAP_EXTRA_MB's 4 GiB +#define LOW_BASE (6 * GiB) // the stack and heap block without a texture heap extra #define STACK_OFF 0x10000000ull #define STACK_SIZE 0x10000000ull #define HEAP_OFF (STACK_OFF + STACK_SIZE + 0x10000000ull) @@ -48,6 +48,9 @@ enum { ATTR_PERMISSION_LOCKED = 16 }; static RtGuestLayout L; static uint64_t mods_lo, mods_hi; // fixed module window +// The stack and heap block: LOW_BASE, plus the texture heap extra (tex_heap_extra_mb or a mod's HeapDef, +// res_tex_heap_extra_bytes) rounded up to a GiB, so only a player who grows the texture heap reserves more (mem_init). +static uint64_t low_size = LOW_BASE; static PlatMutex lock = PLAT_MUTEX_INIT; static uint64_t heap_size, alias_bytes; static uint8_t *alias_pages; // 1 per mapped 4 KiB page of the alias region @@ -78,7 +81,7 @@ static volatile int64_t frz_count[4]; // freezes; thaws by a write fault, by m static int watched(uint64_t addr, uint64_t size) { const RtGuestLayout *g = &L; - return (addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size) || + return (addr >= g->aslr.base && size <= low_size && addr - g->aslr.base <= low_size - size) || (addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size); } @@ -221,10 +224,15 @@ void mem_init(void) { // Stack and heap block, then the alias region above it, both searched below the module window. uint64_t low = 0, alias = 0; - for (uint64_t at = LOW_SEARCH_FROM; at + LOW_SIZE <= mods_lo && !low; at += GiB) - if (reserve_at(at, LOW_SIZE, 1)) low = at; - if (!low) reexec_or_fail("a 10 GiB stack and heap block below 2^39"); - for (uint64_t at = align_up(low + LOW_SIZE, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) + low_size = LOW_BASE + align_up(res_tex_heap_extra_bytes(), GiB); + for (uint64_t at = LOW_SEARCH_FROM; at + low_size <= mods_lo && !low; at += GiB) + if (reserve_at(at, low_size, 1)) low = at; + if (!low) { + char what[64]; + snprintf(what, sizeof what, "a %llu GiB stack and heap block below 2^39", (unsigned long long)(low_size / GiB)); + reexec_or_fail(what); + } + for (uint64_t at = align_up(low + low_size, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) if (reserve_at(at, ALIAS_SIZE, 1)) alias = at; if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39"); @@ -506,7 +514,7 @@ static void taken_gap(uint64_t a, uint64_t *base, uint64_t *size) { uint64_t lo = 0, hi = RT_ADDRESS_LIMIT; RtRange known[3 + MOD_COUNT + 2]; int n = 0; - known[n++] = (RtRange){L.aslr.base, LOW_SIZE}; + known[n++] = (RtRange){L.aslr.base, low_size}; known[n++] = L.alias; known[n++] = L.kernel; known[n++] = L.hostfn; @@ -549,14 +557,14 @@ uint32_t mem_query(uint64_t a, RtMemoryInfo *out) { r.base = 0; r.size = low; r.state = ST_FREE; - } else if (a < low + LOW_SIZE) { + } else if (a < low + low_size) { uint64_t heap = L.heap.base; if (a >= heap && a < heap + heap_size) { r = (RtMemoryInfo){.base = heap, .size = heap_size, .state = ST_NORMAL, .perm = PERM_RW}; } else if (a < heap) { r = (RtMemoryInfo){.base = low, .size = heap - low, .state = ST_FREE}; } else { - r = (RtMemoryInfo){.base = heap + heap_size, .size = low + LOW_SIZE - heap - heap_size, .state = ST_FREE}; + r = (RtMemoryInfo){.base = heap + heap_size, .size = low + low_size - heap - heap_size, .state = ST_FREE}; } } else if (a - L.alias.base < L.alias.size) { uint64_t p = (a - L.alias.base) / PAGE;