From a57ea119face0564f1fdae52fe5119475029b980 Mon Sep 17 00:00:00 2001 From: Herbert Costa <246700963+kurerb@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:57:54 -0300 Subject: [PATCH 1/9] Grow the world texture heap for HD texture packs with tex_heap_extra_mb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HD texture packs made for emulators with 8 GB of DRAM, such as the HDPacks 4K pack, grow "TextureHeap" in their own HeapDef table. Served as a mod, that table replaces the one the port grows for the resolution and the shadow map, and the game crashes at boot; without it the pack's textures do not fit, and the game loads and then starts its panic Blood Moon over and over. tex_heap_extra_mb ([mods], 0 to 4096 MiB, read at start) grows TextureHeap and its parent ゲームグラフィックス in the port's HeapDef, and the application pool and the heap cap by the same amount. LOW_SIZE goes from 6 to 10 GiB so the stack and heap block holds the largest heap. At 0 the pool, the heap cap and the HeapDef are what they were. The selftest's "beyond the pool" MapPhysicalMemory asked for 4 GiB, which a grown pool holds; it now asks for just past the pool. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 3 ++- runtime/host.h | 2 ++ runtime/main.c | 4 +++- runtime/memory.c | 12 +++++------ runtime/resolution.c | 50 +++++++++++++++++++++++++++++++++++++++++--- totk.ini.example | 7 +++++++ 6 files changed, 67 insertions(+), 11 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index ec395f97..a9efd386 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -59,7 +59,7 @@ static const char *const known[] = { "SHADER_MEMO", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -82,6 +82,7 @@ static const struct { const char *key, *section; } sections[] = { {"SHADOW_SCALE", "display"}, {"SHADOW_RANGE", "display"}, {"CONTACT_SHADOWS", "display"}, {"PCSS", "display"}, {"MODS_DIR", "mods"}, {"MODS_ORDER", "mods"}, {"MODS_DISABLED", "mods"}, {"MODS_REBUILD", "mods"}, + {"TEX_HEAP_EXTRA_MB", "mods"}, {"MENU_LANG", "language"}, {"LANG", "language"}, {"OPERATION_MODE", "language"}, {"VOLUME", "audio"}, {"AUDIO_CHANNELS", "audio"}, {"MOTION", "input"}, {"RUMBLE_STRENGTH", "input"}, {"RUMBLE_CURVE", "input"}, {"RUMBLE_TRIGGERS", "input"}, diff --git a/runtime/host.h b/runtime/host.h index 21d0534f..5381c3dd 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -260,6 +260,8 @@ void cam_pack_served(void); // romfs.c serves the // ---- game resolution (resolution.c): TOTK_OUTPUT_RES, TOTK_RENDER_SCALE uint64_t res_heap_extra_bytes(void); // added to the application pool (mem_init), 2 MiB steps uint64_t res_heap_extra_kib(void); // added to the HeapDef texture heap and its parent; 0 = none +uint64_t res_tex_heap_extra_bytes(void); // TOTK_TEX_HEAP_EXTRA_MB: added to the application pool and + // the heap cap (mem_init), 2 MiB steps; 0 = none int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n); // the HeapDef table (.byml.zs) with // the grown heaps, malloc'd; 0 = cannot (logged) void res_heapdef_served(void); // romfs.c serves the grown table diff --git a/runtime/main.c b/runtime/main.c index 49d227a0..d2c68575 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -397,7 +397,9 @@ static void check_memory(void) { CHECK(q.state == 0 && q.base == g->alias.base && q.size == g->alias.size, "QueryMemory on the free alias region"); CHECK(mem_map_physical(a, 0x1000) == 0 && LD64(a + 0x10) == 0 && mem_unmap_physical(a, 0x1000) == 0, "remapped physical memory is not zeroed"); - CHECK(mem_map_physical(a, 0x100000000ull) == 0x10801, "MapPhysicalMemory beyond the pool is not LimitReached"); + // past the pool whatever its size (TOTK_APP_POOL, TOTK_TEX_HEAP_EXTRA_MB): 4 GiB fits a grown one + CHECK(mem_map_physical(a, (g->pool + 0x100000) & ~0xfffffull) == 0x10801, + "MapPhysicalMemory beyond the pool is not LimitReached"); mem_query(RT_HOSTFN_BASE + 0x40, &q); CHECK(q.state == 0x10 && q.base == RT_HOSTFN_BASE, "QueryMemory on the host-function window"); mem_query(g->alias.base + g->alias.size + 0x1000, &q); diff --git a/runtime/memory.c b/runtime/memory.c index 4dcc064c..79ccf70c 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -20,11 +20,11 @@ #define PAGE 0x1000ull #define GiB (1ull << 30) #define LOW_SEARCH_FROM 0x100000000ull -#define LOW_SIZE (6 * GiB) +#define LOW_SIZE (10 * GiB) // the heap at its largest: HEAP_CAP plus TOTK_TEX_HEAP_EXTRA_MB's 4 GiB #define STACK_OFF 0x10000000ull #define STACK_SIZE 0x10000000ull #define HEAP_OFF (STACK_OFF + STACK_SIZE + 0x10000000ull) -#define HEAP_CAP (4 * GiB) +#define HEAP_CAP (4 * GiB) // without TOTK_TEX_HEAP_EXTRA_MB, which raises it by its own size (mem_init) #define ALIAS_SIZE (64 * GiB) #define ALIAS_ALIGN (4 * GiB) #define ALIAS_PAGES (ALIAS_SIZE / PAGE) @@ -223,7 +223,7 @@ void mem_init(void) { uint64_t low = 0, alias = 0; for (uint64_t at = LOW_SEARCH_FROM; at + LOW_SIZE <= mods_lo && !low; at += GiB) if (reserve_at(at, LOW_SIZE, 1)) low = at; - if (!low) reexec_or_fail("a 6 GiB stack and heap block below 2^39"); + if (!low) reexec_or_fail("a 10 GiB stack and heap block below 2^39"); for (uint64_t at = align_up(low + LOW_SIZE, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) if (reserve_at(at, ALIAS_SIZE, 1)) alias = at; if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39"); @@ -238,11 +238,11 @@ void mem_init(void) { L = (RtGuestLayout){ .aslr = {low, RT_ADDRESS_LIMIT - low}, .stack = {low + STACK_OFF, STACK_SIZE}, - .heap = {low + HEAP_OFF, HEAP_CAP}, + .heap = {low + HEAP_OFF, HEAP_CAP + res_tex_heap_extra_bytes()}, .alias = {alias, ALIAS_SIZE}, .kernel = {RT_KERNEL_BASE, RT_KERNEL_SIZE}, .hostfn = {RT_HOSTFN_BASE, RT_HOSTFN_SIZE}, - .pool = pool ? strtoull(pool, NULL, 0) : DEFAULT_POOL + res_heap_extra_bytes(), // resolution.c + .pool = pool ? strtoull(pool, NULL, 0) : DEFAULT_POOL + res_heap_extra_bytes() + res_tex_heap_extra_bytes(), }; if (!L.pool) rt_fatal("memory: TOTK_APP_POOL is 0"); } @@ -276,7 +276,7 @@ static int inside(uint64_t a, uint64_t n, RtRange r) { return a >= r.base && n < // ---- 0x01 SetHeapSize uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { plat_mutex_lock(&lock); - if (size > HEAP_CAP || size % 0x200000 || size + image_bytes() > L.pool) { + if (size > L.heap.size || size % 0x200000 || size + image_bytes() > L.pool) { plat_mutex_unlock(&lock); return R_INVALID_STATE; } diff --git a/runtime/resolution.c b/runtime/resolution.c index 3e70fefa..157e2b00 100644 --- a/runtime/resolution.c +++ b/runtime/resolution.c @@ -189,6 +189,25 @@ uint64_t res_heap_extra_kib(void) { uint64_t res_heap_extra_bytes(void) { return ((res_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; } +// TOTK_TEX_HEAP_EXTRA_MB (0 to 4096): more room in "TextureHeap" and its parent "ゲームグラフィックス", the heap the +// world's textures stream into. HD texture packs made for emulators with 8 GB of DRAM grow it by about 3.3 GiB in their +// own HeapDef; without the room the game runs out and starts its panic Blood Moon over and over. The pool and the heap +// cap grow by the same amount (memory.c mem_init). +static uint64_t tex_heap_extra_kib(void) { + const char *v = cfg_get("TOTK_TEX_HEAP_EXTRA_MB"); + if (!v || !*v) return 0; + char *end; + long mb = strtol(v, &end, 10); + if (end == v || *end || mb < 0 || mb > 4096) { + static int logged; + if (!logged++) rt_log("resolution: TOTK_TEX_HEAP_EXTRA_MB=%s is not 0..4096; 0", v); + return 0; + } + return (uint64_t)mb << 10; +} + +uint64_t res_tex_heap_extra_bytes(void) { return ((tex_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; } + // ---- the HeapDef table. BYML version 2..7, little endian ("YB"): a hash node is {u8 0xC1, u24 count} and count // entries {u24 key index, u8 type, u32 value}; an int (type 0xD1) is stored in the entry, so a size changes in place. typedef struct { uint8_t *p; size_t n; uint32_t keys; } Byml; @@ -227,9 +246,9 @@ static size_t heap_size_field(const Byml *b, uint32_t heaps, const char *heap) { } int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { - uint64_t extra = res_heap_extra_kib(); + uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib(); *out = NULL; - if (!extra) return 0; + if (!extra && !tex) return 0; if (ZSTD_getDictID_fromFrame(in, n)) { rt_log("resolution: the HeapDef table needs a zstd dictionary; the game keeps its own resolution"); return 0; @@ -263,6 +282,26 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { int32_t dta2 = dta + (int32_t)extra, gfx2 = gfx + (int32_t)extra; memcpy(b.p + dta_at, &dta2, 4); memcpy(b.p + gfx_at, &gfx2, 4); + int32_t th = 0, th2 = 0, gg = 0, gg2 = 0; + if (tex) { // TOTK_TEX_HEAP_EXTRA_MB: the world texture heap and its parent grow by the same amount + static const char parent[] = "\xe3\x82\xb2\xe3\x83\xbc\xe3\x83\xa0\xe3\x82\xb0\xe3\x83\xa9\xe3\x83\x95" + "\xe3\x82\xa3\xe3\x83\x83\xe3\x82\xaf\xe3\x82\xb9"; // ゲームグラフィックス + size_t th_at = heap_size_field(&b, heaps, "TextureHeap"), gg_at = heap_size_field(&b, heaps, parent); + if (th_at && gg_at) { + memcpy(&th, b.p + th_at, 4); + memcpy(&gg, b.p + gg_at, 4); + th2 = th + (int32_t)tex, gg2 = gg + (int32_t)tex; + memcpy(b.p + th_at, &th2, 4); + memcpy(b.p + gg_at, &gg2, 4); + } else { + rt_log("resolution: the HeapDef table has no TextureHeap sizes; TOTK_TEX_HEAP_EXTRA_MB left out"); + tex = 0; + if (!extra) { + free(b.p); + return 0; + } + } + } size_t cap = ZSTD_compressBound(b.n); void *z = malloc(cap); size_t zn = z ? ZSTD_compress(z, cap, b.p, b.n, 9) : 0; @@ -273,7 +312,12 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { } *out = z; *out_n = zn; - rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx, gfx2); + if (extra) + rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx, + gfx2); + if (tex) + rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB)", th, + th2, gg, gg2); return 1; } diff --git a/totk.ini.example b/totk.ini.example index 3ef2537b..c3f9f743 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -366,6 +366,13 @@ # the setup's rebuild at start (a few minutes) and starts again after it. off: the game starts with the code it has. # A rebuild that failed or was cancelled is not started again for the same cheats. Default: auto. # mods_rebuild = auto +# Extra room in MiB (0 to 4096) for the heap the world's textures stream into, for HD texture packs. Packs made for +# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs), +# which replaces the port's own: take that file out of the pack and set this instead (about 3400 for the HDPacks 4K +# pack). Without the room the game keeps starting its panic Blood Moon. The memory pool grows by the same amount. +# Read at start. Default: 0. Such a pack also needs much more video memory: astc_bc = 1 ([performance]) keeps its +# textures at a quarter of it. +# tex_heap_extra_mb = 0 [files] # Developer overlay: a folder with RomFS files and a manifest.tsv (tools/make_romfs_patches.py format). Players put From 2a70683007ad0e90397907c8eab1bde6194d682c Mon Sep 17 00:00:00 2001 From: Herbert Costa <246700963+kurerb@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:57:54 -0300 Subject: [PATCH 2/9] Grow the HeapDef for tex_heap_extra_mb at the game's own resolution too heapdef_overlay returned at once when res_heap_extra_kib() was 0, which it is at 1920x1080 with the game's scene size and shadow map, so tex_heap_extra_mb did nothing there and a 1080p player with an HD pack still hit the Blood Moon loop. It now also builds the table when res_tex_heap_extra_bytes() is set. Found by itaskaev2 in review. Co-Authored-By: Claude Opus 5.5 --- runtime/romfs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/romfs.c b/runtime/romfs.c index c062c4b2..c14b9aa6 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -829,12 +829,12 @@ static void restbl_overlay(void) { members, packs, (now_s() - t0) * 1000); } -// The HeapDef table a bigger game resolution needs (resolution.c): its texture heap and that heap's parent grow. The -// base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS original; the grown table is -// written to /resolution/ and served in the base's place. +// The HeapDef table a bigger game resolution or TOTK_TEX_HEAP_EXTRA_MB needs (resolution.c): its texture heaps and +// their parents grow. The base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS +// original; the grown table is written to /resolution/ and served in the base's place. #define HEAPDEF "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" static void heapdef_overlay(void) { - if (!res_heap_extra_kib()) return; + if (!res_heap_extra_kib() && !res_tex_heap_extra_bytes()) return; static const char keeps[] = "the game keeps its own resolution"; Overlay *base; size_t n; From 8ac207f3537add01f41bd564f0ae10cfa93318a1 Mon Sep 17 00:00:00 2001 From: itaskaev2 <165683571+itaskaev2@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:28:25 +0100 Subject: [PATCH 3/9] Grow the texture heap a mod's HeapDef asks for instead of serving that table HD texture packs made for the emulators ship their own HeapDef with TextureHeap and its parent grown for the emulators' 8 GB memory option (HDPacks.com's 4K pack: by 3,419,000 KiB). Served as it is, the game asked for room the application pool does not have and crashed at start (#673); without the file it looped the panic Blood Moon. Before mem_init sizes the pool, resolution.c reads the HeapDef of every switched-on mod (mods.c mods_peek_romfs, which lists the mods' RomFS roots without the rest of the scan: mods_scan needs paths main sets later) and counts the largest TextureHeap growth over the shipped 506,880 KiB like TOTK_TEX_HEAP_EXTRA_MB (the larger of the two, at most 4 GiB). The pool and the heap cap grow by it, and romfs.c heapdef_overlay serves the game's own table grown by that amount in the mod's place, so the other heaps keep the sizes of the game version that runs. Tested with the HDPacks.com 4K Texture Pack v1.0 as players install it, on 1.4.3 and 1.2.1 (RTX 4080 SUPER): the game loads a save and runs at 60 fps with 0 validation errors, and holds 4.5 GB of game memory pools against 1.2 GB without the pack. docs/mods.md, README.md and totk.ini.example say so; other packs come with no guarantee. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- docs/mods.md | 15 +++++++ runtime/host.h | 1 + runtime/mods.c | 51 +++++++++++++++++------ runtime/mods.h | 3 ++ runtime/resolution.c | 98 ++++++++++++++++++++++++++++++++++++-------- runtime/romfs.c | 25 ++++++++++- totk.ini.example | 7 ++-- 8 files changed, 169 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index c0ffc147..38780dce 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,8 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). - Shaders prepared at install instead of during play - Mods made for emulators: unpack a mod into the `mods` folder next to `totk.exe`; switch and order them in the menu. RomFS mods, TKMM's merged output and ExeFS code patches (`.ips`, `.pchtxt`) work; the game rebuilds itself for code patches - ([docs/mods.md](docs/mods.md)) + ([docs/mods.md](docs/mods.md)). HD texture packs work: the port grows the texture heap a pack asks for by itself. Tested + with the HDPacks.com 4K Texture Pack v1.0 on 1.4.3 and 1.2.1; other packs come with no guarantee - Atmosphère cheats: put a cheat file into a mod and switch its cheats in the menu ([docs/cheats.md](docs/cheats.md)) ## Planned diff --git a/docs/mods.md b/docs/mods.md index d26553db..0734f382 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -87,6 +87,21 @@ The port keys a mod by its folder name. A renamed folder counts as a new mod: Rename mod folders before you set them up in the menu, not after. +## HD texture packs + +Texture packs made for the emulators replace the game's textures with larger ones, so the heap the world's textures +stream into must grow. Such a pack grows it in its own `System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs`, sized for +the emulators' 8 GB memory option. The port reads that file at start: the texture heap and the game's memory pool grow +by the pack's amount, and the game gets the port's own heap table with that growth (the log says +`resolution: mod ships a HeapDef with TextureHeap ...`). Nothing needs to be set. For a pack without such a file +that still loops the panic Blood Moon, set `tex_heap_extra_mb` under `[mods]` in `totk.ini` (0 to 4096 MiB). + +Tested: **HDPacks.com Tears of the Kingdom 4K Texture Pack v1.0** (`4kTextures`, 10,407 textures, 6.5 GB) on game 1.4.3 +and 1.2.1, RTX 4080 SUPER: it loads into the game from a save and runs at the same frame rate, with about 3.3 GB more of +textures in memory (4.5 GB of game memory pools against 1.2 GB without it). Other packs are not tested and come with no +guarantee. Such a pack needs a lot of video memory; on a GPU with less than 8 GB, `astc_bc = 1` under `[performance]` +keeps its textures at a quarter of the size. + ## What does not work | On the Mods tab | Why | What to do | diff --git a/runtime/host.h b/runtime/host.h index 5381c3dd..d46336b9 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -262,6 +262,7 @@ uint64_t res_heap_extra_bytes(void); // added to the appli uint64_t res_heap_extra_kib(void); // added to the HeapDef texture heap and its parent; 0 = none uint64_t res_tex_heap_extra_bytes(void); // TOTK_TEX_HEAP_EXTRA_MB: added to the application pool and // the heap cap (mem_init), 2 MiB steps; 0 = none +int res_tex_heap_from_mod(void); // a switched-on mod's HeapDef grows the texture heap int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n); // the HeapDef table (.byml.zs) with // the grown heaps, malloc'd; 0 = cannot (logged) void res_heapdef_served(void); // romfs.c serves the grown table diff --git a/runtime/mods.c b/runtime/mods.c index 4f3ad08c..d960a407 100644 --- a/runtime/mods.c +++ b/runtime/mods.c @@ -338,22 +338,49 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) { return 1; } +// The mods folder into out: TOTK_MODS_DIR, else /mods. 0 when the exe's path is unknown. +static int mods_folder(char *out, size_t cap) { + const char *d = cfg_get("TOTK_MODS_DIR"); + if (d && *d) return snprintf(out, cap, "%s", d) < (int)cap; + char exe[2048]; + if (!plat_exe_path(exe, sizeof exe)) return 0; + char *cut = strrchr(exe, '/'), *back = strrchr(exe, '\\'); + if (back > cut) cut = back; + if (!cut) return 0; + *cut = 0; + return snprintf(out, cap, "%s/mods", exe) < (int)cap; +} + +typedef struct { + const char *folder, *disabled; + void (*cb)(const char *name, const char *romfs, void *ctx); + void *ctx; +} Peek; + +static int peek_cb(const char *name, int kind, uint64_t size, void *ctx) { + (void)size; + const Peek *pk = ctx; + if (kind != PLAT_FS_DIR || listed(pk->disabled, name)) return 1; + char path[2100], romfs[2048]; + if (snprintf(path, sizeof path, "%s/%s", pk->folder, name) >= (int)sizeof path) return 1; + find_root(path, "romfs", romfs, sizeof romfs); + if (!romfs[0]) find_root(path, "romfslite", romfs, sizeof romfs); + if (romfs[0]) pk->cb(name, romfs, pk->ctx); + return 1; +} + +void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), void *ctx) { + char folder[2048]; + if (!mods_folder(folder, sizeof folder) || !plat_dir_exists(folder)) return; + Peek pk = {folder, cfg_get("TOTK_MODS_DISABLED"), cb, ctx}; + plat_fs_list(folder, peek_cb, &pk); +} + void mods_scan(void) { static int done; if (done) return; done = 1; - const char *d = cfg_get("TOTK_MODS_DIR"); - if (d && *d) { - snprintf(dir, sizeof dir, "%s", d); - } else { - char exe[2048]; - if (!plat_exe_path(exe, sizeof exe)) return; - char *cut = strrchr(exe, '/'), *back = strrchr(exe, '\\'); - if (back > cut) cut = back; - if (!cut) return; - *cut = 0; - snprintf(dir, sizeof dir, "%s/mods", exe); - } + if (!mods_folder(dir, sizeof dir)) return; if (!plat_dir_exists(dir)) { rt_log("mods: no folder %s", dir); return; diff --git a/runtime/mods.h b/runtime/mods.h index 98a513db..318a9632 100644 --- a/runtime/mods.h +++ b/runtime/mods.h @@ -40,6 +40,9 @@ typedef struct { } ModInfo; void mods_scan(void); // once; later calls return at once +// Every switched-on mod folder's RomFS root, without the rest of the scan (no ExeFS checks, no order): for mem_init, +// which runs before the paths mods_scan needs are set (resolution.c mod_tex_heap_kib). cb gets the name and root. +void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), void *ctx); const char *mods_dir(void); // the folder scanned (default: /mods) int mods_count(void); ModInfo *mods_at(int i); // 0 = top of the order; NULL out of range diff --git a/runtime/resolution.c b/runtime/resolution.c index 157e2b00..ac4fa75c 100644 --- a/runtime/resolution.c +++ b/runtime/resolution.c @@ -39,6 +39,7 @@ #include "cfg.h" #include "host.h" +#include "mods.h" #include "platform/platform.h" #define BASE_W 1920 @@ -46,6 +47,7 @@ #define SCENE_W 1600 #define SCENE_H 900 #define DTA_KIB 113664u // DynamicTextureAllocatorMem, shipped +#define TEX_HEAP_KIB 506880u // TextureHeap, shipped (1.2.1 and 1.4.3): what a mod's own HeapDef is measured against #define SCENE_MAX 4096 // per side, as MacNativeGames' TOTK_RENDER_RES #define SHADOW_BASE 960 // the sun shadow map's base size (its cascades: a D16 array of 4 layers) @@ -193,7 +195,9 @@ uint64_t res_heap_extra_bytes(void) { return ((res_heap_extra_kib() << 10) + 0x1 // world's textures stream into. HD texture packs made for emulators with 8 GB of DRAM grow it by about 3.3 GiB in their // own HeapDef; without the room the game runs out and starts its panic Blood Moon over and over. The pool and the heap // cap grow by the same amount (memory.c mem_init). -static uint64_t tex_heap_extra_kib(void) { +static uint64_t mod_tex_heap_kib(void); + +static uint64_t cfg_tex_heap_kib(void) { const char *v = cfg_get("TOTK_TEX_HEAP_EXTRA_MB"); if (!v || !*v) return 0; char *end; @@ -206,6 +210,12 @@ static uint64_t tex_heap_extra_kib(void) { return (uint64_t)mb << 10; } +// The larger of TOTK_TEX_HEAP_EXTRA_MB and what an enabled mod's own HeapDef adds to the texture heap (mod_tex_heap_kib). +static uint64_t tex_heap_extra_kib(void) { + uint64_t c = cfg_tex_heap_kib(), m = mod_tex_heap_kib(); + return c > m ? c : m; +} + uint64_t res_tex_heap_extra_bytes(void) { return ((tex_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; } // ---- the HeapDef table. BYML version 2..7, little endian ("YB"): a hash node is {u8 0xC1, u24 count} and count @@ -245,28 +255,82 @@ static size_t heap_size_field(const Byml *b, uint32_t heaps, const char *heap) { return at && u32_at(b, at, &node) ? field(b, node, "Size", 0xd1) : 0; } -int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { - uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib(); - *out = NULL; - if (!extra && !tex) return 0; +// A HeapDef in b's place (zstd frame in[0..n) with its content size, no dictionary): b.p malloc'd and *heaps its +// "Heaps" node; 0 when it is not one (b.p freed). why: what the log line says the caller keeps then. +static int heapdef_open(const void *in, size_t n, Byml *b, uint32_t *heaps, const char *why) { + b->p = NULL; if (ZSTD_getDictID_fromFrame(in, n)) { - rt_log("resolution: the HeapDef table needs a zstd dictionary; the game keeps its own resolution"); + rt_log("resolution: the HeapDef table needs a zstd dictionary; %s", why); return 0; } unsigned long long raw = ZSTD_getFrameContentSize(in, n); if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > (64u << 20)) { - rt_log("resolution: the HeapDef table is not a zstd frame with a size; the game keeps its own resolution"); + rt_log("resolution: the HeapDef table is not a zstd frame with a size; %s", why); return 0; } - Byml b = {malloc((size_t)raw), (size_t)raw, 0}; - if (!b.p) return 0; - size_t got = ZSTD_decompress(b.p, b.n, in, n); - uint32_t root, heaps = 0; + *b = (Byml){malloc((size_t)raw), (size_t)raw, 0}; + if (!b->p) return 0; + size_t got = ZSTD_decompress(b->p, b->n, in, n); + uint32_t root; uint16_t ver = 0; - if (!ZSTD_isError(got) && got == b.n && b.n >= 16) memcpy(&ver, b.p + 2, 2); - size_t heaps_at = 0, dta_at = 0, gfx_at = 0; - if (ver >= 2 && ver <= 7 && !memcmp(b.p, "YB", 2) && u32_at(&b, 4, &b.keys) && u32_at(&b, 12, &root) && - (heaps_at = field(&b, root, "Heaps", 0xc1)) && u32_at(&b, heaps_at, &heaps)) { + if (!ZSTD_isError(got) && got == b->n && b->n >= 16) memcpy(&ver, b->p + 2, 2); + size_t heaps_at = 0; + if (ver >= 2 && ver <= 7 && !memcmp(b->p, "YB", 2) && u32_at(b, 4, &b->keys) && u32_at(b, 12, &root) && + (heaps_at = field(b, root, "Heaps", 0xc1)) && u32_at(b, heaps_at, heaps)) + return 1; + free(b->p); + b->p = NULL; + return 0; +} + +// HD texture packs made for the emulators ship their own HeapDef with "TextureHeap" and its parent grown (HDPacks.com's +// 4K pack for TotK: by 3,419,000 KiB, 3.3 GiB, for the emulators' 8 GB memory option). Served as it is, the game asks +// for that room and the application pool has none: it crashed at start (issue #673). So before mem_init sizes the pool, +// every switched-on mod's HeapDef is read (mods.c mods_peek_romfs: mods_scan needs paths main sets later) and the +// largest TextureHeap growth over the shipped size counts as TOTK_TEX_HEAP_EXTRA_MB (the larger of the two, at most +// 4 GiB). romfs.c heapdef_overlay then serves the game's own table grown by that amount in the mod's place, so the +// other heaps keep the sizes of the game version that runs. +#define HEAPDEF_REL "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" +static struct { int done; uint64_t kib; } MT; + +static void mod_heapdef(const char *name, const char *romfs, void *ctx) { + (void)ctx; + char path[2300]; + if (snprintf(path, sizeof path, "%s/%s", romfs, HEAPDEF_REL) >= (int)sizeof path) return; + uint64_t n = 0; + uint8_t *in = plat_file_read(path, &n); + if (!in) return; + Byml b; + uint32_t heaps; + size_t th_at = 0; + int32_t th = 0; + if (heapdef_open(in, (size_t)n, &b, &heaps, "its texture heap is left out") && + (th_at = heap_size_field(&b, heaps, "TextureHeap"))) + memcpy(&th, b.p + th_at, 4); + free(b.p); + free(in); + int64_t grow = (int64_t)th - (int64_t)TEX_HEAP_KIB; + if (grow > (int64_t)(4096u << 10)) grow = 4096u << 10; + if (grow > 0 && (uint64_t)grow > MT.kib) MT.kib = (uint64_t)grow; + rt_log("resolution: mod %s ships a HeapDef with TextureHeap %d KiB (the game's %u KiB)%s", name, th, TEX_HEAP_KIB, + grow > 0 ? ": the port grows the texture heap and the pool by the difference" : ""); +} + +static uint64_t mod_tex_heap_kib(void) { + if (!MT.done) MT.done = 1, mods_peek_romfs(mod_heapdef, NULL); + return MT.kib; +} + +int res_tex_heap_from_mod(void) { return mod_tex_heap_kib() > 0; } + +int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { + uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib(); + *out = NULL; + if (!extra && !tex) return 0; + Byml b; + uint32_t heaps = 0; + size_t dta_at = 0, gfx_at = 0; + if (heapdef_open(in, n, &b, &heaps, "the game keeps its own resolution")) { dta_at = heap_size_field(&b, heaps, "DynamicTextureAllocatorMem"); gfx_at = heap_size_field(&b, heaps, "Graphics"); } @@ -316,8 +380,8 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx, gfx2); if (tex) - rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB)", th, - th2, gg, gg2); + rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB or a " + "mod's HeapDef)", th, th2, gg, gg2); return 1; } diff --git a/runtime/romfs.c b/runtime/romfs.c index c14b9aa6..38de9ab2 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -535,6 +535,21 @@ static void added_init(void) { // RomFS file `rel` as the game would read it: from an overlay that replaces it (*base), else from the image. malloc'd; // NULL when it is missing, unreadable or 16 MiB or more (logged with `keeps`, what the game does without it). +// RomFS file rel as the RomFS holds it, whatever overlays replace it; NULL (logged) when it cannot be read. +static void *read_original(const char *rel, size_t *n, const char *keeps) { + uint64_t i = lower(rel); + if (i >= file_count || strcmp(entry_name(entry(i)), rel)) return NULL; + uint64_t size = entry(i).size; + void *in = size && size < (16u << 20) ? malloc((size_t)size) : NULL; + if (!in || read_at(image, in, (uint32_t)size, entry(i).offset) != (int64_t)size) { + free(in); + rt_log("RomFS: cannot read %s; %s", rel, keeps); + return NULL; + } + *n = (size_t)size; + return in; +} + static void *read_served(const char *rel, Overlay **base, size_t *n, const char *keeps) { uint64_t i = lower(rel); *base = NULL; @@ -831,7 +846,9 @@ static void restbl_overlay(void) { // The HeapDef table a bigger game resolution or TOTK_TEX_HEAP_EXTRA_MB needs (resolution.c): its texture heaps and // their parents grow. The base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS -// original; the grown table is written to /resolution/ and served in the base's place. +// original; the grown table is written to /resolution/ and served in the base's place. A mod's table whose +// texture heap is grown (an HD texture pack's, resolution.c mod_tex_heap_kib) is not the base: that growth is already +// in the amount, so the RomFS original grows by it and is served in the mod's place. #define HEAPDEF "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" static void heapdef_overlay(void) { if (!res_heap_extra_kib() && !res_tex_heap_extra_bytes()) return; @@ -840,6 +857,12 @@ static void heapdef_overlay(void) { size_t n; void *in = read_served(HEAPDEF, &base, &n, keeps), *out = NULL; size_t out_n = 0; + if (in && base && base->mod && res_tex_heap_from_mod()) { + free(in); + in = read_original(HEAPDEF, &n, keeps); + rt_log("RomFS: %s: the game's own HeapDef with the mod's texture heap growth in place of the mod's table", + mods_at(base->mod - 1)->name); + } int ok = in && res_heapdef_build(in, n, &out, &out_n); free(in); if (ok && serve_generated(HEAPDEF, base, out, out_n, "resolution", "grown heaps", keeps)) res_heapdef_served(); diff --git a/totk.ini.example b/totk.ini.example index c3f9f743..4a09a50c 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -367,9 +367,10 @@ # A rebuild that failed or was cancelled is not started again for the same cheats. Default: auto. # mods_rebuild = auto # Extra room in MiB (0 to 4096) for the heap the world's textures stream into, for HD texture packs. Packs made for -# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs), -# which replaces the port's own: take that file out of the pack and set this instead (about 3400 for the HDPacks 4K -# pack). Without the room the game keeps starting its panic Blood Moon. The memory pool grows by the same amount. +# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs): +# the port reads that growth at start and applies it itself (about 3339 MiB for the HDPacks 4K pack), so such a pack +# needs nothing here. Set it for a pack that ships textures without a HeapDef. Without the room the game keeps starting +# its panic Blood Moon. The memory pool grows by the same amount; the larger of this and a mod's growth counts. # Read at start. Default: 0. Such a pack also needs much more video memory: astc_bc = 1 ([performance]) keeps its # textures at a quarter of it. # tex_heap_extra_mb = 0 From 2ad82557fd73777de3e1071c1dbc09f40ef95cb1 Mon Sep 17 00:00:00 2001 From: itaskaev2 <165683571+itaskaev2@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:43:01 +0100 Subject: [PATCH 4/9] Reserve the larger stack and heap block only for a texture heap extra The block below the module window stays 6 GiB, as on dev, and grows by the texture heap extra (tex_heap_extra_mb or a mod's HeapDef) rounded up to a GiB. A player without an HD pack reserves nothing more. Co-Authored-By: Claude Opus 5.5 --- runtime/memory.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/runtime/memory.c b/runtime/memory.c index 79ccf70c..019c0fab 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -20,7 +20,7 @@ #define PAGE 0x1000ull #define GiB (1ull << 30) #define LOW_SEARCH_FROM 0x100000000ull -#define LOW_SIZE (10 * GiB) // the heap at its largest: HEAP_CAP plus TOTK_TEX_HEAP_EXTRA_MB's 4 GiB +#define LOW_BASE (6 * GiB) // the stack and heap block without a texture heap extra #define STACK_OFF 0x10000000ull #define STACK_SIZE 0x10000000ull #define HEAP_OFF (STACK_OFF + STACK_SIZE + 0x10000000ull) @@ -48,6 +48,9 @@ enum { ATTR_PERMISSION_LOCKED = 16 }; static RtGuestLayout L; static uint64_t mods_lo, mods_hi; // fixed module window +// The stack and heap block: LOW_BASE, plus the texture heap extra (tex_heap_extra_mb or a mod's HeapDef, +// res_tex_heap_extra_bytes) rounded up to a GiB, so only a player who grows the texture heap reserves more (mem_init). +static uint64_t low_size = LOW_BASE; static PlatMutex lock = PLAT_MUTEX_INIT; static uint64_t heap_size, alias_bytes; static uint8_t *alias_pages; // 1 per mapped 4 KiB page of the alias region @@ -78,7 +81,7 @@ static volatile int64_t frz_count[4]; // freezes; thaws by a write fault, by m static int watched(uint64_t addr, uint64_t size) { const RtGuestLayout *g = &L; - return (addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size) || + return (addr >= g->aslr.base && size <= low_size && addr - g->aslr.base <= low_size - size) || (addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size); } @@ -221,10 +224,15 @@ void mem_init(void) { // Stack and heap block, then the alias region above it, both searched below the module window. uint64_t low = 0, alias = 0; - for (uint64_t at = LOW_SEARCH_FROM; at + LOW_SIZE <= mods_lo && !low; at += GiB) - if (reserve_at(at, LOW_SIZE, 1)) low = at; - if (!low) reexec_or_fail("a 10 GiB stack and heap block below 2^39"); - for (uint64_t at = align_up(low + LOW_SIZE, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) + low_size = LOW_BASE + align_up(res_tex_heap_extra_bytes(), GiB); + for (uint64_t at = LOW_SEARCH_FROM; at + low_size <= mods_lo && !low; at += GiB) + if (reserve_at(at, low_size, 1)) low = at; + if (!low) { + char what[64]; + snprintf(what, sizeof what, "a %llu GiB stack and heap block below 2^39", (unsigned long long)(low_size / GiB)); + reexec_or_fail(what); + } + for (uint64_t at = align_up(low + low_size, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) if (reserve_at(at, ALIAS_SIZE, 1)) alias = at; if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39"); @@ -506,7 +514,7 @@ static void taken_gap(uint64_t a, uint64_t *base, uint64_t *size) { uint64_t lo = 0, hi = RT_ADDRESS_LIMIT; RtRange known[3 + MOD_COUNT + 2]; int n = 0; - known[n++] = (RtRange){L.aslr.base, LOW_SIZE}; + known[n++] = (RtRange){L.aslr.base, low_size}; known[n++] = L.alias; known[n++] = L.kernel; known[n++] = L.hostfn; @@ -549,14 +557,14 @@ uint32_t mem_query(uint64_t a, RtMemoryInfo *out) { r.base = 0; r.size = low; r.state = ST_FREE; - } else if (a < low + LOW_SIZE) { + } else if (a < low + low_size) { uint64_t heap = L.heap.base; if (a >= heap && a < heap + heap_size) { r = (RtMemoryInfo){.base = heap, .size = heap_size, .state = ST_NORMAL, .perm = PERM_RW}; } else if (a < heap) { r = (RtMemoryInfo){.base = low, .size = heap - low, .state = ST_FREE}; } else { - r = (RtMemoryInfo){.base = heap + heap_size, .size = low + LOW_SIZE - heap - heap_size, .state = ST_FREE}; + r = (RtMemoryInfo){.base = heap + heap_size, .size = low + low_size - heap - heap_size, .state = ST_FREE}; } } else if (a - L.alias.base < L.alias.size) { uint64_t p = (a - L.alias.base) / PAGE; From 5c8f3ee5c6e9631eb1e6e0105593b316ce001951 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 6 Nov 2026 14:05:44 +0300 Subject: [PATCH 5/9] Keep NVN recording copies off the game threads' critical path. Three game threads flatten recordings, inline them and add control memory at once, and every large copy came from a fresh VirtualAlloc, so each one paid zero-page faults, and release_copies freed them under the exclusive copy lock. Copies are now freed after the lock through a per-thread dead list, blocks of 256 KiB to 16 MiB come from a small best-fit cache, and each copy keeps its pin list so a free or clone no longer walks the records again. Co-Authored-By: Claude Opus 5.5 --- runtime/nvn_api_cmd.c | 160 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 143 insertions(+), 17 deletions(-) diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index 21644b76..874002f7 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -141,10 +141,13 @@ static void pool_drain(void) { // system-managed page file, so the same request succeeds a moment later, and a stall of some frames is better than // ending the game. Frees the pooled buffers first. NULL only after about 30 s without memory; the caller ends the game. // Never called with a lock held. +static void big_drain(void); + static void *realloc_wait(void *old, size_t n, const char *what) { void *p = realloc(old, n); if (p) return p; pool_drain(); + big_drain(); static uint32_t reports; int report = __atomic_fetch_add(&reports, 1, __ATOMIC_RELAXED) < 8; if (report) { @@ -406,8 +409,10 @@ void nvn_pins_stats(uint64_t *live, uint64_t *freed) { // adding control memory again releases the copies recorded there without walking every live copy. 4 KiB pages cost // about 500 lookups per AddControlMemory under the exclusive lock (1.4.3 field: 53 calls a frame, 2 MiB ranges, 3.7 // copies per page), and the recording threads waited on it. -// deferred: 0 unknown, 1 none, 2 some; pins: texture pins its ops name, counted in their refs (pin_refs) -typedef struct { uint64_t magic, bytes, made_at, used_at, deferred, pins; } copy_prefix; +// deferred: 0 unknown, 1 none, 2 some; pins: texture pins its ops name, counted in their refs; pin_v: those pins, in +// record order, or NULL (then pin_refs walks the records); cap: bytes of the block from the large-block cache +// (copy_block), 0 for a plain heap block +typedef struct { uint64_t magic, bytes, made_at, used_at, deferred, pins, cap; const tex_t **pin_v; } copy_prefix; _Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned"); typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy; typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket; @@ -429,15 +434,117 @@ static void zombie_add(copy_prefix *pf) { // copy_lock held zombies[nzombies++] = pf; } -static void copy_free(copy_prefix *pf) { // copy_lock held - if (pf->pins) pin_refs((const uint8_t *)(pf + 1), -1); +// Blocks of large copies are kept for the next ones. 1.4.3 makes two copies of 0.5-2 MiB a frame (the per-frame +// recordings that inline many others); above the heap's direct-allocation threshold (about 512 KiB) every malloc mapped +// fresh zero-filled pages and every free unmapped them, so the copy ran at half the speed of a small one and the free +// held copy_lock for tens of microseconds. A block is reused for a copy that fills at least half of it. +#define BIG_MIN ((size_t)256 << 10) // smaller blocks come from the heap and go back to it +#define BIG_MAX ((size_t)16 << 20) // a larger block is never kept +#define BIG_SLOTS 8 +static PlatMutex big_lock = PLAT_MUTEX_INIT; // lock order: copy_lock, then big_lock +static struct { copy_prefix *p; size_t cap; } big[BIG_SLOTS]; +static uint32_t nbig; + +static void big_drain(void) { + plat_mutex_lock(&big_lock); + while (nbig) free(big[--nbig].p); + plat_mutex_unlock(&big_lock); +} + +// A block for a copy of n bytes after its prefix; prefix.cap says where it goes back (copy_block_free). wait: a failed +// allocation waits for memory (realloc_wait), which is never done with a lock held; else it ends the game at once. +static copy_prefix *copy_block(size_t n, int wait, const char *what) { + size_t total = sizeof(copy_prefix) + n, cap = 0; + copy_prefix *pf = NULL; + if (total >= BIG_MIN && total <= BIG_MAX) { + plat_mutex_lock(&big_lock); + uint32_t best = BIG_SLOTS; + for (uint32_t i = 0; i < nbig; i++) + if (big[i].cap >= total && big[i].cap / 2 <= total && (best == BIG_SLOTS || big[i].cap < big[best].cap)) + best = i; + if (best < BIG_SLOTS) { + pf = big[best].p; + cap = big[best].cap; + big[best] = big[--nbig]; + } + plat_mutex_unlock(&big_lock); + if (!pf) cap = (total + total / 8 + 0xffff) & ~(size_t)0xffff; // room for the next frame's slightly larger one + } + if (!pf) pf = wait ? realloc_wait(NULL, cap ? cap : total, what) : malloc(cap ? cap : total); + if (!pf) rt_fatal("nvn: %s allocation (%zu bytes)", what, cap ? cap : total); + pf->cap = cap; + return pf; +} + +static void copy_block_free(copy_prefix *pf) { + size_t cap = pf->cap; + if (cap) { + plat_mutex_lock(&big_lock); + if (nbig < BIG_SLOTS) { + big[nbig].p = pf; + big[nbig++].cap = cap; + pf = NULL; + } else { + uint32_t small = 0; + for (uint32_t i = 1; i < nbig; i++) + if (big[i].cap < big[small].cap) small = i; + if (big[small].cap < cap) { + copy_prefix *drop = big[small].p; + big[small].p = pf; + big[small].cap = cap; + pf = drop; + } + } + plat_mutex_unlock(&big_lock); + } free(pf); } -static void zombie_sweep(uint64_t replayed) { // copy_lock held +// Counts (delta 1) or uncounts (-1) the pins a copy names: from its list, without walking its records. A copy of a +// per-frame recording is 0.5-2 MiB of records naming a few pins; walking them again at every free cost about 40 us. +static void copy_pins_ref(const copy_prefix *pf, int delta) { + if (!pf->pins) return; + if (!pf->pin_v) { + pin_refs((const uint8_t *)(pf + 1), delta); + return; + } + uint64_t now = __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); + for (uint64_t i = 0; i < pf->pins; i++) pin_ref(pf->pin_v[i], delta, now); +} + +// A copy in no map and no zombie list: nothing else can reach it, so copy_lock need not be held. +static void copy_free(copy_prefix *pf) { + copy_pins_ref(pf, -1); + free((void *)pf->pin_v); + copy_block_free(pf); +} + +// Copies retired under copy_lock and freed after it is released: the frees, the unpinning walk over their records and +// the large blocks kept the lock exclusive for most of release_copies, while the game's other recording threads waited +// for it in copy_touch (1.4.3 town: three threads, 0.6 ms a frame of waiting). +typedef struct { copy_prefix **v; uint32_t n, cap; } dead_list; +static __thread dead_list dead; + +static void dead_add(copy_prefix *pf) { + if (dead.n == dead.cap) { + uint32_t cap = dead.cap ? dead.cap * 2 : 256; + copy_prefix **v = realloc(dead.v, sizeof *v * cap); + if (!v) { copy_free(pf); return; } // out of memory: free it here, under the lock, as before + dead.v = v; + dead.cap = cap; + } + dead.v[dead.n++] = pf; +} + +static void dead_free(void) { // copy_lock not held + for (uint32_t i = 0; i < dead.n; i++) copy_free(dead.v[i]); + dead.n = 0; +} + +static void zombie_sweep(uint64_t replayed) { // copy_lock held; the swept copies go to the dead list for (uint32_t i = 0; i < nzombies;) { if (zombies[i]->used_at <= replayed) { - copy_free(zombies[i]); + dead_add(zombies[i]); zombies[i] = zombies[--nzombies]; } else { i++; @@ -477,13 +584,16 @@ static uint64_t copy_touch(uint64_t handle, uint64_t now, const char *what) { } // The recording [p, p + n), OP_BEGIN to OP_END (the recording buffer is one block: recording emits no OP_JUMP), copied -// into one host block of its exact size. One allocation and one copy: growing the copy record by record cost the -// game's main thread about 1 ms a frame in reallocations and small copies (1.4.3, field). +// into one host block (copy_block: its exact size, or a kept large block). One allocation and one copy: growing the +// copy record by record cost the game's main thread about 1 ms a frame in reallocations and small copies (1.4.3, +// field). The pins the records name are counted and listed (copy_prefix.pin_v) on the way. static uint8_t *flatten_recording(const uint8_t *p, size_t n) { - uint8_t *out = realloc_wait(NULL, sizeof(copy_prefix) + n, "command copy"); - if (!out) rt_fatal("nvn: command copy allocation"); + copy_prefix *block = copy_block(n, 1, "command copy"); + uint8_t *out = (uint8_t *)block; memcpy(out + sizeof(copy_prefix), p, n); + static __thread struct { const tex_t **v; uint64_t cap; } named; // the pins of this recording, reused uint64_t pins_named = 0; + int listed = 1; // 0 after a failed growth: the copy goes without a list for (size_t at = 0;;) { // the records stay checked: a corrupt one stops here, not in the replay op_hdr h; memcpy(&h, p + at, sizeof h); @@ -493,6 +603,14 @@ static uint8_t *flatten_recording(const uint8_t *p, size_t n) { const tex_t *pinned[9]; int k = op_pins(h.op, out + sizeof(copy_prefix) + at + 8, pinned); for (int i = 0; i < k; i++) pin_ref(pinned[i], 1, 0); + if (k && listed && pins_named + (uint64_t)k > named.cap) { + uint64_t cap = named.cap ? named.cap * 2 : 64; + while (cap < pins_named + (uint64_t)k) cap *= 2; + const tex_t **v = realloc((void *)named.v, sizeof *v * cap); + if (v) named.v = v, named.cap = cap; + else listed = 0; + } + if (k && listed) memcpy((void *)(named.v + pins_named), pinned, sizeof *pinned * (size_t)k); pins_named += (uint64_t)k; at += h.size; if (h.op == OP_END) { @@ -500,7 +618,10 @@ static uint8_t *flatten_recording(const uint8_t *p, size_t n) { break; } } - *(copy_prefix *)out = (copy_prefix){.magic = COPY_MAGIC, .bytes = n, .pins = pins_named}; + const tex_t **pin_v = NULL; + if (pins_named && listed && (pin_v = malloc(sizeof *pin_v * pins_named))) + memcpy((void *)pin_v, (const void *)named.v, sizeof *pin_v * pins_named); + *block = (copy_prefix){.magic = COPY_MAGIC, .bytes = n, .pins = pins_named, .cap = block->cap, .pin_v = pin_v}; return out + sizeof(copy_prefix); } @@ -550,7 +671,7 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { copy_bytes -= pf->bytes; pf->magic = 0; if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it - else copy_free(pf); + else dead_add(pf); b->v[i] = b->v[--b->n]; } if (!b->n) { @@ -560,6 +681,7 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { } } plat_mutex_unlock(©_lock); + dead_free(); } // ---------- CommandBuffer: object and memory @@ -1182,11 +1304,14 @@ void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { } out[i] = handles[i]; if (pf->used_at > replayed && pf->used_at != now && pf->deferred != 1) { - size_t sz = sizeof *pf + pf->bytes; - copy_prefix *cl = malloc(sz); - if (!cl) rt_fatal("nvn: out of memory (recording clone, %zu bytes)", sz); - memcpy(cl, pf, sz); - if (cl->pins) pin_refs((const uint8_t *)(cl + 1), 1); + copy_prefix *cl = copy_block(pf->bytes, 0, "recording clone"); // copy_lock held + size_t cap = cl->cap; + memcpy(cl, pf, sizeof *pf + pf->bytes); + cl->cap = cap; + cl->pin_v = NULL; + if (pf->pin_v && (cl->pin_v = malloc(sizeof *cl->pin_v * pf->pins))) + memcpy((void *)cl->pin_v, (const void *)pf->pin_v, sizeof *cl->pin_v * pf->pins); + copy_pins_ref(cl, 1); cl->magic = 0; // not a live handle: the game never sees it cl->used_at = now; zombie_add(cl); @@ -1197,6 +1322,7 @@ void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { } } plat_mutex_unlock(©_lock); + dead_free(); for (int i = 0; i < n; i++) { int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0); if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1; From 3dd16691f1d060df41fe979beb550c9db04feb85 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 6 Nov 2026 14:05:44 +0300 Subject: [PATCH 6/9] Count host calls per thread instead of with a locked add on the slot's cache line. Co-Authored-By: Claude Opus 5.5 --- runtime/hostfn.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/runtime/hostfn.c b/runtime/hostfn.c index 27aa0c15..02bfbecd 100644 --- a/runtime/hostfn.c +++ b/runtime/hostfn.c @@ -35,7 +35,7 @@ typedef struct { const char *name; GuestFn fn; uint64_t original; // observers: the guest definition rt_original calls - uint64_t calls; + uint64_t calls; // with TOTK_TRACE=host only (hostfn_report) int blocking; // runs with the virtual core given up (hostfn_blocking) } Slot; static Slot slots[MAX_HOSTFN]; @@ -44,6 +44,21 @@ static RtHostEntry added[MAX_ADDED]; static int nadded; static int sealed; +// The call count (hostfn_calls): one counter per calling thread, each on its own cache line, written only by that +// thread. A locked add on a per-function counter beside the read-mostly slot fields made the game threads share and +// bounce those lines on every host call (1.4.3 town: about 1% of the game threads' time in hostfn_entry). +#define MAX_COUNTERS 4096 +typedef struct { _Alignas(64) uint64_t n; } Counter; +static Counter counters[MAX_COUNTERS]; +static uint32_t ncounters; +static uint64_t shared_calls; // the threads past MAX_COUNTERS, counted with an atomic add +static __thread uint64_t *thread_calls; + +__attribute__((noinline)) static uint64_t *counter_new(void) { + uint32_t i = __atomic_fetch_add(&ncounters, 1, __ATOMIC_RELAXED); + return thread_calls = i < MAX_COUNTERS ? &counters[i].n : &shared_calls; +} + static GuestFn table_find(const RtHostEntry *start, const RtHostEntry *stop, const char *name) { __asm__("" : "+r"(start), "+r"(stop)); // separate objects: keep the compiler from reasoning about them for (const RtHostEntry *p = start; p < stop; p++) @@ -146,8 +161,11 @@ static void hostfn_entry(Cpu *c) { uint64_t i = (c->pc - RT_HOSTFN_BASE) >> 4; if (__builtin_expect((c->pc & 15) || i >= (uint64_t)nslots, 0)) bad_entry(c); Slot *s = &slots[i]; - uint64_t n = __atomic_fetch_add(&s->calls, 1, __ATOMIC_RELAXED); - if (__builtin_expect(!n && (rt_trace & RT_TRACE_HOSTCALL), 0)) { + uint64_t *k = thread_calls; + if (__builtin_expect(!k, 0)) k = counter_new(); + if (__builtin_expect(k == &shared_calls, 0)) __atomic_fetch_add(k, 1, __ATOMIC_RELAXED); + else __atomic_store_n(k, *k + 1, __ATOMIC_RELAXED); + if (__builtin_expect(rt_trace & RT_TRACE_HOSTCALL, 0) && !__atomic_fetch_add(&s->calls, 1, __ATOMIC_RELAXED)) { char lr[96]; rt_log("host call %s from %s", s->name, ldr_describe(c->x[30] - 4, lr, sizeof lr)); } @@ -178,8 +196,9 @@ void hostfn_seal(void) { } uint64_t hostfn_calls(void) { - uint64_t n = 0; - for (int i = 0; i < nslots; i++) n += __atomic_load_n(&slots[i].calls, __ATOMIC_RELAXED); + uint32_t m = __atomic_load_n(&ncounters, __ATOMIC_RELAXED); + uint64_t n = __atomic_load_n(&shared_calls, __ATOMIC_RELAXED); + for (uint32_t i = 0; i < m && i < MAX_COUNTERS; i++) n += __atomic_load_n(&counters[i].n, __ATOMIC_RELAXED); return n; } From df9ba65a91259fb7b14e2db6ba45a0cef85bdf55 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 6 Nov 2026 15:23:15 +0300 Subject: [PATCH 7/9] Swap a mod's HeapDef for the game's own only when that table grows TextureHeap heapdef_overlay replaced the served mod table whenever any switched-on mod grew the texture heap. With an HD pack and another mod that ships a HeapDef for other heaps on top of it, the other mod's sizes were lost. It now checks the served table itself (resolution.c res_heapdef_tex_grown): only a table with TextureHeap over the shipped size is swapped, any other stays the base and gets the growth on top. A mod HeapDef the port cannot read no longer logs "TextureHeap 0 KiB": the failure is logged with the mod's name. read_original sat between read_served and its comment; each function has its own comment again. Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 3 ++- runtime/resolution.c | 35 ++++++++++++++++++++++++----------- runtime/romfs.c | 21 ++++++++++++--------- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index d46336b9..7bcc152a 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -262,7 +262,8 @@ uint64_t res_heap_extra_bytes(void); // added to the appli uint64_t res_heap_extra_kib(void); // added to the HeapDef texture heap and its parent; 0 = none uint64_t res_tex_heap_extra_bytes(void); // TOTK_TEX_HEAP_EXTRA_MB: added to the application pool and // the heap cap (mem_init), 2 MiB steps; 0 = none -int res_tex_heap_from_mod(void); // a switched-on mod's HeapDef grows the texture heap +int res_heapdef_tex_grown(const void *in, size_t n); // the HeapDef table (.byml.zs) grows TextureHeap over the + // shipped size (an HD texture pack's) int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n); // the HeapDef table (.byml.zs) with // the grown heaps, malloc'd; 0 = cannot (logged) void res_heapdef_served(void); // romfs.c serves the grown table diff --git a/runtime/resolution.c b/runtime/resolution.c index ac4fa75c..20b73d06 100644 --- a/runtime/resolution.c +++ b/runtime/resolution.c @@ -293,22 +293,31 @@ static int heapdef_open(const void *in, size_t n, Byml *b, uint32_t *heaps, cons #define HEAPDEF_REL "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" static struct { int done; uint64_t kib; } MT; -static void mod_heapdef(const char *name, const char *romfs, void *ctx) { - (void)ctx; - char path[2300]; - if (snprintf(path, sizeof path, "%s/%s", romfs, HEAPDEF_REL) >= (int)sizeof path) return; - uint64_t n = 0; - uint8_t *in = plat_file_read(path, &n); - if (!in) return; +// "TextureHeap" of the HeapDef table in[0..n) in KiB; 0 when the table cannot be read or has no such size (the failure +// is logged with why). +static int32_t heapdef_tex_kib(const void *in, size_t n, const char *why) { Byml b; uint32_t heaps; size_t th_at = 0; int32_t th = 0; - if (heapdef_open(in, (size_t)n, &b, &heaps, "its texture heap is left out") && - (th_at = heap_size_field(&b, heaps, "TextureHeap"))) - memcpy(&th, b.p + th_at, 4); + if (!heapdef_open(in, n, &b, &heaps, why)) return 0; + if ((th_at = heap_size_field(&b, heaps, "TextureHeap"))) memcpy(&th, b.p + th_at, 4); + else rt_log("resolution: the HeapDef table has no TextureHeap size; %s", why); free(b.p); + return th; +} + +static void mod_heapdef(const char *name, const char *romfs, void *ctx) { + (void)ctx; + char path[2300], why[400]; + if (snprintf(path, sizeof path, "%s/%s", romfs, HEAPDEF_REL) >= (int)sizeof path) return; + uint64_t n = 0; + uint8_t *in = plat_file_read(path, &n); + if (!in) return; + snprintf(why, sizeof why, "mod %s: its texture heap growth is left out", name); + int32_t th = heapdef_tex_kib(in, (size_t)n, why); free(in); + if (!th) return; int64_t grow = (int64_t)th - (int64_t)TEX_HEAP_KIB; if (grow > (int64_t)(4096u << 10)) grow = 4096u << 10; if (grow > 0 && (uint64_t)grow > MT.kib) MT.kib = (uint64_t)grow; @@ -321,7 +330,11 @@ static uint64_t mod_tex_heap_kib(void) { return MT.kib; } -int res_tex_heap_from_mod(void) { return mod_tex_heap_kib() > 0; } +// Only such a table is swapped for the game's own (romfs.c heapdef_overlay): a mod table that changes other heaps +// keeps them. A table it cannot read stays the base (logged). +int res_heapdef_tex_grown(const void *in, size_t n) { + return heapdef_tex_kib(in, n, "the mod's table stays the base") > (int32_t)TEX_HEAP_KIB; +} int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib(); diff --git a/runtime/romfs.c b/runtime/romfs.c index 38de9ab2..484c6c0d 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -533,9 +533,8 @@ static void added_init(void) { rt_log("RomFS overlay: %zu files added to the RomFS", nadded); } -// RomFS file `rel` as the game would read it: from an overlay that replaces it (*base), else from the image. malloc'd; -// NULL when it is missing, unreadable or 16 MiB or more (logged with `keeps`, what the game does without it). -// RomFS file rel as the RomFS holds it, whatever overlays replace it; NULL (logged) when it cannot be read. +// RomFS file `rel` as the image holds it, whatever overlays replace it. malloc'd; NULL when it is missing, unreadable +// or 16 MiB or more (logged with `keeps`, what the game does without it). static void *read_original(const char *rel, size_t *n, const char *keeps) { uint64_t i = lower(rel); if (i >= file_count || strcmp(entry_name(entry(i)), rel)) return NULL; @@ -550,6 +549,8 @@ static void *read_original(const char *rel, size_t *n, const char *keeps) { return in; } +// RomFS file `rel` as the game would read it: from an overlay that replaces it (*base), else from the image. malloc'd; +// NULL when it is missing, unreadable or 16 MiB or more (logged with `keeps`, what the game does without it). static void *read_served(const char *rel, Overlay **base, size_t *n, const char *keeps) { uint64_t i = lower(rel); *base = NULL; @@ -846,9 +847,10 @@ static void restbl_overlay(void) { // The HeapDef table a bigger game resolution or TOTK_TEX_HEAP_EXTRA_MB needs (resolution.c): its texture heaps and // their parents grow. The base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS -// original; the grown table is written to /resolution/ and served in the base's place. A mod's table whose -// texture heap is grown (an HD texture pack's, resolution.c mod_tex_heap_kib) is not the base: that growth is already -// in the amount, so the RomFS original grows by it and is served in the mod's place. +// original; the grown table is written to /resolution/ and served in the base's place. A served mod table +// whose own TextureHeap is grown (an HD texture pack's, resolution.c res_heapdef_tex_grown) is not the base: its +// growth is already in the amount (resolution.c mod_tex_heap_kib), so the RomFS original grows by it and is served in +// the mod's place. A mod table that changes other heaps only stays the base. #define HEAPDEF "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" static void heapdef_overlay(void) { if (!res_heap_extra_kib() && !res_tex_heap_extra_bytes()) return; @@ -857,11 +859,12 @@ static void heapdef_overlay(void) { size_t n; void *in = read_served(HEAPDEF, &base, &n, keeps), *out = NULL; size_t out_n = 0; - if (in && base && base->mod && res_tex_heap_from_mod()) { + if (in && base && base->mod && res_heapdef_tex_grown(in, n)) { free(in); in = read_original(HEAPDEF, &n, keeps); - rt_log("RomFS: %s: the game's own HeapDef with the mod's texture heap growth in place of the mod's table", - mods_at(base->mod - 1)->name); + if (in) + rt_log("RomFS: %s: the game's own HeapDef with the mod's texture heap growth in place of the mod's table", + mods_at(base->mod - 1)->name); } int ok = in && res_heapdef_build(in, n, &out, &out_n); free(in); From b578d26628ef638f96f234680198447c330ecbfb Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 6 Nov 2026 16:17:38 +0300 Subject: [PATCH 8/9] Keep the recorder queue's head, tail and producer state on separate cache lines and publish head in batches The recorder spins on head while it waits, and tail, put_h, head and the producer's statistics shared one cache line, so every packet moved that line between the two cores: put took 6% of the replay thread in the fort with the recorder on. head, tail and sleeping now have a line each, the producer keeps its own state and the last tail it read, and pub publishes head once 64 words wait (TOTK_REC_PUB; 0 publishes every packet). Every wait publishes everything first. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_rec.c | 88 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 29 deletions(-) diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 8df8444f..95a4463f 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -19,10 +19,20 @@ #define RING_WORDS (1u << 21) // 16 MiB: a part of a submission is ~1-2 MiB of packets in the town static uint64_t ring[RING_WORDS]; -static volatile uint64_t head, tail; // words published by the producer / consumed by the recorder -static volatile int32_t sleeping; // the recorder waits for wake_ev +// The queue state, a cache line per writer. The recorder spins on head while it waits for packets, so a line the +// producer writes on every packet and the recorder reads or writes goes back and forth between their cores each time: +// with tail, Q.put_h, head and the producer's statistics in one line (the linker's order of these statics), handing +// packets over took 6% of the replay thread in the fort, more than building the descriptors. +static struct { + _Alignas(64) volatile uint64_t head; // words published by the producer; the recorder reads it + _Alignas(64) volatile uint64_t tail; // words consumed by the recorder; the producer reads it when it seems full + _Alignas(64) volatile int32_t sleeping; // the recorder waits for wake_ev; written only around a sleep + _Alignas(64) uint64_t Q.put_h; // producer only: words written + uint64_t pub_h; // the last head published + uint64_t tail_seen; // tail as last read, at most the real one: the room it leaves is there + uint64_t pub_words; // TOTK_REC_PUB: publish head once this many words wait (pub) +} Q; static PlatEvent wake_ev; // auto-reset -static uint64_t put_h; // producer: words written (published up to head) static VkCommandBuffer rec_cb; // producer: the part being recorded, whose vkCmd* become packets static VkCommandBuffer r_cb; // recorder: the command buffer its packets go into (set by a BeginCb packet) static int rec_live; // producer: rec_begin started the recorder for rec_cb @@ -33,11 +43,11 @@ static PlatThread *rec_thread; static int rec_prio; // the submitting thread's priority, which the recorder takes static int rec_chk; // TOTK_REC_CHECK (rec_check_handle) static uint64_t rec_delay_ns; // TOTK_REC_DELAY_US (rec_main) -static volatile uint64_t part_h0; // put_h at rec_begin: where the packets of the part being recorded start +static volatile uint64_t part_h0; // Q.put_h at rec_begin: where the packets of the part being recorded start static volatile uint64_t part_fence; // the value of the part being recorded (rec_begin) // Producer statistics, logged by rec_stats_log. -static struct { uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_; +static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_; enum { R_PAD, R_BeginQuery, R_BeginRendering, R_BindDescriptorSets, R_BindIndexBuffer, R_BindPipeline, R_BindVertexBuffers2, @@ -63,14 +73,14 @@ static void rec_main(void *arg) { plat_thread_priority(rec_prio); uint64_t t = 0; for (;;) { - uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); + uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); if (t == h) { - for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); + for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); if (t == h) { - __atomic_store_n(&sleeping, 1, __ATOMIC_SEQ_CST); - h = __atomic_load_n(&head, __ATOMIC_SEQ_CST); + __atomic_store_n(&Q.sleeping, 1, __ATOMIC_SEQ_CST); + h = __atomic_load_n(&Q.head, __ATOMIC_SEQ_CST); if (t == h) plat_event_wait_ns(&wake_ev, 2000000); // a missed wake costs at most 2 ms - __atomic_store_n(&sleeping, 0, __ATOMIC_RELAXED); + __atomic_store_n(&Q.sleeping, 0, __ATOMIC_RELAXED); continue; } } @@ -82,9 +92,9 @@ static void rec_main(void *arg) { uint32_t op = (uint32_t)(w & 0xffff), n = (uint32_t)(w >> 32); if (op != R_PAD) exec_packet(op, &ring[t % RING_WORDS + 1]); t += n; - if ((k & 63) == 63) __atomic_store_n(&tail, t, __ATOMIC_RELEASE); + if ((k & 63) == 63) __atomic_store_n(&Q.tail, t, __ATOMIC_RELEASE); } - __atomic_store_n(&tail, t, __ATOMIC_RELEASE); + __atomic_store_n(&Q.tail, t, __ATOMIC_RELEASE); } return; } @@ -93,48 +103,63 @@ static void rec_main(void *arg) { static void wake(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); - if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); + if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); } +// Room for need words: the tail read last leaves it in all but about one packet in 16 MiB of them. static void wait_space(uint64_t need) { - if (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) <= RING_WORDS) return; + if (Q.put_h + need - Q.tail_seen <= RING_WORDS) return; + Q.tail_seen = __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE); + if (Q.put_h + need - Q.tail_seen <= RING_WORDS) return; RS_.full_waits++; - __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); + Q.pub_h = Q.put_h; + __atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE); wake(); - while (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) > RING_WORDS) _mm_pause(); + while (Q.put_h + need - (Q.tail_seen = __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE)) > RING_WORDS) _mm_pause(); } // Room for a packet of bytes of arguments; the caller fills it and calls pub(). static void *put(uint32_t op, size_t bytes) { - uint64_t n = 1 + (bytes + 7) / 8, pos = put_h % RING_WORDS; + uint64_t n = 1 + (bytes + 7) / 8, pos = Q.put_h % RING_WORDS; if (n > RING_WORDS / 4) rt_fatal("vk_rec: a packet of %llu bytes", (unsigned long long)bytes); if (pos + n > RING_WORDS) { uint64_t pad = RING_WORDS - pos; wait_space(pad); ring[pos] = (uint64_t)R_PAD | pad << 32; - put_h += pad, pos = 0; + Q.put_h += pad, pos = 0; } wait_space(n); ring[pos] = (uint64_t)op | n << 32; // Packers reserve room for optional arrays they may leave unwritten (BeginRendering's depth and stencil); with // TOTK_REC_CHECK those words would still hold handles from the ring's previous lap. if (rec_chk) memset(&ring[pos + 1], 0, (n - 1) * 8); - put_h += n; + Q.put_h += n; RS_.packets++, RS_.words += n; return &ring[pos + 1]; } +// After a packet: head moves once Q.pub_words words wait, so the recorder reads a line the producer is still filling +// once per batch instead of once per packet. Whatever the producer waits for publishes everything first (wait_space, +// rec_drain), and the packets that begin and end a part go out at once (pub_now), so the recorder begins and ends +// command buffers as early as before; TOTK_REC_CHECK publishes every packet, since rec_check_handle searches the ring +// up to head. +static void pub_now(void) { + Q.pub_h = Q.put_h; + __atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE); + if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); +} + static void pub(void) { - __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); - if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); + if (Q.put_h - Q.pub_h >= Q.pub_words) pub_now(); } void rec_drain(void) { - if (!rec_thread || __atomic_load_n(&tail, __ATOMIC_ACQUIRE) == put_h) return; + if (!rec_thread || __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE) == Q.put_h) return; uint64_t t0 = now_ns(); - __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); + Q.pub_h = Q.put_h; + __atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE); wake(); - for (int i = 0; __atomic_load_n(&tail, __ATOMIC_ACQUIRE) != put_h; i++) + for (int i = 0; __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE) != Q.put_h; i++) if (i < 20000) _mm_pause(); else plat_yield(); RS_.drains++, RS_.drain_ns += now_ns() - t0; @@ -175,6 +200,9 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { rec_chk = rt_knob("REC_CHECK", "0"); // debug switches, read before the recorder starts const char *d = cfg_get("TOTK_REC_DELAY_US"); rec_delay_ns = d && *d && atoi(d) > 0 ? (uint64_t)atoi(d) * 1000 : 0; + const char *pw = cfg_get("TOTK_REC_PUB"); // words per publish (pub); 0: every packet + Q.pub_words = rec_chk ? 0 : pw && *pw ? (uint64_t)atoi(pw) : 64; + if (Q.pub_words > RING_WORDS / 8) Q.pub_words = RING_WORDS / 8; rec_thread = plat_thread_new(rec_main, NULL, "vk recorder"); if (!rec_thread) { rt_log("vk_rec: no recorder thread: the replay makes its Vulkan calls itself"); @@ -183,6 +211,8 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { rt_log("vk_rec: the replay's Vulkan commands are made by a recorder thread%s", mode == 2 ? " in the B blocks (TOTK_AB)" : ""); if (rec_chk) rt_log("vk_rec: TOTK_REC_CHECK: deferred and destroyed objects are searched in the ring"); + if (Q.pub_words != 64) rt_log("vk_rec: TOTK_REC_PUB: head published every %llu words", + (unsigned long long)Q.pub_words); if (rec_delay_ns) rt_log("vk_rec: TOTK_REC_DELAY_US: the recorder waits %llu us before each batch", (unsigned long long)(rec_delay_ns / 1000)); } @@ -198,11 +228,11 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { if (!on) return; } RS_.parts_on++; - __atomic_store_n(&part_h0, put_h, __ATOMIC_RELEASE); + __atomic_store_n(&part_h0, Q.put_h, __ATOMIC_RELEASE); __atomic_store_n(&part_fence, fence, __ATOMIC_RELEASE); A_BeginCb *a = put(R_BeginCb, sizeof *a); *a = (A_BeginCb){cb, own, crumb, n}; - pub(); + pub_now(); rec_cb = cb; rec_live = 1; } @@ -241,7 +271,7 @@ int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n) { } A_EndCb *a = put(R_EndCb, sizeof *a); *a = (A_EndCb){crumb, n}; - pub(); + pub_now(); rec_live = 0; sub_pending = 1; return 1; @@ -306,7 +336,7 @@ static const char *const kind_name[REC_KINDS] = {"image", "view", "buffer"}; static volatile int32_t chk_seen[2], chk_named, chk_logged, chk_hz[2][2][REC_KINDS][64]; // [destroy][made][kind][op] void rec_check_handle(int kind, uint64_t handle, int destroy, uint64_t tl, uint64_t gpu) { if (!rec_chk || !handle || kind < 0 || kind >= REC_KINDS || !__atomic_load_n(&rec_live, __ATOMIC_RELAXED)) return; - uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE), p = __atomic_load_n(&part_h0, __ATOMIC_ACQUIRE); + uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE), p = __atomic_load_n(&part_h0, __ATOMIC_ACQUIRE); uint64_t fence = __atomic_load_n(&part_fence, __ATOMIC_ACQUIRE); if (p > h || h - p > RING_WORDS / 2) return; // a part this large is not searched plat_inc32(&chk_seen[destroy != 0]); @@ -317,7 +347,7 @@ void rec_check_handle(int kind, uint64_t handle, int destroy, uint64_t tl, uint6 if (op != R_PAD) for (uint32_t i = 1; i < n; i++) { if (ring[(p + i) % RING_WORDS] != handle) continue; - int made = p < __atomic_load_n(&tail, __ATOMIC_ACQUIRE); + int made = p < __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE); if (!destroy && tl >= fence) { plat_inc32(&chk_named); return; } plat_inc32(&chk_hz[destroy][made][kind][op & 63]); if (plat_inc32(&chk_logged) <= 64) From 0f4e0af554f34aa4c04999da6420d0c9d6389605 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 6 Nov 2026 16:24:56 +0300 Subject: [PATCH 9/9] Declare the recorder queue's put_h field by its own name, which the move into Q had rewritten as Q.put_h. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_rec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 95a4463f..036fe370 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -27,7 +27,7 @@ static struct { _Alignas(64) volatile uint64_t head; // words published by the producer; the recorder reads it _Alignas(64) volatile uint64_t tail; // words consumed by the recorder; the producer reads it when it seems full _Alignas(64) volatile int32_t sleeping; // the recorder waits for wake_ev; written only around a sleep - _Alignas(64) uint64_t Q.put_h; // producer only: words written + _Alignas(64) uint64_t put_h; // producer only: words written uint64_t pub_h; // the last head published uint64_t tail_seen; // tail as last read, at most the real one: the room it leaves is there uint64_t pub_words; // TOTK_REC_PUB: publish head once this many words wait (pub)