diff --git a/README.md b/README.md index c0ffc147..38780dce 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,8 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). - Shaders prepared at install instead of during play - Mods made for emulators: unpack a mod into the `mods` folder next to `totk.exe`; switch and order them in the menu. RomFS mods, TKMM's merged output and ExeFS code patches (`.ips`, `.pchtxt`) work; the game rebuilds itself for code patches - ([docs/mods.md](docs/mods.md)) + ([docs/mods.md](docs/mods.md)). HD texture packs work: the port grows the texture heap a pack asks for by itself. Tested + with the HDPacks.com 4K Texture Pack v1.0 on 1.4.3 and 1.2.1; other packs come with no guarantee - Atmosphère cheats: put a cheat file into a mod and switch its cheats in the menu ([docs/cheats.md](docs/cheats.md)) ## Planned diff --git a/docs/mods.md b/docs/mods.md index d26553db..0734f382 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -87,6 +87,21 @@ The port keys a mod by its folder name. A renamed folder counts as a new mod: Rename mod folders before you set them up in the menu, not after. +## HD texture packs + +Texture packs made for the emulators replace the game's textures with larger ones, so the heap the world's textures +stream into must grow. Such a pack grows it in its own `System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs`, sized for +the emulators' 8 GB memory option. The port reads that file at start: the texture heap and the game's memory pool grow +by the pack's amount, and the game gets the port's own heap table with that growth (the log says +`resolution: mod ships a HeapDef with TextureHeap ...`). Nothing needs to be set. For a pack without such a file +that still loops the panic Blood Moon, set `tex_heap_extra_mb` under `[mods]` in `totk.ini` (0 to 4096 MiB). + +Tested: **HDPacks.com Tears of the Kingdom 4K Texture Pack v1.0** (`4kTextures`, 10,407 textures, 6.5 GB) on game 1.4.3 +and 1.2.1, RTX 4080 SUPER: it loads into the game from a save and runs at the same frame rate, with about 3.3 GB more of +textures in memory (4.5 GB of game memory pools against 1.2 GB without it). Other packs are not tested and come with no +guarantee. Such a pack needs a lot of video memory; on a GPU with less than 8 GB, `astc_bc = 1` under `[performance]` +keeps its textures at a quarter of the size. + ## What does not work | On the Mods tab | Why | What to do | diff --git a/runtime/cfg.c b/runtime/cfg.c index 34ca8d3d..badaa090 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -59,7 +59,7 @@ static const char *const known[] = { "SHADER_MEMO", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -82,6 +82,7 @@ static const struct { const char *key, *section; } sections[] = { {"SHADOW_SCALE", "display"}, {"SHADOW_RANGE", "display"}, {"CONTACT_SHADOWS", "display"}, {"PCSS", "display"}, {"MODS_DIR", "mods"}, {"MODS_ORDER", "mods"}, {"MODS_DISABLED", "mods"}, {"MODS_REBUILD", "mods"}, + {"TEX_HEAP_EXTRA_MB", "mods"}, {"MENU_LANG", "language"}, {"LANG", "language"}, {"OPERATION_MODE", "language"}, {"VOLUME", "audio"}, {"AUDIO_CHANNELS", "audio"}, {"MOTION", "input"}, {"RUMBLE_STRENGTH", "input"}, {"RUMBLE_CURVE", "input"}, {"RUMBLE_TRIGGERS", "input"}, diff --git a/runtime/host.h b/runtime/host.h index 21d0534f..7bcc152a 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -260,6 +260,10 @@ void cam_pack_served(void); // romfs.c serves the // ---- game resolution (resolution.c): TOTK_OUTPUT_RES, TOTK_RENDER_SCALE uint64_t res_heap_extra_bytes(void); // added to the application pool (mem_init), 2 MiB steps uint64_t res_heap_extra_kib(void); // added to the HeapDef texture heap and its parent; 0 = none +uint64_t res_tex_heap_extra_bytes(void); // TOTK_TEX_HEAP_EXTRA_MB: added to the application pool and + // the heap cap (mem_init), 2 MiB steps; 0 = none +int res_heapdef_tex_grown(const void *in, size_t n); // the HeapDef table (.byml.zs) grows TextureHeap over the + // shipped size (an HD texture pack's) int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n); // the HeapDef table (.byml.zs) with // the grown heaps, malloc'd; 0 = cannot (logged) void res_heapdef_served(void); // romfs.c serves the grown table diff --git a/runtime/hostfn.c b/runtime/hostfn.c index 27aa0c15..02bfbecd 100644 --- a/runtime/hostfn.c +++ b/runtime/hostfn.c @@ -35,7 +35,7 @@ typedef struct { const char *name; GuestFn fn; uint64_t original; // observers: the guest definition rt_original calls - uint64_t calls; + uint64_t calls; // with TOTK_TRACE=host only (hostfn_report) int blocking; // runs with the virtual core given up (hostfn_blocking) } Slot; static Slot slots[MAX_HOSTFN]; @@ -44,6 +44,21 @@ static RtHostEntry added[MAX_ADDED]; static int nadded; static int sealed; +// The call count (hostfn_calls): one counter per calling thread, each on its own cache line, written only by that +// thread. A locked add on a per-function counter beside the read-mostly slot fields made the game threads share and +// bounce those lines on every host call (1.4.3 town: about 1% of the game threads' time in hostfn_entry). +#define MAX_COUNTERS 4096 +typedef struct { _Alignas(64) uint64_t n; } Counter; +static Counter counters[MAX_COUNTERS]; +static uint32_t ncounters; +static uint64_t shared_calls; // the threads past MAX_COUNTERS, counted with an atomic add +static __thread uint64_t *thread_calls; + +__attribute__((noinline)) static uint64_t *counter_new(void) { + uint32_t i = __atomic_fetch_add(&ncounters, 1, __ATOMIC_RELAXED); + return thread_calls = i < MAX_COUNTERS ? &counters[i].n : &shared_calls; +} + static GuestFn table_find(const RtHostEntry *start, const RtHostEntry *stop, const char *name) { __asm__("" : "+r"(start), "+r"(stop)); // separate objects: keep the compiler from reasoning about them for (const RtHostEntry *p = start; p < stop; p++) @@ -146,8 +161,11 @@ static void hostfn_entry(Cpu *c) { uint64_t i = (c->pc - RT_HOSTFN_BASE) >> 4; if (__builtin_expect((c->pc & 15) || i >= (uint64_t)nslots, 0)) bad_entry(c); Slot *s = &slots[i]; - uint64_t n = __atomic_fetch_add(&s->calls, 1, __ATOMIC_RELAXED); - if (__builtin_expect(!n && (rt_trace & RT_TRACE_HOSTCALL), 0)) { + uint64_t *k = thread_calls; + if (__builtin_expect(!k, 0)) k = counter_new(); + if (__builtin_expect(k == &shared_calls, 0)) __atomic_fetch_add(k, 1, __ATOMIC_RELAXED); + else __atomic_store_n(k, *k + 1, __ATOMIC_RELAXED); + if (__builtin_expect(rt_trace & RT_TRACE_HOSTCALL, 0) && !__atomic_fetch_add(&s->calls, 1, __ATOMIC_RELAXED)) { char lr[96]; rt_log("host call %s from %s", s->name, ldr_describe(c->x[30] - 4, lr, sizeof lr)); } @@ -178,8 +196,9 @@ void hostfn_seal(void) { } uint64_t hostfn_calls(void) { - uint64_t n = 0; - for (int i = 0; i < nslots; i++) n += __atomic_load_n(&slots[i].calls, __ATOMIC_RELAXED); + uint32_t m = __atomic_load_n(&ncounters, __ATOMIC_RELAXED); + uint64_t n = __atomic_load_n(&shared_calls, __ATOMIC_RELAXED); + for (uint32_t i = 0; i < m && i < MAX_COUNTERS; i++) n += __atomic_load_n(&counters[i].n, __ATOMIC_RELAXED); return n; } diff --git a/runtime/main.c b/runtime/main.c index 49d227a0..d2c68575 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -397,7 +397,9 @@ static void check_memory(void) { CHECK(q.state == 0 && q.base == g->alias.base && q.size == g->alias.size, "QueryMemory on the free alias region"); CHECK(mem_map_physical(a, 0x1000) == 0 && LD64(a + 0x10) == 0 && mem_unmap_physical(a, 0x1000) == 0, "remapped physical memory is not zeroed"); - CHECK(mem_map_physical(a, 0x100000000ull) == 0x10801, "MapPhysicalMemory beyond the pool is not LimitReached"); + // past the pool whatever its size (TOTK_APP_POOL, TOTK_TEX_HEAP_EXTRA_MB): 4 GiB fits a grown one + CHECK(mem_map_physical(a, (g->pool + 0x100000) & ~0xfffffull) == 0x10801, + "MapPhysicalMemory beyond the pool is not LimitReached"); mem_query(RT_HOSTFN_BASE + 0x40, &q); CHECK(q.state == 0x10 && q.base == RT_HOSTFN_BASE, "QueryMemory on the host-function window"); mem_query(g->alias.base + g->alias.size + 0x1000, &q); diff --git a/runtime/memory.c b/runtime/memory.c index 4dcc064c..019c0fab 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -20,11 +20,11 @@ #define PAGE 0x1000ull #define GiB (1ull << 30) #define LOW_SEARCH_FROM 0x100000000ull -#define LOW_SIZE (6 * GiB) +#define LOW_BASE (6 * GiB) // the stack and heap block without a texture heap extra #define STACK_OFF 0x10000000ull #define STACK_SIZE 0x10000000ull #define HEAP_OFF (STACK_OFF + STACK_SIZE + 0x10000000ull) -#define HEAP_CAP (4 * GiB) +#define HEAP_CAP (4 * GiB) // without TOTK_TEX_HEAP_EXTRA_MB, which raises it by its own size (mem_init) #define ALIAS_SIZE (64 * GiB) #define ALIAS_ALIGN (4 * GiB) #define ALIAS_PAGES (ALIAS_SIZE / PAGE) @@ -48,6 +48,9 @@ enum { ATTR_PERMISSION_LOCKED = 16 }; static RtGuestLayout L; static uint64_t mods_lo, mods_hi; // fixed module window +// The stack and heap block: LOW_BASE, plus the texture heap extra (tex_heap_extra_mb or a mod's HeapDef, +// res_tex_heap_extra_bytes) rounded up to a GiB, so only a player who grows the texture heap reserves more (mem_init). +static uint64_t low_size = LOW_BASE; static PlatMutex lock = PLAT_MUTEX_INIT; static uint64_t heap_size, alias_bytes; static uint8_t *alias_pages; // 1 per mapped 4 KiB page of the alias region @@ -78,7 +81,7 @@ static volatile int64_t frz_count[4]; // freezes; thaws by a write fault, by m static int watched(uint64_t addr, uint64_t size) { const RtGuestLayout *g = &L; - return (addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size) || + return (addr >= g->aslr.base && size <= low_size && addr - g->aslr.base <= low_size - size) || (addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size); } @@ -221,10 +224,15 @@ void mem_init(void) { // Stack and heap block, then the alias region above it, both searched below the module window. uint64_t low = 0, alias = 0; - for (uint64_t at = LOW_SEARCH_FROM; at + LOW_SIZE <= mods_lo && !low; at += GiB) - if (reserve_at(at, LOW_SIZE, 1)) low = at; - if (!low) reexec_or_fail("a 6 GiB stack and heap block below 2^39"); - for (uint64_t at = align_up(low + LOW_SIZE, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) + low_size = LOW_BASE + align_up(res_tex_heap_extra_bytes(), GiB); + for (uint64_t at = LOW_SEARCH_FROM; at + low_size <= mods_lo && !low; at += GiB) + if (reserve_at(at, low_size, 1)) low = at; + if (!low) { + char what[64]; + snprintf(what, sizeof what, "a %llu GiB stack and heap block below 2^39", (unsigned long long)(low_size / GiB)); + reexec_or_fail(what); + } + for (uint64_t at = align_up(low + low_size, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) if (reserve_at(at, ALIAS_SIZE, 1)) alias = at; if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39"); @@ -238,11 +246,11 @@ void mem_init(void) { L = (RtGuestLayout){ .aslr = {low, RT_ADDRESS_LIMIT - low}, .stack = {low + STACK_OFF, STACK_SIZE}, - .heap = {low + HEAP_OFF, HEAP_CAP}, + .heap = {low + HEAP_OFF, HEAP_CAP + res_tex_heap_extra_bytes()}, .alias = {alias, ALIAS_SIZE}, .kernel = {RT_KERNEL_BASE, RT_KERNEL_SIZE}, .hostfn = {RT_HOSTFN_BASE, RT_HOSTFN_SIZE}, - .pool = pool ? strtoull(pool, NULL, 0) : DEFAULT_POOL + res_heap_extra_bytes(), // resolution.c + .pool = pool ? strtoull(pool, NULL, 0) : DEFAULT_POOL + res_heap_extra_bytes() + res_tex_heap_extra_bytes(), }; if (!L.pool) rt_fatal("memory: TOTK_APP_POOL is 0"); } @@ -276,7 +284,7 @@ static int inside(uint64_t a, uint64_t n, RtRange r) { return a >= r.base && n < // ---- 0x01 SetHeapSize uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { plat_mutex_lock(&lock); - if (size > HEAP_CAP || size % 0x200000 || size + image_bytes() > L.pool) { + if (size > L.heap.size || size % 0x200000 || size + image_bytes() > L.pool) { plat_mutex_unlock(&lock); return R_INVALID_STATE; } @@ -506,7 +514,7 @@ static void taken_gap(uint64_t a, uint64_t *base, uint64_t *size) { uint64_t lo = 0, hi = RT_ADDRESS_LIMIT; RtRange known[3 + MOD_COUNT + 2]; int n = 0; - known[n++] = (RtRange){L.aslr.base, LOW_SIZE}; + known[n++] = (RtRange){L.aslr.base, low_size}; known[n++] = L.alias; known[n++] = L.kernel; known[n++] = L.hostfn; @@ -549,14 +557,14 @@ uint32_t mem_query(uint64_t a, RtMemoryInfo *out) { r.base = 0; r.size = low; r.state = ST_FREE; - } else if (a < low + LOW_SIZE) { + } else if (a < low + low_size) { uint64_t heap = L.heap.base; if (a >= heap && a < heap + heap_size) { r = (RtMemoryInfo){.base = heap, .size = heap_size, .state = ST_NORMAL, .perm = PERM_RW}; } else if (a < heap) { r = (RtMemoryInfo){.base = low, .size = heap - low, .state = ST_FREE}; } else { - r = (RtMemoryInfo){.base = heap + heap_size, .size = low + LOW_SIZE - heap - heap_size, .state = ST_FREE}; + r = (RtMemoryInfo){.base = heap + heap_size, .size = low + low_size - heap - heap_size, .state = ST_FREE}; } } else if (a - L.alias.base < L.alias.size) { uint64_t p = (a - L.alias.base) / PAGE; diff --git a/runtime/mods.c b/runtime/mods.c index 4f3ad08c..d960a407 100644 --- a/runtime/mods.c +++ b/runtime/mods.c @@ -338,22 +338,49 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) { return 1; } +// The mods folder into out: TOTK_MODS_DIR, else /mods. 0 when the exe's path is unknown. +static int mods_folder(char *out, size_t cap) { + const char *d = cfg_get("TOTK_MODS_DIR"); + if (d && *d) return snprintf(out, cap, "%s", d) < (int)cap; + char exe[2048]; + if (!plat_exe_path(exe, sizeof exe)) return 0; + char *cut = strrchr(exe, '/'), *back = strrchr(exe, '\\'); + if (back > cut) cut = back; + if (!cut) return 0; + *cut = 0; + return snprintf(out, cap, "%s/mods", exe) < (int)cap; +} + +typedef struct { + const char *folder, *disabled; + void (*cb)(const char *name, const char *romfs, void *ctx); + void *ctx; +} Peek; + +static int peek_cb(const char *name, int kind, uint64_t size, void *ctx) { + (void)size; + const Peek *pk = ctx; + if (kind != PLAT_FS_DIR || listed(pk->disabled, name)) return 1; + char path[2100], romfs[2048]; + if (snprintf(path, sizeof path, "%s/%s", pk->folder, name) >= (int)sizeof path) return 1; + find_root(path, "romfs", romfs, sizeof romfs); + if (!romfs[0]) find_root(path, "romfslite", romfs, sizeof romfs); + if (romfs[0]) pk->cb(name, romfs, pk->ctx); + return 1; +} + +void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), void *ctx) { + char folder[2048]; + if (!mods_folder(folder, sizeof folder) || !plat_dir_exists(folder)) return; + Peek pk = {folder, cfg_get("TOTK_MODS_DISABLED"), cb, ctx}; + plat_fs_list(folder, peek_cb, &pk); +} + void mods_scan(void) { static int done; if (done) return; done = 1; - const char *d = cfg_get("TOTK_MODS_DIR"); - if (d && *d) { - snprintf(dir, sizeof dir, "%s", d); - } else { - char exe[2048]; - if (!plat_exe_path(exe, sizeof exe)) return; - char *cut = strrchr(exe, '/'), *back = strrchr(exe, '\\'); - if (back > cut) cut = back; - if (!cut) return; - *cut = 0; - snprintf(dir, sizeof dir, "%s/mods", exe); - } + if (!mods_folder(dir, sizeof dir)) return; if (!plat_dir_exists(dir)) { rt_log("mods: no folder %s", dir); return; diff --git a/runtime/mods.h b/runtime/mods.h index 98a513db..318a9632 100644 --- a/runtime/mods.h +++ b/runtime/mods.h @@ -40,6 +40,9 @@ typedef struct { } ModInfo; void mods_scan(void); // once; later calls return at once +// Every switched-on mod folder's RomFS root, without the rest of the scan (no ExeFS checks, no order): for mem_init, +// which runs before the paths mods_scan needs are set (resolution.c mod_tex_heap_kib). cb gets the name and root. +void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx), void *ctx); const char *mods_dir(void); // the folder scanned (default: /mods) int mods_count(void); ModInfo *mods_at(int i); // 0 = top of the order; NULL out of range diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index 21644b76..874002f7 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -141,10 +141,13 @@ static void pool_drain(void) { // system-managed page file, so the same request succeeds a moment later, and a stall of some frames is better than // ending the game. Frees the pooled buffers first. NULL only after about 30 s without memory; the caller ends the game. // Never called with a lock held. +static void big_drain(void); + static void *realloc_wait(void *old, size_t n, const char *what) { void *p = realloc(old, n); if (p) return p; pool_drain(); + big_drain(); static uint32_t reports; int report = __atomic_fetch_add(&reports, 1, __ATOMIC_RELAXED) < 8; if (report) { @@ -406,8 +409,10 @@ void nvn_pins_stats(uint64_t *live, uint64_t *freed) { // adding control memory again releases the copies recorded there without walking every live copy. 4 KiB pages cost // about 500 lookups per AddControlMemory under the exclusive lock (1.4.3 field: 53 calls a frame, 2 MiB ranges, 3.7 // copies per page), and the recording threads waited on it. -// deferred: 0 unknown, 1 none, 2 some; pins: texture pins its ops name, counted in their refs (pin_refs) -typedef struct { uint64_t magic, bytes, made_at, used_at, deferred, pins; } copy_prefix; +// deferred: 0 unknown, 1 none, 2 some; pins: texture pins its ops name, counted in their refs; pin_v: those pins, in +// record order, or NULL (then pin_refs walks the records); cap: bytes of the block from the large-block cache +// (copy_block), 0 for a plain heap block +typedef struct { uint64_t magic, bytes, made_at, used_at, deferred, pins, cap; const tex_t **pin_v; } copy_prefix; _Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned"); typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy; typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket; @@ -429,15 +434,117 @@ static void zombie_add(copy_prefix *pf) { // copy_lock held zombies[nzombies++] = pf; } -static void copy_free(copy_prefix *pf) { // copy_lock held - if (pf->pins) pin_refs((const uint8_t *)(pf + 1), -1); +// Blocks of large copies are kept for the next ones. 1.4.3 makes two copies of 0.5-2 MiB a frame (the per-frame +// recordings that inline many others); above the heap's direct-allocation threshold (about 512 KiB) every malloc mapped +// fresh zero-filled pages and every free unmapped them, so the copy ran at half the speed of a small one and the free +// held copy_lock for tens of microseconds. A block is reused for a copy that fills at least half of it. +#define BIG_MIN ((size_t)256 << 10) // smaller blocks come from the heap and go back to it +#define BIG_MAX ((size_t)16 << 20) // a larger block is never kept +#define BIG_SLOTS 8 +static PlatMutex big_lock = PLAT_MUTEX_INIT; // lock order: copy_lock, then big_lock +static struct { copy_prefix *p; size_t cap; } big[BIG_SLOTS]; +static uint32_t nbig; + +static void big_drain(void) { + plat_mutex_lock(&big_lock); + while (nbig) free(big[--nbig].p); + plat_mutex_unlock(&big_lock); +} + +// A block for a copy of n bytes after its prefix; prefix.cap says where it goes back (copy_block_free). wait: a failed +// allocation waits for memory (realloc_wait), which is never done with a lock held; else it ends the game at once. +static copy_prefix *copy_block(size_t n, int wait, const char *what) { + size_t total = sizeof(copy_prefix) + n, cap = 0; + copy_prefix *pf = NULL; + if (total >= BIG_MIN && total <= BIG_MAX) { + plat_mutex_lock(&big_lock); + uint32_t best = BIG_SLOTS; + for (uint32_t i = 0; i < nbig; i++) + if (big[i].cap >= total && big[i].cap / 2 <= total && (best == BIG_SLOTS || big[i].cap < big[best].cap)) + best = i; + if (best < BIG_SLOTS) { + pf = big[best].p; + cap = big[best].cap; + big[best] = big[--nbig]; + } + plat_mutex_unlock(&big_lock); + if (!pf) cap = (total + total / 8 + 0xffff) & ~(size_t)0xffff; // room for the next frame's slightly larger one + } + if (!pf) pf = wait ? realloc_wait(NULL, cap ? cap : total, what) : malloc(cap ? cap : total); + if (!pf) rt_fatal("nvn: %s allocation (%zu bytes)", what, cap ? cap : total); + pf->cap = cap; + return pf; +} + +static void copy_block_free(copy_prefix *pf) { + size_t cap = pf->cap; + if (cap) { + plat_mutex_lock(&big_lock); + if (nbig < BIG_SLOTS) { + big[nbig].p = pf; + big[nbig++].cap = cap; + pf = NULL; + } else { + uint32_t small = 0; + for (uint32_t i = 1; i < nbig; i++) + if (big[i].cap < big[small].cap) small = i; + if (big[small].cap < cap) { + copy_prefix *drop = big[small].p; + big[small].p = pf; + big[small].cap = cap; + pf = drop; + } + } + plat_mutex_unlock(&big_lock); + } free(pf); } -static void zombie_sweep(uint64_t replayed) { // copy_lock held +// Counts (delta 1) or uncounts (-1) the pins a copy names: from its list, without walking its records. A copy of a +// per-frame recording is 0.5-2 MiB of records naming a few pins; walking them again at every free cost about 40 us. +static void copy_pins_ref(const copy_prefix *pf, int delta) { + if (!pf->pins) return; + if (!pf->pin_v) { + pin_refs((const uint8_t *)(pf + 1), delta); + return; + } + uint64_t now = __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); + for (uint64_t i = 0; i < pf->pins; i++) pin_ref(pf->pin_v[i], delta, now); +} + +// A copy in no map and no zombie list: nothing else can reach it, so copy_lock need not be held. +static void copy_free(copy_prefix *pf) { + copy_pins_ref(pf, -1); + free((void *)pf->pin_v); + copy_block_free(pf); +} + +// Copies retired under copy_lock and freed after it is released: the frees, the unpinning walk over their records and +// the large blocks kept the lock exclusive for most of release_copies, while the game's other recording threads waited +// for it in copy_touch (1.4.3 town: three threads, 0.6 ms a frame of waiting). +typedef struct { copy_prefix **v; uint32_t n, cap; } dead_list; +static __thread dead_list dead; + +static void dead_add(copy_prefix *pf) { + if (dead.n == dead.cap) { + uint32_t cap = dead.cap ? dead.cap * 2 : 256; + copy_prefix **v = realloc(dead.v, sizeof *v * cap); + if (!v) { copy_free(pf); return; } // out of memory: free it here, under the lock, as before + dead.v = v; + dead.cap = cap; + } + dead.v[dead.n++] = pf; +} + +static void dead_free(void) { // copy_lock not held + for (uint32_t i = 0; i < dead.n; i++) copy_free(dead.v[i]); + dead.n = 0; +} + +static void zombie_sweep(uint64_t replayed) { // copy_lock held; the swept copies go to the dead list for (uint32_t i = 0; i < nzombies;) { if (zombies[i]->used_at <= replayed) { - copy_free(zombies[i]); + dead_add(zombies[i]); zombies[i] = zombies[--nzombies]; } else { i++; @@ -477,13 +584,16 @@ static uint64_t copy_touch(uint64_t handle, uint64_t now, const char *what) { } // The recording [p, p + n), OP_BEGIN to OP_END (the recording buffer is one block: recording emits no OP_JUMP), copied -// into one host block of its exact size. One allocation and one copy: growing the copy record by record cost the -// game's main thread about 1 ms a frame in reallocations and small copies (1.4.3, field). +// into one host block (copy_block: its exact size, or a kept large block). One allocation and one copy: growing the +// copy record by record cost the game's main thread about 1 ms a frame in reallocations and small copies (1.4.3, +// field). The pins the records name are counted and listed (copy_prefix.pin_v) on the way. static uint8_t *flatten_recording(const uint8_t *p, size_t n) { - uint8_t *out = realloc_wait(NULL, sizeof(copy_prefix) + n, "command copy"); - if (!out) rt_fatal("nvn: command copy allocation"); + copy_prefix *block = copy_block(n, 1, "command copy"); + uint8_t *out = (uint8_t *)block; memcpy(out + sizeof(copy_prefix), p, n); + static __thread struct { const tex_t **v; uint64_t cap; } named; // the pins of this recording, reused uint64_t pins_named = 0; + int listed = 1; // 0 after a failed growth: the copy goes without a list for (size_t at = 0;;) { // the records stay checked: a corrupt one stops here, not in the replay op_hdr h; memcpy(&h, p + at, sizeof h); @@ -493,6 +603,14 @@ static uint8_t *flatten_recording(const uint8_t *p, size_t n) { const tex_t *pinned[9]; int k = op_pins(h.op, out + sizeof(copy_prefix) + at + 8, pinned); for (int i = 0; i < k; i++) pin_ref(pinned[i], 1, 0); + if (k && listed && pins_named + (uint64_t)k > named.cap) { + uint64_t cap = named.cap ? named.cap * 2 : 64; + while (cap < pins_named + (uint64_t)k) cap *= 2; + const tex_t **v = realloc((void *)named.v, sizeof *v * cap); + if (v) named.v = v, named.cap = cap; + else listed = 0; + } + if (k && listed) memcpy((void *)(named.v + pins_named), pinned, sizeof *pinned * (size_t)k); pins_named += (uint64_t)k; at += h.size; if (h.op == OP_END) { @@ -500,7 +618,10 @@ static uint8_t *flatten_recording(const uint8_t *p, size_t n) { break; } } - *(copy_prefix *)out = (copy_prefix){.magic = COPY_MAGIC, .bytes = n, .pins = pins_named}; + const tex_t **pin_v = NULL; + if (pins_named && listed && (pin_v = malloc(sizeof *pin_v * pins_named))) + memcpy((void *)pin_v, (const void *)named.v, sizeof *pin_v * pins_named); + *block = (copy_prefix){.magic = COPY_MAGIC, .bytes = n, .pins = pins_named, .cap = block->cap, .pin_v = pin_v}; return out + sizeof(copy_prefix); } @@ -550,7 +671,7 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { copy_bytes -= pf->bytes; pf->magic = 0; if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it - else copy_free(pf); + else dead_add(pf); b->v[i] = b->v[--b->n]; } if (!b->n) { @@ -560,6 +681,7 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { } } plat_mutex_unlock(©_lock); + dead_free(); } // ---------- CommandBuffer: object and memory @@ -1182,11 +1304,14 @@ void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { } out[i] = handles[i]; if (pf->used_at > replayed && pf->used_at != now && pf->deferred != 1) { - size_t sz = sizeof *pf + pf->bytes; - copy_prefix *cl = malloc(sz); - if (!cl) rt_fatal("nvn: out of memory (recording clone, %zu bytes)", sz); - memcpy(cl, pf, sz); - if (cl->pins) pin_refs((const uint8_t *)(cl + 1), 1); + copy_prefix *cl = copy_block(pf->bytes, 0, "recording clone"); // copy_lock held + size_t cap = cl->cap; + memcpy(cl, pf, sizeof *pf + pf->bytes); + cl->cap = cap; + cl->pin_v = NULL; + if (pf->pin_v && (cl->pin_v = malloc(sizeof *cl->pin_v * pf->pins))) + memcpy((void *)cl->pin_v, (const void *)pf->pin_v, sizeof *cl->pin_v * pf->pins); + copy_pins_ref(cl, 1); cl->magic = 0; // not a live handle: the game never sees it cl->used_at = now; zombie_add(cl); @@ -1197,6 +1322,7 @@ void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { } } plat_mutex_unlock(©_lock); + dead_free(); for (int i = 0; i < n; i++) { int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0); if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1; diff --git a/runtime/resolution.c b/runtime/resolution.c index 3e70fefa..20b73d06 100644 --- a/runtime/resolution.c +++ b/runtime/resolution.c @@ -39,6 +39,7 @@ #include "cfg.h" #include "host.h" +#include "mods.h" #include "platform/platform.h" #define BASE_W 1920 @@ -46,6 +47,7 @@ #define SCENE_W 1600 #define SCENE_H 900 #define DTA_KIB 113664u // DynamicTextureAllocatorMem, shipped +#define TEX_HEAP_KIB 506880u // TextureHeap, shipped (1.2.1 and 1.4.3): what a mod's own HeapDef is measured against #define SCENE_MAX 4096 // per side, as MacNativeGames' TOTK_RENDER_RES #define SHADOW_BASE 960 // the sun shadow map's base size (its cascades: a D16 array of 4 layers) @@ -189,6 +191,33 @@ uint64_t res_heap_extra_kib(void) { uint64_t res_heap_extra_bytes(void) { return ((res_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; } +// TOTK_TEX_HEAP_EXTRA_MB (0 to 4096): more room in "TextureHeap" and its parent "ゲームグラフィックス", the heap the +// world's textures stream into. HD texture packs made for emulators with 8 GB of DRAM grow it by about 3.3 GiB in their +// own HeapDef; without the room the game runs out and starts its panic Blood Moon over and over. The pool and the heap +// cap grow by the same amount (memory.c mem_init). +static uint64_t mod_tex_heap_kib(void); + +static uint64_t cfg_tex_heap_kib(void) { + const char *v = cfg_get("TOTK_TEX_HEAP_EXTRA_MB"); + if (!v || !*v) return 0; + char *end; + long mb = strtol(v, &end, 10); + if (end == v || *end || mb < 0 || mb > 4096) { + static int logged; + if (!logged++) rt_log("resolution: TOTK_TEX_HEAP_EXTRA_MB=%s is not 0..4096; 0", v); + return 0; + } + return (uint64_t)mb << 10; +} + +// The larger of TOTK_TEX_HEAP_EXTRA_MB and what an enabled mod's own HeapDef adds to the texture heap (mod_tex_heap_kib). +static uint64_t tex_heap_extra_kib(void) { + uint64_t c = cfg_tex_heap_kib(), m = mod_tex_heap_kib(); + return c > m ? c : m; +} + +uint64_t res_tex_heap_extra_bytes(void) { return ((tex_heap_extra_kib() << 10) + 0x1fffff) & ~0x1fffffull; } + // ---- the HeapDef table. BYML version 2..7, little endian ("YB"): a hash node is {u8 0xC1, u24 count} and count // entries {u24 key index, u8 type, u32 value}; an int (type 0xD1) is stored in the entry, so a size changes in place. typedef struct { uint8_t *p; size_t n; uint32_t keys; } Byml; @@ -226,28 +255,95 @@ static size_t heap_size_field(const Byml *b, uint32_t heaps, const char *heap) { return at && u32_at(b, at, &node) ? field(b, node, "Size", 0xd1) : 0; } -int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { - uint64_t extra = res_heap_extra_kib(); - *out = NULL; - if (!extra) return 0; +// A HeapDef in b's place (zstd frame in[0..n) with its content size, no dictionary): b.p malloc'd and *heaps its +// "Heaps" node; 0 when it is not one (b.p freed). why: what the log line says the caller keeps then. +static int heapdef_open(const void *in, size_t n, Byml *b, uint32_t *heaps, const char *why) { + b->p = NULL; if (ZSTD_getDictID_fromFrame(in, n)) { - rt_log("resolution: the HeapDef table needs a zstd dictionary; the game keeps its own resolution"); + rt_log("resolution: the HeapDef table needs a zstd dictionary; %s", why); return 0; } unsigned long long raw = ZSTD_getFrameContentSize(in, n); if (raw == ZSTD_CONTENTSIZE_UNKNOWN || raw == ZSTD_CONTENTSIZE_ERROR || raw > (64u << 20)) { - rt_log("resolution: the HeapDef table is not a zstd frame with a size; the game keeps its own resolution"); + rt_log("resolution: the HeapDef table is not a zstd frame with a size; %s", why); return 0; } - Byml b = {malloc((size_t)raw), (size_t)raw, 0}; - if (!b.p) return 0; - size_t got = ZSTD_decompress(b.p, b.n, in, n); - uint32_t root, heaps = 0; + *b = (Byml){malloc((size_t)raw), (size_t)raw, 0}; + if (!b->p) return 0; + size_t got = ZSTD_decompress(b->p, b->n, in, n); + uint32_t root; uint16_t ver = 0; - if (!ZSTD_isError(got) && got == b.n && b.n >= 16) memcpy(&ver, b.p + 2, 2); - size_t heaps_at = 0, dta_at = 0, gfx_at = 0; - if (ver >= 2 && ver <= 7 && !memcmp(b.p, "YB", 2) && u32_at(&b, 4, &b.keys) && u32_at(&b, 12, &root) && - (heaps_at = field(&b, root, "Heaps", 0xc1)) && u32_at(&b, heaps_at, &heaps)) { + if (!ZSTD_isError(got) && got == b->n && b->n >= 16) memcpy(&ver, b->p + 2, 2); + size_t heaps_at = 0; + if (ver >= 2 && ver <= 7 && !memcmp(b->p, "YB", 2) && u32_at(b, 4, &b->keys) && u32_at(b, 12, &root) && + (heaps_at = field(b, root, "Heaps", 0xc1)) && u32_at(b, heaps_at, heaps)) + return 1; + free(b->p); + b->p = NULL; + return 0; +} + +// HD texture packs made for the emulators ship their own HeapDef with "TextureHeap" and its parent grown (HDPacks.com's +// 4K pack for TotK: by 3,419,000 KiB, 3.3 GiB, for the emulators' 8 GB memory option). Served as it is, the game asks +// for that room and the application pool has none: it crashed at start (issue #673). So before mem_init sizes the pool, +// every switched-on mod's HeapDef is read (mods.c mods_peek_romfs: mods_scan needs paths main sets later) and the +// largest TextureHeap growth over the shipped size counts as TOTK_TEX_HEAP_EXTRA_MB (the larger of the two, at most +// 4 GiB). romfs.c heapdef_overlay then serves the game's own table grown by that amount in the mod's place, so the +// other heaps keep the sizes of the game version that runs. +#define HEAPDEF_REL "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" +static struct { int done; uint64_t kib; } MT; + +// "TextureHeap" of the HeapDef table in[0..n) in KiB; 0 when the table cannot be read or has no such size (the failure +// is logged with why). +static int32_t heapdef_tex_kib(const void *in, size_t n, const char *why) { + Byml b; + uint32_t heaps; + size_t th_at = 0; + int32_t th = 0; + if (!heapdef_open(in, n, &b, &heaps, why)) return 0; + if ((th_at = heap_size_field(&b, heaps, "TextureHeap"))) memcpy(&th, b.p + th_at, 4); + else rt_log("resolution: the HeapDef table has no TextureHeap size; %s", why); + free(b.p); + return th; +} + +static void mod_heapdef(const char *name, const char *romfs, void *ctx) { + (void)ctx; + char path[2300], why[400]; + if (snprintf(path, sizeof path, "%s/%s", romfs, HEAPDEF_REL) >= (int)sizeof path) return; + uint64_t n = 0; + uint8_t *in = plat_file_read(path, &n); + if (!in) return; + snprintf(why, sizeof why, "mod %s: its texture heap growth is left out", name); + int32_t th = heapdef_tex_kib(in, (size_t)n, why); + free(in); + if (!th) return; + int64_t grow = (int64_t)th - (int64_t)TEX_HEAP_KIB; + if (grow > (int64_t)(4096u << 10)) grow = 4096u << 10; + if (grow > 0 && (uint64_t)grow > MT.kib) MT.kib = (uint64_t)grow; + rt_log("resolution: mod %s ships a HeapDef with TextureHeap %d KiB (the game's %u KiB)%s", name, th, TEX_HEAP_KIB, + grow > 0 ? ": the port grows the texture heap and the pool by the difference" : ""); +} + +static uint64_t mod_tex_heap_kib(void) { + if (!MT.done) MT.done = 1, mods_peek_romfs(mod_heapdef, NULL); + return MT.kib; +} + +// Only such a table is swapped for the game's own (romfs.c heapdef_overlay): a mod table that changes other heaps +// keeps them. A table it cannot read stays the base (logged). +int res_heapdef_tex_grown(const void *in, size_t n) { + return heapdef_tex_kib(in, n, "the mod's table stays the base") > (int32_t)TEX_HEAP_KIB; +} + +int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { + uint64_t extra = res_heap_extra_kib(), tex = tex_heap_extra_kib(); + *out = NULL; + if (!extra && !tex) return 0; + Byml b; + uint32_t heaps = 0; + size_t dta_at = 0, gfx_at = 0; + if (heapdef_open(in, n, &b, &heaps, "the game keeps its own resolution")) { dta_at = heap_size_field(&b, heaps, "DynamicTextureAllocatorMem"); gfx_at = heap_size_field(&b, heaps, "Graphics"); } @@ -263,6 +359,26 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { int32_t dta2 = dta + (int32_t)extra, gfx2 = gfx + (int32_t)extra; memcpy(b.p + dta_at, &dta2, 4); memcpy(b.p + gfx_at, &gfx2, 4); + int32_t th = 0, th2 = 0, gg = 0, gg2 = 0; + if (tex) { // TOTK_TEX_HEAP_EXTRA_MB: the world texture heap and its parent grow by the same amount + static const char parent[] = "\xe3\x82\xb2\xe3\x83\xbc\xe3\x83\xa0\xe3\x82\xb0\xe3\x83\xa9\xe3\x83\x95" + "\xe3\x82\xa3\xe3\x83\x83\xe3\x82\xaf\xe3\x82\xb9"; // ゲームグラフィックス + size_t th_at = heap_size_field(&b, heaps, "TextureHeap"), gg_at = heap_size_field(&b, heaps, parent); + if (th_at && gg_at) { + memcpy(&th, b.p + th_at, 4); + memcpy(&gg, b.p + gg_at, 4); + th2 = th + (int32_t)tex, gg2 = gg + (int32_t)tex; + memcpy(b.p + th_at, &th2, 4); + memcpy(b.p + gg_at, &gg2, 4); + } else { + rt_log("resolution: the HeapDef table has no TextureHeap sizes; TOTK_TEX_HEAP_EXTRA_MB left out"); + tex = 0; + if (!extra) { + free(b.p); + return 0; + } + } + } size_t cap = ZSTD_compressBound(b.n); void *z = malloc(cap); size_t zn = z ? ZSTD_compress(z, cap, b.p, b.n, 9) : 0; @@ -273,7 +389,12 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n) { } *out = z; *out_n = zn; - rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx, gfx2); + if (extra) + rt_log("resolution: HeapDef DynamicTextureAllocatorMem %d -> %d KiB, Graphics %d -> %d KiB", dta, dta2, gfx, + gfx2); + if (tex) + rt_log("resolution: HeapDef TextureHeap %d -> %d KiB and its parent %d -> %d KiB (TOTK_TEX_HEAP_EXTRA_MB or a " + "mod's HeapDef)", th, th2, gg, gg2); return 1; } diff --git a/runtime/romfs.c b/runtime/romfs.c index c062c4b2..484c6c0d 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -533,6 +533,22 @@ static void added_init(void) { rt_log("RomFS overlay: %zu files added to the RomFS", nadded); } +// RomFS file `rel` as the image holds it, whatever overlays replace it. malloc'd; NULL when it is missing, unreadable +// or 16 MiB or more (logged with `keeps`, what the game does without it). +static void *read_original(const char *rel, size_t *n, const char *keeps) { + uint64_t i = lower(rel); + if (i >= file_count || strcmp(entry_name(entry(i)), rel)) return NULL; + uint64_t size = entry(i).size; + void *in = size && size < (16u << 20) ? malloc((size_t)size) : NULL; + if (!in || read_at(image, in, (uint32_t)size, entry(i).offset) != (int64_t)size) { + free(in); + rt_log("RomFS: cannot read %s; %s", rel, keeps); + return NULL; + } + *n = (size_t)size; + return in; +} + // RomFS file `rel` as the game would read it: from an overlay that replaces it (*base), else from the image. malloc'd; // NULL when it is missing, unreadable or 16 MiB or more (logged with `keeps`, what the game does without it). static void *read_served(const char *rel, Overlay **base, size_t *n, const char *keeps) { @@ -829,17 +845,27 @@ static void restbl_overlay(void) { members, packs, (now_s() - t0) * 1000); } -// The HeapDef table a bigger game resolution needs (resolution.c): its texture heap and that heap's parent grow. The -// base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS original; the grown table is -// written to /resolution/ and served in the base's place. +// The HeapDef table a bigger game resolution or TOTK_TEX_HEAP_EXTRA_MB needs (resolution.c): its texture heaps and +// their parents grow. The base is an overlay that replaces the table (a mod's heap sizes stay), else the RomFS +// original; the grown table is written to /resolution/ and served in the base's place. A served mod table +// whose own TextureHeap is grown (an HD texture pack's, resolution.c res_heapdef_tex_grown) is not the base: its +// growth is already in the amount (resolution.c mod_tex_heap_kib), so the RomFS original grows by it and is served in +// the mod's place. A mod table that changes other heaps only stays the base. #define HEAPDEF "System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs" static void heapdef_overlay(void) { - if (!res_heap_extra_kib()) return; + if (!res_heap_extra_kib() && !res_tex_heap_extra_bytes()) return; static const char keeps[] = "the game keeps its own resolution"; Overlay *base; size_t n; void *in = read_served(HEAPDEF, &base, &n, keeps), *out = NULL; size_t out_n = 0; + if (in && base && base->mod && res_heapdef_tex_grown(in, n)) { + free(in); + in = read_original(HEAPDEF, &n, keeps); + if (in) + rt_log("RomFS: %s: the game's own HeapDef with the mod's texture heap growth in place of the mod's table", + mods_at(base->mod - 1)->name); + } int ok = in && res_heapdef_build(in, n, &out, &out_n); free(in); if (ok && serve_generated(HEAPDEF, base, out, out_n, "resolution", "grown heaps", keeps)) res_heapdef_served(); diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 8df8444f..036fe370 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -19,10 +19,20 @@ #define RING_WORDS (1u << 21) // 16 MiB: a part of a submission is ~1-2 MiB of packets in the town static uint64_t ring[RING_WORDS]; -static volatile uint64_t head, tail; // words published by the producer / consumed by the recorder -static volatile int32_t sleeping; // the recorder waits for wake_ev +// The queue state, a cache line per writer. The recorder spins on head while it waits for packets, so a line the +// producer writes on every packet and the recorder reads or writes goes back and forth between their cores each time: +// with tail, Q.put_h, head and the producer's statistics in one line (the linker's order of these statics), handing +// packets over took 6% of the replay thread in the fort, more than building the descriptors. +static struct { + _Alignas(64) volatile uint64_t head; // words published by the producer; the recorder reads it + _Alignas(64) volatile uint64_t tail; // words consumed by the recorder; the producer reads it when it seems full + _Alignas(64) volatile int32_t sleeping; // the recorder waits for wake_ev; written only around a sleep + _Alignas(64) uint64_t put_h; // producer only: words written + uint64_t pub_h; // the last head published + uint64_t tail_seen; // tail as last read, at most the real one: the room it leaves is there + uint64_t pub_words; // TOTK_REC_PUB: publish head once this many words wait (pub) +} Q; static PlatEvent wake_ev; // auto-reset -static uint64_t put_h; // producer: words written (published up to head) static VkCommandBuffer rec_cb; // producer: the part being recorded, whose vkCmd* become packets static VkCommandBuffer r_cb; // recorder: the command buffer its packets go into (set by a BeginCb packet) static int rec_live; // producer: rec_begin started the recorder for rec_cb @@ -33,11 +43,11 @@ static PlatThread *rec_thread; static int rec_prio; // the submitting thread's priority, which the recorder takes static int rec_chk; // TOTK_REC_CHECK (rec_check_handle) static uint64_t rec_delay_ns; // TOTK_REC_DELAY_US (rec_main) -static volatile uint64_t part_h0; // put_h at rec_begin: where the packets of the part being recorded start +static volatile uint64_t part_h0; // Q.put_h at rec_begin: where the packets of the part being recorded start static volatile uint64_t part_fence; // the value of the part being recorded (rec_begin) // Producer statistics, logged by rec_stats_log. -static struct { uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_; +static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_; enum { R_PAD, R_BeginQuery, R_BeginRendering, R_BindDescriptorSets, R_BindIndexBuffer, R_BindPipeline, R_BindVertexBuffers2, @@ -63,14 +73,14 @@ static void rec_main(void *arg) { plat_thread_priority(rec_prio); uint64_t t = 0; for (;;) { - uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); + uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); if (t == h) { - for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); + for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); if (t == h) { - __atomic_store_n(&sleeping, 1, __ATOMIC_SEQ_CST); - h = __atomic_load_n(&head, __ATOMIC_SEQ_CST); + __atomic_store_n(&Q.sleeping, 1, __ATOMIC_SEQ_CST); + h = __atomic_load_n(&Q.head, __ATOMIC_SEQ_CST); if (t == h) plat_event_wait_ns(&wake_ev, 2000000); // a missed wake costs at most 2 ms - __atomic_store_n(&sleeping, 0, __ATOMIC_RELAXED); + __atomic_store_n(&Q.sleeping, 0, __ATOMIC_RELAXED); continue; } } @@ -82,9 +92,9 @@ static void rec_main(void *arg) { uint32_t op = (uint32_t)(w & 0xffff), n = (uint32_t)(w >> 32); if (op != R_PAD) exec_packet(op, &ring[t % RING_WORDS + 1]); t += n; - if ((k & 63) == 63) __atomic_store_n(&tail, t, __ATOMIC_RELEASE); + if ((k & 63) == 63) __atomic_store_n(&Q.tail, t, __ATOMIC_RELEASE); } - __atomic_store_n(&tail, t, __ATOMIC_RELEASE); + __atomic_store_n(&Q.tail, t, __ATOMIC_RELEASE); } return; } @@ -93,48 +103,63 @@ static void rec_main(void *arg) { static void wake(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); - if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); + if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); } +// Room for need words: the tail read last leaves it in all but about one packet in 16 MiB of them. static void wait_space(uint64_t need) { - if (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) <= RING_WORDS) return; + if (Q.put_h + need - Q.tail_seen <= RING_WORDS) return; + Q.tail_seen = __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE); + if (Q.put_h + need - Q.tail_seen <= RING_WORDS) return; RS_.full_waits++; - __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); + Q.pub_h = Q.put_h; + __atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE); wake(); - while (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) > RING_WORDS) _mm_pause(); + while (Q.put_h + need - (Q.tail_seen = __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE)) > RING_WORDS) _mm_pause(); } // Room for a packet of bytes of arguments; the caller fills it and calls pub(). static void *put(uint32_t op, size_t bytes) { - uint64_t n = 1 + (bytes + 7) / 8, pos = put_h % RING_WORDS; + uint64_t n = 1 + (bytes + 7) / 8, pos = Q.put_h % RING_WORDS; if (n > RING_WORDS / 4) rt_fatal("vk_rec: a packet of %llu bytes", (unsigned long long)bytes); if (pos + n > RING_WORDS) { uint64_t pad = RING_WORDS - pos; wait_space(pad); ring[pos] = (uint64_t)R_PAD | pad << 32; - put_h += pad, pos = 0; + Q.put_h += pad, pos = 0; } wait_space(n); ring[pos] = (uint64_t)op | n << 32; // Packers reserve room for optional arrays they may leave unwritten (BeginRendering's depth and stencil); with // TOTK_REC_CHECK those words would still hold handles from the ring's previous lap. if (rec_chk) memset(&ring[pos + 1], 0, (n - 1) * 8); - put_h += n; + Q.put_h += n; RS_.packets++, RS_.words += n; return &ring[pos + 1]; } +// After a packet: head moves once Q.pub_words words wait, so the recorder reads a line the producer is still filling +// once per batch instead of once per packet. Whatever the producer waits for publishes everything first (wait_space, +// rec_drain), and the packets that begin and end a part go out at once (pub_now), so the recorder begins and ends +// command buffers as early as before; TOTK_REC_CHECK publishes every packet, since rec_check_handle searches the ring +// up to head. +static void pub_now(void) { + Q.pub_h = Q.put_h; + __atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE); + if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); +} + static void pub(void) { - __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); - if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); + if (Q.put_h - Q.pub_h >= Q.pub_words) pub_now(); } void rec_drain(void) { - if (!rec_thread || __atomic_load_n(&tail, __ATOMIC_ACQUIRE) == put_h) return; + if (!rec_thread || __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE) == Q.put_h) return; uint64_t t0 = now_ns(); - __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); + Q.pub_h = Q.put_h; + __atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE); wake(); - for (int i = 0; __atomic_load_n(&tail, __ATOMIC_ACQUIRE) != put_h; i++) + for (int i = 0; __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE) != Q.put_h; i++) if (i < 20000) _mm_pause(); else plat_yield(); RS_.drains++, RS_.drain_ns += now_ns() - t0; @@ -175,6 +200,9 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { rec_chk = rt_knob("REC_CHECK", "0"); // debug switches, read before the recorder starts const char *d = cfg_get("TOTK_REC_DELAY_US"); rec_delay_ns = d && *d && atoi(d) > 0 ? (uint64_t)atoi(d) * 1000 : 0; + const char *pw = cfg_get("TOTK_REC_PUB"); // words per publish (pub); 0: every packet + Q.pub_words = rec_chk ? 0 : pw && *pw ? (uint64_t)atoi(pw) : 64; + if (Q.pub_words > RING_WORDS / 8) Q.pub_words = RING_WORDS / 8; rec_thread = plat_thread_new(rec_main, NULL, "vk recorder"); if (!rec_thread) { rt_log("vk_rec: no recorder thread: the replay makes its Vulkan calls itself"); @@ -183,6 +211,8 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { rt_log("vk_rec: the replay's Vulkan commands are made by a recorder thread%s", mode == 2 ? " in the B blocks (TOTK_AB)" : ""); if (rec_chk) rt_log("vk_rec: TOTK_REC_CHECK: deferred and destroyed objects are searched in the ring"); + if (Q.pub_words != 64) rt_log("vk_rec: TOTK_REC_PUB: head published every %llu words", + (unsigned long long)Q.pub_words); if (rec_delay_ns) rt_log("vk_rec: TOTK_REC_DELAY_US: the recorder waits %llu us before each batch", (unsigned long long)(rec_delay_ns / 1000)); } @@ -198,11 +228,11 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { if (!on) return; } RS_.parts_on++; - __atomic_store_n(&part_h0, put_h, __ATOMIC_RELEASE); + __atomic_store_n(&part_h0, Q.put_h, __ATOMIC_RELEASE); __atomic_store_n(&part_fence, fence, __ATOMIC_RELEASE); A_BeginCb *a = put(R_BeginCb, sizeof *a); *a = (A_BeginCb){cb, own, crumb, n}; - pub(); + pub_now(); rec_cb = cb; rec_live = 1; } @@ -241,7 +271,7 @@ int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n) { } A_EndCb *a = put(R_EndCb, sizeof *a); *a = (A_EndCb){crumb, n}; - pub(); + pub_now(); rec_live = 0; sub_pending = 1; return 1; @@ -306,7 +336,7 @@ static const char *const kind_name[REC_KINDS] = {"image", "view", "buffer"}; static volatile int32_t chk_seen[2], chk_named, chk_logged, chk_hz[2][2][REC_KINDS][64]; // [destroy][made][kind][op] void rec_check_handle(int kind, uint64_t handle, int destroy, uint64_t tl, uint64_t gpu) { if (!rec_chk || !handle || kind < 0 || kind >= REC_KINDS || !__atomic_load_n(&rec_live, __ATOMIC_RELAXED)) return; - uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE), p = __atomic_load_n(&part_h0, __ATOMIC_ACQUIRE); + uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE), p = __atomic_load_n(&part_h0, __ATOMIC_ACQUIRE); uint64_t fence = __atomic_load_n(&part_fence, __ATOMIC_ACQUIRE); if (p > h || h - p > RING_WORDS / 2) return; // a part this large is not searched plat_inc32(&chk_seen[destroy != 0]); @@ -317,7 +347,7 @@ void rec_check_handle(int kind, uint64_t handle, int destroy, uint64_t tl, uint6 if (op != R_PAD) for (uint32_t i = 1; i < n; i++) { if (ring[(p + i) % RING_WORDS] != handle) continue; - int made = p < __atomic_load_n(&tail, __ATOMIC_ACQUIRE); + int made = p < __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE); if (!destroy && tl >= fence) { plat_inc32(&chk_named); return; } plat_inc32(&chk_hz[destroy][made][kind][op & 63]); if (plat_inc32(&chk_logged) <= 64) diff --git a/totk.ini.example b/totk.ini.example index 3ef2537b..4a09a50c 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -366,6 +366,14 @@ # the setup's rebuild at start (a few minutes) and starts again after it. off: the game starts with the code it has. # A rebuild that failed or was cancelled is not started again for the same cheats. Default: auto. # mods_rebuild = auto +# Extra room in MiB (0 to 4096) for the heap the world's textures stream into, for HD texture packs. Packs made for +# emulators with 8 GB of DRAM grow it in their own HeapDef table (System/HeapSize/Product.Nin_NX_NVN.HeapDef.byml.zs): +# the port reads that growth at start and applies it itself (about 3339 MiB for the HDPacks 4K pack), so such a pack +# needs nothing here. Set it for a pack that ships textures without a HeapDef. Without the room the game keeps starting +# its panic Blood Moon. The memory pool grows by the same amount; the larger of this and a mod's growth counts. +# Read at start. Default: 0. Such a pack also needs much more video memory: astc_bc = 1 ([performance]) keeps its +# textures at a quarter of it. +# tex_heap_extra_mb = 0 [files] # Developer overlay: a folder with RomFS files and a manifest.tsv (tools/make_romfs_patches.py format). Players put