From a5711539e132612e634e353399c5ee28839ea556 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 12:43:50 +0300 Subject: [PATCH 01/50] Size the code generator's worker pool by free memory so it stops running out of RAM gen.py started one emit worker per CPU, about 1.3 GiB each on 1.4.3, so many-core machines with modest RAM died with a MemoryError during "Translating code to C" (issues 17, 28, probably 66). The default is now min(CPU count, 80% of the free physical memory / 1.5 GiB), at least 1, capped by the memory limit of the enclosing job object if there is one. --jobs still overrides it, and the chosen count and its reason are logged. The memory helper is tools/hostmem.py, shipped with the release tools. Co-Authored-By: Claude Opus 5.5 --- recomp/gen.py | 24 ++++++++++++++++++-- tools/hostmem.py | 52 +++++++++++++++++++++++++++++++++++++++++++ tools/make_release.py | 2 +- 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 tools/hostmem.py diff --git a/recomp/gen.py b/recomp/gen.py index e07396fa..1eaa00f2 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -39,6 +39,7 @@ sys.path.insert(0, str(ROOT / 'tools')) sys.path.insert(0, str(ROOT / 'recomp')) import flags # noqa: E402 +import hostmem # noqa: E402 import jumptables # noqa: E402 import patches # noqa: E402 import regs # noqa: E402 @@ -587,12 +588,29 @@ def remove_stale(moddir, name, nchunks, errors): return removed +# Peak memory of one emit worker on 1.4.3 is 1.3 GiB (1.5 GiB with margin); the parent holds about 0.7 GiB. +WORKER_MEM = 1536 << 20 +MEM_SHARE = 0.8 # of the free memory the workers may take, the rest is for the system and other programs + + +def pick_jobs(requested): + """Worker count and the reason for it: --jobs if given, else the CPU count capped by the free memory.""" + if requested: + return requested, '--jobs' + cpus = os.cpu_count() or 4 + free = hostmem.available_ram() + if free is None: + return cpus, f'{cpus} CPUs, free memory unknown' + by_mem = max(1, int(free * MEM_SHARE) // WORKER_MEM) + return min(cpus, by_mem), f'{cpus} CPUs, {free / 2**30:.1f} GiB free memory allows {by_mem}' + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--exefs', type=Path, required=True) ap.add_argument('--out', type=Path, required=True) ap.add_argument('--modules', default='main,subsdk0,sdk') - ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4) + ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)') ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module') ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most') ap.add_argument('--chunk-bytes', type=int, default=3000000, help='estimated bytes of C per chunk') @@ -651,7 +669,9 @@ def main(): done_at, errors = {}, [] t_emit = time.perf_counter() ctx = mp.get_context('spawn') - with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool: + jobs, why = pick_jobs(args.jobs) + print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True) + with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool: for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)): results[r['module']].append(r) if r['error']: diff --git a/tools/hostmem.py b/tools/hostmem.py new file mode 100644 index 00000000..7916d937 --- /dev/null +++ b/tools/hostmem.py @@ -0,0 +1,52 @@ +"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows).""" +import ctypes +import ctypes.wintypes as wt +import sys + +JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 +LIMIT_PROCESS_MEMORY = 0x100 +LIMIT_JOB_MEMORY = 0x200 + + +class _MemStatus(ctypes.Structure): + _fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \ + [(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')] + + +class _JobLimits(ctypes.Structure): + _fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong), + ('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t), + ('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD), + ('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD), + ('r_ops', ctypes.c_ulonglong), ('w_ops', ctypes.c_ulonglong), ('o_ops', ctypes.c_ulonglong), + ('r_bytes', ctypes.c_ulonglong), ('w_bytes', ctypes.c_ulonglong), ('o_bytes', ctypes.c_ulonglong), + ('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t), + ('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)] + + +def _job_headroom(): + """Bytes left under the memory limit of the job object this process runs in, or None without such a limit.""" + k32 = ctypes.WinDLL('kernel32', use_last_error=True) + k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p] + info = _JobLimits() + # A null handle asks about the job of the calling process. + if not k32.QueryInformationJobObject(None, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info), + ctypes.sizeof(info), None): + return None + if info.LimitFlags & LIMIT_JOB_MEMORY: + return max(0, info.JobMemoryLimit - info.PeakJobMemoryUsed) + if info.LimitFlags & LIMIT_PROCESS_MEMORY: + return info.ProcessMemoryLimit + return None + + +def available_ram(): + """Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows.""" + if sys.platform != 'win32': + return None + status = _MemStatus() + status.dwLength = ctypes.sizeof(status) + if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + return None + headroom = _job_headroom() + return status.avail if headroom is None else min(status.avail, headroom) diff --git a/tools/make_release.py b/tools/make_release.py index 4489c8d8..9fd9d33c 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -97,7 +97,7 @@ LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32' LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. -TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'make_romfs_patches.py', +TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon From 4a961a8b8d86eb3371120f742e19588dd4528fc0 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 12:49:06 +0300 Subject: [PATCH 02/50] Pass the linker map path with -Xlinker so build folders containing a comma link The installer names its temp folder after the game title (Legend of Zelda, The - ...), and clang splits -Wl, values at every comma, so lld received a truncated --Map path and a stray input file. Fixes issue 47. Co-Authored-By: Claude Opus 5.5 --- tools/build_game.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/build_game.py b/tools/build_game.py index 8546c37a..3efcfbea 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -77,7 +77,8 @@ SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden', '-march=x86-64-v3'] RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3'] -LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '"-Wl,--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py +# -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path +LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c) VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator # --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead. From 112fb2dd7423d3c1e7c857befb1ce25d28d99ae4 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 13:21:14 +0300 Subject: [PATCH 03/50] Log the VK_EXT_device_fault report when the device is lost vk_device_fault_report queries vkGetDeviceFaultInfoEXT once per process after VK_ERROR_DEVICE_LOST and logs the description, the faulting addresses with type and precision, and the vendor codes. The submit, upload, occlusion query and wait fatals call it before rt_fatal, and the present path calls it on a failed acquire, blit submit or present. Threads that lose the device together wait until the report is written. The extension counts as usable only when the driver reports the deviceFault feature. No cost unless the device is lost. Co-Authored-By: Claude Opus 5.5 --- runtime/gfx_window.c | 3 +++ runtime/vk.h | 4 +++ runtime/vk_device.c | 59 ++++++++++++++++++++++++++++++++++++++++++ runtime/vk_exec_xfer.c | 5 +++- runtime/vk_memory.c | 5 +++- runtime/vk_sync.c | 15 ++++++++--- 6 files changed, 86 insertions(+), 5 deletions(-) diff --git a/runtime/gfx_window.c b/runtime/gfx_window.c index f62de225..fc449051 100644 --- a/runtime/gfx_window.c +++ b/runtime/gfx_window.c @@ -490,6 +490,7 @@ static void flip_locked(void) { vk_queue_lock(); VkResult r = vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi); vk_queue_unlock(); + vk_device_fault_report(r); if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) InterlockedExchange(&g_dirty, 1); } @@ -763,6 +764,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w if (!create_swapchain()) goto out; r = vkAcquireNextImageKHR(c->device, g_sc, 1000000000ull, g_acquire[slot], VK_NULL_HANDLE, &idx); } + vk_device_fault_report(r); if (r != VK_SUCCESS && r != VK_SUBOPTIMAL_KHR) goto out; // lost, not ready: skip the frame if (r == VK_SUBOPTIMAL_KHR) InterlockedExchange(&g_dirty, 1); vkResetFences(c->device, 1, &g_fence[slot]); @@ -870,6 +872,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w g_flip_idx = idx; if (!defer) flip_locked(); } else { + vk_device_fault_report(r); rt_log("gfx: present submit failed (%d)", (int)r); } g_frame++; diff --git a/runtime/vk.h b/runtime/vk.h index 961f2f35..f6302678 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -69,6 +69,10 @@ VkCtx *vk_ctx(void); // NULL until vk_init() succeeded int vk_init(void); // idempotent; 1 on success, 0 on failure (reason logged with rt_log) void vk_shutdown(void); int vk_selftest(void); // 0 on success +// Call with the result of a failed queue submit, present or wait before stopping on it: VK_ERROR_DEVICE_LOST logs +// what VK_EXT_device_fault reports (description, faulting addresses with their precision, vendor codes) once per +// process. Any other result returns at once. +void vk_device_fault_report(VkResult r); // ---- window and present (gfx_window.c). Host threads only. // vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 01573a38..26881a68 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -267,6 +267,7 @@ static int create_device(void) { else if (!h->geometryShader) miss = "geometryShader"; else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } + if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { @@ -364,6 +365,64 @@ void vk_shutdown(void) { VkCtx *vk_ctx(void) { return g_ready ? &g_vk : NULL; } +static const char *fault_address_type(VkDeviceFaultAddressTypeEXT t) { + switch (t) { + case VK_DEVICE_FAULT_ADDRESS_TYPE_NONE_EXT: return "none"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_READ_INVALID_EXT: return "read invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_WRITE_INVALID_EXT: return "write invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_EXECUTE_INVALID_EXT: return "execute invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_UNKNOWN_EXT: return "instruction pointer (unknown)"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_INVALID_EXT: return "instruction pointer (invalid)"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_FAULT_EXT: return "instruction pointer (fault)"; + default: return "unknown type"; + } +} + +// The query runs once per process. Threads that hit the loss at the same time wait on the lock until it is logged, so +// none of them reaches its rt_fatal and ends the process before the report is written. +void vk_device_fault_report(VkResult r) { + static SRWLOCK lock = SRWLOCK_INIT; + static int done; + if (r != VK_ERROR_DEVICE_LOST) return; + AcquireSRWLockExclusive(&lock); + if (done) { ReleaseSRWLockExclusive(&lock); return; } + done = 1; + VkCtx *c = vk_ctx(); + PFN_vkGetDeviceFaultInfoEXT get = + c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") + : NULL; + if (!get) { + rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point" + : "not enabled"); + ReleaseSRWLockExclusive(&lock); + return; + } + VkDeviceFaultCountsEXT n = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT}; + VkResult q = get(c->device, &n, NULL); + VkDeviceFaultAddressInfoEXT *ai = calloc(n.addressInfoCount + 1, sizeof *ai); + VkDeviceFaultVendorInfoEXT *vi = calloc(n.vendorInfoCount + 1, sizeof *vi); + VkDeviceFaultInfoEXT info = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT, .pAddressInfos = ai, .pVendorInfos = vi}; + uint64_t binary = n.vendorBinarySize; + n.vendorBinarySize = 0; // the vendor binary is not fetched; its size alone is logged + if (q == VK_SUCCESS && ai && vi) q = get(c->device, &n, &info); + if (q != VK_SUCCESS && q != VK_INCOMPLETE) { + rt_log("vk: device lost; vkGetDeviceFaultInfoEXT failed (%d)", (int)q); + } else { + rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description, + n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary); + // The faulting access lies in the precision-aligned range around the reported address. + for (uint32_t i = 0; i < n.addressInfoCount; i++) + rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType), + (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision); + for (uint32_t i = 0; i < n.vendorInfoCount; i++) + rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description, + (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData); + } + free(ai); + free(vi); + ReleaseSRWLockExclusive(&lock); +} + int vk_selftest(void) { if (!vk_init()) return 1; VkCtx *c = &g_vk; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 34548153..a6685bfd 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -211,7 +211,10 @@ static void oq_cb(void *arg) { // completion thread, and with it every later callback, for good. It counts as visible (one sample). VkResult r = vkGetQueryPoolResults(dev, oq_pool, s->q[i], j - i, sizeof v, v, 2 * sizeof v[0], VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); - if (r != VK_SUCCESS && r != VK_NOT_READY) rt_fatal("vk: occlusion query results failed (%d)", (int)r); + if (r != VK_SUCCESS && r != VK_NOT_READY) { + vk_device_fault_report(r); + rt_fatal("vk: occlusion query results failed (%d)", (int)r); + } vkResetQueryPool(dev, oq_pool, s->q[i], j - i); for (uint32_t k = i; k < j; k++) { if (v[2 * (k - i) + 1]) sum += v[2 * (k - i)]; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index cde3b60c..7d8192a6 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1722,7 +1722,10 @@ uint64_t vk_upload_flush(int wait) { vk_queue_lock(); VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: upload submit failed (%d)", (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: upload submit failed (%d)", (int)r); + } if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight); inflight[ninflight++] = open_b; open_active = 0; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index f89ac1a7..2dd32c58 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -186,7 +186,10 @@ static DWORD WINAPI completion_main(void *arg) { VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, .pSemaphores = &c->timeline, .pValues = &want}; VkResult r = vkWaitSemaphores(c->device, &wi, 100000000ull); // 100 ms: stop is checked between waits - if (r != VK_SUCCESS && r != VK_TIMEOUT) rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); + if (r != VK_SUCCESS && r != VK_TIMEOUT) { + vk_device_fault_report(r); + rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); + } vkGetSemaphoreCounterValue(c->device, c->timeline, &reached); AcquireSRWLockExclusive(&st_lock); if (reached > issued) reached = issued; @@ -302,7 +305,10 @@ VkCommandBuffer vk_sync_split(void) { vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); + } vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, @@ -360,7 +366,10 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); + } s->fence = recorded ? fence : 0; vk_mem_note_submit(fence); AcquireSRWLockExclusive(&st_lock); From 67116a387fbd19948c46f9f68add98ecc39d9b39 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 13:27:19 +0300 Subject: [PATCH 04/50] Map the zh-CN and zh-TW game languages to zh-Hans and zh-Hant Choosing Simplified Chinese in the installer wrote lang = zh-CN to totk.ini and the game showed Japanese: ov_language() handed the code to nn::oe::GetDesiredLanguage unchanged, and TotK only accepts zh-Hans and zh-Hant for Chinese. Reported in #72 and #68. ov_language() now maps both old spellings, so existing totk.ini files keep working and cfg.c still accepts them. The F1 menu offers one row per language under the canonical codes. The installer lists only zh-Hans and zh-Hant, writes them, and reads zh-CN or zh-TW from the CLI or from a recorded totk-install.json as the canonical code, so a repair or update of an old install does not fail with check.lang_unknown and rewrites the ini. --probe-services prints the configured language and checks the four Chinese codes. Co-Authored-By: Claude Opus 5.5 --- installer/gui.py | 5 +++-- installer/ini.py | 16 ++++++++++++---- installer/steps.py | 2 +- installer/text/en.json | 2 -- installer/text/ru.json | 2 -- runtime/ov_probe.c | 4 +++- runtime/ov_system.c | 10 +++++++--- runtime/ui/menu.cpp | 2 +- tests/installer_test.py | 10 ++++++++++ 9 files changed, 37 insertions(+), 16 deletions(-) diff --git a/installer/gui.py b/installer/gui.py index 391918de..bbda6a2d 100644 --- a/installer/gui.py +++ b/installer/gui.py @@ -266,8 +266,9 @@ class App: state = form.installed_state() if state is not None: self.update_hint.configure(text=self.cat('ui.update_mode')) - if state.get('lang') in ini.LANGUAGES and not self.lang_touched: - self.vars['lang'].set(self.lang_label(state['lang'])) + recorded = ini.canonical_language(state.get('lang')) + if recorded in ini.LANGUAGES and not self.lang_touched: + self.vars['lang'].set(self.lang_label(recorded)) else: self.update_hint.configure(text='') self.fit() diff --git a/installer/ini.py b/installer/ini.py index 6bcc341f..13147862 100644 --- a/installer/ini.py +++ b/installer/ini.py @@ -9,19 +9,27 @@ import os import re from pathlib import Path -# runtime/ov_system.c: the codes TOTK_LANG accepts. +# runtime/ov_system.c: the codes TOTK_LANG offers. Each language appears once, under the code the game accepts. LANGUAGES = ('ja', 'en-US', 'en-GB', 'fr', 'fr-CA', 'de', 'it', 'es', 'es-419', 'nl', 'pt', 'pt-BR', 'ru', 'ko', - 'zh-CN', 'zh-TW', 'zh-Hans', 'zh-Hant') + 'zh-Hans', 'zh-Hant') +# Older installer builds wrote these two codes; they are read as the canonical ones and rewritten on the next install. +ALIASES = {'zh-CN': 'zh-Hans', 'zh-TW': 'zh-Hant'} # Their text falls back to the Switch shared fonts (runtime/ov_system.c), which the installer does not extract. -NEED_SHARED_FONTS = ('ko', 'zh-CN', 'zh-TW', 'zh-Hans', 'zh-Hant') +NEED_SHARED_FONTS = ('ko', 'zh-Hans', 'zh-Hant') ACTIVE = re.compile(r'^(\s*)(?:totk_)?lang(\s*=\s*)(.*?)(\s*)$', re.IGNORECASE) COMMENTED = re.compile(r'^\s*[#;]\s*(?:totk_)?lang\s*=.*$', re.IGNORECASE) SECTION = re.compile(r'^\s*\[(.*)\]\s*$') +def canonical_language(lang): + """The code LANGUAGES lists for lang: an alias of an older installer build is mapped, anything else is kept.""" + return ALIASES.get(lang, lang) + + def set_language(text, lang): - """text with `lang = `; see the module docstring.""" + """text with `lang = ` (an alias written as its canonical code); see the module docstring.""" + lang = canonical_language(lang) lines = text.splitlines() for i, line in enumerate(lines): m = ACTIVE.match(line) diff --git a/installer/steps.py b/installer/steps.py index c7d9c40a..3984f784 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -203,7 +203,7 @@ class Check(Step): raise InstallError('check.dest_not_folder', path=str(dest)) if ctx.mode == 'install' and dest.is_dir() and any(dest.iterdir()): raise InstallError('check.dest_not_empty', path=str(dest)) - lang = ctx.options.lang or ctx.state.get('lang') + lang = ini.canonical_language(ctx.options.lang or ctx.state.get('lang')) if not lang: raise InstallError('check.lang_missing') if lang not in ini.LANGUAGES: diff --git a/installer/text/en.json b/installer/text/en.json index 92c24197..5964a0f6 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -166,8 +166,6 @@ "lang.pt-BR": "Portuguese (Brazil)", "lang.ru": "Russian", "lang.ko": "Korean", - "lang.zh-CN": "Chinese (Simplified, China)", - "lang.zh-TW": "Chinese (Traditional, Taiwan)", "lang.zh-Hans": "Chinese (Simplified)", "lang.zh-Hant": "Chinese (Traditional)" } diff --git a/installer/text/ru.json b/installer/text/ru.json index ad3e9666..c3a87db8 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -166,8 +166,6 @@ "lang.pt-BR": "Португальский (Бразилия)", "lang.ru": "Русский", "lang.ko": "Корейский", - "lang.zh-CN": "Китайский (упрощённый, КНР)", - "lang.zh-TW": "Китайский (традиционный, Тайвань)", "lang.zh-Hans": "Китайский (упрощённый)", "lang.zh-Hant": "Китайский (традиционный)" } diff --git a/runtime/ov_probe.c b/runtime/ov_probe.c index 487ca5da..4668bf0a 100644 --- a/runtime/ov_probe.c +++ b/runtime/ov_probe.c @@ -250,8 +250,10 @@ static void probe_oe(void) { {NULL, "en-US", "en-US"}, {NULL, "be-BY", "en-US"}, {NULL, "xx", "en-US"}, {NULL, "ja-JP", "ja"}, {NULL, "fr-CA", "fr-CA"}, {NULL, "es-MX", "es-419"}, {NULL, "es_ES", "es"}, {NULL, "zh-Hant-TW", "zh-Hant"}, {NULL, "zh-CN", "zh-Hans"}, {NULL, "en-GB", "en-GB"}, {NULL, "ru", "ru"}, {"ja", "fr-FR", "ja"}, - {"en-US", NULL, "en-US"}}; + {"en-US", NULL, "en-US"}, {"zh-CN", NULL, "zh-Hans"}, {"zh-TW", NULL, "zh-Hant"}, + {"zh-Hans", NULL, "zh-Hans"}, {"zh-Hant", NULL, "zh-Hant"}}; int bad = 0; + printf("oe: configured language (TOTK_LANG=%s) is %s\n", keep_lang ? keep_lang : "", lang()); for (size_t i = 0; i < sizeof cases / sizeof *cases; i++) { set_env("TOTK_LANG", cases[i].lang); set_env("TOTK_LOCALE", cases[i].locale); diff --git a/runtime/ov_system.c b/runtime/ov_system.c index d3d98878..c3b5c6f7 100644 --- a/runtime/ov_system.c +++ b/runtime/ov_system.c @@ -88,12 +88,16 @@ static const char *switch_language(const char *tag) { return "en-US"; } -// The game's language as a Switch language code (TOTK_LANG, else TOTK_LOCALE, else the Windows user locale). +// The game's language as a Switch language code (TOTK_LANG, else TOTK_LOCALE, else the Windows user locale). TotK +// accepts "zh-Hans" and "zh-Hant" for Chinese and falls back to Japanese for "zh-CN" and "zh-TW", so those two +// spellings, which older totk.ini files and menus wrote, are mapped to them. const char *ov_language(void) { - static const char *const codes[] = {"ja", "en-US", "fr", "de", "it", "es", "zh-CN", "ko", "nl", "pt", "ru", - "zh-TW", "en-GB", "fr-CA", "es-419", "zh-Hans", "zh-Hant", "pt-BR"}; + static const char *const codes[] = {"ja", "en-US", "fr", "de", "it", "es", "ko", "nl", "pt", "ru", + "en-GB", "fr-CA", "es-419", "zh-Hans", "zh-Hant", "pt-BR"}; const char *code = cfg_get("TOTK_LANG"); // no fixed default: TOTK_LOCALE or the Windows user locale decides when unset if (code && *code) { + if (!strcmp(code, "zh-CN")) return "zh-Hans"; + if (!strcmp(code, "zh-TW")) return "zh-Hant"; for (size_t i = 0; i < sizeof codes / sizeof *codes; i++) if (!strcmp(code, codes[i])) return codes[i]; rt_fatal("TOTK_LANG '%s' is not a Switch language code", code); diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 64c3c4c0..ccb1bbb0 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -458,7 +458,7 @@ static void row_game_lang(void) { {"en-US", UI_MENU_LANGNAME_EN}, {"ja", UI_MENU_LANGNAME_JA}, {"fr", UI_MENU_LANGNAME_FR}, {"de", UI_MENU_LANGNAME_DE}, {"it", UI_MENU_LANGNAME_IT}, {"es", UI_MENU_LANGNAME_ES}, {"nl", UI_MENU_LANGNAME_NL}, {"ru", UI_MENU_LANGNAME_RU}, - {"zh-CN", UI_MENU_LANGNAME_ZH_HANS}, {"zh-TW", UI_MENU_LANGNAME_ZH_HANT}, {"ko", UI_MENU_LANGNAME_KO}, + {"zh-Hans", UI_MENU_LANGNAME_ZH_HANS}, {"zh-Hant", UI_MENU_LANGNAME_ZH_HANT}, {"ko", UI_MENU_LANGNAME_KO}, }; enum { N = sizeof langs / sizeof *langs + 1 }; const char *vals[N], *names[N]; diff --git a/tests/installer_test.py b/tests/installer_test.py index d9d325e4..d7404929 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -217,6 +217,16 @@ class IniTest(unittest.TestCase): self.assertEqual(ini.set_language('[language]\nlocale = de-DE\n', 'fr'), '[language]\nlang = fr\nlocale = de-DE\n') + def test_old_chinese_codes_are_written_as_the_canonical_ones(self): + self.assertEqual(ini.canonical_language('zh-CN'), 'zh-Hans') + self.assertEqual(ini.canonical_language('zh-TW'), 'zh-Hant') + self.assertEqual(ini.canonical_language('ja'), 'ja') + self.assertNotIn('zh-CN', ini.LANGUAGES) + self.assertNotIn('zh-TW', ini.LANGUAGES) + out = ini.set_language(self.EXAMPLE, 'zh-CN') + self.assertEqual(ini.get_language(out), 'zh-Hans') + self.assertEqual(ini.get_language(ini.set_language(out, 'zh-TW')), 'zh-Hant') + def test_write_keeps_user_lines(self): with tempfile.TemporaryDirectory() as d: d = Path(d) From d8547a84acc7b66fccef0705cdf0393e86cd9abb Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 13:30:09 +0300 Subject: [PATCH 05/50] Offer 90 FPS in the frame rate limit menu GitHub issue #21 asks for a 90 fps limit. fps_limit = 90 in totk.ini already worked, because frame_rate.c accepts any whole rate from 30 to 360 and cfg.c does not validate the values column. The menu row and the documented value lists did not name it. Added 90 to the menu between 60 and 120 (new catalog id UI_MENU_FPS_90 in both languages), to the FPS_LIMIT row in cfg.c and to totk.ini.example. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 +- runtime/frame_rate.c | 2 +- runtime/ui/menu.cpp | 5 +++-- runtime/ui_text.c | 4 ++-- runtime/ui_text.h | 3 ++- totk.ini.example | 2 +- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index 84abed08..f355b982 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -94,7 +94,7 @@ static const CfgMeta metas[] = { {"OUTPUT_RES", "1920x1080", NULL, RESTART}, {"RENDER_SCALE", "game", "game|50|67|75|100|125|150|200", RESTART}, {"UPSCALER", "fsr", "fsr|bilinear", LIVE}, {"SHARPNESS", "80", "0..100", LIVE}, // Game - {"FPS_LIMIT", "60", "30|60|120|144|display", LIVE}, {"FPS60", "1", "0|1", LIVE}, + {"FPS_LIMIT", "60", "30|60|90|120|144|display", LIVE}, {"FPS60", "1", "0|1", LIVE}, {"REALTIME", "1", "0|1", RESTART}, {"CLOTH_REALTIME", "1", "0|1", RESTART}, {"HIGHFPS_FIX", "1", "0|1", RESTART}, {"LANG", "", "ja|en-US|fr|de|it|es|zh-CN|ko|nl|pt|ru|zh-TW|en-GB|fr-CA|es-419|zh-Hans|zh-Hant|pt-BR", RESTART}, diff --git a/runtime/frame_rate.c b/runtime/frame_rate.c index bb360bf8..7bb5c60e 100644 --- a/runtime/frame_rate.c +++ b/runtime/frame_rate.c @@ -90,7 +90,7 @@ void frame_step_init(void) { int frame_step_realtime(void) { return step_word != NULL; } -// The frame limit (menu, TOTK_FPS_LIMIT, live): 30, 60, 120, 144, any whole rate from 30 to 360, or display (the +// The frame limit (menu, TOTK_FPS_LIMIT, live): 30, 60, 90, 120, 144, any whole rate from 30 to 360, or display (the // refresh rate of the window's monitor). Unset, the older TOTK_FPS60=0 still means 30. A limit of 30 switches the set // fps60 off (the shipping minimum interval 2), every other limit switches it on; the controller reads its minimum // interval from the patched sites each frame. With the real-time step the pacer holds frames to the limit; above 60 diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index ccb1bbb0..803c4810 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -476,8 +476,9 @@ static void row_game_lang(void) { // The frame limit (frame_rate.c): unset, the older fps60 = 0 still means 30. The monitor choice shows the rate it gives. static void row_fps_limit() { - static const char *const vals[] = {"30", "60", "120", "144", "display"}; - static const UiTextId ids[] = {UI_MENU_FPS_30, UI_MENU_FPS_60, UI_MENU_FPS_120, UI_MENU_FPS_144, UI_MENU_FPS_DISPLAY}; + static const char *const vals[] = {"30", "60", "90", "120", "144", "display"}; + static const UiTextId ids[] = {UI_MENU_FPS_30, UI_MENU_FPS_60, UI_MENU_FPS_90, UI_MENU_FPS_120, UI_MENU_FPS_144, + UI_MENU_FPS_DISPLAY}; enum { N = sizeof vals / sizeof *vals }; const char *names[N]; for (int i = 0; i < N; i++) names[i] = menu_text(ids[i]); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index f93107ae..acdd619c 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -67,7 +67,7 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_FPS_LIMIT] = "Frame rate limit", [UI_MENU_FPS_LIMIT_DESC] = "The most frames per second the game draws. Above 60 is experimental and needs Game speed " "follows the clock.", - [UI_MENU_FPS_30] = "30 FPS", [UI_MENU_FPS_60] = "60 FPS", [UI_MENU_FPS_120] = "120 FPS", [UI_MENU_FPS_144] = "144 FPS", + [UI_MENU_FPS_30] = "30 FPS", [UI_MENU_FPS_60] = "60 FPS", [UI_MENU_FPS_90] = "90 FPS", [UI_MENU_FPS_120] = "120 FPS", [UI_MENU_FPS_144] = "144 FPS", // the monitor's refresh rate; printf format with the rate in Hz when it is known [UI_MENU_FPS_DISPLAY] = "Monitor rate", [UI_MENU_FPS_DISPLAY_HZ] = "Monitor rate (%.0f Hz)", [UI_MENU_REALTIME] = "Game speed follows the clock", @@ -285,7 +285,7 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_FPS_LIMIT] = "Ограничение частоты кадров", [UI_MENU_FPS_LIMIT_DESC] = "Наибольшее число кадров в секунду. Выше 60 — экспериментальный режим, нужна «Скорость " "игры по часам».", - [UI_MENU_FPS_30] = "30 FPS", [UI_MENU_FPS_60] = "60 FPS", [UI_MENU_FPS_120] = "120 FPS", [UI_MENU_FPS_144] = "144 FPS", + [UI_MENU_FPS_30] = "30 FPS", [UI_MENU_FPS_60] = "60 FPS", [UI_MENU_FPS_90] = "90 FPS", [UI_MENU_FPS_120] = "120 FPS", [UI_MENU_FPS_144] = "144 FPS", [UI_MENU_FPS_DISPLAY] = "Как у монитора", [UI_MENU_FPS_DISPLAY_HZ] = "Как у монитора (%.0f Гц)", [UI_MENU_REALTIME] = "Скорость игры по часам", [UI_MENU_REALTIME_DESC] = "Игра идёт по реальному времени, медленный кадр её не замедляет.", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 6eb805cf..1ea60ba1 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -38,7 +38,8 @@ typedef enum { UI_MENU_SHARPNESS, UI_MENU_SHARPNESS_DESC, // Game - UI_MENU_FPS_LIMIT, UI_MENU_FPS_LIMIT_DESC, UI_MENU_FPS_30, UI_MENU_FPS_60, UI_MENU_FPS_120, UI_MENU_FPS_144, + UI_MENU_FPS_LIMIT, UI_MENU_FPS_LIMIT_DESC, UI_MENU_FPS_30, UI_MENU_FPS_60, UI_MENU_FPS_90, UI_MENU_FPS_120, + UI_MENU_FPS_144, UI_MENU_FPS_DISPLAY, UI_MENU_FPS_DISPLAY_HZ, UI_MENU_REALTIME, UI_MENU_REALTIME_DESC, UI_MENU_CLOTH_REALTIME, UI_MENU_CLOTH_REALTIME_DESC, diff --git a/totk.ini.example b/totk.ini.example index f8213a99..d55d52e9 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -3,7 +3,7 @@ # file. Keys are case-insensitive; '#' and ';' start a comment; [sections] only group lines. [frame rate] -# Frame rate limit: 30, 60, 120, 144, any whole rate from 30 to 360, or display (the refresh rate of the window's +# Frame rate limit: 30, 60, 90, 120, 144, any whole rate from 30 to 360, or display (the refresh rate of the window's # monitor). Above 60 is experimental and needs realtime = 1. Changes while the game runs take effect at once. # fps_limit = 60 # Older key, read only while fps_limit is unset: fps60 = 0 means a limit of 30. From fcaecabad43562f2ed9df408a94269b95fe94e08 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 13:40:04 +0300 Subject: [PATCH 06/50] Make totk.exe per-monitor DPI aware so fullscreen and the window use physical pixels The manifest declared no DPI awareness, so Windows ran the process DPI-unaware: on a 4K monitor at 150% scale the fullscreen window and its swapchain were 2560x1440, the 4K game image was shrunk to it and Windows stretched the result back up, blurred (issues #25, #56, #31). The manifest now declares PerMonitorV2 (true/pm for older Windows 10). The window frame is measured at the window's DPI, a framed window keeps its client size when it moves to a monitor at another scale, WM_DPICHANGED takes the suggested rect or keeps fullscreen over the monitor, and window_size means screen pixels. The menu's minimum text size follows the window's DPI, and the text entry dialog is measured at the game window's DPI. --probe-dpi prints the process awareness and fails unless it is per-monitor v2. Co-Authored-By: Claude Opus 5.5 --- runtime/dpi_probe.c | 33 ++++++++++++++++++ runtime/gfx_window.c | 79 +++++++++++++++++++++++++++++++++---------- runtime/host.h | 1 + runtime/main.c | 4 ++- runtime/swkbd_win.c | 9 +++-- runtime/totk.manifest | 8 ++++- runtime/ui/menu.cpp | 9 +++-- runtime/ui/menu.h | 1 + totk.ini.example | 3 +- 9 files changed, 121 insertions(+), 26 deletions(-) create mode 100644 runtime/dpi_probe.c diff --git a/runtime/dpi_probe.c b/runtime/dpi_probe.c new file mode 100644 index 00000000..a2ddcf66 --- /dev/null +++ b/runtime/dpi_probe.c @@ -0,0 +1,33 @@ +// --probe-dpi: the process DPI awareness that totk.manifest declares, and what the primary monitor reports under it. +// A DPI-unaware process gets every window and monitor size scaled down by the display scale and its output stretched +// back up by Windows; the probe fails unless the process is per-monitor aware (v2). Needs no game files and no GPU. +#include +#include + +#include "host.h" + +static const char *awareness_name(DPI_AWARENESS_CONTEXT ctx) { + if (AreDpiAwarenessContextsEqual(ctx, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)) return "per-monitor aware v2"; + switch (GetAwarenessFromDpiAwarenessContext(ctx)) { + case DPI_AWARENESS_UNAWARE: return "unaware"; + case DPI_AWARENESS_SYSTEM_AWARE: return "system aware"; + case DPI_AWARENESS_PER_MONITOR_AWARE: return "per-monitor aware v1"; + default: return "invalid"; + } +} + +int probe_dpi(void) { + DPI_AWARENESS_CONTEXT ctx = GetThreadDpiAwarenessContext(); + MONITORINFO mi = {sizeof mi}; + HMONITOR mon = MonitorFromPoint((POINT){0, 0}, MONITOR_DEFAULTTOPRIMARY); + int have = GetMonitorInfoW(mon, &mi); + printf("dpi: awareness %s\n", awareness_name(ctx)); + printf("dpi: system dpi %u (scale %u%%)\n", GetDpiForSystem(), GetDpiForSystem() * 100 / 96); + if (have) + printf("dpi: primary monitor %ldx%ld at %ld,%ld; screen metrics %dx%d\n", mi.rcMonitor.right - mi.rcMonitor.left, + mi.rcMonitor.bottom - mi.rcMonitor.top, mi.rcMonitor.left, mi.rcMonitor.top, GetSystemMetrics(SM_CXSCREEN), + GetSystemMetrics(SM_CYSCREEN)); + int ok = AreDpiAwarenessContextsEqual(ctx, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) && have; + printf("probe-dpi: %s\n", ok ? "PASS" : "FAIL (the process is not per-monitor aware v2)"); + return ok ? 0 : 1; +} diff --git a/runtime/gfx_window.c b/runtime/gfx_window.c index f62de225..5d4bd485 100644 --- a/runtime/gfx_window.c +++ b/runtime/gfx_window.c @@ -12,6 +12,10 @@ // The in-game menu (ui/menu.h) draws over the game image in the present's command buffer, after the blit; its keyboard // and mouse come from the window procedure (ui/menu_win.cpp). The window's size and fullscreen state are the settings // TOTK_WINDOW_SIZE and TOTK_FULLSCREEN, applied at start and whenever a cfg_set changes them (window_settings_live). +// +// The process is per-monitor DPI aware (totk.manifest): every size here is physical pixels, TOTK_WINDOW_SIZE included, +// and the swapchain matches the screen pixel for pixel. A window moved to a monitor at another display scale keeps its +// client size (WM_GETDPISCALEDSIZE); the menu takes its text size floor from the window's DPI (gfx_window_dpi). #include #include #include @@ -48,6 +52,7 @@ static CRITICAL_SECTION g_start_lock; // gfx_window_start / vsync start static int g_started, g_wnd_failed; static int g_background; // TOTK_BACKGROUND_WINDOW: the window stays behind the user's windows static int g_fs_pending; // window thread: TOTK_FULLSCREEN waits for the first activation +static volatile LONG g_dpi = USER_DEFAULT_SCREEN_DPI; // the window's DPI, set by the window thread // ---- swapchain and present state (g_pl) static CRITICAL_SECTION g_pl; @@ -189,6 +194,18 @@ static WINDOWPLACEMENT g_saved_place = {sizeof g_saved_place}; // the window b static int is_fullscreen(HWND h) { return !(GetWindowLongW(h, GWL_STYLE) & WS_OVERLAPPEDWINDOW); } +unsigned gfx_window_dpi(void) { return (unsigned)InterlockedCompareExchange(&g_dpi, 0, 0); } + +// The outer size of a framed window whose client area is cw x ch. The frame is measured at the window's DPI, or the +// system DPI before the window exists: plain AdjustWindowRect always takes the system DPI, which is wrong on a monitor +// at another display scale. +static void outer_size(HWND h, int cw, int ch, int *ow, int *oh) { + RECT r = {0, 0, cw, ch}; + AdjustWindowRectExForDpi(&r, WND_STYLE, FALSE, 0, h ? GetDpiForWindow(h) : GetDpiForSystem()); + *ow = r.right - r.left; + *oh = r.bottom - r.top; +} + // TOTK_FULLSCREEN=1: borderless window over the monitor. TOTK_WINDOW_SIZE=x: client area of the window (the // swapchain follows the window; the game image is scaled to it). static int cfg_fullscreen(void) { @@ -242,20 +259,18 @@ static void apply_settings(HWND h) { int size_changed = strcmp(cfg_value("TOTK_WINDOW_SIZE"), g_applied_size) != 0, fs = cfg_fullscreen(), fs_changed = fs != g_applied_fs; note_applied(); if (size_changed) { - int cw, ch; + int cw, ch, ow, oh; cfg_window_size(&cw, &ch); - RECT r = {0, 0, cw, ch}; - AdjustWindowRect(&r, WND_STYLE, FALSE); + outer_size(h, cw, ch, &ow, &oh); if (is_fullscreen(h)) { RECT *n = &g_saved_place.rcNormalPosition; - n->right = n->left + (r.right - r.left); - n->bottom = n->top + (r.bottom - r.top); + n->right = n->left + ow; + n->bottom = n->top + oh; } else { RECT have; if (GetClientRect(h, &have) && (have.right != cw || have.bottom != ch)) { if (IsZoomed(h)) ShowWindow(h, SW_RESTORE); - SetWindowPos(h, NULL, 0, 0, r.right - r.left, r.bottom - r.top, - SWP_NOMOVE | SWP_NOZORDER | SWP_NOACTIVATE); + SetWindowPos(h, NULL, 0, 0, ow, oh, SWP_NOMOVE | SWP_NOZORDER | SWP_NOACTIVATE); } } } @@ -381,6 +396,32 @@ static LRESULT CALLBACK wnd_proc(HWND h, UINT m, WPARAM w, LPARAM l) { case WM_DISPLAYCHANGE: refresh_rate_update(h, 1); break; + case WM_GETDPISCALEDSIZE: // a framed window keeps its client size in pixels on a monitor at another scale + if (!is_fullscreen(h) && !IsZoomed(h)) { + RECT c, r; + if (GetClientRect(h, &c)) { + r = (RECT){0, 0, c.right, c.bottom}; + if (AdjustWindowRectExForDpi(&r, WND_STYLE, FALSE, 0, LOWORD(w))) { + *(SIZE *)l = (SIZE){r.right - r.left, r.bottom - r.top}; + return TRUE; + } + } + } + break; // DefWindowProc: Windows scales the size with the DPI + case WM_DPICHANGED: { // display scale changed, or the window reached a monitor at another scale + const RECT *s = (const RECT *)l; + InterlockedExchange(&g_dpi, LOWORD(w)); + rt_log("gfx: window DPI %u (%u%%)", LOWORD(w), LOWORD(w) * 100u / USER_DEFAULT_SCREEN_DPI); + if (is_fullscreen(h)) { // keep covering the monitor the suggested rect lies on + MONITORINFO mi = {sizeof mi}; + if (GetMonitorInfoW(MonitorFromRect(s, MONITOR_DEFAULTTONEAREST), &mi)) + SetWindowPos(h, NULL, mi.rcMonitor.left, mi.rcMonitor.top, mi.rcMonitor.right - mi.rcMonitor.left, + mi.rcMonitor.bottom - mi.rcMonitor.top, SWP_NOZORDER | SWP_NOACTIVATE); + } else { + SetWindowPos(h, NULL, s->left, s->top, s->right - s->left, s->bottom - s->top, SWP_NOZORDER | SWP_NOACTIVATE); + } + return 0; + } case WM_ACTIVATEAPP: { int in = w != 0; InterlockedExchange(&g_focused, in); @@ -417,23 +458,25 @@ static DWORD WINAPI window_main(void *arg) { GetSystemMetrics(SM_CYSMICON), LR_SHARED), .hCursor = LoadCursorW(NULL, (LPCWSTR)IDC_ARROW), .lpszClassName = L"TotKRecompWindow"}; RegisterClassExW(&wc); - int cw, ch; + int cw, ch, ow, oh; cfg_window_size(&cw, &ch); - RECT r = {0, 0, cw, ch}; - AdjustWindowRect(&r, WND_STYLE, FALSE); + outer_size(NULL, cw, ch, &ow, &oh); // The window opens behind the user's windows without taking focus: it shows the game's blank loading frames for a // while, and must not cover what the player works on meanwhile. The game still counts itself in focus until the // first WM_ACTIVATEAPP, so loading goes on, and keyboard input waits for the foreground (input_win.c). Its taskbar // button flashes until it is brought up, and TOTK_FULLSCREEN waits for that first activation. // TOTK_BACKGROUND_WINDOW=1 (test runs): no flashing, never fullscreen at start, and F11 keeps the window's place. int background = g_background = cfg_get("TOTK_BACKGROUND_WINDOW") && atoi(cfg_get("TOTK_BACKGROUND_WINDOW")); - g_hwnd = CreateWindowExW(0, wc.lpszClassName, L"TotK Recomp", WND_STYLE, CW_USEDEFAULT, CW_USEDEFAULT, - r.right - r.left, r.bottom - r.top, NULL, NULL, inst, NULL); + g_hwnd = CreateWindowExW(0, wc.lpszClassName, L"TotK Recomp", WND_STYLE, CW_USEDEFAULT, CW_USEDEFAULT, ow, oh, NULL, + NULL, inst, NULL); if (!g_hwnd) { rt_log("gfx: CreateWindowEx failed (%lu)", GetLastError()); g_wnd_failed = 1; } else { - SetWindowPos(g_hwnd, HWND_BOTTOM, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW); + // The frame was measured at the system DPI; the window may have opened on a monitor at another scale. + InterlockedExchange(&g_dpi, (LONG)GetDpiForWindow(g_hwnd)); + outer_size(g_hwnd, cw, ch, &ow, &oh); + SetWindowPos(g_hwnd, HWND_BOTTOM, 0, 0, ow, oh, SWP_NOMOVE | SWP_NOACTIVATE | SWP_SHOWWINDOW); if (!background) { FLASHWINFO fw = {.cbSize = sizeof fw, .hwnd = g_hwnd, .dwFlags = FLASHW_TRAY | FLASHW_TIMERNOFG}; FlashWindowEx(&fw); @@ -901,9 +944,9 @@ void vk_present_flip(void) { void gfx_window_resize(int cw, int chh) { gfx_window_start(); - RECT r = {0, 0, cw, chh}; - AdjustWindowRect(&r, WND_STYLE, FALSE); - SetWindowPos(g_hwnd, NULL, 0, 0, r.right - r.left, r.bottom - r.top, SWP_NOMOVE | SWP_NOZORDER | SWP_NOACTIVATE); + int ow, oh; + outer_size(g_hwnd, cw, chh, &ow, &oh); + SetWindowPos(g_hwnd, NULL, 0, 0, ow, oh, SWP_NOMOVE | SWP_NOZORDER | SWP_NOACTIVATE); EnterCriticalSection(&g_pl); create_swapchain(); LeaveCriticalSection(&g_pl); @@ -979,8 +1022,8 @@ void gfx_window_start(void) { LeaveCriticalSection(&g_pl); RECT rc = {0}; GetClientRect(g_hwnd, &rc); - rt_log("gfx: window %ldx%ld, swapchain %ux%u format %d", rc.right, rc.bottom, g_sc_ext.width, g_sc_ext.height, - (int)g_sc_fmt); + rt_log("gfx: window %ldx%ld at DPI %u, swapchain %ux%u format %d", rc.right, rc.bottom, gfx_window_dpi(), + g_sc_ext.width, g_sc_ext.height, (int)g_sc_fmt); } LeaveCriticalSection(&g_start_lock); vsync_ensure(); diff --git a/runtime/host.h b/runtime/host.h index 87b55007..262b7b98 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -281,6 +281,7 @@ void rt_boot(void); // boot.c // modules, registers its host functions, then seals (hostfn_seal, or implicitly at the first rt_call). Return the // process exit code. int probe_cfg(void); // cfg_probe.c +int probe_dpi(void); // dpi_probe.c int probe_loader(void), probe_kernel(void), probe_ipc(void), probe_services(void), probe_fs(void); int probe_gfx_abi(void); // gfx_abi_probe.c int probe_vkdevice(void); // vk_device.c diff --git a/runtime/main.c b/runtime/main.c index 80e80ae6..901804d6 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -4,6 +4,7 @@ // --selftest layout, dispatch table and host-function registry; needs no game files // --probe-loader load and relocate the modules, check them against the measured 1.2.1 values // --probe-kernel, --probe-ipc, --probe-services, --probe-fs host probes +// --probe-dpi the process DPI awareness (per-monitor v2) and the primary monitor's size; no game files // --probe-gfx-abi NVN object layouts against nvn.h's reserved sizes; needs no game files // --probe-nvn, --probe-nvn-tex, --probe-window, --probe-shader, --probe-vkmem, --probe-vkexec graphics probes // --probe-astc the compute ASTC decoder against astc_decode_ldr; needs no game files @@ -472,7 +473,7 @@ static int selftest(void) { static int usage(const char *argv0) { fprintf(stderr, "usage: %s [--exefs DIR] [--romfs DIR] [--profile DIR] [--selftest | --probe-loader | " - "--probe-cfg | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " + "--probe-cfg | --probe-dpi | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " "--probe-vkexec | --probe-astc | --probe-movie | --prepare-shaders | --warm-shaders | --boot |--seconds S [--shot PNG]]\n", argv0); return 2; @@ -559,6 +560,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--selftest")) return selftest(); if (!strcmp(mode, "--probe-loader")) return probe_loader(); if (!strcmp(mode, "--probe-cfg")) return probe_cfg(); + if (!strcmp(mode, "--probe-dpi")) return probe_dpi(); if (!strcmp(mode, "--probe-kernel")) return probe_kernel(); if (!strcmp(mode, "--probe-ipc")) return probe_ipc(); if (!strcmp(mode, "--probe-services")) return probe_services(); diff --git a/runtime/swkbd_win.c b/runtime/swkbd_win.c index e319fc32..9d266978 100644 --- a/runtime/swkbd_win.c +++ b/runtime/swkbd_win.c @@ -217,8 +217,12 @@ static int text_entry_run(const SwkbdRequest *req, uint16_t *out, size_t cap, si Entry e = {.req = req, .armed = 0}; const char *test = cfg_get("TOTK_SWKBD_TEST"); if (test && *test) e.test = !strcmp(test, "cancel") ? -1 : 1; + // The process is per-monitor DPI aware: the font and the layout below are measured at the game window's DPI, the + // monitor the dialog opens on, not at the system DPI that plain SystemParametersInfo and AdjustWindowRectEx use. + HWND owner = gfx_window_handle(); + UINT dpi = owner ? GetDpiForWindow(owner) : GetDpiForSystem(); NONCLIENTMETRICSW ncm = {.cbSize = sizeof ncm}; - SystemParametersInfoW(SPI_GETNONCLIENTMETRICS, sizeof ncm, &ncm, 0); + SystemParametersInfoForDpi(SPI_GETNONCLIENTMETRICS, sizeof ncm, &ncm, 0, dpi); HFONT font = CreateFontIndirectW(&ncm.lfMessageFont); HDC dc = GetDC(NULL); HGDIOBJ old = SelectObject(dc, font); @@ -230,10 +234,9 @@ static int text_entry_run(const SwkbdRequest *req, uint16_t *out, size_t cap, si int y_prompt = pad, h_prompt = u * 5 / 2, y_edit = y_prompt + h_prompt + u / 4, h_edit = u * 9 / 5; int y_hint = y_edit + h_edit + u / 2, y_btn = y_hint + u * 3 / 2, ch = y_btn + bh + pad; - HWND owner = gfx_window_handle(); RECT wr = {0, 0, cw + 2 * pad, ch}; DWORD style = WS_POPUP | WS_CAPTION | WS_SYSMENU, ex = WS_EX_DLGMODALFRAME | WS_EX_CONTROLPARENT; - AdjustWindowRectEx(&wr, style, FALSE, ex); + AdjustWindowRectExForDpi(&wr, style, FALSE, ex, dpi); int ww = wr.right - wr.left, wh = wr.bottom - wr.top, x = CW_USEDEFAULT, y = CW_USEDEFAULT; RECT orc; if (owner && GetWindowRect(owner, &orc)) { diff --git a/runtime/totk.manifest b/runtime/totk.manifest index 8fe37fa1..a0da154b 100644 --- a/runtime/totk.manifest +++ b/runtime/totk.manifest @@ -1,7 +1,11 @@ + and fs_host.c convert with, so an install folder or user profile with a non-ASCII name works. + dpiAwareness PerMonitorV2 (dpiAware true/pm on Windows 10 before 1607): window, monitor and cursor sizes are physical + pixels and the swapchain matches the screen. A DPI-unaware process gets them divided by the display scale and its + image stretched back up by Windows, so 4K at 150% presented a 2560x1440 image, blurred. gfx_window.c follows a + change of the window's DPI (WM_GETDPISCALEDSIZE, WM_DPICHANGED). --> @@ -13,6 +17,8 @@ UTF-8 + true/pm + PerMonitorV2, PerMonitor diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 803c4810..073c57e3 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -171,8 +171,13 @@ static ImGuiStyle g_base_style; // the menu's style at 1080 lines static uint64_t g_last_ns; static uint64_t g_cost_ns, g_cost_frames; // CPU time of layout and recording while open, logged per 600 frames -// Sizes are tuned for 1080 lines and follow the window's height; under 720 lines they stay at the 720 size. -static float menu_scale(uint32_t h) { return (h >= 720 ? h : 720) / 1080.0f; } +// Sizes are tuned for 1080 lines and follow the window's height. Under 720 lines at 100% display scale they stay at the +// 720 size; that floor grows with the window's DPI, so a small window on a 150% or 200% display (sizes are physical +// pixels) still gets text as large as the display scale asks for. +static float menu_scale(uint32_t h) { + float least = 720.0f * (float)gfx_window_dpi() / 96.0f; + return ((float)h >= least ? (float)h : least) / 1080.0f; +} // SavageWindRecomp's colours: dark green panels, gold accents. static void style_init(ImGuiStyle &s) { diff --git a/runtime/ui/menu.h b/runtime/ui/menu.h index 0bf7b6b7..1dffc8ff 100644 --- a/runtime/ui/menu.h +++ b/runtime/ui/menu.h @@ -59,6 +59,7 @@ void gfx_window_apply_settings(void); // TOTK_FULLSCREEN or TOTK_WINDOW_SIZE void gfx_window_request_close(void); // as the window's close button: the process ends int gfx_window_monitor_size(int *w, int *h); // pixels of the monitor the window is on; 0 when unknown double gfx_window_refresh_hz(void); // its refresh rate (TOTK_FPS_LIMIT=display); 0 when unknown +unsigned gfx_window_dpi(void); // the window's DPI (96 at 100% display scale); any thread // The present mode the swapchain got, as a TOTK_VK_PRESENT value ("fifo" when mailbox is missing); "" before the first // swapchain. Any thread. const char *gfx_window_present_mode(void); diff --git a/totk.ini.example b/totk.ini.example index d55d52e9..abccd570 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -95,7 +95,8 @@ [display] # Presentation mode: mailbox (default when the driver offers it, vsync otherwise), fifo (vsync) or immediate. # vk_present = mailbox -# Window client area in pixels (640x360 to 7680x4320), and borderless fullscreen. F11 switches fullscreen and saves it +# Window client area in screen pixels (640x360 to 7680x4320), and borderless fullscreen. Windows display scaling does +# not apply: at 150% scale, 1920x1080 covers the area 1280x720 covers at 100%. F11 switches fullscreen and saves it # here. Changes while the game runs take effect at once. # window_size = 1920x1080 # fullscreen = 0 From 444abcbebba75d8fc58ea0a76c64eaeedf7c2100 Mon Sep 17 00:00:00 2001 From: arthurhfcardoso Date: Fri, 30 Oct 2026 09:26:36 -0300 Subject: [PATCH 07/50] Drop released textures that reload intact when video memory runs low On a 4 GiB GPU the released-texture list (up to 2048 entries, each ASTC texture decoded to RGBA8) filled video memory and the heap spilled into system memory: a Depths lava scene went from 12 ms to 60 ms of GPU time per frame and about 16 fps. Above tex_vram_pressure percent of the device-local budget (default 90, 0 = off), each submit now destroys up to 32 of the oldest released entries. Only entries that come back intact are dropped: mirrored textures the GPU never wrote, which are uploaded again from their storage. A render target exists only in its image; dropping those made textures come back empty (black flashes in the Depths boss fight, about 170 "left uninitialised" aliases a minute). Adds the setting to totk.ini and the Advanced tab, and a perf log line with the released count and drops. Co-Authored-By: Claude Sonnet 5.5 --- runtime/cfg.c | 6 +++--- runtime/ui/menu.cpp | 1 + runtime/ui_text.c | 2 ++ runtime/ui_text.h | 1 + runtime/vk_texture.c | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index 84abed08..f65bdf15 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -48,7 +48,7 @@ static const char *const known[] = { "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", }; @@ -74,7 +74,7 @@ static const struct { const char *key, *section; } sections[] = { {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, - {"TEX_ALIAS", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, + {"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, @@ -114,7 +114,7 @@ static const CfgMeta metas[] = { {"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART}, // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, - {"TEX_ALIAS", "1", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, // render fixes {"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART}, {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART}, diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 64c3c4c0..d0b46ca4 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -663,6 +663,7 @@ static void tab_advanced(float scale) { UI_MENU_MEMORY_VERIFY}; opt_choice("TOTK_MEMORY", UI_MENU_MEMORY, mem, 4, UI_MENU_MEMORY_DESC); opt_toggle("TOTK_TEX_ALIAS", UI_MENU_TEX_ALIAS, UI_MENU_TEX_ALIAS_DESC); + opt_slider("TOTK_TEX_VRAM_PRESSURE", UI_MENU_TEX_VRAM_PRESSURE, 0, 100, UI_MENU_TEX_VRAM_PRESSURE_DESC, 1, "%.0f%%"); opt_group(UI_MENU_GROUP_FIXES); opt_toggle("TOTK_LOD_GUARD", UI_MENU_LOD_GUARD, UI_MENU_LOD_GUARD_DESC); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index f93107ae..b16bf4a7 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -184,6 +184,7 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_MEMORY_AUTO] = "Automatic", [UI_MENU_MEMORY_IMPORT] = "Import", [UI_MENU_MEMORY_MIRROR] = "Mirror", [UI_MENU_MEMORY_VERIFY] = "Verify", [UI_MENU_TEX_ALIAS] = "Texture aliasing", [UI_MENU_TEX_ALIAS_DESC] = "A new texture created over a live texture's memory starts with that content instead of empty.", + [UI_MENU_TEX_VRAM_PRESSURE] = "Texture memory limit", [UI_MENU_TEX_VRAM_PRESSURE_DESC] = "Percent of video memory above which textures the game released are dropped (they come back from storage when used again). 0 keeps them.", [UI_MENU_LOD_GUARD] = "LOD guard", [UI_MENU_LOD_GUARD_DESC] = "Prevents objects flashing in the wrong place or a flat far field for a frame, by waiting for the game to finish rewriting level-of-detail data before the GPU reads it.", [UI_MENU_FIX_C0] = "Constant buffer fix", [UI_MENU_FIX_C0_DESC] = "Fills the driver-reserved constant bank c0 (storage sizes, dispatch size, point size, base instance) instead of reading zeros.", @@ -405,6 +406,7 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_MEMORY_AUTO] = "Автоматически", [UI_MENU_MEMORY_IMPORT] = "Импорт", [UI_MENU_MEMORY_MIRROR] = "Зеркало", [UI_MENU_MEMORY_VERIFY] = "Проверка", [UI_MENU_TEX_ALIAS] = "Наследование содержимого текстур", [UI_MENU_TEX_ALIAS_DESC] = "Новая текстура поверх памяти живой текстуры получает её содержимое, а не пустую.", + [UI_MENU_TEX_VRAM_PRESSURE] = "Предел памяти текстур", [UI_MENU_TEX_VRAM_PRESSURE_DESC] = "Процент видеопамяти, выше которого освобождённые игрой текстуры удаляются (при повторном использовании загружаются снова). 0 — не удалять.", [UI_MENU_LOD_GUARD] = "Защита LOD", [UI_MENU_LOD_GUARD_DESC] = "Не даёт объектам мелькать не на своём месте, а дальнему плану — на кадр становиться плоским: ждёт, пока игра допишет данные уровней детализации, и только потом их читает GPU.", [UI_MENU_FIX_C0] = "Исправление константного буфера", [UI_MENU_FIX_C0_DESC] = "Заполняет зарезервированный драйвером банк констант c0 (размеры хранилищ, размер запуска, размер точки, базовый экземпляр) вместо чтения нулей.", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 6eb805cf..64aa080e 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -117,6 +117,7 @@ typedef enum { UI_MENU_MEMORY, UI_MENU_MEMORY_DESC, UI_MENU_MEMORY_AUTO, UI_MENU_MEMORY_IMPORT, UI_MENU_MEMORY_MIRROR, UI_MENU_MEMORY_VERIFY, UI_MENU_TEX_ALIAS, UI_MENU_TEX_ALIAS_DESC, + UI_MENU_TEX_VRAM_PRESSURE, UI_MENU_TEX_VRAM_PRESSURE_DESC, UI_MENU_LOD_GUARD, UI_MENU_LOD_GUARD_DESC, UI_MENU_FIX_C0, UI_MENU_FIX_C0_DESC, UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC, diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index b65f21e7..abf0ab2d 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -475,6 +475,36 @@ static void sweep_released(void) { while (nreleased > RELEASED_MAX && rel_head) destroy_tex(rel_head); } +// TOTK_TEX_VRAM_PRESSURE (percent of the device-local budget, default 90; 0 = off): above it, each submit drops up to +// PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released images (each ASTC +// one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason above. Only entries +// that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their storage. A render target +// lives only in its image; dropping it brought the texture back empty (black flashes in a Depths boss fight, ~170 +// "left uninitialised" aliases a minute). +#define PRESSURE_STEP 32 +static int vram_pressure = -1; +static uint64_t n_pressure_drops; + +static void sweep_pressure(void) { // tex_lock held + if (vram_pressure < 0) { + const char *s = cfg_get("TOTK_TEX_VRAM_PRESSURE"); + vram_pressure = s && *s ? atoi(s) : 90; + if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = 90; + rt_log("vk: released textures under VRAM pressure: %s%d%%", vram_pressure ? "dropped above " : "off", vram_pressure); + } + if (!vram_pressure || !rel_head || submit_epoch & 7) return; // the budget query every 8 submits + uint64_t used, budget; + int exact; + if (!vk_mem_vram(&used, &budget, &exact) || used <= budget / 100 * (uint64_t)vram_pressure) return; + int dropped = 0; + for (Tex *e = rel_head, *next; e && dropped < PRESSURE_STEP; e = next) { + next = e->rel_next; + if (!e->pub.mirrored || e->pub.gpu_owned || e->gpu_dirty) continue; + destroy_tex(e); + dropped++, n_pressure_drops++; + } +} + // ---------------------------------------------------------------- uploads static uint32_t texel_alpha_one(int format, uint32_t *elem) { switch (format) { @@ -1032,6 +1062,7 @@ uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC void vk_texture_new_submit(void) { lock(); if (!++submit_epoch) submit_epoch = 1; + sweep_pressure(); static uint64_t log_at; uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) / 60; if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) { @@ -1050,6 +1081,9 @@ void vk_texture_new_submit(void) { (unsigned long long)f * 60, (unsigned long long)ck, (unsigned long long)n_checks, (unsigned long long)calls, (unsigned long long)untr); n_checks = 0; + rt_log("perf: frame %llu: released textures kept %d, dropped under VRAM pressure %llu", (unsigned long long)f * 60, + nreleased, (unsigned long long)n_pressure_drops); + n_pressure_drops = 0; } unlock(); } From abb00c500c264e458a444f419b03fbd530def66a Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 15:27:42 +0300 Subject: [PATCH 08/50] Add a diagnostics key for the fence write-back size limit and trace the targets it skips wb_fence_limit (TOTK_WB_FENCE_LIMIT, default 65536) replaces the fixed 64 KiB limit of the write-back done when the CPU waits for a fence. With trace = gfx the log names each dirty target left on the GPU because of its size and each target over 64 KiB that a raised limit wrote back. Meant to test whether the fuse weapon switch bug (#18) and the Purah Pad camera bug (#13, #63) come from a target the game reads on the CPU without an invalidate. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 4 ++- runtime/vk_texture.c | 58 ++++++++++++++++++++++++++++++++++++++------ totk.ini.example | 5 ++++ 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index f355b982..b122690a 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -50,7 +50,7 @@ static const char *const known[] = { "SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", + "WB_FENCE_LIMIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", }; // The totk.ini section a key cfg_set may write goes to when the file lacks it (totk.ini.example's sections). @@ -79,6 +79,7 @@ static const struct { const char *key, *section; } sections[] = { {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -126,6 +127,7 @@ static const CfgMeta metas[] = { {"VK_SUBGROUP_PIN", "1", "0|1", RESTART}, // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index b65f21e7..7b68ffe2 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -841,9 +841,22 @@ const VkTexImage *vk_texture_get(const tex_t *t) { } // Small render targets the CPU reads without NVN's invalidate (vk_texture_write_back_small); small_pending: one may be -// dirty. -enum { WB_SMALL = 64 << 10 }; +// dirty. wb_limit is the largest target (bytes) that goes back when the CPU waits for a fence: TOTK_WB_FENCE_LIMIT, +// read once. wb_trace: TOTK_TRACE=gfx, read together with it. +enum { WB_SMALL_DEFAULT = 64 << 10 }; static int small_pending; +static uint64_t wb_limit = WB_SMALL_DEFAULT; +static int wb_trace; +static int wb_cfg_read; + +static void wb_cfg_init(void) { + if (wb_cfg_read) return; + int v = cfg_int("TOTK_WB_FENCE_LIMIT"); + wb_limit = v < 0 ? 0 : (uint64_t)v; + wb_trace = (rt_trace & RT_TRACE_GFX) != 0; + wb_cfg_read = 1; + if (wb_limit != WB_SMALL_DEFAULT) rt_log("vk: fence write-back limit %llu bytes", (unsigned long long)wb_limit); +} void vk_texture_gpu_written(const tex_t *t) { if (!t || !t->mem) return; @@ -860,7 +873,8 @@ void vk_texture_gpu_written(const tex_t *t) { storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render e->gpu_seq = ++gpu_writes; e->gpu_fence = vk_exec_recording_fence(); - if (e->t.L.size <= WB_SMALL) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE); + wb_cfg_init(); + if (e->t.L.size <= wb_limit) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE); own_add(e); } } @@ -1367,24 +1381,54 @@ void vk_texture_write_back(uint64_t addr, uint64_t size) { // TotK's grass system renders a 128x128 RGBA8 map of cut and trampled grass into a CPU-cached pool every frame and its // CPU code reads the pool bytes without nvnMemoryPoolInvalidateMappedRange (cut reactions: blades, insects, the cut // itself). On the Switch the bytes are there once the frame's fence has passed. So when the -// CPU waits for a fence, mirrored targets up to WB_SMALL bytes rendered by work up to that fence go back to their +// CPU waits for a fence, mirrored targets up to wb_limit bytes rendered by work up to that fence go back to their // storage, unless the CPU rewrote it since. Larger targets still go back on an invalidate, memcpy or ReadTexels. int vk_texture_small_dirty(void) { return __atomic_load_n(&small_pending, __ATOMIC_ACQUIRE); } +// TOTK_TRACE=gfx: a dirty target the fence write-back leaves alone because it is over the limit (what a CPU read +// after the wait would miss), or sends back although it is over the default limit (what a raised limit changed). One +// line per target address and outcome; the table is small and a full one stops logging. +static void wb_note(const Tex *e, int skipped) { + static uint64_t seen[2][64]; + static int n_seen[2]; + for (int i = 0; i < n_seen[skipped]; i++) + if (seen[skipped][i] == e->pub.addr) return; + if (n_seen[skipped] == 64) return; + seen[skipped][n_seen[skipped]++] = e->pub.addr; + const pool_t *p = e->t.pool; + unsigned fl = p ? p->flags : 0; + rt_log("vk: fence write-back %s %#llx %ux%u fmt %#x size %llu pool %s (limit %llu)", + skipped ? "skipped-for-size" : "over-default", (unsigned long long)e->pub.addr, e->t.d.width, e->t.d.height, + e->t.d.format, (unsigned long long)e->t.L.size, + !p ? "none" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_CACHED) ? "cpu-cached" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED) ? "cpu-uncached" : "other", + (unsigned long long)wb_limit); +} + void vk_texture_write_back_small(uint64_t fence) { lock(); + wb_cfg_init(); __atomic_store_n(&small_pending, 0, __ATOMIC_RELAXED); AcquireSRWLockShared(&owned_lock); int n = 0, later = 0; - for (int i = 0; i < nowned; i++) n += owned[i]->gpu_dirty && owned[i]->t.L.size <= WB_SMALL; + for (int i = 0; i < nowned; i++) n += owned[i]->gpu_dirty && owned[i]->t.L.size <= wb_limit; Tex **hit = n ? malloc(sizeof *hit * (size_t)n) : NULL; if (n && !hit) rt_fatal("vk: out of memory for a texture write-back"); n = 0; for (int i = 0; i < nowned; i++) { Tex *e = owned[i]; - if (!e->gpu_dirty || e->t.L.size > WB_SMALL) continue; + if (!e->gpu_dirty) continue; + if (e->t.L.size > wb_limit) { + if (wb_trace && e->gpu_fence <= fence) { + wb_note(e, 1); + perf_count(PERF_CNT_WB_SKIP); + } + continue; + } if (e->gpu_fence > fence) later = 1; - else hit[n++] = e; + else { + if (wb_trace && e->t.L.size > WB_SMALL_DEFAULT) wb_note(e, 0); + hit[n++] = e; + } } ReleaseSRWLockShared(&owned_lock); if (later) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE); diff --git a/totk.ini.example b/totk.ini.example index d55d52e9..411871e2 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -180,3 +180,8 @@ # takes longer than flight_spike_ms (0 to 60000). # flight = 0 # flight_spike_ms = 100 +# Largest render target (bytes, 0 to 1073741824) that goes back to its game memory when the game waits for a GPU fence. +# A game that reads a larger target without invalidating it sees stale bytes at the default. With trace = gfx the log +# names each dirty target over the limit that stays on the GPU ("fence write-back skipped-for-size") and each one over +# 65536 bytes that went back ("over-default"). Raising it costs a copy per such target and wait. +# wb_fence_limit = 65536 From bb1b675381255adc551ad61bab0d89eca2058538 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 15:27:44 +0300 Subject: [PATCH 09/50] Floor the 1.4.3 LOD and pre-actor fade increments at 1 Above about 63.5 fps the fade step trunc(2.1 * ratio) is 0, so objects that start a fade stay translucent or invisible until a slower frame moves them on (issues #10, #37). Switching to 60 fps and back cleared it because the 60 fps step is 1. Port of MacNativeGames' high-FPS fade fix as a fade set under TOTK_HIGHFPS_FIX. Co-Authored-By: Claude Opus 5.5 --- recomp/patches-v143.txt | 12 ++++++++++++ runtime/ui_text.c | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/recomp/patches-v143.txt b/recomp/patches-v143.txt index 16189169..498af650 100644 --- a/recomp/patches-v143.txt +++ b/recomp/patches-v143.txt @@ -47,6 +47,18 @@ site cloth main 28bc98 9400546a @page:x8 bd400104 1e240842 1e240863 9400546a # # s1 is set again by the fmov two instructions on. site bounce main 562a64 bd402920 bd402920 bd403921 1e210800 # ldr s0, [x9, #0x28]; ldr s1, [x9, #0x38]; fmul s0, s0, s1 +# Fades (MacNativeGames game_patches_v143.c patch_fade_increment, upstream 3e9e7f67): the distance/LOD mesh fades and +# the pre-actor fades add trunc(2.1 * ratio) to a counter each frame: 2 at 30 fps, 1 at 60, 0 above about 63.5 fps +# (ratio below 1 / 2.1), where a fade stops and its object stays translucent or invisible until a slower frame moves +# it on. Each site floors the increment at 1 without the flags: clz is 32 only for 0, so increment += clz >> 5. The +# scratch register is one the next instruction writes (w10 at 0x0a7ac4: ldrb w10; w11 at the pre-actor sites: add +# w11), or at 0x95dd24 a temporary the function no longer reads (add, strb, return). +set fade TOTK_HIGHFPS_FIX 1 +site fade main 0a7ac4 1e380008 1e380008 5ac0110a 53057d4a 0b0a0108 # fcvtzs w8, s0; clz w10, w8; lsr w10, w10, #5; add w8, w8, w10 +site fade main 95dd24 1e380008 1e380008 5ac0110a 53057d4a 0b0a0108 # the same, second mesh fade +site fade main 6155e4 1e38000a 1e38000a 5ac0114b 53057d6b 0b0b014a # fcvtzs w10, s0; clz w11, w10; lsr w11, w11, #5; add w10, w10, w11 +site fade main 615660 1e38000a 1e38000a 5ac0114b 53057d6b 0b0b014a # the same, second pre-actor fade path + # Output size (upstream game_patches_v143.c patch_output_res; runtime/resolution.c switches the set on). The swapchain # size nnMain writes comes from page +48; the display configuration's canvas (+0x10) and active output (+0x508) from # page +32, and the configuration goes to page +40. x9 is dead at the publication point. diff --git a/runtime/ui_text.c b/runtime/ui_text.c index f93107ae..519fcbe3 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -74,8 +74,8 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_REALTIME_DESC] = "Steps the game by real time, so a slow frame does not slow the game down.", [UI_MENU_CLOTH_REALTIME] = "Cloth on the clock", [UI_MENU_CLOTH_REALTIME_DESC] = "Steps cloth and hair by real time as well.", - [UI_MENU_HIGHFPS_FIX] = "High frame rate physics fix", - [UI_MENU_HIGHFPS_FIX_DESC] = "Corrects physics that misbehave above 30 frames per second.", + [UI_MENU_HIGHFPS_FIX] = "High frame rate fixes", + [UI_MENU_HIGHFPS_FIX_DESC] = "Corrects physics that misbehave above 30 frames per second, and objects that stay see-through or invisible above 60.", [UI_MENU_GAME_LANG] = "Game language", [UI_MENU_GAME_LANG_DESC] = "Language of the game's text and voices.", [UI_MENU_GAME_LANG_AUTO] = "Automatic", [UI_MENU_OPERATION_MODE] = "Console mode", @@ -291,8 +291,8 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_REALTIME_DESC] = "Игра идёт по реальному времени, медленный кадр её не замедляет.", [UI_MENU_CLOTH_REALTIME] = "Ткань по часам", [UI_MENU_CLOTH_REALTIME_DESC] = "Ткань и волосы тоже идут по реальному времени.", - [UI_MENU_HIGHFPS_FIX] = "Исправление физики на высоком FPS", - [UI_MENU_HIGHFPS_FIX_DESC] = "Исправляет физику, которая ломается при частоте выше 30 кадров в секунду.", + [UI_MENU_HIGHFPS_FIX] = "Исправления для высокого FPS", + [UI_MENU_HIGHFPS_FIX_DESC] = "Исправляет физику, которая ломается при частоте выше 30 кадров в секунду, и объекты, которые выше 60 остаются полупрозрачными или невидимыми.", [UI_MENU_GAME_LANG] = "Язык игры", [UI_MENU_GAME_LANG_DESC] = "Язык текста и озвучки игры.", [UI_MENU_GAME_LANG_AUTO] = "Автоматически", [UI_MENU_OPERATION_MODE] = "Режим консоли", From afa3607259caf92d2991df1815e6d8c9d54984d3 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 15:30:16 +0300 Subject: [PATCH 10/50] Keep the display and the system awake while the game window is up and not minimized Windows turned the monitor off after its idle timeout during play, since gamepad input does not count as activity (#7). Co-Authored-By: Claude Opus 5.5 --- runtime/gfx_window.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/runtime/gfx_window.c b/runtime/gfx_window.c index fc449051..98bc8fe3 100644 --- a/runtime/gfx_window.c +++ b/runtime/gfx_window.c @@ -203,6 +203,16 @@ static void cfg_window_size(int *w, int *h) { *h = y; } +// Windows turns the display off and sleeps after the idle timeout, and gamepad input does not count as activity. While +// the window is up and not minimized it holds the display and the system on, as other games do. The state belongs to +// the calling thread, so only the window thread (window_main and wnd_proc) calls this; the process exit releases it. +static void keep_awake(int on) { + static int cur = -1; + if (on == cur) return; + cur = on; + SetThreadExecutionState(on ? ES_CONTINUOUS | ES_DISPLAY_REQUIRED | ES_SYSTEM_REQUIRED : ES_CONTINUOUS); +} + // F11: borderless window over the whole monitor, and back to the saved window. The swapchain follows through WM_SIZE. // Not Alt+Enter: Enter is the Plus button (input_win.c). A background window (test runs) keeps its place behind the // user's windows and takes no focus. @@ -372,6 +382,7 @@ static LRESULT CALLBACK wnd_proc(HWND h, UINT m, WPARAM w, LPARAM l) { } return 0; case WM_SIZE: + keep_awake(w != SIZE_MINIMIZED); if (w != SIZE_MINIMIZED) InterlockedExchange(&g_dirty, 1); refresh_rate_update(h, 0); return 0; @@ -441,6 +452,7 @@ static DWORD WINAPI window_main(void *arg) { } note_applied(); refresh_rate_update(g_hwnd, 1); + keep_awake(1); } SetEvent(g_wnd_ready); if (g_wnd_failed) return 1; From 389a516f8d15f933337c3f50c75051a3cdf77c36 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 15:35:10 +0300 Subject: [PATCH 11/50] Read the packed pipeline keys of the current MacNativeGames shader list in prepare_shaders Since MacNativeGames f024ef04 the R and B rows keep the key as "::" instead of plain hex, and parse_list stopped with a ValueError on a source checkout. The pinned release list parses as before (identical variants and priorities); the current one gives 58053 variants. Co-Authored-By: Claude Opus 5.5 --- tools/prepare_shaders.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index 4535f708..4bb8985d 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -68,6 +68,15 @@ def read_list(path): return shader_list.history(ROOT / 'refs' / 'MacNativeGames') +def pipeline_variants(key): + """The vertex and fragment variant names of an R/B row's pipeline key. Since MacNativeGames f024ef04 (2026-10-28) the + TotK list keeps keys packed, "::"; older revisions and contribution files + hold the whole key in hex, "\\0\\0" + a Metal descriptor (docs/shader-list.md there).""" + if ':' in key: + return key.split(':', 2)[:2] + return [n.decode('ascii', 'replace') for n in bytes.fromhex(key).split(b'\0')[:2]] + + def parse_list(text): """V rows as (key, deps); priority per shader key from the warm rows (variant names start with the key).""" variants, prio = [], {} @@ -85,10 +94,9 @@ def parse_list(text): elif f[0] == 'C' and len(f) == 3: warm(f[1][:64], f[2]) elif f[0] in ('R', 'B') and len(f) >= 3: - names = bytes.fromhex(f[1]).split(b'\0')[:2] # "\0\0" + a Metal descriptor - for n in names: + for n in pipeline_variants(f[1]): if len(n) >= 64: - warm(n[:64].decode('ascii', 'replace'), f[-1]) + warm(n[:64], f[-1]) return variants, prio From 43704c0d9f8e75516353d7be0aca7a81b8be16ac Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 19:48:38 +0300 Subject: [PATCH 12/50] Turn off the recorder thread on AMD GPUs and GPL on AMD's RDNA 1/2 driver, and log GPU breadcrumbs on a lost device AMD drivers lost the device at boot or at menu opens with the recorder thread; vk_recorder = 0 ended it for three RX 9070 XT users (issue #5), and RADV saw an illegal opcode at the end of a command buffer (issue #40). It now defaults to off on vendor 0x1002; TOTK_VK_RECORDER=1 turns it back on. On AMD's separate driver branch for RDNA 1/2 ("AMD proprietary shader compiler") every device loss came with a cold pipeline cache and draws waiting on fast-linked pipelines, and the recorder setting did not help there. TOTK_GPL now has no fixed default: off on that branch, on elsewhere. VK_AMD_buffer_marker breadcrumbs (vk_crumbs.c) mark the start and end of each submission part, upload batch, ASTC decode chunk and present; on device lost the log names the last point the GPU reached and the last one it finished. vk_fsr.c ensure_images held no queue lock around vkDeviceWaitIdle, which needs every queue externally synchronized. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 +- runtime/gfx_window.c | 3 ++ runtime/vk.h | 19 +++++++++++-- runtime/vk_astc.c | 3 +- runtime/vk_caps.c | 24 +++++++++++++--- runtime/vk_crumbs.c | 67 ++++++++++++++++++++++++++++++++++++++++++++ runtime/vk_device.c | 6 ++++ runtime/vk_fsr.c | 6 +++- runtime/vk_memory.c | 2 ++ runtime/vk_rec.c | 9 +++++- runtime/vk_sync.c | 3 ++ totk.ini.example | 4 ++- 12 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 runtime/vk_crumbs.c diff --git a/runtime/cfg.c b/runtime/cfg.c index b122690a..7a58cfc9 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -112,7 +112,7 @@ static const CfgMeta metas[] = { // shaders and pipelines {"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "fullscreen", "strict|any|fullscreen", RESTART}, {"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART}, - {"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART}, + {"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "", "0|1", RESTART}, // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, {"TEX_ALIAS", "1", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, diff --git a/runtime/gfx_window.c b/runtime/gfx_window.c index ffb7f1cf..718474b3 100644 --- a/runtime/gfx_window.c +++ b/runtime/gfx_window.c @@ -849,6 +849,8 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(cb, &bi); + uint64_t crumb = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + vk_crumb(cb, VK_CRUMB_PRESENT, crumb); VkImage dst = g_sc_img[idx]; sync_barrier(cb, dst, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_2_TRANSFER_BIT, 0, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); @@ -909,6 +911,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w record_thumb(cb, dst, slot); } sync_barrier(cb, dst, last, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, last_stage, last_write, VK_PIPELINE_STAGE_2_NONE, 0); + vk_crumb(cb, VK_CRUMB_PRESENT_END, crumb); vkEndCommandBuffer(cb); VkSemaphoreSubmitInfo wait = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = g_acquire[slot], diff --git a/runtime/vk.h b/runtime/vk.h index f6302678..f7050892 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -36,7 +36,7 @@ typedef struct VkCtx { int validation; // layer active int has_custom_border_color, has_eds, has_eds2, has_eds3, has_vertex_input_dynamic_state, has_conditional_rendering, has_transform_feedback, has_provoking_vertex, has_device_fault, has_memory_budget, - has_robustness2, has_buffer_device_address, has_fragment_shader_interlock, + has_robustness2, has_buffer_device_address, has_fragment_shader_interlock, has_buffer_marker, has_astc_ldr; // textureCompressionASTC_LDR (enabled when reported) // Feedback loops (maxwell.md 1.9, vk_exec_draw.c feedback_prepare): a draw samples a target of its open pass that // an earlier draw of the pass wrote. TOTK_FEEDBACK=0: VK_FEEDBACK_OFF (sampled in GENERAL without a barrier, the @@ -56,8 +56,11 @@ typedef struct { uint32_t sg_size; // requiredSubgroupSize on sg_pin_stages (32) VkShaderStageFlags sg_pin_stages; // 0: no pin, stages run at the device's subgroupSize int gpl; // VK_EXT_graphics_pipeline_library enabled with fast linking (TOTK_GPL=0: off) + int amd; // an AMD GPU (vendor 0x1002), any driver + int amd_sc; // AMD's Windows driver branch for RDNA 1 and 2 ("AMD proprietary shader compiler") } VkCaps; extern VkCaps vk_caps; +void vk_caps_driver(VkPhysicalDevice p); // first, before vkCreateDevice: vk_caps.amd, amd_sc void vk_caps_features(VkPhysicalDeviceFeatures *f); // before vkCreateDevice (TOTK_VK_NO_FLOAT64 clears shaderFloat64) // Before vkCreateDevice, with the driver's graphics pipeline library feature and properties: sets vk_caps.gpl, // 1 = enable the extensions (VK_KHR_pipeline_library, VK_EXT_graphics_pipeline_library) and the feature. @@ -71,9 +74,21 @@ void vk_shutdown(void); int vk_selftest(void); // 0 on success // Call with the result of a failed queue submit, present or wait before stopping on it: VK_ERROR_DEVICE_LOST logs // what VK_EXT_device_fault reports (description, faulting addresses with their precision, vendor codes) once per -// process. Any other result returns at once. +// process, after the GPU breadcrumbs (vk_crumbs_report). Any other result returns at once. void vk_device_fault_report(VkResult r); +// ---- GPU breadcrumbs (vk_crumbs.c): with VK_AMD_buffer_marker, each crumb has the GPU write its code into host memory +// twice, when the command processor reaches it and when all work before it has finished. After a lost device the two +// codes bracket the work that hung. Without the extension every call returns at once. +enum { VK_CRUMB_SUBMIT = 1, VK_CRUMB_SUBMIT_END, VK_CRUMB_UPLOAD, VK_CRUMB_UPLOAD_END, VK_CRUMB_ASTC, VK_CRUMB_PRESENT, + VK_CRUMB_PRESENT_END }; +void vk_crumbs_init(VkCtx *c); // after the allocator exists +void vk_crumbs_release(void); +// Records crumb kind with number n (a timeline value, a batch number; the low 28 bits are kept) into cb. Only the +// thread that records cb may call it: never into a replay part while the recorder thread owns it (vk_rec.h). +void vk_crumb(VkCommandBuffer cb, int kind, uint64_t n); +void vk_crumbs_report(void); // logs the two codes and what they name + // ---- window and present (gfx_window.c). Host threads only. // vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled // to the swapchain extent, into the acquired swapchain image and presents it. src is in layout, which is diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index 70ba539d..62afe30d 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -104,7 +104,7 @@ static struct { VmaAllocation tab_alloc; int tab_uploaded; } G = {.mode = -1}; -static uint64_t n_dispatch, n_texels; +static uint64_t n_dispatch, n_texels, n_chunks; // n_chunks: the breadcrumb number of the next scratch chunk // TOTK_ASTC_TIME=1 (or vk_astc_time_enable): timestamps around the dispatches of each scratch chunk (the decode // alone) and around a whole vk_astc_record call (one texture upload or WriteTexels: copies and barriers included), @@ -449,6 +449,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons bcp[i1 - i0] = (VkBufferCopy){rg[i1].src, in, region_in(&rg[i1], bw, bh)}; in += region_in(&rg[i1], bw, bh), out += region_out(&rg[i1]); } + vk_crumb(cb, VK_CRUMB_ASTC, ++n_chunks); if (chunks) // the previous chunk's dispatches and image copy are done with the scratch mem_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_READ_BIT, diff --git a/runtime/vk_caps.c b/runtime/vk_caps.c index c07b32dc..ca4c527b 100644 --- a/runtime/vk_caps.c +++ b/runtime/vk_caps.c @@ -6,8 +6,8 @@ // - minStorageBufferOffsetAlignment above 4 (TOTK_VK_SSBO_ALIGN= raises it): the translator computes // storage offsets from the guest address rounded down to that alignment (Ryujinx's unaligned storage path), and // vk_exec_draw.c ssbo_info binds the rounded-down address, so shader writes reach guest memory instead of a copy. -// - no graphics pipeline library with fast linking (TOTK_GPL=0 forces it): vk_caps.gpl = 0, the extension stays -// disabled, and pipelines are built whole. +// - no graphics pipeline library with fast linking (TOTK_GPL=0 forces it; the default on AMD's RDNA 1/2 driver): +// vk_caps.gpl = 0, the extension stays disabled, and pipelines are built whole. // - no 32-lane pin (no subgroupSizeControl, 32 outside min..max, or TOTK_VK_SUBGROUP_PIN=0): stages run at the // device's subgroupSize (SPIR-V 1.3 without a required size) and the translator is told that size. // TOTK_VK_SUBGROUP= pins n instead of 32 where the driver allows it. @@ -36,11 +36,25 @@ void vk_caps_features(VkPhysicalDeviceFeatures *f) { vk_caps.float64 = f->shaderFloat64; } +void vk_caps_driver(VkPhysicalDevice p) { + VkPhysicalDeviceDriverProperties drv = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; + VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &drv}; + vkGetPhysicalDeviceProperties2(p, &p2); + vk_caps.amd = p2.properties.vendorID == 0x1002; + // AMD split RDNA 1 and 2 off into a driver branch of their own; its driverInfo reads + // "26.10.1 (AMD proprietary shader compiler)" where the RDNA 3 and 4 driver names LLPC. + vk_caps.amd_sc = drv.driverID == VK_DRIVER_ID_AMD_PROPRIETARY && strstr(drv.driverInfo, "shader compiler"); +} + +// TOTK_GPL=0/1 forces graphics pipeline libraries off or on. Default off on AMD's RDNA 1/2 driver branch: every boot +// that lost the device there had a cold pipeline cache and draws waiting on fast-linked pipelines, and a run with GPL +// off booted (issue #5); Savage Wind Recomp turns GPL off on RDNA 4 under RADV for a GPU hang at the title screen. int vk_caps_gpl(const VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT *f, const VkPhysicalDeviceGraphicsPipelineLibraryPropertiesEXT *p) { gpl_lib = f->graphicsPipelineLibrary; gpl_fast = p->graphicsPipelineLibraryFastLinking; - vk_caps.gpl = gpl_lib && gpl_fast && cfg_bool("TOTK_GPL"); + const char *e = cfg_get("TOTK_GPL"); // no fixed default: it depends on the driver, see above + vk_caps.gpl = gpl_lib && gpl_fast && (e && *e ? strcmp(e, "0") != 0 : !vk_caps.amd_sc); return vk_caps.gpl; } @@ -117,8 +131,10 @@ void vk_caps_init(VkCtx *c) { has_ext(ep, ne, "VK_EXT_graphics_pipeline_library"), depth_ok(p, VK_FORMAT_D16_UNORM), depth_ok(p, VK_FORMAT_D24_UNORM_S8_UINT), depth_ok(p, VK_FORMAT_X8_D24_UNORM_PACK32), depth_ok(p, VK_FORMAT_D32_SFLOAT), depth_ok(p, VK_FORMAT_D32_SFLOAT_S8_UINT)); + const char *ge = cfg_get("TOTK_GPL"); rt_log("vk caps: gpl %s fast-link %s (graphicsPipelineLibrary %u%s)", vk_caps.gpl ? "yes" : "no", - gpl_fast ? "yes" : "no", (unsigned)gpl_lib, gpl_lib && gpl_fast && !vk_caps.gpl ? ", off by TOTK_GPL=0" : ""); + gpl_fast ? "yes" : "no", (unsigned)gpl_lib, + !gpl_lib || !gpl_fast || vk_caps.gpl ? "" : ge && *ge ? ", off by TOTK_GPL=0" : ", off on AMD's RDNA 1/2 driver"); rt_log("vk caps: fallbacks: float64 %s, storage buffers %s %u, subgroup pin %u on stages %#x, translator subgroup vs/fs/cs %u/%u/%u", vk_caps.float64 ? "native" : "softfloat64", vk_caps.sbuf_align ? "rounded down to" : "exact, alignment", (unsigned)l->minStorageBufferOffsetAlignment, want, (unsigned)vk_caps.sg_pin_stages, told[0], told[1], told[2]); diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c new file mode 100644 index 00000000..74a391a2 --- /dev/null +++ b/runtime/vk_crumbs.c @@ -0,0 +1,67 @@ +// GPU breadcrumbs for a lost device (vk.h vk_crumb). VK_AMD_buffer_marker writes a 32-bit code into a buffer from +// the command stream: at TOP_OF_PIPE once the command processor reaches the marker, at BOTTOM_OF_PIPE once every +// command before it has finished. Each crumb writes both, into two words of one host-coherent buffer, so after a +// device loss word 0 names the newest point the GPU reached and word 1 the newest point it had finished: the hang lies +// between them. A code is the crumb kind in the top 4 bits and its number in the low 28. The markers cost two small +// writes per crumb, and crumbs mark submissions, upload batches, ASTC chunks and presents, not draws. +#include +#include + +#include "host.h" +#include "vk.h" + +static PFN_vkCmdWriteBufferMarkerAMD write_marker; +static VkBuffer buf; +static VmaAllocation alloc; +static volatile uint32_t *words; + +void vk_crumbs_init(VkCtx *c) { + if (!c->has_buffer_marker) return; + write_marker = (PFN_vkCmdWriteBufferMarkerAMD)vkGetDeviceProcAddr(c->device, "vkCmdWriteBufferMarkerAMD"); + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 16, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, + .requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT}; + VmaAllocationInfo info; + if (!write_marker || vmaCreateBuffer(c->vma, &bi, &ai, &buf, &alloc, &info) != VK_SUCCESS) { + rt_log("vk: GPU breadcrumbs unavailable"); + write_marker = NULL; + return; + } + words = info.pMappedData; + words[0] = words[1] = 0; + rt_log("vk: GPU breadcrumbs on (VK_AMD_buffer_marker)"); +} + +void vk_crumbs_release(void) { + VkCtx *c = vk_ctx(); + if (buf && c) vmaDestroyBuffer(c->vma, buf, alloc); + buf = VK_NULL_HANDLE, alloc = NULL, words = NULL, write_marker = NULL; +} + +void vk_crumb(VkCommandBuffer cb, int kind, uint64_t n) { + if (!write_marker || !cb) return; + uint32_t code = (uint32_t)kind << 28 | (uint32_t)(n & 0x0fffffff); + write_marker(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, buf, 0, code); + write_marker(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, buf, 4, code); +} + +static void describe(uint32_t code, char *o, size_t size) { + static const char *const kinds[] = {"nothing", "the start of a part of submission", "the end of submission", + "the start of upload batch", "the end of upload batch", + "ASTC decode chunk", "the start of present", + "the end of present"}; + unsigned k = code >> 28; + if (!code) snprintf(o, size, "nothing yet"); + else if (k < sizeof kinds / sizeof *kinds) snprintf(o, size, "%s %u", kinds[k], code & 0x0fffffff); + else snprintf(o, size, "unknown code %#x", code); +} + +void vk_crumbs_report(void) { + if (!words) return; + char reached[96], finished[96]; + describe(words[0], reached, sizeof reached); + describe(words[1], finished, sizeof finished); + rt_log("vk: GPU breadcrumbs: the GPU last reached %s; everything before %s had finished", reached, finished); +} diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 26881a68..98fb84f9 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -233,6 +233,8 @@ static int create_device(void) { CHAIN(lr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_LOCAL_READ_FEATURES_KHR); } if (has_ext(ep, ne, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) { g_vk.has_memory_budget = 1; ext[n_ext++] = VK_EXT_MEMORY_BUDGET_EXTENSION_NAME; } + // GPU breadcrumbs for a lost device (vk_crumbs.c): AMD's drivers and RADV report it. + if (has_ext(ep, ne, VK_AMD_BUFFER_MARKER_EXTENSION_NAME)) { g_vk.has_buffer_marker = 1; ext[n_ext++] = VK_AMD_BUFFER_MARKER_EXTENSION_NAME; } // Graphics pipeline library: chained last so it can be dropped again when vk_caps_gpl declines it. void **gpl_link = NULL; if (has_ext(ep, ne, VK_KHR_PIPELINE_LIBRARY_EXTENSION_NAME) && has_ext(ep, ne, VK_EXT_GRAPHICS_PIPELINE_LIBRARY_EXTENSION_NAME)) { @@ -246,6 +248,7 @@ static int create_device(void) { // The driver fills the chain with what it supports; the same chain then enables all of it (robustBufferAccess // stays on: guest shaders index constant buffers freely and out-of-range reads must stay harmless). vkGetPhysicalDeviceFeatures2(g_vk.phys, &c->f2); + vk_caps_driver(g_vk.phys); vk_caps_features(&c->f2.features); if (gpl_link) { VkPhysicalDeviceGraphicsPipelineLibraryPropertiesEXT gp = { @@ -320,6 +323,7 @@ static int create_device(void) { rt_log("vk: timeline semaphore creation failed (%d)", (int)r); return 0; } + vk_crumbs_init(&g_vk); return 1; } @@ -329,6 +333,7 @@ static void destroy_all(void) { gfx_window_release_vk(); if (g_vk.device) vkDeviceWaitIdle(g_vk.device); if (g_vk.device) vk_astc_release(); + if (g_vk.device) vk_crumbs_release(); if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL); vk_headroom_stop(); if (g_vk.vma) vmaDestroyAllocator(g_vk.vma); @@ -387,6 +392,7 @@ void vk_device_fault_report(VkResult r) { AcquireSRWLockExclusive(&lock); if (done) { ReleaseSRWLockExclusive(&lock); return; } done = 1; + vk_crumbs_report(); VkCtx *c = vk_ctx(); PFN_vkGetDeviceFaultInfoEXT get = c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") diff --git a/runtime/vk_fsr.c b/runtime/vk_fsr.c index ad6fd601..b88ac12c 100644 --- a/runtime/vk_fsr.c +++ b/runtime/vk_fsr.c @@ -99,7 +99,11 @@ static void free_images(VkCtx *c) { static int ensure_images(VkCtx *c, uint32_t w, uint32_t h) { if (F.img[0] && F.w == w && F.h == h) return 1; - vkDeviceWaitIdle(c->device); // a swapchain size change: earlier presents may still read the old images + // A swapchain size change: earlier presents may still read the old images. vkDeviceWaitIdle needs every queue + // externally synchronized, so it holds the queue lock like a submission. + vk_queue_lock(); + vkDeviceWaitIdle(c->device); + vk_queue_unlock(); free_images(c); for (int i = 0; i < 2; i++) { VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 7d8192a6..36e050c1 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1657,6 +1657,7 @@ VkCommandBuffer vk_upload_begin(void) { .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(open_b.cb, &bi); open_b.seq = ++next_seq; + vk_crumb(open_b.cb, VK_CRUMB_UPLOAD, open_b.seq); open_active = 1; return open_b.cb; } @@ -1717,6 +1718,7 @@ uint64_t vk_upload_flush(int wait) { uint64_t seq = next_seq; if (c && open_active) { for (int i = 0; i < open_b.nstg; i++) vmaFlushAllocation(c->vma, open_b.stg[i].alloc, 0, VK_WHOLE_SIZE); + vk_crumb(open_b.cb, VK_CRUMB_UPLOAD_END, open_b.seq); vkEndCommandBuffer(open_b.cb); VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &open_b.cb}; vk_queue_lock(); diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 2329be22..4ebfeb79 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -12,6 +12,7 @@ #include #include +#include "cfg.h" #include "host.h" #include "vk.h" @@ -128,7 +129,13 @@ void rec_drain(void) { void rec_begin(VkCommandBuffer cb) { static int mode = -1; if (mode < 0) { - mode = rt_knob("VK_RECORDER", "1"); + // Default off on AMD GPUs: with the recorder their drivers lost the device within the first frames or at menu + // opens, 1 boot in 3 on an RX 9070 XT, and TOTK_VK_RECORDER=0 ended it for three users (issue #5); under RADV + // the kernel saw an illegal opcode at the end of a command buffer (issue #40). The cause is not found. + const char *set = cfg_get("TOTK_VK_RECORDER"); + mode = rt_knob("VK_RECORDER", vk_caps.amd ? "0" : "1"); + if (!mode && vk_caps.amd && !(set && *set)) + rt_log("vk_rec: no recorder thread on AMD GPUs (TOTK_VK_RECORDER=1 turns it on)"); if (mode) { wake_ev = CreateEventA(NULL, FALSE, FALSE, NULL); rec_thread = wake_ev ? CreateThread(NULL, 0, rec_main, NULL, 0, NULL) : NULL; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 2dd32c58..671a803e 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -314,6 +314,7 @@ VkCommandBuffer vk_sync_split(void) { VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(s->cb, &bi); + vk_crumb(s->cb, VK_CRUMB_SUBMIT, cur_fence); ts_begin(s->cb); rec_begin(s->cb); return s->cb; @@ -342,11 +343,13 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; uint64_t t0 = perf_begin(); vkBeginCommandBuffer(s->cb, &bi); + vk_crumb(s->cb, VK_CRUMB_SUBMIT, fence); ts_begin(s->cb); rec_begin(s->cb); int recorded = vk_exec_record(s->cb, handles, n, fence); s = &slots[cur_slot]; // the last part (vk_sync_split) rec_end(); + vk_crumb(s->cb, VK_CRUMB_SUBMIT_END, fence); // the recorder is done with the part if (recorded) ts_end(s->cb, fence); else ts_drop(); vkEndCommandBuffer(s->cb); diff --git a/totk.ini.example b/totk.ini.example index 1f3b11f7..30c3a2af 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -44,7 +44,7 @@ # pipe_prebuild = 1 # Threads that translate known shaders at start (1 to 64; default: half the CPU threads, at most 16). # prewarm_xlate = 8 -# Vulkan graphics pipeline library. 0 if the driver misbehaves with it. +# Vulkan graphics pipeline library, 0 or 1. Default: on, except on AMD's driver for RDNA 1 and 2 (RX 5000 and 6000). # gpl = 1 # Textures and memory. @@ -87,6 +87,8 @@ # Vulkan. # Vulkan validation layer: checks every call and slows the game down a lot. Needs the layer installed (Vulkan SDK). # vk_validation = 0 +# A thread of its own makes the Vulkan calls of each frame, 0 or 1. Default: on, except on AMD GPUs. +# vk_recorder = 1 # Dynamic vertex input, 0 or 1. Default: on, except on Intel's older integrated GPU drivers (Vulkan 1.3.215 and older). # vk_dyn_vertex = 1 # Pin the shader subgroup size to 32 when the GPU allows it. From 3b5469a83ffd1f90ec1f8f3fbe00b4ab28aa577c Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 20:09:36 +0300 Subject: [PATCH 13/50] Sample compare-sampled R32F and R16 textures of compute passes through a depth twin TotK's compute passes sample a 4x4 R32_SFLOAT texture through a shadow sampler. Vulkan allows depth comparison only on formats with SAMPLED_IMAGE_DEPTH_COMPARISON, which AMD's drivers do not give R32_SFLOAT, so the result there was undefined (20 validation errors in 40 s of field play). Such an image now gets a D32_SFLOAT (or D16_UNORM) twin copied through a buffer right before each dispatch that samples it, and the dispatch samples the twin. Formats the driver compares as they are keep the colour image. TOTK_DEPTH_TWIN=0 turns it off. Design after Savage Wind Recomp's depth twins. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 184 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 182 insertions(+), 2 deletions(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index e9326df5..59ff9cbb 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -366,10 +366,14 @@ static void dummies_init(void) { Z.ready = 1; } -// Releases the stand-ins. The device must be idle; the game path leaves them to process exit like the pipelines. +static void twins_release(VkCtx *c); + +// Releases the stand-ins and the depth twins. The device must be idle; the game path leaves them to process exit like +// the pipelines. void vk_exec_draw_shutdown(void) { if (!Z.ready) return; VkCtx *c = vk_ctx(); + twins_release(c); for (int i = 0; i < DV_COUNT; i++) { if (Z.img[i].sview != Z.img[i].view) vkDestroyImageView(c->device, Z.img[i].sview, NULL); vkDestroyImageView(c->device, Z.img[i].view, NULL); @@ -1146,7 +1150,8 @@ static struct { VkDescriptorImageInfo out; } TD[TD_N]; -static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) { +static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, + VkDescriptorImageInfo *out) { int td_on = td_mode(); VkImage image; if (!h || storage || !td_on) return tex_info_full(ex, r, h, storage, separate, out, &image); @@ -1207,6 +1212,177 @@ static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int sto *out = (VkDescriptorImageInfo){separate || storage ? VK_NULL_HANDLE : samp, view, layout}; } +// ---- depth twins: colour images under a compare sampler in a compute shader +// TotK's compute passes sample an R32_SFLOAT texture through a shadow sampler. Maxwell compares any single-channel +// float or unorm texel; Vulkan only formats with SAMPLED_IMAGE_DEPTH_COMPARISON (VUID-vkCmdDispatch-None-06479), which +// AMD's drivers do not give R32_SFLOAT, so the result there is undefined. Such an image gets a depth twin of the same +// size and bits (R32_SFLOAT -> D32_SFLOAT, R16_UNORM -> D16_UNORM), copied through a buffer right before each dispatch +// that samples it (a dispatch is outside any render pass), and the dispatch samples the twin. Draws keep the colour +// image: a copy cannot go inside their pass. Design: Savage Wind Recomp runtime/nvn/vulkan/exec.c depth_twin_update +// (https://github.com/Revivedx/SavageWindRecomp, MIT); the code is ours. TOTK_DEPTH_TWIN=0: off. +#define TWINS 8 +typedef struct { VkImage src, img; VmaAllocation mem; VkImageView view[2]; uint32_t w, h, layers; VkFormat fmt; } Twin; +static Twin twins[TWINS]; +static unsigned twin_next; +static int twin_dispatch; // op_dispatch*: the resources being bound are a dispatch's +static VkBuffer twin_buf; +static VmaAllocation twin_buf_mem; +static VkDeviceSize twin_buf_cap; + +static void twin_free(void *arg) { // completion thread, after the last submission that may sample it + Twin *t = arg; + VkCtx *c = vk_ctx(); + for (int k = 0; k < 2; k++) + if (t->view[k]) vkDestroyImageView(c->device, t->view[k], NULL); + vmaDestroyImage(c->vma, t->img, t->mem); + free(t); +} + +typedef struct { VkBuffer buf; VmaAllocation mem; } TwinBuf; +static void twin_buf_free(void *arg) { // completion thread, after the last submission that may copy through it + TwinBuf *b = arg; + vmaDestroyBuffer(vk_ctx()->vma, b->buf, b->mem); + free(b); +} + +static VkFormat twin_format(const VkTexImage *img) { + static int on = -1; + if (on < 0) on = rt_knob("DEPTH_TWIN", "1"); + if (!on || (img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) || img->samples > 1 || img->depth > 1) return VK_FORMAT_UNDEFINED; + VkFormat df = img->format == VK_FORMAT_R32_SFLOAT ? VK_FORMAT_D32_SFLOAT + : img->format == VK_FORMAT_R16_UNORM ? VK_FORMAT_D16_UNORM : VK_FORMAT_UNDEFINED; + if (!df) return df; + static int compares[2] = {-1, -1}; // per source format: the driver compares it as it is + int i = df == VK_FORMAT_D16_UNORM; + if (compares[i] < 0) { + VkFormatProperties3 p3 = {.sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3}; + VkFormatProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, .pNext = &p3}; + vkGetPhysicalDeviceFormatProperties2(vk_ctx()->phys, img->format, &p2); + compares[i] = (p3.optimalTilingFeatures & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT) != 0; + rt_log("vk_exec: %s under a compare sampler: %s", i ? "R16_UNORM" : "R32_SFLOAT", + compares[i] ? "compared by the driver" : "sampled through a depth twin in compute passes"); + } + return compares[i] ? VK_FORMAT_UNDEFINED : df; +} + +static void twins_release(VkCtx *c) { + for (int i = 0; i < TWINS; i++) { + for (int k = 0; k < 2; k++) + if (twins[i].view[k]) vkDestroyImageView(c->device, twins[i].view[k], NULL); + if (twins[i].img) vmaDestroyImage(c->vma, twins[i].img, twins[i].mem); + } + memset(twins, 0, sizeof twins); + if (twin_buf) vmaDestroyBuffer(c->vma, twin_buf, twin_buf_mem); + twin_buf = VK_NULL_HANDLE, twin_buf_mem = NULL, twin_buf_cap = 0; +} + +// The twin of img holding img's current texels, the copies recorded into ex->cb; NULL when img needs none or it cannot +// be made. +static Twin *twin_update(VkExec *ex, const VkTexImage *img) { + VkFormat df = twin_format(img); + if (!df) return NULL; + VkCtx *c = vk_ctx(); + uint32_t bpp = df == VK_FORMAT_D32_SFLOAT ? 4 : 2, layers = img->layers ? img->layers : 1; + VkDeviceSize bytes = (VkDeviceSize)img->width * img->height * layers * bpp; + if (bytes > (64u << 20)) return NULL; + if (bytes > twin_buf_cap) { // one copy buffer, grown; the old one goes after the submission being recorded + VkDeviceSize want = twin_buf_cap ? twin_buf_cap * 2 : 1u << 16; + while (want < bytes) want *= 2; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = want, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + VkBuffer nb; + VmaAllocation nm; + if (vmaCreateBuffer(c->vma, &bi, &ai, &nb, &nm, NULL) != VK_SUCCESS) return NULL; + if (twin_buf) { + TwinBuf *old = malloc(sizeof *old); + if (!old) rt_fatal("vk: out of memory (depth twin)"); + *old = (TwinBuf){twin_buf, twin_buf_mem}; + vk_completion_add(vk_sync_issued() + 1, twin_buf_free, old); + } + twin_buf = nb, twin_buf_mem = nm, twin_buf_cap = want; + } + Twin *t = NULL; + for (int i = 0; i < TWINS && !t; i++) + if (twins[i].img && twins[i].src == img->image && twins[i].w == img->width && twins[i].h == img->height && + twins[i].layers == layers && twins[i].fmt == df) + t = &twins[i]; + if (!t) { + t = &twins[twin_next++ % TWINS]; + if (t->img) { // the submission being recorded may still sample the replaced twin + Twin *old = malloc(sizeof *old); + if (!old) rt_fatal("vk: out of memory (depth twin)"); + *old = *t; + vk_completion_add(vk_sync_issued() + 1, twin_free, old); + } + *t = (Twin){.src = img->image, .w = img->width, .h = img->height, .layers = layers, .fmt = df}; + VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, .format = df, + .extent = {img->width, img->height, 1}, .mipLevels = 1, .arrayLayers = layers, + .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (vmaCreateImage(c->vma, &ci, &ai, &t->img, &t->mem, NULL) != VK_SUCCESS) { + *t = (Twin){0}; + return NULL; + } + for (int k = 0; k < 2; k++) { + VkImageViewCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = t->img, + .viewType = k ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D, .format = df, + .subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, k ? layers : 1}}; + if (vkCreateImageView(c->device, &vi, NULL, &t->view[k]) != VK_SUCCESS) t->view[k] = VK_NULL_HANDLE; + } + static int nlog; + if (nlog++ < 8) + rt_log("vk_exec: depth twin %ux%ux%u %s for a compute pass's compare sampler", img->width, img->height, + layers, df == VK_FORMAT_D32_SFLOAT ? "D32_SFLOAT" : "D16_UNORM"); + } + // The source's earlier writes and the twin's earlier readers before the copies; the twin's old texels are dropped. + VkMemoryBarrier2 m0 = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT}; + VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = t->img, .subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, layers}}; + VkDependencyInfo d0 = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &m0, + .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; + vkCmdPipelineBarrier2(ex->cb, &d0); + VkBufferImageCopy src = {.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, layers}, + .imageExtent = {img->width, img->height, 1}}; + vkCmdCopyImageToBuffer(ex->cb, img->image, VK_IMAGE_LAYOUT_GENERAL, twin_buf, 1, &src); + VkMemoryBarrier2 m1 = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT}; + VkDependencyInfo d1 = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &m1}; + vkCmdPipelineBarrier2(ex->cb, &d1); + VkBufferImageCopy dst = {.imageSubresource = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 0, layers}, + .imageExtent = {img->width, img->height, 1}}; + vkCmdCopyBufferToImage(ex->cb, twin_buf, t->img, VK_IMAGE_LAYOUT_GENERAL, 1, &dst); + // The twin's texels before the dispatch; the source's next writers and the buffer's next copy after these copies. + VkMemoryBarrier2 m2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; + VkDependencyInfo d2 = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &m2}; + vkCmdPipelineBarrier2(ex->cb, &d2); + ex->recorded += 5; + return t; +} + +// The image descriptor of slot h; in a dispatch, a compare-sampled colour image the driver cannot compare is swapped +// for its depth twin. +static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) { + tex_info_lookup(ex, r, h, storage, separate, out); + if (!twin_dispatch || storage || separate || !h || !(r->type & 0x400)) return; + VkImageViewType vt = res_view_type(r->type); + if (vt != VK_IMAGE_VIEW_TYPE_2D && vt != VK_IMAGE_VIEW_TYPE_2D_ARRAY) return; + const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL); + Twin *t = img ? twin_update(ex, img) : NULL; + VkImageView v = t ? t->view[vt == VK_IMAGE_VIEW_TYPE_2D_ARRAY] : VK_NULL_HANDLE; + if (v) out->imageView = v, out->imageLayout = VK_IMAGE_LAYOUT_GENERAL; +} + static VkDescriptorSet alloc_set(VkExec *ex, VkDescriptorSetLayout layout) { CbRes *r = cb_res(ex); VkCtx *c = vk_ctx(); @@ -2076,7 +2252,9 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { vk_exec_barrier(ex); // ends the pass; earlier writes are visible to the shader bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); c0_dim[0] = (uint32_t)d->x, c0_dim[1] = (uint32_t)d->y, c0_dim[2] = (uint32_t)d->z; + twin_dispatch = 1; bind_resources(ex, info); + twin_dispatch = 0; vkCmdDispatch(ex->cb, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z); ex->recorded++; if (vk_dump_armed) vk_dump_dispatch(ex, cs, p, info, d); @@ -2102,7 +2280,9 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); c0_dim[0] = c0_dim[1] = c0_dim[2] = 0; B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads + twin_dispatch = 1; bind_resources(ex, info); + twin_dispatch = 0; if (B.c0_valid[2]) { // the shader binds c0 VkBufferCopy cp = {ip.offset, B.c0_off[2] + 0x200, sizeof(VkDispatchIndirectCommand)}; vkCmdCopyBuffer(ex->cb, ip.buffer, B.c0_buf[2], 1, &cp); From 4d7fa271d43bca1172bb7c7df5a6b76f3ce57af7 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 20:15:39 +0300 Subject: [PATCH 14/50] Make only cube textures cube compatible Every square 2D array of 6 or more layers was created cube compatible, though the replay makes cube views only of images whose own target is a cubemap or cubemap array (vk_exec_draw.c get_view). On AMD's Windows driver the flag changes the image layout, and Savage Wind Recomp traced rainbow blocks on terrain that moved with the mip level to it (their issue #59, BotW on an RX 580 and a 7800 XT); our AMD and Intel reports show the same symptom (#42, #67, #81). The flag now goes to cube targets only, on every driver. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_texture.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 4920fa10..9631add8 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -313,7 +313,13 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { if ((d->flags & NVN_TEXTURE_FLAGS_IMAGE) && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1) usage |= VK_IMAGE_USAGE_STORAGE_BIT; VkImageCreateFlags flags = ds ? 0 : VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; // format-reinterpreting views - if (type == VK_IMAGE_TYPE_2D && layers >= 6 && w == h && samples == 1) flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT; + // Cube compatible only for cube targets: the replay makes cube views of nothing else (vk_exec_draw.c get_view checks + // the image's own view type). The flag on every square array of 6+ layers changed how AMD's Windows driver lays such + // an image out, and terrain material arrays sampled through it came out in rainbow blocks that moved with the mip + // level (Savage Wind Recomp issue #59, BotW on an RX 580 and a 7800 XT). + if (type == VK_IMAGE_TYPE_2D && layers >= 6 && w == h && samples == 1 && + (d->target == NVN_TEXTURE_TARGET_CUBEMAP || d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY)) + flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT; // NVN renders into a 3D texture slice by slice (a view's min layer is the depth plane, nvn_mtl.m), e.g. the // grading LUT of the menu background blur. Vulkan needs a 2D view of the slice for that. if (type == VK_IMAGE_TYPE_3D && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT; From 6c27e70249b63bb69c7bdf542dcd14b091522117 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 20:51:22 +0300 Subject: [PATCH 15/50] Raise the version to 0.1.6 Co-Authored-By: Claude Opus 5.5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 17e51c38..c946ee61 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.1 +0.1.6 From fb06577c237d1a9e43353e41ef6c8e4b242886c4 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 20:54:28 +0300 Subject: [PATCH 16/50] Target Windows 10 in the runtime flags so llvm-mingw sees the DPI functions llvm-mingw's headers default to Windows 7 and hide AdjustWindowRectExForDpi, GetDpiForWindow and the other per-monitor DPI functions, so the release toolchain failed on gfx_window.c and dpi_probe.c. MSYS2's headers already default to Windows 10, the port's minimum. Co-Authored-By: Claude Opus 5.5 --- tools/build_game.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/build_game.py b/tools/build_game.py index 3efcfbea..990c2072 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -76,7 +76,9 @@ SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] # Flags for generated code as LAS Recomp builds it (Makefile GENFLAGS, Windows), plus -march for the FP/SIMD lowering. GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden', '-march=x86-64-v3'] -RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3'] +# _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain) to +# Windows 7, which hides the per-monitor DPI functions gfx_window.c and dpi_probe.c call. +RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3', '-D_WIN32_WINNT=0x0A00'] # -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c) From 25b200f94c2c0a64bb9bb5a315a7fcfd8c0b9cf5 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 23:50:02 +0300 Subject: [PATCH 17/50] Draw 3D BC textures through stand-ins and let the recorder end its parts AMD driver 26.10.1 on RDNA2 cards wrote past the end of a 3D BC4 volume made BLOCK_TEXEL_VIEW_COMPATIBLE and 2D_ARRAY_COMPATIBLE, which lost the device at the logo (issue #5). 3D BC textures no longer get block views: draws into a level go to an uncompressed 2D array stand-in that is copied back at the pass end (TOTK_BLOCK_VIEWS_3D=1 restores the old path). Parts the recorder thread recorded and the submitting thread ended turned the scene green, then white, on the same driver once a submission was split. The recorder now writes the closing breadcrumb and ends the part itself (TOTK_REC_END=0 restores the old split). The recorder stays off on AMD by default. Based on the patch Pelipooper posted in issue #5; stand-ins extended to every mip level. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- runtime/cfg.c | 4 +- runtime/vk_exec.c | 156 +++++++++++++++++++++++++++++++++++++++++ runtime/vk_exec.h | 2 + runtime/vk_exec_draw.c | 1 + runtime/vk_rec.c | 37 +++++++++- runtime/vk_rec.h | 6 +- runtime/vk_sync.c | 8 +-- runtime/vk_texture.c | 13 +++- 9 files changed, 217 insertions(+), 12 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index af1c6605..28a29bf7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,7 +82,7 @@ Every guest thread is a host thread. TotK assumes Horizon's scheduler: three app - Vulkan 1.3 back end in place of MacNativeGames' Metal one. At `QueueSubmitCommands` the recorded NVN commands are replayed into Vulkan command buffers (`vk_exec*.c`). - Memory pools. Apple's unified memory let MacNativeGames use the game's pools zero-copy. The back end imports the pools as host memory (`VK_EXT_external_memory_host`). On a discrete GPU, pools bound as buffers are mirrored in device-local memory and the pages the CPU wrote (write watch, above) are uploaded before a replay; idle pool regions are made read-only so the write-watch queries skip them (`vk_memory.c`). GPU-rendered textures are written back when the CPU reads them: on a pool invalidate, on a CPU copy, and for small targets once the CPU waits for a fence (`vk_texture.c`). - Replay thread. `TOTK_REPLAY_ASYNC=1` moves the replay off the submitting thread onto a queue at most one frame behind the game; reads of GPU results wait for the newest writer instead of draining the queue (`vk_backend.c`). It is off by default because no timing has shown a gain yet. -- Recorder thread. The Vulkan calls the replay makes into its command buffer are made by a thread of their own (`vk_rec.c`, after Savage Wind Recomp and LAS Recomp): each `vkCmd*` copies its arguments into a packet of a ring and the recorder makes the calls in order; calls into other command buffers are made at once, and the submit waits until the ring is empty. In the town on an RX 9070 the replay takes 17% less time on the submitting thread. `TOTK_VK_RECORDER=0` makes every call on the submitting thread. +- Recorder thread. The Vulkan calls the replay makes into its command buffer are made by a thread of their own (`vk_rec.c`, after Savage Wind Recomp and LAS Recomp): each `vkCmd*` copies its arguments into a packet of a ring and the recorder makes the calls in order; calls into other command buffers are made at once; the recorder also ends the command buffer, and the submit waits until it has. In the town on an RX 9070 the replay takes 17% less time on the submitting thread. `TOTK_VK_RECORDER=0` makes every call on the submitting thread. - Uniform snapshot. The game rewrites per-object uniform blocks on other threads soon after it submits the frame that reads them, while the replay still records it. At the start of a submission's replay every uniform range it binds from CPU-written pools is copied, and the draws bind the copy (`vk_usnap.c`, after MacNativeGames' diff --git a/runtime/cfg.c b/runtime/cfg.c index 31eb9213..ed31672a 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -35,7 +35,7 @@ static const char *const known[] = { "BIND_LSTICK_RIGHT", "BIND_LSTICK_UP", "BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_RSTICK_DOWN", "BIND_RSTICK_LEFT", "BIND_RSTICK_RIGHT", "BIND_RSTICK_UP", "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", - "BIND_ZL", "BIND_ZR", "CLOTH_REALTIME", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", + "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FPS_LIMIT", "FREEZE_IDLE", "FREE_CORES", "FULLSCREEN", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", @@ -43,7 +43,7 @@ static const char *const known[] = { "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PATCH_OUTPUT", "PATCH_SCENE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", - "PROFILE_MAIN", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", + "PROFILE_MAIN", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index d82c0574..258b38ed 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -23,6 +23,7 @@ #include "host.h" #include "nvn_api.h" #include "nvn_tex.h" +#include "platform/platform.h" #include "tex.h" #include "vk.h" #include "vk_exec.h" @@ -141,11 +142,163 @@ static int view_format(const tex_t *t, const view_t *v, int blocks, VkFormat *ou return 1; } +// ---- 3D BC render targets +// A 3D BC texture has no block views (vk_texture.c create_tex, TOTK_BLOCK_VIEWS_3D): on RDNA2 cards such an image +// made BLOCK_TEXEL_VIEW_COMPATIBLE and 2D_ARRAY_COMPATIBLE took GPU writes past the end of its memory (issue #5). Draws +// into it through an uncompressed view of its blocks (agl's BC4 encoder fills 32x32x32 and 64x64x64 volumes and their +// levels at the logo) go to a stand-in: a 2D array of the blocks of one level in the view's format, one layer per +// depth plane. It starts with the level's content, so a pass that writes part of a plane keeps the rest, and each pass +// end copies the planes it drew into the level (a copy between a compressed and a size-compatible uncompressed format). +typedef struct { VkImage bc, img; VmaAllocation alloc; VkFormat fmt; uint32_t level, w, h, d; } BcStandIn; +#define BC_STANDINS 64 // one per level drawn: 6 volumes of up to 7 levels at the logo +static BcStandIn bcs[BC_STANDINS]; +static int nbcs; +static PlatMutex bcs_lock = PLAT_MUTEX_INIT; // the replay adds and reads; destroy_tex (any thread) removes + +static void image_barrier(VkCommandBuffer cb, VkImage img, VkImageLayout from, uint32_t layers) { + VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_NONE, + .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .oldLayout = from, .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = img, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, layers}}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; + vkCmdPipelineBarrier2(cb, &di); +} + +static void memory_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst, + VkAccessFlags2 da) { + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa, + .dstStageMask = dst, .dstAccessMask = da}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); +} + +// The stand-in of level level of a 3D BC image for draws through view format fmt, made on first use; 0 when none can +// be made (logged). Replay thread. +static int bc_standin(const VkTexImage *img, uint32_t level, VkFormat fmt, BcStandIn *out) { + static int warned; + plat_mutex_lock(&bcs_lock); + int full = nbcs == BC_STANDINS; + for (int i = 0; i < nbcs; i++) + if (bcs[i].bc == img->image && bcs[i].level == level && bcs[i].fmt == fmt) { + *out = bcs[i]; + plat_mutex_unlock(&bcs_lock); + return 1; + } + plat_mutex_unlock(&bcs_lock); + if (full) { + if (!warned++) rt_log("vk: more than %d 3D BC render targets: the others are left out", BC_STANDINS); + return 0; + } + uint32_t lw = level_extent(img->width, level), lh = level_extent(img->height, level); + BcStandIn s = {img->image, VK_NULL_HANDLE, NULL, fmt, level, (lw + 3) / 4, (lh + 3) / 4, + level_extent(img->depth, level)}; + VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, .format = fmt, + .extent = {s.w, s.h, 1}, .mipLevels = 1, .arrayLayers = s.d, .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | + VK_IMAGE_USAGE_TRANSFER_DST_BIT, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (vmaCreateImage(vk_ctx()->vma, &ci, &ai, &s.img, &s.alloc, NULL) != VK_SUCCESS) { + if (!warned++) rt_log("vk: stand-in for a 3D BC render target could not be created: left out"); + return 0; + } + // In the upload batch, which goes to the queue before this part: GENERAL, then the texture's blocks. + VkCommandBuffer cb = vk_upload_begin(); + image_barrier(cb, s.img, VK_IMAGE_LAYOUT_UNDEFINED, s.d); + memory_barrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + VkImageCopy rg = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1}, + .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, s.d}, + .extent = {lw, lh, s.d}}; // texels of the compressed source + vkCmdCopyImage(cb, img->image, VK_IMAGE_LAYOUT_GENERAL, s.img, VK_IMAGE_LAYOUT_GENERAL, 1, &rg); + memory_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + vk_upload_end(); + plat_mutex_lock(&bcs_lock); + bcs[nbcs++] = s; + plat_mutex_unlock(&bcs_lock); + rt_log("vk: 3D BC render target %ux%ux%u level %u drawn through a %ux%u x %u stand-in (format %d)", img->width, + img->height, img->depth, level, s.w, s.h, s.d, (int)fmt); + *out = s; + return 1; +} + +// vk_exec_draw.c vk_exec_image_gone: the stand-in of a BC image that goes, freed once the GPU is done with it. +void vk_exec_bc_gone(VkImage image) { + plat_mutex_lock(&bcs_lock); + for (int i = 0; i < nbcs; i++) + if (bcs[i].bc == image) { + vk_defer_image(bcs[i].img, bcs[i].alloc); + bcs[i--] = bcs[--nbcs]; + } + plat_mutex_unlock(&bcs_lock); +} + +// At a pass end: the planes the pass drew into a stand-in go to its texture. +static void bc_copy_back(VkExec *ex, const VkExecTarget *t) { + plat_mutex_lock(&bcs_lock); + for (int i = 0; i < nbcs; i++) { + if (bcs[i].img != t->image) continue; + memory_barrier(ex->cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + VkImageCopy rg = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, t->base_layer, t->layers}, + .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, bcs[i].level, 0, 1}, + .dstOffset = {0, 0, (int32_t)t->base_layer}, + .extent = {bcs[i].w, bcs[i].h, t->layers}}; // texels of the uncompressed source + vkCmdCopyImage(ex->cb, t->image, VK_IMAGE_LAYOUT_GENERAL, bcs[i].bc, VK_IMAGE_LAYOUT_GENERAL, 1, &rg); + memory_barrier(ex->cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + ex->recorded++; + break; + } + plat_mutex_unlock(&bcs_lock); +} + +// A draw into a 3D BC image without block views through an uncompressed view of its blocks: the target is the +// image's stand-in (bc_standin). 1 when o is filled (target or left out), 0 for any other target. +static int bc_target(const tex_t *t, const view_t *v, int has_v, const VkTexImage *img, VkExecTarget *o) { + int bw, bh; + if (img->type != VK_IMAGE_TYPE_3D || img->block_views || !nvn_fmt_bpb(t->d.format, &bw, &bh) || bw != 4 || bh != 4) + return 0; + VkFormat fmt; + BcStandIn s; + uint32_t level = has_v && v->set_levels ? v->base_level : 0; + if (level >= img->levels) { + warn_once(W_LEVEL, "render target level %u of a %u-level texture: left out", level, img->levels); + return 1; + } + if (!has_v || !v->set_format || !view_format(t, v, 1, &fmt) || !bc_standin(img, level, fmt, &s)) { + warn_once(W_BLOCK_VIEW, "render target format %#x (%ux%u) through view format %#x: no block view, left out", + t->d.format, t->d.width, t->d.height, has_v && v->set_format ? v->format : 0); + return 1; + } + uint32_t base = 0, layers = s.d; + if (v->set_layers) { + base = v->min_layer; + layers = v->nlayers ? v->nlayers : (base < s.d ? s.d - base : 0); + } + if (base >= s.d || !layers) return 1; + if (base + layers > s.d) layers = s.d - base; + o->image = s.img; // not img->image: view_of makes this command buffer's view of the stand-in + o->img = img; + o->format = fmt; + o->view_type = layers > 1 ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D; + o->aspect = VK_IMAGE_ASPECT_COLOR_BIT; + o->level = 0, o->base_layer = base, o->layers = layers; + o->width = s.w, o->height = s.h, o->samples = 1; + o->nvn_format = v->format; + return 1; +} + static void resolve_target(const tex_t *t, const view_t *v, int has_v, int depth_slot, VkExecTarget *o) { memset(o, 0, sizeof *o); o->tex = t; if (!t) return; const VkTexImage *img = vk_texture_get(t); + if (img && !depth_slot && bc_target(t, v, has_v, img, o)) return; // a 3D BC image: its stand-in VkImageUsageFlags need = depth_slot ? VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT : VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; int is3d = img && img->type == VK_IMAGE_TYPE_3D && !depth_slot; // slices drawn through 2D views (2D_ARRAY_COMPATIBLE) if (!img || !(img->usage & need) || (img->type != VK_IMAGE_TYPE_2D && !is3d)) { @@ -469,6 +622,9 @@ void vk_exec_pass_end(VkExec *ex) { vkCmdEndRendering(ex->cb); ex->recorded++; ex->in_pass = 0; + for (int i = 0; i < ex->pass_ncolor && i < 8; i++) // stand-ins of 3D BC targets go to their textures + if (ex->color[i].image && ex->color[i].img && ex->color[i].img->image != ex->color[i].image) + bc_copy_back(ex, &ex->color[i]); if (vk_dump_armed) vk_dump_pass_end(ex); uint64_t stamp = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1; for (int i = 0; i < 8; i++) diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index c81d5c61..6236929e 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -124,6 +124,8 @@ void vk_exec_inline_end(int reopen); // buffers): the replay continues in a new one (vk_sync_split). void vk_exec_draw_new_cb(void); VkBuffer vk_exec_inline_buffer(VkDeviceSize size); +// vk_exec.c: the stand-in a 3D BC render target is drawn through goes with its image (vk_exec_image_gone). +void vk_exec_bc_gone(VkImage image); // Scissor 0 clipped to the render area of the open pass; extent 0: nothing to draw. VkRect2D vk_exec_scissor(const VkExec *ex); void vk_exec_noatt_dims(const VkExec *ex, uint32_t *w, uint32_t *h); // pass size when no attachment is set diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 59ff9cbb..436ba0a2 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -564,6 +564,7 @@ VkImageView vk_exec_target_view(const VkTexImage *img, VkFormat format, VkImageV } void vk_exec_image_gone(VkImage image) { + vk_exec_bc_gone(image); if (!views_by_image.klen) return; for (ViewNode *n = nvn_hm_del(&views_by_image, &image), *next; n; n = next) { next = n->next; diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 4ebfeb79..6ccd30d3 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -38,7 +38,7 @@ enum { R_SetDepthWriteEnable, R_SetFrontFace, R_SetPrimitiveRestartEnable, R_SetPrimitiveTopology, R_SetRasterizerDiscardEnable, R_SetScissorWithCount, R_SetStencilCompareMask, R_SetStencilOp, R_SetStencilReference, R_SetStencilTestEnable, R_SetStencilWriteMask, R_SetViewportWithCount, R_WriteTimestamp, R_WriteTimestamp2, - R_PushTpl, R_Push, R_VertexInput, R_Bool, R_PolygonMode, R_BlendEnable, R_BlendEquation, R_WriteMask, + R_PushTpl, R_Push, R_VertexInput, R_Bool, R_PolygonMode, R_BlendEnable, R_BlendEquation, R_WriteMask, R_EndCb, }; static uint64_t now_ns(void) { @@ -131,7 +131,8 @@ void rec_begin(VkCommandBuffer cb) { if (mode < 0) { // Default off on AMD GPUs: with the recorder their drivers lost the device within the first frames or at menu // opens, 1 boot in 3 on an RX 9070 XT, and TOTK_VK_RECORDER=0 ended it for three users (issue #5); under RADV - // the kernel saw an illegal opcode at the end of a command buffer (issue #40). The cause is not found. + // the kernel saw an illegal opcode at the end of a command buffer (issue #40). Letting the recorder end its own + // parts (rec_end_cb) fixed the white scene on an RX 6700 XT; the default stays off until more AMD cards run it. const char *set = cfg_get("TOTK_VK_RECORDER"); mode = rt_knob("VK_RECORDER", vk_caps.amd ? "0" : "1"); if (!mode && vk_caps.amd && !(set && *set)) @@ -162,6 +163,32 @@ void rec_end(void) { rec_live = 0; } +// TOTK_REC_END (default 1): the thread that recorded the part also ends it, after the breadcrumb crumb (kind 0: none) +// that vk_sync.c writes at the end of a submission. On an RX 6700 XT (driver 26.10.1, issue #5) parts the recorder +// recorded and the submitting thread ended broke when a submission was split (vk_sync.c vk_sync_split): the GPU hung +// or the scene turned green, then white. The handoff in rec_drain orders the two threads, so Vulkan allows the old +// split; the driver did not cope with it. TOTK_REC_END=0 brings the old split back. +typedef struct { int kind; uint64_t n; } A_EndCb; +int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n) { + static int mode = -1; + if (mode < 0) { + mode = rt_knob("REC_END", "1"); + rt_log("vk_rec: a part the recorder records is ended by %s (TOTK_REC_END=%d)", + mode ? "the recorder" : "the submitting thread", mode); + } + if (!rec_live || cb != rec_cb || !mode) { + rec_end(); + if (crumb) vk_crumb(cb, crumb, n); + return 0; + } + A_EndCb *a = put(R_EndCb, sizeof *a); + *a = (A_EndCb){crumb, n}; + pub(); + rec_drain(); + rec_live = 0; + return 1; +} + void rec_stats_log(void) { if (!(rt_trace & RT_TRACE_PERF) || !RS_.parts) return; rt_log("vk_rec: %llu of %llu parts recorded by the recorder: %llu packets (%.1f MiB), %llu drains (%llu for a call " @@ -588,6 +615,12 @@ static void exec_packet(uint32_t op, void *p) { case R_BlendEnable: { A_Targets *a = p; ((PFN_vkCmdSetColorBlendEnableEXT)a->fn)(cb, a->first, a->n, a->v); break; } case R_BlendEquation: { A_Targets *a = p; ((PFN_vkCmdSetColorBlendEquationEXT)a->fn)(cb, a->first, a->n, a->v); break; } case R_WriteMask: { A_Targets *a = p; ((PFN_vkCmdSetColorWriteMaskEXT)a->fn)(cb, a->first, a->n, a->v); break; } + case R_EndCb: { + A_EndCb *a = p; + if (a->kind) vk_crumb(cb, a->kind, a->n); + vkEndCommandBuffer(cb); + break; + } default: rt_fatal("vk_rec: unknown packet %u", op); } } diff --git a/runtime/vk_rec.h b/runtime/vk_rec.h index afbb2937..d4054e1e 100644 --- a/runtime/vk_rec.h +++ b/runtime/vk_rec.h @@ -4,7 +4,8 @@ // ring, in call order, and the recorder makes the real call in that order; calls into any other command buffer (uploads, // the window, probes) are made at once. A call the ring has no packet for (a structure with a pNext chain) waits for // the recorder to empty the ring and is then made at once on the calling thread. rec_end returns once the recorder made -// every call, so ending, submitting and the command pool stay on the submitting thread. +// every call. vk_sync.c ends a part through rec_end_cb: the recorder makes the last calls into a part it recorded, +// vkEndCommandBuffer included; submitting and the command pool stay on the submitting thread. // // Every vkCmd* the runtime calls through vk.h goes through the rc_* wrappers below (the macros at the end); the // extension commands called through function pointers have rec_* wrappers of their own. TOTK_VK_RECORDER=0 makes every @@ -16,6 +17,9 @@ void rec_begin(VkCommandBuffer cb); // the submitting thread, sub_lock held void rec_end(void); // waits for the recorder; no-op when rec_begin did not start it +// The end of part cb: the breadcrumb crumb (0: none, vk_crumbs.c), then vkEndCommandBuffer, both made by the recorder +// when it records cb, and the wait for it (1); otherwise rec_end and the breadcrumb, and the caller ends cb (0). +int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n); void rec_drain(void); // waits until the recorder made every call queued so far void rec_stats_log(void); // TOTK_TRACE=perf: packets, bytes, drains and waits since the last call diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 671a803e..6fb05a5b 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -293,9 +293,8 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { VkCommandBuffer vk_sync_split(void) { VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; - rec_end(); // every command of the part is in cb ts_end(cb, cur_fence); - vkEndCommandBuffer(cb); + if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder vk_upload_flush(0); // the uploads this part asked for precede it in queue order VkCommandBuffer pro = prologue(c, cur_fence); VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, @@ -348,11 +347,10 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { rec_begin(s->cb); int recorded = vk_exec_record(s->cb, handles, n, fence); s = &slots[cur_slot]; // the last part (vk_sync_split) - rec_end(); - vk_crumb(s->cb, VK_CRUMB_SUBMIT_END, fence); // the recorder is done with the part if (recorded) ts_end(s->cb, fence); else ts_drop(); - vkEndCommandBuffer(s->cb); + // The recorder is done with the part: the end-of-submission crumb, then the end, by the recorder when it records. + if (!rec_end_cb(s->cb, VK_CRUMB_SUBMIT_END, fence)) vkEndCommandBuffer(s->cb); perf_replay(t0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. vk_upload_flush(0); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 9631add8..f12e7934 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -328,7 +328,18 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { .samples = (VkSampleCountFlagBits)samples, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = usage, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; int bw, bh, block_views = 0; - if (!ds && samples == 1 && f.vk == f.ideal && nvn_fmt_bpb(d->format, &bw, &bh) && bw == 4 && bh == 4) { + // TOTK_BLOCK_VIEWS_3D (default 0): a 3D BC image also gets block views. On RDNA2 cards (driver 26.10.1, issue #5) + // the 32x32x32 BC4 volume the game creates at the logo, made BLOCK_TEXEL_VIEW_COMPATIBLE and 2D_ARRAY_COMPATIBLE, + // took a GPU write just past the end of its memory: device lost at the first submit after it when the image ended + // its memory block. Draws into a 3D BC image go through a stand-in instead (vk_exec.c bc_target). + static int bv3d = -1; + if (bv3d < 0) { + bv3d = rt_knob("BLOCK_VIEWS_3D", "0"); + rt_log("vk: 3D BC textures %s (TOTK_BLOCK_VIEWS_3D=%d)", bv3d ? "get block views (renderable)" : "have no block views", + bv3d); + } + if (!ds && samples == 1 && f.vk == f.ideal && nvn_fmt_bpb(d->format, &bw, &bh) && bw == 4 && bh == 4 && + (type != VK_IMAGE_TYPE_3D || bv3d)) { VkImageCreateInfo bc = ci; // BC formats: renderable through views of their blocks (block_views_ok) bc.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; bc.flags |= VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; From ed1973a68b449c5712c2da1eaec785605c75c11f Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 31 Oct 2026 00:05:54 +0300 Subject: [PATCH 18/50] Add a stick dead zone setting for drifting gamepads stick_deadzone (0 to 50 percent, default 8) replaces the fixed per-axis cutoff of 2500. The zone is round and the travel past it is scaled to start at 0, so a drifting stick rests centred and full deflection still reaches full. The Controls tab shows it as a slider; --probe-cfg checks the math (input_pad_stick). Co-Authored-By: Claude Opus 5.5 --- docs/controls.md | 2 ++ runtime/cfg.c | 6 +++--- runtime/cfg_probe.c | 11 +++++++++++ runtime/input_map.c | 9 +++++++++ runtime/input_map.h | 4 ++++ runtime/input_win.c | 27 +++++++++++++++++++-------- runtime/ui/menu.cpp | 1 + runtime/ui_text.c | 4 ++++ runtime/ui_text.h | 1 + totk.ini.example | 3 +++ 10 files changed, 57 insertions(+), 11 deletions(-) diff --git a/docs/controls.md b/docs/controls.md index 00f5a5d1..b80fc89c 100644 --- a/docs/controls.md +++ b/docs/controls.md @@ -68,6 +68,8 @@ Mouse motion (raw input) becomes right-stick deflection. Each presented frame ta `TOTK_MOUSE_SENS` (0 to 10, default 1; higher turns faster, 0 turns the mouse camera and the cursor capture off). While the window is the foreground window the cursor is hidden and confined to the client area. It is released on focus loss, minimise, while F11 changes the window, during a size or move loop and while a text entry is open. Esc releases it by hand: the cursor shows, the mouse camera and mouse buttons stop, and the window can be moved, minimised or closed. A mouse click inside the client area captures it again; that click does not reach the game. Activating the window by a click on its title bar or border does not capture, so it can be dragged or closed right away; Alt+Tab or a click in the client area captures as before. `TOTK_INPUT=none` disables the capture too. +`TOTK_STICK_DEADZONE` (0 to 50, default 8) is the gamepad stick dead zone in percent of full deflection, the same for both sticks. It is round: a stick inside the circle reads centred, and the rest of its travel is scaled to start at 0 at the edge, so full deflection still reaches full. Raise it for a stick that drifts. The game's own dead zone applies after it. + The stick direction bits (`StickLLeft` to `StickRDown`) are derived from the sticks at 50% deflection. A gamepad plugged in later is picked up within one second; with none connected the log says `input: no gamepad`. ## Scripted input diff --git a/runtime/cfg.c b/runtime/cfg.c index 31eb9213..322917ab 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -47,7 +47,7 @@ static const char *const known[] = { "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", - "SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "SWKBD_ANSWER", + "SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WB_FENCE_LIMIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -63,7 +63,7 @@ static const struct { const char *key, *section; } sections[] = { {"MENU_LANG", "language"}, {"LANG", "language"}, {"OPERATION_MODE", "language"}, {"VOLUME", "audio"}, {"MOTION", "input"}, {"RUMBLE_STRENGTH", "input"}, {"RUMBLE_CURVE", "input"}, {"RUMBLE_TRIGGERS", "input"}, - {"PAD_LAYOUT", "input"}, {"MOUSE_SENS", "input"}, {"INPUT_RECORD", "input"}, + {"PAD_LAYOUT", "input"}, {"STICK_DEADZONE", "input"}, {"MOUSE_SENS", "input"}, {"INPUT_RECORD", "input"}, {"BIND_A", "input"}, {"BIND_B", "input"}, {"BIND_X", "input"}, {"BIND_Y", "input"}, {"BIND_LCLICK", "input"}, {"BIND_RCLICK", "input"}, {"BIND_L", "input"}, {"BIND_R", "input"}, {"BIND_ZL", "input"}, {"BIND_ZR", "input"}, {"BIND_PLUS", "input"}, {"BIND_MINUS", "input"}, {"BIND_LEFT", "input"}, {"BIND_UP", "input"}, @@ -104,7 +104,7 @@ static const CfgMeta metas[] = { {"VOLUME", "100", "0..100", LIVE}, {"MOTION", "1", "0|1", LIVE}, {"RUMBLE_STRENGTH", "130", "0..300", LIVE}, {"RUMBLE_CURVE", "0.6", "0.2..1.5", LIVE}, {"RUMBLE_TRIGGERS", "0", "0|1", LIVE}, {"PAD_LAYOUT", "auto", "auto|nintendo|xbox", LIVE}, - {"MOUSE_SENS", "1", "0..10", LIVE}, + {"STICK_DEADZONE", "8", "0..50", LIVE}, {"MOUSE_SENS", "1", "0..10", LIVE}, {"MENU_LANG", "", "en|ru", LIVE}, // Advanced: scheduling {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, diff --git a/runtime/cfg_probe.c b/runtime/cfg_probe.c index 3e4ee435..0fc40a42 100644 --- a/runtime/cfg_probe.c +++ b/runtime/cfg_probe.c @@ -261,6 +261,17 @@ int probe_cfg(void) { CHECK((int64_t)st[0] * st[0] + (int64_t)st[1] * st[1] <= 32768ll * 32768, "mouse: clamped to the unit circle"); input_mouse_stick(30, 0, 1.0 / 60, 0, st); CHECK(st[0] == 0 && st[1] == 0, "mouse: sensitivity 0 is off"); + input_pad_stick(3000, -2000, 0.15f, st); + CHECK(st[0] == 0 && st[1] == 0, "pad stick: an 11% drift inside a 15% dead zone reads centred"); + input_pad_stick(4000, -3000, 0, st); + CHECK(st[0] == 4000 && st[1] == -3000, "pad stick: dead zone 0 passes the stick unchanged"); + input_pad_stick(32767, 0, 0.15f, st); + CHECK(st[0] == 32767 && st[1] == 0, "pad stick: full deflection still reaches full"); + input_pad_stick(-32768, -32768, 0.15f, st); + CHECK(st[0] < 0 && st[1] < 0 && st[0] >= -32767 && (int64_t)st[0] * st[0] + (int64_t)st[1] * st[1] <= 32768ll * 32768, + "pad stick: a square gate's corner stays on the unit circle"); + input_pad_stick(0, 32767 / 2 + 1, 0.2f, st); + CHECK(abs(st[1] - 32767 * 3 / 8) <= 3, "pad stick: half deflection past a 20% zone is (0.5 - 0.2) / 0.8 of full"); // Gamepad layouts (input_map.c) CHECK(input_pad_layout_pick("auto", 1) == PAD_LAYOUT_NINTENDO && input_pad_layout_pick(NULL, 0) == PAD_LAYOUT_XBOX, "pad layout: automatic is Nintendo for an east button labelled A, Xbox otherwise"); diff --git a/runtime/input_map.c b/runtime/input_map.c index 7874e7e1..5c5e06c8 100644 --- a/runtime/input_map.c +++ b/runtime/input_map.c @@ -325,6 +325,15 @@ void input_mouse_stick(int32_t dx, int32_t dy, double dt_s, float sens, int32_t out[1] = (int32_t)lrint(y); } +void input_pad_stick(int32_t x, int32_t y, float dead, int32_t out[2]) { + double fx = x / (double)STICK_MAX, fy = y / (double)STICK_MAX, m = sqrt(fx * fx + fy * fy); + out[0] = out[1] = 0; + if (m <= dead) return; + double k = ((m > 1 ? 1 : m) - dead) / (1 - dead) / m; // past the edge (a square gate's corner) the length is 1 + out[0] = (int32_t)lrint(fmax(-1, fmin(1, fx * k)) * STICK_MAX); + out[1] = (int32_t)lrint(fmax(-1, fmin(1, fy * k)) * STICK_MAX); +} + // Minus and Plus held together open or close the menu. *held: latched from the poll where both go down until neither is // down (the caller keeps it). Every poll of the latch reports neither to the game, so releasing one button first does not // leak the other; Minus alone before Plus reaches the game unchanged. With the menu open the game sees a neutral pad. diff --git a/runtime/input_map.h b/runtime/input_map.h index c4c4425b..fe524fa6 100644 --- a/runtime/input_map.h +++ b/runtime/input_map.h @@ -91,6 +91,10 @@ void input_map_stick(const uint8_t down[KEY_COUNT], int stick, int32_t out[2]); // Mouse motion in raw counts over dt_s seconds to right-stick deflection: full deflection at 3000 / sens counts/s. // Clamped to the unit circle, up (negative dy) is +y. void input_mouse_stick(int32_t dx, int32_t dy, double dt_s, float sens, int32_t out[2]); +// A gamepad stick (x, y in -32768..32767, up is +y) through a round dead zone of `dead` (fraction of full deflection, +// 0..0.5): inside the circle the stick reads 0, and the rest of the travel is scaled to start at 0 at the edge, so a +// drifting stick rests at 0 and full deflection still reaches full. The result stays inside the unit square. +void input_pad_stick(int32_t x, int32_t y, float dead, int32_t out[2]); // ---- the remapping rows' edits, on a copy of input_map. Pure. // action's keys (table order) / its gamepad buttons in layout, at most max; returns how many. diff --git a/runtime/input_win.c b/runtime/input_win.c index 61bb7dea..096fef53 100644 --- a/runtime/input_win.c +++ b/runtime/input_win.c @@ -19,7 +19,7 @@ #include "ui/menu.h" #include "vk.h" -enum { STICK_MAX = 32767, STICK_DEAD = 2500, STICK_DIR = 16384, TRIGGER_ON = 16384, RESCAN_MS = 1000 }; +enum { STICK_MAX = 32767, STICK_DIR = 16384, TRIGGER_ON = 16384, RESCAN_MS = 1000 }; static SRWLOCK lock = SRWLOCK_INIT; static int state; // 0 not started, 1 running, -1 disabled or SDL failed @@ -428,9 +428,21 @@ static void mouse_camera(int32_t out[2]) { out[1] = held[1]; } -static int32_t axis(int16_t v) { - int32_t a = v < -STICK_MAX ? -STICK_MAX : v; - return a > -STICK_DEAD && a < STICK_DEAD ? 0 : a; +static float stick_dead = -1; // TOTK_STICK_DEADZONE as a fraction of full deflection; live (cfg_version) +static unsigned stick_dead_seen; + +static float stick_dead_get(void) { + unsigned ver = cfg_version(); + if (stick_dead >= 0 && ver == stick_dead_seen) return stick_dead; + stick_dead_seen = ver; + double v = cfg_double("TOTK_STICK_DEADZONE"); + stick_dead = (float)(v < 0 ? 0 : v > 50 ? 50 : v) / 100.f; + return stick_dead; +} + +// One gamepad stick through the dead zone; SDL's Y grows downwards, the Switch's upwards. +static void stick(SDL_GamepadAxis ax, SDL_GamepadAxis ay, float dead, int32_t out[2]) { + input_pad_stick(SDL_GetGamepadAxis(pad, ax), -(int32_t)SDL_GetGamepadAxis(pad, ay), dead, out); } // Open the first gamepad when none is open or the open one went away; logs once per change. @@ -545,10 +557,9 @@ static int pad_read(uint64_t *b, int32_t sl[2], int32_t sr[2]) { uint8_t pd[PAD_COUNT] = {0}; if (!pad_physical(pd)) return 0; *b |= input_map_pad_buttons(pd, pad_layout_get()); - sl[0] = axis(SDL_GetGamepadAxis(pad, SDL_GAMEPAD_AXIS_LEFTX)); - sl[1] = -axis(SDL_GetGamepadAxis(pad, SDL_GAMEPAD_AXIS_LEFTY)); - sr[0] = axis(SDL_GetGamepadAxis(pad, SDL_GAMEPAD_AXIS_RIGHTX)); - sr[1] = -axis(SDL_GetGamepadAxis(pad, SDL_GAMEPAD_AXIS_RIGHTY)); + float dead = stick_dead_get(); + stick(SDL_GAMEPAD_AXIS_LEFTX, SDL_GAMEPAD_AXIS_LEFTY, dead, sl); + stick(SDL_GAMEPAD_AXIS_RIGHTX, SDL_GAMEPAD_AXIS_RIGHTY, dead, sr); return 1; } diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 7d52baa9..99466940 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -573,6 +573,7 @@ static void tab_controls(float scale) { opt_slider("TOTK_RUMBLE_STRENGTH", UI_MENU_RUMBLE_STRENGTH, 0, 300, UI_MENU_RUMBLE_STRENGTH_DESC, 10, "%.0f %%"); opt_slider("TOTK_RUMBLE_CURVE", UI_MENU_RUMBLE_CURVE, 0.2f, 1.5f, UI_MENU_RUMBLE_CURVE_DESC, 0.05f, "%.2f"); opt_toggle("TOTK_RUMBLE_TRIGGERS", UI_MENU_RUMBLE_TRIGGERS, UI_MENU_RUMBLE_TRIGGERS_DESC); + opt_slider("TOTK_STICK_DEADZONE", UI_MENU_STICK_DEADZONE, 0, 50, UI_MENU_STICK_DEADZONE_DESC, 1, "%.0f %%"); opt_slider("TOTK_MOUSE_SENS", UI_MENU_MOUSE_SENS, 0, 10, UI_MENU_MOUSE_SENS_DESC, 0.1f, "%.1f"); binds_rows(scale); opt_end(); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 0de0818f..b17ebb02 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -97,6 +97,8 @@ static const char *const text_en[UI_TEXT_COUNT] = { "Nintendo for controllers with Nintendo labels and Xbox for the others.", [UI_MENU_PAD_LAYOUT_AUTO] = "Automatic", [UI_MENU_PAD_LAYOUT_NINTENDO] = "Nintendo (A right)", [UI_MENU_PAD_LAYOUT_XBOX] = "Xbox (A bottom)", [UI_MENU_PAD_LAYOUT_IN_USE] = "in use: %s", + [UI_MENU_STICK_DEADZONE] = "Stick dead zone", + [UI_MENU_STICK_DEADZONE_DESC] = "How far a gamepad stick moves before the game sees it. Raise it if the character or the camera moves on its own.", [UI_MENU_MOUSE_SENS] = "Mouse camera sensitivity", [UI_MENU_MOUSE_SENS_DESC] = "How fast the mouse turns the camera.", @@ -315,6 +317,8 @@ static const char *const text_ru[UI_TEXT_COUNT] = { "Nintendo для контроллеров с подписями Nintendo, Xbox для остальных.", [UI_MENU_PAD_LAYOUT_AUTO] = "Автоматически", [UI_MENU_PAD_LAYOUT_NINTENDO] = "Nintendo (A справа)", [UI_MENU_PAD_LAYOUT_XBOX] = "Xbox (A снизу)", [UI_MENU_PAD_LAYOUT_IN_USE] = "сейчас: %s", + [UI_MENU_STICK_DEADZONE] = "Мёртвая зона стиков", + [UI_MENU_STICK_DEADZONE_DESC] = "Насколько нужно отклонить стик геймпада, чтобы игра это заметила. Увеличьте, если персонаж или камера двигаются сами.", [UI_MENU_MOUSE_SENS] = "Чувствительность мыши", [UI_MENU_MOUSE_SENS_DESC] = "Как быстро мышь поворачивает камеру.", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 96e10618..24975758 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -57,6 +57,7 @@ typedef enum { UI_MENU_RUMBLE_TRIGGERS, UI_MENU_RUMBLE_TRIGGERS_DESC, UI_MENU_PAD_LAYOUT, UI_MENU_PAD_LAYOUT_DESC, UI_MENU_PAD_LAYOUT_AUTO, UI_MENU_PAD_LAYOUT_NINTENDO, UI_MENU_PAD_LAYOUT_XBOX, UI_MENU_PAD_LAYOUT_IN_USE, + UI_MENU_STICK_DEADZONE, UI_MENU_STICK_DEADZONE_DESC, UI_MENU_MOUSE_SENS, UI_MENU_MOUSE_SENS_DESC, // Controls tab: the remapping rows (ui/binds_page.cpp) diff --git a/totk.ini.example b/totk.ini.example index 30c3a2af..522e247b 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -150,6 +150,9 @@ # west X, north Y, as an Xbox pad labels them; cross A on a PlayStation pad) or auto (nintendo for a pad with Nintendo # labels, xbox for the others). The bind_ lines below apply on top. Changes while the game runs take effect at once. # pad_layout = auto +# Gamepad stick dead zone in percent of full deflection (0 to 50). Raise it for a stick that drifts: inside the circle +# the stick reads centred, and the rest of its travel is scaled to start there. +# stick_deadzone = 8 # Mouse camera sensitivity (0 to 10; 0 = no mouse camera and no cursor capture). # mouse_sens = 1 # Write the input of this run to a file that replays as a script (docs/controls.md, "Scripted input"). Default: off. From 42a1d32710b290d55e56e68f8986df7d4c217156 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 31 Oct 2026 00:20:41 +0300 Subject: [PATCH 19/50] Skip audio that piles up in the output ring so the sound delay stays short A stalled audio device (a wireless headset waking up, a change of the default device) filled the 340 ms ring, the render thread then dropped new frames and the queue stayed full for the rest of the session: about a third of a second of delay on top of the device's own. ring_pull now skips what is queued beyond the prefill, the device's request and 40 ms. The 5 s level line (trace = audio) reports the queue and the skipped frames. With the SDL audio thread suspended for 1.5 s: before, 325-340 ms queued for the rest of the run; after, 10 ms. Co-Authored-By: Claude Opus 5.5 --- runtime/audio_mixer.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/runtime/audio_mixer.c b/runtime/audio_mixer.c index bbb29ae8..a34885c7 100644 --- a/runtime/audio_mixer.c +++ b/runtime/audio_mixer.c @@ -9,6 +9,9 @@ // single-producer single-consumer ring. The SDL audio thread drains that ring without taking any lock: an empty ring // plays silence and never blocks the render thread or the guest. TOTK_AUDIO=none (or TOTK_AUDIO_OFF) mixes without a // device; TOTK_AUDIO_PREFILL_MS (0..500, default 20) is the queued audio the output waits for after start or underrun. +// The render thread runs on the host clock and the device on its own, and after a stall the render thread catches up in +// a burst, so audio can pile up in the ring; the SDL thread skips what is queued beyond the prefill plus QUEUE_SLACK_MS, +// which keeps the delay from the game to the speakers at prefill plus one device period instead of up to the ring size. #include #include #include @@ -26,6 +29,7 @@ #define MAX_FRAME 1024 #define RING_FRAMES 16384 // stereo frames, about 340 ms at 48 kHz +#define QUEUE_SLACK_MS 40 // queued audio above prefill plus the device's request that ring_pull skips #define SPLITTER_UNUSED 0xffffffffu #define dm_guest_write_prepare(p, n) ((void)(p), (void)(n)) // guest memory is plain host memory on Windows @@ -43,7 +47,7 @@ typedef struct Mixer { SplitInfo *split; int nsplit; SplitDest *dest; int ndest; SDL_AudioStream *stream; int device_ok, rate; float ring[RING_FRAMES * 2]; - _Atomic uint64_t wr, rd, underruns, overruns, missing_frames, callback_frames, callback_nonzero; + _Atomic uint64_t wr, rd, underruns, overruns, missing_frames, callback_frames, callback_nonzero, skipped_frames; _Atomic int playing; int ready; // ready: the callback's own state _Atomic uint64_t flush_until; _Atomic unsigned stop_epoch; unsigned callback_epoch; EffectState *effect; int *effect_order; unsigned effect_reported; @@ -107,16 +111,22 @@ static void ring_pull(Mixer *m, float *out, uint32_t frames) { atomic_store_explicit(&m->rd, wr, memory_order_release); return; } + uint64_t prefill = (uint64_t)m->frame * (uint64_t)m->prefill; + if (prefill < frames) prefill = frames; + if (prefill > RING_FRAMES) prefill = RING_FRAMES; if (!m->ready) { - uint64_t prefill = (uint64_t)m->frame * (uint64_t)m->prefill; - if (prefill < frames) prefill = frames; - if (prefill > RING_FRAMES) prefill = RING_FRAMES; if (have < prefill) { memset(out, 0, (size_t)frames * 2 * sizeof(float)); return; } m->ready = 1; } + uint64_t keep = prefill + frames, slack = (uint64_t)m->rate * QUEUE_SLACK_MS / 1000; + if (have > keep + slack) { // audio piled up (render burst, clock drift): skip the oldest, keep prefill + request + atomic_fetch_add(&m->skipped_frames, have - keep); + rd = wr - keep; + have = keep; + } uint32_t n = frames < have ? frames : (uint32_t)have; uint64_t nonzero = 0; for (uint32_t i = 0; i < n; i++) { @@ -209,12 +219,14 @@ static void level_note(AudioRenderer *r, Mixer *m, const float *lr, int frames) rt_log("audio: first nonzero output after %llu frames", (unsigned long long)r->elapsed_frames); } if (m->level_frames >= (uint64_t)r->param.sample_rate * 5) { + uint64_t wr = atomic_load(&m->wr), queued = wr - atomic_load(&m->rd); rt_log("audio: level rms %.4f peak %.4f over %.1f s; written %llu frames, underruns %llu, overruns %llu, " - "missing frames %llu, callback frames %llu (nonzero %llu)", + "missing frames %llu, callback frames %llu (nonzero %llu), queued %.1f ms, skipped %llu frames", sqrt(m->level_sum / (double)m->level_count), m->level_peak, m->level_frames / (double)r->param.sample_rate, - (unsigned long long)atomic_load(&m->wr), (unsigned long long)atomic_load(&m->underruns), + (unsigned long long)wr, (unsigned long long)atomic_load(&m->underruns), (unsigned long long)atomic_load(&m->overruns), (unsigned long long)atomic_load(&m->missing_frames), - (unsigned long long)atomic_load(&m->callback_frames), (unsigned long long)atomic_load(&m->callback_nonzero)); + (unsigned long long)atomic_load(&m->callback_frames), (unsigned long long)atomic_load(&m->callback_nonzero), + queued * 1000.0 / r->param.sample_rate, (unsigned long long)atomic_load(&m->skipped_frames)); m->level_sum = m->level_peak = 0; m->level_count = m->level_frames = 0; } From 7d89fde2f0fd1cde07285d08b303aeb3c8cfe9a2 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 31 Oct 2026 00:22:08 +0300 Subject: [PATCH 20/50] Drop the six-axis samples from before StartSixAxisSensor motion_restart only flagged the next sample to start a fresh frame, so a read in the frame of the Start returned the previous session's 16 states as live, and for the next 60 ms 15 of the 16 were from before it, with the angle jumping back to zero between two reads. Motion aim was dead when the bow was first drawn and lagged after that. A restart now hides the older samples and zeroes the angle; sampling numbers keep counting. --probe-services checks a restart in the middle of a turn: 16 old states in the Start frame before, none after. Reported with the cause and a harness in #104. Co-Authored-By: Claude Opus 5.5 --- runtime/motion_fusion.c | 13 +++++++++---- runtime/ov_probe.c | 14 +++++++++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/runtime/motion_fusion.c b/runtime/motion_fusion.c index 0b4c56b7..76df5064 100644 --- a/runtime/motion_fusion.c +++ b/runtime/motion_fusion.c @@ -7,8 +7,9 @@ // axis, direction = the controller's three axes in a reference frame whose z is up. The orientation is integrated from // the gyro and, while the controller is not being accelerated (|a| within 15 % of 1 G), pulled toward the measured // gravity (Mahony, gain 0.5/s: about 2 s), as the Switch's sensor fusion does; yaw has no reference and drifts with the -// gyro's bias. The newest MOTION_RING samples are kept; reads are newest first. The feeding thread writes, guest -// threads read, both under the instance's SRW lock. +// gyro's bias. The newest MOTION_RING samples are kept; reads are newest first. A restart (StartSixAxisSensor) drops +// the samples before it: reads return nothing until the next sample, which starts a fresh frame and angle; sampling +// numbers go on. The feeding thread writes, guest threads read, both under the instance's SRW lock. #include #include #include @@ -24,6 +25,7 @@ struct MotionFusion { SRWLOCK lock; MotionState ring[MOTION_RING]; uint64_t produced, last_ns; + uint64_t first; // produced at the last restart: older samples are not read double R[3][3]; // controller -> reference; row i = reference axis i in controller coordinates double angle[3]; int restart; @@ -131,8 +133,8 @@ int motion_read(MotionFusion *m, MotionState *out, int max, uint64_t now) { if (max <= 0) return 0; int n = 0; AcquireSRWLockShared(&m->lock); - if (m->produced && (now < m->last_ns || now - m->last_ns < STALE_NS)) { - uint64_t have = m->produced < MOTION_RING ? m->produced : MOTION_RING; + if (m->produced > m->first && (now < m->last_ns || now - m->last_ns < STALE_NS)) { + uint64_t have = m->produced - m->first < MOTION_RING ? m->produced - m->first : MOTION_RING; n = (int)(have < (uint64_t)max ? have : (uint64_t)max); for (int i = 0; i < n; i++) out[i] = m->ring[(m->produced - 1 - (uint64_t)i) % MOTION_RING]; } @@ -143,5 +145,8 @@ int motion_read(MotionFusion *m, MotionState *out, int max, uint64_t now) { void motion_restart(MotionFusion *m) { AcquireSRWLockExclusive(&m->lock); m->restart = 1; + m->first = m->produced; + m->last_ns = 0; // the first sample after the restart integrates nothing across the stop + memset(m->angle, 0, sizeof m->angle); ReleaseSRWLockExclusive(&m->lock); } diff --git a/runtime/ov_probe.c b/runtime/ov_probe.c index 4668bf0a..c0ffa181 100644 --- a/runtime/ov_probe.c +++ b/runtime/ov_probe.c @@ -552,9 +552,21 @@ static void probe_feedback(uint32_t pid, uint32_t style) { near3(ms[0].dir + 3, -1, 0, 0, 0.01) && near3(ms[0].dir + 6, 0, 0, 1, 0.01), "fusion quarter turn: %d states, sampling %lld, angle %.4f rev, x axis (%.3f %.3f %.3f)", n, (long long)ms[0].sampling, (double)ms[0].angle[2], (double)ms[0].dir[0], (double)ms[0].dir[1], (double)ms[0].dir[2]); + double quarter = ms[0].angle[2]; CHECK(motion_read(m, ms, 16, t + 300000000ull) == 0, "fusion states 300 ms after the last sample"); + // StartSixAxisSensor again in the middle of the turn (the bow drawn again): the states from before it are gone, and + // the first one after it starts a fresh angle in a fresh frame; sampling numbers go on. + for (int i = 0; i < 250; i++) motion_feed(m, down, turn, t += 4000000); + motion_restart(m); + int same = motion_read(m, ms, 16, t + 1000000); + motion_feed(m, down, turn, t += 4000000); + int next = motion_read(m, ms, 16, t + 1000000); + CHECK(same == 0 && next == 1 && ms[0].sampling == 502 && fabs(ms[0].angle[2]) < 0.002 && + near3(ms[0].dir, 1, 0, 0, 0.01), + "fusion restart: %d states in the frame of the Start, %d after one sample (sampling %lld, angle %.4f rev)", same, + next, (long long)ms[0].sampling, (double)ms[0].angle[2]); printf("hid feedback: vibration handles %#x %#x, six-axis %#x, at rest when stopped; fusion quarter turn %.4f rev\n", - vh[0], vh[1], sh[0], (double)ms[0].angle[2]); + vh[0], vh[1], sh[0], quarter); } static void probe_input(void) { From 605662ca87613bfcfaf13045fd26c5c7487742d8 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 31 Oct 2026 00:25:05 +0300 Subject: [PATCH 21/50] Offer 1280x720 and 1600x900 as game resolutions in the menu The game resolution row started at 1920x1080 although resolution.c accepts sizes from 1280x720, so weak GPUs and integrated graphics had to edit totk.ini to draw the game smaller. A 1280x720 run on 1.4.3 renders the field with a 1067x600 scene and an intact interface. Co-Authored-By: Claude Opus 5.5 --- runtime/ui/menu.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 7d52baa9..162894d5 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -414,10 +414,10 @@ static void row_window_size(void) { opt_choice_list("TOTK_WINDOW_SIZE", UI_MENU_WINDOW_SIZE, pv, pn, n, UI_MENU_WINDOW_SIZE_DESC); } -// The game resolution (resolution.c): 16:9 sizes from the game's own 1920x1080 up, and the current value when it is -// another size. +// The game resolution (resolution.c): 16:9 sizes from its smallest, 1280x720 (for weak GPUs), up, and the current value +// when it is another size. static void row_output_res(void) { - static const char *const vals[] = {"1920x1080", "2560x1440", "3200x1800", "3840x2160"}; + static const char *const vals[] = {"1280x720", "1600x900", "1920x1080", "2560x1440", "3200x1800", "3840x2160"}; enum { N = sizeof vals / sizeof *vals }; char shown[N][32]; const char *names[N]; From a76224f8a6ce85221af6df21b45281a7eafdfd73 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 31 Oct 2026 00:42:03 +0300 Subject: [PATCH 22/50] Fail the install when romfs/ is gone or unreadable after cleanup An install whose romfs/ went empty and unreadable after the extraction had verified it (a folder Windows was still deleting) ended with result ok, and the game stopped at its first file read. Cleanup now checks that romfs.bin and index.json are still there, match the verified size and hash record, and that romfs.bin opens; otherwise it fails with cleanup.romfs_unreadable, which tells the player to restart Windows and run the installer on the same folder (resume extracts the game data again). Reported in #107. tests/installer_test.py CleanupTest: an emptied, removed or resized romfs passed cleanup before, fails now. Co-Authored-By: Claude Opus 5.5 --- installer/steps.py | 16 +++++++++++++ installer/text/en.json | 1 + installer/text/ru.json | 1 + tests/installer_test.py | 52 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/installer/steps.py b/installer/steps.py index 3984f784..9d31fb13 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -661,8 +661,24 @@ class Cleanup(Step): remove(stale) for name in ('gen', 'build', 'check', 'shader_list.tsv'): remove(ctx.temp / name) + romfs_problem(dest / 'romfs', expected_update(ctx)) return {} +def romfs_problem(romfs, expected): + """Raise cleanup.romfs_unreadable unless romfs/ still holds the verified image. The game opens romfs.bin at start; + an install whose romfs/ went empty and unreadable after the extraction verified it (a folder Windows was still + deleting, #107) reported success and the game stopped at its first file read.""" + try: + ok = extract.romfs_verified(romfs, expected) and (romfs / 'romfs.bin').stat().st_size == expected['romfs']['size'] + if ok: + with open(romfs / 'romfs.bin', 'rb') as f: + f.read(1) + except (OSError, ValueError): + ok = False + if not ok: + raise InstallError('cleanup.romfs_unreadable', path=str(romfs)) + + def all_steps(): return [Check(), Extract(), Lift(), Compile(), Prepare(), Cleanup()] diff --git a/installer/text/en.json b/installer/text/en.json index 5964a0f6..3425f5a5 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -75,6 +75,7 @@ "prepare.shader_list_missing": "The shader list is missing.", "prepare.romfs_patches_failed": "Writing the game data patches failed (exit code {code}). Details: {log}", "prepare.shaders_failed": "Preparing the shaders failed (exit code {code}). Details: {log}", + "cleanup.romfs_unreadable": "The game data in {path} is missing or cannot be read after the install. Restart Windows, then run the installer again on the same folder: it extracts the game data again and keeps the rest.", "cleanup.ini_failed": "totk.ini cannot be written: {detail}", "warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts, which the installer does not extract; some text may not show.", diff --git a/installer/text/ru.json b/installer/text/ru.json index c3a87db8..b1dc33ae 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -75,6 +75,7 @@ "prepare.shader_list_missing": "Нет списка шейдеров.", "prepare.romfs_patches_failed": "Не удалось записать патчи данных игры (код выхода {code}). Подробности: {log}", "prepare.shaders_failed": "Не удалось подготовить шейдеры (код выхода {code}). Подробности: {log}", + "cleanup.romfs_unreadable": "Данные игры в {path} пропали или не читаются после установки. Перезагрузите Windows и запустите установщик снова на ту же папку: он заново распакует данные игры, остальное сохранится.", "cleanup.ini_failed": "Не удаётся записать totk.ini: {detail}", "warn.lang_needs_fonts": "Для языка «{lang}» нужны системные шрифты Switch, которые установка не извлекает; часть текста может не отображаться.", diff --git a/tests/installer_test.py b/tests/installer_test.py index d7404929..036d897a 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -14,7 +14,7 @@ from types import SimpleNamespace ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) -from installer import ini, runner, state # noqa: E402 +from installer import ini, runner, state, steps # noqa: E402 from installer.events import Reporter # noqa: E402 from installer.layout import Layout # noqa: E402 @@ -241,6 +241,56 @@ class IniTest(unittest.TestCase): self.assertEqual(ini.get_language(text), 'ja') +class CleanupTest(unittest.TestCase): + """The real Cleanup step on a finished install whose romfs/ is intact, emptied or gone (#107).""" + + def setUp(self): + self.steps = steps + self.tmp = tempfile.TemporaryDirectory() + base = Path(self.tmp.name) + self.dest, temp = base / 'dest', base / 'temp' + (self.dest / 'romfs').mkdir(parents=True) + temp.mkdir() + image = b'romfs image' + (self.dest / 'romfs' / 'romfs.bin').write_bytes(image) + digest = steps.extract.sha256(self.dest / 'romfs' / 'romfs.bin') + (self.dest / 'romfs' / 'index.json').write_text(json.dumps({'romfs_size': len(image), 'romfs_sha256': digest})) + self.expected = {'romfs': {'size': len(image), 'sha256': digest}} + self.ctx = SimpleNamespace(reporter=Reporter(out=io.StringIO(), log_path=base / 'x.log'), dest=self.dest, + temp=temp, lang='en-US', state={}, + layout=SimpleNamespace(ini_example=ROOT / 'totk.ini.example')) + self.saved = steps.expected_update + steps.expected_update = lambda ctx: self.expected + + def tearDown(self): + self.steps.expected_update = self.saved + self.ctx.reporter.close() + self.tmp.cleanup() + + def run_cleanup(self): + try: + self.steps.Cleanup().run(self.ctx) + return 'ok' + except self.steps.InstallError as e: + return e.id + + def test_intact_romfs_passes(self): + self.assertEqual(self.run_cleanup(), 'ok') + + def test_emptied_romfs_fails(self): + for p in (self.dest / 'romfs').iterdir(): + p.unlink() + self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable') + + def test_missing_romfs_fails(self): + self.steps.remove(self.dest / 'romfs') + self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable') + + def test_romfs_of_another_size_fails(self): + (self.dest / 'romfs' / 'romfs.bin').write_bytes(b'short') + self.assertEqual(self.run_cleanup(), 'cleanup.romfs_unreadable') + + class EventsTest(unittest.TestCase): def test_lines_are_json_and_the_log_is_english(self): with tempfile.TemporaryDirectory() as d: From 9016b1e21072e35f85b3dcd1758d47431d32151e Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 31 Oct 2026 00:46:24 +0300 Subject: [PATCH 23/50] Add a virtual test gamepad for stick tests without a controller TOTK_TEST_PAD=,,, attaches an SDL virtual gamepad at start with its sticks at those raw values, so the dead zone and the gamepad path can be tested in a run. On the field save: an 11% drift reached the game as 3.3% at the default 8% zone (the old per-axis cutoff passed 9% on x and dropped y) and not at all at 15% (Link stood still); full deflection reached it as full at 15%. Co-Authored-By: Claude Opus 5.5 --- docs/controls.md | 2 ++ runtime/cfg.c | 2 +- runtime/input_win.c | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/controls.md b/docs/controls.md index b80fc89c..a351fdcd 100644 --- a/docs/controls.md +++ b/docs/controls.md @@ -87,6 +87,8 @@ The stick direction bits (`StickLLeft` to `StickRDown`) are derived from the sti `TOTK_INPUT_RECORD=` (`input_record` in `totk.ini`, or Input recording in the menu) writes the merged input in the same format, one line per button press and per stick segment: events before the world clock starts by tick, the rest timed; a button or stick held when the world clock starts is cut there and goes on as a timed event. The file replays as a script, which makes it the repro to attach to a bug report. +`TOTK_TEST_PAD=,,,` attaches an SDL virtual gamepad at start whose sticks rest at these raw SDL axis values (-32768 to 32767, down is +y), so a drifting or held stick can be tested without a controller: the values go through the gamepad mapping and `TOTK_STICK_DEADZONE` like a real pad's, and `TOTK_INPUT_RECORD` shows what reached the game. + ## RenderDoc capture (blocked) `TOTK_RDOC=[,...]` (`runtime/rdoc_win.c`; `TOTK_RDOC_DIR`, `TOTK_RDOC_DLL`) loads RenderDoc before `vkCreateInstance`. RenderDoc hides `VK_EXT_external_memory_host`, which the back end requires for guest pool imports (`runtime/vk_memory.c`), so device creation fails under it. A capture needs a copy-based pool fallback first. diff --git a/runtime/cfg.c b/runtime/cfg.c index 322917ab..336b24e3 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -48,7 +48,7 @@ static const char *const known[] = { "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WB_FENCE_LIMIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", }; diff --git a/runtime/input_win.c b/runtime/input_win.c index 096fef53..d4f0e78d 100644 --- a/runtime/input_win.c +++ b/runtime/input_win.c @@ -515,12 +515,44 @@ static void binds_refresh(void) { input_map_configure(); } +// TOTK_TEST_PAD=,,, (test runs): an SDL virtual gamepad whose sticks rest at these raw SDL axis values +// (-32768..32767, down is +y), attached before the first scan so it is the gamepad the game reads: drift and full +// deflection go through the same mapping and dead zone as a real pad. +static void test_pad_attach(void) { + static SDL_Joystick *joy; // kept open: a virtual joystick's axes are set through an open handle + const char *e = cfg_get("TOTK_TEST_PAD"); + int v[4]; + if (!e || !*e) return; + if (sscanf(e, "%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3]) != 4) { + rt_log("input: TOTK_TEST_PAD=%s is not ,,,; no test gamepad", e); + return; + } + SDL_VirtualJoystickDesc d; + SDL_INIT_INTERFACE(&d); + d.type = SDL_JOYSTICK_TYPE_GAMEPAD; + d.naxes = SDL_GAMEPAD_AXIS_COUNT; + d.nbuttons = SDL_GAMEPAD_BUTTON_COUNT; + d.name = "TotK test gamepad"; + SDL_JoystickID id = SDL_AttachVirtualJoystick(&d); + joy = id ? SDL_OpenJoystick(id) : NULL; + if (!joy) { + rt_log("input: test gamepad not attached (%s)", SDL_GetError()); + return; + } + static const SDL_GamepadAxis axes[4] = {SDL_GAMEPAD_AXIS_LEFTX, SDL_GAMEPAD_AXIS_LEFTY, SDL_GAMEPAD_AXIS_RIGHTX, + SDL_GAMEPAD_AXIS_RIGHTY}; + for (int i = 0; i < 4; i++) + SDL_SetJoystickVirtualAxis(joy, axes[i], (Sint16)(v[i] < -32768 ? -32768 : v[i] > 32767 ? 32767 : v[i])); + rt_log("input: test gamepad attached, sticks at %d,%d %d,%d", v[0], v[1], v[2], v[3]); +} + static void start(void) { binds_refresh(); const char *e = cfg_get("TOTK_INPUT"); if (e && !strcmp(e, "none")) { state = -1; rt_log("input: disabled (TOTK_INPUT=none)"); return; } if (!SDL_Init(SDL_INIT_GAMEPAD)) { state = -1; rt_log("input: SDL gamepad init failed (%s), keyboard only", SDL_GetError()); } else state = 1; + if (state == 1) test_pad_attach(); if (state == 1) { scan_pads(); if (!pad) rt_log("input: no gamepad (keyboard only until one is connected)"); From ada341e2b11472a65454d783ac4a019ff8101a50 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 31 Oct 2026 07:12:48 +0300 Subject: [PATCH 24/50] Pace audio rendering by the output device The render thread ran on its own 5 ms timer while the device consumed on its own clock, so the queue crept: on DirectSound it grew 2.7 ms every 5 s, which fills the 340 ms ring in about ten minutes of play. The render thread now renders the next frame when the device has taken the ring below max(prefill, two device periods), as the audio hardware paces the Switch's DSP and as LASRecomp and SavageWindRecomp pace their renderers by the SDL stream. Without a device, while the renderer is stopped and while the device takes nothing for 50 ms (at least four periods) it keeps the timer, so a stalled device does not stall the game; the ring trim then drops what piled up, measured from the target. 60 s on the monsters save: WASAPI 15 ms queued, DirectSound 27-40 ms without growth, no underruns. With the SDL audio thread suspended for 1.5 s the game held 60 fps and the queue was back at 15 ms after one trim. Co-Authored-By: Claude Opus 5.5 --- runtime/audio_mixer.c | 44 ++++++++++++++++++++++++++++++++++++------- runtime/audren.c | 31 ++++++++++++++++++++---------- runtime/audren.h | 4 ++++ 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/runtime/audio_mixer.c b/runtime/audio_mixer.c index a34885c7..6dc625b8 100644 --- a/runtime/audio_mixer.c +++ b/runtime/audio_mixer.c @@ -9,9 +9,13 @@ // single-producer single-consumer ring. The SDL audio thread drains that ring without taking any lock: an empty ring // plays silence and never blocks the render thread or the guest. TOTK_AUDIO=none (or TOTK_AUDIO_OFF) mixes without a // device; TOTK_AUDIO_PREFILL_MS (0..500, default 20) is the queued audio the output waits for after start or underrun. -// The render thread runs on the host clock and the device on its own, and after a stall the render thread catches up in -// a burst, so audio can pile up in the ring; the SDL thread skips what is queued beyond the prefill plus QUEUE_SLACK_MS, -// which keeps the delay from the game to the speakers at prefill plus one device period instead of up to the ring size. +// The device's clock paces the render thread, as the audio hardware paces the Switch's DSP (audio_mixer_pace_ns): a +// frame is rendered when the ring holds less than the output target, max(prefill, two device periods), so the queue and +// the delay from the game to the speakers stay there and the host and device clocks cannot drift apart. LASRecomp and +// SavageWindRecomp pace their renderers by the SDL stream the same way. Without a device, while the renderer is +// stopped and while the device takes nothing (a stall), the render thread keeps real time on its own timer; what piled +// up meanwhile is skipped on the SDL thread once the device resumes (more than QUEUE_SLACK_MS above the target plus the +// device's request), as MacNativeGames' audio_mixer.c trims its ring after a device stall. #include #include #include @@ -29,7 +33,8 @@ #define MAX_FRAME 1024 #define RING_FRAMES 16384 // stereo frames, about 340 ms at 48 kHz -#define QUEUE_SLACK_MS 40 // queued audio above prefill plus the device's request that ring_pull skips +#define QUEUE_SLACK_MS 40 // queued audio above the target plus the device's request that ring_pull skips +#define STALL_MS 50 // the device took nothing this long (at least 4 periods): the timer paces #define SPLITTER_UNUSED 0xffffffffu #define dm_guest_write_prepare(p, n) ((void)(p), (void)(n)) // guest memory is plain host memory on Windows @@ -49,6 +54,9 @@ typedef struct Mixer { float ring[RING_FRAMES * 2]; _Atomic uint64_t wr, rd, underruns, overruns, missing_frames, callback_frames, callback_nonzero, skipped_frames; _Atomic int playing; int ready; // ready: the callback's own state + int target, period; // output target and device period, frames (output_open) + uint64_t stall_ns; + _Atomic uint64_t pulled_ns; // kern_now_ns of the device's last request, 0 before the first _Atomic uint64_t flush_until; _Atomic unsigned stop_epoch; unsigned callback_epoch; EffectState *effect; int *effect_order; unsigned effect_reported; uint64_t *sink_address; uint32_t *sink_size, *sink_offset, *sink_old, *sink_written; @@ -121,8 +129,9 @@ static void ring_pull(Mixer *m, float *out, uint32_t frames) { } m->ready = 1; } - uint64_t keep = prefill + frames, slack = (uint64_t)m->rate * QUEUE_SLACK_MS / 1000; - if (have > keep + slack) { // audio piled up (render burst, clock drift): skip the oldest, keep prefill + request + uint64_t keep = (prefill > (uint64_t)m->target ? prefill : (uint64_t)m->target) + frames; + uint64_t slack = (uint64_t)m->rate * QUEUE_SLACK_MS / 1000; + if (have > keep + slack) { // piled up while the device stalled: skip the oldest, keep target + request atomic_fetch_add(&m->skipped_frames, have - keep); rd = wr - keep; have = keep; @@ -149,6 +158,7 @@ static void SDLCALL sdl_pull(void *user, SDL_AudioStream *stream, int additional (void)total; Mixer *m = user; float tmp[2 * 1024]; + atomic_store_explicit(&m->pulled_ns, kern_now_ns(), memory_order_release); while (additional > 0) { uint32_t frames = (uint32_t)additional / (2 * sizeof(float)); if (!frames) break; @@ -182,8 +192,28 @@ static void output_open(Mixer *m, int rate) { return; } m->device_ok = 1; + SDL_AudioSpec dev; + int period = 0; + if (!SDL_GetAudioDeviceFormat(SDL_GetAudioStreamDevice(m->stream), &dev, &period) || period <= 0) period = m->frame; + m->period = period; + m->target = m->frame * m->prefill > 2 * period ? m->frame * m->prefill : 2 * period; + if (m->target > RING_FRAMES / 2) m->target = RING_FRAMES / 2; + uint64_t period_ns = (uint64_t)period * 1000000000ull / (uint64_t)rate; + m->stall_ns = 4 * period_ns > STALL_MS * 1000000ull ? 4 * period_ns : STALL_MS * 1000000ull; const char *name = SDL_GetAudioDeviceName(SDL_GetAudioStreamDevice(m->stream)); - rt_log("audio: SDL device '%s' opened, %d Hz stereo float, prefill %d ms", name ? name : "?", rate, m->prefill * 5); + rt_log("audio: SDL device '%s' opened, %d Hz stereo float, prefill %d ms, device period %d frames; the device paces " + "rendering at %.1f ms queued", name ? name : "?", rate, m->prefill * 5, period, m->target * 1000.0 / rate); +} + +int64_t audio_mixer_pace_ns(AudioRenderer *r, uint64_t now) { + Mixer *m = r->mixer; + if (!m || !m->device_ok || !atomic_load_explicit(&m->playing, memory_order_acquire)) return -1; + uint64_t pulled = atomic_load_explicit(&m->pulled_ns, memory_order_acquire); + if (!pulled || (now > pulled && now - pulled > m->stall_ns)) return -1; + uint64_t have = atomic_load(&m->wr) - atomic_load_explicit(&m->rd, memory_order_acquire); + if (have < (uint64_t)m->target) return 0; + int64_t ns = (int64_t)((have - (uint64_t)m->target + 1) * 1000000000ull / (uint64_t)m->rate); + return ns < 500000 ? 500000 : ns; // until the device has taken it below the target } // Render thread only. diff --git a/runtime/audren.c b/runtime/audren.c index f031d031..8a379c0b 100644 --- a/runtime/audren.c +++ b/runtime/audren.c @@ -316,7 +316,8 @@ static void request_update(AudioRenderer *r, const uint8_t *input, uint64_t in_s ReleaseSRWLockExclusive(&r->lock); } -// ---- render tick: one frame every sample_count / sample_rate seconds, on a host thread +// ---- render tick: one frame each time the output device has taken the ring below its target (audio_mixer_pace_ns), +// or every sample_count / sample_rate seconds of the host clock without a device, while stopped or while it stalls static void advance_voice(AudioRenderer *r, ServerVoice *v, double out_seconds) { if (!v->in.in_use || v->state != SV_STARTED || !v->in.sample_rate) return; double want = v->fraction + out_seconds * v->in.sample_rate * (v->in.pitch > 0 ? v->in.pitch : 1.0); @@ -348,6 +349,12 @@ static void advance_voice(AudioRenderer *r, ServerVoice *v, double out_seconds) } } +static void wait_ns(HANDLE timer, uint64_t ns) { + LARGE_INTEGER due = {.QuadPart = -(LONGLONG)((ns + 99) / 100)}; // relative, 100 ns units + if (!SetWaitableTimer(timer, &due, 0, NULL, NULL, FALSE)) rt_fatal("audren: SetWaitableTimer failed (error %lu)", GetLastError()); + WaitForSingleObject(timer, INFINITE); +} + static DWORD WINAPI render_thread(void *p) { AudioRenderer *r = p; SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); @@ -358,6 +365,19 @@ static DWORD WINAPI render_thread(void *p) { const double frame = (double)r->param.sample_count / r->param.sample_rate; uint64_t next = kern_now_ns(); while (InterlockedCompareExchange(&r->running, 0, 0)) { + uint64_t now = kern_now_ns(); + int64_t pace = audio_mixer_pace_ns(r, now); + if (pace > 0) { + wait_ns(timer, (uint64_t)pace); + continue; + } + if (pace == 0) { + next = now + frame_ns; // where the timer goes on if the device stops pacing + } else { + if (next > now) wait_ns(timer, next - now); + else if (now - next > 100000000ull) next = now; // fell far behind (debugger, sleep): resynchronise + next += frame_ns; + } AcquireSRWLockExclusive(&r->lock); if (r->state == 0) { voices_prepare(r); @@ -374,15 +394,6 @@ static DWORD WINAPI render_thread(void *p) { } ReleaseSRWLockExclusive(&r->lock); kern_event_signal(r->event); - next += frame_ns; - uint64_t now = kern_now_ns(); - if (next > now) { - LARGE_INTEGER due = {.QuadPart = -(LONGLONG)((next - now + 99) / 100)}; // relative, 100 ns units - if (!SetWaitableTimer(timer, &due, 0, NULL, NULL, FALSE)) - rt_fatal("audren: SetWaitableTimer failed (error %lu)", GetLastError()); - WaitForSingleObject(timer, INFINITE); - } else if (now - next > 100000000ull) - next = now; // fell far behind (debugger, sleep): resynchronise } CloseHandle(timer); return 0; diff --git a/runtime/audren.h b/runtime/audren.h index 92da770d..6dc74bc7 100644 --- a/runtime/audren.h +++ b/runtime/audren.h @@ -180,5 +180,9 @@ void audio_mixer_update_effects(AudioRenderer *r); void audio_mixer_effect_result(AudioRenderer *r, int id, void *output, unsigned size); uint32_t audio_mixer_sink_written(AudioRenderer *r, int id); void audio_mixer_render(AudioRenderer *r, double seconds); // after audio_voices_prepare, before voices advance +// Device pacing for the render thread at kern_now_ns `now`: 0 renders the next frame now, a positive value waits that +// many ns (the ring is at its target), -1 leaves the pace to the render thread's timer (no device, renderer stopped, +// device stalled). +int64_t audio_mixer_pace_ns(AudioRenderer *r, uint64_t now); int audio_buffer_empty(const AudrenWaveBuffer *b); // no samples to decode: played as soon as it is reached void audio_device_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs); // IAudioDevice From 31fff747767155a039f8a0a3f276fc36e35a671a Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 31 Oct 2026 07:23:04 +0300 Subject: [PATCH 25/50] Log the memory commit charge with every fatal error #94 ended on a realloc of 1 MiB that failed with 64 GB of commit allowed, and the log could not tell whether the process had grown towards the limit or the system ran short. rt_fatal now adds one line with the process's private bytes and the system's commit in use and limit (plat_commit_info). Co-Authored-By: Claude Opus 5.5 --- runtime/dispatch.c | 6 ++++++ runtime/platform/platform.h | 3 +++ runtime/platform/win32.c | 12 ++++++++++++ 3 files changed, 21 insertions(+) diff --git a/runtime/dispatch.c b/runtime/dispatch.c index e9bc46a9..71e326ab 100644 --- a/runtime/dispatch.c +++ b/runtime/dispatch.c @@ -12,6 +12,7 @@ // is also fixed at init; they are looked up only after the flat table misses. #include "dispatch.h" #include "host.h" +#include "platform/platform.h" #include #include @@ -51,6 +52,11 @@ void rt_fatal(const char *fmt, ...) { rt_vlogf(fmt, ap); rt_log_raw("\n", 1); va_end(ap); + // A failed allocation (malloc, realloc, VirtualAlloc) means the commit limit was reached: this line tells a leak of + // ours (process private bytes near the limit) from a machine short of page file. + double proc, used, limit; + plat_commit_info(&proc, &used, &limit); + rt_log("fatal: memory commit: this process %.0f MB, the system %.0f of %.0f MB", proc, used, limit); fflush(stdout); fflush(stderr); _Exit(70); diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 17898d8f..c5761651 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -116,6 +116,9 @@ void plat_cpu_load(PlatLoad *l, float *total_pct, float *proc_pct, float *temp_c void plat_gpu_load(PlatLoad *l, float *load_pct, float *vram_used_mb); // Physical memory in MB: in use and installed on the machine, and this process's working set. void plat_mem_info(float *used_mb, float *total_mb, float *proc_mb); +// Commit charge in MB: this process's private bytes, the system's commit in use and its limit (RAM plus page files). +// An allocation fails when the system's reaches the limit, however much physical memory is free. -1 when unknown. +void plat_commit_info(double *proc_mb, double *used_mb, double *limit_mb); // ---- child processes: a program with stdin and stdout piped to us (stderr discarded), no console window. typedef struct PlatChild PlatChild; diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index c053d4d5..6ab2fbdb 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -613,6 +613,18 @@ void plat_mem_info(float *used_mb, float *total_mb, float *proc_mb) { if (K32GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof pmc)) *proc_mb = (float)(pmc.WorkingSetSize >> 20); } +void plat_commit_info(double *proc_mb, double *used_mb, double *limit_mb) { + *proc_mb = *used_mb = *limit_mb = -1; + MEMORYSTATUSEX ms = {.dwLength = sizeof ms}; + if (GlobalMemoryStatusEx(&ms)) { + *limit_mb = (double)(ms.ullTotalPageFile >> 20); + *used_mb = (double)((ms.ullTotalPageFile - ms.ullAvailPageFile) >> 20); + } + PROCESS_MEMORY_COUNTERS_EX pmc; + if (K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS *)&pmc, sizeof pmc)) + *proc_mb = (double)(pmc.PrivateUsage >> 20); +} + // ---- child processes struct PlatChild { From 2fcb09d609f47030e37abb3abcd995a24995d432 Mon Sep 17 00:00:00 2001 From: Pasha Kagan Date: Fri, 30 Oct 2026 07:26:13 +0200 Subject: [PATCH 26/50] Read shader translations, use masks and stored inputs from shader-cache.pack before the loose files --- runtime/gfx_shader.c | 147 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 136 insertions(+), 11 deletions(-) diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 4df23fea..15e2343a 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -18,6 +18,11 @@ // (64 hex digits) overrides it. A translator change therefore changes every cache key without a hand-made KEY_VERSION // bump; KEY_VERSION is for changes on this side (key layout, how the output is used). // +// Pack: tools/pack_shader_cache.py folds the .spv, .res.json, .use and in/*.in files into /shader-cache.pack +// (one file instead of four per shader: ~117,000 files after an install) and deletes them; the installer runs it after +// preparing the shaders. The pack is read-only here. Every read looks in the pack first and then for the loose file, +// and new translations are written as loose files as before, until the tool runs again (section "cache pack"). +// // Stored inputs: every translation also writes /in/.in (magic "TKSI"; stage, the full answers, the // control section, the code up to its length, c1), so a pipeline key record (vk_pipeline.c), which names its shaders by // input id, can be translated again under a new translator without guest memory (gfx_shader_load_input, @@ -125,6 +130,94 @@ static INIT_ONCE paths_once = INIT_ONCE_STATIC_INIT; #define read_file plat_file_read #define write_file plat_file_write +// ---- cache pack (/shader-cache.pack, written by tools/pack_shader_cache.py) +// Little-endian: "TKSPACK1", u32 count, u32 0, then count entries sorted by (kind, name), then the data. name is the +// 32 bytes the loose file's 64 hex digits spell. The index is read once (48 bytes a file: ~5.6 MB after an install); +// the data is read per lookup with positioned reads on one handle, safe from every thread. A pack that fails a check is +// ignored as a whole and the loose files are used. +enum { PACK_SPV = 1, PACK_RES = 2, PACK_USE = 3, PACK_IN = 4 }; +typedef struct { uint8_t name[32]; uint32_t kind, size; uint64_t offset; } PackEnt; +_Static_assert(sizeof(PackEnt) == 48, "shader-cache.pack entry layout"); +static HANDLE pack_file = INVALID_HANDLE_VALUE; +static PackEnt *pack_ix; +static uint32_t pack_n; + +static int pack_cmp(const PackEnt *a, uint32_t kind, const uint8_t name[32]) { + if (a->kind != kind) return a->kind < kind ? -1 : 1; + return memcmp(a->name, name, 32); +} + +static int pack_read_at(uint64_t offset, void *out, uint32_t n) { + OVERLAPPED o = {0}; + o.Offset = (DWORD)offset, o.OffsetHigh = (DWORD)(offset >> 32); + DWORD got = 0; + return ReadFile(pack_file, out, n, &got, &o) && got == n; +} + +// Opens /shader-cache.pack and reads its index (init_paths). Nothing happens when there is none. +static void pack_open(void) { + char p[PATH_CAP + 32]; + wchar_t w[PATH_CAP + 32]; + snprintf(p, sizeof p, "%s\\shader-cache.pack", cache_dir); + if (!MultiByteToWideChar(CP_UTF8, 0, p, -1, w, PATH_CAP + 32)) return; + HANDLE h = CreateFileW(w, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + if (h == INVALID_HANDLE_VALUE) return; + LARGE_INTEGER size; + uint8_t head[16]; + uint32_t n = 0; + const char *why = NULL; + PackEnt *ix = NULL; + pack_file = h; + if (!GetFileSizeEx(h, &size) || !pack_read_at(0, head, sizeof head) || memcmp(head, "TKSPACK1", 8)) why = "header"; + if (!why) { + memcpy(&n, head + 8, 4); + if (n > (1u << 24) || (uint64_t)n * sizeof *ix + sizeof head > (uint64_t)size.QuadPart || + !(ix = malloc((size_t)n * sizeof *ix + 1))) + why = "index size"; + else if (!pack_read_at(sizeof head, ix, n * (uint32_t)sizeof *ix)) why = "index read"; + } + for (uint32_t i = 0; !why && i < n; i++) { + if (ix[i].offset + ix[i].size > (uint64_t)size.QuadPart) why = "entry out of bounds"; + else if (i && pack_cmp(&ix[i - 1], ix[i].kind, ix[i].name) >= 0) why = "index order"; + } + if (why) { + rt_log("gfx_shader: %s is not a usable cache pack (%s); reading the loose files", p, why); + free(ix); + CloseHandle(h); + pack_file = INVALID_HANDLE_VALUE; + return; + } + pack_ix = ix, pack_n = n; + rt_log("gfx_shader: cache pack %s: %u files", p, n); +} + +static const PackEnt *pack_find(uint32_t kind, const uint8_t name[32]) { + size_t lo = 0, hi = pack_n; + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + int c = pack_cmp(&pack_ix[mid], kind, name); + if (!c) return &pack_ix[mid]; + if (c < 0) lo = mid + 1; + else hi = mid; + } + return NULL; +} + +// A packed file as plat_file_read returns a loose one: malloc'd, NUL after the last byte; NULL when absent or empty. +static uint8_t *pack_read(uint32_t kind, const uint8_t name[32], uint64_t *size) { + const PackEnt *e = pack_find(kind, name); + uint8_t *buf = e && e->size ? malloc((size_t)e->size + 1) : NULL; + if (!buf) return NULL; + if (!pack_read_at(e->offset, buf, e->size)) { + free(buf); + return NULL; + } + buf[e->size] = 0; + *size = e->size; + return buf; +} + // Appends a UTF-8 string as a JSON string body: ASCII as is, quotes and backslashes escaped, the rest as \uXXXX // (UTF-16 units), so the service's stdin encoding does not matter. static void json_esc(char **o, const char *s) { @@ -218,6 +311,7 @@ static BOOL CALLBACK init_paths(PINIT_ONCE once, PVOID param, PVOID *ctx) { if (cache_cfg && *cache_cfg && strlen(cache_cfg) < sizeof cache_dir) strcpy(cache_dir, cache_cfg); else if (exe_dir[0]) snprintf(cache_dir, sizeof cache_dir, "%s\\shader-cache", exe_dir); xlate_identity(); + if (cache_dir[0]) pack_open(); use_preload_start(); return TRUE; } @@ -468,15 +562,15 @@ static int spirv_complete(const uint32_t *w, uint64_t n) { return i == n; } -// Reads .spv and .res.json; NULL if either is missing or malformed (the caller translates again and -// overwrites both). iid: the input id, NULL when the caller sets it. -static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { +// Reads .spv and .res.json from the pack (packed 1) or the loose files; NULL if either is missing or +// malformed. iid: the input id, NULL when the caller sets it. +static GfxShader *disk_load_from(const uint8_t key[32], const uint8_t *iid, int packed) { static volatile LONG n_bad; char hex[65], p[PATH_CAP + 80]; key_hex(key, hex); uint64_t spv_size = 0, res_size = 0; snprintf(p, PATH_CAP + 80, "%s\\%s.res.json", cache_dir, hex); - uint8_t *res = read_file(p, &res_size); + uint8_t *res = packed ? pack_read(PACK_RES, key, &res_size) : read_file(p, &res_size); if (!res) return NULL; if (!reflection_valid((const char *)res, res_size)) { LONG n = InterlockedIncrement(&n_bad); @@ -485,7 +579,7 @@ static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { return NULL; } snprintf(p, PATH_CAP + 80, "%s\\%s.spv", cache_dir, hex); - uint8_t *spv = read_file(p, &spv_size); + uint8_t *spv = packed ? pack_read(PACK_SPV, key, &spv_size) : read_file(p, &spv_size); uint32_t magic = 0; if (spv && spv_size >= 20) memcpy(&magic, spv, 4); if (!spv || spv_size % 4 || magic != 0x07230203 || !spirv_complete((const uint32_t *)spv, spv_size / 4)) { @@ -503,6 +597,13 @@ static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { return s; } +// The pack, then the loose files (a newer translation of a key the pack holds a bad copy of); NULL when neither has a +// valid pair (the caller translates again and writes loose files). +static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { + GfxShader *s = pack_n ? disk_load_from(key, iid, 1) : NULL; + return s ? s : disk_load_from(key, iid, 0); +} + // ---- request building static const char *const tex_names[] = {NULL, "1d", "1darray", "buffer", "2d", "2darray", "2dms", "2dmsarray", "3d", @@ -751,10 +852,25 @@ static DWORD WINAPI use_preload(void *arg) { (void)arg; uint64_t t0 = plat_now_ns(); char pattern[PATH_CAP + 16], p[PATH_CAP + 80]; + int n = 0, bad = 0; + for (uint32_t i = 0; i < pack_n; i++) { // the packed masks; the loose files below add the masks found since + uint32_t u[5]; + UseFile **slot, *x; + if (pack_ix[i].kind != PACK_USE) continue; + if (pack_ix[i].size != sizeof u || !pack_read_at(pack_ix[i].offset, u, sizeof u) || u[0] != USE_MAGIC || + u[1] != USE_VERSION) { + bad++; + continue; + } + if (!*(slot = use_file_slot(pack_ix[i].name)) && (x = malloc(sizeof *x))) { + memcpy(x->name, pack_ix[i].name, 32); + x->next = NULL, x->m.slots = u[2], x->m.attrs = u[3], x->m.fouts = u[4]; + *slot = x, n++; + } + } snprintf(pattern, sizeof pattern, "%s\\*.use", cache_dir); WIN32_FIND_DATAA fd; HANDLE f = FindFirstFileExA(pattern, FindExInfoBasic, &fd, FindExSearchNameMatch, NULL, FIND_FIRST_EX_LARGE_FETCH); - int n = 0, bad = 0; if (f != INVALID_HANDLE_VALUE) { do { UseFile e = {0}; @@ -824,7 +940,8 @@ static int use_find(const uint8_t h[32], UseMask *m) { char p[PATH_CAP + 80]; use_path_of(name, p); uint64_t n = 0; - uint32_t *f = (uint32_t *)read_file(p, &n); + uint32_t *f = (uint32_t *)pack_read(PACK_USE, name, &n); + if (!f) f = (uint32_t *)read_file(p, &n); known = f && n == 20 && f[0] == USE_MAGIC && f[1] == USE_VERSION; if (known) m->slots = f[2], m->attrs = f[3], m->fouts = f[4]; free(f); @@ -1105,7 +1222,7 @@ static void in_store(const uint8_t iid[32], int stage, const uint8_t *control, c const GfxShaderAnswers *a) { char p[PATH_CAP + 80], tmp[PATH_CAP + 120]; in_path(iid, p); - if (file_exists(p)) return; + if (pack_find(PACK_IN, iid) || file_exists(p)) return; GfxShaderAnswers zero = {0}; InHeader h = {IN_MAGIC, IN_VERSION, (uint32_t)stage, r->length, r->c1_len, sizeof(GfxShaderAnswers)}; size_t n = sizeof h + sizeof zero + CONTROL_SIZE + r->length + r->c1_len; @@ -1127,7 +1244,8 @@ static int in_load(const uint8_t iid[32], Inputs *in) { char p[PATH_CAP + 80]; in_path(iid, p); uint64_t n = 0; - uint8_t *f = read_file(p, &n); + uint8_t *f = pack_read(PACK_IN, iid, &n); + if (!f) f = read_file(p, &n); InHeader h; if (!f || n < sizeof h) return free(f), 0; memcpy(&h, f, sizeof h); @@ -1425,7 +1543,7 @@ static DWORD WINAPI warm_worker(void *arg) { key_of_input(iid, key); key_hex(key, hex); snprintf(p, PATH_CAP + 80, "%s\\%s.spv", cache_dir, hex); - if (file_exists(p)) InterlockedIncrement(&w->other_cached); + if (pack_find(PACK_SPV, key) || file_exists(p)) InterlockedIncrement(&w->other_cached); else gfx_shader_load_input(iid, 1); } InterlockedIncrement64(&w->done); @@ -1463,7 +1581,14 @@ static size_t other_inputs(const uint8_t (*named)[32], size_t n_named, uint8_t ( char dir[PATH_CAP + 80]; snprintf(dir, sizeof dir, "%s\\in", cache_dir); InList l = {.named = named, .n_named = n_named}; - plat_file_list(dir, ".in", in_collect, &l); + char name[68]; + for (uint32_t i = 0; i < pack_n; i++) { // the packed inputs, under the name their loose file would have + if (pack_ix[i].kind != PACK_IN) continue; + key_hex(pack_ix[i].name, name); + memcpy(name + 64, ".in", 4); + if (!in_collect(name, &l)) break; + } + plat_file_list(dir, ".in", in_collect, &l); // in_store writes no loose file for a packed input: no duplicates return *out = l.v, l.n; } From bc84ac0b1fc6e62949aa83ccb6830ed3b5f3e343 Mon Sep 17 00:00:00 2001 From: Pasha Kagan Date: Fri, 30 Oct 2026 07:27:03 +0200 Subject: [PATCH 27/50] Add tools/pack_shader_cache.py, which folds the shader cache's loose files into shader-cache.pack --- tools/pack_shader_cache.py | 140 +++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 tools/pack_shader_cache.py diff --git a/tools/pack_shader_cache.py b/tools/pack_shader_cache.py new file mode 100644 index 00000000..0b326d5d --- /dev/null +++ b/tools/pack_shader_cache.py @@ -0,0 +1,140 @@ +"""Fold the loose files of a shader cache into /shader-cache.pack and delete them. + +The translator writes four files per shader into the cache (runtime/gfx_shader.c): .spv, .res.json, +.use and in/.in, about 117,000 files after an install, which are slow to copy, back up, scan and sync. +totk.exe reads the pack first and the loose files after it, and keeps writing new translations as loose files, so +this tool can run again at any time the game is closed: it merges the existing pack with the loose files (a loose file +wins over a packed one of the same name) and writes the pack anew. The installer runs it after preparing the shaders. + +Layout, little-endian (gfx_shader.c, section "cache pack"): b'TKSPACK1', u32 count, u32 0, then count entries of 48 +bytes sorted by (kind, name): name (the 32 bytes the file name's 64 hex digits spell), u32 kind (1 .spv, 2 .res.json, +3 .use, 4 in/*.in), u32 size, u64 offset; then the data. + + python tools\\pack_shader_cache.py \\shader-cache [--keep] + +--keep leaves the loose files in place. Close the game first: a running totk.exe has read the old pack's index. +""" +import argparse +import os +import re +import struct +import sys +from pathlib import Path + +MAGIC = b'TKSPACK1' +PACK = 'shader-cache.pack' +KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4} +NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)') +USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks + + +def loose_files(cache): + """{(kind, name): path} of the cache's loose files; *.in only in in/.""" + found = {} + for folder, kinds in ((cache, (1, 2, 3)), (cache / 'in', (4,))): + if not folder.is_dir(): + continue + with os.scandir(folder) as it: + for e in it: + m = NAME.fullmatch(e.name) + if not m or not e.is_file(): + continue + kind = KINDS[m.group(2)] + if kind in kinds: + found[(kind, bytes.fromhex(m.group(1)))] = Path(e.path) + return found + + +def read_pack(path): + """{(kind, name): (offset, size)} of an existing pack; {} when there is none or it fails a check.""" + try: + f = open(path, 'rb') + except FileNotFoundError: + return {} + with f: + size = os.fstat(f.fileno()).st_size + head = f.read(16) + if len(head) != 16 or head[:8] != MAGIC: + print(f'{path}: not a cache pack, rebuilt from the loose files only', flush=True) + return {} + count = struct.unpack_from(' 0 + + +def pack(cache, keep=False): + cache = Path(cache) + out = cache / PACK + packed = read_pack(out) + loose = loose_files(cache) + keys = sorted(set(packed) | set(loose)) + tmp = out.with_name(PACK + '.tmp') + index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys) + old = open(out, 'rb') if packed else None + try: + with open(tmp, 'wb') as f: + f.write(bytes(offset)) # header and index, written once the offsets are known + for key in keys: + data = None + if key in loose: # a valid loose file, else the packed copy, else nothing + try: + data = loose[key].read_bytes() + except OSError: + pass + if data is not None and valid(key[0], data): + written.append(loose[key]) + else: + data = None + if data is None and key in packed: + old.seek(packed[key][0]) + data = old.read(packed[key][1]) + if data is None or not valid(key[0], data): + skipped += 1 + continue + f.write(data) + index.append(struct.pack('<32sIIQ', key[1], key[0], len(data), offset)) + offset += len(data) + f.seek(0) + f.write(MAGIC + struct.pack(' Date: Fri, 30 Oct 2026 07:27:51 +0200 Subject: [PATCH 28/50] Test tools/pack_shader_cache.py on a fake shader cache --- tests/pack_shader_cache_test.py | 111 ++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/pack_shader_cache_test.py diff --git a/tests/pack_shader_cache_test.py b/tests/pack_shader_cache_test.py new file mode 100644 index 00000000..53356ef7 --- /dev/null +++ b/tests/pack_shader_cache_test.py @@ -0,0 +1,111 @@ +"""tools/pack_shader_cache.py on a fake shader cache: the pack layout gfx_shader.c reads (sorted index, bounds, data), +loose files folded in and deleted, a second run merging new loose files, a loose file winning over its packed copy, +malformed files skipped and --keep. No game files; stdlib only. + +Run: python tests/pack_shader_cache_test.py +""" +import hashlib +import struct +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +import pack_shader_cache as psc # noqa: E402 + +USE = struct.pack('<5I', 0x4d554b54, 1, 3, 0, 1) # gfx_shader.c use_store + + +def name(seed): + return hashlib.sha256(seed.encode()).hexdigest() + + +def read_pack(path): + """{(kind, name hex): bytes}, checking what gfx_shader.c pack_open checks.""" + b = path.read_bytes() + assert b[:8] == psc.MAGIC + count = struct.unpack_from('= 16 + 48 * count + assert last is None or (last[0], last[1]) < (kind, nm), 'index not sorted by (kind, name)' + last = (kind, nm) + out[(kind, nm.hex())] = b[offset:offset + size] + return out + + +class PackTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.cache = Path(self.tmp.name) / 'shader-cache' + (self.cache / 'in').mkdir(parents=True) + + def tearDown(self): + self.tmp.cleanup() + + def shader(self, seed, spv=b'\x03\x02\x23\x07spirv', res=b'{"x":1}'): + k = name(seed) + (self.cache / f'{k}.spv').write_bytes(spv + seed.encode()) + (self.cache / f'{k}.res.json').write_bytes(res) + (self.cache / f'{name("use" + seed)}.use').write_bytes(USE) + (self.cache / 'in').mkdir(exist_ok=True) # in_store makes it again after a pack removed it + (self.cache / 'in' / f'{name("in" + seed)}.in').write_bytes(b'TKSI' + seed.encode()) + return k + + def test_pack_holds_every_file_and_deletes_the_loose_ones(self): + keys = [self.shader(str(i)) for i in range(50)] + (self.cache / 'notes.txt').write_text('kept') + (self.cache / f'{keys[0]}.code').write_bytes(b'kept too') + n, new, skipped, _ = psc.pack(self.cache) + self.assertEqual((n, new, skipped), (200, 200, 0)) + got = read_pack(self.cache / psc.PACK) + self.assertEqual(got[(1, keys[3])], b'\x03\x02\x23\x07spirv3') + self.assertEqual(got[(2, keys[3])], b'{"x":1}') + self.assertEqual(got[(3, name('use3'))], USE) + self.assertEqual(got[(4, name('in3'))], b'TKSI3') + left = sorted(p.name for p in self.cache.iterdir()) + self.assertEqual(left, sorted([psc.PACK, 'notes.txt', f'{keys[0]}.code'])) # in/ removed when empty + + def test_second_run_merges_new_loose_files_and_loose_wins(self): + a = self.shader('a') + psc.pack(self.cache) + b = self.shader('b') + (self.cache / f'{a}.res.json').write_bytes(b'{"newer":1}') + n, new, _, _ = psc.pack(self.cache) + got = read_pack(self.cache / psc.PACK) + self.assertEqual(n, 8) + self.assertEqual(new, 5) + self.assertEqual(got[(2, a)], b'{"newer":1}') + self.assertEqual(got[(1, b)], b'\x03\x02\x23\x07spirvb') + + def test_malformed_files_are_skipped_or_replaced_by_the_packed_copy(self): + a = self.shader('a') + psc.pack(self.cache) + (self.cache / f'{name("usea")}.use').write_bytes(b'short') # bad loose copy: the packed one stays + (self.cache / f'{name("bad")}.use').write_bytes(b'short') # bad and nowhere else: skipped + (self.cache / f'{name("empty")}.spv').write_bytes(b'') + n, _, skipped, _ = psc.pack(self.cache) + got = read_pack(self.cache / psc.PACK) + self.assertEqual(got[(3, name('usea'))], USE) + self.assertNotIn((3, name('bad')), got) + self.assertEqual((n, skipped), (4, 2)) + self.assertTrue((self.cache / f'{name("bad")}.use').exists()) # never deleted unless packed + + def test_keep_leaves_the_loose_files(self): + k = self.shader('a') + psc.pack(self.cache, keep=True) + self.assertTrue((self.cache / f'{k}.spv').exists()) + self.assertEqual(len(read_pack(self.cache / psc.PACK)), 4) + + def test_a_broken_pack_is_rebuilt_from_the_loose_files(self): + self.shader('a') + (self.cache / psc.PACK).write_bytes(b'garbage') + n, _, _, _ = psc.pack(self.cache) + self.assertEqual(n, 4) + + +if __name__ == '__main__': + unittest.main() From 00da267dcd092f75a6c040aeca3b8b7d287e9a49 Mon Sep 17 00:00:00 2001 From: Pasha Kagan Date: Fri, 30 Oct 2026 07:29:01 +0200 Subject: [PATCH 29/50] Run tools/pack_shader_cache.py at the end of the prepare step --- installer/steps.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/installer/steps.py b/installer/steps.py index 9d31fb13..b2109fa1 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -587,7 +587,8 @@ class Prepare(Step): def key(self, ctx): layout = ctx.layout xlate_id = layout.xlate.with_suffix('.id') - tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py')] + tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py', + 'pack_shader_cache.py')] return fingerprint('prepare-1', ctx.keys['compile'], layout.xlate, xlate_id, shader_list_text(layout), *tools) def current(self, ctx, record): @@ -638,6 +639,12 @@ class Prepare(Step): dest / 'exefs', '--list', shader_list], parse) if code or not (dest / 'shader-prepared.bin').is_file(): raise InstallError('prepare.shaders_failed', code=code, log=str(log)) + + # One shader-cache.pack instead of four files per shader (~117,000): totk.exe reads the pack first. + rep.progress('prepare', 'pack', force=True) + code, log = run_tool(ctx, 'prepare', layout.tools / 'pack_shader_cache.py', [dest / 'shader-cache']) + if code: + raise InstallError('prepare.pack_failed', code=code, log=str(log)) return {'xlate_sha256': file_sha256(xlate)} From dc5bd01a152bb8c7fabc9e39fc469b90c1f5ebfd Mon Sep 17 00:00:00 2001 From: Pasha Kagan Date: Fri, 30 Oct 2026 07:29:42 +0200 Subject: [PATCH 30/50] List tools/pack_shader_cache.py among the files a release needs --- installer/layout.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installer/layout.py b/installer/layout.py index 655c7c22..affcb63b 100644 --- a/installer/layout.py +++ b/installer/layout.py @@ -72,7 +72,8 @@ class Layout: def missing(self): """Paths the steps need that do not exist.""" need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py', - self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', self.clang, + self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', + self.clang, self.deps / 'include', self.deps / 'lib', self.runtime, self.runtime_lib, self.ninja, self.hactool, self.xlate, *self.dlls, *(f(v) for v in SUPPORTED for f in (self.patches, self.expected))] if self.shader_list: From 06ae18fe933fa280d8b5973ec09369d72fe0d40a Mon Sep 17 00:00:00 2001 From: Pasha Kagan Date: Fri, 30 Oct 2026 07:30:32 +0200 Subject: [PATCH 31/50] Add the English texts of the pack phase and its failure --- installer/text/en.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/installer/text/en.json b/installer/text/en.json index 3425f5a5..5e8a5912 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -26,6 +26,7 @@ "phase.prepare.seed": "Reading the shaders from the game data", "phase.prepare.translate": "Translating the shaders", "phase.prepare.libraries": "Building the shaders for the graphics driver", + "phase.prepare.pack": "Packing the shader cache into one file", "phase.cleanup.ini": "Writing totk.ini", "phase.cleanup.temp": "Removing temporary files", @@ -76,6 +77,7 @@ "prepare.romfs_patches_failed": "Writing the game data patches failed (exit code {code}). Details: {log}", "prepare.shaders_failed": "Preparing the shaders failed (exit code {code}). Details: {log}", "cleanup.romfs_unreadable": "The game data in {path} is missing or cannot be read after the install. Restart Windows, then run the installer again on the same folder: it extracts the game data again and keeps the rest.", + "prepare.pack_failed": "Packing the shader cache failed (exit code {code}). Details: {log}", "cleanup.ini_failed": "totk.ini cannot be written: {detail}", "warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts, which the installer does not extract; some text may not show.", From 82af209b2b3a9f80ebf301878aabb35010837fd9 Mon Sep 17 00:00:00 2001 From: Pasha Kagan Date: Fri, 30 Oct 2026 07:31:03 +0200 Subject: [PATCH 32/50] Add the Russian texts of the pack phase and its failure --- installer/text/ru.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/installer/text/ru.json b/installer/text/ru.json index b1dc33ae..fa4d3a76 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -26,6 +26,7 @@ "phase.prepare.seed": "Чтение шейдеров из данных игры", "phase.prepare.translate": "Трансляция шейдеров", "phase.prepare.libraries": "Сборка шейдеров для драйвера видеокарты", + "phase.prepare.pack": "Упаковка кэша шейдеров в один файл", "phase.cleanup.ini": "Запись totk.ini", "phase.cleanup.temp": "Удаление временных файлов", @@ -76,6 +77,7 @@ "prepare.romfs_patches_failed": "Не удалось записать патчи данных игры (код выхода {code}). Подробности: {log}", "prepare.shaders_failed": "Не удалось подготовить шейдеры (код выхода {code}). Подробности: {log}", "cleanup.romfs_unreadable": "Данные игры в {path} пропали или не читаются после установки. Перезагрузите Windows и запустите установщик снова на ту же папку: он заново распакует данные игры, остальное сохранится.", + "prepare.pack_failed": "Не удалось упаковать кэш шейдеров (код выхода {code}). Подробности: {log}", "cleanup.ini_failed": "Не удаётся записать totk.ini: {detail}", "warn.lang_needs_fonts": "Для языка «{lang}» нужны системные шрифты Switch, которые установка не извлекает; часть текста может не отображаться.", From 58b15ed06977cd91a5b17a2dbc2c9f4f4b5c2393 Mon Sep 17 00:00:00 2001 From: Pasha Kagan Date: Fri, 30 Oct 2026 07:31:33 +0200 Subject: [PATCH 33/50] Copy tools/pack_shader_cache.py into the release --- tools/make_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make_release.py b/tools/make_release.py index 9fd9d33c..9c3c5f92 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -98,7 +98,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py', - 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py'] + 'pack_shader_cache.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon From 4d871c7920490ca82b3e024317424db396803300 Mon Sep 17 00:00:00 2001 From: Pasha Kagan Date: Fri, 30 Oct 2026 07:31:56 +0200 Subject: [PATCH 34/50] Describe the shader cache pack in the installer documentation --- docs/installer.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/installer.md b/docs/installer.md index b6157b83..a3158d28 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -24,7 +24,7 @@ The install folder (`main.c` `portable_paths()` reads this layout; it stays port exefs/, romfs/ extracted from the player's files profile/ saves logs/ totk.log of the last run, totk.1.log to totk.4.log of the runs before it - shader and pipeline caches prepared by the setup + shader and pipeline caches prepared by the setup; shader-cache/shader-cache.pack holds the translations totk-install.json what built this folder: release version, input hashes, finished steps ``` @@ -50,7 +50,9 @@ keeps `profile/`. Saves carry over both ways: 1.4.3 converts a 1.2.1 save on loa hashes and the game version's patch page) and link them with the prebuilt runtime library into `totk.exe` (`build_game.py --runtime-lib`; about 3 min on 24 cores). 5. **Prepare** the RomFS patches and the shaders (`tools/make_romfs_patches.py`, `tools/prepare_shaders.py`, - `totk.exe --prepare-shaders`) in the install folder. + `totk.exe --prepare-shaders`) in the install folder, then fold the shader cache's loose files (four per shader, about + 117,000) into `shader-cache/shader-cache.pack` (`tools/pack_shader_cache.py`). totk.exe reads the pack first and + writes the translations it makes in play as loose files; running the tool again with the game closed folds those in. 6. **Clean up** the temp folder (generated C and objects, about 1.3 GB). A cancelled or failed setup resumes from the last finished step (`totk-install.json`). From 70a83837b0ca76001df310e482a209935ddb64b4 Mon Sep 17 00:00:00 2001 From: Seth <29849850+shin2344234@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:54:59 -0500 Subject: [PATCH 35/50] Let the back paddles be bound SDL reports the back buttons of an Xbox Elite, DualSense Edge or Steam Controller as four paddle buttons, but the pad table stopped at the triggers, so they could not be mapped. They are now pad inputs lpaddle1, rpaddle1, lpaddle2 and rpaddle2, with no default action, bindable from totk.ini or the remapping rows. --- docs/controls.md | 2 +- runtime/cfg_probe.c | 7 +++++++ runtime/input_map.c | 2 +- runtime/input_map.h | 5 ++++- runtime/input_win.c | 3 +++ runtime/ui/binds_page.cpp | 3 ++- runtime/ui_text.c | 4 ++++ runtime/ui_text.h | 2 +- totk.ini.example | 1 + 9 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/controls.md b/docs/controls.md index a351fdcd..0cceb44a 100644 --- a/docs/controls.md +++ b/docs/controls.md @@ -46,7 +46,7 @@ The *Controls* tab of the settings menu lists, under the button layout, every Sw The tab writes `totk.ini` lines, one per Switch button or stick direction: `bind_