From b358b8f1f16e0300ffc70505e2f85c5e31c036ee Mon Sep 17 00:00:00 2001 From: itaskaev2 <165683571+itaskaev2@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:51:23 +0100 Subject: [PATCH 1/5] Keep pipeline builds apart from the cache copy, so Intel's driver does not fault in it Co-Authored-By: Claude Opus 5.5 --- runtime/vk_pipeline.c | 50 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index dedf7a37..031ee897 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -101,6 +101,12 @@ static struct { VkPipeStats stats; } P = {.lock = PLAT_MUTEX_INIT}; +// Builds into P.cache take it shared, vk_pipeline_save takes it to copy the cache out. Vulkan lets +// vkGetPipelineCacheData run alongside builds into a cache created without the EXTERNALLY_SYNCHRONIZED bit, but +// Intel's Windows driver (101.7085, Iris Xe) faults in it (igvk64.dll, reading 0x170) when the first autosave meets the +// boot's pipeline builds (#681). Nothing is taken while it is held. +static PlatMutex cache_use = PLAT_MUTEX_INIT; + typedef struct Layout { struct Layout *next; uint64_t hash; @@ -1440,7 +1446,9 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay if (k->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; if (k->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; uint64_t p0 = perf_begin(); + plat_mutex_lock_shared(&cache_use); VkResult r = vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe); + plat_mutex_unlock_shared(&cache_use); perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it if (r != VK_SUCCESS) { rt_log("vk_pipeline: vkCreateGraphicsPipelines failed (%d)", (int)r); @@ -1591,7 +1599,9 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil; pi.pColorBlendState = &cb, pi.pMultisampleState = &ms; } + plat_mutex_lock_shared(&cache_use); if (vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + plat_mutex_unlock_shared(&cache_use); out: if (m) vkDestroyShaderModule(P.device, m, NULL); free(patched); @@ -1803,7 +1813,8 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc // shaders (gfx_shader_get results) live for the process (gfx_shader_drop_memory is the shader probe's, which builds no // pipelines), layouts until vk_pipeline_shutdown, which drains the queue first. Workers call vkCreateGraphicsPipelines // without P.lock: P.cache is created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT, so the driver -// synchronises it, vkGetPipelineCacheData in vk_pipeline_save included. Compute pipelines stay synchronous: a skipped +// synchronises the builds among themselves. vk_pipeline_save still keeps them apart from vkGetPipelineCacheData (see +// cache_use). Compute pipelines stay synchronous: a skipped // dispatch can change what the game computes. // // Two queues: draws' misses (A.head) and the boot prebuild (A.low, see "pipeline keys" below). Workers take the low @@ -2512,7 +2523,7 @@ static void prebuild_start(void) { plat_mutex_unlock(&P.lock); } -// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised). +// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. Takes cache_use shared, like the graphics builds. static VkPipeline compute_create(const GfxShader *cs, const Layout *l) { uint64_t words; const uint32_t *code = gfx_shader_spirv(cs, &words); @@ -2525,7 +2536,9 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l) { .layout = l->layout}; pin_subgroup32(&ci.stage, &rqc, code, words); VkPipeline pipe = VK_NULL_HANDLE; + plat_mutex_lock_shared(&cache_use); if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + plat_mutex_unlock_shared(&cache_use); vkDestroyShaderModule(P.device, m, NULL); return pipe; } @@ -3290,7 +3303,8 @@ void vk_pipeline_stats(VkPipeStats *s) { static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache -void vk_pipeline_save(void) { +// One cache save; wait: queue for cache_use (the exit saves) rather than try for it (the autosave). +static void cache_save(int wait) { if (P.ready) keys_flush(); if (P.ready && P.gpl) { rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws " @@ -3317,23 +3331,39 @@ void vk_pipeline_save(void) { } gfx_shader_record_flush(1); if (!P.ready || !P.cache || !P.cache_path[0]) return; - // Not under P.lock: the driver synchronises P.cache, and holding P.lock through the copy and the file write (about - // 60 ms on the 1.4.3 fight) stopped every pipeline lookup of the replay, a freeze at each 20 s autosave. + // Not under P.lock: holding it through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped every + // pipeline lookup of the replay, a freeze at each 20 s autosave. Only the copy takes cache_use, so no pipeline is + // being built into P.cache while the driver reads it. plat_mutex_lock(&save_lock); if (!P.cache) { // vk_pipeline_shutdown destroyed it plat_mutex_unlock(&save_lock); return; } + if (wait) { + plat_mutex_lock(&cache_use); + } else { + // The autosave never queues for cache_use: a queued exclusive request on an SRW lock holds back new shared + // ones, so the drawing thread's next build would wait for a worker's build to end. It tries for about 2 s + // and otherwise saves on its next round. + int got = 0; + for (int i = 0; i < 1000 && !(got = plat_mutex_trylock(&cache_use)); i++) plat_sleep_ms(2); + if (!got) { + plat_mutex_unlock(&save_lock); + return; + } + } size_t size = 0; void *data = NULL; - if (vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) && - vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS) { - if (!plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path); - } + int ok = vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) && + vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS; + plat_mutex_unlock(&cache_use); + if (ok && !plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path); free(data); plat_mutex_unlock(&save_lock); } +void vk_pipeline_save(void) { cache_save(1); } + // The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it // runs: at most every 20 s, and only after new pipelines were built. New pipeline keys are appended every 2 s. static void autosave_once(void) { @@ -3347,7 +3377,7 @@ static void autosave_once(void) { vk_pipeline_stats(&s); if (s.graphics + s.compute == last_n) return; last_n = s.graphics + s.compute, last_ms = now; - vk_pipeline_save(); + cache_save(0); } // On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to From 57037ccc2978131f1bb9a422f23d9df2218c5e05 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 7 Nov 2026 16:38:26 +0300 Subject: [PATCH 2/5] Deswizzle block-linear textures on the GPU when their storage is an imported pool The replay thread deswizzled every block-linear texture on the CPU, and entering a new area stalled it for 50-236 ms. Now a compute pass reads the texture straight from the pool's imported buffer and writes a linear copy into a device-local scratch pool; the upload batch copies it into the image, or the ASTC decoder takes it. Large texture loads submit the batch early so the staging and scratch pools do not grow on the replay thread, and both pools are made at device creation. On fight2 the render thread waits about 270 ms on textures per run instead of 900, and the load frame drops from 145 to about 80 ms. Mean fps is unchanged. TOTK_DESWIZZLE=cpu turns it off and check compares every upload with the CPU result (0 of 196k regions differ). Shadow mode (Linux/Wine) and textures outside the device's limits stay on the CPU. Co-Authored-By: Claude Opus 5.5 --- runtime/bl_deswizzle_spv.h | 111 +++++++++++++++ runtime/cfg.c | 2 +- runtime/shaders/bl_deswizzle.comp | 37 +++++ runtime/vk.h | 27 ++++ runtime/vk_astc.c | 6 +- runtime/vk_deswizzle.c | 176 ++++++++++++++++++++++++ runtime/vk_device.c | 3 +- runtime/vk_memory.c | 66 +++++++++ runtime/vk_texture.c | 216 +++++++++++++++++++++++++++++- 9 files changed, 633 insertions(+), 11 deletions(-) create mode 100644 runtime/bl_deswizzle_spv.h create mode 100644 runtime/shaders/bl_deswizzle.comp create mode 100644 runtime/vk_deswizzle.c diff --git a/runtime/bl_deswizzle_spv.h b/runtime/bl_deswizzle_spv.h new file mode 100644 index 00000000..4bb18500 --- /dev/null +++ b/runtime/bl_deswizzle_spv.h @@ -0,0 +1,111 @@ +// Generated by tools/build_shaders.py from runtime/shaders/bl_deswizzle.comp; do not edit. +// source sha256 bd2065290e315e85d2fcdb609d812c15ba9a109d7506b3105850a5860677b0ef +#pragma once +#include +static const uint32_t bl_deswizzle_spv[] = { + 0x07230203, 0x00010300, 0x000d000b, 0x000000d0, 0x00000000, 0x00020011, 0x00000001, 0x0006000b, + 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001, + 0x0007000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000b, 0x0000001f, 0x00060010, + 0x00000004, 0x00000011, 0x00000040, 0x00000001, 0x00000001, 0x00040047, 0x0000000b, 0x0000000b, + 0x0000001a, 0x00030047, 0x00000010, 0x00000002, 0x00050048, 0x00000010, 0x00000000, 0x00000023, + 0x00000000, 0x00050048, 0x00000010, 0x00000001, 0x00000023, 0x00000004, 0x00050048, 0x00000010, + 0x00000002, 0x00000023, 0x00000008, 0x00050048, 0x00000010, 0x00000003, 0x00000023, 0x0000000c, + 0x00050048, 0x00000010, 0x00000004, 0x00000023, 0x00000010, 0x00050048, 0x00000010, 0x00000005, + 0x00000023, 0x00000014, 0x00050048, 0x00000010, 0x00000006, 0x00000023, 0x00000018, 0x00050048, + 0x00000010, 0x00000007, 0x00000023, 0x0000001c, 0x00050048, 0x00000010, 0x00000008, 0x00000023, + 0x00000020, 0x00050048, 0x00000010, 0x00000009, 0x00000023, 0x00000024, 0x00050048, 0x00000010, + 0x0000000a, 0x00000023, 0x00000028, 0x00040047, 0x0000001f, 0x0000000b, 0x0000001b, 0x00040047, + 0x000000ad, 0x00000006, 0x00000010, 0x00030047, 0x000000ae, 0x00000002, 0x00040048, 0x000000ae, + 0x00000000, 0x00000019, 0x00050048, 0x000000ae, 0x00000000, 0x00000023, 0x00000000, 0x00030047, + 0x000000b0, 0x00000019, 0x00040047, 0x000000b0, 0x00000021, 0x00000001, 0x00040047, 0x000000b0, + 0x00000022, 0x00000000, 0x00040047, 0x000000c1, 0x00000006, 0x00000010, 0x00030047, 0x000000c2, + 0x00000002, 0x00040048, 0x000000c2, 0x00000000, 0x00000018, 0x00050048, 0x000000c2, 0x00000000, + 0x00000023, 0x00000000, 0x00030047, 0x000000c4, 0x00000018, 0x00040047, 0x000000c4, 0x00000021, + 0x00000000, 0x00040047, 0x000000c4, 0x00000022, 0x00000000, 0x00040047, 0x000000cd, 0x0000000b, + 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006, + 0x00000020, 0x00000000, 0x00040017, 0x00000009, 0x00000006, 0x00000003, 0x00040020, 0x0000000a, + 0x00000001, 0x00000009, 0x0004003b, 0x0000000a, 0x0000000b, 0x00000001, 0x0004002b, 0x00000006, + 0x0000000c, 0x00000001, 0x00040020, 0x0000000d, 0x00000001, 0x00000006, 0x000d001e, 0x00000010, + 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, + 0x00000006, 0x00000006, 0x00000006, 0x00040020, 0x00000011, 0x00000009, 0x00000010, 0x0004003b, + 0x00000011, 0x00000012, 0x00000009, 0x00040015, 0x00000013, 0x00000020, 0x00000001, 0x0004002b, + 0x00000013, 0x00000014, 0x0000000a, 0x00040020, 0x00000015, 0x00000009, 0x00000006, 0x0004002b, + 0x00000006, 0x00000019, 0x00000000, 0x0004002b, 0x00000006, 0x0000001d, 0x00000040, 0x0004003b, + 0x0000000a, 0x0000001f, 0x00000001, 0x0004002b, 0x00000013, 0x00000024, 0x00000002, 0x00020014, + 0x00000027, 0x0004002b, 0x00000013, 0x0000002d, 0x00000006, 0x0004002b, 0x00000013, 0x00000032, + 0x00000007, 0x0004002b, 0x00000006, 0x00000037, 0x00000020, 0x0004002b, 0x00000006, 0x00000048, + 0x00000005, 0x0004002b, 0x00000006, 0x0000004c, 0x0000001f, 0x0004002b, 0x00000013, 0x0000004f, + 0x00000008, 0x0004002b, 0x00000013, 0x00000052, 0x00000009, 0x0004002b, 0x00000006, 0x0000006a, + 0x00000004, 0x0004002b, 0x00000006, 0x0000006f, 0x00000002, 0x0004002b, 0x00000006, 0x00000078, + 0x00000008, 0x0004002b, 0x00000006, 0x00000083, 0x00000003, 0x0004002b, 0x00000013, 0x00000093, + 0x00000003, 0x0004002b, 0x00000013, 0x0000009b, 0x00000004, 0x0004002b, 0x00000013, 0x000000a4, + 0x00000005, 0x00040017, 0x000000ac, 0x00000006, 0x00000004, 0x0003001d, 0x000000ad, 0x000000ac, + 0x0003001e, 0x000000ae, 0x000000ad, 0x00040020, 0x000000af, 0x0000000c, 0x000000ae, 0x0004003b, + 0x000000af, 0x000000b0, 0x0000000c, 0x0004002b, 0x00000013, 0x000000b1, 0x00000000, 0x0004002b, + 0x00000013, 0x000000b2, 0x00000001, 0x0003001d, 0x000000c1, 0x000000ac, 0x0003001e, 0x000000c2, + 0x000000c1, 0x00040020, 0x000000c3, 0x0000000c, 0x000000c2, 0x0004003b, 0x000000c3, 0x000000c4, + 0x0000000c, 0x00040020, 0x000000c9, 0x0000000c, 0x000000ac, 0x0006002c, 0x00000009, 0x000000cd, + 0x0000001d, 0x0000000c, 0x0000000c, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, + 0x000200f8, 0x00000005, 0x000300f7, 0x000000ce, 0x00000000, 0x000300fb, 0x00000019, 0x000000cf, + 0x000200f8, 0x000000cf, 0x00050041, 0x0000000d, 0x0000000e, 0x0000000b, 0x0000000c, 0x0004003d, + 0x00000006, 0x0000000f, 0x0000000e, 0x00050041, 0x00000015, 0x00000016, 0x00000012, 0x00000014, + 0x0004003d, 0x00000006, 0x00000017, 0x00000016, 0x00050084, 0x00000006, 0x00000018, 0x0000000f, + 0x00000017, 0x00050041, 0x0000000d, 0x0000001a, 0x0000000b, 0x00000019, 0x0004003d, 0x00000006, + 0x0000001b, 0x0000001a, 0x00050080, 0x00000006, 0x0000001c, 0x00000018, 0x0000001b, 0x00050084, + 0x00000006, 0x0000001e, 0x0000001c, 0x0000001d, 0x00050041, 0x0000000d, 0x00000020, 0x0000001f, + 0x00000019, 0x0004003d, 0x00000006, 0x00000021, 0x00000020, 0x00050080, 0x00000006, 0x00000022, + 0x0000001e, 0x00000021, 0x00050041, 0x00000015, 0x00000025, 0x00000012, 0x00000024, 0x0004003d, + 0x00000006, 0x00000026, 0x00000025, 0x000500ae, 0x00000027, 0x00000028, 0x00000022, 0x00000026, + 0x000300f7, 0x0000002a, 0x00000000, 0x000400fa, 0x00000028, 0x00000029, 0x0000002a, 0x000200f8, + 0x00000029, 0x000200f9, 0x000000ce, 0x000200f8, 0x0000002a, 0x00050041, 0x00000015, 0x0000002e, + 0x00000012, 0x0000002d, 0x0004003d, 0x00000006, 0x0000002f, 0x0000002e, 0x000500c4, 0x00000006, + 0x00000030, 0x0000000c, 0x0000002f, 0x00050041, 0x00000015, 0x00000033, 0x00000012, 0x00000032, + 0x0004003d, 0x00000006, 0x00000034, 0x00000033, 0x000500c4, 0x00000006, 0x00000035, 0x0000000c, + 0x00000034, 0x00050080, 0x00000006, 0x0000003c, 0x0000002f, 0x00000034, 0x000500c4, 0x00000006, + 0x0000003d, 0x00000037, 0x0000003c, 0x00050086, 0x00000006, 0x00000041, 0x00000022, 0x0000003d, + 0x00050089, 0x00000006, 0x00000045, 0x00000022, 0x0000003d, 0x000500c2, 0x00000006, 0x00000049, + 0x00000045, 0x00000048, 0x000500c7, 0x00000006, 0x0000004d, 0x00000045, 0x0000004c, 0x00050041, + 0x00000015, 0x00000050, 0x00000012, 0x0000004f, 0x0004003d, 0x00000006, 0x00000051, 0x00000050, + 0x00050041, 0x00000015, 0x00000053, 0x00000012, 0x00000052, 0x0004003d, 0x00000006, 0x00000054, + 0x00000053, 0x00050084, 0x00000006, 0x00000055, 0x00000051, 0x00000054, 0x00050086, 0x00000006, + 0x00000059, 0x00000041, 0x00000055, 0x00050089, 0x00000006, 0x0000005d, 0x00000041, 0x00000055, + 0x00050086, 0x00000006, 0x00000062, 0x0000005d, 0x00000051, 0x00050089, 0x00000006, 0x00000067, + 0x0000005d, 0x00000051, 0x00050084, 0x00000006, 0x0000006b, 0x00000067, 0x0000006a, 0x000500c2, + 0x00000006, 0x0000006d, 0x0000004d, 0x0000006a, 0x000500c7, 0x00000006, 0x0000006e, 0x0000006d, + 0x0000000c, 0x00050084, 0x00000006, 0x00000070, 0x0000006e, 0x0000006f, 0x00050080, 0x00000006, + 0x00000071, 0x0000006b, 0x00000070, 0x000500c2, 0x00000006, 0x00000073, 0x0000004d, 0x0000000c, + 0x000500c7, 0x00000006, 0x00000074, 0x00000073, 0x0000000c, 0x00050080, 0x00000006, 0x00000075, + 0x00000071, 0x00000074, 0x00050084, 0x00000006, 0x00000079, 0x00000062, 0x00000078, 0x00050084, + 0x00000006, 0x0000007b, 0x00000079, 0x00000030, 0x00050089, 0x00000006, 0x0000007e, 0x00000049, + 0x00000030, 0x00050084, 0x00000006, 0x0000007f, 0x0000007e, 0x00000078, 0x00050080, 0x00000006, + 0x00000080, 0x0000007b, 0x0000007f, 0x000500c2, 0x00000006, 0x00000082, 0x0000004d, 0x0000006f, + 0x000500c7, 0x00000006, 0x00000084, 0x00000082, 0x00000083, 0x00050084, 0x00000006, 0x00000085, + 0x00000084, 0x0000006f, 0x00050080, 0x00000006, 0x00000086, 0x00000080, 0x00000085, 0x000500c7, + 0x00000006, 0x00000088, 0x00000045, 0x0000000c, 0x00050080, 0x00000006, 0x00000089, 0x00000086, + 0x00000088, 0x00050084, 0x00000006, 0x0000008d, 0x00000059, 0x00000035, 0x00050086, 0x00000006, + 0x00000090, 0x00000049, 0x00000030, 0x00050080, 0x00000006, 0x00000091, 0x0000008d, 0x00000090, + 0x00050041, 0x00000015, 0x00000094, 0x00000012, 0x00000093, 0x0004003d, 0x00000006, 0x00000095, + 0x00000094, 0x000500ae, 0x00000027, 0x00000096, 0x00000075, 0x00000095, 0x000400a8, 0x00000027, + 0x00000097, 0x00000096, 0x000300f7, 0x00000099, 0x00000000, 0x000400fa, 0x00000097, 0x00000098, + 0x00000099, 0x000200f8, 0x00000098, 0x00050041, 0x00000015, 0x0000009c, 0x00000012, 0x0000009b, + 0x0004003d, 0x00000006, 0x0000009d, 0x0000009c, 0x000500ae, 0x00000027, 0x0000009e, 0x00000089, + 0x0000009d, 0x000200f9, 0x00000099, 0x000200f8, 0x00000099, 0x000700f5, 0x00000027, 0x0000009f, + 0x00000096, 0x0000002a, 0x0000009e, 0x00000098, 0x000400a8, 0x00000027, 0x000000a0, 0x0000009f, + 0x000300f7, 0x000000a2, 0x00000000, 0x000400fa, 0x000000a0, 0x000000a1, 0x000000a2, 0x000200f8, + 0x000000a1, 0x00050041, 0x00000015, 0x000000a5, 0x00000012, 0x000000a4, 0x0004003d, 0x00000006, + 0x000000a6, 0x000000a5, 0x000500ae, 0x00000027, 0x000000a7, 0x00000091, 0x000000a6, 0x000200f9, + 0x000000a2, 0x000200f8, 0x000000a2, 0x000700f5, 0x00000027, 0x000000a8, 0x0000009f, 0x00000099, + 0x000000a7, 0x000000a1, 0x000300f7, 0x000000aa, 0x00000000, 0x000400fa, 0x000000a8, 0x000000a9, + 0x000000aa, 0x000200f8, 0x000000a9, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000aa, 0x00050041, + 0x00000015, 0x000000b3, 0x00000012, 0x000000b2, 0x0004003d, 0x00000006, 0x000000b4, 0x000000b3, + 0x00050041, 0x00000015, 0x000000b6, 0x00000012, 0x0000009b, 0x0004003d, 0x00000006, 0x000000b7, + 0x000000b6, 0x00050084, 0x00000006, 0x000000b8, 0x00000091, 0x000000b7, 0x00050080, 0x00000006, + 0x000000ba, 0x000000b8, 0x00000089, 0x00050084, 0x00000006, 0x000000bd, 0x000000ba, 0x00000095, + 0x00050080, 0x00000006, 0x000000be, 0x000000b4, 0x000000bd, 0x00050080, 0x00000006, 0x000000c0, + 0x000000be, 0x00000075, 0x00050041, 0x00000015, 0x000000c5, 0x00000012, 0x000000b1, 0x0004003d, + 0x00000006, 0x000000c6, 0x000000c5, 0x00050080, 0x00000006, 0x000000c8, 0x000000c6, 0x00000022, + 0x00060041, 0x000000c9, 0x000000ca, 0x000000c4, 0x000000b1, 0x000000c8, 0x0004003d, 0x000000ac, + 0x000000cb, 0x000000ca, 0x00060041, 0x000000c9, 0x000000cc, 0x000000b0, 0x000000b1, 0x000000c0, + 0x0003003e, 0x000000cc, 0x000000cb, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000ce, 0x000100fd, + 0x00010038, +}; diff --git a/runtime/cfg.c b/runtime/cfg.c index b913d3b8..a660c26c 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -40,7 +40,7 @@ static const char *const known[] = { "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", - "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", + "DESC_TEMPLATE", "DESWIZZLE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DYN_DELTA", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FRAME_QUEUE", "FREEZE_IDLE", "FREE_CORES", diff --git a/runtime/shaders/bl_deswizzle.comp b/runtime/shaders/bl_deswizzle.comp new file mode 100644 index 00000000..f7a56e72 --- /dev/null +++ b/runtime/shaders/bl_deswizzle.comp @@ -0,0 +1,37 @@ +#version 450 +// Block-linear to linear copy of one texture level and layer on the GPU (vk_deswizzle.c), the same layout as +// tex_deswizzle.c nvn_bl_offset: GOB = 64 bytes x 8 rows; a block is 1 GOB wide, 2^th GOBs high and 2^td deep, its +// GOBs ordered y first, then z; blocks fill the level x first, then y, then z. +// One invocation per 16-byte segment of the block-linear storage, in storage order, so the reads (from the pool's +// imported buffer: guest memory, over PCIe on a discrete GPU) are contiguous; the writes go to a device-local buffer +// with rows of pitch16 segments. Segments outside the level's rows, slices or row width are padding and skipped. +layout(local_size_x = 64) in; +layout(std430, set = 0, binding = 0) readonly buffer Src { uvec4 src[]; }; +layout(std430, set = 0, binding = 1) writeonly buffer Dst { uvec4 dst[]; }; +layout(push_constant) uniform Push { + uint src_base; // first segment of the level and layer in the src binding + uint dst_base; // first segment of its linear copy in the dst binding + uint segs; // segments to visit: whole blocks covering the level + uint pitch16; // segments per linear row (row bytes rounded up to 16) + uint rows, slices, th, td; + uint gobs_x, blocks_y; + uint nx; // workgroups along x (the dispatch is 2D when one dimension is not enough) +} pc; + +void main() { + uint s = (gl_WorkGroupID.y * pc.nx + gl_WorkGroupID.x) * 64u + gl_LocalInvocationID.x; + if (s >= pc.segs) return; + uint bh = 1u << pc.th, bd = 1u << pc.td; + uint per_block = 32u << (pc.th + pc.td); // 512-byte GOBs of 32 segments each + uint block = s / per_block, inb = s % per_block; + uint gob = inb >> 5u, ing = inb & 31u; + uint per_layer = pc.gobs_x * pc.blocks_y; + uint bz = block / per_layer, rem = block % per_layer; + uint by = rem / pc.gobs_x, bx = rem % pc.gobs_x; + // Inside a GOB, segment bits: 4 = x % 64 / 32, 3..2 = y % 8 / 2, 1 = x % 32 / 16, 0 = y % 2. + uint xseg = bx * 4u + ((ing >> 4u) & 1u) * 2u + ((ing >> 1u) & 1u); + uint y = by * 8u * bh + (gob % bh) * 8u + ((ing >> 2u) & 3u) * 2u + (ing & 1u); + uint z = bz * bd + gob / bh; + if (xseg >= pc.pitch16 || y >= pc.rows || z >= pc.slices) return; + dst[pc.dst_base + (z * pc.rows + y) * pc.pitch16 + xseg] = src[pc.src_base + s]; +} diff --git a/runtime/vk.h b/runtime/vk.h index abe40f57..1f210afb 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -329,6 +329,12 @@ void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestro void vk_defer_view(VkImageView view); void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc); +// Device-local scratch for the upload batch (storage, transfer src and dst), from a pool of its own (vk_memory.c). +VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc); +// Staging and scratch bytes the open upload batch holds until it completes. Large texture loads submit the batch early +// (vk_texture.c UPLOAD_HELD_MAX) so the GPU starts on it and its buffers come back before the pools need new blocks. +VkDeviceSize vk_upload_held(void); +void vk_upload_pools_init(void); // vk_device.c, after the allocator: the staging and scratch pools and their blocks // ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c. enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK }; int vk_astc_mode(void); // decided once per device (logged); VK_ASTC_CPU before vk_init @@ -352,6 +358,27 @@ void vk_astc_time_enable(int on); int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us); // totals so far void vk_astc_time_summary(void); // logs the totals void vk_astc_release(void); // vk_device.c, before the device goes: frees the decoder's objects +// ---- Block-linear deswizzle on the GPU (vk_deswizzle.c). TOTK_DESWIZZLE = auto | gpu | cpu | check. +enum { VK_DESWZ_CPU, VK_DESWZ_GPU, VK_DESWZ_CHECK }; +int vk_deswz_mode(void); // decided once per device (logged); VK_DESWZ_CPU before vk_init +// One level and layer: its storage starts at byte src of the bound source range and covers segs 16-byte segments +// (whole blocks, cut at the storage end); its linear copy goes to byte dst of the destination, rows of pitch16 segments, +// rows x slices of them. th, td, gobs_x, blocks_y: as tex_deswizzle.c nvn_bl_offset. +typedef struct { + VkDeviceSize src, dst; + uint32_t segs, pitch16, rows, slices, th, td, gobs_x, blocks_y; +} VkDeswzRegion; +// Records into cb a dispatch per region reading [src_off, src_off + src_size) of src (a storage buffer) and writing +// dst (device-local, dst_size bytes). The caller records the barriers around it. 0 when nothing was recorded: no +// pipeline, src_off not 16-byte aligned, a range over the device's storage-buffer limit, a region outside the ranges. +int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst, + VkDeviceSize dst_size, const VkDeswzRegion *rg, int n); +// 1 when vk_deswz_record will take these regions (their dst offsets aside) with destinations of up to max_dst bytes. +int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst); +void vk_deswz_release(void); // vk_device.c, before the device goes +// 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission +// sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch). +int vk_mem_span_live(const VkMemSpan *sp); void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value void vk_mem_collect(void); // frees what has completed typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats; diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index 0b91e70f..ccfa4235 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -463,13 +463,9 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons cap_in = (cap_in + al - 1) / al * al; cap_out = (cap_out + al - 1) / al * al; if (!bc) cap_bc = 0; - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = cap_in + cap_out + cap_bc, - .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | - VK_BUFFER_USAGE_TRANSFER_DST_BIT}; - VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; VkBuffer sb; VmaAllocation sa; - if (vmaCreateBuffer(c->vma, &bi, &aci, &sb, &sa, NULL) != VK_SUCCESS) { + if (vk_scratch_buffer(cap_in + cap_out + cap_bc, &sb, &sa) != VK_SUCCESS) { if (plat_inc32(&n_fail) <= 16) rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded", (unsigned long long)(cap_in + cap_out)); diff --git a/runtime/vk_deswizzle.c b/runtime/vk_deswizzle.c new file mode 100644 index 00000000..d70f0886 --- /dev/null +++ b/runtime/vk_deswizzle.c @@ -0,0 +1,176 @@ +// Block-linear textures deswizzled by the GPU (runtime/shaders/bl_deswizzle.comp): vk_texture.c upload binds the +// texture's storage in its pool's imported buffer and lets a compute pass write the linear copy that a buffer-to-image +// copy (or the ASTC decoder) then takes, instead of tex_deswizzle.c walking every 16 bytes on the replay thread +// (50-236 ms frames on entering new areas, 2026-11-07 play sessions). The same design as MacNativeGames' TotK port +// (d2r_gfx nvn_mtl.m: deswizzle from the pool at GPU execution time), written for Vulkan from our CPU layout code. +// +// TOTK_DESWIZZLE = auto (default) / gpu: the compute pass where the storage is an imported pool (vk_mem_span_live), +// the CPU otherwise; cpu: always the CPU; check: gpu, and every upload is also deswizzled on the CPU and compared +// (waits for the GPU; for testing only). +#include +#include + +#include "bl_deswizzle_spv.h" +#include "cfg.h" +#include "host.h" +#include "vk.h" +#include "vk_rec.h" +#include "platform/platform.h" + +typedef struct { + uint32_t src_base, dst_base, segs, pitch16, rows, slices, th, td, gobs_x, blocks_y, nx; +} Push; + +static PlatMutex g_lock = PLAT_MUTEX_INIT; +static struct { + VkDevice dev; + int mode; // VK_DESWZ_*; -1 before the first vk_deswz_mode with a device + VkDescriptorSetLayout dsl; + VkPipelineLayout layout; + VkPipeline pipe; + PFN_vkCmdPushDescriptorSetKHR push; +} G = {.mode = -1}; +static volatile int64_t n_regions, n_bytes; + +static int create_pipeline(VkCtx *c) { + G.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR"); + if (!G.push) return 0; + VkDescriptorSetLayoutBinding b[2]; + for (int i = 0; i < 2; i++) + b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}; + VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, + .bindingCount = 2, .pBindings = b}; + if (vkCreateDescriptorSetLayout(c->device, &dci, NULL, &G.dsl) != VK_SUCCESS) return 0; + VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push)}; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, + .pSetLayouts = &G.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr}; + if (vkCreatePipelineLayout(c->device, &li, NULL, &G.layout) != VK_SUCCESS) return 0; + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bl_deswizzle_spv, + .pCode = bl_deswizzle_spv}; + VkShaderModule m = VK_NULL_HANDLE; + if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = G.layout}; + VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.pipe); + vkDestroyShaderModule(c->device, m, NULL); + if (r != VK_SUCCESS) G.pipe = VK_NULL_HANDLE; + return r == VK_SUCCESS; +} + +int vk_deswz_mode(void) { + VkCtx *c = vk_ctx(); + if (!c) return VK_DESWZ_CPU; + plat_mutex_lock_shared(&g_lock); + int m = G.dev == c->device ? G.mode : -1; + plat_mutex_unlock_shared(&g_lock); + if (m >= 0) return m; + plat_mutex_lock(&g_lock); + if (G.dev != c->device) { // the old device's objects went with it (probes recreate the device) + memset(&G, 0, sizeof G); + G.dev = c->device; + G.mode = -1; + } + if (G.mode < 0) { + const char *e = cfg_get("TOTK_DESWIZZLE"); + if (!e || !*e) e = "auto"; + int want = !strcmp(e, "cpu") || !strcmp(e, "0") ? VK_DESWZ_CPU : !strcmp(e, "check") ? VK_DESWZ_CHECK : VK_DESWZ_GPU; + if (strcmp(e, "auto") && strcmp(e, "gpu") && strcmp(e, "1") && strcmp(e, "cpu") && strcmp(e, "0") && + strcmp(e, "check")) + rt_log("vk: TOTK_DESWIZZLE=%s is not auto, gpu, cpu or check: auto", e); + if (want != VK_DESWZ_CPU && !create_pipeline(c)) { + rt_log("vk: the deswizzle compute pass is unavailable on %s: textures deswizzle on the CPU", c->props.deviceName); + want = VK_DESWZ_CPU; + } + G.mode = want; + rt_log("vk: block-linear textures deswizzle on the %s%s", want == VK_DESWZ_CPU ? "CPU" : "GPU where the storage is " + "an imported pool", want == VK_DESWZ_CHECK ? ", checked against the CPU (TOTK_DESWIZZLE=check)" : ""); + } + m = G.mode; + plat_mutex_unlock(&g_lock); + return m; +} + +static uint32_t max_groups_x(const VkCtx *c) { + uint32_t m = c->props.limits.maxComputeWorkGroupCount[0]; + return m > 65535 ? 65535 : m; +} + +// The bound source range starts at the alignment below src_off. +static VkDeviceSize src_lo(const VkCtx *c, VkDeviceSize src_off) { + VkDeviceSize al = c->props.limits.minStorageBufferOffsetAlignment; + if (al < 16) al = 16; + return src_off / al * al; +} + +int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst) { + VkCtx *c = vk_ctx(); + if (!c || !n || (src_off & 15) || vk_deswz_mode() == VK_DESWZ_CPU) return 0; + VkDeviceSize range = src_off + src_size - src_lo(c, src_off), lim = c->props.limits.maxStorageBufferRange; + if (range > lim || max_dst > lim) return 0; + uint32_t maxg = max_groups_x(c); + for (int i = 0; i < n; i++) { // the 32-bit segment indices of the shader stay inside the bindings + uint64_t groups = ((uint64_t)rg[i].segs + 63) / 64, out = (uint64_t)rg[i].pitch16 * rg[i].rows * rg[i].slices; + if ((rg[i].src & 15) || rg[i].src + (uint64_t)rg[i].segs * 16 > src_size || !rg[i].segs || + (src_off - src_lo(c, src_off) + rg[i].src) / 16 + rg[i].segs > UINT32_MAX || out > UINT32_MAX || + out * 16 > max_dst || (groups + maxg - 1) / maxg > c->props.limits.maxComputeWorkGroupCount[1]) + return 0; + } + return 1; +} + +int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst, + VkDeviceSize dst_size, const VkDeswzRegion *rg, int n) { + VkCtx *c = vk_ctx(); + if (!vk_deswz_fits(src_off, src_size, rg, n, dst_size)) return 0; + for (int i = 0; i < n; i++) + if ((rg[i].dst & 15) || rg[i].dst + (uint64_t)rg[i].pitch16 * 16 * rg[i].rows * rg[i].slices > dst_size) return 0; + plat_mutex_lock_shared(&g_lock); + VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE; + plat_mutex_unlock_shared(&g_lock); + if (!pipe) return 0; + VkDeviceSize lo = src_lo(c, src_off), range = src_off + src_size - lo; + uint32_t maxg = max_groups_x(c); + VkDescriptorBufferInfo dbi[2] = {{src, lo, range}, {dst, 0, dst_size}}; + VkWriteDescriptorSet w[2]; + for (int i = 0; i < 2; i++) + w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i, + .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .pBufferInfo = &dbi[i]}; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); + rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 2, w); + uint64_t bytes = 0; + for (int i = 0; i < n; i++) { + const VkDeswzRegion *r = &rg[i]; + uint64_t groups = ((uint64_t)r->segs + 63) / 64; + uint32_t nx = groups < maxg ? (uint32_t)groups : maxg, ny = (uint32_t)((groups + nx - 1) / nx); + Push pc = {.src_base = (uint32_t)((src_off - lo + r->src) / 16), .dst_base = (uint32_t)(r->dst / 16), + .segs = r->segs, .pitch16 = r->pitch16, .rows = r->rows, .slices = r->slices, .th = r->th, .td = r->td, + .gobs_x = r->gobs_x, .blocks_y = r->blocks_y, .nx = nx}; + vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); + vkCmdDispatch(cb, nx, ny, 1); + bytes += (uint64_t)r->segs * 16; + } + int64_t before = plat_add64(&n_regions, n) - n; + plat_add64(&n_bytes, (int64_t)bytes); + if (!before || (before + n) / 4096 != before / 4096) + rt_log("vk: deswizzle: %lld regions, %.1f MiB of block-linear storage deswizzled on the GPU so far", + (long long)(before + n), (double)n_bytes / 1048576.0); + return 1; +} + +void vk_deswz_release(void) { + VkCtx *c = vk_ctx(); + plat_mutex_lock(&g_lock); + if (c && G.dev == c->device) { + if (G.pipe) vkDestroyPipeline(c->device, G.pipe, NULL); + if (G.layout) vkDestroyPipelineLayout(c->device, G.layout, NULL); + if (G.dsl) vkDestroyDescriptorSetLayout(c->device, G.dsl, NULL); + } + memset(&G, 0, sizeof G); + G.mode = -1; + plat_mutex_unlock(&g_lock); +} diff --git a/runtime/vk_device.c b/runtime/vk_device.c index a063955e..b0845994 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -376,6 +376,7 @@ static void destroy_all(void) { gfx_window_release_vk(); if (g_vk.device) vkDeviceWaitIdle(g_vk.device); if (g_vk.device) vk_astc_release(); + if (g_vk.device) vk_deswz_release(); if (g_vk.device) vk_crumbs_release(); if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL); vk_headroom_stop(); @@ -397,7 +398,7 @@ int vk_init(void) { plat_mutex_lock(&g_lock); if (!g_ready) { memset(&g_vk, 0, sizeof g_vk); - if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_headroom_start(), vk_dlaa_init(); + if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_upload_pools_init(), vk_headroom_start(), vk_dlaa_init(); else destroy_all(); } int ok = g_ready; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 6c6b1a8b..f19a1b88 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -416,6 +416,10 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { return resolve_locked(addr, size, out); } +int vk_mem_span_live(const VkMemSpan *sp) { + return sp->buffer && sp->pool && !sp->alloc && !shadow_on && ((const PoolRec *)sp->pool)->kind == POOL_IMPORT; +} + // The pool index lookup under the lock; a non-overlapped import or physical pool goes into the address's cache slot. static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) { memset(out, 0, sizeof *out); @@ -2152,6 +2156,7 @@ typedef struct { uint64_t seq; Staging *stg; int nstg, cap; + VkDeviceSize held; // staging and scratch bytes freed when the batch completes (vk_upload_held) } Batch; enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA }; @@ -2305,10 +2310,68 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { else { if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg); open_b.stg[open_b.nstg++] = (Staging){b, a}; + open_b.held += bi.size; } return info.pMappedData; } +// Device-local scratch of the upload batch (the GPU deswizzle's linear copies, the ASTC decoder's blocks and texels): +// a pool of its own, like staging, so the replay thread neither waits on the default pool's lock while another thread +// makes a block there (the headroom thread: about 8 ms for 256 MiB) nor makes one itself for a few MiB. One block +// of SCR_BLOCK is kept from the first use; larger buffers than half a block go to the default pool. +#define SCR_BLOCK (64ull << 20) +static VmaPool scr_pool; +static int scr_pool_tried; + +VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { + VkCtx *c = vk_ctx(); + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (!__atomic_load_n(&scr_pool_tried, __ATOMIC_ACQUIRE)) { + plat_once(&cmd_once, cmd_init, NULL); + plat_rmutex_lock(&cmd_lock); + if (!scr_pool_tried) { + // two blocks with 8 GiB or more of device-local memory: fight2's first world frame deswizzles about + // 280 MiB, more than the GPU finishes while the replay records, and the pool grew by four blocks there + VkDeviceSize vram = 0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT && c->mem_props.memoryHeaps[h].size > vram) + vram = c->mem_props.memoryHeaps[h].size; + uint32_t type; + VmaPoolCreateInfo pi = {.blockSize = SCR_BLOCK, .minBlockCount = vram >= (8ull << 30) ? 2 : 1}; + if (vmaFindMemoryTypeIndexForBufferInfo(c->vma, &bi, &aci, &type) == VK_SUCCESS) { + pi.memoryTypeIndex = type; + if (vmaCreatePool(c->vma, &pi, &scr_pool) != VK_SUCCESS) scr_pool = NULL; + } + rt_log("vk: upload scratch pool %s (%u blocks of %llu MiB kept)", scr_pool ? "created" : "unavailable: default pool", + (unsigned)pi.minBlockCount, (unsigned long long)(SCR_BLOCK >> 20)); + __atomic_store_n(&scr_pool_tried, 1, __ATOMIC_RELEASE); + } + plat_rmutex_unlock(&cmd_lock); + } + if (scr_pool && bi.size <= SCR_BLOCK / 2) aci.pool = scr_pool; + VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, buf, alloc, NULL); + if (r == VK_SUCCESS && open_active) open_b.held += bi.size; // the caller defers its free to the open batch + return r; +} + +VkDeviceSize vk_upload_held(void) { return open_active ? open_b.held : 0; } + +// At device creation: the staging and scratch pools with their kept blocks, made through a first small buffer each +// (freed at once), so the first world frame does not allocate them (staging was first used there once the GPU took +// the early texture uploads). +void vk_upload_pools_init(void) { + VkCtx *c = vk_ctx(); + if (!c) return; + VkBuffer b; + VmaAllocation a; + vk_staging(16, &b, &a); + vmaDestroyBuffer(c->vma, b, a); + if (vk_scratch_buffer(16, &b, &a) == VK_SUCCESS) vmaDestroyBuffer(c->vma, b, a); +} + uint64_t vk_upload_flush(int wait) { plat_once(&cmd_once, cmd_init, NULL); plat_rmutex_lock(&cmd_lock); @@ -2434,6 +2497,9 @@ void vk_mem_shutdown(void) { if (stg_pool) vmaDestroyPool(c->vma, stg_pool); // after retire: no staging buffer is left in it stg_pool = NULL; stg_pool_tried = 0; + if (scr_pool) vmaDestroyPool(c->vma, scr_pool); // deferred scratch buffers went with collect_locked above + scr_pool = NULL; + scr_pool_tried = 0; plat_rmutex_unlock(&cmd_lock); plat_mutex_lock(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index aafe5ff7..3f72948c 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -819,15 +819,20 @@ static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar, // large ASTC array decodes to RGBA8 in one go: a staging buffer of 676682336 bytes failed (-2) on an Intel UHD with // 8 GB of shared memory and the game closed before the title screen. #define UPLOAD_PART (32ull << 20) // fits the staging pool (vk_staging takes up to half of its 64 MiB block) +// The open upload batch is submitted between chunks (and parts of the CPU path) once its staging and scratch pass this: +// on fight2's first world frame one batch took hundreds of MiB, and the staging and scratch pools grew by 64 MiB blocks +// on the replay thread. Submitting earlier keeps the queue order (the batch still precedes the part that samples the +// images) and lets the GPU finish and return the buffers sooner. +#define UPLOAD_HELD_MAX (48ull << 20) -// Copies the whole storage into the image (every level and layer the image has). -static void upload(Tex *e) { +static int upload_gpu(Tex *e); + +// The CPU side of upload: deswizzles (and decodes or expands) into staging buffers, then copies them into the image. +static void upload_cpu(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; const tex_t *t = &e->t; const nvn_texdesc *d = &t->d; int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); - if (!bpb || !t->mem) return; - uint64_t p0 = perf_begin(), u0 = kern_now_ns(); int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D; uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers; int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb; @@ -908,6 +913,7 @@ static void upload(Tex *e) { } barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_upload_end(); + if (!acheck && vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0); } if (parts > 1) log_limited(&n_split, "texture %#llx fmt %#x %ux%u, %u levels, %u layers: %.1f MiB staged in %d parts", @@ -924,7 +930,201 @@ static void upload(Tex *e) { free(raw_copy); free(ar); free(rg); +} + +// Block-linear storage in an imported pool, deswizzled by the GPU (vk_deswizzle.c): the replay thread only records +// the dispatches, the image copies and (ASTC) the decode into the upload batch the CPU path also uses, which runs before +// the part that samples the image. The GPU reads the pool when the batch runs, as the Switch's GPU reads it at +// execution; a CPU write after make_fresh's write-watch reset still marks the pages and brings another upload. Each +// chunk of regions gets a device-local scratch buffer of at most DESWZ_CHUNK bytes (one region, when larger), freed +// once the batch completes. TOTK_DESWIZZLE=check compares every chunk with the CPU deswizzle (waits for the GPU). +// 0 when not handled, before anything is recorded: the CPU path takes the texture. +#define DESWZ_CHUNK (32ull << 20) + +// Uploads by path since the last perf summary (TOTK_TRACE=perf): 0 GPU, else why the CPU took it. +enum { UP_GPU, UP_MODE, UP_LINEAR, UP_RGB, UP_ASTC_CPU, UP_BPB, UP_NOT_LIVE, UP_GEOM, UP_LIMITS, UP_SCRATCH, UP_WHY }; +static const char *const up_why_name[UP_WHY] = {"gpu", "cpu mode", "linear", "rgb", "astc on cpu", "bytes per block", + "not an imported pool", "layout", "device limits", "scratch"}; +static uint64_t up_n[UP_WHY], up_bytes[UP_WHY], up_ns[UP_WHY]; +static int up_why; // upload_gpu's reason for returning 0 (texture lock held) + +static int upload_gpu(Tex *e) { + static volatile int32_t n_scratch_fail, n_undecoded, n_bad; + static volatile int64_t n_checked, n_differ; + const tex_t *t = &e->t; + const nvn_texdesc *d = &t->d; + int mode = vk_deswz_mode(); + up_why = mode == VK_DESWZ_CPU ? UP_MODE : nvn_tex_is_linear(d) ? UP_LINEAR : e->f.expand_rgb ? UP_RGB : UP_GPU; + if (up_why) return 0; + int astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu(); + if (astc && (!agpu || vk_astc_mode() == VK_ASTC_CHECK)) return up_why = UP_ASTC_CPU, 0; // or ASTC's check mode + int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); + if (!bpb || 16 % bpb) return up_why = UP_BPB, 0; // the rows of the linear copy are padded to 16 bytes + VkMemSpan sp; + if (!vk_mem_resolve(e->pub.addr, t->L.size, &sp) || !vk_mem_span_live(&sp)) return up_why = UP_NOT_LIVE, 0; + int one_d = e->pub.type == VK_IMAGE_TYPE_1D, is3d = d->target == NVN_TEXTURE_TARGET_3D; + uint32_t levels = e->pub.levels, layers = e->pub.layers, n = levels * layers, k = 0; + VkDeswzRegion *dr = calloc(n, sizeof *dr); + VkBufferImageCopy *rg = calloc(n, sizeof *rg); + VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL; + if (!dr || !rg || (agpu && !ar)) rt_fatal("vk: out of memory (texture upload regions)"); + int ok = 1; + VkDeviceSize max_dst = DESWZ_CHUNK; + for (uint32_t lv = 0; ok && lv < levels; lv++) + for (uint32_t ly = 0; ok && ly < layers; ly++) { + uint32_t row_bytes, rows, slices, gbpb; + uint64_t base; + unsigned th, td; + uint32_t w = mip(d->width, lv), h = one_d ? 1 : mip(d->height, lv), sl = is3d ? mip(d->depth, lv) : 1; + if (nvn_deswizzle_geom(d, lv, ly, &row_bytes, &rows, &slices, &base, &th, &td, &gbpb) || gbpb != (uint32_t)bpb || + base >= t->L.size || rows != (h + bh - 1) / bh || slices != sl) { + ok = 0; + break; + } + uint64_t bhg = 1ull << th, bdg = 1ull << td, gobs_x = (row_bytes + 63) / 64; + uint64_t blocks_y = (rows + 8 * bhg - 1) / (8 * bhg), blocks_z = (slices + bdg - 1) / bdg; + uint64_t segs = gobs_x * blocks_y * blocks_z * (32ull << (th + td)), room = (t->L.size - base) / 16; + if (segs > room) segs = room; + uint32_t pitch16 = (row_bytes + 15) / 16; + VkDeviceSize out = (VkDeviceSize)pitch16 * 16 * rows * slices; + if (segs > UINT32_MAX || gobs_x * blocks_y > UINT32_MAX) { ok = 0; break; } + if (out > max_dst) max_dst = out; + dr[k] = (VkDeswzRegion){.src = base, .segs = (uint32_t)segs, .pitch16 = pitch16, .rows = rows, .slices = slices, + .th = th, .td = td, .gobs_x = (uint32_t)gobs_x, .blocks_y = (uint32_t)blocks_y}; + rg[k] = (VkBufferImageCopy){.bufferRowLength = pitch16 * 16 / (uint32_t)bpb * (uint32_t)bw, + .bufferImageHeight = rows * (uint32_t)bh, + .imageSubresource = {e->pub.aspect, lv, ly, 1}, .imageExtent = {w, h, sl}}; + if (agpu) ar[k] = (VkAstcRegion){.w = w, .h = h, .slices = sl, .level = lv, .layer = ly}; + k++; + } + if (!ok || k != n || !vk_deswz_fits(sp.offset, t->L.size, dr, (int)n, max_dst)) { + free(dr), free(rg), free(ar); + return up_why = !ok || k != n ? UP_GEOM : UP_LIMITS, 0; + } + VkCtx *c = vk_ctx(); + VkCommandBuffer cb = vk_upload_begin(); + // Earlier GPU writes into the pool (copies, shaders) before the reads; the image's earlier uses before its copy. + barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) { + VkDeviceSize size = 0; + for (k1 = k0; k1 < n; k1++) { + VkDeviceSize out = (VkDeviceSize)dr[k1].pitch16 * 16 * dr[k1].rows * dr[k1].slices; + if (k1 > k0 && size + out > DESWZ_CHUNK) break; + dr[k1].dst = size, rg[k1].bufferOffset = size; + if (agpu) ar[k1].src = size; + size += out; + } + VkBuffer sb; + VmaAllocation sa; + if (vk_scratch_buffer(size, &sb, &sa) != VK_SUCCESS) { + log_limited(&n_scratch_fail, "texture %#llx: deswizzle scratch of %llu bytes failed%s", + (unsigned long long)e->pub.addr, (unsigned long long)size, + k0 ? "; its later levels left as they were" : "; deswizzled on the CPU"); + if (!k0) { + vk_upload_end(); + free(dr), free(rg), free(ar); + return up_why = UP_SCRATCH, 0; + } + break; + } + if (!vk_deswz_record(cb, sp.buffer, sp.offset, t->L.size, sb, size, dr + k0, (int)(k1 - k0))) { + log_limited(&n_bad, "texture %#llx: deswizzle of levels %u-%u not recorded; left as they were", + (unsigned long long)e->pub.addr, k0, k1 - 1); // vk_deswz_fits passed: not expected + vk_defer_buffer(sb, sa); + break; + } + barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, + VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); + if (agpu) { + uint64_t a0 = perf_begin(); + if (!vk_astc_record(cb, sb, size, ar + k0, (int)(k1 - k0), e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, + e->f.astc_bc, e->pub.image, e->pub.aspect, VK_NULL_HANDLE)) + log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was", + (unsigned long long)e->pub.addr, d->format, d->width, d->height); + perf_end(PERF_ASTC, a0); + } else { + vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, k1 - k0, rg + k0); + } + if (mode == VK_DESWZ_CHECK) { // the linear copy back to the host, compared with the CPU deswizzle + VkBuffer rb; + VmaAllocation ra; + const uint8_t *rmap = vk_staging(size, &rb, &ra); + VkBufferCopy cp = {0, 0, size}; + vkCmdCopyBuffer(cb, sb, rb, 1, &cp); + barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT, + VK_ACCESS_2_HOST_READ_BIT); + vk_defer_buffer(sb, sa); + vk_upload_end(); + vk_upload_flush(1); + vmaInvalidateAllocation(c->vma, ra, 0, VK_WHOLE_SIZE); + for (uint32_t i = k0; i < k1; i++) { + const VkDeswzRegion *r = &dr[i]; + size_t pitch = (size_t)r->pitch16 * 16, bytes = pitch * r->rows * r->slices; + uint8_t *cpu = malloc(bytes); + if (!cpu) rt_fatal("vk: out of memory (deswizzle check)"); + uint32_t row_bytes = 0, rows, slices, gbpb; + uint64_t base; + unsigned th, td; + nvn_deswizzle_geom(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, &row_bytes, + &rows, &slices, &base, &th, &td, &gbpb); + nvn_deswizzle(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, t->mem, cpu, pitch); + uint64_t bad = 0, first = UINT64_MAX; + for (uint64_t row = 0; row < (uint64_t)r->rows * r->slices; row++) + if (memcmp(cpu + row * pitch, rmap + r->dst + row * pitch, row_bytes)) { + bad++; + if (first == UINT64_MAX) first = row; + } + free(cpu); + plat_add64(&n_checked, 1); + if (bad) { + plat_add64(&n_differ, 1); + rt_log("vk: deswizzle check: texture %#llx fmt %#x %ux%u level %u layer %u: %llu of %u rows differ, the " + "first %llu (th %u td %u, %u rows of %u bytes, %u slices)", (unsigned long long)e->pub.addr, + d->format, d->width, d->height, rg[i].imageSubresource.mipLevel, + rg[i].imageSubresource.baseArrayLayer, (unsigned long long)bad, r->rows * r->slices, + (unsigned long long)first, r->th, r->td, r->rows, row_bytes, r->slices); + } + } + vmaDestroyBuffer(c->vma, rb, ra); + if ((n_checked & 1023) == 0 || n_checked < 2) + rt_log("vk: deswizzle check: %lld regions compared, %lld differ", (long long)n_checked, (long long)n_differ); + cb = vk_upload_begin(); + barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + continue; + } + vk_defer_buffer(sb, sa); + if (k1 < n && vk_upload_held() > UPLOAD_HELD_MAX) { // submit what is recorded, go on in a new batch + barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW); + vk_upload_end(); + vk_upload_flush(0); + cb = vk_upload_begin(); + barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + } + } + barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW); + vk_upload_end(); + if (vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0); + free(dr), free(rg), free(ar); + return 1; +} + +// Copies the whole storage into the image (every level and layer the image has): on the GPU where it can +// (upload_gpu), else on the CPU. +static void upload(Tex *e) { + const tex_t *t = &e->t; + const nvn_texdesc *d = &t->d; + int bw, bh; + if (!nvn_fmt_bpb(d->format, &bw, &bh) || !t->mem) return; + uint64_t p0 = perf_begin(), u0 = kern_now_ns(); + int why = UP_GPU; + if (!upload_gpu(e)) why = up_why, upload_cpu(e); uint64_t dt = kern_now_ns() - u0; + up_n[why]++, up_bytes[why] += t->L.size, up_ns[why] += dt; if (e->uploaded) { n_reuploads++, perf_count(PERF_CNT_TEX_REUP); ru_n++, ru_bytes += t->L.size, ru_ns += dt; @@ -1404,6 +1604,14 @@ void vk_texture_new_submit(void) { (unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6, (unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6); ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = 0; + char ub[512]; + size_t uo = 0; + for (int i = 0; i < UP_WHY; i++) + if (up_n[i] && uo < sizeof ub) + uo += (size_t)snprintf(ub + uo, sizeof ub - uo, "%s %s %llu (%.1f MiB, %.2f ms)", uo ? "," : "", + up_why_name[i], (unsigned long long)up_n[i], up_bytes[i] / 1048576.0, up_ns[i] / 1e6); + if (uo) rt_log("perf: frame %llu: texture uploads by path:%s", (unsigned long long)f * 60, ub); + memset(up_n, 0, sizeof up_n), memset(up_bytes, 0, sizeof up_bytes), memset(up_ns, 0, sizeof up_ns); uint64_t ck, calls, untr; vk_mem_snap_stats(&ck, &calls, &untr); rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu", From 5b5273b541ba2891884d6314e0956e8fa00f6e85 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 7 Nov 2026 16:53:43 +0300 Subject: [PATCH 3/5] Log how long each pipeline cache save copies and waits for builds The copy time bounds how long a compute miss on the drawing thread can wait for the save now that builds and the copy exclude each other; a put-off autosave is logged too. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_pipeline.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 031ee897..57142b21 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -3339,6 +3339,7 @@ static void cache_save(int wait) { plat_mutex_unlock(&save_lock); return; } + uint64_t t0 = qpc(); if (wait) { plat_mutex_lock(&cache_use); } else { @@ -3348,15 +3349,21 @@ static void cache_save(int wait) { int got = 0; for (int i = 0; i < 1000 && !(got = plat_mutex_trylock(&cache_use)); i++) plat_sleep_ms(2); if (!got) { + rt_log("vk_pipeline: cache save put off, pipelines were being built for %.0f ms", ms_since(t0)); plat_mutex_unlock(&save_lock); return; } } + uint64_t t1 = qpc(); size_t size = 0; void *data = NULL; int ok = vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) && vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS; plat_mutex_unlock(&cache_use); + // The copy time is how long a pipeline build on the drawing thread (a compute miss, under P.lock) can wait for + // this save; the wait is how long the save waited for builds in flight. + rt_log("vk_pipeline: cache %s: %.1f MB copied in %.1f ms, after %.1f ms waiting for builds", wait ? "save" : "autosave", + (double)size / 1e6, ms_since(t1), (double)(t1 - t0) / 1e6); if (ok && !plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path); free(data); plat_mutex_unlock(&save_lock); From dc45de0e9a3f951bc99c7a1d52eeb42ac7b29470 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 7 Nov 2026 19:16:50 +0300 Subject: [PATCH 4/5] Stamp deferred frees with the submission being recorded, whichever thread destroys the object With the synchronous replay (the default since 7be68ec) a game thread finalizing a texture destroys released texture cache entries while another game thread records a submission that slot_resolve may already have handed one of them. defer() saw the recording's value only on the recording thread and stamped the others with the last submitted value, so the completion thread freed the image before the submission naming it ran: "write invalid" device faults on NVIDIA (#788, #802). vk_sync_submit now notes the value at the start of the recording, and every deferral made from then on waits for it. A fight run counted 10 such deferrals during the world load. Device faults now log the deferred-free state, and with VK_EXT_device_address_binding_report the object that held the faulting address and whether it was unbound before the fault (TOTK_ADDR_REPORT=0 turns the map off). The census line counts deferrals made off the recording thread during a recording. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 5 ++- runtime/vk_device.c | 96 +++++++++++++++++++++++++++++++++++++++++---- runtime/vk_memory.c | 47 +++++++++++++++------- runtime/vk_sync.c | 7 ++-- 4 files changed, 128 insertions(+), 27 deletions(-) diff --git a/runtime/vk.h b/runtime/vk.h index 1f210afb..0dfd738d 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -379,7 +379,10 @@ void vk_deswz_release(void); // vk_device.c, before the device goes // 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission // sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch). int vk_mem_span_live(const VkMemSpan *sp); -void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value +// vk_sync.c, when it starts recording the submission that will signal ctx->timeline with this value: objects destroyed +// from then on are freed once the timeline reaches it. +void vk_mem_note_fence(uint64_t timeline_value); +void vk_mem_fault_log(void); // vk_device_fault_report: the deferred-free state, lock-free void vk_mem_collect(void); // frees what has completed typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats; void vk_mem_stats(VkMemStats *s); diff --git a/runtime/vk_device.c b/runtime/vk_device.c index b0845994..31fbc1a4 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -16,9 +16,69 @@ static VkCtx g_vk; static int g_ready; static PlatMutex g_lock = PLAT_MUTEX_INIT; +// ---- GPU address map for device faults +// VK_EXT_device_address_binding_report: the driver reports each bind and unbind of an object's GPU virtual range through +// the debug messenger. The newest AB_N events stay in a ring written without a lock (one atomic add each), so a device +// fault names the object behind its address and whether it was unbound before the fault, which is memory freed while +// a submission still used it. One event per image, buffer or memory block made or freed, none per draw. +// TOTK_ADDR_REPORT=0 turns it off. +#define AB_N (1u << 15) +typedef struct { uint64_t seq, base, size, handle, issued, done; int32_t type; uint32_t unbind; } AbEvent; +static AbEvent *ab_ring; +static uint64_t ab_head; +static int ab_want; // the instance has the messenger for it (create_instance) + +static void ab_record(const VkDebugUtilsMessengerCallbackDataEXT *d) { + const VkDeviceAddressBindingCallbackDataEXT *b = NULL; + for (const VkBaseInStructure *p = d->pNext; p; p = p->pNext) + if (p->sType == VK_STRUCTURE_TYPE_DEVICE_ADDRESS_BINDING_CALLBACK_DATA_EXT) b = (const void *)p; + if (!b || !ab_ring) return; + uint64_t i = __atomic_fetch_add(&ab_head, 1, __ATOMIC_RELAXED); + AbEvent *e = &ab_ring[i & (AB_N - 1)]; + __atomic_store_n(&e->seq, 0, __ATOMIC_RELAXED); + e->base = b->baseAddress, e->size = b->size; + e->handle = d->objectCount ? d->pObjects[0].objectHandle : 0; + e->type = d->objectCount ? (int32_t)d->pObjects[0].objectType : 0; + e->unbind = b->bindingType == VK_DEVICE_ADDRESS_BINDING_TYPE_UNBIND_EXT; + e->issued = vk_sync_issued(), e->done = vk_sync_fence_done(); + __atomic_store_n(&e->seq, i + 1, __ATOMIC_RELEASE); +} + +static const char *ab_type_name(int32_t t, char *buf, size_t n) { + switch (t) { + case VK_OBJECT_TYPE_IMAGE: return "image"; + case VK_OBJECT_TYPE_BUFFER: return "buffer"; + case VK_OBJECT_TYPE_DEVICE_MEMORY: return "memory"; + default: snprintf(buf, n, "object type %d", (int)t); return buf; + } +} + +// The newest events whose range meets the precision-aligned range around a fault address, newest first. +static void ab_report(uint64_t addr, uint64_t precision) { + if (!ab_ring) return; + uint64_t lo = precision > 1 ? addr & ~(precision - 1) : addr, hi = lo + (precision > 1 ? precision : 1); + uint64_t head = __atomic_load_n(&ab_head, __ATOMIC_ACQUIRE), n = head < AB_N ? head : AB_N; + int found = 0; + for (uint64_t k = 1; k <= n && found < 6; k++) { + const AbEvent *e = &ab_ring[(head - k) & (AB_N - 1)]; + if (__atomic_load_n(&e->seq, __ATOMIC_ACQUIRE) != head - k + 1) continue; + if (e->base >= hi || e->base + e->size <= lo) continue; + found++; + char tn[32]; + rt_log("vk: fault address %#llx: %s %#llx [%#llx, +%#llx] %s %llu address events before the fault, " + "submitted %llu, done %llu", (unsigned long long)addr, ab_type_name(e->type, tn, sizeof tn),(unsigned long long)e->handle, + (unsigned long long)e->base, (unsigned long long)e->size, e->unbind ? "unbound" : "bound", + (unsigned long long)k, (unsigned long long)e->issued, (unsigned long long)e->done); + } + if (!found) rt_log("vk: fault address %#llx: no binding in the last %llu address events", (unsigned long long)addr, + (unsigned long long)n); +} + static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type, const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) { - (void)type, (void)user; + (void)user; + if (type & VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT) { ab_record(d); return VK_FALSE; } + if (sev < VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) return VK_FALSE; // info, asked for the address events int err = (sev & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) != 0; if (err) plat_inc32(&g_vk.validation_errors); static volatile int32_t logged; @@ -76,7 +136,10 @@ static int create_instance(void) { rt_log("vk: TOTK_VK_VALIDATION set but %s is not installed; continuing without", layers[0]); validate = 0; } - if (validate) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; + const char *ar = cfg_get("TOTK_ADDR_REPORT"); + ab_want = !(ar && !strcmp(ar, "0")) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + if (ab_want && !ab_ring && !(ab_ring = calloc(AB_N, sizeof *ab_ring))) ab_want = 0; + if (validate || ab_want) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; if (vk_hdr_wanted() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats ext[n_ext++] = VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME; vk_dlaa_instance_exts(ext, &n_ext, 12); @@ -89,16 +152,18 @@ static int create_instance(void) { VkResult r = vkCreateInstance(&ci, NULL, &g_vk.instance); if (r != VK_SUCCESS) { rt_log("vk: vkCreateInstance failed (%d); is a Vulkan 1.3 driver installed?", (int)r); return 0; } g_vk.validation = validate; - if (validate) { + if (validate || ab_want) { PFN_vkCreateDebugUtilsMessengerEXT f = (PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkCreateDebugUtilsMessengerEXT"); VkDebugUtilsMessengerCreateInfoEXT mi = { .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT, - .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT, - .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | - VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT, + .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT | + (ab_want ? VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT : 0), + .messageType = (validate ? VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT : 0) | + (ab_want ? VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT : 0), .pfnUserCallback = debug_cb}; - if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger); + if (!f || f(g_vk.instance, &mi, NULL, &g_vk.messenger) != VK_SUCCESS) ab_want = 0; } return 1; } @@ -217,6 +282,7 @@ typedef struct { VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl; VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbl; VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr; + VkPhysicalDeviceAddressBindingReportFeaturesEXT abr; } FeatureChain; static int create_device(void) { @@ -251,6 +317,13 @@ static int create_device(void) { // Ryujinx's SPIR-V declares fragment shader interlock whenever the host reports it (vk_pipeline.c spirv_fixup). OPTIONAL_EXT(has_fragment_shader_interlock, VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME, fsi, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT); + // The GPU address map for device faults (ab_record), with the messenger made for it + int has_abr = 0; + if (ab_want && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME)) { + has_abr = 1; + ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME; + CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT); + } // Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering int has_fbl = 0, has_lr = 0; if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) && @@ -314,6 +387,10 @@ static int create_device(void) { else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension + rt_log("vk: GPU address map for device faults: %s", + has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report; TOTK_ADDR_REPORT=0 turns it off)" + : !ab_want ? "off (TOTK_ADDR_REPORT=0 or no VK_EXT_debug_utils)" + : "unavailable (no VK_EXT_device_address_binding_report)"); g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { @@ -438,6 +515,7 @@ void vk_device_fault_report(VkResult r) { done = 1; vk_crumbs_report(); vk_pass_marks_report(); + vk_mem_fault_log(); VkCtx *c = vk_ctx(); PFN_vkGetDeviceFaultInfoEXT get = c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") @@ -462,9 +540,11 @@ void vk_device_fault_report(VkResult r) { rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description, n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary); // The faulting access lies in the precision-aligned range around the reported address. - for (uint32_t i = 0; i < n.addressInfoCount; i++) + for (uint32_t i = 0; i < n.addressInfoCount; i++) { rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType), (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision); + ab_report(ai[i].reportedAddress, ai[i].addressPrecision); + } for (uint32_t i = 0; i < n.vendorInfoCount; i++) rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description, (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index f19a1b88..22b04e1c 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -511,6 +511,9 @@ static void dirty_frame(uint64_t frame); static void mirror_log(uint64_t frame); static void shadow_log(uint64_t frame); static int dead_pending(void); +// Deferrals made off the recording thread while another thread recorded a submission (cmd_lock, defer), the newest +// one's stamp: the case the stamp from the recording start covers. In the census line and the device fault report. +static uint64_t n_cross_defer, cross_defer_tl; static void vram_log(uint64_t frame); @@ -546,10 +549,11 @@ void vk_mem_census(uint64_t frame) { } plat_mutex_unlock_shared(&pool_lock); rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%d without a GPU buffer), physical %llu pools %.1f MiB, " - "virtual %llu pools %.1f MiB of address space", + "virtual %llu pools %.1f MiB of address space; %llu objects deferred off the recording thread during a recording", (unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0, n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0, - (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0); + (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0, + (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED)); static const char *const kinds[] = {"", "import", "physical", "virtual"}; for (int j = 0; j < ng; j++) { char f[160]; @@ -2174,7 +2178,8 @@ static int nspare, cap_spare; static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq) static Dead *dead; static int ndead, cap_dead; -static uint64_t tl_noted; // newest timeline value vk_sync.c reported submitted +static uint64_t tl_noted; // value of the submission being recorded, else of the last one (vk_mem_note_fence) +static _Thread_local uint64_t tl_noted_here; // the value this thread noted: it records that submission static void cmd_init(void *p) { (void)p; @@ -2409,17 +2414,17 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) { plat_rmutex_lock(&cmd_lock); if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead); // The open batch may record into the object: it counts as the newest user. The GPU must also be past every - // submission that names it. tl_noted covers the submitted ones; an object the replay lets go of while it records - // a submission (vk_exec_recording_fence) may be named by that one too, not yet submitted: vk_texture.c - // slot_resolve hands a draw a released entry, and entry_of destroys released entries at that address later in - // the same submission. Stamped with the last submitted value, the object went as soon as the GPU reached it, - // before the submission that names it ran, and with the recorder thread (vk_rec.c) behind, before the recorder - // made the call that names it. LAS Recomp and Savage Wind Recomp stamp every deferral with the submission being - // recorded (runtime/nvn/vulkan/textures.c defer: vk_pending_seq). TOTK_DEFER_STAMP=0 brings back the old stamp. - static int stamp = -1; - if (stamp < 0) stamp = rt_knob("DEFER_STAMP", "1") != 0; - uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0; - if (rf > tl) tl = rf; + // submission that names it, including the one being recorded, which may name it and is not submitted yet: + // vk_texture.c slot_resolve hands a draw a released entry, and entry_of, sweep_released and the VRAM pressure drop + // destroy released entries. tl_noted is that submission's value from the start of its recording (vk_sync.c + // vk_sync_submit), whichever thread destroys the object. With the synchronous replay a game thread finalizing a + // texture (vk_backend.c vk_be_texture_release) destroys entries while another game thread records; stamped with + // the last submitted value, as the recording thread's own fence was the only one seen, the object was freed once + // the GPU reached that value, before the submission naming it ran: "write invalid" device faults on NVIDIA. LAS + // Recomp and Savage Wind Recomp stamp every deferral with the submission being recorded (runtime/nvn/vulkan/ + // textures.c defer: vk_pending_seq). + uint64_t tl = tl_noted; + if (tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl; dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl}; rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl, vk_sync_fence_done()); @@ -2442,10 +2447,22 @@ void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) { if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc); } -void vk_mem_note_submit(uint64_t timeline_value) { +// The device fault report: no lock, since a thread that holds cmd_lock may be waiting for the report's own lock +// (vk_upload_flush reports a failed submit with cmd_lock held). +void vk_mem_fault_log(void) { + rt_log("vk: deferred frees at the fault: %d pending, stamp %llu, submitted %llu, done %llu; %llu deferred off the " + "recording thread during a recording (newest stamp %llu)", + __atomic_load_n(&ndead, __ATOMIC_RELAXED), (unsigned long long)__atomic_load_n(&tl_noted, __ATOMIC_RELAXED), + (unsigned long long)vk_sync_issued(), (unsigned long long)vk_sync_fence_done(), + (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED), + (unsigned long long)__atomic_load_n(&cross_defer_tl, __ATOMIC_RELAXED)); +} + +void vk_mem_note_fence(uint64_t timeline_value) { plat_once(&cmd_once, cmd_init, NULL); plat_rmutex_lock(&cmd_lock); if (timeline_value > tl_noted) tl_noted = timeline_value; + tl_noted_here = timeline_value; plat_rmutex_unlock(&cmd_lock); } diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index e3162d12..3b90c026 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -309,7 +309,7 @@ void vk_sync_start(void) { if (reserved < now) reserved = now; stop = 0; plat_mutex_unlock(&st_lock); - vk_mem_note_submit(now); + vk_mem_note_fence(now); completion_thread = plat_thread_new(completion_main, c, "vk completion"); if (!completion_thread) rt_fatal("vk: cannot start the completion thread"); __atomic_store_n(&started, 1, __ATOMIC_RELEASE); @@ -379,7 +379,6 @@ VkCommandBuffer vk_sync_split(void) { vk_device_fault_report(r); rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); } - vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence); ts_begin(s->cb); @@ -404,6 +403,9 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1; cur_fence = fence; + // Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c + // defer), since the recording may already name it. + vk_mem_note_fence(fence); CmdSlot *s = take_slot(c, fence); uint64_t t0 = perf_begin(); rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence); // begun by the recorder when it records the part @@ -434,7 +436,6 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); } s->fence = recorded ? fence : 0; - vk_mem_note_submit(fence); plat_mutex_lock(&st_lock); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); From 36a83bd78fcacdca0fd69311869840f463f58cf6 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 7 Nov 2026 20:05:49 +0300 Subject: [PATCH 5/5] Keep pool memory and texture records until the submission being recorded is done vk_pool_fini freed import, physical and virtual pools right after vkQueueWaitIdle, which covers submitted work only; with the synchronous replay another game thread may be recording a submission that names the pool. Every pool kind now retires through the completion queue at the later of the recording's value and the next one. destroy_tex freed the texture record while other threads still held &e->pub (render targets for a pass, storage bindings writing wr_fence through it): they read a stale VkImage and cached views by that handle. The record is now freed with its image's stamp (vk_defer_free), and views of a dropped entry are deferred at once instead of cached. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 6 ++++++ runtime/vk_exec.c | 1 + runtime/vk_exec_draw.c | 18 +++++++++++----- runtime/vk_hdr.c | 2 +- runtime/vk_memory.c | 48 ++++++++++++++++++++++++++++++------------ runtime/vk_texture.c | 8 ++++++- 6 files changed, 63 insertions(+), 20 deletions(-) diff --git a/runtime/vk.h b/runtime/vk.h index 0dfd738d..91a67370 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -329,6 +329,9 @@ void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestro void vk_defer_view(VkImageView view); void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc); +// free(p) once the GPU is past the submission being recorded, like the deferrals above: host records that a recording +// may still read through a pointer taken before another thread dropped them (vk_texture.c destroy_tex). +void vk_defer_free(void *p); // Device-local scratch for the upload batch (storage, transfer src and dst), from a pool of its own (vk_memory.c). VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc); // Staging and scratch bytes the open upload batch holds until it completes. Large texture loads submit the batch early @@ -414,6 +417,9 @@ typedef struct { uint8_t scale; // 1, or TOTK_SHADOW_SCALE for a shadow map made larger than the game's (vk_texture.c // shadow_scale_of): width and height above are the larger ones, the game's regions // are multiplied by it + uint8_t gone; // the entry was dropped (vk_texture.c destroy_tex, under the texture lock): image and + // record live until the submission being recorded is done, and nothing may be cached + // by its image handle, which the driver can hand out again after that } VkTexImage; // The window texture the backend presents (vk_backend.c): *image and *format become the float picture the game copied // into it, when the last draw into it was that copy and the swapchain is HDR10. diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index b35831c8..6408d2be 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -183,6 +183,7 @@ static void memory_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAcce // be made (logged). Replay thread. static int bc_standin(const VkTexImage *img, uint32_t level, VkFormat fmt, BcStandIn *out) { static int warned; + if (img->gone) return 0; // dropped since the lookup: a stand-in kept by its handle would outlive it plat_mutex_lock(&bcs_lock); int full = nbcs == BC_STANDINS; for (int i = 0; i < nbcs; i++) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index eee270c9..7bede297 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -593,6 +593,10 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c) VkImageView view; if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE; + if (img->gone) { // dropped since the lookup: its views were purged and a new one would outlive the image + vk_defer_view(view); + return view; + } ViewNode *n = malloc(sizeof *n); if (!n) rt_fatal("vk: out of memory"); *n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view}; @@ -619,11 +623,15 @@ VkImageView vk_exec_target_view(const VkTexImage *img, VkFormat format, VkImageV if (blocks) ci.pNext = &use; // not the BC image's sampling and transfer usages if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) rt_fatal("vk: render target view (format %d) could not be created", (int)format); - ViewNode *n = malloc(sizeof *n); - if (!n) rt_fatal("vk: out of memory"); - *n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view}; - nvn_hm_put(&views_by_image, &img->image, n); - nvn_hm_put(&view_of, &k, (void *)view); + if (img->gone) { // dropped since resolve_target (get_view) + vk_defer_view(view); + } else { + ViewNode *n = malloc(sizeof *n); + if (!n) rt_fatal("vk: out of memory"); + *n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view}; + nvn_hm_put(&views_by_image, &img->image, n); + nvn_hm_put(&view_of, &k, (void *)view); + } } vk_texture_unlock(); return view; diff --git a/runtime/vk_hdr.c b/runtime/vk_hdr.c index e8480bb2..806ffd1b 100644 --- a/runtime/vk_hdr.c +++ b/runtime/vk_hdr.c @@ -272,7 +272,7 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) { } if (ntex == 1 && slot >= 0 && slot < 128 && ex->tex[1][slot]) { const VkTexImage *s = vk_texture_by_id(NVN_HANDLE_TEX(ex->tex[1][slot]), NULL, NULL); - if (s && is_float(s->format) && s->width == t->width && s->height == t->height && s->samples == 1) + if (s && !s->gone && is_float(s->format) && s->width == t->width && s->height == t->height && s->samples == 1) src = s->image, fmt = s->format; } plat_mutex_lock(&rec_lock); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 22b04e1c..3e30682f 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -78,8 +78,9 @@ static int shadow_on; // shadow mode: imported pools get shad static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size); static void shadow_touch(PoolRec *r); static void shadow_forget(PoolRec *r); -static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown) -static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record +static void pool_retiring(PoolRec *r); // vk_pool_fini: kept until pool_retire (or vk_mem_shutdown) +static void pool_retire(void *arg); // completion thread: destroys a finalized pool's record +static uint64_t retire_value(void); // the timeline value after which a finalized pool's record may go static void grow(void **arr, int *cap, size_t elem); static PlatMutex pool_lock = PLAT_MUTEX_INIT; @@ -333,10 +334,14 @@ void vk_pool_fini(pool_t *p) { vk_queue_unlock(); } vk_mem_collect(); - // A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may - // name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued). - if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r); - else destroy_rec(c, r); + // The record, its buffers and its memory outlive the submission being recorded, of any pool kind. The wait above + // covers submitted work only. With the synchronous replay another game thread may be recording right now, and that + // recording may already name this pool: a mirrored pool's sync copies and write-backs, an import pool's buffer + // through a texture upload or a released cache entry (vk_texture.c slot_resolve), a physical pool's memory under + // the images vk_texture_range_gone just deferred to that submission. The game frees a pool only once its own GPU + // work is done, so only the port's work can name it. Freed here, the submission ran with freed memory. + pool_retiring(r); + vk_completion_add(retire_value(), pool_retire, r); } // Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c); apply them in order. @@ -1398,19 +1403,19 @@ static void mirror_forget(PoolRec *r) { } } -// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran. +// Finalized pools waiting for pool_retire; vk_mem_shutdown destroys those whose callback never ran. static PoolRec **retiring; static int nretiring, cretiring; static PlatMutex retire_lock = PLAT_MUTEX_INIT; -static void mirror_retiring(PoolRec *r) { +static void pool_retiring(PoolRec *r) { plat_mutex_lock(&retire_lock); if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring); retiring[nretiring++] = r; plat_mutex_unlock(&retire_lock); } -static void mirror_retire(void *arg) { +static void pool_retire(void *arg) { plat_mutex_lock(&retire_lock); for (int i = 0; i < nretiring; i++) if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; } @@ -2163,7 +2168,7 @@ typedef struct { VkDeviceSize held; // staging and scratch bytes freed when the batch completes (vk_upload_held) } Batch; -enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA }; +enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST }; // DEAD_HOST: free(handle) typedef struct { int kind; uint64_t handle; VmaAllocation alloc; uint64_t seq, tl; } Dead; static PlatRMutex cmd_lock; @@ -2222,6 +2227,10 @@ static void collect_locked(VkCtx *c) { for (int i = 0; i < ndead; i++) { Dead *d = &dead[i]; if (d->seq > done_seq || d->tl > tl) { dead[k++] = *d; continue; } + if (d->kind == DEAD_HOST) { + free((void *)(uintptr_t)d->handle); + continue; + } rec_check_handle(d->kind == DEAD_VIEW ? REC_VIEW : d->kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, d->handle, 1, d->tl, tl); switch (d->kind) { @@ -2424,10 +2433,11 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) { // Recomp and Savage Wind Recomp stamp every deferral with the submission being recorded (runtime/nvn/vulkan/ // textures.c defer: vk_pending_seq). uint64_t tl = tl_noted; - if (tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl; + if (kind != DEAD_HOST && tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl; dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl}; - rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl, - vk_sync_fence_done()); + if (kind != DEAD_HOST) + rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl, + vk_sync_fence_done()); plat_rmutex_unlock(&cmd_lock); } @@ -2446,6 +2456,7 @@ void vk_defer_view(VkImageView view) { if (view) defer(DEAD_VIEW, (uint64_t)(uin void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) { if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc); } +void vk_defer_free(void *p) { if (p) defer(DEAD_HOST, (uint64_t)(uintptr_t)p, NULL); } // The device fault report: no lock, since a thread that holds cmd_lock may be waiting for the report's own lock // (vk_upload_flush reports a failed submit with cmd_lock held). @@ -2466,6 +2477,17 @@ void vk_mem_note_fence(uint64_t timeline_value) { plat_rmutex_unlock(&cmd_lock); } +// The later of the submission being recorded (tl_noted, noted from its start: vk_sync.c vk_sync_submit) and the next +// one to be submitted. +static uint64_t retire_value(void) { + plat_once(&cmd_once, cmd_init, NULL); + plat_rmutex_lock(&cmd_lock); + uint64_t v = vk_sync_issued() + 1; + if (tl_noted > v) v = tl_noted; + plat_rmutex_unlock(&cmd_lock); + return v; +} + void vk_mem_collect(void) { VkCtx *c = vk_ctx(); if (!c) return; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 3f72948c..d461f561 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -597,10 +597,16 @@ static void destroy_tex(Tex *e) { if (p->next_at == e) { p->next_at = e->next_at; break; } } if (e->released) nreleased--, rel_unlink(e); + // Another thread may hold &e->pub, taken under the lock and used after it (vk_texture_get, vk_texture_by_id: render + // targets for a whole pass, vk_exec.c resolve_target, vk_texture_storage_bound_locked writes through it). Freed + // here, that thread read a stale VkImage from freed memory, made views of it and cached them by that handle. The + // record goes with the image, once the submission being recorded is done; gone keeps views of it out of the caches. + e->pub.gone = 1; vk_exec_image_gone(e->pub.image); vk_defer_image(e->pub.image, e->alloc); free(e->snap); - free(e); + e->snap = NULL; + vk_defer_free(e); } // Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries the