From dc45de0e9a3f951bc99c7a1d52eeb42ac7b29470 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 7 Nov 2026 19:16:50 +0300 Subject: [PATCH] Stamp deferred frees with the submission being recorded, whichever thread destroys the object With the synchronous replay (the default since 7be68ec) a game thread finalizing a texture destroys released texture cache entries while another game thread records a submission that slot_resolve may already have handed one of them. defer() saw the recording's value only on the recording thread and stamped the others with the last submitted value, so the completion thread freed the image before the submission naming it ran: "write invalid" device faults on NVIDIA (#788, #802). vk_sync_submit now notes the value at the start of the recording, and every deferral made from then on waits for it. A fight run counted 10 such deferrals during the world load. Device faults now log the deferred-free state, and with VK_EXT_device_address_binding_report the object that held the faulting address and whether it was unbound before the fault (TOTK_ADDR_REPORT=0 turns the map off). The census line counts deferrals made off the recording thread during a recording. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 5 ++- runtime/vk_device.c | 96 +++++++++++++++++++++++++++++++++++++++++---- runtime/vk_memory.c | 47 +++++++++++++++------- runtime/vk_sync.c | 7 ++-- 4 files changed, 128 insertions(+), 27 deletions(-) diff --git a/runtime/vk.h b/runtime/vk.h index 1f210afb..0dfd738d 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -379,7 +379,10 @@ void vk_deswz_release(void); // vk_device.c, before the device goes // 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission // sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch). int vk_mem_span_live(const VkMemSpan *sp); -void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value +// vk_sync.c, when it starts recording the submission that will signal ctx->timeline with this value: objects destroyed +// from then on are freed once the timeline reaches it. +void vk_mem_note_fence(uint64_t timeline_value); +void vk_mem_fault_log(void); // vk_device_fault_report: the deferred-free state, lock-free void vk_mem_collect(void); // frees what has completed typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats; void vk_mem_stats(VkMemStats *s); diff --git a/runtime/vk_device.c b/runtime/vk_device.c index b0845994..31fbc1a4 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -16,9 +16,69 @@ static VkCtx g_vk; static int g_ready; static PlatMutex g_lock = PLAT_MUTEX_INIT; +// ---- GPU address map for device faults +// VK_EXT_device_address_binding_report: the driver reports each bind and unbind of an object's GPU virtual range through +// the debug messenger. The newest AB_N events stay in a ring written without a lock (one atomic add each), so a device +// fault names the object behind its address and whether it was unbound before the fault, which is memory freed while +// a submission still used it. One event per image, buffer or memory block made or freed, none per draw. +// TOTK_ADDR_REPORT=0 turns it off. +#define AB_N (1u << 15) +typedef struct { uint64_t seq, base, size, handle, issued, done; int32_t type; uint32_t unbind; } AbEvent; +static AbEvent *ab_ring; +static uint64_t ab_head; +static int ab_want; // the instance has the messenger for it (create_instance) + +static void ab_record(const VkDebugUtilsMessengerCallbackDataEXT *d) { + const VkDeviceAddressBindingCallbackDataEXT *b = NULL; + for (const VkBaseInStructure *p = d->pNext; p; p = p->pNext) + if (p->sType == VK_STRUCTURE_TYPE_DEVICE_ADDRESS_BINDING_CALLBACK_DATA_EXT) b = (const void *)p; + if (!b || !ab_ring) return; + uint64_t i = __atomic_fetch_add(&ab_head, 1, __ATOMIC_RELAXED); + AbEvent *e = &ab_ring[i & (AB_N - 1)]; + __atomic_store_n(&e->seq, 0, __ATOMIC_RELAXED); + e->base = b->baseAddress, e->size = b->size; + e->handle = d->objectCount ? d->pObjects[0].objectHandle : 0; + e->type = d->objectCount ? (int32_t)d->pObjects[0].objectType : 0; + e->unbind = b->bindingType == VK_DEVICE_ADDRESS_BINDING_TYPE_UNBIND_EXT; + e->issued = vk_sync_issued(), e->done = vk_sync_fence_done(); + __atomic_store_n(&e->seq, i + 1, __ATOMIC_RELEASE); +} + +static const char *ab_type_name(int32_t t, char *buf, size_t n) { + switch (t) { + case VK_OBJECT_TYPE_IMAGE: return "image"; + case VK_OBJECT_TYPE_BUFFER: return "buffer"; + case VK_OBJECT_TYPE_DEVICE_MEMORY: return "memory"; + default: snprintf(buf, n, "object type %d", (int)t); return buf; + } +} + +// The newest events whose range meets the precision-aligned range around a fault address, newest first. +static void ab_report(uint64_t addr, uint64_t precision) { + if (!ab_ring) return; + uint64_t lo = precision > 1 ? addr & ~(precision - 1) : addr, hi = lo + (precision > 1 ? precision : 1); + uint64_t head = __atomic_load_n(&ab_head, __ATOMIC_ACQUIRE), n = head < AB_N ? head : AB_N; + int found = 0; + for (uint64_t k = 1; k <= n && found < 6; k++) { + const AbEvent *e = &ab_ring[(head - k) & (AB_N - 1)]; + if (__atomic_load_n(&e->seq, __ATOMIC_ACQUIRE) != head - k + 1) continue; + if (e->base >= hi || e->base + e->size <= lo) continue; + found++; + char tn[32]; + rt_log("vk: fault address %#llx: %s %#llx [%#llx, +%#llx] %s %llu address events before the fault, " + "submitted %llu, done %llu", (unsigned long long)addr, ab_type_name(e->type, tn, sizeof tn),(unsigned long long)e->handle, + (unsigned long long)e->base, (unsigned long long)e->size, e->unbind ? "unbound" : "bound", + (unsigned long long)k, (unsigned long long)e->issued, (unsigned long long)e->done); + } + if (!found) rt_log("vk: fault address %#llx: no binding in the last %llu address events", (unsigned long long)addr, + (unsigned long long)n); +} + static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type, const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) { - (void)type, (void)user; + (void)user; + if (type & VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT) { ab_record(d); return VK_FALSE; } + if (sev < VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) return VK_FALSE; // info, asked for the address events int err = (sev & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) != 0; if (err) plat_inc32(&g_vk.validation_errors); static volatile int32_t logged; @@ -76,7 +136,10 @@ static int create_instance(void) { rt_log("vk: TOTK_VK_VALIDATION set but %s is not installed; continuing without", layers[0]); validate = 0; } - if (validate) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; + const char *ar = cfg_get("TOTK_ADDR_REPORT"); + ab_want = !(ar && !strcmp(ar, "0")) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + if (ab_want && !ab_ring && !(ab_ring = calloc(AB_N, sizeof *ab_ring))) ab_want = 0; + if (validate || ab_want) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; if (vk_hdr_wanted() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats ext[n_ext++] = VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME; vk_dlaa_instance_exts(ext, &n_ext, 12); @@ -89,16 +152,18 @@ static int create_instance(void) { VkResult r = vkCreateInstance(&ci, NULL, &g_vk.instance); if (r != VK_SUCCESS) { rt_log("vk: vkCreateInstance failed (%d); is a Vulkan 1.3 driver installed?", (int)r); return 0; } g_vk.validation = validate; - if (validate) { + if (validate || ab_want) { PFN_vkCreateDebugUtilsMessengerEXT f = (PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkCreateDebugUtilsMessengerEXT"); VkDebugUtilsMessengerCreateInfoEXT mi = { .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT, - .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT, - .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | - VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT, + .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT | + (ab_want ? VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT : 0), + .messageType = (validate ? VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT : 0) | + (ab_want ? VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT : 0), .pfnUserCallback = debug_cb}; - if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger); + if (!f || f(g_vk.instance, &mi, NULL, &g_vk.messenger) != VK_SUCCESS) ab_want = 0; } return 1; } @@ -217,6 +282,7 @@ typedef struct { VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl; VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbl; VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr; + VkPhysicalDeviceAddressBindingReportFeaturesEXT abr; } FeatureChain; static int create_device(void) { @@ -251,6 +317,13 @@ static int create_device(void) { // Ryujinx's SPIR-V declares fragment shader interlock whenever the host reports it (vk_pipeline.c spirv_fixup). OPTIONAL_EXT(has_fragment_shader_interlock, VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME, fsi, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT); + // The GPU address map for device faults (ab_record), with the messenger made for it + int has_abr = 0; + if (ab_want && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME)) { + has_abr = 1; + ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME; + CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT); + } // Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering int has_fbl = 0, has_lr = 0; if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) && @@ -314,6 +387,10 @@ static int create_device(void) { else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension + rt_log("vk: GPU address map for device faults: %s", + has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report; TOTK_ADDR_REPORT=0 turns it off)" + : !ab_want ? "off (TOTK_ADDR_REPORT=0 or no VK_EXT_debug_utils)" + : "unavailable (no VK_EXT_device_address_binding_report)"); g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { @@ -438,6 +515,7 @@ void vk_device_fault_report(VkResult r) { done = 1; vk_crumbs_report(); vk_pass_marks_report(); + vk_mem_fault_log(); VkCtx *c = vk_ctx(); PFN_vkGetDeviceFaultInfoEXT get = c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") @@ -462,9 +540,11 @@ void vk_device_fault_report(VkResult r) { rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description, n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary); // The faulting access lies in the precision-aligned range around the reported address. - for (uint32_t i = 0; i < n.addressInfoCount; i++) + for (uint32_t i = 0; i < n.addressInfoCount; i++) { rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType), (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision); + ab_report(ai[i].reportedAddress, ai[i].addressPrecision); + } for (uint32_t i = 0; i < n.vendorInfoCount; i++) rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description, (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index f19a1b88..22b04e1c 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -511,6 +511,9 @@ static void dirty_frame(uint64_t frame); static void mirror_log(uint64_t frame); static void shadow_log(uint64_t frame); static int dead_pending(void); +// Deferrals made off the recording thread while another thread recorded a submission (cmd_lock, defer), the newest +// one's stamp: the case the stamp from the recording start covers. In the census line and the device fault report. +static uint64_t n_cross_defer, cross_defer_tl; static void vram_log(uint64_t frame); @@ -546,10 +549,11 @@ void vk_mem_census(uint64_t frame) { } plat_mutex_unlock_shared(&pool_lock); rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%d without a GPU buffer), physical %llu pools %.1f MiB, " - "virtual %llu pools %.1f MiB of address space", + "virtual %llu pools %.1f MiB of address space; %llu objects deferred off the recording thread during a recording", (unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0, n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0, - (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0); + (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0, + (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED)); static const char *const kinds[] = {"", "import", "physical", "virtual"}; for (int j = 0; j < ng; j++) { char f[160]; @@ -2174,7 +2178,8 @@ static int nspare, cap_spare; static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq) static Dead *dead; static int ndead, cap_dead; -static uint64_t tl_noted; // newest timeline value vk_sync.c reported submitted +static uint64_t tl_noted; // value of the submission being recorded, else of the last one (vk_mem_note_fence) +static _Thread_local uint64_t tl_noted_here; // the value this thread noted: it records that submission static void cmd_init(void *p) { (void)p; @@ -2409,17 +2414,17 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) { plat_rmutex_lock(&cmd_lock); if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead); // The open batch may record into the object: it counts as the newest user. The GPU must also be past every - // submission that names it. tl_noted covers the submitted ones; an object the replay lets go of while it records - // a submission (vk_exec_recording_fence) may be named by that one too, not yet submitted: vk_texture.c - // slot_resolve hands a draw a released entry, and entry_of destroys released entries at that address later in - // the same submission. Stamped with the last submitted value, the object went as soon as the GPU reached it, - // before the submission that names it ran, and with the recorder thread (vk_rec.c) behind, before the recorder - // made the call that names it. LAS Recomp and Savage Wind Recomp stamp every deferral with the submission being - // recorded (runtime/nvn/vulkan/textures.c defer: vk_pending_seq). TOTK_DEFER_STAMP=0 brings back the old stamp. - static int stamp = -1; - if (stamp < 0) stamp = rt_knob("DEFER_STAMP", "1") != 0; - uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0; - if (rf > tl) tl = rf; + // submission that names it, including the one being recorded, which may name it and is not submitted yet: + // vk_texture.c slot_resolve hands a draw a released entry, and entry_of, sweep_released and the VRAM pressure drop + // destroy released entries. tl_noted is that submission's value from the start of its recording (vk_sync.c + // vk_sync_submit), whichever thread destroys the object. With the synchronous replay a game thread finalizing a + // texture (vk_backend.c vk_be_texture_release) destroys entries while another game thread records; stamped with + // the last submitted value, as the recording thread's own fence was the only one seen, the object was freed once + // the GPU reached that value, before the submission naming it ran: "write invalid" device faults on NVIDIA. LAS + // Recomp and Savage Wind Recomp stamp every deferral with the submission being recorded (runtime/nvn/vulkan/ + // textures.c defer: vk_pending_seq). + uint64_t tl = tl_noted; + if (tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl; dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl}; rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl, vk_sync_fence_done()); @@ -2442,10 +2447,22 @@ void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) { if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc); } -void vk_mem_note_submit(uint64_t timeline_value) { +// The device fault report: no lock, since a thread that holds cmd_lock may be waiting for the report's own lock +// (vk_upload_flush reports a failed submit with cmd_lock held). +void vk_mem_fault_log(void) { + rt_log("vk: deferred frees at the fault: %d pending, stamp %llu, submitted %llu, done %llu; %llu deferred off the " + "recording thread during a recording (newest stamp %llu)", + __atomic_load_n(&ndead, __ATOMIC_RELAXED), (unsigned long long)__atomic_load_n(&tl_noted, __ATOMIC_RELAXED), + (unsigned long long)vk_sync_issued(), (unsigned long long)vk_sync_fence_done(), + (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED), + (unsigned long long)__atomic_load_n(&cross_defer_tl, __ATOMIC_RELAXED)); +} + +void vk_mem_note_fence(uint64_t timeline_value) { plat_once(&cmd_once, cmd_init, NULL); plat_rmutex_lock(&cmd_lock); if (timeline_value > tl_noted) tl_noted = timeline_value; + tl_noted_here = timeline_value; plat_rmutex_unlock(&cmd_lock); } diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index e3162d12..3b90c026 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -309,7 +309,7 @@ void vk_sync_start(void) { if (reserved < now) reserved = now; stop = 0; plat_mutex_unlock(&st_lock); - vk_mem_note_submit(now); + vk_mem_note_fence(now); completion_thread = plat_thread_new(completion_main, c, "vk completion"); if (!completion_thread) rt_fatal("vk: cannot start the completion thread"); __atomic_store_n(&started, 1, __ATOMIC_RELEASE); @@ -379,7 +379,6 @@ VkCommandBuffer vk_sync_split(void) { vk_device_fault_report(r); rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); } - vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence); ts_begin(s->cb); @@ -404,6 +403,9 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1; cur_fence = fence; + // Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c + // defer), since the recording may already name it. + vk_mem_note_fence(fence); CmdSlot *s = take_slot(c, fence); uint64_t t0 = perf_begin(); rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence); // begun by the recorder when it records the part @@ -434,7 +436,6 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); } s->fence = recorded ? fence : 0; - vk_mem_note_submit(fence); plat_mutex_lock(&st_lock); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work);