diff --git a/runtime/vk.h b/runtime/vk.h index 1f210afb..0dfd738d 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -379,7 +379,10 @@ void vk_deswz_release(void); // vk_device.c, before the device goes // 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission // sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch). int vk_mem_span_live(const VkMemSpan *sp); -void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value +// vk_sync.c, when it starts recording the submission that will signal ctx->timeline with this value: objects destroyed +// from then on are freed once the timeline reaches it. +void vk_mem_note_fence(uint64_t timeline_value); +void vk_mem_fault_log(void); // vk_device_fault_report: the deferred-free state, lock-free void vk_mem_collect(void); // frees what has completed typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats; void vk_mem_stats(VkMemStats *s); diff --git a/runtime/vk_device.c b/runtime/vk_device.c index b0845994..31fbc1a4 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -16,9 +16,69 @@ static VkCtx g_vk; static int g_ready; static PlatMutex g_lock = PLAT_MUTEX_INIT; +// ---- GPU address map for device faults +// VK_EXT_device_address_binding_report: the driver reports each bind and unbind of an object's GPU virtual range through +// the debug messenger. The newest AB_N events stay in a ring written without a lock (one atomic add each), so a device +// fault names the object behind its address and whether it was unbound before the fault, which is memory freed while +// a submission still used it. One event per image, buffer or memory block made or freed, none per draw. +// TOTK_ADDR_REPORT=0 turns it off. +#define AB_N (1u << 15) +typedef struct { uint64_t seq, base, size, handle, issued, done; int32_t type; uint32_t unbind; } AbEvent; +static AbEvent *ab_ring; +static uint64_t ab_head; +static int ab_want; // the instance has the messenger for it (create_instance) + +static void ab_record(const VkDebugUtilsMessengerCallbackDataEXT *d) { + const VkDeviceAddressBindingCallbackDataEXT *b = NULL; + for (const VkBaseInStructure *p = d->pNext; p; p = p->pNext) + if (p->sType == VK_STRUCTURE_TYPE_DEVICE_ADDRESS_BINDING_CALLBACK_DATA_EXT) b = (const void *)p; + if (!b || !ab_ring) return; + uint64_t i = __atomic_fetch_add(&ab_head, 1, __ATOMIC_RELAXED); + AbEvent *e = &ab_ring[i & (AB_N - 1)]; + __atomic_store_n(&e->seq, 0, __ATOMIC_RELAXED); + e->base = b->baseAddress, e->size = b->size; + e->handle = d->objectCount ? d->pObjects[0].objectHandle : 0; + e->type = d->objectCount ? (int32_t)d->pObjects[0].objectType : 0; + e->unbind = b->bindingType == VK_DEVICE_ADDRESS_BINDING_TYPE_UNBIND_EXT; + e->issued = vk_sync_issued(), e->done = vk_sync_fence_done(); + __atomic_store_n(&e->seq, i + 1, __ATOMIC_RELEASE); +} + +static const char *ab_type_name(int32_t t, char *buf, size_t n) { + switch (t) { + case VK_OBJECT_TYPE_IMAGE: return "image"; + case VK_OBJECT_TYPE_BUFFER: return "buffer"; + case VK_OBJECT_TYPE_DEVICE_MEMORY: return "memory"; + default: snprintf(buf, n, "object type %d", (int)t); return buf; + } +} + +// The newest events whose range meets the precision-aligned range around a fault address, newest first. +static void ab_report(uint64_t addr, uint64_t precision) { + if (!ab_ring) return; + uint64_t lo = precision > 1 ? addr & ~(precision - 1) : addr, hi = lo + (precision > 1 ? precision : 1); + uint64_t head = __atomic_load_n(&ab_head, __ATOMIC_ACQUIRE), n = head < AB_N ? head : AB_N; + int found = 0; + for (uint64_t k = 1; k <= n && found < 6; k++) { + const AbEvent *e = &ab_ring[(head - k) & (AB_N - 1)]; + if (__atomic_load_n(&e->seq, __ATOMIC_ACQUIRE) != head - k + 1) continue; + if (e->base >= hi || e->base + e->size <= lo) continue; + found++; + char tn[32]; + rt_log("vk: fault address %#llx: %s %#llx [%#llx, +%#llx] %s %llu address events before the fault, " + "submitted %llu, done %llu", (unsigned long long)addr, ab_type_name(e->type, tn, sizeof tn),(unsigned long long)e->handle, + (unsigned long long)e->base, (unsigned long long)e->size, e->unbind ? "unbound" : "bound", + (unsigned long long)k, (unsigned long long)e->issued, (unsigned long long)e->done); + } + if (!found) rt_log("vk: fault address %#llx: no binding in the last %llu address events", (unsigned long long)addr, + (unsigned long long)n); +} + static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type, const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) { - (void)type, (void)user; + (void)user; + if (type & VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT) { ab_record(d); return VK_FALSE; } + if (sev < VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) return VK_FALSE; // info, asked for the address events int err = (sev & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) != 0; if (err) plat_inc32(&g_vk.validation_errors); static volatile int32_t logged; @@ -76,7 +136,10 @@ static int create_instance(void) { rt_log("vk: TOTK_VK_VALIDATION set but %s is not installed; continuing without", layers[0]); validate = 0; } - if (validate) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; + const char *ar = cfg_get("TOTK_ADDR_REPORT"); + ab_want = !(ar && !strcmp(ar, "0")) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + if (ab_want && !ab_ring && !(ab_ring = calloc(AB_N, sizeof *ab_ring))) ab_want = 0; + if (validate || ab_want) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; if (vk_hdr_wanted() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats ext[n_ext++] = VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME; vk_dlaa_instance_exts(ext, &n_ext, 12); @@ -89,16 +152,18 @@ static int create_instance(void) { VkResult r = vkCreateInstance(&ci, NULL, &g_vk.instance); if (r != VK_SUCCESS) { rt_log("vk: vkCreateInstance failed (%d); is a Vulkan 1.3 driver installed?", (int)r); return 0; } g_vk.validation = validate; - if (validate) { + if (validate || ab_want) { PFN_vkCreateDebugUtilsMessengerEXT f = (PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkCreateDebugUtilsMessengerEXT"); VkDebugUtilsMessengerCreateInfoEXT mi = { .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT, - .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT, - .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | - VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT, + .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT | + (ab_want ? VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT : 0), + .messageType = (validate ? VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT : 0) | + (ab_want ? VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT : 0), .pfnUserCallback = debug_cb}; - if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger); + if (!f || f(g_vk.instance, &mi, NULL, &g_vk.messenger) != VK_SUCCESS) ab_want = 0; } return 1; } @@ -217,6 +282,7 @@ typedef struct { VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl; VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbl; VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr; + VkPhysicalDeviceAddressBindingReportFeaturesEXT abr; } FeatureChain; static int create_device(void) { @@ -251,6 +317,13 @@ static int create_device(void) { // Ryujinx's SPIR-V declares fragment shader interlock whenever the host reports it (vk_pipeline.c spirv_fixup). OPTIONAL_EXT(has_fragment_shader_interlock, VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME, fsi, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT); + // The GPU address map for device faults (ab_record), with the messenger made for it + int has_abr = 0; + if (ab_want && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME)) { + has_abr = 1; + ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME; + CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT); + } // Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering int has_fbl = 0, has_lr = 0; if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) && @@ -314,6 +387,10 @@ static int create_device(void) { else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension + rt_log("vk: GPU address map for device faults: %s", + has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report; TOTK_ADDR_REPORT=0 turns it off)" + : !ab_want ? "off (TOTK_ADDR_REPORT=0 or no VK_EXT_debug_utils)" + : "unavailable (no VK_EXT_device_address_binding_report)"); g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { @@ -438,6 +515,7 @@ void vk_device_fault_report(VkResult r) { done = 1; vk_crumbs_report(); vk_pass_marks_report(); + vk_mem_fault_log(); VkCtx *c = vk_ctx(); PFN_vkGetDeviceFaultInfoEXT get = c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") @@ -462,9 +540,11 @@ void vk_device_fault_report(VkResult r) { rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description, n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary); // The faulting access lies in the precision-aligned range around the reported address. - for (uint32_t i = 0; i < n.addressInfoCount; i++) + for (uint32_t i = 0; i < n.addressInfoCount; i++) { rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType), (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision); + ab_report(ai[i].reportedAddress, ai[i].addressPrecision); + } for (uint32_t i = 0; i < n.vendorInfoCount; i++) rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description, (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index f19a1b88..22b04e1c 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -511,6 +511,9 @@ static void dirty_frame(uint64_t frame); static void mirror_log(uint64_t frame); static void shadow_log(uint64_t frame); static int dead_pending(void); +// Deferrals made off the recording thread while another thread recorded a submission (cmd_lock, defer), the newest +// one's stamp: the case the stamp from the recording start covers. In the census line and the device fault report. +static uint64_t n_cross_defer, cross_defer_tl; static void vram_log(uint64_t frame); @@ -546,10 +549,11 @@ void vk_mem_census(uint64_t frame) { } plat_mutex_unlock_shared(&pool_lock); rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%d without a GPU buffer), physical %llu pools %.1f MiB, " - "virtual %llu pools %.1f MiB of address space", + "virtual %llu pools %.1f MiB of address space; %llu objects deferred off the recording thread during a recording", (unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0, n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0, - (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0); + (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0, + (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED)); static const char *const kinds[] = {"", "import", "physical", "virtual"}; for (int j = 0; j < ng; j++) { char f[160]; @@ -2174,7 +2178,8 @@ static int nspare, cap_spare; static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq) static Dead *dead; static int ndead, cap_dead; -static uint64_t tl_noted; // newest timeline value vk_sync.c reported submitted +static uint64_t tl_noted; // value of the submission being recorded, else of the last one (vk_mem_note_fence) +static _Thread_local uint64_t tl_noted_here; // the value this thread noted: it records that submission static void cmd_init(void *p) { (void)p; @@ -2409,17 +2414,17 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) { plat_rmutex_lock(&cmd_lock); if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead); // The open batch may record into the object: it counts as the newest user. The GPU must also be past every - // submission that names it. tl_noted covers the submitted ones; an object the replay lets go of while it records - // a submission (vk_exec_recording_fence) may be named by that one too, not yet submitted: vk_texture.c - // slot_resolve hands a draw a released entry, and entry_of destroys released entries at that address later in - // the same submission. Stamped with the last submitted value, the object went as soon as the GPU reached it, - // before the submission that names it ran, and with the recorder thread (vk_rec.c) behind, before the recorder - // made the call that names it. LAS Recomp and Savage Wind Recomp stamp every deferral with the submission being - // recorded (runtime/nvn/vulkan/textures.c defer: vk_pending_seq). TOTK_DEFER_STAMP=0 brings back the old stamp. - static int stamp = -1; - if (stamp < 0) stamp = rt_knob("DEFER_STAMP", "1") != 0; - uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0; - if (rf > tl) tl = rf; + // submission that names it, including the one being recorded, which may name it and is not submitted yet: + // vk_texture.c slot_resolve hands a draw a released entry, and entry_of, sweep_released and the VRAM pressure drop + // destroy released entries. tl_noted is that submission's value from the start of its recording (vk_sync.c + // vk_sync_submit), whichever thread destroys the object. With the synchronous replay a game thread finalizing a + // texture (vk_backend.c vk_be_texture_release) destroys entries while another game thread records; stamped with + // the last submitted value, as the recording thread's own fence was the only one seen, the object was freed once + // the GPU reached that value, before the submission naming it ran: "write invalid" device faults on NVIDIA. LAS + // Recomp and Savage Wind Recomp stamp every deferral with the submission being recorded (runtime/nvn/vulkan/ + // textures.c defer: vk_pending_seq). + uint64_t tl = tl_noted; + if (tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl; dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl}; rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl, vk_sync_fence_done()); @@ -2442,10 +2447,22 @@ void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) { if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc); } -void vk_mem_note_submit(uint64_t timeline_value) { +// The device fault report: no lock, since a thread that holds cmd_lock may be waiting for the report's own lock +// (vk_upload_flush reports a failed submit with cmd_lock held). +void vk_mem_fault_log(void) { + rt_log("vk: deferred frees at the fault: %d pending, stamp %llu, submitted %llu, done %llu; %llu deferred off the " + "recording thread during a recording (newest stamp %llu)", + __atomic_load_n(&ndead, __ATOMIC_RELAXED), (unsigned long long)__atomic_load_n(&tl_noted, __ATOMIC_RELAXED), + (unsigned long long)vk_sync_issued(), (unsigned long long)vk_sync_fence_done(), + (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED), + (unsigned long long)__atomic_load_n(&cross_defer_tl, __ATOMIC_RELAXED)); +} + +void vk_mem_note_fence(uint64_t timeline_value) { plat_once(&cmd_once, cmd_init, NULL); plat_rmutex_lock(&cmd_lock); if (timeline_value > tl_noted) tl_noted = timeline_value; + tl_noted_here = timeline_value; plat_rmutex_unlock(&cmd_lock); } diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index e3162d12..3b90c026 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -309,7 +309,7 @@ void vk_sync_start(void) { if (reserved < now) reserved = now; stop = 0; plat_mutex_unlock(&st_lock); - vk_mem_note_submit(now); + vk_mem_note_fence(now); completion_thread = plat_thread_new(completion_main, c, "vk completion"); if (!completion_thread) rt_fatal("vk: cannot start the completion thread"); __atomic_store_n(&started, 1, __ATOMIC_RELEASE); @@ -379,7 +379,6 @@ VkCommandBuffer vk_sync_split(void) { vk_device_fault_report(r); rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); } - vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence); ts_begin(s->cb); @@ -404,6 +403,9 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1; cur_fence = fence; + // Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c + // defer), since the recording may already name it. + vk_mem_note_fence(fence); CmdSlot *s = take_slot(c, fence); uint64_t t0 = perf_begin(); rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence); // begun by the recorder when it records the part @@ -434,7 +436,6 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); } s->fence = recorded ? fence : 0; - vk_mem_note_submit(fence); plat_mutex_lock(&st_lock); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work);