Merge task/deferstamp: deferred frees stamped with the submission being recorded (NVIDIA write-invalid faults, #788, #802) and the GPU address map for device faults

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

runtime/vk.h

@@ -379,7 +379,10 @@ void vk_deswz_release(void);            // vk_device.c, before the device goes
 // 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission
 // sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch).
 int vk_mem_span_live(const VkMemSpan *sp);
-void vk_mem_note_submit(uint64_t timeline_value);  // vk_sync.c: after each submit that signals ctx->timeline with this value
+// vk_sync.c, when it starts recording the submission that will signal ctx->timeline with this value: objects destroyed
+// from then on are freed once the timeline reaches it.
+void vk_mem_note_fence(uint64_t timeline_value);
+void vk_mem_fault_log(void);   // vk_device_fault_report: the deferred-free state, lock-free
 void vk_mem_collect(void);                         // frees what has completed
 typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats;
 void vk_mem_stats(VkMemStats *s);

runtime/vk_device.c

@@ -16,9 +16,69 @@ static VkCtx g_vk;
 static int g_ready;
 static PlatMutex g_lock = PLAT_MUTEX_INIT;
 
+// ---- GPU address map for device faults
+// VK_EXT_device_address_binding_report: the driver reports each bind and unbind of an object's GPU virtual range through
+// the debug messenger. The newest AB_N events stay in a ring written without a lock (one atomic add each), so a device
+// fault names the object behind its address and whether it was unbound before the fault, which is memory freed while
+// a submission still used it. One event per image, buffer or memory block made or freed, none per draw.
+// TOTK_ADDR_REPORT=0 turns it off.
+#define AB_N (1u << 15)
+typedef struct { uint64_t seq, base, size, handle, issued, done; int32_t type; uint32_t unbind; } AbEvent;
+static AbEvent *ab_ring;
+static uint64_t ab_head;
+static int ab_want;   // the instance has the messenger for it (create_instance)
+
+static void ab_record(const VkDebugUtilsMessengerCallbackDataEXT *d) {
+    const VkDeviceAddressBindingCallbackDataEXT *b = NULL;
+    for (const VkBaseInStructure *p = d->pNext; p; p = p->pNext)
+        if (p->sType == VK_STRUCTURE_TYPE_DEVICE_ADDRESS_BINDING_CALLBACK_DATA_EXT) b = (const void *)p;
+    if (!b || !ab_ring) return;
+    uint64_t i = __atomic_fetch_add(&ab_head, 1, __ATOMIC_RELAXED);
+    AbEvent *e = &ab_ring[i & (AB_N - 1)];
+    __atomic_store_n(&e->seq, 0, __ATOMIC_RELAXED);
+    e->base = b->baseAddress, e->size = b->size;
+    e->handle = d->objectCount ? d->pObjects[0].objectHandle : 0;
+    e->type = d->objectCount ? (int32_t)d->pObjects[0].objectType : 0;
+    e->unbind = b->bindingType == VK_DEVICE_ADDRESS_BINDING_TYPE_UNBIND_EXT;
+    e->issued = vk_sync_issued(), e->done = vk_sync_fence_done();
+    __atomic_store_n(&e->seq, i + 1, __ATOMIC_RELEASE);
+}
+
+static const char *ab_type_name(int32_t t, char *buf, size_t n) {
+    switch (t) {
+    case VK_OBJECT_TYPE_IMAGE: return "image";
+    case VK_OBJECT_TYPE_BUFFER: return "buffer";
+    case VK_OBJECT_TYPE_DEVICE_MEMORY: return "memory";
+    default: snprintf(buf, n, "object type %d", (int)t); return buf;
+    }
+}
+
+// The newest events whose range meets the precision-aligned range around a fault address, newest first.
+static void ab_report(uint64_t addr, uint64_t precision) {
+    if (!ab_ring) return;
+    uint64_t lo = precision > 1 ? addr & ~(precision - 1) : addr, hi = lo + (precision > 1 ? precision : 1);
+    uint64_t head = __atomic_load_n(&ab_head, __ATOMIC_ACQUIRE), n = head < AB_N ? head : AB_N;
+    int found = 0;
+    for (uint64_t k = 1; k <= n && found < 6; k++) {
+        const AbEvent *e = &ab_ring[(head - k) & (AB_N - 1)];
+        if (__atomic_load_n(&e->seq, __ATOMIC_ACQUIRE) != head - k + 1) continue;
+        if (e->base >= hi || e->base + e->size <= lo) continue;
+        found++;
+        char tn[32];
+        rt_log("vk: fault address %#llx: %s %#llx [%#llx, +%#llx] %s %llu address events before the fault, "
+               "submitted %llu, done %llu", (unsigned long long)addr, ab_type_name(e->type, tn, sizeof tn),(unsigned long long)e->handle,
+               (unsigned long long)e->base, (unsigned long long)e->size, e->unbind ? "unbound" : "bound",
+               (unsigned long long)k, (unsigned long long)e->issued, (unsigned long long)e->done);
+    }
+    if (!found) rt_log("vk: fault address %#llx: no binding in the last %llu address events", (unsigned long long)addr,
+                       (unsigned long long)n);
+}
+
 static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type,
                                                const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) {
-    (void)type, (void)user;
+    (void)user;
+    if (type & VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT) { ab_record(d); return VK_FALSE; }
+    if (sev < VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) return VK_FALSE;   // info, asked for the address events
     int err = (sev & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) != 0;
     if (err) plat_inc32(&g_vk.validation_errors);
     static volatile int32_t logged;
@@ -76,7 +136,10 @@ static int create_instance(void) {
         rt_log("vk: TOTK_VK_VALIDATION set but %s is not installed; continuing without", layers[0]);
         validate = 0;
     }
-    if (validate) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
+    const char *ar = cfg_get("TOTK_ADDR_REPORT");
+    ab_want = !(ar && !strcmp(ar, "0")) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
+    if (ab_want && !ab_ring && !(ab_ring = calloc(AB_N, sizeof *ab_ring))) ab_want = 0;
+    if (validate || ab_want) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
     if (vk_hdr_wanted() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME))   // HDR10 surface formats
         ext[n_ext++] = VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME;
     vk_dlaa_instance_exts(ext, &n_ext, 12);
@@ -89,16 +152,18 @@ static int create_instance(void) {
     VkResult r = vkCreateInstance(&ci, NULL, &g_vk.instance);
     if (r != VK_SUCCESS) { rt_log("vk: vkCreateInstance failed (%d); is a Vulkan 1.3 driver installed?", (int)r); return 0; }
     g_vk.validation = validate;
-    if (validate) {
+    if (validate || ab_want) {
         PFN_vkCreateDebugUtilsMessengerEXT f =
             (PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkCreateDebugUtilsMessengerEXT");
         VkDebugUtilsMessengerCreateInfoEXT mi = {
             .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
-            .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
-            .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
-                           VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
+            .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT |
+                               (ab_want ? VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT : 0),
+            .messageType = (validate ? VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
+                                       VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT : 0) |
+                           (ab_want ? VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT : 0),
             .pfnUserCallback = debug_cb};
-        if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger);
+        if (!f || f(g_vk.instance, &mi, NULL, &g_vk.messenger) != VK_SUCCESS) ab_want = 0;
     }
     return 1;
 }
@@ -217,6 +282,7 @@ typedef struct {
     VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl;
     VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbl;
     VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr;
+    VkPhysicalDeviceAddressBindingReportFeaturesEXT abr;
 } FeatureChain;
 
 static int create_device(void) {
@@ -251,6 +317,13 @@ static int create_device(void) {
     // Ryujinx's SPIR-V declares fragment shader interlock whenever the host reports it (vk_pipeline.c spirv_fixup).
     OPTIONAL_EXT(has_fragment_shader_interlock, VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME, fsi,
                  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT);
+    // The GPU address map for device faults (ab_record), with the messenger made for it
+    int has_abr = 0;
+    if (ab_want && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME)) {
+        has_abr = 1;
+        ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME;
+        CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT);
+    }
     // Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering
     int has_fbl = 0, has_lr = 0;
     if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) &&
@@ -314,6 +387,10 @@ static int create_device(void) {
     else if (!h->textureCompressionBC) miss = "textureCompressionBC";
     if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; }
     if (!c->fault.deviceFault) g_vk.has_device_fault = 0;   // vkGetDeviceFaultInfoEXT needs the feature, not just the extension
+    rt_log("vk: GPU address map for device faults: %s",
+           has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report; TOTK_ADDR_REPORT=0 turns it off)"
+           : !ab_want ? "off (TOTK_ADDR_REPORT=0 or no VK_EXT_debug_utils)"
+                      : "unavailable (no VK_EXT_device_address_binding_report)");
     g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress;
     g_vk.has_astc_ldr = h->textureCompressionASTC_LDR;
     {
@@ -438,6 +515,7 @@ void vk_device_fault_report(VkResult r) {
     done = 1;
     vk_crumbs_report();
     vk_pass_marks_report();
+    vk_mem_fault_log();
     VkCtx *c = vk_ctx();
     PFN_vkGetDeviceFaultInfoEXT get =
         c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT")
@@ -462,9 +540,11 @@ void vk_device_fault_report(VkResult r) {
         rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description,
                n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary);
         // The faulting access lies in the precision-aligned range around the reported address.
-        for (uint32_t i = 0; i < n.addressInfoCount; i++)
+        for (uint32_t i = 0; i < n.addressInfoCount; i++) {
             rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType),
                    (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision);
+            ab_report(ai[i].reportedAddress, ai[i].addressPrecision);
+        }
         for (uint32_t i = 0; i < n.vendorInfoCount; i++)
             rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description,
                    (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData);

runtime/vk_memory.c

@@ -511,6 +511,9 @@ static void dirty_frame(uint64_t frame);
 static void mirror_log(uint64_t frame);
 static void shadow_log(uint64_t frame);
 static int dead_pending(void);
+// Deferrals made off the recording thread while another thread recorded a submission (cmd_lock, defer), the newest
+// one's stamp: the case the stamp from the recording start covers. In the census line and the device fault report.
+static uint64_t n_cross_defer, cross_defer_tl;
 
 static void vram_log(uint64_t frame);
 
@@ -546,10 +549,11 @@ void vk_mem_census(uint64_t frame) {
     }
     plat_mutex_unlock_shared(&pool_lock);
     rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%d without a GPU buffer), physical %llu pools %.1f MiB, "
-           "virtual %llu pools %.1f MiB of address space",
+           "virtual %llu pools %.1f MiB of address space; %llu objects deferred off the recording thread during a recording",
            (unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0,
            n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0,
-           (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0);
+           (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0,
+           (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED));
     static const char *const kinds[] = {"", "import", "physical", "virtual"};
     for (int j = 0; j < ng; j++) {
         char f[160];
@@ -2174,7 +2178,8 @@ static int nspare, cap_spare;
 static uint64_t next_seq, done_seq;   // batches handed out / completed (all batches up to done_seq)
 static Dead *dead;
 static int ndead, cap_dead;
-static uint64_t tl_noted;             // newest timeline value vk_sync.c reported submitted
+static uint64_t tl_noted;             // value of the submission being recorded, else of the last one (vk_mem_note_fence)
+static _Thread_local uint64_t tl_noted_here;   // the value this thread noted: it records that submission
 
 static void cmd_init(void *p) {
     (void)p;
@@ -2409,17 +2414,17 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) {
     plat_rmutex_lock(&cmd_lock);
     if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead);
     // The open batch may record into the object: it counts as the newest user. The GPU must also be past every
-    // submission that names it. tl_noted covers the submitted ones; an object the replay lets go of while it records
-    // a submission (vk_exec_recording_fence) may be named by that one too, not yet submitted: vk_texture.c
-    // slot_resolve hands a draw a released entry, and entry_of destroys released entries at that address later in
-    // the same submission. Stamped with the last submitted value, the object went as soon as the GPU reached it,
-    // before the submission that names it ran, and with the recorder thread (vk_rec.c) behind, before the recorder
-    // made the call that names it. LAS Recomp and Savage Wind Recomp stamp every deferral with the submission being
-    // recorded (runtime/nvn/vulkan/textures.c defer: vk_pending_seq). TOTK_DEFER_STAMP=0 brings back the old stamp.
-    static int stamp = -1;
-    if (stamp < 0) stamp = rt_knob("DEFER_STAMP", "1") != 0;
-    uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0;
-    if (rf > tl) tl = rf;
+    // submission that names it, including the one being recorded, which may name it and is not submitted yet:
+    // vk_texture.c slot_resolve hands a draw a released entry, and entry_of, sweep_released and the VRAM pressure drop
+    // destroy released entries. tl_noted is that submission's value from the start of its recording (vk_sync.c
+    // vk_sync_submit), whichever thread destroys the object. With the synchronous replay a game thread finalizing a
+    // texture (vk_backend.c vk_be_texture_release) destroys entries while another game thread records; stamped with
+    // the last submitted value, as the recording thread's own fence was the only one seen, the object was freed once
+    // the GPU reached that value, before the submission naming it ran: "write invalid" device faults on NVIDIA. LAS
+    // Recomp and Savage Wind Recomp stamp every deferral with the submission being recorded (runtime/nvn/vulkan/
+    // textures.c defer: vk_pending_seq).
+    uint64_t tl = tl_noted;
+    if (tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl;
     dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl};
     rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl,
                      vk_sync_fence_done());
@@ -2442,10 +2447,22 @@ void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) {
     if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
 }
 
-void vk_mem_note_submit(uint64_t timeline_value) {
+// The device fault report: no lock, since a thread that holds cmd_lock may be waiting for the report's own lock
+// (vk_upload_flush reports a failed submit with cmd_lock held).
+void vk_mem_fault_log(void) {
+    rt_log("vk: deferred frees at the fault: %d pending, stamp %llu, submitted %llu, done %llu; %llu deferred off the "
+           "recording thread during a recording (newest stamp %llu)",
+           __atomic_load_n(&ndead, __ATOMIC_RELAXED), (unsigned long long)__atomic_load_n(&tl_noted, __ATOMIC_RELAXED),
+           (unsigned long long)vk_sync_issued(), (unsigned long long)vk_sync_fence_done(),
+           (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED),
+           (unsigned long long)__atomic_load_n(&cross_defer_tl, __ATOMIC_RELAXED));
+}
+
+void vk_mem_note_fence(uint64_t timeline_value) {
     plat_once(&cmd_once, cmd_init, NULL);
     plat_rmutex_lock(&cmd_lock);
     if (timeline_value > tl_noted) tl_noted = timeline_value;
+    tl_noted_here = timeline_value;
     plat_rmutex_unlock(&cmd_lock);
 }
 

runtime/vk_sync.c

@@ -309,7 +309,7 @@ void vk_sync_start(void) {
         if (reserved < now) reserved = now;
         stop = 0;
         plat_mutex_unlock(&st_lock);
-        vk_mem_note_submit(now);
+        vk_mem_note_fence(now);
         completion_thread = plat_thread_new(completion_main, c, "vk completion");
         if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
         __atomic_store_n(&started, 1, __ATOMIC_RELEASE);
@@ -379,7 +379,6 @@ VkCommandBuffer vk_sync_split(void) {
         vk_device_fault_report(r);
         rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r);
     }
-    vk_mem_note_submit(cur_fence);   // objects destroyed from here on may be in use by the part just submitted
     CmdSlot *s = take_slot(c, cur_fence);   // the submitted part keeps its slot until the value is reached
     rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence);
     ts_begin(s->cb);
@@ -404,6 +403,9 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
     vk_mem_dirty_submit();   // TOTK_TRACE=perf: what a mirror would upload before this submit
     uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1;
     cur_fence = fence;
+    // Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c
+    // defer), since the recording may already name it.
+    vk_mem_note_fence(fence);
     CmdSlot *s = take_slot(c, fence);
     uint64_t t0 = perf_begin();
     rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence);   // begun by the recorder when it records the part
@@ -434,7 +436,6 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
         rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r);
     }
     s->fence = recorded ? fence : 0;
-    vk_mem_note_submit(fence);
     plat_mutex_lock(&st_lock);
     __atomic_store_n(&issued, fence, __ATOMIC_RELEASE);
     plat_cond_broadcast(&cv_work);