From 112fb2dd7423d3c1e7c857befb1ce25d28d99ae4 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 13:21:14 +0300 Subject: [PATCH] Log the VK_EXT_device_fault report when the device is lost vk_device_fault_report queries vkGetDeviceFaultInfoEXT once per process after VK_ERROR_DEVICE_LOST and logs the description, the faulting addresses with type and precision, and the vendor codes. The submit, upload, occlusion query and wait fatals call it before rt_fatal, and the present path calls it on a failed acquire, blit submit or present. Threads that lose the device together wait until the report is written. The extension counts as usable only when the driver reports the deviceFault feature. No cost unless the device is lost. Co-Authored-By: Claude Opus 5.5 --- runtime/gfx_window.c | 3 +++ runtime/vk.h | 4 +++ runtime/vk_device.c | 59 ++++++++++++++++++++++++++++++++++++++++++ runtime/vk_exec_xfer.c | 5 +++- runtime/vk_memory.c | 5 +++- runtime/vk_sync.c | 15 ++++++++--- 6 files changed, 86 insertions(+), 5 deletions(-) diff --git a/runtime/gfx_window.c b/runtime/gfx_window.c index f62de225..fc449051 100644 --- a/runtime/gfx_window.c +++ b/runtime/gfx_window.c @@ -490,6 +490,7 @@ static void flip_locked(void) { vk_queue_lock(); VkResult r = vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi); vk_queue_unlock(); + vk_device_fault_report(r); if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) InterlockedExchange(&g_dirty, 1); } @@ -763,6 +764,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w if (!create_swapchain()) goto out; r = vkAcquireNextImageKHR(c->device, g_sc, 1000000000ull, g_acquire[slot], VK_NULL_HANDLE, &idx); } + vk_device_fault_report(r); if (r != VK_SUCCESS && r != VK_SUBOPTIMAL_KHR) goto out; // lost, not ready: skip the frame if (r == VK_SUBOPTIMAL_KHR) InterlockedExchange(&g_dirty, 1); vkResetFences(c->device, 1, &g_fence[slot]); @@ -870,6 +872,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w g_flip_idx = idx; if (!defer) flip_locked(); } else { + vk_device_fault_report(r); rt_log("gfx: present submit failed (%d)", (int)r); } g_frame++; diff --git a/runtime/vk.h b/runtime/vk.h index 961f2f35..f6302678 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -69,6 +69,10 @@ VkCtx *vk_ctx(void); // NULL until vk_init() succeeded int vk_init(void); // idempotent; 1 on success, 0 on failure (reason logged with rt_log) void vk_shutdown(void); int vk_selftest(void); // 0 on success +// Call with the result of a failed queue submit, present or wait before stopping on it: VK_ERROR_DEVICE_LOST logs +// what VK_EXT_device_fault reports (description, faulting addresses with their precision, vendor codes) once per +// process. Any other result returns at once. +void vk_device_fault_report(VkResult r); // ---- window and present (gfx_window.c). Host threads only. // vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 01573a38..26881a68 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -267,6 +267,7 @@ static int create_device(void) { else if (!h->geometryShader) miss = "geometryShader"; else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } + if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { @@ -364,6 +365,64 @@ void vk_shutdown(void) { VkCtx *vk_ctx(void) { return g_ready ? &g_vk : NULL; } +static const char *fault_address_type(VkDeviceFaultAddressTypeEXT t) { + switch (t) { + case VK_DEVICE_FAULT_ADDRESS_TYPE_NONE_EXT: return "none"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_READ_INVALID_EXT: return "read invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_WRITE_INVALID_EXT: return "write invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_EXECUTE_INVALID_EXT: return "execute invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_UNKNOWN_EXT: return "instruction pointer (unknown)"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_INVALID_EXT: return "instruction pointer (invalid)"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_FAULT_EXT: return "instruction pointer (fault)"; + default: return "unknown type"; + } +} + +// The query runs once per process. Threads that hit the loss at the same time wait on the lock until it is logged, so +// none of them reaches its rt_fatal and ends the process before the report is written. +void vk_device_fault_report(VkResult r) { + static SRWLOCK lock = SRWLOCK_INIT; + static int done; + if (r != VK_ERROR_DEVICE_LOST) return; + AcquireSRWLockExclusive(&lock); + if (done) { ReleaseSRWLockExclusive(&lock); return; } + done = 1; + VkCtx *c = vk_ctx(); + PFN_vkGetDeviceFaultInfoEXT get = + c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") + : NULL; + if (!get) { + rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point" + : "not enabled"); + ReleaseSRWLockExclusive(&lock); + return; + } + VkDeviceFaultCountsEXT n = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT}; + VkResult q = get(c->device, &n, NULL); + VkDeviceFaultAddressInfoEXT *ai = calloc(n.addressInfoCount + 1, sizeof *ai); + VkDeviceFaultVendorInfoEXT *vi = calloc(n.vendorInfoCount + 1, sizeof *vi); + VkDeviceFaultInfoEXT info = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT, .pAddressInfos = ai, .pVendorInfos = vi}; + uint64_t binary = n.vendorBinarySize; + n.vendorBinarySize = 0; // the vendor binary is not fetched; its size alone is logged + if (q == VK_SUCCESS && ai && vi) q = get(c->device, &n, &info); + if (q != VK_SUCCESS && q != VK_INCOMPLETE) { + rt_log("vk: device lost; vkGetDeviceFaultInfoEXT failed (%d)", (int)q); + } else { + rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description, + n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary); + // The faulting access lies in the precision-aligned range around the reported address. + for (uint32_t i = 0; i < n.addressInfoCount; i++) + rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType), + (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision); + for (uint32_t i = 0; i < n.vendorInfoCount; i++) + rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description, + (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData); + } + free(ai); + free(vi); + ReleaseSRWLockExclusive(&lock); +} + int vk_selftest(void) { if (!vk_init()) return 1; VkCtx *c = &g_vk; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 34548153..a6685bfd 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -211,7 +211,10 @@ static void oq_cb(void *arg) { // completion thread, and with it every later callback, for good. It counts as visible (one sample). VkResult r = vkGetQueryPoolResults(dev, oq_pool, s->q[i], j - i, sizeof v, v, 2 * sizeof v[0], VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); - if (r != VK_SUCCESS && r != VK_NOT_READY) rt_fatal("vk: occlusion query results failed (%d)", (int)r); + if (r != VK_SUCCESS && r != VK_NOT_READY) { + vk_device_fault_report(r); + rt_fatal("vk: occlusion query results failed (%d)", (int)r); + } vkResetQueryPool(dev, oq_pool, s->q[i], j - i); for (uint32_t k = i; k < j; k++) { if (v[2 * (k - i) + 1]) sum += v[2 * (k - i)]; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index cde3b60c..7d8192a6 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1722,7 +1722,10 @@ uint64_t vk_upload_flush(int wait) { vk_queue_lock(); VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: upload submit failed (%d)", (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: upload submit failed (%d)", (int)r); + } if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight); inflight[ninflight++] = open_b; open_active = 0; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index f89ac1a7..2dd32c58 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -186,7 +186,10 @@ static DWORD WINAPI completion_main(void *arg) { VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, .pSemaphores = &c->timeline, .pValues = &want}; VkResult r = vkWaitSemaphores(c->device, &wi, 100000000ull); // 100 ms: stop is checked between waits - if (r != VK_SUCCESS && r != VK_TIMEOUT) rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); + if (r != VK_SUCCESS && r != VK_TIMEOUT) { + vk_device_fault_report(r); + rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); + } vkGetSemaphoreCounterValue(c->device, c->timeline, &reached); AcquireSRWLockExclusive(&st_lock); if (reached > issued) reached = issued; @@ -302,7 +305,10 @@ VkCommandBuffer vk_sync_split(void) { vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); + } vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, @@ -360,7 +366,10 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); + } s->fence = recorded ? fence : 0; vk_mem_note_submit(fence); AcquireSRWLockExclusive(&st_lock);