diff --git a/runtime/gfx_window.c b/runtime/gfx_window.c index f62de225..fc449051 100644 --- a/runtime/gfx_window.c +++ b/runtime/gfx_window.c @@ -490,6 +490,7 @@ static void flip_locked(void) { vk_queue_lock(); VkResult r = vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi); vk_queue_unlock(); + vk_device_fault_report(r); if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) InterlockedExchange(&g_dirty, 1); } @@ -763,6 +764,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w if (!create_swapchain()) goto out; r = vkAcquireNextImageKHR(c->device, g_sc, 1000000000ull, g_acquire[slot], VK_NULL_HANDLE, &idx); } + vk_device_fault_report(r); if (r != VK_SUCCESS && r != VK_SUBOPTIMAL_KHR) goto out; // lost, not ready: skip the frame if (r == VK_SUBOPTIMAL_KHR) InterlockedExchange(&g_dirty, 1); vkResetFences(c->device, 1, &g_fence[slot]); @@ -870,6 +872,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w g_flip_idx = idx; if (!defer) flip_locked(); } else { + vk_device_fault_report(r); rt_log("gfx: present submit failed (%d)", (int)r); } g_frame++; diff --git a/runtime/vk.h b/runtime/vk.h index 961f2f35..f6302678 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -69,6 +69,10 @@ VkCtx *vk_ctx(void); // NULL until vk_init() succeeded int vk_init(void); // idempotent; 1 on success, 0 on failure (reason logged with rt_log) void vk_shutdown(void); int vk_selftest(void); // 0 on success +// Call with the result of a failed queue submit, present or wait before stopping on it: VK_ERROR_DEVICE_LOST logs +// what VK_EXT_device_fault reports (description, faulting addresses with their precision, vendor codes) once per +// process. Any other result returns at once. +void vk_device_fault_report(VkResult r); // ---- window and present (gfx_window.c). Host threads only. // vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 01573a38..26881a68 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -267,6 +267,7 @@ static int create_device(void) { else if (!h->geometryShader) miss = "geometryShader"; else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } + if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { @@ -364,6 +365,64 @@ void vk_shutdown(void) { VkCtx *vk_ctx(void) { return g_ready ? &g_vk : NULL; } +static const char *fault_address_type(VkDeviceFaultAddressTypeEXT t) { + switch (t) { + case VK_DEVICE_FAULT_ADDRESS_TYPE_NONE_EXT: return "none"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_READ_INVALID_EXT: return "read invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_WRITE_INVALID_EXT: return "write invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_EXECUTE_INVALID_EXT: return "execute invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_UNKNOWN_EXT: return "instruction pointer (unknown)"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_INVALID_EXT: return "instruction pointer (invalid)"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_FAULT_EXT: return "instruction pointer (fault)"; + default: return "unknown type"; + } +} + +// The query runs once per process. Threads that hit the loss at the same time wait on the lock until it is logged, so +// none of them reaches its rt_fatal and ends the process before the report is written. +void vk_device_fault_report(VkResult r) { + static SRWLOCK lock = SRWLOCK_INIT; + static int done; + if (r != VK_ERROR_DEVICE_LOST) return; + AcquireSRWLockExclusive(&lock); + if (done) { ReleaseSRWLockExclusive(&lock); return; } + done = 1; + VkCtx *c = vk_ctx(); + PFN_vkGetDeviceFaultInfoEXT get = + c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") + : NULL; + if (!get) { + rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point" + : "not enabled"); + ReleaseSRWLockExclusive(&lock); + return; + } + VkDeviceFaultCountsEXT n = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT}; + VkResult q = get(c->device, &n, NULL); + VkDeviceFaultAddressInfoEXT *ai = calloc(n.addressInfoCount + 1, sizeof *ai); + VkDeviceFaultVendorInfoEXT *vi = calloc(n.vendorInfoCount + 1, sizeof *vi); + VkDeviceFaultInfoEXT info = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT, .pAddressInfos = ai, .pVendorInfos = vi}; + uint64_t binary = n.vendorBinarySize; + n.vendorBinarySize = 0; // the vendor binary is not fetched; its size alone is logged + if (q == VK_SUCCESS && ai && vi) q = get(c->device, &n, &info); + if (q != VK_SUCCESS && q != VK_INCOMPLETE) { + rt_log("vk: device lost; vkGetDeviceFaultInfoEXT failed (%d)", (int)q); + } else { + rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description, + n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary); + // The faulting access lies in the precision-aligned range around the reported address. + for (uint32_t i = 0; i < n.addressInfoCount; i++) + rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType), + (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision); + for (uint32_t i = 0; i < n.vendorInfoCount; i++) + rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description, + (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData); + } + free(ai); + free(vi); + ReleaseSRWLockExclusive(&lock); +} + int vk_selftest(void) { if (!vk_init()) return 1; VkCtx *c = &g_vk; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 34548153..a6685bfd 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -211,7 +211,10 @@ static void oq_cb(void *arg) { // completion thread, and with it every later callback, for good. It counts as visible (one sample). VkResult r = vkGetQueryPoolResults(dev, oq_pool, s->q[i], j - i, sizeof v, v, 2 * sizeof v[0], VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); - if (r != VK_SUCCESS && r != VK_NOT_READY) rt_fatal("vk: occlusion query results failed (%d)", (int)r); + if (r != VK_SUCCESS && r != VK_NOT_READY) { + vk_device_fault_report(r); + rt_fatal("vk: occlusion query results failed (%d)", (int)r); + } vkResetQueryPool(dev, oq_pool, s->q[i], j - i); for (uint32_t k = i; k < j; k++) { if (v[2 * (k - i) + 1]) sum += v[2 * (k - i)]; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index cde3b60c..7d8192a6 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1722,7 +1722,10 @@ uint64_t vk_upload_flush(int wait) { vk_queue_lock(); VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: upload submit failed (%d)", (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: upload submit failed (%d)", (int)r); + } if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight); inflight[ninflight++] = open_b; open_active = 0; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index f89ac1a7..2dd32c58 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -186,7 +186,10 @@ static DWORD WINAPI completion_main(void *arg) { VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, .pSemaphores = &c->timeline, .pValues = &want}; VkResult r = vkWaitSemaphores(c->device, &wi, 100000000ull); // 100 ms: stop is checked between waits - if (r != VK_SUCCESS && r != VK_TIMEOUT) rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); + if (r != VK_SUCCESS && r != VK_TIMEOUT) { + vk_device_fault_report(r); + rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); + } vkGetSemaphoreCounterValue(c->device, c->timeline, &reached); AcquireSRWLockExclusive(&st_lock); if (reached > issued) reached = issued; @@ -302,7 +305,10 @@ VkCommandBuffer vk_sync_split(void) { vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); + } vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, @@ -360,7 +366,10 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); - if (r != VK_SUCCESS) rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); + } s->fence = recorded ? fence : 0; vk_mem_note_submit(fence); AcquireSRWLockExclusive(&st_lock);