From d6715029e5865f4c33fda151d74cccce84de3f5c Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 15 Nov 2026 18:55:39 +0300 Subject: [PATCH] A device loss logs the last descriptor pool events, and a pool with two owners is logged at once The replay keeps a ring of the last 1024 descriptor pool events (a CbRes made, a pool handed to a CbRes from the spare list or new, a pool reset or destroyed), a few per submission and nothing per draw. vk_device_fault_report logs the newest 256 after a device loss. Each pool's owner is checked under spare_lock as the pool is handed out and as it is reset: a pool handed out while another CbRes holds it, or reset by a CbRes that does not hold it, is logged at once with the 64 events before it. On a GTX 1050 Ti with TOTK_BIND_INC on, the validation layer reported a descriptor pool reset while a command buffer used it, then binds of its sets, before a device lost that slower diagnostic runs did not bring back; the next one names the pool's owners. --probe-vkexec gains a set batch part (60 submissions of batched sets with splits and empty submissions beside them), a pool owner part (one pool handed to two CbRes, then reset by the one that no longer holds it: two conflicts) and calls the device loss report on the working device to check that it comes out and returns. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 3 + runtime/vk_device.c | 1 + runtime/vk_exec.c | 7 ++ runtime/vk_exec_draw.c | 143 ++++++++++++++++++++++++++++++++++++++++ runtime/vk_exec_probe.c | 99 ++++++++++++++++++++++++++++ 5 files changed, 253 insertions(+) diff --git a/runtime/vk.h b/runtime/vk.h index b9f1e585..548069c3 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -421,6 +421,9 @@ int vk_mem_span_live(const VkMemSpan *sp); // from then on are freed once the timeline reaches it. void vk_mem_note_fence(uint64_t timeline_value); void vk_mem_fault_log(void); // vk_device_fault_report: the deferred-free state, lock-free +// vk_device_fault_report: the last descriptor pool events (vk_exec_draw.c journal: CbRes made, pools handed out and +// reset), lock-free. +void vk_exec_pool_journal_log(void); void vk_mem_collect(void); // frees what has completed // After a failed allocation, on a thread that may hold the texture lock and the upload lock (cmd_lock) but no VMA // lock: frees the headroom blocks and the completed upload batches and deferred objects, and requests the release diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 6d047228..0db73ba1 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -546,6 +546,7 @@ void vk_device_fault_report(VkResult r) { vk_crumbs_report(); vk_pass_marks_report(); vk_mem_fault_log(); + vk_exec_pool_journal_log(); VkCtx *c = vk_ctx(); PFN_vkGetDeviceFaultInfoEXT get = c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index c2592b48..0df3ce01 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -932,6 +932,13 @@ static void maybe_split(VkExec *ex) { n_parts++; } +// --probe-vkexec: a split every `draws` draws between passes, parts of at least `us` (0: no minimum); the parts made +// so far. +uint64_t vk_exec_split_set(int draws, int us) { + split_n = draws, split_ns = us > 0 ? (uint64_t)us * 1000 : 0; + return n_parts; +} + // The fixed-function state ops: writes into ex alone, applied by the replay and the state scan. 0: not one of them. static int state_op(VkExec *ex, uint32_t op, const void *a) { switch (op) { diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 7af5df4e..ddbb004c 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -81,7 +81,10 @@ typedef struct { int npools, cpools; Range *written; int nwritten, cwritten; + uint64_t fence; // the value cb_done waits for + uint64_t serial; // unique per CbRes, for the descriptor pool journal } CbRes; +static uint64_t cbres_serial; // atomic // Descriptor pools and full-size transient chunks go back to these lists when their command buffer completes, instead // of being destroyed and created again for the next one: each creation is a driver memory allocation (on Windows an @@ -93,6 +96,136 @@ static VkDescriptorPool spare_pool[SPARE_POOLS]; static Chunk spare_chunk[SPARE_CHUNKS]; static int nspare_pool, nspare_chunk; +// ---- Descriptor pool journal, always on: what a device loss needs to tell whether a descriptor pool was reset under a +// command buffer that still used its sets (a set bound after its pool's reset loses the device on NVIDIA). A ring keeps +// the last PJ_N events: a CbRes made, a pool handed to a CbRes (from the spare list or new), a pool reset or destroyed. +// That is a few events a submission and nothing per draw. The ring is logged +// after a device loss (vk_device_fault_report). The owner of each pool is checked as it is handed out and as it is +// reset, under spare_lock: a pool handed out while another CbRes holds it, or reset by a CbRes that does not hold it, +// is logged at once (at most 4 times) with the events before it. +#define PJ_N 1024 // power of two +#define PJ_SHOW 64 // events logged with an owner conflict +#define PJ_SHOW_LOST 256 // events logged after a device loss +#define PO_N 4096 // pool owner table, open addressing, power of two +enum { PJ_CBRES = 1, PJ_SPARE, PJ_NEW, PJ_RESET, PJ_DESTROY, PJ_KINDS }; +typedef struct { uint64_t pool, serial, fence, issued; uint32_t kind, tid; } PjEvent; +static PjEvent pj[PJ_N]; +static uint64_t pj_n; // atomic: events so far +typedef struct { uint64_t pool, serial, fence; } PoolOwner; // pool 0: empty slot, 1: slot of a destroyed pool +static PoolOwner po[PO_N]; // spare_lock +static int po_used; // spare_lock: slots ever taken +static volatile int32_t pool_bad; // owner conflicts found +static uint64_t aset_batches; // alloc_set_batched: batches of more than one set (replay context threads, atomic) + +void vk_exec_draw_counts(uint64_t *batches, int *stale, uint64_t *journal) { // --probe-vkexec + *batches = __atomic_load_n(&aset_batches, __ATOMIC_RELAXED); + *stale = __atomic_load_n(&pool_bad, __ATOMIC_RELAXED); + *journal = __atomic_load_n(&pj_n, __ATOMIC_RELAXED); +} + +static void pj_add(int kind, VkDescriptorPool pool, uint64_t serial, uint64_t fence) { + uint64_t i = __atomic_fetch_add(&pj_n, 1, __ATOMIC_RELAXED) & (PJ_N - 1); + pj[i] = (PjEvent){(uint64_t)(uintptr_t)pool, serial, fence, vk_sync_issued(), (uint32_t)kind, plat_thread_id()}; +} + +// The last `last` events, oldest first. Lock-free: after a device loss the threads that write the ring may be stuck +// holding locks; an event written meanwhile can show torn. +static void pj_log(int last) { + static const char *const kinds[PJ_KINDS] = {"?", "CbRes made", "pool from the spare list", "new pool", "pool reset", + "pool reset and destroyed"}; + uint64_t n = __atomic_load_n(&pj_n, __ATOMIC_ACQUIRE), k = n < (uint64_t)last ? n : (uint64_t)last; + rt_log("vk_exec: descriptor pool journal, the last %llu of %llu events (thread: event, pool, CbRes, its value; the " + "newest value submitted then); GPU done %llu, submitted %llu", (unsigned long long)k, (unsigned long long)n, + (unsigned long long)vk_sync_fence_done(), (unsigned long long)vk_sync_issued()); + for (uint64_t i = n - k; i < n; i++) { + const PjEvent *e = &pj[i & (PJ_N - 1)]; + rt_log("vk_exec: %u: %s, pool %#llx, CbRes %llu, value %llu; submitted %llu", e->tid, + kinds[e->kind < PJ_KINDS ? e->kind : 0], (unsigned long long)e->pool, (unsigned long long)e->serial, + (unsigned long long)e->fence, (unsigned long long)e->issued); + } +} + +void vk_exec_pool_journal_log(void) { pj_log(PJ_SHOW_LOST); } + +static void pool_take(VkDescriptorPool p, const CbRes *r, int spare); +static int pool_release(VkDescriptorPool p, const CbRes *r, int destroyed, uint64_t *other, uint64_t *of); +static void pool_conflict(const char *what, uint64_t pool, const CbRes *r, uint64_t other, uint64_t other_fence); + +// --probe-vkexec: two CbRes of the probe's own, never submitted, take one pool handle that names no Vulkan object, then +// the one that no longer holds it resets it. Each step must log an owner conflict with the journal: returns the +// conflicts found (2). The handle's slot is freed and the conflict count restored, so later checks start clean. +int vk_exec_pool_journal_selftest(void) { + CbRes a = {.serial = UINT64_MAX - 1, .fence = 1}, b = {.serial = UINT64_MAX - 2, .fence = 2}; + VkDescriptorPool p = (VkDescriptorPool)(uintptr_t)0x7e57d00d000ull; + int32_t bad0 = __atomic_load_n(&pool_bad, __ATOMIC_RELAXED); + pool_take(p, &a, 0); + pool_take(p, &b, 1); // a holds it + uint64_t other = 0, of = 0; + plat_mutex_lock(&spare_lock); + int bad = pool_release(p, &a, 1, &other, &of); // b holds it + plat_mutex_unlock(&spare_lock); + pj_add(PJ_DESTROY, p, a.serial, a.fence); + if (bad) pool_conflict("reset by", (uint64_t)(uintptr_t)p, &a, other, of); + int found = __atomic_load_n(&pool_bad, __ATOMIC_RELAXED) - bad0; + __atomic_store_n(&pool_bad, bad0, __ATOMIC_RELAXED); + return found; +} + +static uint32_t po_hash(uint64_t h) { return (uint32_t)((h * 0x9E3779B97F4A7C15ull) >> 40) & (PO_N - 1); } + +// spare_lock held: the pool's slot, made when add is set; NULL when it has none (the table filled up: no check). +static PoolOwner *po_slot(uint64_t h, int add) { + PoolOwner *free_slot = NULL; + for (uint32_t i = po_hash(h), k = 0; k < PO_N; k++, i = (i + 1) & (PO_N - 1)) { + PoolOwner *e = &po[i]; + if (e->pool == h) return e; + if (e->pool == 1 && !free_slot) free_slot = e; + if (!e->pool) { + if (!add) return NULL; + if (!free_slot) { + if (po_used >= PO_N / 4 * 3) return NULL; + po_used++, free_slot = e; + } + break; + } + } + if (!add || !free_slot) return NULL; + *free_slot = (PoolOwner){h, 0, 0}; + return free_slot; +} + +static void pool_conflict(const char *what, uint64_t pool, const CbRes *r, uint64_t other, uint64_t other_fence) { + if (plat_inc32(&pool_bad) > 4) return; + rt_log("vk_exec: descriptor pool %#llx %s CbRes %llu (value %llu) while %s%llu (value %llu) holds it; submitted " + "%llu, done %llu, thread %u", (unsigned long long)pool, what, (unsigned long long)r->serial, + (unsigned long long)r->fence, other ? "CbRes " : "no CbRes, ", (unsigned long long)other, + (unsigned long long)other_fence, (unsigned long long)vk_sync_issued(), + (unsigned long long)vk_sync_fence_done(), plat_thread_id()); + pj_log(PJ_SHOW); +} + +// alloc_set: pool p goes to CbRes r, from the spare list (spare) or new. +static void pool_take(VkDescriptorPool p, const CbRes *r, int spare) { + uint64_t h = (uint64_t)(uintptr_t)p, other = 0, of = 0; + plat_mutex_lock(&spare_lock); + PoolOwner *e = po_slot(h, 1); + if (e) other = e->serial, of = e->fence, e->serial = r->serial, e->fence = r->fence; + plat_mutex_unlock(&spare_lock); + pj_add(spare ? PJ_SPARE : PJ_NEW, p, r->serial, r->fence); + if (other) pool_conflict("handed to", h, r, other, of); +} + +// cb_done, spare_lock held, after the pool's reset: 1 when CbRes r did not hold it. *other: who did. +static int pool_release(VkDescriptorPool p, const CbRes *r, int destroyed, uint64_t *other, uint64_t *of) { + PoolOwner *e = po_slot((uint64_t)(uintptr_t)p, 0); + if (!e) return 0; // not in the table: unknown, not checked + int bad = e->serial != r->serial; + *other = e->serial, *of = e->fence; + if (destroyed) e->pool = 1, e->serial = e->fence = 0; + else e->serial = e->fence = 0; + return bad; +} + // State of the command buffer being recorded that the Vulkan command buffer itself holds. typedef struct { VkPipeline pipe[2]; // bound per bind point: 0 graphics, 1 compute @@ -392,10 +525,14 @@ static void cb_done(void *arg) { // completion thread for (int i = 0; i < r->nwritten; i++) vk_gpu_written(r->written[i].addr, r->written[i].size); for (int i = 0; i < r->npools; i++) { vkResetDescriptorPool(c->device, r->pools[i], 0); + uint64_t other = 0, of = 0; plat_mutex_lock(&spare_lock); int kept = nspare_pool < SPARE_POOLS; if (kept) spare_pool[nspare_pool++] = r->pools[i]; + int bad = pool_release(r->pools[i], r, !kept, &other, &of); plat_mutex_unlock(&spare_lock); + pj_add(kept ? PJ_RESET : PJ_DESTROY, r->pools[i], r->serial, r->fence); + if (bad) pool_conflict("reset by", (uint64_t)(uintptr_t)r->pools[i], r, other, of); if (!kept) vkDestroyDescriptorPool(c->device, r->pools[i], NULL); } for (int i = 0; i < r->nchunks; i++) { @@ -437,6 +574,9 @@ static CbRes *cb_res(VkExec *ex) { if (D->cur && D->cur_fence == ex->fence) return D->cur; D->cur = calloc(1, sizeof *D->cur); if (!D->cur) rt_fatal("vk: out of memory"); + D->cur->fence = ex->fence; + D->cur->serial = __atomic_add_fetch(&cbres_serial, 1, __ATOMIC_RELAXED); + pj_add(PJ_CBRES, VK_NULL_HANDLE, D->cur->serial, ex->fence); D->cur_fence = ex->fence; memset(&D->B, 0, sizeof D->B); vk_completion_add(ex->fence, cb_done, D->cur); @@ -1882,7 +2022,9 @@ static VkDescriptorSet alloc_set(VkExec *ex, VkDescriptorSetLayout layout) { plat_mutex_lock(&spare_lock); if (nspare_pool) pool = spare_pool[--nspare_pool]; plat_mutex_unlock(&spare_lock); + int spare = pool != VK_NULL_HANDLE; if (!pool && vkCreateDescriptorPool(c->device, &pi, NULL, &pool) != VK_SUCCESS) rt_fatal("vk: descriptor pool failed"); + pool_take(pool, r, spare); r->pools[r->npools++] = pool; } rt_fatal("vk: descriptor set allocation failed twice"); @@ -1911,6 +2053,7 @@ static VkDescriptorSet alloc_set_batched(VkExec *ex, VkDescriptorSetLayout layou if (vkAllocateDescriptorSets(vk_ctx()->device, &ai, got) == VK_SUCCESS) { for (int i = 1; i < want; i++) c->set[i - 1] = got[i]; c->n = want - 1, c->fence = ex->fence; + __atomic_add_fetch(&aset_batches, 1, __ATOMIC_RELAXED); return got[0]; } // the pool is nearly full: one set, from a new pool when needed } diff --git a/runtime/vk_exec_probe.c b/runtime/vk_exec_probe.c index e6574ef4..37db3fba 100644 --- a/runtime/vk_exec_probe.c +++ b/runtime/vk_exec_probe.c @@ -455,6 +455,86 @@ static int test_marks(const GfxBackend *b, pool_t *pool, tex_t *color, prog_t *g return 0; } +// ---- set batch part: SB_SUBMITS submissions in a row, none waited for, each of SB_PASSES passes of SB_DRAWS draws +// whose viewport, and so the support buffer of every set, changes from draw to draw: each draw writes a new +// allocated set (set 1 with independent layouts), which TOTK_BIND_INC takes from batches (vk_exec_draw.c +// alloc_set_batched). A split after every 3 draws between passes (vk_exec.c maybe_split), the recorder on (its +// default), and empty submissions from a second thread as the present thread makes them. A set bound after its pool +// was reset is a validation error ("Invalid VkDescriptorSet", with TOTK_VK_VALIDATION=1); a pool with two owners is +// a line of vk_exec_draw.c's descriptor pool journal. On a GTX 1050 Ti it was a lost device. Skipped where no draw allocates its sets in batches. +#define SB_SUBMITS 60 +#define SB_PASSES 4 +#define SB_DRAWS 6 + +uint64_t vk_exec_split_set(int draws, int us); // vk_exec.c +void vk_exec_draw_counts(uint64_t *batches, int *stale, uint64_t *journal); // vk_exec_draw.c +int vk_exec_pool_journal_selftest(void); // vk_exec_draw.c + +static volatile int sb_stop; +static volatile int32_t sb_empty; +static int sb_batches, sb_parts, sb_skipped, sb_validated; + +static void sb_present(void *arg) { + (void)arg; + while (!__atomic_load_n(&sb_stop, __ATOMIC_ACQUIRE)) { + vk_sync_submit(NULL, 0); + plat_inc32(&sb_empty); + plat_sleep_ns(200000); + } +} + +static int test_batches(const GfxBackend *b, prog_t *prog, const pool_t *pool, tex_t *color) { + static _Alignas(8) uint8_t stream[16384]; + uint64_t batches0, journal0; + int stale0; + vk_exec_draw_counts(&batches0, &stale0, &journal0); + long errors0 = vk_validation_errors_total(); + uint64_t parts0 = vk_exec_split_set(3, 0); + sb_stop = 0; + PlatThread *t = plat_thread_new(sb_present, NULL, "probe present"); + CHECK(t, "no thread for the empty submissions"); + uint64_t f = 0; + for (int s = 0; s < SB_SUBMITS; s++) { + uint8_t *q = draw_prefix(stream, prog, pool, color); + for (int p = 0; p < SB_PASSES; p++) { + for (int d = 0; d < SB_DRAWS; d++) { + p_viewports vp; + memset(&vp, 0, sizeof vp); + vp.n = 1, vp.v[0][2] = (float)(TEX_W - (s * SB_PASSES * SB_DRAWS + p * SB_DRAWS + d) % 97), vp.v[0][3] = TEX_H; + vp.v[0][5] = 1.0f; + q = put(q, OP_VIEWPORTS, &vp, sizeof vp); + const p_draw dr = {NVN_DRAW_PRIMITIVE_TRIANGLES, 0, 3, 0, 0}; + q = put(q, OP_DRAW, &dr, sizeof dr); + } + const p_barrier bar = {0, 0}; + q = put(q, OP_BARRIER, &bar, sizeof bar); + } + q = put(q, OP_END, NULL, 0); + CHECK(q <= stream + sizeof stream, "set batch op stream overflow"); + uint64_t handle = (uint64_t)(uintptr_t)stream; + f = b->submit(&handle, 1); // the replay records it before submit returns: the stream is free again + } + __atomic_store_n(&sb_stop, 1, __ATOMIC_RELEASE); + plat_thread_join(t); + CHECK(b->fence_wait(f, 20000000000ull), "fence_wait of the last set batch submission timed out"); + b->finish(); + int split_us = cfg_int("TOTK_SPLIT_US"); + sb_parts = (int)(vk_exec_split_set(cfg_int("TOTK_SPLIT_DRAWS") > 0 ? cfg_int("TOTK_SPLIT_DRAWS") : 0, split_us) - parts0); + uint64_t batches, journal; + int stale; + vk_exec_draw_counts(&batches, &stale, &journal); + long errors = vk_validation_errors_total() - errors0; + CHECK(!errors && stale == stale0, "%ld validation errors and %d doubly owned descriptor pool lines (vk_exec: ...) over %d " + "submissions of batched sets", errors, stale - stale0, SB_SUBMITS); + // Each submission that drew made a CbRes and took at least one pool, and the pools of the earlier ones were reset. + CHECK(journal - journal0 >= 2 * SB_SUBMITS, "the descriptor pool journal took %llu events over %d submissions", + (unsigned long long)(journal - journal0), SB_SUBMITS); + sb_batches = (int)(batches - batches0), sb_validated = vk_ctx()->validation != 0; + sb_skipped = !sb_batches; + CHECK(sb_skipped || sb_parts >= SB_SUBMITS, "%d parts over %d submissions: the replay did not split", sb_parts, SB_SUBMITS); + return 0; +} + static int test_draw(const GfxBackend *b, uint8_t *guest, int *n_drawn, int *n_tries, int *n_cs) { char root[PLAT_PATH_MAX * 2], path[PLAT_PATH_MAX * 2 + 64]; CHECK(probe_census_dir(root, sizeof root), "no build/v%d/shaders/jobs.json above the exe (run tools/shader_census.py)", @@ -602,6 +682,9 @@ static int test_draw(const GfxBackend *b, uint8_t *guest, int *n_drawn, int *n_t ind_dispatches++; } if (test_marks(b, &pool, &color, hit, cs_prog)) return 1; + if (test_batches(b, hit, &pool, &color)) return 1; + int conflicts = vk_exec_pool_journal_selftest(); + CHECK(conflicts == 2, "the descriptor pool owner check found %d of 2 conflicts", conflicts); b->texture_release(&color); b->pool_fini(&pool); free(pairs), free(cs), free(e), free(jobs); @@ -1043,9 +1126,25 @@ int probe_vkexec(void) { else printf("probe-vkexec: pass marks part passed (pass, dispatch, pass in one submission: the GPU's last started and " "finished mark is the last pass, ring records exact); the report reads:\n%s\n", marks_text); + if (sb_skipped) + printf("probe-vkexec: set batch part skipped (no draw allocated its descriptor sets in batches: TOTK_BIND_INC off " + "or every set pushed)\n"); + else + printf("probe-vkexec: set batch part passed (%d submissions of %d draws, each writing a new set: %d batches, %d " + "parts, %d empty submissions from a second thread, no stale pool, %s)\n", + SB_SUBMITS, SB_PASSES * SB_DRAWS, sb_batches, sb_parts, sb_empty, + sb_validated ? "0 validation errors" : "validation layer off: TOTK_VK_VALIDATION=1 checks the binds"); printf("probe-vkexec: empty pool part passed (a zero-byte pool at the start of another: no Vulkan objects, lookups " "there find the other pool before and after its Finalize)\n"); printf("probe-vkexec: out-of-memory part passed (2 failed staging buffers waited for, 1 failed texture image retried, " "1 failed draw-data chunk waited for)\n"); + printf("probe-vkexec: pool owner part passed (a pool handed to a second CbRes and reset by one that no longer held it: " + "2 conflicts logged with the journal)\n"); + // The device loss report on a working device, as a device loss calls it: breadcrumbs, pass marks, deferred frees and + // the descriptor pool journal must come out without a crash. Last: the report runs once per process. + printf("probe-vkexec: device loss report follows (a test: the device is not lost)\n"); + fflush(stdout); + vk_device_fault_report(VK_ERROR_DEVICE_LOST); + printf("probe-vkexec: device loss report part passed\n"); return 0; }