diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 238a2bc5..62eeaef2 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -315,6 +315,7 @@ static void replay_log(uint64_t frame, int total) { // vk_backend_shutdown. void gfx_exit_log(void) { static int32_t done; + rec_check_log(); if (gfx_replay_async() && r_started && !plat_xchg32(&done, 1)) replay_log(game_frames, 1); } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index d8f57e23..d4242a80 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1807,6 +1807,8 @@ static void collect_locked(VkCtx *c) { for (int i = 0; i < ndead; i++) { Dead *d = &dead[i]; if (d->seq > done_seq || d->tl > tl) { dead[k++] = *d; continue; } + rec_check_handle(d->kind == DEAD_VIEW ? REC_VIEW : d->kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, d->handle, 1, + d->tl, tl); switch (d->kind) { case DEAD_IMAGE: vkDestroyImage(c->device, (VkImage)(uintptr_t)d->handle, NULL); break; case DEAD_IMAGE_VMA: vmaDestroyImage(c->vma, (VkImage)(uintptr_t)d->handle, d->alloc); break; @@ -1951,6 +1953,8 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) { uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0; if (rf > tl) tl = rf; dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl}; + rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl, + vk_sync_fence_done()); plat_rmutex_unlock(&cmd_lock); } diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index d48fa0b7..28bf7ab8 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -26,6 +26,10 @@ static VkCommandBuffer rec_cb; // the part being recorded (set while the static int rec_live; // producer: rec_begin started the recorder for rec_cb static PlatThread *rec_thread; static int rec_prio; // the submitting thread's priority, which the recorder takes +static int rec_chk; // TOTK_REC_CHECK (rec_check_handle) +static uint64_t rec_delay_ns; // TOTK_REC_DELAY_US (rec_main) +static volatile uint64_t part_h0; // put_h at rec_begin: where the packets of the part being recorded start +static volatile uint64_t part_fence; // the value of the part being recorded (rec_begin) // Producer statistics, logged by rec_stats_log. static struct { uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on; } RS_; @@ -65,6 +69,9 @@ static void rec_main(void *arg) { continue; } } + // TOTK_REC_DELAY_US (debug): the recorder falls this far behind each batch it finds, as on a slow machine, + // so objects the replay lets go of meet packets it has not made yet (TOTK_REC_CHECK). + if (rec_delay_ns) plat_sleep_ns(rec_delay_ns); for (int k = 0; t != h; k++) { uint64_t w = ring[t % RING_WORDS]; uint32_t op = (uint32_t)(w & 0xffff), n = (uint32_t)(w >> 32); @@ -104,6 +111,9 @@ static void *put(uint32_t op, size_t bytes) { } wait_space(n); ring[pos] = (uint64_t)op | n << 32; + // Packers reserve room for optional arrays they may leave unwritten (BeginRendering's depth and stencil); with + // TOTK_REC_CHECK those words would still hold handles from the ring's previous lap. + if (rec_chk) memset(&ring[pos + 1], 0, (n - 1) * 8); put_h += n; RS_.packets++, RS_.words += n; return &ring[pos + 1]; @@ -125,7 +135,7 @@ void rec_drain(void) { RS_.drains++, RS_.drain_ns += now_ns() - t0; } -void rec_begin(VkCommandBuffer cb) { +void rec_begin(VkCommandBuffer cb, uint64_t fence) { static int mode = -1; if (mode < 0) { // Default off on AMD GPUs. With the recorder those drivers lost the device or hung the GPU at startup on @@ -141,6 +151,9 @@ void rec_begin(VkCommandBuffer cb) { if (mode) { plat_event_init(&wake_ev, 0, 0); rec_prio = plat_thread_priority_get(); + rec_chk = rt_knob("REC_CHECK", "0"); // debug switches, read before the recorder starts + const char *d = cfg_get("TOTK_REC_DELAY_US"); + rec_delay_ns = d && *d && atoi(d) > 0 ? (uint64_t)atoi(d) * 1000 : 0; rec_thread = plat_thread_new(rec_main, NULL, "vk recorder"); if (!rec_thread) { rt_log("vk_rec: no recorder thread: the replay makes its Vulkan calls itself"); @@ -148,6 +161,9 @@ void rec_begin(VkCommandBuffer cb) { } else { rt_log("vk_rec: the replay's Vulkan commands are made by a recorder thread%s", mode == 2 ? " in the B blocks (TOTK_AB)" : ""); + if (rec_chk) rt_log("vk_rec: TOTK_REC_CHECK: deferred and destroyed objects are searched in the ring"); + if (rec_delay_ns) rt_log("vk_rec: TOTK_REC_DELAY_US: the recorder waits %llu us before each batch", + (unsigned long long)(rec_delay_ns / 1000)); } } } @@ -155,6 +171,8 @@ void rec_begin(VkCommandBuffer cb) { if (!RT_KNOB_ON(mode)) return; RS_.parts_on++; rec_cb = cb; // the recorder is idle: rec_end drained it + __atomic_store_n(&part_h0, put_h, __ATOMIC_RELEASE); + __atomic_store_n(&part_fence, fence, __ATOMIC_RELEASE); rec_live = 1; } @@ -203,6 +221,64 @@ void rec_stats_log(void) { memset(&RS_, 0, sizeof RS_); } +// TOTK_REC_CHECK=1 (debug, recorder on): the packets of the part being recorded are searched word by word for an object +// vk_memory.c lets go of. put() zeroes each packet's room, so only words a packer wrote can match. A hit is a hazard +// - at a deferral (destroy 0) when the stamp tl is below the part's value: vk_memory.c collect_locked may destroy the +// object once the GPU reaches tl, before the part ran and, with the recorder behind, before it made the call; +// - at a destruction (destroy 1) always: a packet not made yet hands the driver a dead handle, a made one leaves it +// in a command buffer not yet submitted. +// "Made" means before the tail the recorder published, which trails it by up to 64 packets. Hazards are logged (the +// first 64) and counted by kind, packet and case for rec_check_log; named deferrals that wait long enough are only +// counted. Reads race the producer only in words below head, which stay put until the ring +// wraps (16 MiB later). +static const char *const kind_name[REC_KINDS] = {"image", "view", "buffer"}; +static volatile int32_t chk_seen[2], chk_named, chk_logged, chk_hz[2][2][REC_KINDS][64]; // [destroy][made][kind][op] +void rec_check_handle(int kind, uint64_t handle, int destroy, uint64_t tl, uint64_t gpu) { + if (!rec_chk || !handle || kind < 0 || kind >= REC_KINDS || !__atomic_load_n(&rec_live, __ATOMIC_RELAXED)) return; + uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE), p = __atomic_load_n(&part_h0, __ATOMIC_ACQUIRE); + uint64_t fence = __atomic_load_n(&part_fence, __ATOMIC_ACQUIRE); + if (p > h || h - p > RING_WORDS / 2) return; // a part this large is not searched + plat_inc32(&chk_seen[destroy != 0]); + while (p < h) { + uint64_t w = ring[p % RING_WORDS]; + uint32_t op = (uint32_t)(w & 0xffff), n = (uint32_t)(w >> 32); + if (!n) return; // overwritten meanwhile + if (op != R_PAD) + for (uint32_t i = 1; i < n; i++) { + if (ring[(p + i) % RING_WORDS] != handle) continue; + int made = p < __atomic_load_n(&tail, __ATOMIC_ACQUIRE); + if (!destroy && tl >= fence) { plat_inc32(&chk_named); return; } + plat_inc32(&chk_hz[destroy][made][kind][op & 63]); + if (plat_inc32(&chk_logged) <= 64) + rt_log("vk_rec: check: HAZARD: %s %#llx %s while packet op %u (byte %u of its arguments) of the " + "part being recorded names it, %s; deferral stamp %llu, part value %llu, GPU at %llu", + kind_name[kind], (unsigned long long)handle, destroy ? "destroyed" : "deferred", op, + 8 * (i - 1), made ? "made" : "not known made", (unsigned long long)tl, + (unsigned long long)fence, (unsigned long long)gpu); + return; + } + p += n; + } +} + +void rec_check_log(void) { + static volatile int32_t said; + if (!rec_chk || plat_xchg32(&said, 1)) return; + int any = 0; + for (int d = 0; d < 2; d++) + for (int m = 0; m < 2; m++) + for (int k = 0; k < REC_KINDS; k++) + for (int op = 0; op < 64; op++) + if (chk_hz[d][m][k][op]) { + any = 1; + rt_log("vk_rec: check summary: %d hazards: %s %s, packet op %d %s", (int)chk_hz[d][m][k][op], + kind_name[k], d ? "destroyed" : "deferred", op, m ? "made" : "not known made"); + } + rt_log("vk_rec: check summary: %s; searched for %d objects deferred and %d destroyed while a part was recorded; %d " + "deferred ones the part named waited for its value", any ? "hazards above" : "no hazards", (int)chk_seen[0], + (int)chk_seen[1], (int)chk_named); +} + #define OWNED(cb) (rec_live && (cb) == rec_cb) // A call the ring takes no packet for: everything queued first, then the call here. #define FALLBACK() (rec_drain(), RS_.fallbacks++) diff --git a/runtime/vk_rec.h b/runtime/vk_rec.h index 53ae1464..c4a1faf8 100644 --- a/runtime/vk_rec.h +++ b/runtime/vk_rec.h @@ -15,13 +15,19 @@ // MIT, Copyright (c) 2026 Peiikai): the driver's work behind the commands depends on nothing the replay decides next. #pragma once -void rec_begin(VkCommandBuffer cb); // the submitting thread, sub_lock held +void rec_begin(VkCommandBuffer cb, uint64_t fence); // the submitting thread, sub_lock held; fence: the part's value void rec_end(void); // waits for the recorder; no-op when rec_begin did not start it // The end of part cb: the breadcrumb crumb (0: none, vk_crumbs.c), then vkEndCommandBuffer, both made by the recorder // when it records cb, and the wait for it (1); otherwise rec_end and the breadcrumb, and the caller ends cb (0). int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n); void rec_drain(void); // waits until the recorder made every call queued so far void rec_stats_log(void); // TOTK_TRACE=perf: packets, bytes, drains and waits since the last call +// TOTK_REC_CHECK=1 (debug, recorder on): vk_memory.c reports each object it defers (destroy 0, stamped tl) or destroys +// (destroy 1) with the value the GPU reached (gpu); a packet of the part being recorded that still names it is logged +// as a hazard when the object can go before the part ran. rec_check_log: the counts by kind and packet, at exit. +enum { REC_IMAGE, REC_VIEW, REC_BUFFER, REC_KINDS }; +void rec_check_handle(int kind, uint64_t handle, int destroy, uint64_t tl, uint64_t gpu); +void rec_check_log(void); // Extension commands called through device function pointers. void rec_push_tpl(PFN_vkCmdPushDescriptorSetWithTemplateKHR fn, VkCommandBuffer cb, VkDescriptorUpdateTemplate tpl, diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 8a4acdcf..ae9ee33b 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -389,7 +389,7 @@ VkCommandBuffer vk_sync_split(void) { vkBeginCommandBuffer(s->cb, &bi); vk_crumb(s->cb, VK_CRUMB_SUBMIT, cur_fence); ts_begin(s->cb); - rec_begin(s->cb); + rec_begin(s->cb, cur_fence); return s->cb; } @@ -418,7 +418,7 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vkBeginCommandBuffer(s->cb, &bi); vk_crumb(s->cb, VK_CRUMB_SUBMIT, fence); ts_begin(s->cb); - rec_begin(s->cb); + rec_begin(s->cb, fence); int recorded = vk_exec_record(s->cb, handles, n, fence); s = &slots[cur_slot]; // the last part (vk_sync_split) if (recorded) ts_end(s->cb, fence);