// Submission, fences and the completion thread of the Vulkan back end. // One timeline semaphore (VkCtx.timeline) orders everything: each submission records its command handles into a // primary command buffer (vk_exec_record) and signals the next timeline value, which is the NVN fence value // the front end hands to syncs. The completion thread waits on the timeline and, in submit order, runs the CPU side // effects queued for each value (event writes, counter reports, sync state), then frees the memory and texture objects whose users // completed. A value counts as done only after its side effects ran, so a guest woken by fence_wait sees them. // // The submission shape (one timeline value per submission, signalled at ALL_COMMANDS, vkWaitSemaphores for waits, a // failed wait treated as a lost device) follows LAS Recomp runtime/nvn/vulkan/exec.c:39-110, 355-397 // (https://github.com/peiikai/LASRecomp @ da35b83a9611d046a9a681aae668d7085954d37f, MIT, Copyright (c) 2026 Peiikai). // Side effects written after GPU completion, as MacNativeGames does: games/totk/third_party/d2r_gfx/gfx/nvn/hle_nvn.c:7-9 // (https://github.com/ly05010419/MacNativeGames @ b75be8cfa704df3701e86130c8ced268e79d6b11, MIT, Copyright (c) 2026 // jhaizhou; origin d2r-mobile). #include #include #include "gfx.h" #include "host.h" #include "nvn_api.h" #include "vk.h" #include "platform/platform.h" #define WEAK __attribute__((weak)) typedef struct { uint64_t fence; void (*fn)(void *); void *arg; } Pending; typedef struct { VkCommandBuffer cb; uint64_t fence; } CmdSlot; // reusable once the timeline reached fence static PlatMutex start_lock = PLAT_MUTEX_INIT; static int started; static PlatThread *completion_thread; // Submissions: one at a time, so timeline values are signalled in the order they are handed out. static PlatMutex sub_lock = PLAT_MUTEX_INIT; static VkCommandPool cmd_pool; static CmdSlot *slots; static int nslots; static int cur_slot; // the slot vk_sync_submit records into now (an index: take_slot may move slots) static uint64_t cur_fence; // the value of the submission being recorded // Shared with the completion thread (st_lock). static PlatMutex st_lock = PLAT_MUTEX_INIT; static PlatCond cv_work = PLAT_COND_INIT; // a submission, a callback or stop static PlatCond cv_done = PLAT_COND_INIT; // done advanced static uint64_t issued, done; // newest submitted value; newest value whose side effects ran static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued) static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; // ---- GPU time (TOTK_TRACE=perf): each submission part (vk_sync_split) writes a timestamp when it starts and when it // ends, into a ring of query pairs. The completion thread reads a pair once the part's value is reached and adds the // part's interval to the busy time, counting overlap with the previous part once. Off (one log line) when the queue // family has no timestamp bits or the period is 0. Upload batches (vk_memory.c) are not timed. #define TS_PAIRS 1024 static VkQueryPool ts_pool; static int ts_state; // 0 not set up, 1 on, -1 off static double ts_period; // ns per tick static uint64_t ts_mask; // valid bits of a timestamp static uint32_t ts_next; // next pair (atomic: parts recorded on other threads take pairs too) static volatile uint8_t ts_busy[TS_PAIRS]; // written until the completion thread read the pair static int ts_cur = -1; // the pair of the part being recorded, -1 untimed static uint64_t ts_last_end; // completion thread: end of the latest part read (ticks) static volatile int64_t ts_busy_ns, ts_parts, ts_missed; // GPU idle (with calibrated timestamps): the gap before each timed part, split by whether the part was submitted after // the previous one had ended (the GPU waited for the CPU: starved) or before (it waited for untimed work or a wait). static uint64_t ts_sub_ns[TS_PAIRS]; // CPU time the part was handed to vkQueueSubmit2 static int ts_unsent[4], ts_n_unsent; // pairs closed and not yet submitted (sub_lock) static uint64_t ts_last_end_ns; // completion thread: ts_last_end on the CPU clock static volatile int64_t ts_idle_ns, ts_starved_ns, ts_gaps, ts_max_gap_ns, ts_queued_ns; static int64_t cal_tick, cal_ns, cal_at; // completion thread: a calibrated (GPU tick, CPU ns) pair and when it was taken static PFN_vkGetCalibratedTimestampsKHR cal_fn; // On the first submission that wants timing (perf_gpu_timing: TOTK_TRACE=perf or the full performance overlay). static void ts_setup(VkCtx *c) { ts_state = -1; uint32_t nq = 0; vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL); VkQueueFamilyProperties *qp = calloc(nq ? nq : 1, sizeof *qp); if (!qp) return; vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp); uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0; free(qp); float period = c->props.limits.timestampPeriod; if (!bits || period <= 0) { rt_log("vk: no GPU timestamps on this queue (valid bits %u, period %.3f ns): the perf line shows no GPU time", bits, period); return; } VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP, .queryCount = 2 * TS_PAIRS}; VkResult r = vkCreateQueryPool(c->device, &qi, NULL, &ts_pool); if (r != VK_SUCCESS) { rt_log("vk: timestamp query pool failed (%d): the perf line shows no GPU time", (int)r); return; } ts_period = period; ts_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1; ts_state = 1; } typedef struct { uint32_t pair; } TsRead; // Completion thread: refresh the GPU tick / CPU clock pair every 250 ms; 0 when there is no calibration. The host // domain is the clock plat_now_ns reads (QPC on Windows, CLOCK_MONOTONIC elsewhere), plat_counter_hz ticks a second. static int cal_refresh(VkCtx *c) { if (!c->has_calibrated) return 0; if (!cal_fn) { cal_fn = (PFN_vkGetCalibratedTimestampsKHR)vkGetDeviceProcAddr(c->device, c->has_calibrated == 2 ? "vkGetCalibratedTimestampsKHR" : "vkGetCalibratedTimestampsEXT"); if (!cal_fn) { c->has_calibrated = 0; return 0; } } int64_t now = (int64_t)plat_now_ns(); if (cal_at && now - cal_at < 250000000) return 1; VkCalibratedTimestampInfoKHR ci[2] = {{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_DEVICE_KHR}, #ifdef _WIN32 {.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_QUERY_PERFORMANCE_COUNTER_KHR}}; #else {.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR}}; #endif uint64_t v[2], dev; if (cal_fn(c->device, 2, ci, v, &dev) != VK_SUCCESS) { if (!cal_at) { // the domain pair is not supported: no split rt_log("vk: calibrated timestamps failed: the perf line shows no GPU idle split"); c->has_calibrated = 0; } return cal_at != 0; } cal_tick = (int64_t)(v[0] & ts_mask), cal_ns = (int64_t)((double)v[1] * 1e9 / (double)plat_counter_hz()), cal_at = now; return 1; } static int64_t tick_ns(uint64_t t) { return cal_ns + (int64_t)((double)((int64_t)t - cal_tick) * ts_period); } static void max64(volatile int64_t *m, int64_t v) { int64_t o = __atomic_load_n(m, __ATOMIC_RELAXED); while (v > o && !__atomic_compare_exchange_n(m, &o, v, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} } // Just before vkQueueSubmit2 (sub_lock held): the parts closed since the last submit leave the CPU now. static void ts_submitted(void) { uint64_t now = plat_now_ns(); for (int i = 0; i < ts_n_unsent; i++) ts_sub_ns[ts_unsent[i]] = now; ts_n_unsent = 0; } static void ts_read_cb(void *arg) { // completion thread, in submission order TsRead *t = arg; VkCtx *c = vk_ctx(); uint64_t v[2]; if (vkGetQueryPoolResults(c->device, ts_pool, 2 * t->pair, 2, sizeof v, v, sizeof v[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) { uint64_t s = v[0] & ts_mask, e = v[1] & ts_mask; if (e >= s) { if (cal_refresh(c)) { int64_t sn = tick_ns(s), en = tick_ns(e), sub = (int64_t)ts_sub_ns[t->pair]; if (ts_last_end_ns && sn > ts_last_end_ns) { int64_t gap = sn - ts_last_end_ns, late = sub - ts_last_end_ns; plat_add64(&ts_idle_ns, gap); if (late > 0) plat_add64(&ts_starved_ns, late < gap ? late : gap); if (gap > 500000) plat_inc64(&ts_gaps); max64(&ts_max_gap_ns, gap); } if (sub && sn > sub) plat_add64(&ts_queued_ns, sn - sub); if (en > ts_last_end_ns) ts_last_end_ns = en; } uint64_t from = s > ts_last_end ? s : ts_last_end; if (e > from) { int64_t ns = (int64_t)((double)(e - from) * ts_period); plat_add64(&ts_busy_ns, ns); perf_gpu_busy((uint64_t)ns); } if (e > ts_last_end) ts_last_end = e; plat_inc64(&ts_parts); } else plat_inc64(&ts_missed); // the counter wrapped } else plat_inc64(&ts_missed); __atomic_store_n(&ts_busy[t->pair], 0, __ATOMIC_RELEASE); free(t); } static int ts_take(void) { uint32_t p = __atomic_fetch_add(&ts_next, 1, __ATOMIC_RELAXED) % TS_PAIRS; if (__atomic_exchange_n(&ts_busy[p], 1, __ATOMIC_ACQUIRE)) { plat_inc64(&ts_missed); return -1; } return (int)p; } // At the start of a command buffer (sub_lock held): the pair it times, -1 untimed. static int ts_open(VkCommandBuffer cb) { if (ts_state != 1 || !perf_gpu_timing()) { if (ts_state || !perf_gpu_timing()) return -1; ts_setup(vk_ctx()); // the overlay turned full: timestamps from now on if (ts_state != 1) return -1; } int t = ts_take(); if (t < 0) return -1; uint32_t p = (uint32_t)t; vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2); vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p); return (int)p; } // At its end, before vkEndCommandBuffer; fence: the value whose completion covers it. static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence) { if (pair < 0) return; vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1); ts_sub_ns[pair] = 0; if (ts_n_unsent < 4) ts_unsent[ts_n_unsent++] = pair; TsRead *t = malloc(sizeof *t); if (!t) rt_fatal("vk: out of memory"); t->pair = (uint32_t)pair; vk_completion_add(fence, ts_read_cb, t); } int vk_sync_ts_open(VkCommandBuffer cb) { if (__atomic_load_n(&ts_state, __ATOMIC_ACQUIRE) != 1 || !perf_gpu_timing()) return -1; int p = ts_take(); if (p < 0) return -1; vkCmdResetQueryPool(cb, ts_pool, 2 * (uint32_t)p, 2); vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)p); return p; } void vk_sync_ts_close(VkCommandBuffer cb, int pair, uint64_t fence) { ts_close(cb, pair, fence); } // The pair of the part being recorded. static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); } static void ts_end(VkCommandBuffer cb, uint64_t fence) { ts_close(cb, ts_cur, fence); ts_cur = -1; } // An empty submission records no command buffer: its pair is never written. static void ts_drop(void) { if (ts_cur < 0) return; __atomic_store_n(&ts_busy[ts_cur], 0, __ATOMIC_RELEASE); ts_cur = -1; } static void gpu_stats(PerfGpu *g) { g->have_gpu = ts_state == 1; g->gpu_ns = (uint64_t)plat_xchg64(&ts_busy_ns, 0); g->gpu_parts = (uint64_t)plat_xchg64(&ts_parts, 0); g->gpu_missed = (uint64_t)plat_xchg64(&ts_missed, 0); g->have_vram = vk_mem_vram(&g->vram_used, &g->vram_budget, &g->vram_exact); g->have_idle = vk_ctx()->has_calibrated != 0 && cal_at != 0; g->idle_ns = (uint64_t)plat_xchg64(&ts_idle_ns, 0); g->starved_ns = (uint64_t)plat_xchg64(&ts_starved_ns, 0); g->queued_ns = (uint64_t)plat_xchg64(&ts_queued_ns, 0); g->idle_gaps = (uint64_t)plat_xchg64(&ts_gaps, 0); g->max_gap_ns = (uint64_t)plat_xchg64(&ts_max_gap_ns, 0); } // Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted. WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { (void)cb, (void)fence; for (int i = 0; i < n; i++) { const op_hdr *h = handles[i] ? nvn_op_first(handles[i]) : NULL; if (!h || h->op != OP_BEGIN || h->size < sizeof *h || nvn_op_next(h)->op != OP_END) rt_fatal("vk: op replay is not linked"); } return 0; } static void completion_main(void *arg) { VkCtx *c = arg; plat_mutex_lock(&st_lock); for (;;) { while (!stop && done == issued && !(qn && queue[qhead].fence <= done)) plat_cond_wait(&cv_work, &st_lock); if (stop) break; uint64_t reached = done; if (issued > done) { uint64_t want = done + 1; plat_mutex_unlock(&st_lock); VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, .pSemaphores = &c->timeline, .pValues = &want}; VkResult r = vkWaitSemaphores(c->device, &wi, 100000000ull); // 100 ms: stop is checked between waits if (r != VK_SUCCESS && r != VK_TIMEOUT) { vk_device_fault_report(r); rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); } vkGetSemaphoreCounterValue(c->device, c->timeline, &reached); plat_mutex_lock(&st_lock); if (reached > issued) reached = issued; } while (qn && queue[qhead].fence <= reached) { Pending p = queue[qhead++]; qn--; plat_mutex_unlock(&st_lock); p.fn(p.arg); plat_mutex_lock(&st_lock); } if (reached > done) { plat_mutex_unlock(&st_lock); vk_mem_collect(); plat_mutex_lock(&st_lock); __atomic_store_n(&done, reached, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_done); } } plat_mutex_unlock(&st_lock); return; } void vk_sync_start(void) { if (__atomic_load_n(&started, __ATOMIC_ACQUIRE)) return; plat_mutex_lock(&start_lock); if (!started) { if (!vk_init()) rt_fatal("vk: no usable Vulkan device (see the vk: lines above; TOTK_GPU=null runs without a GPU)"); VkCtx *c = vk_ctx(); VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = c->gfx_family}; if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: submission command pool"); if (!ts_state && perf_gpu_timing()) ts_setup(c); perf_gpu_stats = gpu_stats; uint64_t now = 0; vkGetSemaphoreCounterValue(c->device, c->timeline, &now); if (issued > now) { // a new device after vk_shutdown: values keep increasing for the front end VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline, .value = issued}; if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) rt_fatal("vk: timeline restart at %llu", (unsigned long long)issued); now = issued; } plat_mutex_lock(&st_lock); issued = done = now; if (reserved < now) reserved = now; stop = 0; plat_mutex_unlock(&st_lock); vk_mem_note_fence(now); completion_thread = plat_thread_new(completion_main, c, "vk completion"); if (!completion_thread) rt_fatal("vk: cannot start the completion thread"); __atomic_store_n(&started, 1, __ATOMIC_RELEASE); vk_astc_mode(); // decides TOTK_ASTC and builds the ASTC decoder now, not at the first ASTC upload } plat_mutex_unlock(&start_lock); } // The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move // slots[]. static int claim_slot(VkCtx *c, uint64_t fence) { uint64_t gpu = 0; vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu); for (int i = 0; i < nslots; i++) if (slots[i].fence <= gpu) { slots[i].fence = fence; return i; } rec_drain(); // the part pool is the recorder's while it has packets (vk_rec.h): it is idle for the allocation CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1)); if (!n) rt_fatal("vk: out of memory (command buffers)"); slots = n; CmdSlot *s = &slots[nslots]; VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation"); s->fence = fence; return nslots++; } static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { cur_slot = claim_slot(c, fence); return &slots[cur_slot]; } // The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is // submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the // part was recorded, sub_lock held; may move slots[]. // First brings the shadow storage up to date with the CPU's writes, which needs no command buffer. // direct: the calling thread records, ends and submits it itself while the part being replayed stays the recorder's // (vk_sync_part, which the caller made the recorder idle for). static VkCommandBuffer prologue(VkCtx *c, uint64_t fence, int direct) { // Shadow storage: the CPU's writes go in before the part is submitted (TOTK_SYNC_AHEAD: queued for the GPU or // made after the GPU finished the readers the game may think done, vk_memory.c). vk_mem_shadow_sync(fence); if (!vk_mem_mirror_pending() && !vk_mem_shadow_deferred()) return VK_NULL_HANDLE; int i = claim_slot(c, fence); // before reading slots: the claim may move it VkCommandBuffer cb = slots[i].cb; if (direct) { VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(cb, &bi); } else { rec_begin(cb, fence, 0, 0); // a command buffer of the part pool: the recorder's like the parts } int pair = ts_open(cb); vk_mem_shadow_prologue(cb, fence); // before the mirror copies, which read the shadow these bring up to date vk_mem_mirror_prologue(cb); ts_close(cb, pair, fence); if (direct || !rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); return cb; } // A chunk of a chunked replay (vk_exec.c chunks_submit), while the submission's own part is open on the recorder. The // recorder begins, records, ends and submits the parts it owns (vk_rec.c rec_end_cb), so the prologue stays off it: // the recorder is made idle first, as the part pool allows one thread at a time, and the prologue is recorded, ended // and submitted here without taking the recorder's part away. void vk_sync_part(VkCommandBuffer cb) { VkCtx *c = vk_ctx(); vk_upload_flush(0); rec_drain(); VkCommandBuffer pro = prologue(c, cur_fence, 1); VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}}; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1, .pCommandBufferInfos = pro ? cbi : cbi + 1}; vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); if (r != VK_SUCCESS) { vk_device_fault_report(r); rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); } } VkCommandBuffer vk_sync_split(void) { VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; ts_end(cb, cur_fence); if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder vk_upload_flush(0); // the uploads this part asked for precede it in queue order VkCommandBuffer pro = prologue(c, cur_fence, 0); VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}}; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1, .pCommandBufferInfos = pro ? cbi : cbi + 1}; ts_submitted(); VkResult r = rec_submit(&si); // by the recorder when it recorded the part if (r != VK_SUCCESS) { vk_device_fault_report(r); rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); } CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence); ts_begin(s->cb); return s->cb; } // Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed // out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value. uint64_t vk_sync_reserve(void) { vk_sync_start(); plat_mutex_lock(&st_lock); if (reserved < issued) reserved = issued; uint64_t v = ++reserved; plat_mutex_unlock(&st_lock); return v; } uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_sync_start(); VkCtx *c = vk_ctx(); plat_mutex_lock(&sub_lock); vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1; cur_fence = fence; // Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c // defer), since the recording may already name it. vk_mem_note_fence(fence); CmdSlot *s = take_slot(c, fence); uint64_t t0 = perf_begin(); rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence); // begun by the recorder when it records the part ts_begin(s->cb); int recorded = vk_exec_record(s->cb, handles, n, fence); s = &slots[cur_slot]; // the last part (vk_sync_split) if (recorded) ts_end(s->cb, fence); else ts_drop(); // The recorder is done with the part: the end-of-submission crumb, then the end, by the recorder when it records. if (!rec_end_cb(s->cb, VK_CRUMB_SUBMIT_END, fence)) vkEndCommandBuffer(s->cb); perf_replay(t0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. vk_upload_flush(0); VkCommandBuffer pro = prologue(c, fence, 0); s = &slots[cur_slot]; VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}}; VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u), .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1, .pSignalSemaphoreInfos = &sig}; ts_submitted(); VkResult r = rec_submit(&si); // by the recorder when it recorded the part if (r != VK_SUCCESS) { vk_device_fault_report(r); rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); } s->fence = recorded ? fence : 0; plat_mutex_lock(&st_lock); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); plat_mutex_unlock(&st_lock); static unsigned subs; if (++subs % 4096 == 0) rec_stats_log(); plat_mutex_unlock(&sub_lock); return fence; } static _Thread_local VkDeferred *deferred; void vk_completion_defer(VkDeferred *d) { deferred = d; } static void queue_put(uint64_t fence, void (*fn)(void *), void *arg) { // st_lock held if (qhead + qn == qcap) { if (qhead) { memmove(queue, queue + qhead, sizeof *queue * (size_t)qn); qhead = 0; } else { int cap = qcap ? qcap * 2 : 256; Pending *q = realloc(queue, sizeof *q * (size_t)cap); if (!q) rt_fatal("vk: out of memory (completion queue)"); queue = q, qcap = cap; } } queue[qhead + qn++] = (Pending){fence, fn, arg}; // The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's, // for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it. if (fence <= done) plat_cond_broadcast(&cv_work); } void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) { VkDeferred *d = deferred; if (d) { if (d->n == d->cap) { d->cap = d->cap ? d->cap * 2 : 64; d->items = realloc(d->items, sizeof(Pending) * (size_t)d->cap); if (!d->items) rt_fatal("vk: out of memory (completion queue)"); } ((Pending *)d->items)[d->n++] = (Pending){fence, fn, arg}; return; } plat_mutex_lock(&st_lock); queue_put(fence, fn, arg); plat_mutex_unlock(&st_lock); } void vk_completion_flush(VkDeferred *d) { plat_mutex_lock(&st_lock); for (int i = 0; i < d->n; i++) { const Pending *p = &((Pending *)d->items)[i]; queue_put(p->fence, p->fn, p->arg); } plat_mutex_unlock(&st_lock); d->n = 0; } static int fence_wait_impl(uint64_t value, uint64_t timeout_ns); int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) { uint64_t p0 = perf_begin(); int ok = fence_wait_impl(value, timeout_ns); perf_end(PERF_FENCE, p0); perf_gpu_wait(p0); return ok; } static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { vk_sync_start(); plat_mutex_lock(&st_lock); int ok; if (value > issued && value > reserved) { ok = 0; // never submitted nor queued: never reached (as in the null back end) } else { uint64_t t0 = kern_now_ns(); while (done < value) { if (timeout_ns < (1ull << 62)) { uint64_t el = kern_now_ns() - t0; if (el >= timeout_ns) break; plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el); } else { plat_cond_wait(&cv_done, &st_lock); } } ok = done >= value; } plat_mutex_unlock(&st_lock); return ok; } uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); } uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); } void vk_sync_finish(void) { vk_sync_start(); vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX); vk_upload_flush(1); } void vk_sync_shutdown(void) { plat_mutex_lock(&start_lock); if (started) { vk_sync_finish(); plat_mutex_lock(&st_lock); stop = 1; plat_cond_broadcast(&cv_work); plat_mutex_unlock(&st_lock); plat_thread_join(completion_thread); completion_thread = NULL; plat_mutex_lock(&st_lock); qhead = qn = 0; // callbacks of values never submitted plat_mutex_unlock(&st_lock); plat_mutex_lock(&sub_lock); VkCtx *c = vk_ctx(); if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL); perf_gpu_stats = NULL; if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL); ts_pool = VK_NULL_HANDLE; ts_state = 0; ts_last_end = 0; memset((void *)ts_busy, 0, sizeof ts_busy); cmd_pool = VK_NULL_HANDLE; free(slots); slots = NULL; nslots = 0; plat_mutex_unlock(&sub_lock); __atomic_store_n(&started, 0, __ATOMIC_RELEASE); } plat_mutex_unlock(&start_lock); }