diff --git a/runtime/cfg.c b/runtime/cfg.c index 76c3279d..be7febbe 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -52,7 +52,7 @@ static const char *const known[] = { "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_BENCH", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", - "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REC_SPIN_US", "REEXEC", + "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_ASYNC", "REC_END", "REC_SPIN_US", "REEXEC", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index 472acb41..ca698c7e 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -33,9 +33,20 @@ #define VSYNC_HZ 60 // ---- queue lock +// The recorder queues the parts it records and submits them later (vk_rec.c rec_submit): the first lock of a thread +// waits until the parts queued before it are submitted, so whatever it puts on the queue comes after them, as when +// the submitting thread submitted them itself. Not inside a held lock: the recorder takes this lock to submit. static PlatRMutex g_ql; -void vk_queue_lock(void) { plat_rmutex_lock(&g_ql); } -void vk_queue_unlock(void) { plat_rmutex_unlock(&g_ql); } +static _Thread_local int ql_depth; +void vk_queue_lock(void) { + if (!ql_depth) rec_queue_wait(); + plat_rmutex_lock(&g_ql); + ql_depth++; +} +void vk_queue_unlock(void) { + ql_depth--; + plat_rmutex_unlock(&g_ql); +} // ---- window state shared with the back end static VkSurfaceKHR g_surface; diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 0706dbcb..5d9a746d 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -39,6 +39,12 @@ static int rec_live; // producer: rec_begin started the record static int end_mode = -1; // TOTK_REC_END (rec_own) static int sub_pending; // producer: the recorder ended a part that rec_submit has not handed over yet static volatile VkResult r_sub_res; // recorder: the result of its last vkQueueSubmit2, read after the drain +static volatile VkResult r_sub_err; // recorder: the first vkQueueSubmit2 that failed (rec_submit reports it) +static volatile uint64_t q_subs; // producer: submissions queued for the recorder (rec_submit) +static volatile uint64_t r_subs; // recorder: submissions it handed to the driver (submit_run) +static _Thread_local int on_rec; // the calling thread is the recorder +static int rec_async = -1; // TOTK_REC_ASYNC (rec_submit) +static struct { uint64_t n, ns; } RQ_; // waits in rec_queue_wait, logged by rec_stats_log static PlatThread *rec_thread; static int rec_prio; // the submitting thread's priority, which the recorder takes static int rec_chk; // TOTK_REC_CHECK (rec_check_handle) @@ -50,7 +56,7 @@ static volatile uint64_t part_fence; // the value of the part being recorded ( static uint64_t r_sleeps; // recorder: sleeps on wake_ev, read by rec_stats_log // Producer statistics, logged by rec_stats_log. -static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_; +static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits, wakes; } RS_; enum { R_PAD, R_BeginQuery, R_BeginRendering, R_BindDescriptorSets, R_BindIndexBuffer, R_BindPipeline, R_BindVertexBuffers2, @@ -74,6 +80,7 @@ static void exec_packet(uint32_t op, void *p); static void rec_main(void *arg) { (void)arg; plat_thread_priority(rec_prio); + on_rec = 1; uint64_t t = 0; for (;;) { uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); @@ -122,7 +129,7 @@ static void rec_main(void *arg) { static void wake(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); - if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); + if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev), RS_.wakes++; } // Room for need words: the tail read last leaves it in all but about one packet in 16 MiB of them. @@ -162,14 +169,21 @@ static void *put(uint32_t op, size_t bytes) { // rec_drain), and the packets that begin and end a part go out at once (pub_now), so the recorder begins and ends // command buffers as early as before; TOTK_REC_CHECK publishes every packet, since rec_check_handle searches the ring // up to head. +// The batch is 64 words (512 bytes) where the recorder polls before it sleeps, 8192 (64 KiB) where it does not (fewer +// than 6 usable logical processors, rec_spin_ns): there it is mostly asleep when a batch goes out, and each publish +// cost the producer an event set and the recorder a wake-up. Savage Wind Recomp measured that at 6.7% of its render +// thread's busy time on 4 cores and moved to 64 KiB there (runtime/nvn/vulkan/record.c rec_init, publish_every). +// TOTK_REC_PUB= sets the batch; =ab: 64 words in the A blocks of TOTK_AB, 8192 in the B blocks. +static int pub_ab; static void pub_now(void) { Q.pub_h = Q.put_h; __atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE); - if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); + if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev), RS_.wakes++; } static void pub(void) { - if (Q.put_h - Q.pub_h >= Q.pub_words) pub_now(); + uint64_t w = pub_ab ? (rt_ab() ? 8192 : 64) : Q.pub_words; + if (Q.put_h - Q.pub_h >= w) pub_now(); } void rec_drain(void) { @@ -224,7 +238,8 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { int spin_us = sp && *sp && !rec_spin_ab ? atoi(sp) : 60; rec_spin_ns = spin_us > 0 && plat_cpu_usable() >= 6 ? (uint64_t)spin_us * 1000 : 0; const char *pw = cfg_get("TOTK_REC_PUB"); // words per publish (pub); 0: every packet - Q.pub_words = rec_chk ? 0 : pw && *pw ? (uint64_t)atoi(pw) : 64; + pub_ab = !rec_chk && pw && !strcmp(pw, "ab"); + Q.pub_words = rec_chk ? 0 : pw && *pw && !pub_ab ? (uint64_t)atoi(pw) : rec_spin_ns ? 64 : 8192; if (Q.pub_words > RING_WORDS / 8) Q.pub_words = RING_WORDS / 8; rec_thread = plat_thread_new(rec_main, NULL, "vk recorder"); if (!rec_thread) { @@ -239,8 +254,8 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { (unsigned long long)(rec_spin_ns / 1000), rec_spin_ab ? " in the B blocks" : ""); else rt_log("vk_rec: the recorder polls 4096 pauses before it sleeps (TOTK_REC_SPIN_US)"); - if (Q.pub_words != 64) rt_log("vk_rec: TOTK_REC_PUB: head published every %llu words", - (unsigned long long)Q.pub_words); + if (pub_ab) rt_log("vk_rec: head published every 64 words, 8192 in the B blocks (TOTK_REC_PUB=ab)"); + else rt_log("vk_rec: head published every %llu words (TOTK_REC_PUB)", (unsigned long long)Q.pub_words); if (rec_delay_ns) rt_log("vk_rec: TOTK_REC_DELAY_US: the recorder waits %llu us before each batch", (unsigned long long)(rec_delay_ns / 1000)); } @@ -323,11 +338,20 @@ VkResult rec_submit(const VkSubmitInfo2 *si) { a->ncb = si->commandBufferInfoCount, a->nsig = si->signalSemaphoreInfoCount; for (uint32_t i = 0; i < a->ncb; i++) a->cbi[i] = si->pCommandBufferInfos[i]; if (a->nsig) a->sig = si->pSignalSemaphoreInfos[0]; - pub(); - // Waited for here, so the queue holds every earlier submission when the caller goes on, as with the recorder off: - // the present, uploads, readbacks and device waits on other threads rely on that order. - rec_drain(); + __atomic_store_n(&q_subs, q_subs + 1, __ATOMIC_RELEASE); RS_.submits++; + // TOTK_REC_ASYNC=1 (default): the submitting thread goes on while the recorder catches up with the part and submits + // it; it waited here for 0.29 ms a frame in the town on a Ryzen 7900X, 0.34 ms limited to 4 cores. The queue order + // the present, uploads, readbacks and device waits rely on holds through vk_queue_lock (rec_queue_wait). The values + // the part signals are counted as issued at once (vk_sync.c): waiting for one on the host before its submission is + // legal for a timeline semaphore. =0: waits here for the submission, as before; =ab with the A/B blocks. + if (rec_async < 0) rec_async = rt_knob("REC_ASYNC", "1"); + if (RT_KNOB_ON(rec_async)) { + pub_now(); + return __atomic_load_n(&r_sub_err, __ATOMIC_ACQUIRE); + } + pub(); + rec_drain(); return r_sub_res; } @@ -336,19 +360,37 @@ static void submit_run(const A_Submit *a) { .pCommandBufferInfos = a->cbi, .signalSemaphoreInfoCount = a->nsig, .pSignalSemaphoreInfos = a->nsig ? &a->sig : NULL}; vk_queue_lock(); - r_sub_res = vkQueueSubmit2(vk_ctx()->gfx_queue, 1, &si, VK_NULL_HANDLE); + VkResult r = vkQueueSubmit2(vk_ctx()->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); + r_sub_res = r; + if (r != VK_SUCCESS && __atomic_load_n(&r_sub_err, __ATOMIC_RELAXED) == VK_SUCCESS) + __atomic_store_n(&r_sub_err, r, __ATOMIC_RELEASE); + __atomic_store_n(&r_subs, r_subs + 1, __ATOMIC_RELEASE); +} + +void rec_queue_wait(void) { + if (!rec_thread || on_rec) return; + uint64_t want = __atomic_load_n(&q_subs, __ATOMIC_ACQUIRE); + if (__atomic_load_n(&r_subs, __ATOMIC_ACQUIRE) >= want) return; + uint64_t t0 = now_ns(); + for (int i = 0; __atomic_load_n(&r_subs, __ATOMIC_ACQUIRE) < want; i++) + if (i < 20000) _mm_pause(); + else plat_yield(); + __atomic_add_fetch(&RQ_.n, 1, __ATOMIC_RELAXED); + __atomic_add_fetch(&RQ_.ns, now_ns() - t0, __ATOMIC_RELAXED); } void rec_stats_log(void) { if (!(rt_trace & RT_TRACE_PERF) || !RS_.parts) return; rt_log("vk_rec: %llu of %llu parts recorded by the recorder: %llu packets (%.1f MiB), %llu drains (%llu for a call " "without a packet) waited %.2f ms in all, ring full %llu times, %llu submissions made by the recorder, " - "recorder slept %llu times", + "recorder slept %llu times and was woken %llu times, queue users waited for its submissions %llu times (%.2f ms)", (unsigned long long)RS_.parts_on, (unsigned long long)RS_.parts, (unsigned long long)RS_.packets, RS_.words * 8 / 1048576.0, (unsigned long long)RS_.drains, (unsigned long long)RS_.fallbacks, RS_.drain_ns / 1e6, (unsigned long long)RS_.full_waits, (unsigned long long)RS_.submits, - (unsigned long long)__atomic_exchange_n(&r_sleeps, 0, __ATOMIC_RELAXED)); + (unsigned long long)__atomic_exchange_n(&r_sleeps, 0, __ATOMIC_RELAXED), (unsigned long long)RS_.wakes, + (unsigned long long)__atomic_exchange_n(&RQ_.n, 0, __ATOMIC_RELAXED), + __atomic_exchange_n(&RQ_.ns, 0, __ATOMIC_RELAXED) / 1e6); memset(&RS_, 0, sizeof RS_); } diff --git a/runtime/vk_rec.h b/runtime/vk_rec.h index bbceff90..28c737af 100644 --- a/runtime/vk_rec.h +++ b/runtime/vk_rec.h @@ -5,8 +5,10 @@ // command buffer (uploads, the window, probes) are made at once. A call the ring has no packet for (a structure with a // pNext chain) waits for the recorder to empty the ring and is then made at once on the calling thread. With // TOTK_REC_END=1 (default) the recorder also begins, ends and submits the parts it records, so a part's command buffer -// never changes threads; rec_submit returns once the recorder submitted it. The submitting thread allocates from the -// part pool only while the recorder is idle (vk_sync.c claim_slot). +// never changes threads. rec_submit queues the submission and returns at once (TOTK_REC_ASYNC=1, default; 0: once the +// recorder submitted it); every other user of the queue waits in vk_queue_lock until the recorder submitted what was +// queued before (rec_queue_wait), so queue order stays the order of the calls. The submitting thread allocates from +// the part pool only while the recorder is idle (vk_sync.c claim_slot). // // Every vkCmd* the runtime calls through vk.h goes through the rc_* wrappers below (the macros at the end); the // extension commands called through function pointers have rec_* wrappers of their own. TOTK_VK_RECORDER=0 makes every @@ -24,9 +26,13 @@ void rec_end(void); // waits for the recorder; no-op when rec_ // (1: rec_submit hands the part over); otherwise rec_end and the breadcrumb, and the caller ends cb (0). int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n); // vkQueueSubmit2 of si (at most two command buffers, one signal, no wait, no fence) under vk_queue_lock: by the -// recorder after a part it ended, waited for, else at once. +// recorder after a part it ended, else at once. Queued for the recorder, it returns the first failure of an earlier +// submission the recorder made (VK_SUCCESS if none), since this one's result is not known yet. VkResult rec_submit(const VkSubmitInfo2 *si); void rec_drain(void); // waits until the recorder made every call queued so far +// vk_queue_lock, before it takes the lock (not on the recorder, not when the thread holds the lock already): waits +// until the recorder submitted every submission queued so far. +void rec_queue_wait(void); void rec_stats_log(void); // TOTK_TRACE=perf: packets, bytes, drains and waits since the last call // TOTK_REC_CHECK=1 (debug, recorder on): vk_memory.c reports each object it defers (destroy 0, stamped tl) or destroys // (destroy 1) with the value the GPU reached (gpu); a packet of the part being recorded that still names it is logged