Let the recorder submit parts without the submitting thread waiting for it

rec_submit queued each part's submission and then drained the ring until the recorder had submitted it: 0.29 ms a
frame in the town on a Ryzen 7900X, 0.34 ms on 4 cores. Now it publishes the submission and returns
(TOTK_REC_ASYNC=1, default). The queue order the present, uploads, readbacks and device waits rely on is kept by
vk_queue_lock: a thread's first lock waits until the recorder submitted what was queued before (rec_queue_wait).
The recorder itself and a thread that already holds the lock skip the wait. rec_submit reports the first failed
submission the recorder made. Town, in-run A/B: replay -4.1% on all cores, -2.9% on 4 cores / 8 threads; queue
users waited about 0.02 ms a frame. fight2 with TOTK_REC_CHECK and the town under sync validation are clean.

Where the recorder does not poll (fewer than 6 usable logical processors) the producer now publishes the ring head
every 8192 words instead of 64, as Savage Wind Recomp does (runtime/nvn/vulkan/record.c rec_init): each publish
there costs an event set and a wake-up. Town on 4 logical processors: frame interval -1.3%. TOTK_REC_PUB=ab
compares both; the perf log counts the recorder's wake-ups and the queue waits.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-08 12:02:53 +03:00
1 parent 1446f3b06b
commit 90fa6e1788
4 files changed
+79 -20

No files matched your search

+1 -1
View File
@@ -52,7 +52,7 @@ static const char *const known[] = {
"MV_SURVEY",
"ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
"PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_BENCH", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES",
"PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REC_SPIN_US", "REEXEC",
"PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_ASYNC", "REC_END", "REC_SPIN_US", "REEXEC",
"RENDER_SCALE", "REPLAY_ASYNC",
"REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
+13 -2
View File
@@ -33,9 +33,20 @@
#define VSYNC_HZ 60
// ---- queue lock
// The recorder queues the parts it records and submits them later (vk_rec.c rec_submit): the first lock of a thread
// waits until the parts queued before it are submitted, so whatever it puts on the queue comes after them, as when
// the submitting thread submitted them itself. Not inside a held lock: the recorder takes this lock to submit.
static PlatRMutex g_ql;
void vk_queue_lock(void) { plat_rmutex_lock(&g_ql); }
void vk_queue_unlock(void) { plat_rmutex_unlock(&g_ql); }
static _Thread_local int ql_depth;
void vk_queue_lock(void) {
if (!ql_depth) rec_queue_wait();
plat_rmutex_lock(&g_ql);
ql_depth++;
}
void vk_queue_unlock(void) {
ql_depth--;
plat_rmutex_unlock(&g_ql);
}
// ---- window state shared with the back end
static VkSurfaceKHR g_surface;
+56 -14
View File
@@ -39,6 +39,12 @@ static int rec_live; // producer: rec_begin started the record
static int end_mode = -1; // TOTK_REC_END (rec_own)
static int sub_pending; // producer: the recorder ended a part that rec_submit has not handed over yet
static volatile VkResult r_sub_res; // recorder: the result of its last vkQueueSubmit2, read after the drain
static volatile VkResult r_sub_err; // recorder: the first vkQueueSubmit2 that failed (rec_submit reports it)
static volatile uint64_t q_subs; // producer: submissions queued for the recorder (rec_submit)
static volatile uint64_t r_subs; // recorder: submissions it handed to the driver (submit_run)
static _Thread_local int on_rec; // the calling thread is the recorder
static int rec_async = -1; // TOTK_REC_ASYNC (rec_submit)
static struct { uint64_t n, ns; } RQ_; // waits in rec_queue_wait, logged by rec_stats_log
static PlatThread *rec_thread;
static int rec_prio; // the submitting thread's priority, which the recorder takes
static int rec_chk; // TOTK_REC_CHECK (rec_check_handle)
@@ -50,7 +56,7 @@ static volatile uint64_t part_fence; // the value of the part being recorded (
static uint64_t r_sleeps; // recorder: sleeps on wake_ev, read by rec_stats_log
// Producer statistics, logged by rec_stats_log.
static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_;
static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits, wakes; } RS_;
enum {
R_PAD, R_BeginQuery, R_BeginRendering, R_BindDescriptorSets, R_BindIndexBuffer, R_BindPipeline, R_BindVertexBuffers2,
@@ -74,6 +80,7 @@ static void exec_packet(uint32_t op, void *p);
static void rec_main(void *arg) {
(void)arg;
plat_thread_priority(rec_prio);
on_rec = 1;
uint64_t t = 0;
for (;;) {
uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE);
@@ -122,7 +129,7 @@ static void rec_main(void *arg) {
static void wake(void) {
__atomic_thread_fence(__ATOMIC_SEQ_CST);
if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev);
if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev), RS_.wakes++;
}
// Room for need words: the tail read last leaves it in all but about one packet in 16 MiB of them.
@@ -162,14 +169,21 @@ static void *put(uint32_t op, size_t bytes) {
// rec_drain), and the packets that begin and end a part go out at once (pub_now), so the recorder begins and ends
// command buffers as early as before; TOTK_REC_CHECK publishes every packet, since rec_check_handle searches the ring
// up to head.
// The batch is 64 words (512 bytes) where the recorder polls before it sleeps, 8192 (64 KiB) where it does not (fewer
// than 6 usable logical processors, rec_spin_ns): there it is mostly asleep when a batch goes out, and each publish
// cost the producer an event set and the recorder a wake-up. Savage Wind Recomp measured that at 6.7% of its render
// thread's busy time on 4 cores and moved to 64 KiB there (runtime/nvn/vulkan/record.c rec_init, publish_every).
// TOTK_REC_PUB=<words> sets the batch; =ab: 64 words in the A blocks of TOTK_AB, 8192 in the B blocks.
static int pub_ab;
static void pub_now(void) {
Q.pub_h = Q.put_h;
__atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE);
if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev);
if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev), RS_.wakes++;
}
static void pub(void) {
if (Q.put_h - Q.pub_h >= Q.pub_words) pub_now();
uint64_t w = pub_ab ? (rt_ab() ? 8192 : 64) : Q.pub_words;
if (Q.put_h - Q.pub_h >= w) pub_now();
}
void rec_drain(void) {
@@ -224,7 +238,8 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) {
int spin_us = sp && *sp && !rec_spin_ab ? atoi(sp) : 60;
rec_spin_ns = spin_us > 0 && plat_cpu_usable() >= 6 ? (uint64_t)spin_us * 1000 : 0;
const char *pw = cfg_get("TOTK_REC_PUB"); // words per publish (pub); 0: every packet
Q.pub_words = rec_chk ? 0 : pw && *pw ? (uint64_t)atoi(pw) : 64;
pub_ab = !rec_chk && pw && !strcmp(pw, "ab");
Q.pub_words = rec_chk ? 0 : pw && *pw && !pub_ab ? (uint64_t)atoi(pw) : rec_spin_ns ? 64 : 8192;
if (Q.pub_words > RING_WORDS / 8) Q.pub_words = RING_WORDS / 8;
rec_thread = plat_thread_new(rec_main, NULL, "vk recorder");
if (!rec_thread) {
@@ -239,8 +254,8 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) {
(unsigned long long)(rec_spin_ns / 1000), rec_spin_ab ? " in the B blocks" : "");
else
rt_log("vk_rec: the recorder polls 4096 pauses before it sleeps (TOTK_REC_SPIN_US)");
if (Q.pub_words != 64) rt_log("vk_rec: TOTK_REC_PUB: head published every %llu words",
(unsigned long long)Q.pub_words);
if (pub_ab) rt_log("vk_rec: head published every 64 words, 8192 in the B blocks (TOTK_REC_PUB=ab)");
else rt_log("vk_rec: head published every %llu words (TOTK_REC_PUB)", (unsigned long long)Q.pub_words);
if (rec_delay_ns) rt_log("vk_rec: TOTK_REC_DELAY_US: the recorder waits %llu us before each batch",
(unsigned long long)(rec_delay_ns / 1000));
}
@@ -323,11 +338,20 @@ VkResult rec_submit(const VkSubmitInfo2 *si) {
a->ncb = si->commandBufferInfoCount, a->nsig = si->signalSemaphoreInfoCount;
for (uint32_t i = 0; i < a->ncb; i++) a->cbi[i] = si->pCommandBufferInfos[i];
if (a->nsig) a->sig = si->pSignalSemaphoreInfos[0];
pub();
// Waited for here, so the queue holds every earlier submission when the caller goes on, as with the recorder off:
// the present, uploads, readbacks and device waits on other threads rely on that order.
rec_drain();
__atomic_store_n(&q_subs, q_subs + 1, __ATOMIC_RELEASE);
RS_.submits++;
// TOTK_REC_ASYNC=1 (default): the submitting thread goes on while the recorder catches up with the part and submits
// it; it waited here for 0.29 ms a frame in the town on a Ryzen 7900X, 0.34 ms limited to 4 cores. The queue order
// the present, uploads, readbacks and device waits rely on holds through vk_queue_lock (rec_queue_wait). The values
// the part signals are counted as issued at once (vk_sync.c): waiting for one on the host before its submission is
// legal for a timeline semaphore. =0: waits here for the submission, as before; =ab with the A/B blocks.
if (rec_async < 0) rec_async = rt_knob("REC_ASYNC", "1");
if (RT_KNOB_ON(rec_async)) {
pub_now();
return __atomic_load_n(&r_sub_err, __ATOMIC_ACQUIRE);
}
pub();
rec_drain();
return r_sub_res;
}
@@ -336,19 +360,37 @@ static void submit_run(const A_Submit *a) {
.pCommandBufferInfos = a->cbi, .signalSemaphoreInfoCount = a->nsig,
.pSignalSemaphoreInfos = a->nsig ? &a->sig : NULL};
vk_queue_lock();
r_sub_res = vkQueueSubmit2(vk_ctx()->gfx_queue, 1, &si, VK_NULL_HANDLE);
VkResult r = vkQueueSubmit2(vk_ctx()->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
r_sub_res = r;
if (r != VK_SUCCESS && __atomic_load_n(&r_sub_err, __ATOMIC_RELAXED) == VK_SUCCESS)
__atomic_store_n(&r_sub_err, r, __ATOMIC_RELEASE);
__atomic_store_n(&r_subs, r_subs + 1, __ATOMIC_RELEASE);
}
void rec_queue_wait(void) {
if (!rec_thread || on_rec) return;
uint64_t want = __atomic_load_n(&q_subs, __ATOMIC_ACQUIRE);
if (__atomic_load_n(&r_subs, __ATOMIC_ACQUIRE) >= want) return;
uint64_t t0 = now_ns();
for (int i = 0; __atomic_load_n(&r_subs, __ATOMIC_ACQUIRE) < want; i++)
if (i < 20000) _mm_pause();
else plat_yield();
__atomic_add_fetch(&RQ_.n, 1, __ATOMIC_RELAXED);
__atomic_add_fetch(&RQ_.ns, now_ns() - t0, __ATOMIC_RELAXED);
}
void rec_stats_log(void) {
if (!(rt_trace & RT_TRACE_PERF) || !RS_.parts) return;
rt_log("vk_rec: %llu of %llu parts recorded by the recorder: %llu packets (%.1f MiB), %llu drains (%llu for a call "
"without a packet) waited %.2f ms in all, ring full %llu times, %llu submissions made by the recorder, "
"recorder slept %llu times",
"recorder slept %llu times and was woken %llu times, queue users waited for its submissions %llu times (%.2f ms)",
(unsigned long long)RS_.parts_on, (unsigned long long)RS_.parts, (unsigned long long)RS_.packets,
RS_.words * 8 / 1048576.0, (unsigned long long)RS_.drains, (unsigned long long)RS_.fallbacks,
RS_.drain_ns / 1e6, (unsigned long long)RS_.full_waits, (unsigned long long)RS_.submits,
(unsigned long long)__atomic_exchange_n(&r_sleeps, 0, __ATOMIC_RELAXED));
(unsigned long long)__atomic_exchange_n(&r_sleeps, 0, __ATOMIC_RELAXED), (unsigned long long)RS_.wakes,
(unsigned long long)__atomic_exchange_n(&RQ_.n, 0, __ATOMIC_RELAXED),
__atomic_exchange_n(&RQ_.ns, 0, __ATOMIC_RELAXED) / 1e6);
memset(&RS_, 0, sizeof RS_);
}
+9 -3
View File
@@ -5,8 +5,10 @@
// command buffer (uploads, the window, probes) are made at once. A call the ring has no packet for (a structure with a
// pNext chain) waits for the recorder to empty the ring and is then made at once on the calling thread. With
// TOTK_REC_END=1 (default) the recorder also begins, ends and submits the parts it records, so a part's command buffer
// never changes threads; rec_submit returns once the recorder submitted it. The submitting thread allocates from the
// part pool only while the recorder is idle (vk_sync.c claim_slot).
// never changes threads. rec_submit queues the submission and returns at once (TOTK_REC_ASYNC=1, default; 0: once the
// recorder submitted it); every other user of the queue waits in vk_queue_lock until the recorder submitted what was
// queued before (rec_queue_wait), so queue order stays the order of the calls. The submitting thread allocates from
// the part pool only while the recorder is idle (vk_sync.c claim_slot).
//
// Every vkCmd* the runtime calls through vk.h goes through the rc_* wrappers below (the macros at the end); the
// extension commands called through function pointers have rec_* wrappers of their own. TOTK_VK_RECORDER=0 makes every
@@ -24,9 +26,13 @@ void rec_end(void); // waits for the recorder; no-op when rec_
// (1: rec_submit hands the part over); otherwise rec_end and the breadcrumb, and the caller ends cb (0).
int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n);
// vkQueueSubmit2 of si (at most two command buffers, one signal, no wait, no fence) under vk_queue_lock: by the
// recorder after a part it ended, waited for, else at once.
// recorder after a part it ended, else at once. Queued for the recorder, it returns the first failure of an earlier
// submission the recorder made (VK_SUCCESS if none), since this one's result is not known yet.
VkResult rec_submit(const VkSubmitInfo2 *si);
void rec_drain(void); // waits until the recorder made every call queued so far
// vk_queue_lock, before it takes the lock (not on the recorder, not when the thread holds the lock already): waits
// until the recorder submitted every submission queued so far.
void rec_queue_wait(void);
void rec_stats_log(void); // TOTK_TRACE=perf: packets, bytes, drains and waits since the last call
// TOTK_REC_CHECK=1 (debug, recorder on): vk_memory.c reports each object it defers (destroy 0, stamped tl) or destroys
// (destroy 1) with the value the GPU reached (gpu); a packet of the part being recorded that still names it is logged