Poll briefly before kernel waits and the recorder sleep

Guest waits and waits for a virtual core usually end within microseconds. A thread now polls for 20 us with the
global lock released before it sleeps on its condition variable, so a quick wake costs no OS wake-up
(TOTK_KSPIN_US, 0 turns it off). The recorder polls by the clock for 60 us instead of 4096 pauses, which last a few
us on Zen 4 and about 140 us on Coffee Lake (TOTK_REC_SPIN_US). Both stay off with fewer than 6 usable logical
processors. Savage Wind Recomp polls the same way before it parks.

Machine A, town save, in-run A/B: frame interval -1.8% on average for the kernel poll (5 of 7 block pairs), -0.7%
with the process limited to 4 cores / 8 threads; the recorder poll is neutral there.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

runtime/cfg.c

@@ -45,14 +45,14 @@ static const char *const known[] = {
     "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FRAME_QUEUE",
     "FREEZE_IDLE", "FREE_CORES",
     "FULLSCREEN", "FXAA", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE",
-    "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
+    "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "KSPIN_US", "LANG", "LIB_PREBUILD",
     "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MENU_LANG", "MENU_TEST",
     "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_AIM_SCALE", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_ASPECT", "MOVIE_HOST", "MOVIE_PACE",
     "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
     "MV_SURVEY",
     "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
     "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_BENCH", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES",
-    "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC",
+    "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REC_SPIN_US", "REEXEC",
     "RENDER_SCALE", "REPLAY_ASYNC",
     "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
     "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",

runtime/kernel.c

@@ -246,6 +246,44 @@ static void wake_sync_waiters(void) {
     }
 }
 
+// ---- a short poll before a sleep
+// Most guest waits and waits for a virtual core end within microseconds. The thread polls for TOTK_KSPIN_US (default
+// 20) with G released before it sleeps on its condition variable: a wake in that time costs no OS wake-up (4-12 us
+// on a desktop Ryzen, more from a deep C-state) and the waker's signal finds nobody asleep. Off with fewer than 6 logical
+// processors, where the poll would take a CPU that a running thread needs; TOTK_KSPIN_US=0 turns it off. Savage Wind
+// Recomp polls the same way before it parks (runtime/ksync.c cond_wait). TOTK_KSPIN_US=ab polls 20 us in the
+// B blocks of TOTK_AB only.
+static int64_t g_kspin_ns = -1;
+static int g_kspin_ab;
+
+static uint64_t kspin_ns(void) {
+    int64_t v = __atomic_load_n(&g_kspin_ns, __ATOMIC_RELAXED);
+    if (v < 0) {
+        const char *e = cfg_get("TOTK_KSPIN_US");
+        g_kspin_ab = e && !strcmp(e, "ab");
+        long long us = e && *e && !g_kspin_ab ? atoll(e) : 20;
+        uint32_t cpus = plat_cpu_usable();
+        v = us <= 0 || cpus < 6 ? 0 : us * 1000;
+        __atomic_store_n(&g_kspin_ns, v, __ATOMIC_RELAXED);
+        rt_log("kernel: waits poll %lld us before they sleep%s (TOTK_KSPIN_US, %u logical processors)",
+               (long long)(v / 1000), g_kspin_ab ? " in the B blocks" : "", cpus);
+    }
+    return v && (!g_kspin_ab || rt_ab()) ? (uint64_t)v : 0;
+}
+
+// Holding G: releases it and polls *p until it reaches at_least or ns pass, then takes G again. 1 if it was reached.
+static int kspin(const int *p, int at_least, uint64_t ns) {
+    uint64_t end = kern_now_ns() + ns;
+    int ok = 0;
+    unlock();
+    for (unsigned i = 1; !(ok = __atomic_load_n(p, __ATOMIC_ACQUIRE) >= at_least); i++) {
+        plat_pause();
+        if (!(i & 31) && kern_now_ns() >= end) break;
+    }
+    lock();
+    return ok;
+}
+
 // SleepConditionVariableSRW counts milliseconds: round up (late is legal, early is not) and re-check the deadline
 // against the performance counter, because the wait may also end early.
 static uint32_t block_inner(RtThread *t, int kind, int64_t ns) {
@@ -262,6 +300,8 @@ static uint32_t block_inner(RtThread *t, int kind, int64_t ns) {
     uint64_t now = kern_now_ns();
     int forever = ns < 0 || (uint64_t)ns > UINT64_MAX - now;
     uint64_t deadline = forever ? 0 : now + (uint64_t)ns;
+    uint64_t poll = kspin_ns();
+    if (poll) kspin(&t->woke, 1, forever || poll < (uint64_t)ns ? poll : (uint64_t)ns);
     while (!t->woke) {
         if (forever) {
             plat_cond_wait(&t->cv, &G);
@@ -991,6 +1031,9 @@ static void core_take(RtThread *t) {
         while (__atomic_load_n(&t->cur_core, __ATOMIC_RELAXED) < 0 && kern_now_ns() < end) plat_pause();
         lock();
         if (t->cur_core >= 0) g_ap_spin_ok++;
+    } else if (t->cur_core < 0) {   // a holder may block or yield any moment (kspin_ns)
+        uint64_t poll = kspin_ns();
+        if (poll) kspin(&t->cur_core, 0, poll);
     }
     int sampled = 0;
     while (t->cur_core < 0) {

runtime/vk_rec.c

@@ -43,8 +43,11 @@ static PlatThread *rec_thread;
 static int rec_prio;                   // the submitting thread's priority, which the recorder takes
 static int rec_chk;                    // TOTK_REC_CHECK (rec_check_handle)
 static uint64_t rec_delay_ns;          // TOTK_REC_DELAY_US (rec_main)
+static uint64_t rec_spin_ns;           // TOTK_REC_SPIN_US (rec_main); 0: 4096 pauses
+static int rec_spin_ab;                // TOTK_REC_SPIN_US=ab: the timed poll in the B blocks of TOTK_AB only
 static volatile uint64_t part_h0;      // Q.put_h at rec_begin: where the packets of the part being recorded start
 static volatile uint64_t part_fence;   // the value of the part being recorded (rec_begin)
+static uint64_t r_sleeps;              // recorder: sleeps on wake_ev, read by rec_stats_log
 
 // Producer statistics, logged by rec_stats_log.
 static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_;
@@ -75,11 +78,27 @@ static void rec_main(void *arg) {
     for (;;) {
         uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE);
         if (t == h) {
-            for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE);
+            // Poll for the next packets before sleeping: a sleep costs the producer an event set and the recorder
+            // an OS wake-up. The poll is timed (TOTK_REC_SPIN_US, default 60) because a pause takes 8-150 cycles
+            // depending on the CPU: 4096 of them are ~140 us on a Coffee Lake i7 and a few us on Zen 4. With fewer than
+            // 6 logical processors it stays 4096 pauses. Savage Wind Recomp's recorder polls 60 us the same way.
+            if (rec_spin_ns && (!rec_spin_ab || rt_ab())) {
+                uint64_t t0 = now_ns();
+                for (unsigned i = 1; t == h; i++) {
+                    _mm_pause();
+                    h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE);
+                    if (!(i & 31) && now_ns() - t0 >= rec_spin_ns) break;
+                }
+            } else {
+                for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE);
+            }
             if (t == h) {
                 __atomic_store_n(&Q.sleeping, 1, __ATOMIC_SEQ_CST);
                 h = __atomic_load_n(&Q.head, __ATOMIC_SEQ_CST);
-                if (t == h) plat_event_wait_ns(&wake_ev, 2000000);   // a missed wake costs at most 2 ms
+                if (t == h) {
+                    __atomic_add_fetch(&r_sleeps, 1, __ATOMIC_RELAXED);
+                    plat_event_wait_ns(&wake_ev, 2000000);   // a missed wake costs at most 2 ms
+                }
                 __atomic_store_n(&Q.sleeping, 0, __ATOMIC_RELAXED);
                 continue;
             }
@@ -200,6 +219,10 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) {
             rec_chk = rt_knob("REC_CHECK", "0");   // debug switches, read before the recorder starts
             const char *d = cfg_get("TOTK_REC_DELAY_US");
             rec_delay_ns = d && *d && atoi(d) > 0 ? (uint64_t)atoi(d) * 1000 : 0;
+            const char *sp = cfg_get("TOTK_REC_SPIN_US");
+            rec_spin_ab = sp && !strcmp(sp, "ab");
+            int spin_us = sp && *sp && !rec_spin_ab ? atoi(sp) : 60;
+            rec_spin_ns = spin_us > 0 && plat_cpu_usable() >= 6 ? (uint64_t)spin_us * 1000 : 0;
             const char *pw = cfg_get("TOTK_REC_PUB");   // words per publish (pub); 0: every packet
             Q.pub_words = rec_chk ? 0 : pw && *pw ? (uint64_t)atoi(pw) : 64;
             if (Q.pub_words > RING_WORDS / 8) Q.pub_words = RING_WORDS / 8;
@@ -211,6 +234,11 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) {
                 rt_log("vk_rec: the replay's Vulkan commands are made by a recorder thread%s",
                        mode == 2 ? " in the B blocks (TOTK_AB)" : "");
                 if (rec_chk) rt_log("vk_rec: TOTK_REC_CHECK: deferred and destroyed objects are searched in the ring");
+                if (rec_spin_ns)
+                    rt_log("vk_rec: the recorder polls %llu us before it sleeps%s (TOTK_REC_SPIN_US)",
+                           (unsigned long long)(rec_spin_ns / 1000), rec_spin_ab ? " in the B blocks" : "");
+                else
+                    rt_log("vk_rec: the recorder polls 4096 pauses before it sleeps (TOTK_REC_SPIN_US)");
                 if (Q.pub_words != 64) rt_log("vk_rec: TOTK_REC_PUB: head published every %llu words",
                                               (unsigned long long)Q.pub_words);
                 if (rec_delay_ns) rt_log("vk_rec: TOTK_REC_DELAY_US: the recorder waits %llu us before each batch",
@@ -315,10 +343,12 @@ static void submit_run(const A_Submit *a) {
 void rec_stats_log(void) {
     if (!(rt_trace & RT_TRACE_PERF) || !RS_.parts) return;
     rt_log("vk_rec: %llu of %llu parts recorded by the recorder: %llu packets (%.1f MiB), %llu drains (%llu for a call "
-           "without a packet) waited %.2f ms in all, ring full %llu times, %llu submissions made by the recorder",
+           "without a packet) waited %.2f ms in all, ring full %llu times, %llu submissions made by the recorder, "
+           "recorder slept %llu times",
            (unsigned long long)RS_.parts_on, (unsigned long long)RS_.parts, (unsigned long long)RS_.packets,
            RS_.words * 8 / 1048576.0, (unsigned long long)RS_.drains, (unsigned long long)RS_.fallbacks,
-           RS_.drain_ns / 1e6, (unsigned long long)RS_.full_waits, (unsigned long long)RS_.submits);
+           RS_.drain_ns / 1e6, (unsigned long long)RS_.full_waits, (unsigned long long)RS_.submits,
+           (unsigned long long)__atomic_exchange_n(&r_sleeps, 0, __ATOMIC_RELAXED));
     memset(&RS_, 0, sizeof RS_);
 }