diff --git a/runtime/cfg.c b/runtime/cfg.c index a660c26c..76c3279d 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -45,14 +45,14 @@ static const char *const known[] = { "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FRAME_QUEUE", "FREEZE_IDLE", "FREE_CORES", "FULLSCREEN", "FXAA", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE", - "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", + "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "KSPIN_US", "LANG", "LIB_PREBUILD", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MENU_LANG", "MENU_TEST", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_AIM_SCALE", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_ASPECT", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_BENCH", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", - "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", + "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REC_SPIN_US", "REEXEC", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", diff --git a/runtime/kernel.c b/runtime/kernel.c index e8fb48b4..00aaa963 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -246,6 +246,44 @@ static void wake_sync_waiters(void) { } } +// ---- a short poll before a sleep +// Most guest waits and waits for a virtual core end within microseconds. The thread polls for TOTK_KSPIN_US (default +// 20) with G released before it sleeps on its condition variable: a wake in that time costs no OS wake-up (4-12 us +// on a desktop Ryzen, more from a deep C-state) and the waker's signal finds nobody asleep. Off with fewer than 6 logical +// processors, where the poll would take a CPU that a running thread needs; TOTK_KSPIN_US=0 turns it off. Savage Wind +// Recomp polls the same way before it parks (runtime/ksync.c cond_wait). TOTK_KSPIN_US=ab polls 20 us in the +// B blocks of TOTK_AB only. +static int64_t g_kspin_ns = -1; +static int g_kspin_ab; + +static uint64_t kspin_ns(void) { + int64_t v = __atomic_load_n(&g_kspin_ns, __ATOMIC_RELAXED); + if (v < 0) { + const char *e = cfg_get("TOTK_KSPIN_US"); + g_kspin_ab = e && !strcmp(e, "ab"); + long long us = e && *e && !g_kspin_ab ? atoll(e) : 20; + uint32_t cpus = plat_cpu_usable(); + v = us <= 0 || cpus < 6 ? 0 : us * 1000; + __atomic_store_n(&g_kspin_ns, v, __ATOMIC_RELAXED); + rt_log("kernel: waits poll %lld us before they sleep%s (TOTK_KSPIN_US, %u logical processors)", + (long long)(v / 1000), g_kspin_ab ? " in the B blocks" : "", cpus); + } + return v && (!g_kspin_ab || rt_ab()) ? (uint64_t)v : 0; +} + +// Holding G: releases it and polls *p until it reaches at_least or ns pass, then takes G again. 1 if it was reached. +static int kspin(const int *p, int at_least, uint64_t ns) { + uint64_t end = kern_now_ns() + ns; + int ok = 0; + unlock(); + for (unsigned i = 1; !(ok = __atomic_load_n(p, __ATOMIC_ACQUIRE) >= at_least); i++) { + plat_pause(); + if (!(i & 31) && kern_now_ns() >= end) break; + } + lock(); + return ok; +} + // SleepConditionVariableSRW counts milliseconds: round up (late is legal, early is not) and re-check the deadline // against the performance counter, because the wait may also end early. static uint32_t block_inner(RtThread *t, int kind, int64_t ns) { @@ -262,6 +300,8 @@ static uint32_t block_inner(RtThread *t, int kind, int64_t ns) { uint64_t now = kern_now_ns(); int forever = ns < 0 || (uint64_t)ns > UINT64_MAX - now; uint64_t deadline = forever ? 0 : now + (uint64_t)ns; + uint64_t poll = kspin_ns(); + if (poll) kspin(&t->woke, 1, forever || poll < (uint64_t)ns ? poll : (uint64_t)ns); while (!t->woke) { if (forever) { plat_cond_wait(&t->cv, &G); @@ -991,6 +1031,9 @@ static void core_take(RtThread *t) { while (__atomic_load_n(&t->cur_core, __ATOMIC_RELAXED) < 0 && kern_now_ns() < end) plat_pause(); lock(); if (t->cur_core >= 0) g_ap_spin_ok++; + } else if (t->cur_core < 0) { // a holder may block or yield any moment (kspin_ns) + uint64_t poll = kspin_ns(); + if (poll) kspin(&t->cur_core, 0, poll); } int sampled = 0; while (t->cur_core < 0) { diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 036fe370..0706dbcb 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -43,8 +43,11 @@ static PlatThread *rec_thread; static int rec_prio; // the submitting thread's priority, which the recorder takes static int rec_chk; // TOTK_REC_CHECK (rec_check_handle) static uint64_t rec_delay_ns; // TOTK_REC_DELAY_US (rec_main) +static uint64_t rec_spin_ns; // TOTK_REC_SPIN_US (rec_main); 0: 4096 pauses +static int rec_spin_ab; // TOTK_REC_SPIN_US=ab: the timed poll in the B blocks of TOTK_AB only static volatile uint64_t part_h0; // Q.put_h at rec_begin: where the packets of the part being recorded start static volatile uint64_t part_fence; // the value of the part being recorded (rec_begin) +static uint64_t r_sleeps; // recorder: sleeps on wake_ev, read by rec_stats_log // Producer statistics, logged by rec_stats_log. static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_; @@ -75,11 +78,27 @@ static void rec_main(void *arg) { for (;;) { uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); if (t == h) { - for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); + // Poll for the next packets before sleeping: a sleep costs the producer an event set and the recorder + // an OS wake-up. The poll is timed (TOTK_REC_SPIN_US, default 60) because a pause takes 8-150 cycles + // depending on the CPU: 4096 of them are ~140 us on a Coffee Lake i7 and a few us on Zen 4. With fewer than + // 6 logical processors it stays 4096 pauses. Savage Wind Recomp's recorder polls 60 us the same way. + if (rec_spin_ns && (!rec_spin_ab || rt_ab())) { + uint64_t t0 = now_ns(); + for (unsigned i = 1; t == h; i++) { + _mm_pause(); + h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); + if (!(i & 31) && now_ns() - t0 >= rec_spin_ns) break; + } + } else { + for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE); + } if (t == h) { __atomic_store_n(&Q.sleeping, 1, __ATOMIC_SEQ_CST); h = __atomic_load_n(&Q.head, __ATOMIC_SEQ_CST); - if (t == h) plat_event_wait_ns(&wake_ev, 2000000); // a missed wake costs at most 2 ms + if (t == h) { + __atomic_add_fetch(&r_sleeps, 1, __ATOMIC_RELAXED); + plat_event_wait_ns(&wake_ev, 2000000); // a missed wake costs at most 2 ms + } __atomic_store_n(&Q.sleeping, 0, __ATOMIC_RELAXED); continue; } @@ -200,6 +219,10 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { rec_chk = rt_knob("REC_CHECK", "0"); // debug switches, read before the recorder starts const char *d = cfg_get("TOTK_REC_DELAY_US"); rec_delay_ns = d && *d && atoi(d) > 0 ? (uint64_t)atoi(d) * 1000 : 0; + const char *sp = cfg_get("TOTK_REC_SPIN_US"); + rec_spin_ab = sp && !strcmp(sp, "ab"); + int spin_us = sp && *sp && !rec_spin_ab ? atoi(sp) : 60; + rec_spin_ns = spin_us > 0 && plat_cpu_usable() >= 6 ? (uint64_t)spin_us * 1000 : 0; const char *pw = cfg_get("TOTK_REC_PUB"); // words per publish (pub); 0: every packet Q.pub_words = rec_chk ? 0 : pw && *pw ? (uint64_t)atoi(pw) : 64; if (Q.pub_words > RING_WORDS / 8) Q.pub_words = RING_WORDS / 8; @@ -211,6 +234,11 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) { rt_log("vk_rec: the replay's Vulkan commands are made by a recorder thread%s", mode == 2 ? " in the B blocks (TOTK_AB)" : ""); if (rec_chk) rt_log("vk_rec: TOTK_REC_CHECK: deferred and destroyed objects are searched in the ring"); + if (rec_spin_ns) + rt_log("vk_rec: the recorder polls %llu us before it sleeps%s (TOTK_REC_SPIN_US)", + (unsigned long long)(rec_spin_ns / 1000), rec_spin_ab ? " in the B blocks" : ""); + else + rt_log("vk_rec: the recorder polls 4096 pauses before it sleeps (TOTK_REC_SPIN_US)"); if (Q.pub_words != 64) rt_log("vk_rec: TOTK_REC_PUB: head published every %llu words", (unsigned long long)Q.pub_words); if (rec_delay_ns) rt_log("vk_rec: TOTK_REC_DELAY_US: the recorder waits %llu us before each batch", @@ -315,10 +343,12 @@ static void submit_run(const A_Submit *a) { void rec_stats_log(void) { if (!(rt_trace & RT_TRACE_PERF) || !RS_.parts) return; rt_log("vk_rec: %llu of %llu parts recorded by the recorder: %llu packets (%.1f MiB), %llu drains (%llu for a call " - "without a packet) waited %.2f ms in all, ring full %llu times, %llu submissions made by the recorder", + "without a packet) waited %.2f ms in all, ring full %llu times, %llu submissions made by the recorder, " + "recorder slept %llu times", (unsigned long long)RS_.parts_on, (unsigned long long)RS_.parts, (unsigned long long)RS_.packets, RS_.words * 8 / 1048576.0, (unsigned long long)RS_.drains, (unsigned long long)RS_.fallbacks, - RS_.drain_ns / 1e6, (unsigned long long)RS_.full_waits, (unsigned long long)RS_.submits); + RS_.drain_ns / 1e6, (unsigned long long)RS_.full_waits, (unsigned long long)RS_.submits, + (unsigned long long)__atomic_exchange_n(&r_sleeps, 0, __ATOMIC_RELAXED)); memset(&RS_, 0, sizeof RS_); }