diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index a5bcbe3e..82dce1dd 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -49,7 +49,7 @@ uint64_t nvn_new_serial(void); // next tex_t / prog_ // that assumption too early. void nvn_sync_ahead_present(void); uint64_t nvn_sync_ahead_guard(void); -int nvn_sync_ahead_on(void); // 1 when the frame-ahead wait is in effect: TOTK_SYNC_AHEAD on and not refused (no shadow storage, no known wait site) +int nvn_sync_ahead_on(void); // 1 when the frame-ahead wait is in effect: TOTK_SYNC_AHEAD=1 and a wait site known for this version int vk_mem_shadow_on(void); // vk_memory.c: 1 with shadow storage on (valid once the Vulkan back end is up) void vk_mem_shadow_fence(void); // vk_memory.c: returns once every shadow sync that read the old guard is done // vk_memory.c: the game is about to read [addr, addr+size) of pool storage; waits for a GPU write into it that the game diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 5d6a1879..afa2da6c 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -483,10 +483,17 @@ NVN_API(nvnQueueFenceSync) { // GPU (about 11 ms from the submit), and the wait sets the frame time. With the switch on, that wait, and only that // one, waits for the frame before it instead. What the game reads back of GPU results after the wait can then be a // frame old; a buffer that holds no older result is waited for when the game invalidates it (vk_memory.c -// vk_mem_shadow_read). The game's CPU writes to memory the GPU still reads reach the GPU only through shadow copies, which are -// queued for the GPU or wait (nvn_sync_ahead_guard). The wait still lasts until the replay thread submitted the asked -// value, so no copy of that value itself picks up those writes. pa_fence: the newest fence at the latest present and -// the one before, present thread only for writes. pa_guard: the newest value the game was told is done early. +// vk_mem_shadow_read). The wait still lasts until the replay thread submitted the asked value, so every copy the +// replay makes of the game's memory at record time (uniform snapshots, CPU texture uploads, the staged mirror syncs) +// holds the asked frame's bytes before the game writes the next. Two forms, by shadow storage: +// - shadow on: the game's CPU writes to memory the GPU still reads reach the GPU only through shadow copies, which are +// queued for the GPU or wait (nvn_sync_ahead_guard). +// - shadow off (in place, Savage Wind's form): the GPU reads the game's own memory; the back end's guards (vk_ahead.c +// vk_ahead_init) keep its GPU-timeline reads off memory the game writes after the wait, and a value whose result the +// game reads at once (vk_ahead_oneshot: a one-off small target, a samples-passed report with +// TOTK_SYNC_AHEAD_QUERIES) is waited for, only up to that value. +// pa_fence: the newest fence at the latest present and the one before, present thread only for writes. pa_guard: the +// newest value the game was told is done early. static uint64_t pa_fence[2], pa_guard; // The return address of that wait. 1.1.0: 0x7000b39aa4, 1.2.1: 0x7000b23dfc, 1.4.3: 0x70002db99c. The second // nvnSyncWait in 1.1.0 f_7000b39a00 and 1.4.3 f_70002db8ac uses the per-frame sync, like on 1.2.1; the first @@ -502,14 +509,6 @@ static int sync_mode(void) { if (mode < 0) { const char *v = cfg_value("TOTK_SYNC_AHEAD"); int on = strcmp(v, "0") != 0 && strcmp(v, "check") != 0, check = !strcmp(v, "check"); - // Without shadow storage the GPU reads the game's own memory, which the game writes ahead: nothing guards - // that. The Vulkan back end sets the mode while it starts, before the game's first present or wait; the null - // back end never does. - if (on && !vk_mem_shadow_on()) { - rt_log("nvn: TOTK_SYNC_AHEAD needs shadow storage (shadow = 1 in totk.ini or TOTK_SHADOW=1 on native Windows); " - "the game waits for the GPU as before"); - on = 0; - } if ((on || check) && !frame_wait_lr()) { rt_log("nvn: TOTK_SYNC_AHEAD: no frame wait site is known for this game version; the game waits for the " "GPU as before"); @@ -518,7 +517,11 @@ static int sync_mode(void) { // The depth query patch (set qstamp, patches-v143.txt) is switched by TOTK_SYNC_AHEAD too: a refused switch // leaves the game's own frame number check, as with the switch off. if (!on) rt_patch_switch(rt_patch_set("qstamp"), 0); - if (on) rt_log("nvn: TOTK_SYNC_AHEAD: a wait for the frame presented last waits for the frame before it"); + // The Vulkan back end decides the mode while it starts (vk_memory.c mirror_mode), after it knows whether shadow + // storage is on, before the game's first present or wait. + if (on) + rt_log("nvn: TOTK_SYNC_AHEAD: a wait for the frame presented last waits for the frame before it (%s)", + vk_mem_shadow_on() ? "shadow storage" : "in place, with the back end's guards"); mode = on ? 1 : check ? 2 : 0; } return mode; @@ -542,12 +545,18 @@ NVN_API(nvnSyncWait) { uint64_t want = s->fence, lag = s->fence; // Only the game's per-frame wait lags, and not a poll (timeout 0). int mode = timeout && c->x[30] == frame_wait_lr() ? sync_mode() : 0, behind = b->fence_done() < want; - if (mode == 2) vk_ahead_frame_entry(); + // The detector (vk_ahead.c): check mode, or the switch on with TOTK_SYNC_AHEAD_CHECK=1. + int det = mode && vk_ahead_on; + if (det) vk_ahead_frame_entry(); // check: every frame as if the GPU were behind (the question is what the game writes, not this machine's timing) if (mode == 2 || (mode && behind)) { uint64_t p0 = __atomic_load_n(&pa_fence[0], __ATOMIC_ACQUIRE); uint64_t p1 = __atomic_load_n(&pa_fence[1], __ATOMIC_ACQUIRE); if (want <= p0 && want > p1) lag = p1; + if (mode == 1 && lag < want) { // a value in (lag, want] whose result the game reads at once: up to it + uint64_t o = vk_ahead_oneshot(lag, want); + if (o > lag) lag = o; + } if (mode == 1) want = lag; } if (want != s->fence) { // the replay of the asked value is done first: the guard then covers every copy of it @@ -561,9 +570,9 @@ NVN_API(nvnSyncWait) { vk_mem_shadow_fence(); // a shadow sync that read the old guard finishes before the game writes again } if (b->fence_done() >= want) { - if (mode == 2) vk_ahead_frame_lag(s->fence, lag, behind); + if (det) vk_ahead_frame_lag(s->fence, lag, behind); b->cpu_synced(want == s->fence ? s->fence : b->fence_done()); - if (mode == 2) vk_ahead_frame_done(1); + if (det) vk_ahead_frame_done(1); ret_x(c, NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED); return; } @@ -572,9 +581,9 @@ NVN_API(nvnSyncWait) { NVN_BLOCKING(); ok = b->fence_wait(want, timeout); } - if (mode == 2) vk_ahead_frame_lag(s->fence, lag, behind); + if (det) vk_ahead_frame_lag(s->fence, lag, behind); if (ok) b->cpu_synced(want == s->fence ? s->fence : b->fence_done()); - if (mode == 2) vk_ahead_frame_done(ok); + if (det) vk_ahead_frame_done(ok); ret_x(c, ok ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED); } // void nvnEventBuilderSetDefaults(NVNeventBuilder *b) diff --git a/runtime/vk.h b/runtime/vk.h index b9179a1e..02c5a766 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -282,6 +282,7 @@ uint32_t vk_mem_ahead_observe(const uint64_t (*iv)[2], int n, uint8_t *untracked uint64_t vk_mem_ahead_written(uint64_t lo, uint64_t hi, uint32_t since, void (*fn)(uint64_t, uint64_t, void *), void *arg); int vk_mem_pool_info(uint64_t addr, uint64_t *lo, uint64_t *size, uint32_t *flags, int *mirrored); // 0: no pool +void vk_mem_ahead_log(void); // TOTK_SYNC_AHEAD=1 in place: the guards' waits and copies over the run, at exit void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size); // A shader's storage-buffer write: goes back through the write-back merge (vk_memory.c), which keeps the words the GPU // did not change. @@ -531,6 +532,9 @@ uint64_t vk_texture_last_write(const tex_t *t); // timeline value of the newe size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img); // bytes ReadTexels spans; 0 for a bad region int vk_texture_small_dirty(void); // a small mirrored target holds GPU texels its storage lacks void vk_texture_write_back_small(uint64_t fence); // writes those rendered by work up to fence back to storage +// TOTK_SYNC_AHEAD=1 in place: the newest value in (lo, hi] that rendered a dirty small target not written back in the +// last 16 frames (a one-off read), else 0 +uint64_t vk_texture_oneshot(uint64_t lo, uint64_t hi); // vk_sync_submit, after the replay recorded value fence into cb (no render pass open, cb the recorder's part): copies // the small targets that value rendered into staging so their write-back later needs no GPU wait (TOTK_WB_EAGER). // cb VK_NULL_HANDLE: the value recorded no part, the candidates for it are dropped. diff --git a/runtime/vk_ahead.c b/runtime/vk_ahead.c index 64c56d78..ed69e732 100644 --- a/runtime/vk_ahead.c +++ b/runtime/vk_ahead.c @@ -96,15 +96,70 @@ static Offender off[OFF_N]; static int noff, nofflog; static uint64_t last_log_ns; +VkAheadGuards vk_ahead_g; +static uint64_t oneshot_q; // the newest value with a samples-passed report (TOTK_SYNC_AHEAD_QUERIES) +static uint64_t n_oneshot_tex, n_oneshot_q; // game thread: frame waits made exact up to a one-off value + void vk_ahead_init(void) { - vk_ahead_on = !strcmp(cfg_value("TOTK_SYNC_AHEAD"), "check"); + int check = !strcmp(cfg_value("TOTK_SYNC_AHEAD"), "check"); + memset(&vk_ahead_g, 0, sizeof vk_ahead_g); + if (!check && !vk_mem_shadow_on() && nvn_sync_ahead_on()) { + VkAheadGuards *g = &vk_ahead_g; + g->lag = 1; + g->msync = rt_knob("SYNC_AHEAD_MSYNC", "1"); + g->deswz = rt_knob("SYNC_AHEAD_DESWZ", "1"); + g->reads = rt_knob("SYNC_AHEAD_READS", "1"); + g->oneshot = rt_knob("SYNC_AHEAD_ONESHOT", "1"); + // Off: TotK reads its samples-passed reports two updates after it asked (1.2.1 f_70010c3298: a query's state + // goes 1 -> 2 or 3 at one update, 2 -> 3 at the next, and the report is read in state 3), by then the frame + // wait one frame late has covered that frame. Savage Wind (BOTW reads them right after the wait) has it on Mali. + g->queries = rt_knob("SYNC_AHEAD_QUERIES", "0"); + rt_log("sync-ahead: in place (no shadow storage), memory %s; guards: mirror syncs staged %d, deswizzle of " + "rewritten textures on the CPU %d, invalidates wait for the writer %d, one-off small targets %d, " + "samples-passed reports %d", vk_mirror_on ? "mirror" : "import", g->msync, g->deswz, g->reads, g->oneshot, + g->queries); + if (!vk_mirror_on) + rt_log("sync-ahead: warning: import memory: vertex, index and storage data are read in place on the GPU, " + "no guard covers them"); + } + vk_ahead_on = check || (vk_ahead_g.lag && rt_knob("SYNC_AHEAD_CHECK", "0")); if (!vk_ahead_on) return; notes = malloc(sizeof *notes * NOTE_MAX); take = malloc(sizeof *take * NOTE_MAX); if (!notes || !take) rt_fatal("vk: out of memory (sync-ahead check)"); - rt_log("sync-ahead check: the frame wait stays exact; logging what a one-frame-late wait without shadow storage " - "would race (%s)", vk_mem_shadow_on() ? "shadow storage is on: the counts describe the in-place form only " - "approximately" : "in place"); + if (vk_ahead_g.lag) + rt_log("sync-ahead check: with the late wait itself: logging what the CPU writes after the wait into memory " + "frames in flight still read on the GPU timeline, after the guards"); + else + rt_log("sync-ahead check: the frame wait stays exact; logging what a one-frame-late wait without shadow storage " + "would race (%s)", vk_mem_shadow_on() ? "shadow storage is on: the counts describe the in-place form only " + "approximately" : "in place"); +} + +void vk_ahead_query(uint64_t fence) { + uint64_t v = __atomic_load_n(&oneshot_q, __ATOMIC_RELAXED); + while (fence > v && !__atomic_compare_exchange_n(&oneshot_q, &v, fence, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {} +} + +uint64_t vk_ahead_oneshot(uint64_t lo, uint64_t hi) { + if (!vk_ahead_g.lag) return 0; + uint64_t v = 0; + if (vk_ahead_g.queries) { + uint64_t q = __atomic_load_n(&oneshot_q, __ATOMIC_ACQUIRE); + if (q > lo && q <= hi && q > vk_sync_fence_done()) v = q, n_oneshot_q++; + } + if (vk_ahead_g.oneshot && v < hi) { + uint64_t t = vk_texture_oneshot(v > lo ? v : lo, hi); + if (t) v = t, n_oneshot_tex++; + } + return v; +} + +void vk_ahead_log_guards(void) { + if (!vk_ahead_g.lag) return; + vk_mem_ahead_log(); + rt_log("sync-ahead: frame waits made exact up to a one-off value: %llu for small targets, %llu for " + "samples-passed reports", (unsigned long long)n_oneshot_tex, (unsigned long long)n_oneshot_q); } static uint64_t note_fence(uint64_t fence) { @@ -198,9 +253,10 @@ void vk_ahead_gpu_write(uint64_t addr, uint64_t size, int kind, uint64_t fence) void vk_ahead_invalidate(uint64_t addr, uint64_t size) { if (!vk_ahead_on || !size) return; plat_mutex_lock(&ah_lock); - uint64_t now = vk_sync_issued(), stale = 0; + // with the late wait itself, after the guard: a write done by now is no stale read + uint64_t now = vk_sync_issued(), stale = 0, done = vk_ahead_g.lag ? vk_sync_fence_done() : 0; for (int i = 0; i < ngw; i++) - if (gw[i].fence > win_lo && gw[i].fence <= win_hi && gw[i].lo < addr + size && gw[i].hi > addr) { + if (gw[i].fence > win_lo && gw[i].fence <= win_hi && gw[i].fence > done && gw[i].lo < addr + size && gw[i].hi > addr) { uint64_t a = gw[i].lo > addr ? gw[i].lo : addr, b = gw[i].hi < addr + size ? gw[i].hi : addr + size; stale += b - a; } diff --git a/runtime/vk_ahead.h b/runtime/vk_ahead.h index 8d4a1b9a..2c02dac0 100644 --- a/runtime/vk_ahead.h +++ b/runtime/vk_ahead.h @@ -1,5 +1,7 @@ // TOTK_SYNC_AHEAD=check (vk_ahead.c): measures what answering the frame wait one frame late without shadow storage -// would race, while the wait stays exact. Off: vk_ahead_on is 0 and every hook is one test of it. +// would race, while the wait stays exact. Off: vk_ahead_on is 0 and every hook is one test of it. The same detector +// runs with the late wait itself (TOTK_SYNC_AHEAD=1, shadow off) under TOTK_SYNC_AHEAD_CHECK=1. +// TOTK_SYNC_AHEAD=1 without shadow storage: vk_ahead_g.lag, and the guards of that form (vk_ahead_init). #pragma once #include @@ -12,7 +14,19 @@ enum { AH_VB, AH_IB, AH_UBO, AH_SSBO, AH_IND, AH_COPY, AH_MSYNC, AH_DESWZ, AH_WB enum { AHW_WRITE, AHW_COUNTER, AHW_SAMPLES, AHW_EVENT, AHW_N }; extern int vk_ahead_on; -void vk_ahead_init(void); // vk_memory.c mirror_mode, once at device start: reads TOTK_SYNC_AHEAD +// The in-place form's switches, read once by vk_ahead_init; every guard is 0 without lag. A/B: TOTK_SYNC_AHEAD_. +// msync: the mirror syncs copy the synced pages into staging on the CPU at submit (vk_memory.c vk_mem_mirror_prologue); +// deswz: a texture re-uploaded after the CPU rewrote it takes the CPU deswizzle (vk_texture.c upload_gpu); reads: an +// invalidate of a range not read lately waits for the pending GPU write into it (vk_memory.c vk_mem_shadow_read); +// oneshot: a one-off small target is waited for, up to its value (vk_ahead_oneshot); queries (default 0): so is a +// samples-passed report. +typedef struct { int lag, msync, deswz, reads, oneshot, queries; } VkAheadGuards; +extern VkAheadGuards vk_ahead_g; +void vk_ahead_init(void); // vk_memory.c mirror_mode, once at device start, shadow mode known: reads TOTK_SYNC_AHEAD +// The newest value in (lo, hi] whose GPU result the game reads right after the frame wait (0: none): the in-place +// frame wait waits up to it. Game thread. +uint64_t vk_ahead_oneshot(uint64_t lo, uint64_t hi); +void vk_ahead_query(uint64_t fence); // a samples-passed report into CPU-visible memory, by the value being recorded // [addr, addr+size) of guest memory is read (or, for WB and WBM, written) on the GPU timeline by the submission of value // fence (0: the one being recorded, else the next one). void vk_ahead_read(uint64_t addr, uint64_t size, int cls, uint64_t fence); @@ -29,4 +43,5 @@ void vk_ahead_frame_entry(void); void vk_ahead_frame_lag(uint64_t asked, uint64_t lagged, int behind); void vk_ahead_frame_done(int ok); void vk_ahead_log_final(void); // at exit +void vk_ahead_log_guards(void); // at exit: what the in-place form's waits did int vk_ahead_selftest(uint64_t addr, uint64_t size); // --probe-vkmem: 0 passed diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index bde23fbe..8efaa1bb 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -479,6 +479,7 @@ void gfx_exit_log(void) { vk_texture_eager_log(); vk_mem_shadow_read_log(); vk_ahead_log_final(); + vk_ahead_log_guards(); if (gfx_replay_async() && r_started && !plat_xchg32(&done, 1)) replay_log(game_frames, 1); } diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index bb8723ac..cd001345 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -292,6 +292,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { rt_fatal("vk: counter report at %#llx is in no memory pool", (unsigned long long)p->addr); vk_mem_count(m.cls, VK_MUSE_GPU_WRITE, sizeof(NVNcounterData)); if (vk_ahead_on) vk_ahead_gpu_write(p->addr, sizeof(NVNcounterData), samples ? AHW_SAMPLES : AHW_COUNTER, ex->fence); + if (samples && vk_ahead_g.queries) vk_ahead_query(ex->fence); // the in-place frame wait waits up to it if (samples) oq_report(ex, p->addr); else { uint64_t *a = malloc(sizeof *a); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 1e17ee7c..109fef9d 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1373,7 +1373,80 @@ static SRead sread[SREAD_N]; // shadow_lock: the ranges the gam static uint64_t s_read_waits, s_read_ticks; // shadow_lock: since the last log static uint64_t r_read_waits, r_read_ticks, r_read_max; // shadow_lock: over the run (vk_mem_shadow_read_log) +// ---------------------------------------------------------------- TOTK_SYNC_AHEAD=1 in place (vk_ahead.h) +// Without shadow storage a GPU write into CPU-visible memory lands in guest memory on the GPU timeline: directly, or +// through the mirror's write-back at the end of its submission (vk_mem_mirror_end). Once the frame wait returns a frame +// early those bytes can still be on their way. aw: the writes recorded and not known done, {lo, hi, fence}; pruned +// when a value completes (aw_prune), bounded: a full list joins a new write with its nearest entry (wider, never +// narrower, so a reader waits for at least what it must). Used by the invalidates below and the staged mirror syncs. +#define AW_N 2048 +typedef struct { uint64_t lo, hi, fence; } AWrite; +static PlatMutex aw_lock = PLAT_MUTEX_INIT; // after every other lock of this file +static AWrite aw[AW_N]; +static int naw, aw_peak; +static uint64_t aw_joined; +// Run totals (aw_lock): invalidate waits, staged sync bytes and calls, CPU time of the copies, waits for an earlier +// write-back into a synced range +static uint64_t ah_rd_waits, ah_rd_ns, ah_rd_max, ah_stage_bytes, ah_stage_n, ah_stage_ns, ah_ov_waits, ah_ov_ns, ah_ov_max; +static uint64_t ah_inplace_bytes; // mirror sync bytes still copied from guest memory on the GPU (TOTK_SYNC_AHEAD_MSYNC=0) + +static void aw_prune(uint64_t done) { // aw_lock + int k = 0; + for (int i = 0; i < naw; i++) + if (aw[i].fence > done) aw[k++] = aw[i]; + naw = k; +} + +static void aw_add(uint64_t lo, uint64_t hi, uint64_t fence) { + if (hi <= lo) return; + plat_mutex_lock(&aw_lock); + for (int i = naw; i-- > 0 && i >= naw - 16;) // the same range again, as the replay notes it draw after draw + if (aw[i].lo == lo && aw[i].hi == hi) { + if (fence > aw[i].fence) aw[i].fence = fence; + plat_mutex_unlock(&aw_lock); + return; + } + if (naw == AW_N) aw_prune(vk_sync_fence_done()); + if (naw == AW_N) { + int m = 0; + uint64_t best = UINT64_MAX; + for (int i = 0; i < naw; i++) { + uint64_t d = aw[i].hi < lo ? lo - aw[i].hi : aw[i].lo > hi ? aw[i].lo - hi : 0; + if (d < best) best = d, m = i; + } + if (lo < aw[m].lo) aw[m].lo = lo; + if (hi > aw[m].hi) aw[m].hi = hi; + if (fence > aw[m].fence) aw[m].fence = fence; + aw_joined++; + } else { + aw[naw++] = (AWrite){lo, hi, fence}; + if (naw > aw_peak) aw_peak = naw; + } + plat_mutex_unlock(&aw_lock); +} + +// aw_lock: the newest value in (done, below) that writes into [lo, hi), 0: none +static uint64_t aw_writer(uint64_t lo, uint64_t hi, uint64_t done, uint64_t below) { + uint64_t w = 0; + for (int i = 0; i < naw; i++) + if (aw[i].fence > done && aw[i].fence < below && aw[i].fence > w && aw[i].lo < hi && aw[i].hi > lo) w = aw[i].fence; + return w; +} + +static void ah_time(uint64_t *n, uint64_t *ns, uint64_t *mx, uint64_t dt) { + plat_mutex_lock(&aw_lock); + (*n)++, *ns += dt; + if (mx && dt > *mx) *mx = dt; + plat_mutex_unlock(&aw_lock); +} + +// The in-place form's vk_mem_shadow_read: the rule of the shadow form (below), with the writer's own value waited for +// instead of the guard, which the writer may be far behind of. Only values up to the guard: a write of a value the game +// was not told is done is the game reading ahead of its own wait. +static void ahead_read(uint64_t addr, uint64_t size); + void vk_mem_shadow_read(uint64_t addr, uint64_t size) { + if (vk_ahead_g.reads && size) ahead_read(addr, size); if (vk_ahead_on) vk_ahead_invalidate(addr, size); if (!shadow_on || !size || !nvn_sync_ahead_guard()) return; plat_mutex_lock(&shadow_lock); @@ -1404,6 +1477,44 @@ void vk_mem_shadow_read(uint64_t addr, uint64_t size) { plat_mutex_unlock(&shadow_lock); } +static void ahead_read(uint64_t addr, uint64_t size) { + uint64_t g = nvn_sync_ahead_guard(); + if (!g) return; + uint64_t done = vk_sync_fence_done(); + plat_mutex_lock(&shadow_lock); // sread: the shadow form's table, unused without shadow storage + uint64_t now = vk_sync_issued(); + SRead *e = &sread[0]; + for (int i = 0; i < SREAD_N; i++) { + if (sread[i].addr == addr && sread[i].size == size) { e = &sread[i]; break; } + if (sread[i].at < e->at) e = &sread[i]; + } + int known = e->addr == addr && e->size == size && e->at + SREAD_AGE >= now; + plat_mutex_lock(&aw_lock); + aw_prune(done); + uint64_t w = aw_writer(addr, addr + size, done, UINT64_MAX), wait = !known && w && w <= g ? w : 0; + plat_mutex_unlock(&aw_lock); + if (known || !w || wait) *e = (SRead){addr, size, now}; + plat_mutex_unlock(&shadow_lock); + if (!wait) return; + uint64_t t0 = qpc(); + { + NVN_BLOCKING(); + vk_sync_fence_wait(wait, UINT64_MAX); + } + ah_time(&ah_rd_waits, &ah_rd_ns, &ah_rd_max, qpc() - t0); +} + +void vk_mem_ahead_log(void) { + plat_mutex_lock(&aw_lock); + rt_log("mem: sync-ahead in place: the game waited for a GPU write it reads %llu times, %.1f ms in all, longest " + "%.2f ms; mirror syncs staged on the CPU %.1f MiB in %llu prologues, %.1f ms, waited for an earlier " + "write-back %llu times, %.1f ms, longest %.2f ms; synced in place %.1f MiB; pending writes at most %d, %llu " + "joined", (unsigned long long)ah_rd_waits, qpc_ms(ah_rd_ns), qpc_ms(ah_rd_max), ah_stage_bytes / 1048576.0, + (unsigned long long)ah_stage_n, qpc_ms(ah_stage_ns), (unsigned long long)ah_ov_waits, qpc_ms(ah_ov_ns), + qpc_ms(ah_ov_max), ah_inplace_bytes / 1048576.0, aw_peak, (unsigned long long)aw_joined); + plat_mutex_unlock(&aw_lock); +} + // The run total of the waits above, once at exit, while frame-ahead is in effect: without TOTK_TRACE=perf the shadow // log lines come only at frames 600 and 1800. void vk_mem_shadow_read_log(void) { @@ -1450,6 +1561,10 @@ int vk_mem_shadow_deferred(void) { return ndq; } // have written those shadow bytes since. void vk_mem_gpu_pending(uint64_t addr, uint64_t size) { if (vk_ahead_on) vk_ahead_gpu_write(addr, size, AHW_WRITE, 0); + if (vk_ahead_g.lag && size) { + uint64_t f = vk_exec_recording_fence(); + aw_add(addr, addr + size, f ? f : vk_sync_issued() + 1); + } if (!shadow_on || !size) return; uint64_t lo = addr, hi = addr + size; plat_mutex_lock_shared(&pool_lock); @@ -2429,17 +2544,111 @@ int vk_mem_mirror_pending(void) { return n; } +#define STG_BLOCK (64ull << 20) // staging pool blocks (vk_staging) +typedef struct { VkBuffer b; VmaAllocation a; } StgFree; +static void free_shadow_stg(void *arg) { + StgFree *f = arg; + vmaDestroyBuffer(vk_ctx()->vma, f->b, f->a); + free(f); +} + +// TOTK_SYNC_AHEAD=1 in place (vk_ahead_g.msync): the sealed sync ranges a[0..n) are copied out of guest memory into +// staging now, on the replay thread, and the GPU copies the staging into the mirrors: the frame wait returns only after +// the replay submitted the value (nvn_api_core.c gfx_replay_wait_value), so the GPU gets this frame's bytes even when +// it runs the copy after the game wrote the next frame's. A range an earlier value still writes back into (aw) would +// lose those GPU words, which a copy on the GPU timeline would have read after them: that value is waited for first +// (mirror_lock released meanwhile; the sealed entries stay, as only the prologue takes them). Staging buffers of at +// most STG_BLOCK / 2, freed when fence completes. Replay thread, mirror_lock held. +static void stage_ranges(VkCommandBuffer cb, MRange *a, int n, uint64_t fence) { + uint64_t done = vk_sync_fence_done(), w = 0; + plat_mutex_lock(&aw_lock); + aw_prune(done); + for (int i = 0; i < n && naw; i++) { + uint64_t lo = a[i].r->import_lo + a[i].off, x = aw_writer(lo, lo + a[i].size, done, fence); + if (x > w) w = x; + } + plat_mutex_unlock(&aw_lock); + if (w) { + plat_mutex_unlock(&mirror_lock); + uint64_t t0 = qpc(); + vk_sync_fence_wait(w, UINT64_MAX); + ah_time(&ah_ov_waits, &ah_ov_ns, &ah_ov_max, qpc() - t0); + plat_mutex_lock(&mirror_lock); + a = pend; // the list may have grown meanwhile + } + uint64_t t0 = qpc(), bytes = 0; + qsort(a, (size_t)n, sizeof *a, range_cmp); + const uint64_t lim = STG_BLOCK / 2; + int i = 0; + uint64_t at_i = 0; // bytes of a[i] already staged + while (i < n) { + uint64_t total = 0, jo = at_i; + for (int j = i; j < n && total < lim;) { + uint64_t lim_j = a[j].r->import_hi - a[j].r->import_lo, sz = a[j].off >= lim_j ? 0 + : a[j].off + a[j].size > lim_j ? lim_j - a[j].off : a[j].size; + uint64_t left = sz - jo, take = left < lim - total ? left : lim - total; + total += take; + if (take == left) j++, jo = 0; + else jo += take; + } + if (!total) break; + VkBuffer sb; + VmaAllocation sa; + uint8_t *p = vk_staging(total, &sb, &sa); + int k = 0; + PoolRec *kr = NULL; + for (uint64_t at = 0; at < total;) { + PoolRec *r = a[i].r; + uint64_t lim_i = r->import_hi - r->import_lo, sz = a[i].off >= lim_i ? 0 + : a[i].off + a[i].size > lim_i ? lim_i - a[i].off : a[i].size; + uint64_t left = sz - at_i, take = left < total - at ? left : total - at; + if (take) { + if (kr != r && k) vkCmdCopyBuffer(cb, sb, kr->mbuf, (uint32_t)k, mregions), k = 0; + kr = r; + memcpy(p + at, (const void *)(uintptr_t)(r->import_lo + a[i].off + at_i), take); + if (k == cregions) grow((void **)&mregions, &cregions, sizeof *mregions); + mregions[k++] = (VkBufferCopy){at, a[i].off + at_i, take}; + } + at += take, at_i += take; + if (at_i == sz) i++, at_i = 0; + } + if (k) vkCmdCopyBuffer(cb, sb, kr->mbuf, (uint32_t)k, mregions); + vmaFlushAllocation(vk_ctx()->vma, sa, 0, VK_WHOLE_SIZE); + StgFree *f = malloc(sizeof *f); + if (!f) rt_fatal("vk: out of memory"); + f->b = sb, f->a = sa; + vk_completion_add(fence, free_shadow_stg, f); + bytes += total; + mrep.upload_copies += (uint64_t)k; + } + mrep.upload_bytes += bytes; + uint64_t dt = qpc() - t0; + plat_mutex_lock(&aw_lock); + ah_stage_bytes += bytes, ah_stage_n++, ah_stage_ns += dt; + plat_mutex_unlock(&aw_lock); +} + void vk_mem_mirror_prologue(VkCommandBuffer cb, uint64_t fence) { plat_mutex_lock(&mirror_lock); if (!pend_seal && snl_copied == snl_seal) { plat_mutex_unlock(&mirror_lock); return; } - if (vk_ahead_on) // the copies read guest memory when the GPU runs them + int staged = vk_ahead_g.msync && pend_seal; + if (vk_ahead_on && !staged) // the copies read guest memory when the GPU runs them for (int i = 0; i < pend_seal; i++) vk_ahead_read(pend[i].r->import_lo + pend[i].off, pend[i].size, AH_MSYNC, fence); mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); - copy_ranges(cb, pend, pend_seal, 1, &mrep.upload_bytes, &mrep.upload_copies); + if (staged) stage_ranges(cb, pend, pend_seal, fence); + else { + uint64_t b0 = mrep.upload_bytes; + copy_ranges(cb, pend, pend_seal, 1, &mrep.upload_bytes, &mrep.upload_copies); + if (vk_ahead_g.lag) { + plat_mutex_lock(&aw_lock); + ah_inplace_bytes += mrep.upload_bytes - b0; + plat_mutex_unlock(&aw_lock); + } + } if (snl_copied < snl_seal) { // the snapshots read what the uploads above wrote if (pend_seal) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); @@ -2550,6 +2759,9 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT); int m = wb_merge_record(cb); + if (m && vk_ahead_g.lag) + for (int i = 0; i < nsnl; i++) + aw_add(snl[i].r->import_lo + snl[i].off, snl[i].r->import_lo + snl[i].off + snl[i].size, fence); if (m && vk_ahead_on) // the merge writes guest memory when the GPU runs it for (int i = 0; i < nsnl; i++) vk_ahead_read(snl[i].r->import_lo + snl[i].off, snl[i].size, AH_WBM, fence); if (!m) { // no range list: copy them plainly @@ -2571,6 +2783,9 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); for (int i = 0; i < nwbl; i++) wbl[i].r->wb_sub = msub; + if (vk_ahead_g.lag) + for (int i = 0; i < nwbl; i++) + aw_add(wbl[i].r->import_lo + wbl[i].off, wbl[i].r->import_lo + wbl[i].off + wbl[i].size, fence); if (vk_ahead_on) for (int i = 0; i < nwbl; i++) vk_ahead_read(wbl[i].r->import_lo + wbl[i].off, wbl[i].size, AH_WB, fence); mrep.wb_ranges += (uint64_t)nwbl; @@ -2843,7 +3058,6 @@ void vk_upload_end(void) { plat_rmutex_unlock(&cmd_lock); } // the headroom thread (vk_headroom.c) or a burst of uploads held every staging allocation of the render thread: one // took 47-50 ms, for a few bytes, in the first frame after a save load. Larger buffers than a block go to the default // pool. -#define STG_BLOCK (64ull << 20) #define STG_MIN_BLOCKS 2 static VmaPool stg_pool; static int stg_pool_tried; @@ -2911,12 +3125,6 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { return info.pMappedData; } -typedef struct { VkBuffer b; VmaAllocation a; } StgFree; -static void free_shadow_stg(void *arg) { - StgFree *f = arg; - vmaDestroyBuffer(vk_ctx()->vma, f->b, f->a); - free(f); -} // Records the deferred shadow copies (vk_mem_shadow_sync queued the ranges): the guest bytes go into staging buffers // now, the GPU copies them into the pools' shadows. The barrier before waits for every earlier command, writes and diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index b7761fac..9bc6300d 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1035,9 +1035,11 @@ static void upload_cpu(Tex *e) { #define DESWZ_CHUNK (32ull << 20) // Uploads by path since the last perf summary (TOTK_TRACE=perf): 0 GPU, else why the CPU took it. -enum { UP_GPU, UP_MODE, UP_LINEAR, UP_RGB, UP_ASTC_CPU, UP_BPB, UP_NOT_LIVE, UP_GEOM, UP_LIMITS, UP_SCRATCH, UP_WHY }; +enum { UP_GPU, UP_MODE, UP_LINEAR, UP_RGB, UP_ASTC_CPU, UP_BPB, UP_NOT_LIVE, UP_GEOM, UP_LIMITS, UP_SCRATCH, UP_AHEAD, + UP_WHY }; static const char *const up_why_name[UP_WHY] = {"gpu", "cpu mode", "linear", "rgb", "astc on cpu", "bytes per block", - "not an imported pool", "layout", "device limits", "scratch"}; + "not an imported pool", "layout", "device limits", "scratch", + "frame ahead"}; static uint64_t up_n[UP_WHY], up_bytes[UP_WHY], up_ns[UP_WHY]; static int up_why; // upload_gpu's reason for returning 0 (texture lock held) @@ -1049,6 +1051,10 @@ static int upload_gpu(Tex *e) { int mode = vk_deswz_mode(); up_why = mode == VK_DESWZ_CPU ? UP_MODE : nvn_tex_is_linear(d) ? UP_LINEAR : e->f.expand_rgb ? UP_RGB : UP_GPU; if (up_why) return 0; + // TOTK_SYNC_AHEAD=1 in place (vk_ahead.h): the batch reads the pool when the GPU runs it, which can be after the + // game was told the frame is done and writes the storage again. A texture the CPU rewrote after its last upload is + // one the game writes while the game runs: deswizzled on the CPU now, from the bytes of this frame. + if (vk_ahead_g.deswz && e->uploaded) return up_why = UP_AHEAD, 0; int astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu(); if (astc && (!agpu || vk_astc_mode() == VK_ASTC_CHECK)) return up_why = UP_ASTC_CPU, 0; // or ASTC's check mode int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); @@ -2489,6 +2495,41 @@ uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size) { return f; } +// TOTK_SYNC_AHEAD=1 in place: the small targets written back lately, with the frame of the write-back. A target not in +// the table within ONEOFF_FRAMES is a one-off read (a photo, a pick), which the frame wait waits for up to the value +// that rendered it (vk_texture_oneshot); a target read back every frame or in turns gets the frame before. Adapted from +// Savage Wind Recomp runtime/nvn/vulkan/exec.c readback_schedule (https://github.com/Revivedx/SavageWindRecomp @ +// bf6146c14c37fee8781a601e5a544b93dff6d75e, MIT, Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, +// Copyright (c) 2026 Peiikai). tex_lock. +#define ONEOFF_N 64 +#define ONEOFF_FRAMES 16 +static struct { uint64_t addr, frame; } oneoff_last[ONEOFF_N]; + +static int oneoff_streaming(uint64_t addr, uint64_t fr, int note) { + int k = 0; + while (k < ONEOFF_N - 1 && oneoff_last[k].addr && oneoff_last[k].addr != addr) k++; + int streaming = oneoff_last[k].addr == addr && oneoff_last[k].frame + ONEOFF_FRAMES >= fr; + if (note) oneoff_last[k].addr = addr, oneoff_last[k].frame = fr; + return streaming; +} + +uint64_t vk_texture_oneshot(uint64_t lo, uint64_t hi) { + if (!__atomic_load_n(&small_pending, __ATOMIC_ACQUIRE)) return 0; // nothing dirty left for a later value + uint64_t fr = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), v = 0; + lock(); + wb_cfg_init(); + plat_mutex_lock_shared(&owned_lock); + for (int i = 0; i < nowned; i++) { + const Tex *e = owned[i]; + if (e->gpu_dirty && e->gpu_fence > lo && e->gpu_fence <= hi && e->gpu_fence > v && wb_wanted(e) && + !oneoff_streaming(e->pub.addr, fr, 0)) + v = e->gpu_fence; + } + plat_mutex_unlock_shared(&owned_lock); + unlock(); + return v; +} + // The image is the newest content only while nothing wrote the storage since the GPU render: TotK finalizes grass maps, // reuses their storage for new data and memcpys that data on, and writing the old image back first handed the copy // a stale map or stale resource bytes (wrong textures in the distance). Every hit is checked before any is written @@ -2511,6 +2552,10 @@ void vk_texture_write_back(uint64_t addr, uint64_t size) { } write_back_n(hit, k); for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB); + if (vk_ahead_g.oneshot) { + uint64_t fr = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + for (int i = 0; i < k; i++) oneoff_streaming(hit[i]->pub.addr, fr, 1); + } for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP); free(hit); unlock(); @@ -2585,6 +2630,10 @@ void vk_texture_write_back_small(uint64_t fence) { } write_back_n(hit, k); for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB); + if (vk_ahead_g.oneshot) { + uint64_t fr = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + for (int i = 0; i < k; i++) oneoff_streaming(hit[i]->pub.addr, fr, 1); + } if (vk_ahead_on) for (int i = 0; i < k; i++) vk_ahead_small(hit[i]->pub.addr, hit[i]->gpu_fence); for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP); diff --git a/tests/sync_ahead_check.py b/tests/sync_ahead_check.py index 192e6b56..d616dd92 100644 --- a/tests/sync_ahead_check.py +++ b/tests/sync_ahead_check.py @@ -1,4 +1,5 @@ -"""Exercise the production sync-ahead wait selection with a deterministic back end (TOTK_SYNC_AHEAD 0, 1, check). +"""Exercise the production sync-ahead wait selection with a deterministic back end (TOTK_SYNC_AHEAD 0, 1, check; shadow +storage on and off, the in-place form's wait up to a one-off value). python tests/sync_ahead_check.py [--cc path/to/clang] """ @@ -40,7 +41,13 @@ static const char *cfg_value(const char *name) { assert(!strcmp(name, "TOTK_SYNC_AHEAD")); return enabled == 2 ? "check" : enabled ? "1" : "0"; } -static int entry_calls, lag_calls, done_calls, done_ok; +static int entry_calls, lag_calls, done_calls, done_ok, oneshot_calls; +static int vk_ahead_on; // the detector: check mode, or the switch on with TOTK_SYNC_AHEAD_CHECK=1 (det) +static uint64_t oneshot, oneshot_lo, oneshot_hi; // the back end's one-off value (vk_ahead_oneshot: in (lo, hi] or 0) +static uint64_t vk_ahead_oneshot(uint64_t lo, uint64_t hi) { + oneshot_calls++, oneshot_lo = lo, oneshot_hi = hi; + return oneshot > lo && oneshot <= hi ? oneshot : 0; +} static uint64_t lag_asked, lag_value; static int lag_behind; static void vk_ahead_frame_entry(void) { assert(!lag_calls && !done_calls); entry_calls++; } @@ -70,8 +77,9 @@ PRODUCTION static uint32_t seed = 0x19ed634b; static uint32_t random_word(void) { seed ^= seed << 13; seed ^= seed >> 17; return seed ^= seed << 5; } int main(int argc, char **argv) { - assert(argc == 4); + assert(argc == 5); rt_game_version = atoi(argv[1]); enabled = atoi(argv[2]); shadow = atoi(argv[3]); + int det = atoi(argv[4]); int supported = rt_game_version == 110 || rt_game_version == 121 || rt_game_version == 143; uint64_t site = rt_game_version == 110 ? 0x7000b39aa4ull : rt_game_version == 121 ? 0x7000b23dfcull : rt_game_version == 143 ? 0x70002db99cull : 0; @@ -79,11 +87,12 @@ int main(int argc, char **argv) { qstamp_on = rt_game_version == 143 && enabled; last = 20; nvn_sync_ahead_present(); last = 40; nvn_sync_ahead_present(); - int on = enabled == 1 && shadow && supported, check = enabled == 2 && supported; + int on = enabled == 1 && supported, check = enabled == 2 && supported; // shadow off: the in-place form + vk_ahead_on = check || (on && det); assert(pa_fence[0] == (on || check ? 40 : 0)); assert(pa_fence[1] == (on || check ? 20 : 0)); assert(qstamp_on == (rt_game_version == 143 && on)); // a refused switch and check leave the game's check - int relaxed_cases = 0, polls = 0, other_waits = 0; + int relaxed_cases = 0, polls = 0, other_waits = 0, oneshot_cases = 0; for (int i = 0; i < 100000; i++) { sync_t sync = {.fenced = random_word() % 2, .fence = random_word() % 65}; uint64_t timeout = i % 3 == 0 ? 0 : i % 3 == 1 ? 1 : UINT64_MAX; @@ -91,25 +100,36 @@ int main(int argc, char **argv) { completed = random_word() % 65; uint64_t initial_done = completed, target = sync.fence; succeed = random_word() % 2; + oneshot = random_word() % 4 ? 0 : random_word() % 65; int lagcase = sync.fenced && timeout && lr == site && completed < target && target > 20 && target <= 40; - relaxed = on && lagcase; - int checked = check && sync.fenced && timeout && lr == site; + // the late wait waits for 20, or for a one-off value in (20, target] the back end names + uint64_t late = lagcase && oneshot > 20 && oneshot <= target ? oneshot : 20; + relaxed = on && lagcase && late < target; + int oneshot_asked = on && lagcase; + oneshot_cases += oneshot_asked && late > 20; + int checked = vk_ahead_on && sync.fenced && timeout && lr == site; int checkcase = checked && target > 20 && target <= 40; // check: as if the GPU were behind relaxed_cases += relaxed; polls += timeout == 0; other_waits += lr != site; - if (relaxed) target = 20; - replay_calls = shadow_calls = sync_calls = wait_calls = 0; + if (relaxed) target = late; + replay_calls = shadow_calls = sync_calls = wait_calls = oneshot_calls = 0; entry_calls = lag_calls = done_calls = 0; synced = waited = waited_timeout = replay_value = UINT64_MAX; pa_guard = 0; Cpu cpu = {0}; cpu.x[0] = (uintptr_t)&sync; cpu.x[1] = timeout; cpu.x[30] = lr; nvnSyncWait(&cpu); assert(replay_calls == relaxed && shadow_calls == relaxed); + assert(oneshot_calls == oneshot_asked); + if (oneshot_asked) assert(oneshot_lo == 20 && oneshot_hi == sync.fence); assert(entry_calls == checked && lag_calls == checked && done_calls == checked); - if (checked) { // exact wait, the late one's value told to the check + if (checked && check) { // exact wait, the late one's value told to the check assert(lag_asked == sync.fence && lag_value == (checkcase ? 20 : sync.fence)); assert(lag_behind == (initial_done < sync.fence)); assert(done_ok == (initial_done >= target || succeed)); } + if (checked && on) { // the late wait itself: the value it waited for told to the detector + assert(lag_asked == sync.fence && lag_value == (relaxed ? late : sync.fence)); + assert(done_ok == (initial_done >= target || succeed)); + } assert(pa_guard == (relaxed ? sync.fence : 0)); if (relaxed) assert(replay_value == sync.fence); if (!sync.fenced) { @@ -125,16 +145,18 @@ int main(int argc, char **argv) { } } assert(polls > 0 && other_waits > 0); - assert((relaxed_cases > 0) == on); + assert((relaxed_cases > 0) == on && (oneshot_cases > 0) == on); // Raising the guard for an older request must not lower a guard already published by another waiter. if (on) { sync_t sync = {.fenced = 1, .fence = 35}; - completed = 25; pa_guard = 39; relaxed = 1; + completed = 25; pa_guard = 39; relaxed = 1; oneshot = 0; + replay_calls = shadow_calls = sync_calls = wait_calls = entry_calls = lag_calls = done_calls = 0; Cpu cpu = {0}; cpu.x[0] = (uintptr_t)&sync; cpu.x[1] = UINT64_MAX; cpu.x[30] = site; nvnSyncWait(&cpu); assert(pa_guard == 39); } - printf("sync-ahead version %d enabled %d shadow %d: 100000 wait selections passed (%d relaxed)\n", rt_game_version, enabled, shadow, relaxed_cases); + printf("sync-ahead version %d enabled %d shadow %d detector %d: 100000 wait selections passed (%d relaxed, %d up " + "to a one-off value)\n", rt_game_version, enabled, shadow, det, relaxed_cases, oneshot_cases); } ''' @@ -155,7 +177,8 @@ def main(): for version in (110, 121, 143, 999): for enabled in (0, 1, 2): for shadow in (0, 1): - subprocess.run([str(exe), str(version), str(enabled), str(shadow)], check=True) + for det in (0, 1): + subprocess.run([str(exe), str(version), str(enabled), str(shadow), str(det)], check=True) if __name__ == "__main__":