From 14af7e7347540eb5a76801a06af80493b96b6bfa Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 25 Oct 2026 19:24:39 +0300 Subject: [PATCH] Replay NVN submissions on a queued replay thread behind TOTK_REPLAY_ASYNC, with drains for calls that read or free GPU state Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 1 + runtime/nvn_api.h | 5 + runtime/nvn_api_cmd.c | 80 ++++++++++++- runtime/nvn_api_core.c | 27 ++++- runtime/prof_win.c | 17 ++- runtime/vk.h | 2 + runtime/vk_backend.c | 254 +++++++++++++++++++++++++++++++++++++---- runtime/vk_sync.c | 17 ++- 8 files changed, 368 insertions(+), 35 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index 58e59ea1..3deb7aa2 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -28,6 +28,7 @@ void prof_write(void); // idempotent; boot.c calls it b uint64_t perf_begin(void); // start time of a wait; pair with perf_end void perf_end(int kind, uint64_t t0); // counts render-thread time only void perf_replay(uint64_t t0); // replay (NVN -> Vulkan recording) time on any thread, waits inside it included +void perf_replay_thread(void); // the calling thread replays for the render thread: perf_end counts its waits // Events per 60-frame window: graphics pipelines finished by the background workers, draws skipped while pending, // first texture uploads and re-uploads after CPU writes, draws skipped while a shader translates in the background, // draws that waited for their pending pipeline and got it, draws into a target the previous frame did not render that diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index fb5b31d3..23e41e30 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -51,6 +51,11 @@ void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the ga // CPU class (0 CPU_CACHED, 1 CPU_UNCACHED, 2 other) since the previous call. void nvn_flush_stats(uint64_t out[2][3][2]); void nvn_submit_prepare(const uint64_t *handles, int n); // weak no-op; the command-buffer slice reads deferred binds +// Asynchronous replay: as nvn_submit_prepare, writing the handles to replay to out (a clone where a still queued +// submission uses the same recording with other deferred binds) +void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n); +int gfx_replay_async(void); // 1: the back end replays submissions on its own thread +uint64_t gfx_replayed_submits(void); // newest submit number (nvn_submit_count) replayed #define NVN_HANDLE_TEX(h) ((int)((h) & 0xfffff)) // combined texture handle = texture id | sampler id << 20 #define NVN_HANDLE_SAMP(h) ((int)(((h) >> 20) & 0xfff)) extern GfxStats gfx_counters; // behind gfx_stats(); add with __atomic_fetch_add. frames: diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index bea13352..04c9cccf 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -160,7 +160,7 @@ static const tex_t *tex_pin(const NVNtexture *nt) { // ---------- recording copies // A handle points just past a copy_prefix. Copies are indexed by the 4 KiB page of their control-memory position, so // adding control memory again releases the copies recorded there without walking every live copy. -typedef struct { uint64_t magic, bytes, made_at, used_at; } copy_prefix; +typedef struct { uint64_t magic, bytes, made_at, used_at, deferred; } copy_prefix; // deferred: 0 unknown, 1 none, 2 some _Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned"); typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy; typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket; @@ -168,6 +168,30 @@ static SRWLOCK copy_lock = SRWLOCK_INIT; // guards the two maps, the counters static nvn_hmap copy_pages = {.klen = 8}; // page -> copy_bucket static nvn_hmap copy_live = {.klen = 8}; // handle -> copy_prefix of a live copy static uint64_t ncopies, copy_bytes, copies_made, nsubmits; +// Asynchronous replay: copies released by the game, and per-submission clones, that a queued submission still reads. +// Freed once gfx_replayed_submits() reaches their used_at. Empty with synchronous replay. +static copy_prefix **zombies; +static uint32_t nzombies, cap_zombies; + +static void zombie_add(copy_prefix *pf) { // copy_lock held + if (nzombies == cap_zombies) { + cap_zombies = cap_zombies ? cap_zombies * 2 : 64; + zombies = realloc(zombies, sizeof *zombies * cap_zombies); + if (!zombies) rt_fatal("nvn: out of memory (released recording copies)"); + } + zombies[nzombies++] = pf; +} + +static void zombie_sweep(uint64_t replayed) { // copy_lock held + for (uint32_t i = 0; i < nzombies;) { + if (zombies[i]->used_at <= replayed) { + free(zombies[i]); + zombies[i] = zombies[--nzombies]; + } else { + i++; + } + } +} static copy_prefix *copy_find(uint64_t handle) { // copy_lock held return nvn_hm_get(©_live, &handle); @@ -258,7 +282,9 @@ static void copy_add(uintptr_t lo, uintptr_t hi, uint8_t *copy) { // Adding control memory again retires every recording copy made in it (the driver's handle lived there). static void release_copies(uintptr_t lo, uintptr_t hi) { if (hi <= lo) return; + uint64_t replayed = gfx_replayed_submits(); AcquireSRWLockExclusive(©_lock); + if (nzombies) zombie_sweep(replayed); for (uintptr_t pg = lo >> COPY_PAGE_SHIFT; pg <= (hi - 1) >> COPY_PAGE_SHIFT; pg++) { copy_bucket *b = nvn_hm_get(©_pages, &pg); if (!b) continue; @@ -271,7 +297,8 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { ncopies--; copy_bytes -= pf->bytes; pf->magic = 0; - free(pf); + if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it + else free(pf); b->v[i] = b->v[--b->n]; } if (!b->n) { @@ -836,16 +863,17 @@ NVN_API(nvnCommandBufferSetPatchSize) { (void)c; } // What the console reads at the submit call is read here, on the game thread: every handle must be a live recording, // and deferred binds take the handles the game's arrays hold now (the game fills its tables for the next frame as soon // as the submit returns). The result is written into the recording copy. -static uint64_t def_reads, def_changed; +static uint64_t def_reads, def_changed, n_clones; -static void resolve_deferred(const uint8_t *p, int depth) { +static int resolve_deferred(const uint8_t *p, int depth) { // returns the number of deferred binds + int nd = 0; for (;;) { op_hdr h; memcpy(&h, p, sizeof h); if (h.size < 8 || (h.size & 7) || h.op >= OP_COUNT_) rt_fatal("nvn: corrupt recording at %p while resolving deferred binds (op %u size %u)", (const void *)p, h.op, h.size); - if (h.op == OP_END) return; + if (h.op == OP_END) return nd; if (h.op == OP_JUMP) { memcpy(&p, p + 8, 8); continue; @@ -854,10 +882,11 @@ static void resolve_deferred(const uint8_t *p, int depth) { uint64_t t; memcpy(&t, p + 8, 8); if (depth >= 16) rt_fatal("nvn: deferred bind resolution: call depth over 16"); - resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1); + nd += resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1); } else if (h.op == OP_BIND_TEX || h.op == OP_BIND_IMG || h.op == OP_BIND_SAMP) { p_bind_handles *b = (p_bind_handles *)(uintptr_t)(p + 8); if (b->src && b->n > 0 && b->n <= 32) { + nd++; uint64_t cur[32]; memcpy(cur, (const void *)(uintptr_t)b->src, 8 * (size_t)b->n); __atomic_fetch_add(&def_reads, 1, __ATOMIC_RELAXED); @@ -879,6 +908,45 @@ void nvn_submit_prepare(const uint64_t *handles, int n) { for (int i = 0; i < n; i++) resolve_deferred((const uint8_t *)(uintptr_t)handles[i], 0); } +// Asynchronous replay. A copy a queued submission still reads keeps that submission's deferred binds: when it holds +// deferred binds (or is not known to hold none), this submission gets a clone, resolved here and freed after its replay. +void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { + uint64_t now = __atomic_add_fetch(&nsubmits, 1, __ATOMIC_RELAXED); + uint64_t replayed = gfx_replayed_submits(); + AcquireSRWLockExclusive(©_lock); + if (nzombies) zombie_sweep(replayed); + for (int i = 0; i < n; i++) { + copy_prefix *pf = copy_find(handles[i]); + if (!pf) { + ReleaseSRWLockExclusive(©_lock); + rt_fatal("nvn: QueueSubmitCommands: command handle %#llx is not a live recording (released or never recorded)", + (unsigned long long)handles[i]); + } + out[i] = handles[i]; + if (pf->used_at > replayed && pf->used_at != now && pf->deferred != 1) { + size_t sz = sizeof *pf + pf->bytes; + copy_prefix *cl = malloc(sz); + if (!cl) rt_fatal("nvn: out of memory (recording clone, %zu bytes)", sz); + memcpy(cl, pf, sz); + cl->magic = 0; // not a live handle: the game never sees it + cl->used_at = now; + zombie_add(cl); + out[i] = (uintptr_t)(cl + 1); + __atomic_fetch_add(&n_clones, 1, __ATOMIC_RELAXED); + } else { + pf->used_at = now; + } + } + ReleaseSRWLockExclusive(©_lock); + for (int i = 0; i < n; i++) { + int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0); + if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1; + } + if (now % 7200 == 0) + rt_log("nvn: asynchronous replay at submit %llu: %llu recording clones so far, %u copies waiting for the replay", + (unsigned long long)now, (unsigned long long)__atomic_load_n(&n_clones, __ATOMIC_RELAXED), nzombies); +} + uint64_t nvn_submit_count(void) { return __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); } const char *nvn_op_name(uint32_t op) { diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 0010d3b6..9dda8b20 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -58,6 +58,11 @@ static void ov_memcpy(Cpu *c) { HOST_OVERRIDE("memcpy", ov_memcpy); __attribute__((weak)) void nvn_submit_prepare(const uint64_t *handles, int n) { (void)handles, (void)n; } +__attribute__((weak)) void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { + memcpy(out, handles, sizeof *out * (size_t)n); +} +__attribute__((weak)) int gfx_replay_async(void) { return 0; } +__attribute__((weak)) uint64_t gfx_replayed_submits(void) { return nvn_submit_count(); } // ---------- pools: GPU window (section 3.1) and the address index static SRWLOCK pool_lock = SRWLOCK_INIT; // guards gva and index @@ -560,7 +565,9 @@ NVN_API(nvnQueueFinish) { static uint64_t lod_entered, lod_issued, lod_fence[LOD_RING]; // void nvnQueueSubmitCommands(NVNqueue *q, int n, const NVNcommandHandle *handles): replayed on this thread with its -// core given up (section 2.2); the newest fence value is what QueueFenceSync records +// core given up (section 2.2), or queued for the replay thread (TOTK_REPLAY_ASYNC, vk_backend.c); the newest fence +// value is what QueueFenceSync records. With the queue the fence covers replay and GPU, so the LOD guard below waits +// for both. NVN_API(nvnQueueSubmitCommands) { queue_t *q = arg_p(c, 0); int n = (int)arg_x(c, 1); @@ -569,8 +576,22 @@ NVN_API(nvnQueueSubmitCommands) { NVN_BLOCKING(); uint64_t no = __atomic_add_fetch(&lod_entered, 1, __ATOMIC_SEQ_CST); q->submits++; - nvn_submit_prepare(h, n); - uint64_t f = gfx_backend()->submit(h, n), prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED); + uint64_t f; + if (gfx_replay_async()) { + // Prepare and queue under one lock: submit numbers follow queue order (gfx_replayed_submits). + static SRWLOCK submit_lock = SRWLOCK_INIT; + uint64_t local[32], *hs = n <= 32 ? local : malloc(sizeof *hs * (size_t)n); + if (!hs) rt_fatal("nvn: out of memory (submit of %d handles)", n); + AcquireSRWLockExclusive(&submit_lock); + nvn_submit_prepare_async(h, hs, n); + f = gfx_backend()->submit(hs, n); + ReleaseSRWLockExclusive(&submit_lock); + if (hs != local) free(hs); + } else { + nvn_submit_prepare(h, n); + f = gfx_backend()->submit(h, n); + } + uint64_t prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED); while (f > prev && !__atomic_compare_exchange_n(&last_fence, &prev, f, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {} __atomic_store_n(&lod_fence[no % LOD_RING], f, __ATOMIC_RELAXED); __atomic_store_n(&lod_issued, no, __ATOMIC_SEQ_CST); diff --git a/runtime/prof_win.c b/runtime/prof_win.c index d81954ba..35b7fe4f 100644 --- a/runtime/prof_win.c +++ b/runtime/prof_win.c @@ -269,11 +269,22 @@ static uint64_t wait_ns[PERF_N], frame_ns[PERF_N], worst_ns[PERF_N]; // window uint64_t perf_begin(void) { return kern_now_ns(); } +static volatile DWORD replay_tid; // asynchronous replay (vk_backend.c): its waits hold up the frame too + +void perf_replay_thread(void) { replay_tid = GetCurrentThreadId(); } + void perf_end(int kind, uint64_t t0) { - if (!t0 || GetCurrentThreadId() != render_tid) return; + if (!t0) return; + DWORD tid = GetCurrentThreadId(); + if (tid != render_tid && tid != replay_tid) return; uint64_t d = kern_now_ns() - t0; - wait_ns[kind] += d; - frame_ns[kind] += d; + if (tid == render_tid && !replay_tid) { // one thread adds: no atomics on the synchronous path + wait_ns[kind] += d; + frame_ns[kind] += d; + return; + } + __atomic_fetch_add(&wait_ns[kind], d, __ATOMIC_RELAXED); + __atomic_fetch_add(&frame_ns[kind], d, __ATOMIC_RELAXED); } static volatile LONG64 other_wait_ns; // perf_gpu_wait on threads other than the render thread, over the window diff --git a/runtime/vk.h b/runtime/vk.h index 91373871..f58c2880 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -286,6 +286,7 @@ void rt_write_png(const char *path, const uint8_t *rgba, uint32_t w, uint32_t h) // once the GPU reached it and the completion thread ran its side effects. Guest or host threads. void vk_sync_start(void); // vk_init (fatal without a device) and the completion thread; idempotent uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value +uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted uint64_t vk_sync_fence_done(void); // highest done value uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1) @@ -308,6 +309,7 @@ void rdoc_init(void); // runtime/rdoc_win.c: TOTK_RDOC debug int rdoc_active(void); void rdoc_after_present(uint64_t frame); void vk_present_drain(void); // returns once every queued present was submitted to gfx_queue +int vk_replay_async(void); // TOTK_REPLAY_ASYNC (default 1): submissions replay on the replay thread void vk_program_release(const prog_t *p); // GfxBackend.program_release: pipelines (T6b); weak no-op in vk_backend.c void vk_backend_shutdown(void); // presents, vk_sync_shutdown, vk_mem_shutdown, vk_shutdown in that order diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 8193f0d6..3dd6d0c8 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -10,8 +10,19 @@ // Back-edge pacing: the present thread submits the blit at once, then holds the host present (vkQueuePresentKHR) until // the frame's display time from the pacer (nvn_api_obj.c), so present-to-present intervals stay even whatever the // replay took. The blit is already on the queue, so the wait blocks no submission. +// +// Asynchronous replay (TOTK_REPLAY_ASYNC, default 1; 0 keeps the synchronous path unchanged): nvnQueueSubmitCommands +// and nvnQueuePresentTexture only queue. Each item gets its timeline value at enqueue (vk_sync_reserve), in queue +// order; the replay thread runs the synchronous submit and present bodies in that order, so every value is signalled +// by the submission it was handed out for. The game runs at most one frame ahead: queueing present N waits until the +// replay thread handled present N-1. Calls that read or free GPU state that queued work may use drain the queue first +// (replay_drain), each cause counted. Design and the orderings it keeps: build/v121-replayq/NOTES.md. #include +#include +#include +#include +#include "cfg.h" #include "flight.h" #include "gfx.h" #include "host.h" @@ -21,7 +32,7 @@ #define WEAK __attribute__((weak)) -typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued; } PresentJob; +typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued, value; } PresentJob; static SRWLOCK pl = SRWLOCK_INIT; static CONDITION_VARIABLE pcv = CONDITION_VARIABLE_INIT; // job queued, job taken, job submitted @@ -52,7 +63,10 @@ static DWORD WINAPI present_main(void *arg) { ReleaseSRWLockExclusive(&pl); if (flight_on) vk_thumbs_frame(j.frame); vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.w, j.h, j.has_crop ? j.crop : NULL); - vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too + uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too + if (j.value && f != j.value) + rt_fatal("vk: the present of frame %llu signalled %llu, queued as %llu", (unsigned long long)j.frame, + (unsigned long long)f, (unsigned long long)j.value); AcquireSRWLockExclusive(&pl); busy = 0; WakeAllConditionVariable(&pcv); @@ -84,11 +98,11 @@ void vk_present_drain(void) { perf_end(PERF_DRAIN, p0); } -static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { +// The present after the frame count: resolves the image and hands the blit to the present thread. queued: when the game +// presented; value: the timeline value queued for this present (asynchronous replay), 0 for the next one (synchronous). +static void present_image(const tex_t *t, const int crop[4], uint64_t at_ns, uint64_t queued, uint64_t frame, + uint64_t value) { static volatile long n_skipped; - vk_sync_start(); - uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); - perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads vk_exec_frame_log(t, frame); vk_mem_census(frame); if (vk_dump_armed) vk_dump_present(frame); @@ -97,10 +111,15 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { if (!img || !(img->aspect & VK_IMAGE_ASPECT_COLOR_BIT)) { if (InterlockedIncrement(&n_skipped) <= 8) rt_log("vk: present of texture %p (format %#x) skipped: no colour image", (const void *)t, t ? t->d.format : 0); + if (value) { // the queued value is still signalled, in order + vk_present_drain(); + uint64_t f = vk_sync_submit(NULL, 0); + if (f != value) rt_fatal("vk: skipped present signalled %llu, queued as %llu", (unsigned long long)f, (unsigned long long)value); + } return; } PresentJob j = {.image = img->image, .format = img->format, .w = (int)img->width, .h = (int)img->height, .frame = frame, - .at = at_ns, .queued = kern_now_ns()}; + .at = at_ns, .queued = queued, .value = value}; if (crop) { j.has_crop = 1; for (int i = 0; i < 4; i++) j.crop[i] = crop[i]; @@ -121,35 +140,223 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { if (rdoc_active()) { vk_present_drain(); rdoc_after_present(frame); } } -// Pipelines (T6b) drop a released program's pipelines here. -WEAK void vk_program_release(const prog_t *p) { (void)p; } +// ---- Asynchronous replay +enum { RI_SUBMIT, RI_PRESENT }; +typedef struct { + int kind, n, has_crop, crop[4]; + uint64_t value, submit_no, at, queued; + uint64_t *h; + const tex_t *tex; +} ReplayItem; +#define RQ_CAP 64 +enum { DR_POOL_FINI, DR_MAP_VIRTUAL, DR_TEX_RELEASE, DR_PROG_RELEASE, DR_FINISH, DR_READ_TEXELS, DR_WRITE_TEXELS, + DR_GUEST_READ, DR_FRAME, DR_FULL, DR_N }; +static const char *const dr_name[DR_N] = {"pool_fini", "map_virtual", "texture_release", "program_release", "finish", + "read_texels", "write_texels", "guest_read", "frame bound", "queue full"}; -static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); } -static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); vk_present_drain(); vk_pool_fini(p); } -static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) { - vk_sync_start(); - vk_present_drain(); - return vk_pool_map_virtual(v, n, requests); +static SRWLOCK rl = SRWLOCK_INIT; +static CONDITION_VARIABLE rcv = CONDITION_VARIABLE_INIT; // item queued, item taken, item done +static ReplayItem rq[RQ_CAP]; +static int rq_head, rq_n, r_busy, r_started; +static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames; +// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run. +static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N]; + +int vk_replay_async(void) { + static int on = -1; + if (on < 0) { + const char *e = cfg_get("TOTK_REPLAY_ASYNC"); + on = !(e && !strcmp(e, "0")); + rt_log("vk: replay %s (TOTK_REPLAY_ASYNC=%d)", + on ? "on its own thread, at most one frame behind the game" : "on the submitting thread", on); + } + return on; } -static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); } -static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); } -static void vk_be_texture_release(const tex_t *t) { vk_sync_start(); vk_present_drain(); vk_texture_release(t); } -static void vk_be_program_release(const prog_t *p) { vk_program_release(p); } -static uint64_t vk_be_submit(const uint64_t *handles, int n) { + +// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end. +int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); } + +// The newest submit number (nvn_submit_count) whose replay is done: recording copies it used may be freed. +uint64_t gfx_replayed_submits(void) { + if (!gfx_replay_async()) return nvn_submit_count(); + AcquireSRWLockShared(&rl); + uint64_t v = r_submit_done; + ReleaseSRWLockShared(&rl); + return v; +} + +static uint64_t submit_sync(const uint64_t *handles, int n) { vk_sync_start(); vk_present_drain(); vk_texture_new_submit(); return vk_sync_submit(handles, n); } -static void vk_be_finish(void) { vk_sync_start(); vk_present_drain(); vk_sync_finish(); } + +static DWORD WINAPI replay_main(void *arg) { + (void)arg; + perf_replay_thread(); + AcquireSRWLockExclusive(&rl); + for (;;) { + while (!rq_n) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0); + ReplayItem it = rq[rq_head]; + rq_head = (rq_head + 1) % RQ_CAP; + rq_n--; + r_busy = 1; + WakeAllConditionVariable(&rcv); + ReleaseSRWLockExclusive(&rl); + if (it.kind == RI_SUBMIT) { + uint64_t f = submit_sync(it.h, it.n); + if (f != it.value) + rt_fatal("vk: replayed submission signalled %llu, queued as %llu", (unsigned long long)f, + (unsigned long long)it.value); + free(it.h); + } else { + uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); + present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value); + } + AcquireSRWLockExclusive(&rl); + if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no; + else r_presents_done++; + r_busy = 0; + WakeAllConditionVariable(&rcv); + } + return 0; +} + +static void wait_counted(int why, uint64_t t0) { // rl held + uint64_t d = kern_now_ns() - t0; + dr_waits[why]++, dr_twaits[why]++; + dr_ns[why] += d, dr_tns[why] += d; + perf_end(PERF_DRAIN, t0); +} + +// Queues an item (waits while the queue is full) and returns its value, reserved under rl so values follow queue order. +static uint64_t replay_enqueue(ReplayItem *it) { + AcquireSRWLockExclusive(&rl); + if (!r_started) { + HANDLE h = CreateThread(NULL, 0, replay_main, NULL, 0, NULL); + if (!h) rt_fatal("vk: cannot start the replay thread (error %lu)", GetLastError()); + CloseHandle(h); + r_started = 1; + } + if (rq_n == RQ_CAP) { + uint64_t p0 = perf_begin(); + dr_calls[DR_FULL]++, dr_tcalls[DR_FULL]++; + while (rq_n == RQ_CAP) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0); + wait_counted(DR_FULL, p0); + } + it->value = vk_sync_reserve(); + rq[(rq_head + rq_n) % RQ_CAP] = *it; + rq_n++; + WakeAllConditionVariable(&rcv); + ReleaseSRWLockExclusive(&rl); + return it->value; +} + +// Returns once every queued item was replayed and submitted. Back-end entries only: the replay thread never calls them. +static void replay_drain(int why) { + if (!gfx_replay_async()) return; + AcquireSRWLockExclusive(&rl); + dr_calls[why]++, dr_tcalls[why]++; + if (rq_n || r_busy) { + uint64_t p0 = perf_begin(); + while (rq_n || r_busy) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0); + wait_counted(why, p0); + } + ReleaseSRWLockExclusive(&rl); +} + +// What made the game wait for the replay: since the last line (every 1800 game frames), or over the run (at exit). +static void replay_log(uint64_t frame, int total) { + char buf[1024]; + int k = 0; + AcquireSRWLockExclusive(&rl); + const uint64_t *c = total ? dr_tcalls : dr_calls, *w = total ? dr_twaits : dr_waits, *t = total ? dr_tns : dr_ns; + for (int i = 0; i < DR_N && k < (int)sizeof buf - 96; i++) + if (c[i] || w[i]) + k += snprintf(buf + k, sizeof buf - (size_t)k, "%s %s %llu calls, %llu waited %.1f ms", k ? ";" : "", dr_name[i], + (unsigned long long)c[i], (unsigned long long)w[i], t[i] / 1e6); + memset(dr_calls, 0, sizeof dr_calls); + memset(dr_waits, 0, sizeof dr_waits); + memset(dr_ns, 0, sizeof dr_ns); + ReleaseSRWLockExclusive(&rl); + rt_log("vk: replay queue %s frame %llu:%s", total ? "over the run to" : "at", (unsigned long long)frame, + k ? buf : " no waits"); +} + +static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { + vk_sync_start(); + if (!gfx_replay_async()) { + uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); + perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads + present_image(t, crop, at_ns, kern_now_ns(), frame, 0); + return; + } + // Frame bound: the replay thread handled the previous present before this one is queued. + AcquireSRWLockExclusive(&rl); + dr_calls[DR_FRAME]++, dr_tcalls[DR_FRAME]++; + if (r_presents_done < r_presents_queued) { + uint64_t p0 = perf_begin(); + while (r_presents_done < r_presents_queued) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0); + wait_counted(DR_FRAME, p0); + } + r_presents_queued++; + ReleaseSRWLockExclusive(&rl); + uint64_t frame = ++game_frames; + perf_present(frame); + if (frame % 1800 == 0) replay_log(frame, 0); + ReplayItem it = {.kind = RI_PRESENT, .tex = t, .at = at_ns, .queued = kern_now_ns()}; + if (crop) { + it.has_crop = 1; + for (int i = 0; i < 4; i++) it.crop[i] = crop[i]; + } + replay_enqueue(&it); +} + +// Pipelines (T6b) drop a released program's pipelines here. +WEAK void vk_program_release(const prog_t *p) { (void)p; } + +static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); } +static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); replay_drain(DR_POOL_FINI); vk_present_drain(); vk_pool_fini(p); } +static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) { + vk_sync_start(); + replay_drain(DR_MAP_VIRTUAL); + vk_present_drain(); + return vk_pool_map_virtual(v, n, requests); +} +static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); } +static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); } +static void vk_be_texture_release(const tex_t *t) { + vk_sync_start(); + replay_drain(DR_TEX_RELEASE); + vk_present_drain(); + vk_texture_release(t); +} +static void vk_be_program_release(const prog_t *p) { replay_drain(DR_PROG_RELEASE); vk_program_release(p); } +// Asynchronous: the handles (nvn_submit_prepare_async ran under the front end's submit lock) are copied and queued; +// the value returned is the one the replay will signal. +static uint64_t vk_be_submit(const uint64_t *handles, int n) { + if (!gfx_replay_async()) return submit_sync(handles, n); + vk_sync_start(); + ReplayItem it = {.kind = RI_SUBMIT, .n = n, .submit_no = nvn_submit_count()}; + if (n > 0) { + it.h = malloc(sizeof *it.h * (size_t)n); + if (!it.h) rt_fatal("vk: out of memory (replay queue)"); + memcpy(it.h, handles, sizeof *it.h * (size_t)n); + } + return replay_enqueue(&it); +} +static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); } static void vk_be_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) { vk_sync_start(); + replay_drain(DR_READ_TEXELS); vk_read_texels(t, v, r, dst, row, img); } static void vk_be_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row, uint64_t img) { vk_sync_start(); + replay_drain(DR_WRITE_TEXELS); vk_present_drain(); vk_write_texels(t, v, r, src, row, img); } @@ -157,9 +364,12 @@ static void vk_be_guest_written(uint64_t addr, uint64_t size) { vk_sync_start(); static void vk_be_guest_read(uint64_t addr, uint64_t size) { if (!vk_texture_cpu_stale(addr, size)) return; vk_sync_start(); + replay_drain(DR_GUEST_READ); vk_present_drain(); vk_texture_write_back(addr, size); } +// After a fence wait: the value is done, so its replay is too; a later submission the replay records meanwhile marks +// its targets with a later fence, which write_back_small leaves alone. No drain. static void vk_be_cpu_synced(uint64_t fence) { if (!vk_texture_small_dirty()) return; vk_sync_start(); @@ -189,6 +399,8 @@ const GfxBackend gfx_vulkan_backend = { }; void vk_backend_shutdown(void) { + replay_drain(DR_FINISH); + if (gfx_replay_async() && r_started) replay_log(game_frames, 1); vk_present_drain(); vk_sync_shutdown(); vk_mem_shutdown(); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 2870b3cb..b3a2a1bb 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -42,6 +42,7 @@ static SRWLOCK st_lock = SRWLOCK_INIT; static CONDITION_VARIABLE cv_work = CONDITION_VARIABLE_INIT; // a submission, a callback or stop static CONDITION_VARIABLE cv_done = CONDITION_VARIABLE_INIT; // done advanced static uint64_t issued, done; // newest submitted value; newest value whose side effects ran +static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued) static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; @@ -223,6 +224,7 @@ void vk_sync_start(void) { } AcquireSRWLockExclusive(&st_lock); issued = done = now; + if (reserved < now) reserved = now; stop = 0; ReleaseSRWLockExclusive(&st_lock); vk_mem_note_submit(now); @@ -301,6 +303,17 @@ VkCommandBuffer vk_sync_split(void) { return s->cb; } +// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed +// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value. +uint64_t vk_sync_reserve(void) { + vk_sync_start(); + AcquireSRWLockExclusive(&st_lock); + if (reserved < issued) reserved = issued; + uint64_t v = ++reserved; + ReleaseSRWLockExclusive(&st_lock); + return v; +} + uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_sync_start(); VkCtx *c = vk_ctx(); @@ -378,8 +391,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { vk_sync_start(); AcquireSRWLockExclusive(&st_lock); int ok; - if (value > issued) { - ok = 0; // never submitted: never reached (as in the null back end) + if (value > issued && value > reserved) { + ok = 0; // never submitted nor queued: never reached (as in the null back end) } else { uint64_t t0 = kern_now_ns(); while (done < value) {