diff --git a/runtime/boot.c b/runtime/boot.c index b4ea789e..9776a527 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -14,6 +14,7 @@ #include "cfg.h" #include "gfx.h" +#include "nvn_api.h" #include "vk.h" #define SYM_INIT_START "_ZN2nn4init5StartEmmPFvvES2_S2_" @@ -200,6 +201,7 @@ static void save_shot(const tex_t *t, const char *path) { view_t view; memset(&view, 0, sizeof view); NVNcopyRegion r = {.width = (int)w, .height = (int)h, .depth = 1}; + if (gfx_replay_async()) gfx_backend()->finish(); // the presented frame may still be queued for the replay gfx_backend()->read_texels(t, &view, &r, px, 4ull * w, 4ull * w * h); if (bgra) for (uint64_t i = 0; i < 4ull * w * h; i += 4) { @@ -238,6 +240,7 @@ static void series_shot(const tex_t *t, uint64_t n) { view_t view; memset(&view, 0, sizeof view); NVNcopyRegion r = {.width = (int)w, .height = (int)h, .depth = 1}; + if (gfx_replay_async()) gfx_backend()->finish(); // the presented frame may still be queued for the replay gfx_backend()->read_texels(t, &view, &r, px, 4ull * w, 4ull * w * h); for (uint32_t y = 0; y < oh; y++) for (uint32_t x = 0; x < ow; x++) { diff --git a/runtime/cfg.c b/runtime/cfg.c index f5c2be89..05152af3 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -34,7 +34,7 @@ static const char *const known[] = { "MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", - "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", + "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS", diff --git a/runtime/host.h b/runtime/host.h index 43aef4be..ca196ba4 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -28,6 +28,7 @@ void prof_write(void); // idempotent; boot.c calls it b uint64_t perf_begin(void); // start time of a wait; pair with perf_end void perf_end(int kind, uint64_t t0); // counts render-thread time only void perf_replay(uint64_t t0); // replay (NVN -> Vulkan recording) time on any thread, waits inside it included +void perf_replay_thread(void); // the calling thread replays for the render thread: perf_end counts its waits // Events per 60-frame window: graphics pipelines finished by the background workers, draws skipped while pending, // first texture uploads and re-uploads after CPU writes, draws skipped while a shader translates in the background, // draws that waited for their pending pipeline and got it, draws into a target the previous frame did not render that @@ -41,6 +42,7 @@ enum { PERF_CNT_PIPE_ASYNC, PERF_CNT_PIPE_SKIP, PERF_CNT_TEX_UP, PERF_CNT_TEX_RE void perf_count(int kind); void perf_present(uint64_t frame); void perf_summary_flush(void); // prints the frame-time summary of the running period (exit); the 60 s lines come from perf_present +void gfx_exit_log(void); // back end's lines for the exit summary (weak no-op in prof_win.c) void perf_gpu_wait(uint64_t t0); // a wait for a GPU fence ended (any thread; the render thread's also counts as PERF_FENCE) // GPU side of the 60-frame perf line, filled by the graphics back end when it set perf_gpu_stats. gpu_ns: GPU busy time // since the previous call (timestamp queries, overlaps counted once), have_gpu 0 when the device has no timestamps. diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index fb5b31d3..23e41e30 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -51,6 +51,11 @@ void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the ga // CPU class (0 CPU_CACHED, 1 CPU_UNCACHED, 2 other) since the previous call. void nvn_flush_stats(uint64_t out[2][3][2]); void nvn_submit_prepare(const uint64_t *handles, int n); // weak no-op; the command-buffer slice reads deferred binds +// Asynchronous replay: as nvn_submit_prepare, writing the handles to replay to out (a clone where a still queued +// submission uses the same recording with other deferred binds) +void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n); +int gfx_replay_async(void); // 1: the back end replays submissions on its own thread +uint64_t gfx_replayed_submits(void); // newest submit number (nvn_submit_count) replayed #define NVN_HANDLE_TEX(h) ((int)((h) & 0xfffff)) // combined texture handle = texture id | sampler id << 20 #define NVN_HANDLE_SAMP(h) ((int)(((h) >> 20) & 0xfff)) extern GfxStats gfx_counters; // behind gfx_stats(); add with __atomic_fetch_add. frames: diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index bea13352..04c9cccf 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -160,7 +160,7 @@ static const tex_t *tex_pin(const NVNtexture *nt) { // ---------- recording copies // A handle points just past a copy_prefix. Copies are indexed by the 4 KiB page of their control-memory position, so // adding control memory again releases the copies recorded there without walking every live copy. -typedef struct { uint64_t magic, bytes, made_at, used_at; } copy_prefix; +typedef struct { uint64_t magic, bytes, made_at, used_at, deferred; } copy_prefix; // deferred: 0 unknown, 1 none, 2 some _Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned"); typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy; typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket; @@ -168,6 +168,30 @@ static SRWLOCK copy_lock = SRWLOCK_INIT; // guards the two maps, the counters static nvn_hmap copy_pages = {.klen = 8}; // page -> copy_bucket static nvn_hmap copy_live = {.klen = 8}; // handle -> copy_prefix of a live copy static uint64_t ncopies, copy_bytes, copies_made, nsubmits; +// Asynchronous replay: copies released by the game, and per-submission clones, that a queued submission still reads. +// Freed once gfx_replayed_submits() reaches their used_at. Empty with synchronous replay. +static copy_prefix **zombies; +static uint32_t nzombies, cap_zombies; + +static void zombie_add(copy_prefix *pf) { // copy_lock held + if (nzombies == cap_zombies) { + cap_zombies = cap_zombies ? cap_zombies * 2 : 64; + zombies = realloc(zombies, sizeof *zombies * cap_zombies); + if (!zombies) rt_fatal("nvn: out of memory (released recording copies)"); + } + zombies[nzombies++] = pf; +} + +static void zombie_sweep(uint64_t replayed) { // copy_lock held + for (uint32_t i = 0; i < nzombies;) { + if (zombies[i]->used_at <= replayed) { + free(zombies[i]); + zombies[i] = zombies[--nzombies]; + } else { + i++; + } + } +} static copy_prefix *copy_find(uint64_t handle) { // copy_lock held return nvn_hm_get(©_live, &handle); @@ -258,7 +282,9 @@ static void copy_add(uintptr_t lo, uintptr_t hi, uint8_t *copy) { // Adding control memory again retires every recording copy made in it (the driver's handle lived there). static void release_copies(uintptr_t lo, uintptr_t hi) { if (hi <= lo) return; + uint64_t replayed = gfx_replayed_submits(); AcquireSRWLockExclusive(©_lock); + if (nzombies) zombie_sweep(replayed); for (uintptr_t pg = lo >> COPY_PAGE_SHIFT; pg <= (hi - 1) >> COPY_PAGE_SHIFT; pg++) { copy_bucket *b = nvn_hm_get(©_pages, &pg); if (!b) continue; @@ -271,7 +297,8 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { ncopies--; copy_bytes -= pf->bytes; pf->magic = 0; - free(pf); + if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it + else free(pf); b->v[i] = b->v[--b->n]; } if (!b->n) { @@ -836,16 +863,17 @@ NVN_API(nvnCommandBufferSetPatchSize) { (void)c; } // What the console reads at the submit call is read here, on the game thread: every handle must be a live recording, // and deferred binds take the handles the game's arrays hold now (the game fills its tables for the next frame as soon // as the submit returns). The result is written into the recording copy. -static uint64_t def_reads, def_changed; +static uint64_t def_reads, def_changed, n_clones; -static void resolve_deferred(const uint8_t *p, int depth) { +static int resolve_deferred(const uint8_t *p, int depth) { // returns the number of deferred binds + int nd = 0; for (;;) { op_hdr h; memcpy(&h, p, sizeof h); if (h.size < 8 || (h.size & 7) || h.op >= OP_COUNT_) rt_fatal("nvn: corrupt recording at %p while resolving deferred binds (op %u size %u)", (const void *)p, h.op, h.size); - if (h.op == OP_END) return; + if (h.op == OP_END) return nd; if (h.op == OP_JUMP) { memcpy(&p, p + 8, 8); continue; @@ -854,10 +882,11 @@ static void resolve_deferred(const uint8_t *p, int depth) { uint64_t t; memcpy(&t, p + 8, 8); if (depth >= 16) rt_fatal("nvn: deferred bind resolution: call depth over 16"); - resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1); + nd += resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1); } else if (h.op == OP_BIND_TEX || h.op == OP_BIND_IMG || h.op == OP_BIND_SAMP) { p_bind_handles *b = (p_bind_handles *)(uintptr_t)(p + 8); if (b->src && b->n > 0 && b->n <= 32) { + nd++; uint64_t cur[32]; memcpy(cur, (const void *)(uintptr_t)b->src, 8 * (size_t)b->n); __atomic_fetch_add(&def_reads, 1, __ATOMIC_RELAXED); @@ -879,6 +908,45 @@ void nvn_submit_prepare(const uint64_t *handles, int n) { for (int i = 0; i < n; i++) resolve_deferred((const uint8_t *)(uintptr_t)handles[i], 0); } +// Asynchronous replay. A copy a queued submission still reads keeps that submission's deferred binds: when it holds +// deferred binds (or is not known to hold none), this submission gets a clone, resolved here and freed after its replay. +void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { + uint64_t now = __atomic_add_fetch(&nsubmits, 1, __ATOMIC_RELAXED); + uint64_t replayed = gfx_replayed_submits(); + AcquireSRWLockExclusive(©_lock); + if (nzombies) zombie_sweep(replayed); + for (int i = 0; i < n; i++) { + copy_prefix *pf = copy_find(handles[i]); + if (!pf) { + ReleaseSRWLockExclusive(©_lock); + rt_fatal("nvn: QueueSubmitCommands: command handle %#llx is not a live recording (released or never recorded)", + (unsigned long long)handles[i]); + } + out[i] = handles[i]; + if (pf->used_at > replayed && pf->used_at != now && pf->deferred != 1) { + size_t sz = sizeof *pf + pf->bytes; + copy_prefix *cl = malloc(sz); + if (!cl) rt_fatal("nvn: out of memory (recording clone, %zu bytes)", sz); + memcpy(cl, pf, sz); + cl->magic = 0; // not a live handle: the game never sees it + cl->used_at = now; + zombie_add(cl); + out[i] = (uintptr_t)(cl + 1); + __atomic_fetch_add(&n_clones, 1, __ATOMIC_RELAXED); + } else { + pf->used_at = now; + } + } + ReleaseSRWLockExclusive(©_lock); + for (int i = 0; i < n; i++) { + int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0); + if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1; + } + if (now % 7200 == 0) + rt_log("nvn: asynchronous replay at submit %llu: %llu recording clones so far, %u copies waiting for the replay", + (unsigned long long)now, (unsigned long long)__atomic_load_n(&n_clones, __ATOMIC_RELAXED), nzombies); +} + uint64_t nvn_submit_count(void) { return __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); } const char *nvn_op_name(uint32_t op) { diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 0010d3b6..9dda8b20 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -58,6 +58,11 @@ static void ov_memcpy(Cpu *c) { HOST_OVERRIDE("memcpy", ov_memcpy); __attribute__((weak)) void nvn_submit_prepare(const uint64_t *handles, int n) { (void)handles, (void)n; } +__attribute__((weak)) void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { + memcpy(out, handles, sizeof *out * (size_t)n); +} +__attribute__((weak)) int gfx_replay_async(void) { return 0; } +__attribute__((weak)) uint64_t gfx_replayed_submits(void) { return nvn_submit_count(); } // ---------- pools: GPU window (section 3.1) and the address index static SRWLOCK pool_lock = SRWLOCK_INIT; // guards gva and index @@ -560,7 +565,9 @@ NVN_API(nvnQueueFinish) { static uint64_t lod_entered, lod_issued, lod_fence[LOD_RING]; // void nvnQueueSubmitCommands(NVNqueue *q, int n, const NVNcommandHandle *handles): replayed on this thread with its -// core given up (section 2.2); the newest fence value is what QueueFenceSync records +// core given up (section 2.2), or queued for the replay thread (TOTK_REPLAY_ASYNC, vk_backend.c); the newest fence +// value is what QueueFenceSync records. With the queue the fence covers replay and GPU, so the LOD guard below waits +// for both. NVN_API(nvnQueueSubmitCommands) { queue_t *q = arg_p(c, 0); int n = (int)arg_x(c, 1); @@ -569,8 +576,22 @@ NVN_API(nvnQueueSubmitCommands) { NVN_BLOCKING(); uint64_t no = __atomic_add_fetch(&lod_entered, 1, __ATOMIC_SEQ_CST); q->submits++; - nvn_submit_prepare(h, n); - uint64_t f = gfx_backend()->submit(h, n), prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED); + uint64_t f; + if (gfx_replay_async()) { + // Prepare and queue under one lock: submit numbers follow queue order (gfx_replayed_submits). + static SRWLOCK submit_lock = SRWLOCK_INIT; + uint64_t local[32], *hs = n <= 32 ? local : malloc(sizeof *hs * (size_t)n); + if (!hs) rt_fatal("nvn: out of memory (submit of %d handles)", n); + AcquireSRWLockExclusive(&submit_lock); + nvn_submit_prepare_async(h, hs, n); + f = gfx_backend()->submit(hs, n); + ReleaseSRWLockExclusive(&submit_lock); + if (hs != local) free(hs); + } else { + nvn_submit_prepare(h, n); + f = gfx_backend()->submit(h, n); + } + uint64_t prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED); while (f > prev && !__atomic_compare_exchange_n(&last_fence, &prev, f, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {} __atomic_store_n(&lod_fence[no % LOD_RING], f, __ATOMIC_RELAXED); __atomic_store_n(&lod_issued, no, __ATOMIC_SEQ_CST); diff --git a/runtime/prof_win.c b/runtime/prof_win.c index 1468b9f9..90cd26be 100644 --- a/runtime/prof_win.c +++ b/runtime/prof_win.c @@ -269,11 +269,22 @@ static uint64_t wait_ns[PERF_N], frame_ns[PERF_N], worst_ns[PERF_N]; // window uint64_t perf_begin(void) { return kern_now_ns(); } +static volatile DWORD replay_tid; // asynchronous replay (vk_backend.c): its waits hold up the frame too + +void perf_replay_thread(void) { replay_tid = GetCurrentThreadId(); } + void perf_end(int kind, uint64_t t0) { - if (!t0 || GetCurrentThreadId() != render_tid) return; + if (!t0) return; + DWORD tid = GetCurrentThreadId(); + if (tid != render_tid && tid != replay_tid) return; uint64_t d = kern_now_ns() - t0; - wait_ns[kind] += d; - frame_ns[kind] += d; + if (tid == render_tid && !replay_tid) { // one thread adds: no atomics on the synchronous path + wait_ns[kind] += d; + frame_ns[kind] += d; + return; + } + __atomic_fetch_add(&wait_ns[kind], d, __ATOMIC_RELAXED); + __atomic_fetch_add(&frame_ns[kind], d, __ATOMIC_RELAXED); } static volatile LONG64 other_wait_ns; // perf_gpu_wait on threads other than the render thread, over the window @@ -339,8 +350,13 @@ static void ft_report(uint64_t now, const char *tag) { ft.t0 = now; } +__attribute__((weak)) void gfx_exit_log(void) {} + void perf_summary_flush(void) { // at exit (prof_write) - if (summary_t0) ft_report(kern_now_ns(), "exit"); + if (summary_t0) { + ft_report(kern_now_ns(), "exit"); + gfx_exit_log(); + } summary_t0 = 0; } diff --git a/runtime/vk.h b/runtime/vk.h index 91373871..3d334efa 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -261,6 +261,9 @@ VkSampler vk_sampler_by_id_locked(int id); uint32_t vk_texture_slots_gen(void); uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage +void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission +uint64_t vk_texture_last_write(const tex_t *t); // timeline value of the newest submission recorded to write it; 0: none +size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img); // bytes ReadTexels spans; 0 for a bad region int vk_texture_small_dirty(void); // a small mirrored target holds GPU texels its storage lacks void vk_texture_write_back_small(uint64_t fence); // writes those rendered by work up to fence back to storage void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written); // vk_dump @@ -268,6 +271,7 @@ int vk_texture_slot_swz(int id, uint8_t swz[4]); // vk_dump: swizzle of the d int vk_texture_read_image(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img); // always from the image (GPU copy, waited for); 0 on failure int vk_texture_cpu_stale(uint64_t addr, uint64_t size); // a GPU-written mirrored texture there has stale storage +uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size); // newest recorded writer of such a texture; 0: none void vk_texture_write_back(uint64_t addr, uint64_t size); // writes those images back into their storage (waits) void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); // vk_pool_fini: drop the images there void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads); @@ -286,6 +290,7 @@ void rt_write_png(const char *path, const uint8_t *rgba, uint32_t w, uint32_t h) // once the GPU reached it and the completion thread ran its side effects. Guest or host threads. void vk_sync_start(void); // vk_init (fatal without a device) and the completion thread; idempotent uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value +uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted uint64_t vk_sync_fence_done(void); // highest done value uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1) @@ -308,6 +313,7 @@ void rdoc_init(void); // runtime/rdoc_win.c: TOTK_RDOC debug int rdoc_active(void); void rdoc_after_present(uint64_t frame); void vk_present_drain(void); // returns once every queued present was submitted to gfx_queue +int vk_replay_async(void); // TOTK_REPLAY_ASYNC (default 1): submissions replay on the replay thread void vk_program_release(const prog_t *p); // GfxBackend.program_release: pipelines (T6b); weak no-op in vk_backend.c void vk_backend_shutdown(void); // presents, vk_sync_shutdown, vk_mem_shutdown, vk_shutdown in that order diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 8193f0d6..03515864 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -10,8 +10,19 @@ // Back-edge pacing: the present thread submits the blit at once, then holds the host present (vkQueuePresentKHR) until // the frame's display time from the pacer (nvn_api_obj.c), so present-to-present intervals stay even whatever the // replay took. The blit is already on the queue, so the wait blocks no submission. +// +// Asynchronous replay (TOTK_REPLAY_ASYNC, default 1; 0 keeps the synchronous path unchanged): nvnQueueSubmitCommands +// and nvnQueuePresentTexture only queue. Each item gets its timeline value at enqueue (vk_sync_reserve), in queue +// order; the replay thread runs the synchronous submit and present bodies in that order, so every value is signalled +// by the submission it was handed out for. The game runs at most one frame ahead: queueing present N waits until the +// replay thread handled present N-1. Calls that read or free GPU state that queued work may use drain the queue first +// (replay_drain), each cause counted. Design and the orderings it keeps: build/v121-replayq/NOTES.md. #include +#include +#include +#include +#include "cfg.h" #include "flight.h" #include "gfx.h" #include "host.h" @@ -21,7 +32,7 @@ #define WEAK __attribute__((weak)) -typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued; } PresentJob; +typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued, value; } PresentJob; static SRWLOCK pl = SRWLOCK_INIT; static CONDITION_VARIABLE pcv = CONDITION_VARIABLE_INIT; // job queued, job taken, job submitted @@ -52,7 +63,10 @@ static DWORD WINAPI present_main(void *arg) { ReleaseSRWLockExclusive(&pl); if (flight_on) vk_thumbs_frame(j.frame); vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.w, j.h, j.has_crop ? j.crop : NULL); - vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too + uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too + if (j.value && f != j.value) + rt_fatal("vk: the present of frame %llu signalled %llu, queued as %llu", (unsigned long long)j.frame, + (unsigned long long)f, (unsigned long long)j.value); AcquireSRWLockExclusive(&pl); busy = 0; WakeAllConditionVariable(&pcv); @@ -84,11 +98,11 @@ void vk_present_drain(void) { perf_end(PERF_DRAIN, p0); } -static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { +// The present after the frame count: resolves the image and hands the blit to the present thread. queued: when the game +// presented; value: the timeline value queued for this present (asynchronous replay), 0 for the next one (synchronous). +static void present_image(const tex_t *t, const int crop[4], uint64_t at_ns, uint64_t queued, uint64_t frame, + uint64_t value) { static volatile long n_skipped; - vk_sync_start(); - uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); - perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads vk_exec_frame_log(t, frame); vk_mem_census(frame); if (vk_dump_armed) vk_dump_present(frame); @@ -97,10 +111,15 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { if (!img || !(img->aspect & VK_IMAGE_ASPECT_COLOR_BIT)) { if (InterlockedIncrement(&n_skipped) <= 8) rt_log("vk: present of texture %p (format %#x) skipped: no colour image", (const void *)t, t ? t->d.format : 0); + if (value) { // the queued value is still signalled, in order + vk_present_drain(); + uint64_t f = vk_sync_submit(NULL, 0); + if (f != value) rt_fatal("vk: skipped present signalled %llu, queued as %llu", (unsigned long long)f, (unsigned long long)value); + } return; } PresentJob j = {.image = img->image, .format = img->format, .w = (int)img->width, .h = (int)img->height, .frame = frame, - .at = at_ns, .queued = kern_now_ns()}; + .at = at_ns, .queued = queued, .value = value}; if (crop) { j.has_crop = 1; for (int i = 0; i < 4; i++) j.crop[i] = crop[i]; @@ -121,45 +140,369 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { if (rdoc_active()) { vk_present_drain(); rdoc_after_present(frame); } } -// Pipelines (T6b) drop a released program's pipelines here. -WEAK void vk_program_release(const prog_t *p) { (void)p; } +// ---- Asynchronous replay +enum { RI_SUBMIT, RI_PRESENT, RI_RELEASE }; +typedef struct { + int kind, n, has_crop, crop[4]; + uint64_t value, submit_no, at, queued; + uint64_t *h; + const tex_t *tex; // RI_RELEASE: a copy the item owns (the game may reuse the object's memory) +} ReplayItem; +#define RQ_CAP 1024 // a frame queues 1 submit and 1 present, and up to a few hundred releases +enum { DR_POOL_FINI, DR_MAP_VIRTUAL, DR_TEX_RELEASE, DR_PROG_RELEASE, DR_FINISH, DR_READ_TEXELS, DR_WRITE_TEXELS, + DR_GUEST_READ, DR_FRAME, DR_FULL, DR_CHECK, DR_N }; +static const char *const dr_name[DR_N] = { + "pool_fini", "map_virtual", "texture_release (queued)", "program_release", "finish", "read_texels (writer fence)", + "write_texels", "guest_read (writer fence)", "frame bound", "queue full", "check drain"}; -static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); } -static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); vk_present_drain(); vk_pool_fini(p); } -static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) { - vk_sync_start(); - vk_present_drain(); - return vk_pool_map_virtual(v, n, requests); +static SRWLOCK rl = SRWLOCK_INIT; +static CONDITION_VARIABLE rcv = CONDITION_VARIABLE_INIT; // item queued, item taken, item done +static ReplayItem rq[RQ_CAP]; +static int rq_head, rq_n, r_busy, r_started; +static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames; +// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run. +static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N]; +// TOTK_REPLAY_CHECK: ReadTexels compared with a read after a full drain (under rl): checked, differing, differing +// although no submission recorded a write to the texture in between, differing with the queue idle at the first read. +static uint64_t ck_reads, ck_diff, ck_diff_unexplained, ck_diff_idle; + +int vk_replay_async(void) { + static int on = -1; + if (on < 0) { + const char *e = cfg_get("TOTK_REPLAY_ASYNC"); + on = !(e && !strcmp(e, "0")); + rt_log("vk: replay %s (TOTK_REPLAY_ASYNC=%d)", + on ? "on its own thread, at most one frame behind the game" : "on the submitting thread", on); + } + return on; } -static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); } -static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); } -static void vk_be_texture_release(const tex_t *t) { vk_sync_start(); vk_present_drain(); vk_texture_release(t); } -static void vk_be_program_release(const prog_t *p) { vk_program_release(p); } -static uint64_t vk_be_submit(const uint64_t *handles, int n) { + +// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end. +int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); } + +// The newest submit number (nvn_submit_count) whose replay is done: recording copies it used may be freed. +uint64_t gfx_replayed_submits(void) { + if (!gfx_replay_async()) return nvn_submit_count(); + AcquireSRWLockShared(&rl); + uint64_t v = r_submit_done; + ReleaseSRWLockShared(&rl); + return v; +} + +static uint64_t submit_sync(const uint64_t *handles, int n) { vk_sync_start(); vk_present_drain(); vk_texture_new_submit(); return vk_sync_submit(handles, n); } -static void vk_be_finish(void) { vk_sync_start(); vk_present_drain(); vk_sync_finish(); } + +static DWORD WINAPI replay_main(void *arg) { + (void)arg; + perf_replay_thread(); + AcquireSRWLockExclusive(&rl); + for (;;) { + while (!rq_n) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0); + ReplayItem it = rq[rq_head]; + rq_head = (rq_head + 1) % RQ_CAP; + rq_n--; + r_busy = 1; + WakeAllConditionVariable(&rcv); + ReleaseSRWLockExclusive(&rl); + if (it.kind == RI_SUBMIT) { + uint64_t f = submit_sync(it.h, it.n); + if (f != it.value) + rt_fatal("vk: replayed submission signalled %llu, queued as %llu", (unsigned long long)f, + (unsigned long long)it.value); + free(it.h); + } else if (it.kind == RI_RELEASE) { + vk_present_drain(); + vk_texture_release(it.tex); + free((void *)it.tex); + } else { + uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); + present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value); + } + AcquireSRWLockExclusive(&rl); + if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no; + else r_presents_done++; + r_busy = 0; + WakeAllConditionVariable(&rcv); + } + return 0; +} + +static void wait_counted(int why, uint64_t t0) { // rl held + uint64_t d = kern_now_ns() - t0; + dr_waits[why]++, dr_twaits[why]++; + dr_ns[why] += d, dr_tns[why] += d; + perf_end(PERF_DRAIN, t0); +} + +// A guest thread waiting for the replay gives its virtual core up, as around any blocking host wait (NVN_BLOCKING): +// guest_read (memcpy, pool invalidate), pool and texture finalize run without it, and a drain lasts up to a replay. +// Called and returning with rl held; rl is not held while the core is taken back. +static int core_away_unlocked(void) { + ReleaseSRWLockExclusive(&rl); + int away = kern_core_away(); + AcquireSRWLockExclusive(&rl); + return away; +} +static void core_back_unlocked(int *away) { + if (!*away) return; + ReleaseSRWLockExclusive(&rl); + kern_core_back(away); + AcquireSRWLockExclusive(&rl); +} + +// Queues an item (waits while the queue is full) and returns its value, reserved under rl so values follow queue order. +static uint64_t replay_enqueue(ReplayItem *it) { + AcquireSRWLockExclusive(&rl); + if (!r_started) { + HANDLE h = CreateThread(NULL, 0, replay_main, NULL, 0, NULL); + if (!h) rt_fatal("vk: cannot start the replay thread (error %lu)", GetLastError()); + CloseHandle(h); + r_started = 1; + } + // rl is dropped while the core is taken back, and another thread may fill the queue again meanwhile: check again. + while (rq_n == RQ_CAP) { + uint64_t p0 = perf_begin(); + dr_calls[DR_FULL]++, dr_tcalls[DR_FULL]++; + int away = core_away_unlocked(); + while (rq_n == RQ_CAP) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0); + wait_counted(DR_FULL, p0); + core_back_unlocked(&away); + } + it->value = it->kind == RI_RELEASE ? 0 : vk_sync_reserve(); // a release signals nothing + rq[(rq_head + rq_n) % RQ_CAP] = *it; + rq_n++; + WakeAllConditionVariable(&rcv); + ReleaseSRWLockExclusive(&rl); + return it->value; +} + +// Returns once every queued item was replayed and submitted. Back-end entries only: the replay thread never calls them. +static void replay_drain(int why) { + if (!gfx_replay_async()) return; + AcquireSRWLockExclusive(&rl); + dr_calls[why]++, dr_tcalls[why]++; + if (rq_n || r_busy) { + uint64_t p0 = perf_begin(); + int away = core_away_unlocked(); + while (rq_n || r_busy) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0); + wait_counted(why, p0); + core_back_unlocked(&away); + } + ReleaseSRWLockExclusive(&rl); +} + +// What made the game wait for the replay: since the last line (every 1800 game frames), or over the run (at exit). +static void replay_log(uint64_t frame, int total) { + char buf[1024]; + int k = 0; + AcquireSRWLockExclusive(&rl); + const uint64_t *c = total ? dr_tcalls : dr_calls, *w = total ? dr_twaits : dr_waits, *t = total ? dr_tns : dr_ns; + for (int i = 0; i < DR_N && k < (int)sizeof buf - 96; i++) + if (c[i] || w[i]) + k += snprintf(buf + k, sizeof buf - (size_t)k, "%s %s %llu calls, %llu waited %.1f ms", k ? ";" : "", dr_name[i], + (unsigned long long)c[i], (unsigned long long)w[i], t[i] / 1e6); + if (ck_reads && k < (int)sizeof buf - 128) + k += snprintf(buf + k, sizeof buf - (size_t)k, "; check (run): %llu reads, %llu differ, %llu without a recorded write, %llu with the queue idle", + (unsigned long long)ck_reads, (unsigned long long)ck_diff, (unsigned long long)ck_diff_unexplained, + (unsigned long long)ck_diff_idle); + memset(dr_calls, 0, sizeof dr_calls); + memset(dr_waits, 0, sizeof dr_waits); + memset(dr_ns, 0, sizeof dr_ns); + ReleaseSRWLockExclusive(&rl); + rt_log("vk: replay queue %s frame %llu:%s", total ? "over the run to" : "at", (unsigned long long)frame, + k ? buf : " no waits"); +} + +// The run total, once: the game exits through prof_write (boot.c _exit, the window close), probes through +// vk_backend_shutdown. +void gfx_exit_log(void) { + static LONG done; + if (gfx_replay_async() && r_started && !InterlockedExchange(&done, 1)) replay_log(game_frames, 1); +} + +static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { + vk_sync_start(); + if (!gfx_replay_async()) { + uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); + perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads + present_image(t, crop, at_ns, kern_now_ns(), frame, 0); + return; + } + // Frame bound: the replay thread handled the previous present before this one is queued. + AcquireSRWLockExclusive(&rl); + dr_calls[DR_FRAME]++, dr_tcalls[DR_FRAME]++; + if (r_presents_done < r_presents_queued) { + uint64_t p0 = perf_begin(); + while (r_presents_done < r_presents_queued) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0); + wait_counted(DR_FRAME, p0); + } + r_presents_queued++; + ReleaseSRWLockExclusive(&rl); + uint64_t frame = ++game_frames; + perf_present(frame); + if (frame % 1800 == 0) replay_log(frame, 0); + ReplayItem it = {.kind = RI_PRESENT, .tex = t, .at = at_ns, .queued = kern_now_ns()}; + if (crop) { + it.has_crop = 1; + for (int i = 0; i < 4; i++) it.crop[i] = crop[i]; + } + replay_enqueue(&it); +} + +// Pipelines (T6b) drop a released program's pipelines here. +WEAK void vk_program_release(const prog_t *p) { (void)p; } + +static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); } +static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); replay_drain(DR_POOL_FINI); vk_present_drain(); vk_pool_fini(p); } +static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) { + vk_sync_start(); + replay_drain(DR_MAP_VIRTUAL); + vk_present_drain(); + return vk_pool_map_virtual(v, n, requests); +} +static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); } +static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); } +// Asynchronous: the release is queued behind the submissions before it, so their replay still resolves the texture's +// slots to its image (a released slot follows another live texture at the storage), and the ones after it see it +// released, the order of the synchronous replay. vk_texture_release reads only the object's key fields: a copy. +static void vk_be_texture_release(const tex_t *t) { + vk_sync_start(); + if (!gfx_replay_async()) { + vk_present_drain(); + vk_texture_release(t); + return; + } + if (!t || !t->mem) return; + tex_t *copy = malloc(sizeof *copy); + if (!copy) rt_fatal("vk: out of memory (replay queue)"); + *copy = *t; + ReplayItem it = {.kind = RI_RELEASE, .tex = copy}; + AcquireSRWLockExclusive(&rl); + dr_calls[DR_TEX_RELEASE]++, dr_tcalls[DR_TEX_RELEASE]++; + ReleaseSRWLockExclusive(&rl); + replay_enqueue(&it); +} +static void vk_be_program_release(const prog_t *p) { replay_drain(DR_PROG_RELEASE); vk_program_release(p); } +// Asynchronous: the handles (nvn_submit_prepare_async ran under the front end's submit lock) are copied and queued; +// the value returned is the one the replay will signal. +static uint64_t vk_be_submit(const uint64_t *handles, int n) { + if (!gfx_replay_async()) return submit_sync(handles, n); + vk_sync_start(); + ReplayItem it = {.kind = RI_SUBMIT, .n = n, .submit_no = nvn_submit_count()}; + if (n > 0) { + it.h = malloc(sizeof *it.h * (size_t)n); + if (!it.h) rt_fatal("vk: out of memory (replay queue)"); + memcpy(it.h, handles, sizeof *it.h * (size_t)n); + } + return replay_enqueue(&it); +} +static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); } +static int replay_check(void) { + static int on = -1; + if (on < 0) { + const char *e = cfg_get("TOTK_REPLAY_CHECK"); + on = e && strcmp(e, "0"); + if (on) rt_log("vk: TOTK_REPLAY_CHECK: every ReadTexels is read again after a full replay drain and compared"); + } + return on; +} + +// ReadTexels with the replay queued: waits for the newest submission recorded to write the texture (replayed and +// completed), not for the whole queue. The readback copy goes through the upload batch on the same queue, after every +// submission the replay has submitted. Submissions still queued or being recorded run after the copy, as on hardware +// when the GPU has not reached them yet: the game only sees their texels after waiting on a sync, and every sync +// value covers replay and GPU, so a read the game synchronised sees them. TOTK_REPLAY_CHECK compares with a full drain. static void vk_be_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) { vk_sync_start(); + if (!gfx_replay_async()) { + vk_read_texels(t, v, r, dst, row, img); + return; + } + uint64_t f = vk_texture_last_write(t); + AcquireSRWLockExclusive(&rl); + dr_calls[DR_READ_TEXELS]++, dr_tcalls[DR_READ_TEXELS]++; + int idle = !rq_n && !r_busy; + ReleaseSRWLockExclusive(&rl); + if (f && vk_sync_fence_done() < f) { + uint64_t t0 = kern_now_ns(); + vk_sync_fence_wait(f, UINT64_MAX); // counts as a fence wait in perf + uint64_t d = kern_now_ns() - t0; + AcquireSRWLockExclusive(&rl); + dr_waits[DR_READ_TEXELS]++, dr_twaits[DR_READ_TEXELS]++; + dr_ns[DR_READ_TEXELS] += d, dr_tns[DR_READ_TEXELS] += d; + ReleaseSRWLockExclusive(&rl); + } vk_read_texels(t, v, r, dst, row, img); + if (!replay_check()) return; + size_t n = vk_texture_texel_span(t, v, r, row, img); + uint8_t *again = n ? malloc(n) : NULL; + if (!again) return; + memcpy(again, dst, n); // bytes between the rows stay the caller's + uint64_t f0 = vk_texture_last_write(t); + replay_drain(DR_CHECK); + uint64_t f1 = vk_texture_last_write(t); + vk_read_texels(t, v, r, again, row, img); + int differ = memcmp(again, dst, n) != 0; + static volatile long n_logged; + if (differ && InterlockedIncrement(&n_logged) <= 40) { + size_t at = 0; + while (at < n && again[at] == dst[at]) at++; + rt_log("vk: replay check: ReadTexels of %p (format %#x, %dx%dx%d at %d,%d,%d) differs at byte %zu of %zu; writer " + "fence %llu at the read, %llu after the drain, done %llu; queue %s at the read", + (void *)t->mem, t->d.format, r->width, r->height, r->depth, r->xoffset, r->yoffset, r->zoffset, at, n, + (unsigned long long)f0, (unsigned long long)f1, (unsigned long long)vk_sync_fence_done(), idle ? "idle" : "busy"); + } + AcquireSRWLockExclusive(&rl); + ck_reads++; + if (differ) { + ck_diff++; + if (f1 == f0) ck_diff_unexplained++; + if (idle) ck_diff_idle++; + } + ReleaseSRWLockExclusive(&rl); + free(again); } static void vk_be_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row, uint64_t img) { vk_sync_start(); + replay_drain(DR_WRITE_TEXELS); vk_present_drain(); vk_write_texels(t, v, r, src, row, img); } static void vk_be_guest_written(uint64_t addr, uint64_t size) { vk_sync_start(); vk_guest_written(addr, size); } +// A CPU read of GPU-written mirrored storage (memcpy, pool invalidate). With the replay queued it waits for the newest +// submission recorded to write the textures there, not for the whole queue: the write-back copy then runs after their +// renders, and clears gpu_dirty only for content that is on the queue. Submissions not replayed yet render after the +// copy, as on hardware when the GPU has not reached them; they mark the texture dirty again for a later read. static void vk_be_guest_read(uint64_t addr, uint64_t size) { if (!vk_texture_cpu_stale(addr, size)) return; vk_sync_start(); + if (gfx_replay_async()) { + uint64_t f = vk_texture_dirty_fence(addr, size); + AcquireSRWLockExclusive(&rl); + dr_calls[DR_GUEST_READ]++, dr_tcalls[DR_GUEST_READ]++; + ReleaseSRWLockExclusive(&rl); + if (f && vk_sync_fence_done() < f) { + uint64_t t0 = kern_now_ns(); + NVN_BLOCKING(); + vk_sync_fence_wait(f, UINT64_MAX); + uint64_t d = kern_now_ns() - t0; + AcquireSRWLockExclusive(&rl); + dr_waits[DR_GUEST_READ]++, dr_twaits[DR_GUEST_READ]++; + dr_ns[DR_GUEST_READ] += d, dr_tns[DR_GUEST_READ] += d; + ReleaseSRWLockExclusive(&rl); + } + } vk_present_drain(); vk_texture_write_back(addr, size); } +// After a fence wait: the value is done, so its replay is too; a later submission the replay records meanwhile marks +// its targets with a later fence, which write_back_small leaves alone. No drain. static void vk_be_cpu_synced(uint64_t fence) { if (!vk_texture_small_dirty()) return; vk_sync_start(); @@ -189,6 +532,8 @@ const GfxBackend gfx_vulkan_backend = { }; void vk_backend_shutdown(void) { + replay_drain(DR_FINISH); + gfx_exit_log(); vk_present_drain(); vk_sync_shutdown(); vk_mem_shutdown(); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index c2b19645..6f00f7c1 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -922,6 +922,7 @@ static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int hv; const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv); if (img) view = get_view(ex, img, &v, hv, r->type, storage); + if (img && storage) vk_texture_storage_bound_locked(img); if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT; if (!storage && !separate) samp = vk_sampler_by_id_locked((int)(h >> 20 & 0xfff)); } diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 2870b3cb..b3a2a1bb 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -42,6 +42,7 @@ static SRWLOCK st_lock = SRWLOCK_INIT; static CONDITION_VARIABLE cv_work = CONDITION_VARIABLE_INIT; // a submission, a callback or stop static CONDITION_VARIABLE cv_done = CONDITION_VARIABLE_INIT; // done advanced static uint64_t issued, done; // newest submitted value; newest value whose side effects ran +static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued) static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; @@ -223,6 +224,7 @@ void vk_sync_start(void) { } AcquireSRWLockExclusive(&st_lock); issued = done = now; + if (reserved < now) reserved = now; stop = 0; ReleaseSRWLockExclusive(&st_lock); vk_mem_note_submit(now); @@ -301,6 +303,17 @@ VkCommandBuffer vk_sync_split(void) { return s->cb; } +// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed +// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value. +uint64_t vk_sync_reserve(void) { + vk_sync_start(); + AcquireSRWLockExclusive(&st_lock); + if (reserved < issued) reserved = issued; + uint64_t v = ++reserved; + ReleaseSRWLockExclusive(&st_lock); + return v; +} + uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_sync_start(); VkCtx *c = vk_ctx(); @@ -378,8 +391,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { vk_sync_start(); AcquireSRWLockExclusive(&st_lock); int ok; - if (value > issued) { - ok = 0; // never submitted: never reached (as in the null back end) + if (value > issued && value > reserved) { + ok = 0; // never submitted nor queued: never reached (as in the null back end) } else { uint64_t t0 = kern_now_ns(); while (done < value) { diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index ec0e012d..5ad2b006 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -52,6 +52,8 @@ typedef struct Tex { uint32_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes) uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one) + uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear, + // storage binding); ReadTexels waits for it instead of draining the replay queue uint8_t listed; // in owned[] uint32_t wt_n; // WriteTexels into the storage since the last upload } Tex; @@ -819,6 +821,8 @@ void vk_texture_gpu_written(const tex_t *t) { if (e) { make_fresh(e); // a partial render keeps what the storage held before e->pub.gpu_owned = 1; + uint64_t f = vk_exec_recording_fence(); + if (f > e->wr_fence) e->wr_fence = f; if (e->pub.mirrored) { e->gpu_dirty = 1; storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render @@ -1240,6 +1244,28 @@ static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) { return n; } +// A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c +// tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels. +void vk_texture_storage_bound_locked(const VkTexImage *img) { + Tex *e = (Tex *)img; // pub is the first member + uint64_t f = vk_exec_recording_fence(); + if (f > e->wr_fence) e->wr_fence = f; +} + +uint64_t vk_texture_last_write(const tex_t *t) { + lock(); + Tex *e = entry_of(t); + uint64_t f = e ? e->wr_fence : 0; + unlock(); + return f; +} + +size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) { + Region g; + if (!region_of(t, v, r, row, img, &g)) return 0; + return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb; +} + int vk_texture_cpu_stale(uint64_t addr, uint64_t size) { if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0; AcquireSRWLockShared(&owned_lock); @@ -1248,6 +1274,21 @@ int vk_texture_cpu_stale(uint64_t addr, uint64_t size) { return dirty; } +// The newest submission recorded to write a GPU-written mirrored texture overlapping [addr, addr+size); 0: none. +uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size) { + if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0; + lock(); + uint64_t f = 0; + for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) { + const Tex *e = owned[i]; + if (!e->gpu_dirty || e->pub.addr + e->t.L.size <= addr) continue; + if (e->gpu_fence > f) f = e->gpu_fence; + if (e->wr_fence > f) f = e->wr_fence; + } + unlock(); + return f; +} + // The image is the newest content only while nothing wrote the storage since the GPU render: TotK finalizes grass maps, // reuses their storage for new data and memcpys that data on, and writing the old image back first handed the copy // a stale map or stale resource bytes (wrong textures in the distance). Every hit is checked before any is written