mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 12:45:57 +00:00
Merge the replay of NVN submissions on a queued thread, with reads waiting for the newest writer instead of a drain, behind TOTK_REPLAY_ASYNC
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
84abfaaf2c
12 files changed
+558
-37
No files matched your search
@@ -14,6 +14,7 @@
|
||||
|
||||
#include "cfg.h"
|
||||
#include "gfx.h"
|
||||
#include "nvn_api.h"
|
||||
#include "vk.h"
|
||||
|
||||
#define SYM_INIT_START "_ZN2nn4init5StartEmmPFvvES2_S2_"
|
||||
@@ -200,6 +201,7 @@ static void save_shot(const tex_t *t, const char *path) {
|
||||
view_t view;
|
||||
memset(&view, 0, sizeof view);
|
||||
NVNcopyRegion r = {.width = (int)w, .height = (int)h, .depth = 1};
|
||||
if (gfx_replay_async()) gfx_backend()->finish(); // the presented frame may still be queued for the replay
|
||||
gfx_backend()->read_texels(t, &view, &r, px, 4ull * w, 4ull * w * h);
|
||||
if (bgra)
|
||||
for (uint64_t i = 0; i < 4ull * w * h; i += 4) {
|
||||
@@ -238,6 +240,7 @@ static void series_shot(const tex_t *t, uint64_t n) {
|
||||
view_t view;
|
||||
memset(&view, 0, sizeof view);
|
||||
NVNcopyRegion r = {.width = (int)w, .height = (int)h, .depth = 1};
|
||||
if (gfx_replay_async()) gfx_backend()->finish(); // the presented frame may still be queued for the replay
|
||||
gfx_backend()->read_texels(t, &view, &r, px, 4ull * w, 4ull * w * h);
|
||||
for (uint32_t y = 0; y < oh; y++)
|
||||
for (uint32_t x = 0; x < ow; x++) {
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ static const char *const known[] = {
|
||||
"MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
|
||||
"PREWARM_XLATE",
|
||||
"PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC",
|
||||
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
|
||||
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
|
||||
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS",
|
||||
"SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL",
|
||||
"SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS",
|
||||
|
||||
@@ -28,6 +28,7 @@ void prof_write(void); // idempotent; boot.c calls it b
|
||||
uint64_t perf_begin(void); // start time of a wait; pair with perf_end
|
||||
void perf_end(int kind, uint64_t t0); // counts render-thread time only
|
||||
void perf_replay(uint64_t t0); // replay (NVN -> Vulkan recording) time on any thread, waits inside it included
|
||||
void perf_replay_thread(void); // the calling thread replays for the render thread: perf_end counts its waits
|
||||
// Events per 60-frame window: graphics pipelines finished by the background workers, draws skipped while pending,
|
||||
// first texture uploads and re-uploads after CPU writes, draws skipped while a shader translates in the background,
|
||||
// draws that waited for their pending pipeline and got it, draws into a target the previous frame did not render that
|
||||
@@ -41,6 +42,7 @@ enum { PERF_CNT_PIPE_ASYNC, PERF_CNT_PIPE_SKIP, PERF_CNT_TEX_UP, PERF_CNT_TEX_RE
|
||||
void perf_count(int kind);
|
||||
void perf_present(uint64_t frame);
|
||||
void perf_summary_flush(void); // prints the frame-time summary of the running period (exit); the 60 s lines come from perf_present
|
||||
void gfx_exit_log(void); // back end's lines for the exit summary (weak no-op in prof_win.c)
|
||||
void perf_gpu_wait(uint64_t t0); // a wait for a GPU fence ended (any thread; the render thread's also counts as PERF_FENCE)
|
||||
// GPU side of the 60-frame perf line, filled by the graphics back end when it set perf_gpu_stats. gpu_ns: GPU busy time
|
||||
// since the previous call (timestamp queries, overlaps counted once), have_gpu 0 when the device has no timestamps.
|
||||
|
||||
@@ -51,6 +51,11 @@ void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the ga
|
||||
// CPU class (0 CPU_CACHED, 1 CPU_UNCACHED, 2 other) since the previous call.
|
||||
void nvn_flush_stats(uint64_t out[2][3][2]);
|
||||
void nvn_submit_prepare(const uint64_t *handles, int n); // weak no-op; the command-buffer slice reads deferred binds
|
||||
// Asynchronous replay: as nvn_submit_prepare, writing the handles to replay to out (a clone where a still queued
|
||||
// submission uses the same recording with other deferred binds)
|
||||
void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n);
|
||||
int gfx_replay_async(void); // 1: the back end replays submissions on its own thread
|
||||
uint64_t gfx_replayed_submits(void); // newest submit number (nvn_submit_count) replayed
|
||||
#define NVN_HANDLE_TEX(h) ((int)((h) & 0xfffff)) // combined texture handle = texture id | sampler id << 20
|
||||
#define NVN_HANDLE_SAMP(h) ((int)(((h) >> 20) & 0xfff))
|
||||
extern GfxStats gfx_counters; // behind gfx_stats(); add with __atomic_fetch_add. frames:
|
||||
|
||||
+74
-6
@@ -160,7 +160,7 @@ static const tex_t *tex_pin(const NVNtexture *nt) {
|
||||
// ---------- recording copies
|
||||
// A handle points just past a copy_prefix. Copies are indexed by the 4 KiB page of their control-memory position, so
|
||||
// adding control memory again releases the copies recorded there without walking every live copy.
|
||||
typedef struct { uint64_t magic, bytes, made_at, used_at; } copy_prefix;
|
||||
typedef struct { uint64_t magic, bytes, made_at, used_at, deferred; } copy_prefix; // deferred: 0 unknown, 1 none, 2 some
|
||||
_Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned");
|
||||
typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy;
|
||||
typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket;
|
||||
@@ -168,6 +168,30 @@ static SRWLOCK copy_lock = SRWLOCK_INIT; // guards the two maps, the counters
|
||||
static nvn_hmap copy_pages = {.klen = 8}; // page -> copy_bucket
|
||||
static nvn_hmap copy_live = {.klen = 8}; // handle -> copy_prefix of a live copy
|
||||
static uint64_t ncopies, copy_bytes, copies_made, nsubmits;
|
||||
// Asynchronous replay: copies released by the game, and per-submission clones, that a queued submission still reads.
|
||||
// Freed once gfx_replayed_submits() reaches their used_at. Empty with synchronous replay.
|
||||
static copy_prefix **zombies;
|
||||
static uint32_t nzombies, cap_zombies;
|
||||
|
||||
static void zombie_add(copy_prefix *pf) { // copy_lock held
|
||||
if (nzombies == cap_zombies) {
|
||||
cap_zombies = cap_zombies ? cap_zombies * 2 : 64;
|
||||
zombies = realloc(zombies, sizeof *zombies * cap_zombies);
|
||||
if (!zombies) rt_fatal("nvn: out of memory (released recording copies)");
|
||||
}
|
||||
zombies[nzombies++] = pf;
|
||||
}
|
||||
|
||||
static void zombie_sweep(uint64_t replayed) { // copy_lock held
|
||||
for (uint32_t i = 0; i < nzombies;) {
|
||||
if (zombies[i]->used_at <= replayed) {
|
||||
free(zombies[i]);
|
||||
zombies[i] = zombies[--nzombies];
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static copy_prefix *copy_find(uint64_t handle) { // copy_lock held
|
||||
return nvn_hm_get(©_live, &handle);
|
||||
@@ -258,7 +282,9 @@ static void copy_add(uintptr_t lo, uintptr_t hi, uint8_t *copy) {
|
||||
// Adding control memory again retires every recording copy made in it (the driver's handle lived there).
|
||||
static void release_copies(uintptr_t lo, uintptr_t hi) {
|
||||
if (hi <= lo) return;
|
||||
uint64_t replayed = gfx_replayed_submits();
|
||||
AcquireSRWLockExclusive(©_lock);
|
||||
if (nzombies) zombie_sweep(replayed);
|
||||
for (uintptr_t pg = lo >> COPY_PAGE_SHIFT; pg <= (hi - 1) >> COPY_PAGE_SHIFT; pg++) {
|
||||
copy_bucket *b = nvn_hm_get(©_pages, &pg);
|
||||
if (!b) continue;
|
||||
@@ -271,7 +297,8 @@ static void release_copies(uintptr_t lo, uintptr_t hi) {
|
||||
ncopies--;
|
||||
copy_bytes -= pf->bytes;
|
||||
pf->magic = 0;
|
||||
free(pf);
|
||||
if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it
|
||||
else free(pf);
|
||||
b->v[i] = b->v[--b->n];
|
||||
}
|
||||
if (!b->n) {
|
||||
@@ -836,16 +863,17 @@ NVN_API(nvnCommandBufferSetPatchSize) { (void)c; }
|
||||
// What the console reads at the submit call is read here, on the game thread: every handle must be a live recording,
|
||||
// and deferred binds take the handles the game's arrays hold now (the game fills its tables for the next frame as soon
|
||||
// as the submit returns). The result is written into the recording copy.
|
||||
static uint64_t def_reads, def_changed;
|
||||
static uint64_t def_reads, def_changed, n_clones;
|
||||
|
||||
static void resolve_deferred(const uint8_t *p, int depth) {
|
||||
static int resolve_deferred(const uint8_t *p, int depth) { // returns the number of deferred binds
|
||||
int nd = 0;
|
||||
for (;;) {
|
||||
op_hdr h;
|
||||
memcpy(&h, p, sizeof h);
|
||||
if (h.size < 8 || (h.size & 7) || h.op >= OP_COUNT_)
|
||||
rt_fatal("nvn: corrupt recording at %p while resolving deferred binds (op %u size %u)", (const void *)p,
|
||||
h.op, h.size);
|
||||
if (h.op == OP_END) return;
|
||||
if (h.op == OP_END) return nd;
|
||||
if (h.op == OP_JUMP) {
|
||||
memcpy(&p, p + 8, 8);
|
||||
continue;
|
||||
@@ -854,10 +882,11 @@ static void resolve_deferred(const uint8_t *p, int depth) {
|
||||
uint64_t t;
|
||||
memcpy(&t, p + 8, 8);
|
||||
if (depth >= 16) rt_fatal("nvn: deferred bind resolution: call depth over 16");
|
||||
resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1);
|
||||
nd += resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1);
|
||||
} else if (h.op == OP_BIND_TEX || h.op == OP_BIND_IMG || h.op == OP_BIND_SAMP) {
|
||||
p_bind_handles *b = (p_bind_handles *)(uintptr_t)(p + 8);
|
||||
if (b->src && b->n > 0 && b->n <= 32) {
|
||||
nd++;
|
||||
uint64_t cur[32];
|
||||
memcpy(cur, (const void *)(uintptr_t)b->src, 8 * (size_t)b->n);
|
||||
__atomic_fetch_add(&def_reads, 1, __ATOMIC_RELAXED);
|
||||
@@ -879,6 +908,45 @@ void nvn_submit_prepare(const uint64_t *handles, int n) {
|
||||
for (int i = 0; i < n; i++) resolve_deferred((const uint8_t *)(uintptr_t)handles[i], 0);
|
||||
}
|
||||
|
||||
// Asynchronous replay. A copy a queued submission still reads keeps that submission's deferred binds: when it holds
|
||||
// deferred binds (or is not known to hold none), this submission gets a clone, resolved here and freed after its replay.
|
||||
void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) {
|
||||
uint64_t now = __atomic_add_fetch(&nsubmits, 1, __ATOMIC_RELAXED);
|
||||
uint64_t replayed = gfx_replayed_submits();
|
||||
AcquireSRWLockExclusive(©_lock);
|
||||
if (nzombies) zombie_sweep(replayed);
|
||||
for (int i = 0; i < n; i++) {
|
||||
copy_prefix *pf = copy_find(handles[i]);
|
||||
if (!pf) {
|
||||
ReleaseSRWLockExclusive(©_lock);
|
||||
rt_fatal("nvn: QueueSubmitCommands: command handle %#llx is not a live recording (released or never recorded)",
|
||||
(unsigned long long)handles[i]);
|
||||
}
|
||||
out[i] = handles[i];
|
||||
if (pf->used_at > replayed && pf->used_at != now && pf->deferred != 1) {
|
||||
size_t sz = sizeof *pf + pf->bytes;
|
||||
copy_prefix *cl = malloc(sz);
|
||||
if (!cl) rt_fatal("nvn: out of memory (recording clone, %zu bytes)", sz);
|
||||
memcpy(cl, pf, sz);
|
||||
cl->magic = 0; // not a live handle: the game never sees it
|
||||
cl->used_at = now;
|
||||
zombie_add(cl);
|
||||
out[i] = (uintptr_t)(cl + 1);
|
||||
__atomic_fetch_add(&n_clones, 1, __ATOMIC_RELAXED);
|
||||
} else {
|
||||
pf->used_at = now;
|
||||
}
|
||||
}
|
||||
ReleaseSRWLockExclusive(©_lock);
|
||||
for (int i = 0; i < n; i++) {
|
||||
int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0);
|
||||
if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1;
|
||||
}
|
||||
if (now % 7200 == 0)
|
||||
rt_log("nvn: asynchronous replay at submit %llu: %llu recording clones so far, %u copies waiting for the replay",
|
||||
(unsigned long long)now, (unsigned long long)__atomic_load_n(&n_clones, __ATOMIC_RELAXED), nzombies);
|
||||
}
|
||||
|
||||
uint64_t nvn_submit_count(void) { return __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); }
|
||||
|
||||
const char *nvn_op_name(uint32_t op) {
|
||||
|
||||
+24
-3
@@ -58,6 +58,11 @@ static void ov_memcpy(Cpu *c) {
|
||||
HOST_OVERRIDE("memcpy", ov_memcpy);
|
||||
|
||||
__attribute__((weak)) void nvn_submit_prepare(const uint64_t *handles, int n) { (void)handles, (void)n; }
|
||||
__attribute__((weak)) void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) {
|
||||
memcpy(out, handles, sizeof *out * (size_t)n);
|
||||
}
|
||||
__attribute__((weak)) int gfx_replay_async(void) { return 0; }
|
||||
__attribute__((weak)) uint64_t gfx_replayed_submits(void) { return nvn_submit_count(); }
|
||||
|
||||
// ---------- pools: GPU window (section 3.1) and the address index
|
||||
static SRWLOCK pool_lock = SRWLOCK_INIT; // guards gva and index
|
||||
@@ -560,7 +565,9 @@ NVN_API(nvnQueueFinish) {
|
||||
static uint64_t lod_entered, lod_issued, lod_fence[LOD_RING];
|
||||
|
||||
// void nvnQueueSubmitCommands(NVNqueue *q, int n, const NVNcommandHandle *handles): replayed on this thread with its
|
||||
// core given up (section 2.2); the newest fence value is what QueueFenceSync records
|
||||
// core given up (section 2.2), or queued for the replay thread (TOTK_REPLAY_ASYNC, vk_backend.c); the newest fence
|
||||
// value is what QueueFenceSync records. With the queue the fence covers replay and GPU, so the LOD guard below waits
|
||||
// for both.
|
||||
NVN_API(nvnQueueSubmitCommands) {
|
||||
queue_t *q = arg_p(c, 0);
|
||||
int n = (int)arg_x(c, 1);
|
||||
@@ -569,8 +576,22 @@ NVN_API(nvnQueueSubmitCommands) {
|
||||
NVN_BLOCKING();
|
||||
uint64_t no = __atomic_add_fetch(&lod_entered, 1, __ATOMIC_SEQ_CST);
|
||||
q->submits++;
|
||||
nvn_submit_prepare(h, n);
|
||||
uint64_t f = gfx_backend()->submit(h, n), prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED);
|
||||
uint64_t f;
|
||||
if (gfx_replay_async()) {
|
||||
// Prepare and queue under one lock: submit numbers follow queue order (gfx_replayed_submits).
|
||||
static SRWLOCK submit_lock = SRWLOCK_INIT;
|
||||
uint64_t local[32], *hs = n <= 32 ? local : malloc(sizeof *hs * (size_t)n);
|
||||
if (!hs) rt_fatal("nvn: out of memory (submit of %d handles)", n);
|
||||
AcquireSRWLockExclusive(&submit_lock);
|
||||
nvn_submit_prepare_async(h, hs, n);
|
||||
f = gfx_backend()->submit(hs, n);
|
||||
ReleaseSRWLockExclusive(&submit_lock);
|
||||
if (hs != local) free(hs);
|
||||
} else {
|
||||
nvn_submit_prepare(h, n);
|
||||
f = gfx_backend()->submit(h, n);
|
||||
}
|
||||
uint64_t prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED);
|
||||
while (f > prev && !__atomic_compare_exchange_n(&last_fence, &prev, f, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {}
|
||||
__atomic_store_n(&lod_fence[no % LOD_RING], f, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&lod_issued, no, __ATOMIC_SEQ_CST);
|
||||
|
||||
+20
-4
@@ -269,11 +269,22 @@ static uint64_t wait_ns[PERF_N], frame_ns[PERF_N], worst_ns[PERF_N]; // window
|
||||
|
||||
uint64_t perf_begin(void) { return kern_now_ns(); }
|
||||
|
||||
static volatile DWORD replay_tid; // asynchronous replay (vk_backend.c): its waits hold up the frame too
|
||||
|
||||
void perf_replay_thread(void) { replay_tid = GetCurrentThreadId(); }
|
||||
|
||||
void perf_end(int kind, uint64_t t0) {
|
||||
if (!t0 || GetCurrentThreadId() != render_tid) return;
|
||||
if (!t0) return;
|
||||
DWORD tid = GetCurrentThreadId();
|
||||
if (tid != render_tid && tid != replay_tid) return;
|
||||
uint64_t d = kern_now_ns() - t0;
|
||||
wait_ns[kind] += d;
|
||||
frame_ns[kind] += d;
|
||||
if (tid == render_tid && !replay_tid) { // one thread adds: no atomics on the synchronous path
|
||||
wait_ns[kind] += d;
|
||||
frame_ns[kind] += d;
|
||||
return;
|
||||
}
|
||||
__atomic_fetch_add(&wait_ns[kind], d, __ATOMIC_RELAXED);
|
||||
__atomic_fetch_add(&frame_ns[kind], d, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static volatile LONG64 other_wait_ns; // perf_gpu_wait on threads other than the render thread, over the window
|
||||
@@ -339,8 +350,13 @@ static void ft_report(uint64_t now, const char *tag) {
|
||||
ft.t0 = now;
|
||||
}
|
||||
|
||||
__attribute__((weak)) void gfx_exit_log(void) {}
|
||||
|
||||
void perf_summary_flush(void) { // at exit (prof_write)
|
||||
if (summary_t0) ft_report(kern_now_ns(), "exit");
|
||||
if (summary_t0) {
|
||||
ft_report(kern_now_ns(), "exit");
|
||||
gfx_exit_log();
|
||||
}
|
||||
summary_t0 = 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -261,6 +261,9 @@ VkSampler vk_sampler_by_id_locked(int id);
|
||||
uint32_t vk_texture_slots_gen(void);
|
||||
uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered
|
||||
void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage
|
||||
void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission
|
||||
uint64_t vk_texture_last_write(const tex_t *t); // timeline value of the newest submission recorded to write it; 0: none
|
||||
size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img); // bytes ReadTexels spans; 0 for a bad region
|
||||
int vk_texture_small_dirty(void); // a small mirrored target holds GPU texels its storage lacks
|
||||
void vk_texture_write_back_small(uint64_t fence); // writes those rendered by work up to fence back to storage
|
||||
void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written); // vk_dump
|
||||
@@ -268,6 +271,7 @@ int vk_texture_slot_swz(int id, uint8_t swz[4]); // vk_dump: swizzle of the d
|
||||
int vk_texture_read_image(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row,
|
||||
uint64_t img); // always from the image (GPU copy, waited for); 0 on failure
|
||||
int vk_texture_cpu_stale(uint64_t addr, uint64_t size); // a GPU-written mirrored texture there has stale storage
|
||||
uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size); // newest recorded writer of such a texture; 0: none
|
||||
void vk_texture_write_back(uint64_t addr, uint64_t size); // writes those images back into their storage (waits)
|
||||
void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); // vk_pool_fini: drop the images there
|
||||
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads);
|
||||
@@ -286,6 +290,7 @@ void rt_write_png(const char *path, const uint8_t *rgba, uint32_t w, uint32_t h)
|
||||
// once the GPU reached it and the completion thread ran its side effects. Guest or host threads.
|
||||
void vk_sync_start(void); // vk_init (fatal without a device) and the completion thread; idempotent
|
||||
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
|
||||
uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals
|
||||
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
|
||||
uint64_t vk_sync_fence_done(void); // highest done value
|
||||
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
|
||||
@@ -308,6 +313,7 @@ void rdoc_init(void); // runtime/rdoc_win.c: TOTK_RDOC debug
|
||||
int rdoc_active(void);
|
||||
void rdoc_after_present(uint64_t frame);
|
||||
void vk_present_drain(void); // returns once every queued present was submitted to gfx_queue
|
||||
int vk_replay_async(void); // TOTK_REPLAY_ASYNC (default 1): submissions replay on the replay thread
|
||||
void vk_program_release(const prog_t *p); // GfxBackend.program_release: pipelines (T6b); weak no-op in vk_backend.c
|
||||
void vk_backend_shutdown(void); // presents, vk_sync_shutdown, vk_mem_shutdown, vk_shutdown in that order
|
||||
|
||||
|
||||
+366
-21
@@ -10,8 +10,19 @@
|
||||
// Back-edge pacing: the present thread submits the blit at once, then holds the host present (vkQueuePresentKHR) until
|
||||
// the frame's display time from the pacer (nvn_api_obj.c), so present-to-present intervals stay even whatever the
|
||||
// replay took. The blit is already on the queue, so the wait blocks no submission.
|
||||
//
|
||||
// Asynchronous replay (TOTK_REPLAY_ASYNC, default 1; 0 keeps the synchronous path unchanged): nvnQueueSubmitCommands
|
||||
// and nvnQueuePresentTexture only queue. Each item gets its timeline value at enqueue (vk_sync_reserve), in queue
|
||||
// order; the replay thread runs the synchronous submit and present bodies in that order, so every value is signalled
|
||||
// by the submission it was handed out for. The game runs at most one frame ahead: queueing present N waits until the
|
||||
// replay thread handled present N-1. Calls that read or free GPU state that queued work may use drain the queue first
|
||||
// (replay_drain), each cause counted. Design and the orderings it keeps: build/v121-replayq/NOTES.md.
|
||||
#include <windows.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "cfg.h"
|
||||
#include "flight.h"
|
||||
#include "gfx.h"
|
||||
#include "host.h"
|
||||
@@ -21,7 +32,7 @@
|
||||
|
||||
#define WEAK __attribute__((weak))
|
||||
|
||||
typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued; } PresentJob;
|
||||
typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued, value; } PresentJob;
|
||||
|
||||
static SRWLOCK pl = SRWLOCK_INIT;
|
||||
static CONDITION_VARIABLE pcv = CONDITION_VARIABLE_INIT; // job queued, job taken, job submitted
|
||||
@@ -52,7 +63,10 @@ static DWORD WINAPI present_main(void *arg) {
|
||||
ReleaseSRWLockExclusive(&pl);
|
||||
if (flight_on) vk_thumbs_frame(j.frame);
|
||||
vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.w, j.h, j.has_crop ? j.crop : NULL);
|
||||
vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too
|
||||
uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too
|
||||
if (j.value && f != j.value)
|
||||
rt_fatal("vk: the present of frame %llu signalled %llu, queued as %llu", (unsigned long long)j.frame,
|
||||
(unsigned long long)f, (unsigned long long)j.value);
|
||||
AcquireSRWLockExclusive(&pl);
|
||||
busy = 0;
|
||||
WakeAllConditionVariable(&pcv);
|
||||
@@ -84,11 +98,11 @@ void vk_present_drain(void) {
|
||||
perf_end(PERF_DRAIN, p0);
|
||||
}
|
||||
|
||||
static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
|
||||
// The present after the frame count: resolves the image and hands the blit to the present thread. queued: when the game
|
||||
// presented; value: the timeline value queued for this present (asynchronous replay), 0 for the next one (synchronous).
|
||||
static void present_image(const tex_t *t, const int crop[4], uint64_t at_ns, uint64_t queued, uint64_t frame,
|
||||
uint64_t value) {
|
||||
static volatile long n_skipped;
|
||||
vk_sync_start();
|
||||
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
|
||||
perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads
|
||||
vk_exec_frame_log(t, frame);
|
||||
vk_mem_census(frame);
|
||||
if (vk_dump_armed) vk_dump_present(frame);
|
||||
@@ -97,10 +111,15 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
|
||||
if (!img || !(img->aspect & VK_IMAGE_ASPECT_COLOR_BIT)) {
|
||||
if (InterlockedIncrement(&n_skipped) <= 8)
|
||||
rt_log("vk: present of texture %p (format %#x) skipped: no colour image", (const void *)t, t ? t->d.format : 0);
|
||||
if (value) { // the queued value is still signalled, in order
|
||||
vk_present_drain();
|
||||
uint64_t f = vk_sync_submit(NULL, 0);
|
||||
if (f != value) rt_fatal("vk: skipped present signalled %llu, queued as %llu", (unsigned long long)f, (unsigned long long)value);
|
||||
}
|
||||
return;
|
||||
}
|
||||
PresentJob j = {.image = img->image, .format = img->format, .w = (int)img->width, .h = (int)img->height, .frame = frame,
|
||||
.at = at_ns, .queued = kern_now_ns()};
|
||||
.at = at_ns, .queued = queued, .value = value};
|
||||
if (crop) {
|
||||
j.has_crop = 1;
|
||||
for (int i = 0; i < 4; i++) j.crop[i] = crop[i];
|
||||
@@ -121,45 +140,369 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
|
||||
if (rdoc_active()) { vk_present_drain(); rdoc_after_present(frame); }
|
||||
}
|
||||
|
||||
// Pipelines (T6b) drop a released program's pipelines here.
|
||||
WEAK void vk_program_release(const prog_t *p) { (void)p; }
|
||||
// ---- Asynchronous replay
|
||||
enum { RI_SUBMIT, RI_PRESENT, RI_RELEASE };
|
||||
typedef struct {
|
||||
int kind, n, has_crop, crop[4];
|
||||
uint64_t value, submit_no, at, queued;
|
||||
uint64_t *h;
|
||||
const tex_t *tex; // RI_RELEASE: a copy the item owns (the game may reuse the object's memory)
|
||||
} ReplayItem;
|
||||
#define RQ_CAP 1024 // a frame queues 1 submit and 1 present, and up to a few hundred releases
|
||||
enum { DR_POOL_FINI, DR_MAP_VIRTUAL, DR_TEX_RELEASE, DR_PROG_RELEASE, DR_FINISH, DR_READ_TEXELS, DR_WRITE_TEXELS,
|
||||
DR_GUEST_READ, DR_FRAME, DR_FULL, DR_CHECK, DR_N };
|
||||
static const char *const dr_name[DR_N] = {
|
||||
"pool_fini", "map_virtual", "texture_release (queued)", "program_release", "finish", "read_texels (writer fence)",
|
||||
"write_texels", "guest_read (writer fence)", "frame bound", "queue full", "check drain"};
|
||||
|
||||
static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); }
|
||||
static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); vk_present_drain(); vk_pool_fini(p); }
|
||||
static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) {
|
||||
vk_sync_start();
|
||||
vk_present_drain();
|
||||
return vk_pool_map_virtual(v, n, requests);
|
||||
static SRWLOCK rl = SRWLOCK_INIT;
|
||||
static CONDITION_VARIABLE rcv = CONDITION_VARIABLE_INIT; // item queued, item taken, item done
|
||||
static ReplayItem rq[RQ_CAP];
|
||||
static int rq_head, rq_n, r_busy, r_started;
|
||||
static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames;
|
||||
// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run.
|
||||
static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N];
|
||||
// TOTK_REPLAY_CHECK: ReadTexels compared with a read after a full drain (under rl): checked, differing, differing
|
||||
// although no submission recorded a write to the texture in between, differing with the queue idle at the first read.
|
||||
static uint64_t ck_reads, ck_diff, ck_diff_unexplained, ck_diff_idle;
|
||||
|
||||
int vk_replay_async(void) {
|
||||
static int on = -1;
|
||||
if (on < 0) {
|
||||
const char *e = cfg_get("TOTK_REPLAY_ASYNC");
|
||||
on = !(e && !strcmp(e, "0"));
|
||||
rt_log("vk: replay %s (TOTK_REPLAY_ASYNC=%d)",
|
||||
on ? "on its own thread, at most one frame behind the game" : "on the submitting thread", on);
|
||||
}
|
||||
return on;
|
||||
}
|
||||
static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); }
|
||||
static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); }
|
||||
static void vk_be_texture_release(const tex_t *t) { vk_sync_start(); vk_present_drain(); vk_texture_release(t); }
|
||||
static void vk_be_program_release(const prog_t *p) { vk_program_release(p); }
|
||||
static uint64_t vk_be_submit(const uint64_t *handles, int n) {
|
||||
|
||||
// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end.
|
||||
int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); }
|
||||
|
||||
// The newest submit number (nvn_submit_count) whose replay is done: recording copies it used may be freed.
|
||||
uint64_t gfx_replayed_submits(void) {
|
||||
if (!gfx_replay_async()) return nvn_submit_count();
|
||||
AcquireSRWLockShared(&rl);
|
||||
uint64_t v = r_submit_done;
|
||||
ReleaseSRWLockShared(&rl);
|
||||
return v;
|
||||
}
|
||||
|
||||
static uint64_t submit_sync(const uint64_t *handles, int n) {
|
||||
vk_sync_start();
|
||||
vk_present_drain();
|
||||
vk_texture_new_submit();
|
||||
return vk_sync_submit(handles, n);
|
||||
}
|
||||
static void vk_be_finish(void) { vk_sync_start(); vk_present_drain(); vk_sync_finish(); }
|
||||
|
||||
static DWORD WINAPI replay_main(void *arg) {
|
||||
(void)arg;
|
||||
perf_replay_thread();
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
for (;;) {
|
||||
while (!rq_n) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
|
||||
ReplayItem it = rq[rq_head];
|
||||
rq_head = (rq_head + 1) % RQ_CAP;
|
||||
rq_n--;
|
||||
r_busy = 1;
|
||||
WakeAllConditionVariable(&rcv);
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
if (it.kind == RI_SUBMIT) {
|
||||
uint64_t f = submit_sync(it.h, it.n);
|
||||
if (f != it.value)
|
||||
rt_fatal("vk: replayed submission signalled %llu, queued as %llu", (unsigned long long)f,
|
||||
(unsigned long long)it.value);
|
||||
free(it.h);
|
||||
} else if (it.kind == RI_RELEASE) {
|
||||
vk_present_drain();
|
||||
vk_texture_release(it.tex);
|
||||
free((void *)it.tex);
|
||||
} else {
|
||||
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
|
||||
present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value);
|
||||
}
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no;
|
||||
else r_presents_done++;
|
||||
r_busy = 0;
|
||||
WakeAllConditionVariable(&rcv);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void wait_counted(int why, uint64_t t0) { // rl held
|
||||
uint64_t d = kern_now_ns() - t0;
|
||||
dr_waits[why]++, dr_twaits[why]++;
|
||||
dr_ns[why] += d, dr_tns[why] += d;
|
||||
perf_end(PERF_DRAIN, t0);
|
||||
}
|
||||
|
||||
// A guest thread waiting for the replay gives its virtual core up, as around any blocking host wait (NVN_BLOCKING):
|
||||
// guest_read (memcpy, pool invalidate), pool and texture finalize run without it, and a drain lasts up to a replay.
|
||||
// Called and returning with rl held; rl is not held while the core is taken back.
|
||||
static int core_away_unlocked(void) {
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
int away = kern_core_away();
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
return away;
|
||||
}
|
||||
static void core_back_unlocked(int *away) {
|
||||
if (!*away) return;
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
kern_core_back(away);
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
}
|
||||
|
||||
// Queues an item (waits while the queue is full) and returns its value, reserved under rl so values follow queue order.
|
||||
static uint64_t replay_enqueue(ReplayItem *it) {
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
if (!r_started) {
|
||||
HANDLE h = CreateThread(NULL, 0, replay_main, NULL, 0, NULL);
|
||||
if (!h) rt_fatal("vk: cannot start the replay thread (error %lu)", GetLastError());
|
||||
CloseHandle(h);
|
||||
r_started = 1;
|
||||
}
|
||||
// rl is dropped while the core is taken back, and another thread may fill the queue again meanwhile: check again.
|
||||
while (rq_n == RQ_CAP) {
|
||||
uint64_t p0 = perf_begin();
|
||||
dr_calls[DR_FULL]++, dr_tcalls[DR_FULL]++;
|
||||
int away = core_away_unlocked();
|
||||
while (rq_n == RQ_CAP) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
|
||||
wait_counted(DR_FULL, p0);
|
||||
core_back_unlocked(&away);
|
||||
}
|
||||
it->value = it->kind == RI_RELEASE ? 0 : vk_sync_reserve(); // a release signals nothing
|
||||
rq[(rq_head + rq_n) % RQ_CAP] = *it;
|
||||
rq_n++;
|
||||
WakeAllConditionVariable(&rcv);
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
return it->value;
|
||||
}
|
||||
|
||||
// Returns once every queued item was replayed and submitted. Back-end entries only: the replay thread never calls them.
|
||||
static void replay_drain(int why) {
|
||||
if (!gfx_replay_async()) return;
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
dr_calls[why]++, dr_tcalls[why]++;
|
||||
if (rq_n || r_busy) {
|
||||
uint64_t p0 = perf_begin();
|
||||
int away = core_away_unlocked();
|
||||
while (rq_n || r_busy) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
|
||||
wait_counted(why, p0);
|
||||
core_back_unlocked(&away);
|
||||
}
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
}
|
||||
|
||||
// What made the game wait for the replay: since the last line (every 1800 game frames), or over the run (at exit).
|
||||
static void replay_log(uint64_t frame, int total) {
|
||||
char buf[1024];
|
||||
int k = 0;
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
const uint64_t *c = total ? dr_tcalls : dr_calls, *w = total ? dr_twaits : dr_waits, *t = total ? dr_tns : dr_ns;
|
||||
for (int i = 0; i < DR_N && k < (int)sizeof buf - 96; i++)
|
||||
if (c[i] || w[i])
|
||||
k += snprintf(buf + k, sizeof buf - (size_t)k, "%s %s %llu calls, %llu waited %.1f ms", k ? ";" : "", dr_name[i],
|
||||
(unsigned long long)c[i], (unsigned long long)w[i], t[i] / 1e6);
|
||||
if (ck_reads && k < (int)sizeof buf - 128)
|
||||
k += snprintf(buf + k, sizeof buf - (size_t)k, "; check (run): %llu reads, %llu differ, %llu without a recorded write, %llu with the queue idle",
|
||||
(unsigned long long)ck_reads, (unsigned long long)ck_diff, (unsigned long long)ck_diff_unexplained,
|
||||
(unsigned long long)ck_diff_idle);
|
||||
memset(dr_calls, 0, sizeof dr_calls);
|
||||
memset(dr_waits, 0, sizeof dr_waits);
|
||||
memset(dr_ns, 0, sizeof dr_ns);
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
rt_log("vk: replay queue %s frame %llu:%s", total ? "over the run to" : "at", (unsigned long long)frame,
|
||||
k ? buf : " no waits");
|
||||
}
|
||||
|
||||
// The run total, once: the game exits through prof_write (boot.c _exit, the window close), probes through
|
||||
// vk_backend_shutdown.
|
||||
void gfx_exit_log(void) {
|
||||
static LONG done;
|
||||
if (gfx_replay_async() && r_started && !InterlockedExchange(&done, 1)) replay_log(game_frames, 1);
|
||||
}
|
||||
|
||||
static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
|
||||
vk_sync_start();
|
||||
if (!gfx_replay_async()) {
|
||||
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
|
||||
perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads
|
||||
present_image(t, crop, at_ns, kern_now_ns(), frame, 0);
|
||||
return;
|
||||
}
|
||||
// Frame bound: the replay thread handled the previous present before this one is queued.
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
dr_calls[DR_FRAME]++, dr_tcalls[DR_FRAME]++;
|
||||
if (r_presents_done < r_presents_queued) {
|
||||
uint64_t p0 = perf_begin();
|
||||
while (r_presents_done < r_presents_queued) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
|
||||
wait_counted(DR_FRAME, p0);
|
||||
}
|
||||
r_presents_queued++;
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
uint64_t frame = ++game_frames;
|
||||
perf_present(frame);
|
||||
if (frame % 1800 == 0) replay_log(frame, 0);
|
||||
ReplayItem it = {.kind = RI_PRESENT, .tex = t, .at = at_ns, .queued = kern_now_ns()};
|
||||
if (crop) {
|
||||
it.has_crop = 1;
|
||||
for (int i = 0; i < 4; i++) it.crop[i] = crop[i];
|
||||
}
|
||||
replay_enqueue(&it);
|
||||
}
|
||||
|
||||
// Pipelines (T6b) drop a released program's pipelines here.
|
||||
WEAK void vk_program_release(const prog_t *p) { (void)p; }
|
||||
|
||||
static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); }
|
||||
static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); replay_drain(DR_POOL_FINI); vk_present_drain(); vk_pool_fini(p); }
|
||||
static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) {
|
||||
vk_sync_start();
|
||||
replay_drain(DR_MAP_VIRTUAL);
|
||||
vk_present_drain();
|
||||
return vk_pool_map_virtual(v, n, requests);
|
||||
}
|
||||
static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); }
|
||||
static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); }
|
||||
// Asynchronous: the release is queued behind the submissions before it, so their replay still resolves the texture's
|
||||
// slots to its image (a released slot follows another live texture at the storage), and the ones after it see it
|
||||
// released, the order of the synchronous replay. vk_texture_release reads only the object's key fields: a copy.
|
||||
static void vk_be_texture_release(const tex_t *t) {
|
||||
vk_sync_start();
|
||||
if (!gfx_replay_async()) {
|
||||
vk_present_drain();
|
||||
vk_texture_release(t);
|
||||
return;
|
||||
}
|
||||
if (!t || !t->mem) return;
|
||||
tex_t *copy = malloc(sizeof *copy);
|
||||
if (!copy) rt_fatal("vk: out of memory (replay queue)");
|
||||
*copy = *t;
|
||||
ReplayItem it = {.kind = RI_RELEASE, .tex = copy};
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
dr_calls[DR_TEX_RELEASE]++, dr_tcalls[DR_TEX_RELEASE]++;
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
replay_enqueue(&it);
|
||||
}
|
||||
static void vk_be_program_release(const prog_t *p) { replay_drain(DR_PROG_RELEASE); vk_program_release(p); }
|
||||
// Asynchronous: the handles (nvn_submit_prepare_async ran under the front end's submit lock) are copied and queued;
|
||||
// the value returned is the one the replay will signal.
|
||||
static uint64_t vk_be_submit(const uint64_t *handles, int n) {
|
||||
if (!gfx_replay_async()) return submit_sync(handles, n);
|
||||
vk_sync_start();
|
||||
ReplayItem it = {.kind = RI_SUBMIT, .n = n, .submit_no = nvn_submit_count()};
|
||||
if (n > 0) {
|
||||
it.h = malloc(sizeof *it.h * (size_t)n);
|
||||
if (!it.h) rt_fatal("vk: out of memory (replay queue)");
|
||||
memcpy(it.h, handles, sizeof *it.h * (size_t)n);
|
||||
}
|
||||
return replay_enqueue(&it);
|
||||
}
|
||||
static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); }
|
||||
static int replay_check(void) {
|
||||
static int on = -1;
|
||||
if (on < 0) {
|
||||
const char *e = cfg_get("TOTK_REPLAY_CHECK");
|
||||
on = e && strcmp(e, "0");
|
||||
if (on) rt_log("vk: TOTK_REPLAY_CHECK: every ReadTexels is read again after a full replay drain and compared");
|
||||
}
|
||||
return on;
|
||||
}
|
||||
|
||||
// ReadTexels with the replay queued: waits for the newest submission recorded to write the texture (replayed and
|
||||
// completed), not for the whole queue. The readback copy goes through the upload batch on the same queue, after every
|
||||
// submission the replay has submitted. Submissions still queued or being recorded run after the copy, as on hardware
|
||||
// when the GPU has not reached them yet: the game only sees their texels after waiting on a sync, and every sync
|
||||
// value covers replay and GPU, so a read the game synchronised sees them. TOTK_REPLAY_CHECK compares with a full drain.
|
||||
static void vk_be_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row,
|
||||
uint64_t img) {
|
||||
vk_sync_start();
|
||||
if (!gfx_replay_async()) {
|
||||
vk_read_texels(t, v, r, dst, row, img);
|
||||
return;
|
||||
}
|
||||
uint64_t f = vk_texture_last_write(t);
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
dr_calls[DR_READ_TEXELS]++, dr_tcalls[DR_READ_TEXELS]++;
|
||||
int idle = !rq_n && !r_busy;
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
if (f && vk_sync_fence_done() < f) {
|
||||
uint64_t t0 = kern_now_ns();
|
||||
vk_sync_fence_wait(f, UINT64_MAX); // counts as a fence wait in perf
|
||||
uint64_t d = kern_now_ns() - t0;
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
dr_waits[DR_READ_TEXELS]++, dr_twaits[DR_READ_TEXELS]++;
|
||||
dr_ns[DR_READ_TEXELS] += d, dr_tns[DR_READ_TEXELS] += d;
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
}
|
||||
vk_read_texels(t, v, r, dst, row, img);
|
||||
if (!replay_check()) return;
|
||||
size_t n = vk_texture_texel_span(t, v, r, row, img);
|
||||
uint8_t *again = n ? malloc(n) : NULL;
|
||||
if (!again) return;
|
||||
memcpy(again, dst, n); // bytes between the rows stay the caller's
|
||||
uint64_t f0 = vk_texture_last_write(t);
|
||||
replay_drain(DR_CHECK);
|
||||
uint64_t f1 = vk_texture_last_write(t);
|
||||
vk_read_texels(t, v, r, again, row, img);
|
||||
int differ = memcmp(again, dst, n) != 0;
|
||||
static volatile long n_logged;
|
||||
if (differ && InterlockedIncrement(&n_logged) <= 40) {
|
||||
size_t at = 0;
|
||||
while (at < n && again[at] == dst[at]) at++;
|
||||
rt_log("vk: replay check: ReadTexels of %p (format %#x, %dx%dx%d at %d,%d,%d) differs at byte %zu of %zu; writer "
|
||||
"fence %llu at the read, %llu after the drain, done %llu; queue %s at the read",
|
||||
(void *)t->mem, t->d.format, r->width, r->height, r->depth, r->xoffset, r->yoffset, r->zoffset, at, n,
|
||||
(unsigned long long)f0, (unsigned long long)f1, (unsigned long long)vk_sync_fence_done(), idle ? "idle" : "busy");
|
||||
}
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
ck_reads++;
|
||||
if (differ) {
|
||||
ck_diff++;
|
||||
if (f1 == f0) ck_diff_unexplained++;
|
||||
if (idle) ck_diff_idle++;
|
||||
}
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
free(again);
|
||||
}
|
||||
static void vk_be_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row,
|
||||
uint64_t img) {
|
||||
vk_sync_start();
|
||||
replay_drain(DR_WRITE_TEXELS);
|
||||
vk_present_drain();
|
||||
vk_write_texels(t, v, r, src, row, img);
|
||||
}
|
||||
static void vk_be_guest_written(uint64_t addr, uint64_t size) { vk_sync_start(); vk_guest_written(addr, size); }
|
||||
// A CPU read of GPU-written mirrored storage (memcpy, pool invalidate). With the replay queued it waits for the newest
|
||||
// submission recorded to write the textures there, not for the whole queue: the write-back copy then runs after their
|
||||
// renders, and clears gpu_dirty only for content that is on the queue. Submissions not replayed yet render after the
|
||||
// copy, as on hardware when the GPU has not reached them; they mark the texture dirty again for a later read.
|
||||
static void vk_be_guest_read(uint64_t addr, uint64_t size) {
|
||||
if (!vk_texture_cpu_stale(addr, size)) return;
|
||||
vk_sync_start();
|
||||
if (gfx_replay_async()) {
|
||||
uint64_t f = vk_texture_dirty_fence(addr, size);
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
dr_calls[DR_GUEST_READ]++, dr_tcalls[DR_GUEST_READ]++;
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
if (f && vk_sync_fence_done() < f) {
|
||||
uint64_t t0 = kern_now_ns();
|
||||
NVN_BLOCKING();
|
||||
vk_sync_fence_wait(f, UINT64_MAX);
|
||||
uint64_t d = kern_now_ns() - t0;
|
||||
AcquireSRWLockExclusive(&rl);
|
||||
dr_waits[DR_GUEST_READ]++, dr_twaits[DR_GUEST_READ]++;
|
||||
dr_ns[DR_GUEST_READ] += d, dr_tns[DR_GUEST_READ] += d;
|
||||
ReleaseSRWLockExclusive(&rl);
|
||||
}
|
||||
}
|
||||
vk_present_drain();
|
||||
vk_texture_write_back(addr, size);
|
||||
}
|
||||
// After a fence wait: the value is done, so its replay is too; a later submission the replay records meanwhile marks
|
||||
// its targets with a later fence, which write_back_small leaves alone. No drain.
|
||||
static void vk_be_cpu_synced(uint64_t fence) {
|
||||
if (!vk_texture_small_dirty()) return;
|
||||
vk_sync_start();
|
||||
@@ -189,6 +532,8 @@ const GfxBackend gfx_vulkan_backend = {
|
||||
};
|
||||
|
||||
void vk_backend_shutdown(void) {
|
||||
replay_drain(DR_FINISH);
|
||||
gfx_exit_log();
|
||||
vk_present_drain();
|
||||
vk_sync_shutdown();
|
||||
vk_mem_shutdown();
|
||||
|
||||
@@ -922,6 +922,7 @@ static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage,
|
||||
int hv;
|
||||
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
|
||||
if (img) view = get_view(ex, img, &v, hv, r->type, storage);
|
||||
if (img && storage) vk_texture_storage_bound_locked(img);
|
||||
if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT;
|
||||
if (!storage && !separate) samp = vk_sampler_by_id_locked((int)(h >> 20 & 0xfff));
|
||||
}
|
||||
|
||||
+15
-2
@@ -42,6 +42,7 @@ static SRWLOCK st_lock = SRWLOCK_INIT;
|
||||
static CONDITION_VARIABLE cv_work = CONDITION_VARIABLE_INIT; // a submission, a callback or stop
|
||||
static CONDITION_VARIABLE cv_done = CONDITION_VARIABLE_INIT; // done advanced
|
||||
static uint64_t issued, done; // newest submitted value; newest value whose side effects ran
|
||||
static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued)
|
||||
static int stop;
|
||||
static Pending *queue; // side effects in the order they were added
|
||||
static int qhead, qn, qcap;
|
||||
@@ -223,6 +224,7 @@ void vk_sync_start(void) {
|
||||
}
|
||||
AcquireSRWLockExclusive(&st_lock);
|
||||
issued = done = now;
|
||||
if (reserved < now) reserved = now;
|
||||
stop = 0;
|
||||
ReleaseSRWLockExclusive(&st_lock);
|
||||
vk_mem_note_submit(now);
|
||||
@@ -301,6 +303,17 @@ VkCommandBuffer vk_sync_split(void) {
|
||||
return s->cb;
|
||||
}
|
||||
|
||||
// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed
|
||||
// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value.
|
||||
uint64_t vk_sync_reserve(void) {
|
||||
vk_sync_start();
|
||||
AcquireSRWLockExclusive(&st_lock);
|
||||
if (reserved < issued) reserved = issued;
|
||||
uint64_t v = ++reserved;
|
||||
ReleaseSRWLockExclusive(&st_lock);
|
||||
return v;
|
||||
}
|
||||
|
||||
uint64_t vk_sync_submit(const uint64_t *handles, int n) {
|
||||
vk_sync_start();
|
||||
VkCtx *c = vk_ctx();
|
||||
@@ -378,8 +391,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
|
||||
vk_sync_start();
|
||||
AcquireSRWLockExclusive(&st_lock);
|
||||
int ok;
|
||||
if (value > issued) {
|
||||
ok = 0; // never submitted: never reached (as in the null back end)
|
||||
if (value > issued && value > reserved) {
|
||||
ok = 0; // never submitted nor queued: never reached (as in the null back end)
|
||||
} else {
|
||||
uint64_t t0 = kern_now_ns();
|
||||
while (done < value) {
|
||||
|
||||
@@ -52,6 +52,8 @@ typedef struct Tex {
|
||||
uint32_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded
|
||||
uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes)
|
||||
uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one)
|
||||
uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear,
|
||||
// storage binding); ReadTexels waits for it instead of draining the replay queue
|
||||
uint8_t listed; // in owned[]
|
||||
uint32_t wt_n; // WriteTexels into the storage since the last upload
|
||||
} Tex;
|
||||
@@ -819,6 +821,8 @@ void vk_texture_gpu_written(const tex_t *t) {
|
||||
if (e) {
|
||||
make_fresh(e); // a partial render keeps what the storage held before
|
||||
e->pub.gpu_owned = 1;
|
||||
uint64_t f = vk_exec_recording_fence();
|
||||
if (f > e->wr_fence) e->wr_fence = f;
|
||||
if (e->pub.mirrored) {
|
||||
e->gpu_dirty = 1;
|
||||
storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render
|
||||
@@ -1240,6 +1244,28 @@ static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) {
|
||||
return n;
|
||||
}
|
||||
|
||||
// A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c
|
||||
// tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels.
|
||||
void vk_texture_storage_bound_locked(const VkTexImage *img) {
|
||||
Tex *e = (Tex *)img; // pub is the first member
|
||||
uint64_t f = vk_exec_recording_fence();
|
||||
if (f > e->wr_fence) e->wr_fence = f;
|
||||
}
|
||||
|
||||
uint64_t vk_texture_last_write(const tex_t *t) {
|
||||
lock();
|
||||
Tex *e = entry_of(t);
|
||||
uint64_t f = e ? e->wr_fence : 0;
|
||||
unlock();
|
||||
return f;
|
||||
}
|
||||
|
||||
size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) {
|
||||
Region g;
|
||||
if (!region_of(t, v, r, row, img, &g)) return 0;
|
||||
return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb;
|
||||
}
|
||||
|
||||
int vk_texture_cpu_stale(uint64_t addr, uint64_t size) {
|
||||
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
|
||||
AcquireSRWLockShared(&owned_lock);
|
||||
@@ -1248,6 +1274,21 @@ int vk_texture_cpu_stale(uint64_t addr, uint64_t size) {
|
||||
return dirty;
|
||||
}
|
||||
|
||||
// The newest submission recorded to write a GPU-written mirrored texture overlapping [addr, addr+size); 0: none.
|
||||
uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size) {
|
||||
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
|
||||
lock();
|
||||
uint64_t f = 0;
|
||||
for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) {
|
||||
const Tex *e = owned[i];
|
||||
if (!e->gpu_dirty || e->pub.addr + e->t.L.size <= addr) continue;
|
||||
if (e->gpu_fence > f) f = e->gpu_fence;
|
||||
if (e->wr_fence > f) f = e->wr_fence;
|
||||
}
|
||||
unlock();
|
||||
return f;
|
||||
}
|
||||
|
||||
// The image is the newest content only while nothing wrote the storage since the GPU render: TotK finalizes grass maps,
|
||||
// reuses their storage for new data and memcpys that data on, and writing the old image back first handed the copy
|
||||
// a stale map or stale resource bytes (wrong textures in the distance). Every hit is checked before any is written
|
||||
|
||||
Reference in new issue
Block a user