diff --git a/runtime/cfg.c b/runtime/cfg.c index 3fed6617..9c8d30b2 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -34,7 +34,7 @@ static const char *const known[] = { "MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", - "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", + "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS", diff --git a/runtime/vk.h b/runtime/vk.h index f58c2880..453ec635 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -261,6 +261,9 @@ VkSampler vk_sampler_by_id_locked(int id); uint32_t vk_texture_slots_gen(void); uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage +void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission +uint64_t vk_texture_last_write(const tex_t *t); // timeline value of the newest submission recorded to write it; 0: none +size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img); // bytes ReadTexels spans; 0 for a bad region int vk_texture_small_dirty(void); // a small mirrored target holds GPU texels its storage lacks void vk_texture_write_back_small(uint64_t fence); // writes those rendered by work up to fence back to storage void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written); // vk_dump diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 5ae049a8..a2eafffc 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -150,9 +150,10 @@ typedef struct { } ReplayItem; #define RQ_CAP 64 enum { DR_POOL_FINI, DR_MAP_VIRTUAL, DR_TEX_RELEASE, DR_PROG_RELEASE, DR_FINISH, DR_READ_TEXELS, DR_WRITE_TEXELS, - DR_GUEST_READ, DR_FRAME, DR_FULL, DR_N }; + DR_GUEST_READ, DR_FRAME, DR_FULL, DR_CHECK, DR_N }; static const char *const dr_name[DR_N] = {"pool_fini", "map_virtual", "texture_release", "program_release", "finish", - "read_texels", "write_texels", "guest_read", "frame bound", "queue full"}; + "read_texels (writer fence)", "write_texels", "guest_read", "frame bound", "queue full", + "check drain"}; static SRWLOCK rl = SRWLOCK_INIT; static CONDITION_VARIABLE rcv = CONDITION_VARIABLE_INIT; // item queued, item taken, item done @@ -161,6 +162,9 @@ static int rq_head, rq_n, r_busy, r_started; static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames; // Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run. static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N]; +// TOTK_REPLAY_CHECK: ReadTexels compared with a read after a full drain (under rl): checked, differing, differing +// although no submission recorded a write to the texture in between, differing with the queue idle at the first read. +static uint64_t ck_reads, ck_diff, ck_diff_unexplained, ck_diff_idle; int vk_replay_async(void) { static int on = -1; @@ -276,6 +280,10 @@ static void replay_log(uint64_t frame, int total) { if (c[i] || w[i]) k += snprintf(buf + k, sizeof buf - (size_t)k, "%s %s %llu calls, %llu waited %.1f ms", k ? ";" : "", dr_name[i], (unsigned long long)c[i], (unsigned long long)w[i], t[i] / 1e6); + if (ck_reads && k < (int)sizeof buf - 128) + k += snprintf(buf + k, sizeof buf - (size_t)k, "; check (run): %llu reads, %llu differ, %llu without a recorded write, %llu with the queue idle", + (unsigned long long)ck_reads, (unsigned long long)ck_diff, (unsigned long long)ck_diff_unexplained, + (unsigned long long)ck_diff_idle); memset(dr_calls, 0, sizeof dr_calls); memset(dr_waits, 0, sizeof dr_waits); memset(dr_ns, 0, sizeof dr_ns); @@ -354,11 +362,71 @@ static uint64_t vk_be_submit(const uint64_t *handles, int n) { return replay_enqueue(&it); } static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); } +static int replay_check(void) { + static int on = -1; + if (on < 0) { + const char *e = cfg_get("TOTK_REPLAY_CHECK"); + on = e && strcmp(e, "0"); + if (on) rt_log("vk: TOTK_REPLAY_CHECK: every ReadTexels is read again after a full replay drain and compared"); + } + return on; +} + +// ReadTexels with the replay queued: waits for the newest submission recorded to write the texture (replayed and +// completed), not for the whole queue. The readback copy goes through the upload batch on the same queue, after every +// submission the replay has submitted. Submissions still queued or being recorded run after the copy, as on hardware +// when the GPU has not reached them yet: the game only sees their texels after waiting on a sync, and every sync +// value covers replay and GPU, so a read the game synchronised sees them. TOTK_REPLAY_CHECK compares with a full drain. static void vk_be_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) { vk_sync_start(); - replay_drain(DR_READ_TEXELS); + if (!gfx_replay_async()) { + vk_read_texels(t, v, r, dst, row, img); + return; + } + uint64_t f = vk_texture_last_write(t); + AcquireSRWLockExclusive(&rl); + dr_calls[DR_READ_TEXELS]++, dr_tcalls[DR_READ_TEXELS]++; + int idle = !rq_n && !r_busy; + ReleaseSRWLockExclusive(&rl); + if (f && vk_sync_fence_done() < f) { + uint64_t t0 = kern_now_ns(); + vk_sync_fence_wait(f, UINT64_MAX); // counts as a fence wait in perf + uint64_t d = kern_now_ns() - t0; + AcquireSRWLockExclusive(&rl); + dr_waits[DR_READ_TEXELS]++, dr_twaits[DR_READ_TEXELS]++; + dr_ns[DR_READ_TEXELS] += d, dr_tns[DR_READ_TEXELS] += d; + ReleaseSRWLockExclusive(&rl); + } vk_read_texels(t, v, r, dst, row, img); + if (!replay_check()) return; + size_t n = vk_texture_texel_span(t, v, r, row, img); + uint8_t *again = n ? malloc(n) : NULL; + if (!again) return; + memcpy(again, dst, n); // bytes between the rows stay the caller's + uint64_t f0 = vk_texture_last_write(t); + replay_drain(DR_CHECK); + uint64_t f1 = vk_texture_last_write(t); + vk_read_texels(t, v, r, again, row, img); + int differ = memcmp(again, dst, n) != 0; + static volatile long n_logged; + if (differ && InterlockedIncrement(&n_logged) <= 40) { + size_t at = 0; + while (at < n && again[at] == dst[at]) at++; + rt_log("vk: replay check: ReadTexels of %p (format %#x, %dx%dx%d at %d,%d,%d) differs at byte %zu of %zu; writer " + "fence %llu at the read, %llu after the drain, done %llu; queue %s at the read", + (void *)t->mem, t->d.format, r->width, r->height, r->depth, r->xoffset, r->yoffset, r->zoffset, at, n, + (unsigned long long)f0, (unsigned long long)f1, (unsigned long long)vk_sync_fence_done(), idle ? "idle" : "busy"); + } + AcquireSRWLockExclusive(&rl); + ck_reads++; + if (differ) { + ck_diff++; + if (f1 == f0) ck_diff_unexplained++; + if (idle) ck_diff_idle++; + } + ReleaseSRWLockExclusive(&rl); + free(again); } static void vk_be_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row, uint64_t img) { diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index c2b19645..6f00f7c1 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -922,6 +922,7 @@ static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int hv; const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv); if (img) view = get_view(ex, img, &v, hv, r->type, storage); + if (img && storage) vk_texture_storage_bound_locked(img); if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT; if (!storage && !separate) samp = vk_sampler_by_id_locked((int)(h >> 20 & 0xfff)); } diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 752e75ef..317adbc2 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -52,6 +52,8 @@ typedef struct Tex { uint32_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes) uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one) + uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear, + // storage binding); ReadTexels waits for it instead of draining the replay queue uint8_t listed; // in owned[] } Tex; @@ -803,6 +805,8 @@ void vk_texture_gpu_written(const tex_t *t) { if (e) { make_fresh(e); // a partial render keeps what the storage held before e->pub.gpu_owned = 1; + uint64_t f = vk_exec_recording_fence(); + if (f > e->wr_fence) e->wr_fence = f; if (e->pub.mirrored) { e->gpu_dirty = 1; storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render @@ -1219,6 +1223,28 @@ static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) { return n; } +// A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c +// tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels. +void vk_texture_storage_bound_locked(const VkTexImage *img) { + Tex *e = (Tex *)img; // pub is the first member + uint64_t f = vk_exec_recording_fence(); + if (f > e->wr_fence) e->wr_fence = f; +} + +uint64_t vk_texture_last_write(const tex_t *t) { + lock(); + Tex *e = entry_of(t); + uint64_t f = e ? e->wr_fence : 0; + unlock(); + return f; +} + +size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) { + Region g; + if (!region_of(t, v, r, row, img, &g)) return 0; + return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb; +} + int vk_texture_cpu_stale(uint64_t addr, uint64_t size) { if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0; AcquireSRWLockShared(&owned_lock);