Wait for the texture's newest writer instead of draining the replay queue in ReadTexels, with TOTK_REPLAY_CHECK comparing against a full drain

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-25 23:27:49 +03:00
1 parent 1c1b3cf073
commit 469c2be1e7
5 files changed
+102 -4

No files matched your search

+1 -1
View File
@@ -34,7 +34,7 @@ static const char *const known[] = {
"MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
"PREWARM_XLATE",
"PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC",
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS",
"SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL",
"SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS",
+3
View File
@@ -261,6 +261,9 @@ VkSampler vk_sampler_by_id_locked(int id);
uint32_t vk_texture_slots_gen(void);
uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered
void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage
void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission
uint64_t vk_texture_last_write(const tex_t *t); // timeline value of the newest submission recorded to write it; 0: none
size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img); // bytes ReadTexels spans; 0 for a bad region
int vk_texture_small_dirty(void); // a small mirrored target holds GPU texels its storage lacks
void vk_texture_write_back_small(uint64_t fence); // writes those rendered by work up to fence back to storage
void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written); // vk_dump
+71 -3
View File
@@ -150,9 +150,10 @@ typedef struct {
} ReplayItem;
#define RQ_CAP 64
enum { DR_POOL_FINI, DR_MAP_VIRTUAL, DR_TEX_RELEASE, DR_PROG_RELEASE, DR_FINISH, DR_READ_TEXELS, DR_WRITE_TEXELS,
DR_GUEST_READ, DR_FRAME, DR_FULL, DR_N };
DR_GUEST_READ, DR_FRAME, DR_FULL, DR_CHECK, DR_N };
static const char *const dr_name[DR_N] = {"pool_fini", "map_virtual", "texture_release", "program_release", "finish",
"read_texels", "write_texels", "guest_read", "frame bound", "queue full"};
"read_texels (writer fence)", "write_texels", "guest_read", "frame bound", "queue full",
"check drain"};
static SRWLOCK rl = SRWLOCK_INIT;
static CONDITION_VARIABLE rcv = CONDITION_VARIABLE_INIT; // item queued, item taken, item done
@@ -161,6 +162,9 @@ static int rq_head, rq_n, r_busy, r_started;
static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames;
// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run.
static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N];
// TOTK_REPLAY_CHECK: ReadTexels compared with a read after a full drain (under rl): checked, differing, differing
// although no submission recorded a write to the texture in between, differing with the queue idle at the first read.
static uint64_t ck_reads, ck_diff, ck_diff_unexplained, ck_diff_idle;
int vk_replay_async(void) {
static int on = -1;
@@ -276,6 +280,10 @@ static void replay_log(uint64_t frame, int total) {
if (c[i] || w[i])
k += snprintf(buf + k, sizeof buf - (size_t)k, "%s %s %llu calls, %llu waited %.1f ms", k ? ";" : "", dr_name[i],
(unsigned long long)c[i], (unsigned long long)w[i], t[i] / 1e6);
if (ck_reads && k < (int)sizeof buf - 128)
k += snprintf(buf + k, sizeof buf - (size_t)k, "; check (run): %llu reads, %llu differ, %llu without a recorded write, %llu with the queue idle",
(unsigned long long)ck_reads, (unsigned long long)ck_diff, (unsigned long long)ck_diff_unexplained,
(unsigned long long)ck_diff_idle);
memset(dr_calls, 0, sizeof dr_calls);
memset(dr_waits, 0, sizeof dr_waits);
memset(dr_ns, 0, sizeof dr_ns);
@@ -354,11 +362,71 @@ static uint64_t vk_be_submit(const uint64_t *handles, int n) {
return replay_enqueue(&it);
}
static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); }
static int replay_check(void) {
static int on = -1;
if (on < 0) {
const char *e = cfg_get("TOTK_REPLAY_CHECK");
on = e && strcmp(e, "0");
if (on) rt_log("vk: TOTK_REPLAY_CHECK: every ReadTexels is read again after a full replay drain and compared");
}
return on;
}
// ReadTexels with the replay queued: waits for the newest submission recorded to write the texture (replayed and
// completed), not for the whole queue. The readback copy goes through the upload batch on the same queue, after every
// submission the replay has submitted. Submissions still queued or being recorded run after the copy, as on hardware
// when the GPU has not reached them yet: the game only sees their texels after waiting on a sync, and every sync
// value covers replay and GPU, so a read the game synchronised sees them. TOTK_REPLAY_CHECK compares with a full drain.
static void vk_be_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row,
uint64_t img) {
vk_sync_start();
replay_drain(DR_READ_TEXELS);
if (!gfx_replay_async()) {
vk_read_texels(t, v, r, dst, row, img);
return;
}
uint64_t f = vk_texture_last_write(t);
AcquireSRWLockExclusive(&rl);
dr_calls[DR_READ_TEXELS]++, dr_tcalls[DR_READ_TEXELS]++;
int idle = !rq_n && !r_busy;
ReleaseSRWLockExclusive(&rl);
if (f && vk_sync_fence_done() < f) {
uint64_t t0 = kern_now_ns();
vk_sync_fence_wait(f, UINT64_MAX); // counts as a fence wait in perf
uint64_t d = kern_now_ns() - t0;
AcquireSRWLockExclusive(&rl);
dr_waits[DR_READ_TEXELS]++, dr_twaits[DR_READ_TEXELS]++;
dr_ns[DR_READ_TEXELS] += d, dr_tns[DR_READ_TEXELS] += d;
ReleaseSRWLockExclusive(&rl);
}
vk_read_texels(t, v, r, dst, row, img);
if (!replay_check()) return;
size_t n = vk_texture_texel_span(t, v, r, row, img);
uint8_t *again = n ? malloc(n) : NULL;
if (!again) return;
memcpy(again, dst, n); // bytes between the rows stay the caller's
uint64_t f0 = vk_texture_last_write(t);
replay_drain(DR_CHECK);
uint64_t f1 = vk_texture_last_write(t);
vk_read_texels(t, v, r, again, row, img);
int differ = memcmp(again, dst, n) != 0;
static volatile long n_logged;
if (differ && InterlockedIncrement(&n_logged) <= 40) {
size_t at = 0;
while (at < n && again[at] == dst[at]) at++;
rt_log("vk: replay check: ReadTexels of %p (format %#x, %dx%dx%d at %d,%d,%d) differs at byte %zu of %zu; writer "
"fence %llu at the read, %llu after the drain, done %llu; queue %s at the read",
(void *)t->mem, t->d.format, r->width, r->height, r->depth, r->xoffset, r->yoffset, r->zoffset, at, n,
(unsigned long long)f0, (unsigned long long)f1, (unsigned long long)vk_sync_fence_done(), idle ? "idle" : "busy");
}
AcquireSRWLockExclusive(&rl);
ck_reads++;
if (differ) {
ck_diff++;
if (f1 == f0) ck_diff_unexplained++;
if (idle) ck_diff_idle++;
}
ReleaseSRWLockExclusive(&rl);
free(again);
}
static void vk_be_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row,
uint64_t img) {
+1
View File
@@ -922,6 +922,7 @@ static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage,
int hv;
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
if (img) view = get_view(ex, img, &v, hv, r->type, storage);
if (img && storage) vk_texture_storage_bound_locked(img);
if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT;
if (!storage && !separate) samp = vk_sampler_by_id_locked((int)(h >> 20 & 0xfff));
}
+26
View File
@@ -52,6 +52,8 @@ typedef struct Tex {
uint32_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded
uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes)
uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one)
uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear,
// storage binding); ReadTexels waits for it instead of draining the replay queue
uint8_t listed; // in owned[]
} Tex;
@@ -803,6 +805,8 @@ void vk_texture_gpu_written(const tex_t *t) {
if (e) {
make_fresh(e); // a partial render keeps what the storage held before
e->pub.gpu_owned = 1;
uint64_t f = vk_exec_recording_fence();
if (f > e->wr_fence) e->wr_fence = f;
if (e->pub.mirrored) {
e->gpu_dirty = 1;
storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render
@@ -1219,6 +1223,28 @@ static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) {
return n;
}
// A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c
// tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels.
void vk_texture_storage_bound_locked(const VkTexImage *img) {
Tex *e = (Tex *)img; // pub is the first member
uint64_t f = vk_exec_recording_fence();
if (f > e->wr_fence) e->wr_fence = f;
}
uint64_t vk_texture_last_write(const tex_t *t) {
lock();
Tex *e = entry_of(t);
uint64_t f = e ? e->wr_fence : 0;
unlock();
return f;
}
size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) {
Region g;
if (!region_of(t, v, r, row, img, &g)) return 0;
return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb;
}
int vk_texture_cpu_stale(uint64_t addr, uint64_t size) {
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
AcquireSRWLockShared(&owned_lock);