From 35b3d37ea3970be5a06bf33dbaa10cf25a44df5b Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 8 Nov 2026 19:53:06 +0300 Subject: [PATCH] Make a chunk's new texture entries follow replay order, and add a knob that records chunks out of order entry_of decides at record time whether a new texture takes the content of a live GPU-written texture at its storage (alias_copy) or the storage bytes. In a chunked replay the source's GPU ownership is set when a chunk records its render, so the choice followed the order the threads got to the chunks: a later chunk could take the old bytes before an earlier chunk recorded the render, and a mirrored texture keeps them. Now a chunk that makes an entry while a live colour texture of its size sits at the storage waits until the chunks before it are recorded (vk_exec_chunk_settle, the texture lock released for the wait), skips sources the scan revives at a later op, and ignores GPU ownership a later chunk set in this submission. slot_resolve and vk_texture_by_id_locked no longer read tslots through a pointer taken before the wait. TOTK_REPLAY_SKEW_US (debug) makes the even chunks wait before they record, so the chunk after each one records first in every submission. With it at 3000 us and 4 threads the field save runs clean (grass drawn, 0 validation errors); the alias path did not fire there (0 copied, 0 filled from storage), so this does not reproduce the missing grass reported on RADV, and the fix there stays unconfirmed. Co-Authored-By: Claude Opus 5.5 --- docs/settings.md | 1 + runtime/cfg.c | 3 +- runtime/vk_exec.c | 31 ++++++++++++++++ runtime/vk_exec.h | 5 +++ runtime/vk_texture.c | 85 +++++++++++++++++++++++++++++++++++--------- 5 files changed, 108 insertions(+), 17 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index eebf62e0..7a24e5f9 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -185,6 +185,7 @@ Many rows below are developer tools that only act together with another setting | `replay_chunks` (default `0`) | Most chunks a submission is cut into when `replay_workers` is set; `0` uses twice the number of recording threads. | Fewer chunks leave threads idle at the end of a submission, more add scan and submit work per chunk; no effect while `replay_workers` is `0`. | Caution | | `replay_chunk_draws` (default `128`) | Least draws a chunk of the chunked replay holds before it may end at a pass. | Small values make many short chunks, large ones leave big submissions on few threads; no effect while `replay_workers` is `0`. | Caution | | `replay_state_check` (default `0`) | `1` compares the replay state after the last chunk of a chunked submission with the state of the scan and logs differences. | Adds a comparison per submission and log lines on a mismatch; no effect while `replay_workers` is `0`. | Caution | +| `replay_skew_us` (default `0`) | Debug: the even chunks of a chunked submission wait this many microseconds before they record, so the chunk after each one records first. | Shows a recording that depends on which thread gets to a chunk first in every submission; slows the replay; no effect with fewer than 2 `replay_workers`. | Caution | | `restart_wait` | Set by the runtime when it restarts itself; names the old process by id and creation time so the new process waits for it to exit. | Meant for the runtime's own use; the code does not describe a manual value. | Unknown | | `spin` (default `1`) | `0` makes a guest thread that polls memory keep its virtual core. | Polling threads can starve the thread whose item they wait for, so frames can stall. | Caution | | `spin_handoff_us` (default `50`) | Microseconds a polling wait lasts before it gives its virtual core to a waiting thread. | A shorter value hands cores over sooner; a longer one keeps them longer. | Caution | diff --git a/runtime/cfg.c b/runtime/cfg.c index 852a9eb2..b4f084cb 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -53,7 +53,7 @@ static const char *const known[] = { "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCORE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_BENCH", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_ASYNC", "REC_END", "REC_SPIN_US", "REEXEC", - "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHUNKS", "REPLAY_CHUNK_DRAWS", "REPLAY_STATE_CHECK", "REPLAY_WORKERS", + "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHUNKS", "REPLAY_CHUNK_DRAWS", "REPLAY_SKEW_US", "REPLAY_STATE_CHECK", "REPLAY_WORKERS", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", @@ -165,6 +165,7 @@ static const CfgMeta metas[] = { {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, {"REPLAY_WORKERS", "0", "0..16", RESTART}, {"REPLAY_CHUNKS", "0", "0..64", RESTART}, {"REPLAY_CHUNK_DRAWS", "128", "1..100000", RESTART}, {"REPLAY_STATE_CHECK", "0", "0|1", RESTART}, + {"REPLAY_SKEW_US", "0", "0..100000", RESTART}, {"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART}, // shaders and pipelines {"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "fullscreen", "strict|any|fullscreen", RESTART}, diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 5c6c88a8..a28b0a1b 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -54,6 +54,7 @@ static PlatMutex fw_lock = PLAT_MUTEX_INIT; // the recording threads of a chun uint64_t vk_exec_recording_fence(void) { return x_cur ? x_cur->fence : 0; } uint32_t vk_exec_replay_pos(void) { return x_cur && x_cur->chunked ? x_cur->pos : UINT32_MAX; } +static _Thread_local int chunk_k = -1; // the chunk this thread records (chunk_record), -1 none // A context's counters go into vk_exec_frame (the submitting thread, after the recording). @@ -1072,6 +1073,7 @@ typedef struct { uint64_t addr, size; uint32_t at; } ScanDrop; static struct { int workers, chunks, min_draws, check; + int skew_us; // TOTK_REPLAY_SKEW_US (chunk_record) VkExec s; // the scan's state Chunk *ch; // chunks + 1 entries, never moved while threads record int nch; @@ -1097,6 +1099,7 @@ static struct { uint64_t subs, chunked, nchunks, scan_ns, first_ns, last_ns, rec_ns, rec_max_ns, bad, prep_ns, start0_ns; uint64_t rec_t0; // vk_exec_record's start uint64_t held[HOLD_N]; // TOTK_TRACE=perf: chunks held, by scan_hold's reason + uint64_t settles, settle_ns; // TOTK_TRACE=perf: vk_exec_chunk_settle waits (atomic: recording threads) uint64_t mirror_ns, usnap_ns, ready_ns; // vk_mem_mirror_begin, vk_usnap_begin, both done after t0 } SC = {.workers = -1}; @@ -1278,6 +1281,10 @@ static VkCommandBuffer rec_cb(Rec *r, uint64_t fence) { // Chunk k on the calling thread with context r: from its state copy up to the next chunk's first op, into a command // buffer of r's pool, ending with every image in GENERAL. static void chunk_record(Rec *r, int k) { + // TOTK_REPLAY_SKEW_US (debug): the even chunks wait this long before they record, so with several threads the + // chunk after each one records first. A recording that depends on the order the threads get to the chunks, rather + // than on replay order, then shows in every submission instead of a few launches in thirty. + if (SC.skew_us > 0 && nthreads && !(k & 1)) plat_sleep_ns((uint64_t)SC.skew_us * 1000); uint64_t t0 = kern_now_ns(); VkExec *ex = &r->ex; Chunk *c = &SC.ch[k]; @@ -1287,6 +1294,7 @@ static void chunk_record(Rec *r, int k) { ex->fence = RP.fence, ex->recorded = 0, ex->in_pass = 0, ex->nviews = 0, ex->barrier_at = -1, ex->nfbl = 0; ex->part_start = 0; x_cur = ex; + chunk_k = k; vk_completion_defer(&c->defer); vk_mem_count_local(r->traffic); int ts = vk_sync_ts_open(ex->cb); @@ -1307,6 +1315,7 @@ static void chunk_record(Rec *r, int k) { vk_mem_count_local(NULL); vk_mem_count_flush(r->traffic); x_cur = NULL; + chunk_k = -1; vkEndCommandBuffer(ex->cb); c->cb = ex->cb, c->by = ex, c->ns = kern_now_ns() - t0; plat_mutex_lock(&RP.m); @@ -1413,6 +1422,22 @@ static void chunks_wait(int k) { plat_mutex_unlock(&RP.m); } +int vk_exec_chunk_behind(void) { + for (int i = chunk_k - 1; i >= 0; i--) + if (!__atomic_load_n(&SC.ch[i].done, __ATOMIC_ACQUIRE)) return 1; + return 0; +} + +// The chunks before take no lock this one holds and wait only for chunks before them, so they all get done: a chunk +// is taken only after every chunk before it was (chunk_take), and the DLSS records wait the same way (chunks_wait). +void vk_exec_chunk_settle(void) { + if (chunk_k <= 0) return; + uint64_t t0 = kern_now_ns(); + chunks_wait(chunk_k); + __atomic_add_fetch(&SC.settles, 1, __ATOMIC_RELAXED); + __atomic_add_fetch(&SC.settle_ns, kern_now_ns() - t0, __ATOMIC_RELAXED); +} + static void scan_job(VkExecJob *j, const op_hdr *h) { static int full; j->chunk = SC.nch - 1; @@ -1603,6 +1628,9 @@ static int chunked_on(void) { if (SC.chunks <= 0) SC.chunks = 2 * SC.workers; SC.min_draws = cfg_int("TOTK_REPLAY_CHUNK_DRAWS"); SC.check = cfg_bool("TOTK_REPLAY_STATE_CHECK"); + SC.skew_us = cfg_int("TOTK_REPLAY_SKEW_US"); + if (SC.workers && SC.skew_us > 0) + rt_log("vk: TOTK_REPLAY_SKEW_US: the even chunks wait %d us before they record", SC.skew_us); if (SC.workers) rt_log("vk: replay in up to %d chunks per submission of at least %d draws each, %d recording thread%s%s", SC.chunks, SC.min_draws, SC.workers, SC.workers > 1 ? "s" : "", SC.check ? ", state checked" : ""); @@ -1669,6 +1697,9 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t rt_log("perf: chunked replay: chunks held per submission: %.2f for a new texture, %.2f for DLSS, %.2f for " "unknown storage writes", SC.held[HOLD_TEX] / (double)SC.chunked, SC.held[HOLD_DLSS] / (double)SC.chunked, SC.held[HOLD_WRITES] / (double)SC.chunked); + rt_log("perf: chunked replay: %llu chunks waited for the chunks before them to make a texture (%.3f ms in all)", + (unsigned long long)__atomic_exchange_n(&SC.settles, 0, __ATOMIC_RELAXED), + __atomic_exchange_n(&SC.settle_ns, 0, __ATOMIC_RELAXED) / 1e6); memset(SC.held, 0, sizeof SC.held); char b[768]; int o = 0; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index fe799392..37579804 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -224,6 +224,11 @@ extern VkExecFrame vk_exec_frame; void vk_exec_frame_written(const tex_t *t); // vk_texture_gpu_written: t rendered or copied into uint64_t vk_exec_recording_fence(void); // fence of the submission this thread records, else 0 uint32_t vk_exec_replay_pos(void); // the op this thread records in a chunk, else UINT32_MAX +// A chunk that is about to make a texture entry whose content depends on the textures at its storage (vk_texture.c +// entry_of): whether a chunk before it is still being recorded, and the wait until none is. The caller holds no lock +// but the texture lock, which it releases for the wait. +int vk_exec_chunk_behind(void); +void vk_exec_chunk_settle(void); void vk_exec_frame_log(const tex_t *presented, uint64_t frame); // TOTK_DUMP= (vk_dump.c): between presents frame-1 and frame, every pass's targets and every dispatch's storage diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 99ecb49b..a2a9ea41 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -47,6 +47,7 @@ typedef struct Tex { uint32_t checked; // submit_epoch of the last check from a draw binding int uploaded, released; uint32_t rev_epoch, rev_pos; // the state scan made it live in that submission at that op (vk_texture_revive) + uint32_t own_epoch, own_pos; // a chunk made it GPU-owned in that submission at that op (vk_texture_gpu_written) uint32_t n_up; // uploads of this entry (vk_texture_debug) int last_ch; // last vk_mem_changed result of make_fresh: 1 written, 0 clean, -1 not tracked struct Tex *next_at; // other entries at the same storage address @@ -146,11 +147,17 @@ static void tex_init(void *p) { __atomic_store_n(&tex_ready, 1, __ATOMIC_RELEASE); return; } +static _Thread_local int tex_depth; // how often this thread holds tex_lock (entry_of releases it for a wait) +static _Thread_local uint32_t tex_settles; // entry_of waits of this thread (vk_texture_by_id_locked) static void lock(void) { if (!__atomic_load_n(&tex_ready, __ATOMIC_ACQUIRE)) plat_once(&tex_once, tex_init, NULL); plat_rmutex_lock(&tex_lock); + tex_depth++; +} +static void unlock(void) { + tex_depth--; + plat_rmutex_unlock(&tex_lock); } -static void unlock(void) { plat_rmutex_unlock(&tex_lock); } static uint32_t umax(uint32_t a, uint32_t b) { return a > b ? a : b; } static uint32_t mip(uint32_t v, unsigned lv) { return umax(v >> lv, 1); } @@ -1234,10 +1241,20 @@ static int alias_compatible(const Tex *e, const Tex *src) { return eb && eb == sb && ew == sw && eh == sh; } -// src holds content the storage bytes do not: the GPU wrote it and, for a mirrored texture, it was not written back -// and the CPU has not rewritten the storage since (the new entry's upload then reads the newer bytes). -static int alias_has_gpu_content(const Tex *src) { - if (!src->pub.gpu_owned) return 0; +// Whether e is live at op pos of a chunked replay: the state scan is ahead of the chunks, and an entry it revived at a +// later op is still released for this one. Two 4K targets at one address, RGBA8 and R8 with memory of their own, were +// swapped whenever the scan got past the second's SET_RT before a chunk sampling the storage got its turn, and large +// parts of the scene flickered. UINT32_MAX: the state now. +static int live_at(const Tex *e, uint32_t pos) { + return !e->released && !(e->rev_epoch == submit_epoch && e->rev_pos > pos); +} + +// src holds content the storage bytes do not, at op pos of a chunked replay (UINT32_MAX: the state now): the GPU wrote +// it and, for a mirrored texture, it was not written back and the CPU has not rewritten the storage since (the new +// entry's upload then reads the newer bytes). A chunk at a later op that rendered src first does not count: in replay +// order that render comes after the new entry's first use. +static int alias_has_gpu_content(const Tex *src, uint32_t pos) { + if (!src->pub.gpu_owned || (src->own_epoch == submit_epoch && src->own_pos > pos)) return 0; if (!src->pub.mirrored) return 1; uint32_t now; return src->gpu_dirty && !storage_rewritten(src, src->gpu_gen, &now); @@ -1298,10 +1315,38 @@ static void entry_live(Tex *e) { e->released = 0, nreleased--, rel_unlink(e), tex_gen_bump(e->key.addr); } +// A live colour entry at e's storage of e's size, as of op pos: the new entry may take its content (alias_copy). +static int alias_candidate(const TexKey *k, const tex_t *t, uint32_t pos) { + for (const Tex *p = nvn_hm_get(&by_addr, &k->addr); p; p = p->next_at) + if (live_at(p, pos) && !p->f.depth && !p->f.stencil && p->t.L.size == t->L.size) return 1; + return 0; +} + static Tex *entry_of(const tex_t *t) { TexKey k; make_key(t, &k); Tex *e = nvn_hm_get(&by_key, &k); + uint32_t pos = vk_exec_replay_pos(); + // A chunk of a chunked replay makes the entry as of its own op: whether it takes the content of a texture at its + // storage (alias_copy) depends on what the chunks before it render there, and those may still be recording on + // other threads. Taken from a render they had not recorded yet, the new image got the storage's old bytes instead: + // on an RX 7800 XT the grass near Link was missing in 3 of 29 launches with 4 threads (a map texture drawn once + // per tile, kept from then on). With a live colour texture of its size at the storage, the chunk waits until the + // chunks before it are recorded, the texture lock released for the wait (as often as this thread holds it). + if (!e && pos != UINT32_MAX && tex_alias != 0 && alias_candidate(&k, t, pos) && vk_exec_chunk_behind()) { + tex_t own = *t; // t may point into tslots, which a registration moves while the lock is released + t = &own; + int d = tex_depth; + for (int i = 0; i < d; i++) unlock(); + vk_exec_chunk_settle(); + for (int i = 0; i < d; i++) lock(); + tex_settles++; + if ((e = nvn_hm_get(&by_key, &k))) { // another chunk made it meanwhile + entry_live(e); + e->t = *t; + return e; + } + } if (!e) { if (tex_alias < 0) { tex_alias = cfg_bool("TOTK_TEX_ALIAS"); @@ -1319,8 +1364,9 @@ static Tex *entry_of(const tex_t *t) { nx = p->next_at; if (p->released) { destroy_tex(p); continue; } alias = 1; + if (!live_at(p, pos)) continue; // the scan revives it at a later op: released at this chunk's if (tex_alias && (!src || (same_but_swizzle(&p->key, &k) && !same_but_swizzle(&src->key, &k))) && - alias_compatible(e, p) && alias_has_gpu_content(p)) + alias_compatible(e, p) && alias_has_gpu_content(p, pos)) src = p; } if (alias) perf_count(PERF_CNT_TEX_ALIAS); @@ -1423,6 +1469,10 @@ void vk_texture_gpu_written(const tex_t *t) { Tex *e = entry_of(t); if (e) { make_fresh(e); // a partial render keeps what the storage held before + if (!e->pub.gpu_owned) { // entry_of's alias choice of a chunk at an earlier op leaves this render out + uint32_t pos = vk_exec_replay_pos(); + e->own_epoch = pos == UINT32_MAX ? 0 : submit_epoch, e->own_pos = pos; + } e->pub.gpu_owned = 1; uint64_t f = vk_exec_recording_fence(); if (f > e->wr_fence) e->wr_fence = f; @@ -1511,14 +1561,6 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { unlock(); } -// Whether e is live at op pos of a chunked replay: the state scan is ahead of the chunks, and an entry it revived at a -// later op is still released for this one. Two 4K targets at one address, RGBA8 and R8 with memory of their own, were -// swapped whenever the scan got past the second's SET_RT before a chunk sampling the storage got its turn, and large -// parts of the scene flickered. UINT32_MAX: the state now. -static int live_at(const Tex *e, uint32_t pos) { - return !e->released && !(e->rev_epoch == submit_epoch && e->rev_pos > pos); -} - // The entry slot s resolves to at op pos, and the view and has_view a caller gets with it (the slot's, with the // descriptor's swizzle when the entry is another descriptor at the same storage). NULL when the slot names no texture. static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view, uint32_t pos) { @@ -1546,7 +1588,8 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view, uint32_t pos) } if (!e) e = any ? any : own; } - if (!e) e = s->t.mem ? entry_of(&s->t) : NULL; + TexSlot copy; // entry_of may release the lock (a chunk's wait), and a registration then moves tslots + if (!e && s->t.mem) copy = *s, s = ©, e = entry_of(&s->t); // A followed texture is another descriptor at the same storage: its image holds the content, but sampling takes the // slot descriptor's swizzle. The UI renders text into RGBA8 and samples it through a (A,A,A,A) descriptor; with the // render target's RGBA swizzle the text blur saw R = 1 everywhere and drew solid boxes behind tips and labels. @@ -1594,6 +1637,7 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { view_t v; int hv; Tex *f = slot_resolve(s, &v, &hv, pos); + s = &tslots[id]; // slot_resolve may have released the lock if (f && (!f->uploaded || f->checked != submit_epoch)) { make_fresh_snap(f); f->checked = submit_epoch; @@ -1605,7 +1649,16 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may int hv; // succeed later - s->ce = slot_resolve(s, &v, &hv, UINT32_MAX); // entry_of may create an entry and bump g: g read before it + uint32_t settles = tex_settles; + Tex *r = slot_resolve(s, &v, &hv, UINT32_MAX); // entry_of may create an entry and bump g: g read before it + s = &tslots[id]; + if (settles != tex_settles) { // the lock was released for a wait: the slot may be another texture by now + if (r && (!r->uploaded || r->checked != submit_epoch)) make_fresh_snap(r), r->checked = submit_epoch; + if (view) *view = has_view ? v : s->v; + if (has_view) *has_view = view ? hv : s->has_view; + return r ? &r->pub : NULL; + } + s->ce = r; s->cview = v, s->chas_view = (uint8_t)hv; s->cgen = g; s->cepoch = 0;