mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 12:45:57 +00:00
Make a chunk's new texture entries follow replay order, and add a knob that records chunks out of order
entry_of decides at record time whether a new texture takes the content of a live GPU-written texture at its storage (alias_copy) or the storage bytes. In a chunked replay the source's GPU ownership is set when a chunk records its render, so the choice followed the order the threads got to the chunks: a later chunk could take the old bytes before an earlier chunk recorded the render, and a mirrored texture keeps them. Now a chunk that makes an entry while a live colour texture of its size sits at the storage waits until the chunks before it are recorded (vk_exec_chunk_settle, the texture lock released for the wait), skips sources the scan revives at a later op, and ignores GPU ownership a later chunk set in this submission. slot_resolve and vk_texture_by_id_locked no longer read tslots through a pointer taken before the wait. TOTK_REPLAY_SKEW_US (debug) makes the even chunks wait before they record, so the chunk after each one records first in every submission. With it at 3000 us and 4 threads the field save runs clean (grass drawn, 0 validation errors); the alias path did not fire there (0 copied, 0 filled from storage), so this does not reproduce the missing grass reported on RADV, and the fix there stays unconfirmed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fd4427cb17
commit
35b3d37ea3
5 files changed
+108
-17
No files matched your search
+2
-1
@@ -53,7 +53,7 @@ static const char *const known[] = {
|
||||
"ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCORE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
|
||||
"PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_BENCH", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES",
|
||||
"PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_ASYNC", "REC_END", "REC_SPIN_US", "REEXEC",
|
||||
"RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHUNKS", "REPLAY_CHUNK_DRAWS", "REPLAY_STATE_CHECK", "REPLAY_WORKERS",
|
||||
"RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_CHUNKS", "REPLAY_CHUNK_DRAWS", "REPLAY_SKEW_US", "REPLAY_STATE_CHECK", "REPLAY_WORKERS",
|
||||
"REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
|
||||
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
|
||||
"SHADER_MEMO", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR",
|
||||
@@ -165,6 +165,7 @@ static const CfgMeta metas[] = {
|
||||
{"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART},
|
||||
{"REPLAY_WORKERS", "0", "0..16", RESTART}, {"REPLAY_CHUNKS", "0", "0..64", RESTART},
|
||||
{"REPLAY_CHUNK_DRAWS", "128", "1..100000", RESTART}, {"REPLAY_STATE_CHECK", "0", "0|1", RESTART},
|
||||
{"REPLAY_SKEW_US", "0", "0..100000", RESTART},
|
||||
{"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART},
|
||||
// shaders and pipelines
|
||||
{"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "fullscreen", "strict|any|fullscreen", RESTART},
|
||||
|
||||
@@ -54,6 +54,7 @@ static PlatMutex fw_lock = PLAT_MUTEX_INIT; // the recording threads of a chun
|
||||
|
||||
uint64_t vk_exec_recording_fence(void) { return x_cur ? x_cur->fence : 0; }
|
||||
uint32_t vk_exec_replay_pos(void) { return x_cur && x_cur->chunked ? x_cur->pos : UINT32_MAX; }
|
||||
static _Thread_local int chunk_k = -1; // the chunk this thread records (chunk_record), -1 none
|
||||
|
||||
|
||||
// A context's counters go into vk_exec_frame (the submitting thread, after the recording).
|
||||
@@ -1072,6 +1073,7 @@ typedef struct { uint64_t addr, size; uint32_t at; } ScanDrop;
|
||||
|
||||
static struct {
|
||||
int workers, chunks, min_draws, check;
|
||||
int skew_us; // TOTK_REPLAY_SKEW_US (chunk_record)
|
||||
VkExec s; // the scan's state
|
||||
Chunk *ch; // chunks + 1 entries, never moved while threads record
|
||||
int nch;
|
||||
@@ -1097,6 +1099,7 @@ static struct {
|
||||
uint64_t subs, chunked, nchunks, scan_ns, first_ns, last_ns, rec_ns, rec_max_ns, bad, prep_ns, start0_ns;
|
||||
uint64_t rec_t0; // vk_exec_record's start
|
||||
uint64_t held[HOLD_N]; // TOTK_TRACE=perf: chunks held, by scan_hold's reason
|
||||
uint64_t settles, settle_ns; // TOTK_TRACE=perf: vk_exec_chunk_settle waits (atomic: recording threads)
|
||||
uint64_t mirror_ns, usnap_ns, ready_ns; // vk_mem_mirror_begin, vk_usnap_begin, both done after t0
|
||||
} SC = {.workers = -1};
|
||||
|
||||
@@ -1278,6 +1281,10 @@ static VkCommandBuffer rec_cb(Rec *r, uint64_t fence) {
|
||||
// Chunk k on the calling thread with context r: from its state copy up to the next chunk's first op, into a command
|
||||
// buffer of r's pool, ending with every image in GENERAL.
|
||||
static void chunk_record(Rec *r, int k) {
|
||||
// TOTK_REPLAY_SKEW_US (debug): the even chunks wait this long before they record, so with several threads the
|
||||
// chunk after each one records first. A recording that depends on the order the threads get to the chunks, rather
|
||||
// than on replay order, then shows in every submission instead of a few launches in thirty.
|
||||
if (SC.skew_us > 0 && nthreads && !(k & 1)) plat_sleep_ns((uint64_t)SC.skew_us * 1000);
|
||||
uint64_t t0 = kern_now_ns();
|
||||
VkExec *ex = &r->ex;
|
||||
Chunk *c = &SC.ch[k];
|
||||
@@ -1287,6 +1294,7 @@ static void chunk_record(Rec *r, int k) {
|
||||
ex->fence = RP.fence, ex->recorded = 0, ex->in_pass = 0, ex->nviews = 0, ex->barrier_at = -1, ex->nfbl = 0;
|
||||
ex->part_start = 0;
|
||||
x_cur = ex;
|
||||
chunk_k = k;
|
||||
vk_completion_defer(&c->defer);
|
||||
vk_mem_count_local(r->traffic);
|
||||
int ts = vk_sync_ts_open(ex->cb);
|
||||
@@ -1307,6 +1315,7 @@ static void chunk_record(Rec *r, int k) {
|
||||
vk_mem_count_local(NULL);
|
||||
vk_mem_count_flush(r->traffic);
|
||||
x_cur = NULL;
|
||||
chunk_k = -1;
|
||||
vkEndCommandBuffer(ex->cb);
|
||||
c->cb = ex->cb, c->by = ex, c->ns = kern_now_ns() - t0;
|
||||
plat_mutex_lock(&RP.m);
|
||||
@@ -1413,6 +1422,22 @@ static void chunks_wait(int k) {
|
||||
plat_mutex_unlock(&RP.m);
|
||||
}
|
||||
|
||||
int vk_exec_chunk_behind(void) {
|
||||
for (int i = chunk_k - 1; i >= 0; i--)
|
||||
if (!__atomic_load_n(&SC.ch[i].done, __ATOMIC_ACQUIRE)) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// The chunks before take no lock this one holds and wait only for chunks before them, so they all get done: a chunk
|
||||
// is taken only after every chunk before it was (chunk_take), and the DLSS records wait the same way (chunks_wait).
|
||||
void vk_exec_chunk_settle(void) {
|
||||
if (chunk_k <= 0) return;
|
||||
uint64_t t0 = kern_now_ns();
|
||||
chunks_wait(chunk_k);
|
||||
__atomic_add_fetch(&SC.settles, 1, __ATOMIC_RELAXED);
|
||||
__atomic_add_fetch(&SC.settle_ns, kern_now_ns() - t0, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static void scan_job(VkExecJob *j, const op_hdr *h) {
|
||||
static int full;
|
||||
j->chunk = SC.nch - 1;
|
||||
@@ -1603,6 +1628,9 @@ static int chunked_on(void) {
|
||||
if (SC.chunks <= 0) SC.chunks = 2 * SC.workers;
|
||||
SC.min_draws = cfg_int("TOTK_REPLAY_CHUNK_DRAWS");
|
||||
SC.check = cfg_bool("TOTK_REPLAY_STATE_CHECK");
|
||||
SC.skew_us = cfg_int("TOTK_REPLAY_SKEW_US");
|
||||
if (SC.workers && SC.skew_us > 0)
|
||||
rt_log("vk: TOTK_REPLAY_SKEW_US: the even chunks wait %d us before they record", SC.skew_us);
|
||||
if (SC.workers)
|
||||
rt_log("vk: replay in up to %d chunks per submission of at least %d draws each, %d recording thread%s%s",
|
||||
SC.chunks, SC.min_draws, SC.workers, SC.workers > 1 ? "s" : "", SC.check ? ", state checked" : "");
|
||||
@@ -1669,6 +1697,9 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
|
||||
rt_log("perf: chunked replay: chunks held per submission: %.2f for a new texture, %.2f for DLSS, %.2f for "
|
||||
"unknown storage writes", SC.held[HOLD_TEX] / (double)SC.chunked, SC.held[HOLD_DLSS] / (double)SC.chunked,
|
||||
SC.held[HOLD_WRITES] / (double)SC.chunked);
|
||||
rt_log("perf: chunked replay: %llu chunks waited for the chunks before them to make a texture (%.3f ms in all)",
|
||||
(unsigned long long)__atomic_exchange_n(&SC.settles, 0, __ATOMIC_RELAXED),
|
||||
__atomic_exchange_n(&SC.settle_ns, 0, __ATOMIC_RELAXED) / 1e6);
|
||||
memset(SC.held, 0, sizeof SC.held);
|
||||
char b[768];
|
||||
int o = 0;
|
||||
|
||||
@@ -224,6 +224,11 @@ extern VkExecFrame vk_exec_frame;
|
||||
void vk_exec_frame_written(const tex_t *t); // vk_texture_gpu_written: t rendered or copied into
|
||||
uint64_t vk_exec_recording_fence(void); // fence of the submission this thread records, else 0
|
||||
uint32_t vk_exec_replay_pos(void); // the op this thread records in a chunk, else UINT32_MAX
|
||||
// A chunk that is about to make a texture entry whose content depends on the textures at its storage (vk_texture.c
|
||||
// entry_of): whether a chunk before it is still being recorded, and the wait until none is. The caller holds no lock
|
||||
// but the texture lock, which it releases for the wait.
|
||||
int vk_exec_chunk_behind(void);
|
||||
void vk_exec_chunk_settle(void);
|
||||
void vk_exec_frame_log(const tex_t *presented, uint64_t frame);
|
||||
|
||||
// TOTK_DUMP=<frame> (vk_dump.c): between presents frame-1 and frame, every pass's targets and every dispatch's storage
|
||||
|
||||
+69
-16
@@ -47,6 +47,7 @@ typedef struct Tex {
|
||||
uint32_t checked; // submit_epoch of the last check from a draw binding
|
||||
int uploaded, released;
|
||||
uint32_t rev_epoch, rev_pos; // the state scan made it live in that submission at that op (vk_texture_revive)
|
||||
uint32_t own_epoch, own_pos; // a chunk made it GPU-owned in that submission at that op (vk_texture_gpu_written)
|
||||
uint32_t n_up; // uploads of this entry (vk_texture_debug)
|
||||
int last_ch; // last vk_mem_changed result of make_fresh: 1 written, 0 clean, -1 not tracked
|
||||
struct Tex *next_at; // other entries at the same storage address
|
||||
@@ -146,11 +147,17 @@ static void tex_init(void *p) {
|
||||
__atomic_store_n(&tex_ready, 1, __ATOMIC_RELEASE);
|
||||
return;
|
||||
}
|
||||
static _Thread_local int tex_depth; // how often this thread holds tex_lock (entry_of releases it for a wait)
|
||||
static _Thread_local uint32_t tex_settles; // entry_of waits of this thread (vk_texture_by_id_locked)
|
||||
static void lock(void) {
|
||||
if (!__atomic_load_n(&tex_ready, __ATOMIC_ACQUIRE)) plat_once(&tex_once, tex_init, NULL);
|
||||
plat_rmutex_lock(&tex_lock);
|
||||
tex_depth++;
|
||||
}
|
||||
static void unlock(void) {
|
||||
tex_depth--;
|
||||
plat_rmutex_unlock(&tex_lock);
|
||||
}
|
||||
static void unlock(void) { plat_rmutex_unlock(&tex_lock); }
|
||||
|
||||
static uint32_t umax(uint32_t a, uint32_t b) { return a > b ? a : b; }
|
||||
static uint32_t mip(uint32_t v, unsigned lv) { return umax(v >> lv, 1); }
|
||||
@@ -1234,10 +1241,20 @@ static int alias_compatible(const Tex *e, const Tex *src) {
|
||||
return eb && eb == sb && ew == sw && eh == sh;
|
||||
}
|
||||
|
||||
// src holds content the storage bytes do not: the GPU wrote it and, for a mirrored texture, it was not written back
|
||||
// and the CPU has not rewritten the storage since (the new entry's upload then reads the newer bytes).
|
||||
static int alias_has_gpu_content(const Tex *src) {
|
||||
if (!src->pub.gpu_owned) return 0;
|
||||
// Whether e is live at op pos of a chunked replay: the state scan is ahead of the chunks, and an entry it revived at a
|
||||
// later op is still released for this one. Two 4K targets at one address, RGBA8 and R8 with memory of their own, were
|
||||
// swapped whenever the scan got past the second's SET_RT before a chunk sampling the storage got its turn, and large
|
||||
// parts of the scene flickered. UINT32_MAX: the state now.
|
||||
static int live_at(const Tex *e, uint32_t pos) {
|
||||
return !e->released && !(e->rev_epoch == submit_epoch && e->rev_pos > pos);
|
||||
}
|
||||
|
||||
// src holds content the storage bytes do not, at op pos of a chunked replay (UINT32_MAX: the state now): the GPU wrote
|
||||
// it and, for a mirrored texture, it was not written back and the CPU has not rewritten the storage since (the new
|
||||
// entry's upload then reads the newer bytes). A chunk at a later op that rendered src first does not count: in replay
|
||||
// order that render comes after the new entry's first use.
|
||||
static int alias_has_gpu_content(const Tex *src, uint32_t pos) {
|
||||
if (!src->pub.gpu_owned || (src->own_epoch == submit_epoch && src->own_pos > pos)) return 0;
|
||||
if (!src->pub.mirrored) return 1;
|
||||
uint32_t now;
|
||||
return src->gpu_dirty && !storage_rewritten(src, src->gpu_gen, &now);
|
||||
@@ -1298,10 +1315,38 @@ static void entry_live(Tex *e) {
|
||||
e->released = 0, nreleased--, rel_unlink(e), tex_gen_bump(e->key.addr);
|
||||
}
|
||||
|
||||
// A live colour entry at e's storage of e's size, as of op pos: the new entry may take its content (alias_copy).
|
||||
static int alias_candidate(const TexKey *k, const tex_t *t, uint32_t pos) {
|
||||
for (const Tex *p = nvn_hm_get(&by_addr, &k->addr); p; p = p->next_at)
|
||||
if (live_at(p, pos) && !p->f.depth && !p->f.stencil && p->t.L.size == t->L.size) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static Tex *entry_of(const tex_t *t) {
|
||||
TexKey k;
|
||||
make_key(t, &k);
|
||||
Tex *e = nvn_hm_get(&by_key, &k);
|
||||
uint32_t pos = vk_exec_replay_pos();
|
||||
// A chunk of a chunked replay makes the entry as of its own op: whether it takes the content of a texture at its
|
||||
// storage (alias_copy) depends on what the chunks before it render there, and those may still be recording on
|
||||
// other threads. Taken from a render they had not recorded yet, the new image got the storage's old bytes instead:
|
||||
// on an RX 7800 XT the grass near Link was missing in 3 of 29 launches with 4 threads (a map texture drawn once
|
||||
// per tile, kept from then on). With a live colour texture of its size at the storage, the chunk waits until the
|
||||
// chunks before it are recorded, the texture lock released for the wait (as often as this thread holds it).
|
||||
if (!e && pos != UINT32_MAX && tex_alias != 0 && alias_candidate(&k, t, pos) && vk_exec_chunk_behind()) {
|
||||
tex_t own = *t; // t may point into tslots, which a registration moves while the lock is released
|
||||
t = &own;
|
||||
int d = tex_depth;
|
||||
for (int i = 0; i < d; i++) unlock();
|
||||
vk_exec_chunk_settle();
|
||||
for (int i = 0; i < d; i++) lock();
|
||||
tex_settles++;
|
||||
if ((e = nvn_hm_get(&by_key, &k))) { // another chunk made it meanwhile
|
||||
entry_live(e);
|
||||
e->t = *t;
|
||||
return e;
|
||||
}
|
||||
}
|
||||
if (!e) {
|
||||
if (tex_alias < 0) {
|
||||
tex_alias = cfg_bool("TOTK_TEX_ALIAS");
|
||||
@@ -1319,8 +1364,9 @@ static Tex *entry_of(const tex_t *t) {
|
||||
nx = p->next_at;
|
||||
if (p->released) { destroy_tex(p); continue; }
|
||||
alias = 1;
|
||||
if (!live_at(p, pos)) continue; // the scan revives it at a later op: released at this chunk's
|
||||
if (tex_alias && (!src || (same_but_swizzle(&p->key, &k) && !same_but_swizzle(&src->key, &k))) &&
|
||||
alias_compatible(e, p) && alias_has_gpu_content(p))
|
||||
alias_compatible(e, p) && alias_has_gpu_content(p, pos))
|
||||
src = p;
|
||||
}
|
||||
if (alias) perf_count(PERF_CNT_TEX_ALIAS);
|
||||
@@ -1423,6 +1469,10 @@ void vk_texture_gpu_written(const tex_t *t) {
|
||||
Tex *e = entry_of(t);
|
||||
if (e) {
|
||||
make_fresh(e); // a partial render keeps what the storage held before
|
||||
if (!e->pub.gpu_owned) { // entry_of's alias choice of a chunk at an earlier op leaves this render out
|
||||
uint32_t pos = vk_exec_replay_pos();
|
||||
e->own_epoch = pos == UINT32_MAX ? 0 : submit_epoch, e->own_pos = pos;
|
||||
}
|
||||
e->pub.gpu_owned = 1;
|
||||
uint64_t f = vk_exec_recording_fence();
|
||||
if (f > e->wr_fence) e->wr_fence = f;
|
||||
@@ -1511,14 +1561,6 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
|
||||
unlock();
|
||||
}
|
||||
|
||||
// Whether e is live at op pos of a chunked replay: the state scan is ahead of the chunks, and an entry it revived at a
|
||||
// later op is still released for this one. Two 4K targets at one address, RGBA8 and R8 with memory of their own, were
|
||||
// swapped whenever the scan got past the second's SET_RT before a chunk sampling the storage got its turn, and large
|
||||
// parts of the scene flickered. UINT32_MAX: the state now.
|
||||
static int live_at(const Tex *e, uint32_t pos) {
|
||||
return !e->released && !(e->rev_epoch == submit_epoch && e->rev_pos > pos);
|
||||
}
|
||||
|
||||
// The entry slot s resolves to at op pos, and the view and has_view a caller gets with it (the slot's, with the
|
||||
// descriptor's swizzle when the entry is another descriptor at the same storage). NULL when the slot names no texture.
|
||||
static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view, uint32_t pos) {
|
||||
@@ -1546,7 +1588,8 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view, uint32_t pos)
|
||||
}
|
||||
if (!e) e = any ? any : own;
|
||||
}
|
||||
if (!e) e = s->t.mem ? entry_of(&s->t) : NULL;
|
||||
TexSlot copy; // entry_of may release the lock (a chunk's wait), and a registration then moves tslots
|
||||
if (!e && s->t.mem) copy = *s, s = ©, e = entry_of(&s->t);
|
||||
// A followed texture is another descriptor at the same storage: its image holds the content, but sampling takes the
|
||||
// slot descriptor's swizzle. The UI renders text into RGBA8 and samples it through a (A,A,A,A) descriptor; with the
|
||||
// render target's RGBA swizzle the text blur saw R = 1 everywhere and drew solid boxes behind tips and labels.
|
||||
@@ -1594,6 +1637,7 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
|
||||
view_t v;
|
||||
int hv;
|
||||
Tex *f = slot_resolve(s, &v, &hv, pos);
|
||||
s = &tslots[id]; // slot_resolve may have released the lock
|
||||
if (f && (!f->uploaded || f->checked != submit_epoch)) {
|
||||
make_fresh_snap(f);
|
||||
f->checked = submit_epoch;
|
||||
@@ -1605,7 +1649,16 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
|
||||
if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the
|
||||
view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may
|
||||
int hv; // succeed later
|
||||
s->ce = slot_resolve(s, &v, &hv, UINT32_MAX); // entry_of may create an entry and bump g: g read before it
|
||||
uint32_t settles = tex_settles;
|
||||
Tex *r = slot_resolve(s, &v, &hv, UINT32_MAX); // entry_of may create an entry and bump g: g read before it
|
||||
s = &tslots[id];
|
||||
if (settles != tex_settles) { // the lock was released for a wait: the slot may be another texture by now
|
||||
if (r && (!r->uploaded || r->checked != submit_epoch)) make_fresh_snap(r), r->checked = submit_epoch;
|
||||
if (view) *view = has_view ? v : s->v;
|
||||
if (has_view) *has_view = view ? hv : s->has_view;
|
||||
return r ? &r->pub : NULL;
|
||||
}
|
||||
s->ce = r;
|
||||
s->cview = v, s->chas_view = (uint8_t)hv;
|
||||
s->cgen = g;
|
||||
s->cepoch = 0;
|
||||
|
||||
Reference in new issue
Block a user