diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 9c262928..d6a48643 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -11,6 +11,7 @@ GPU semantics behind the graphics rows: [maxwell.md](../maxwell.md). The method | Symptom | Usual causes | First check | |---|---|---| | magenta, brown or stale textures; caves or sky wrong | the CPU filled the texture and the GPU never got the upload (no flush on the console) | frame dump: the texture's upload count and mirror state in `passes.txt` | +| rain shows only its splashes on the ground, no falling drops; waterfall splashes and mist blink | a GPU write-back copies a whole range over particle records the CPU spawned in it (`vk_memory.c`: the write-back merge, and with `shadow = 1` the shadow's pending ranges) | a run with `TOTK_WB_MERGE=0` or `TOTK_SHADOW=0` against one without | | effects that never happen: cut grass stays up, fire does not spread | the CPU reads a GPU-rendered texture that was never written back | which texture the game reads after its fence; write-back of that target | | black scene, black dialog background or loading map | deferred binds read at record time; a 3D target rejected; a texture compare ignoring the channel mapping | frame dump: the first pass whose target goes black | | lime silhouettes, rectangles in the sky | a target outside the shader's output map overwritten instead of kept | dump: the target's previous contents versus after the pass | diff --git a/docs/settings.md b/docs/settings.md index af489a70..e1717a0f 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -153,7 +153,7 @@ Many rows below are developer tools that only act together with another setting | `vram_headroom` (default `1`) | `0` turns off the spare half-block of video memory that a background thread keeps per memory type. | Texture allocations on the render thread can wait for a new block again. | Caution | | `vs_outs` | `all` declares all 32 vertex shader output locations; otherwise only the union of the paired fragment inputs. | Larger vertex shaders and new pipelines; unwritten outputs read the same as before. | Caution | | `wb_exact` (default `1`) | `0` decides write-back of GPU-written textures by page only. | A target that shares a page with data the CPU writes can go unwritten, so a name label next to the reticle can blink. | Caution | -| `wb_merge` (default `1`) | `0` copies GPU-written ranges back as plain copies instead of writing word by word the words that changed. | In mirror memory mode the GPU's copy can overwrite words the CPU wrote, which breaks effects such as falling rain and waterfall splashes. | Caution | +| `wb_merge` (default `1`) | `0` copies GPU-written ranges back as plain copies instead of writing word by word the words that changed; with `shadow = 1` it also leaves ranges the GPU has yet to write back out of the copy into the shadow. | In mirror memory mode, and with `shadow = 1`, the GPU's copy can overwrite words the CPU wrote, which breaks effects such as falling rain and waterfall splashes. | Caution | ## Motion vectors and DLSS diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 9b959184..1e73828e 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -989,14 +989,21 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { // - CPU -> shadow: right before each submit (vk_mem_shadow_sync), the pools the replay resolved since the last one are // queried for written pages; pages stamped after a pool's sgen are copied in by the CPU. // - GPU -> guest: the replay notes each GPU write into CPU-visible memory at record time (vk_mem_gpu_pending) and -// reports it at the fence (vk_gpu_written), which copies those bytes back into guest memory. Until then the CPU -// copies skip them: a page the CPU and the GPU both write keeps the GPU's bytes. +// reports it at the fence (vk_gpu_written), which copies the bytes the GPU changed back into guest memory. +// Both directions go word by word while a write is pending, as the write-back merge does for imported pools: TotK's +// particle emitters write a new particle's whole record into a free slot of a buffer the simulation shaders update +// every frame, so that buffer is always pending. A whole-range copy back put the shadow's stale words over the fresh +// spawns, and a CPU copy that skipped pending ranges never let them reach the GPU: falling rain showed only its +// splashes. Each pending range keeps a base, the words guest memory and the shadow last agreed on (the shadow's bytes +// when the range became pending). The CPU copy takes the words where guest memory differs from the base; the fence +// copy back takes the words where the shadow differs from it. A word both changed takes the GPU's. TOTK_WB_MERGE=0 +// skips the pending ranges in the CPU copy and copies them back whole, as before. // TOTK_SHADOW=1/0 (shadow in totk.ini) forces the mode on or off; the default is on for Linux and Wine. -typedef struct { uint64_t lo, hi; } SRange; +typedef struct { uint64_t lo, hi; uint8_t *base; int refs; } SRange; static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock static PoolRec **stouch; // shadow_lock: pools resolved since the last sync static int nstouch, cstouch; -static SRange *spend; // shadow_lock: GPU writes recorded and not copied back yet +static SRange *spend; // shadow_lock: distinct GPU writes recorded and not copied back yet static int nspend, cspend; static int64_t s_live_n, s_live_bytes; // shadow_lock static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks; // shadow_lock: since the last log @@ -1062,15 +1069,102 @@ static void shadow_forget(PoolRec *r) { plat_mutex_unlock(&shadow_lock); } -// shadow_lock held: copies guest [a, b) into the shadow, leaving out the bytes of GPU writes not copied back yet -// (spend from index i on). +// TOTK_WB_MERGE: the write-back merge of mirrors (wb_merge_record) and the word merge of the shadow's pending ranges. +static int merge_on = -1; +static int merge_wanted(void) { + if (merge_on < 0) { + merge_on = cfg_bool("TOTK_WB_MERGE"); + rt_log("mem: write-back merge %s (TOTK_WB_MERGE)", merge_on ? "on" : "off"); + } + return merge_on; +} + +// The word at byte offset i of p's n bytes; a last word shorter than 4 bytes reads as zero-extended. The common case +// is a fixed 4-byte load: a memcpy of variable size is a library call per word. +static uint32_t word_at(const uint8_t *p, uint64_t i, uint64_t n) { + uint32_t v = 0; + if (n - i >= 4) memcpy(&v, p + i, 4); + else memcpy(&v, p + i, n - i); + return v; +} + +// A run of differing words from *at on, as byte offsets [*at, *end) of x and y (n bytes, the last word may be shorter); +// 0 when none is left. +static int diff_run(const uint8_t *x, const uint8_t *y, uint64_t n, uint64_t *at, uint64_t *end) { + uint64_t i = *at; + while (i + 64 <= n && !memcmp(x + i, y + i, 64)) i += 64; // most words are equal: skip them a block at a time + while (i < n && word_at(x, i, n) == word_at(y, i, n)) i += 4; + if (i >= n) return 0; + uint64_t j = i + 4; + while (j < n && word_at(x, j, n) != word_at(y, j, n)) j += 4; + *at = i, *end = j < n ? j : n; + return 1; +} + +// The pending ranges with a base over a guest range, found once per merge rather than once per differing run: the +// game's particle buffers differ in many short runs, and about a hundred ranges are pending at a time. n < 0: more than +// BASES_NEAR, every range is checked. +#define BASES_NEAR 16 +typedef struct { int n, at[BASES_NEAR]; } BaseSet; + +// shadow_lock held: the pending ranges with a base over guest [a, b). Valid while spend is unchanged. +static void bases_over(uint64_t a, uint64_t b, BaseSet *s) { + s->n = 0; + for (int i = 0; i < nspend; i++) { + if (!spend[i].base || spend[i].lo >= b || spend[i].hi <= a) continue; + if (s->n == BASES_NEAR) { s->n = -1; return; } + s->at[s->n++] = i; + } +} + +// shadow_lock held: src's bytes become guest [a, b) in p's base, where they overlap. +static void base_put(const SRange *p, uint64_t a, uint64_t b, const uint8_t *src) { + if (!p->base || p->lo >= b || p->hi <= a) return; + uint64_t x = a > p->lo ? a : p->lo, y = b < p->hi ? b : p->hi; + memcpy(p->base + (x - p->lo), src + (x - a), y - x); +} + +// shadow_lock held: src's bytes become guest [a, b) in the base of every pending range over them; s holds those +// ranges for a span that contains [a, b). +static void base_set(const BaseSet *s, uint64_t a, uint64_t b, const uint8_t *src) { + if (s->n < 0) { + for (int i = 0; i < nspend; i++) base_put(&spend[i], a, b, src); + return; + } + for (int k = 0; k < s->n; k++) base_put(&spend[s->at[k]], a, b, src); +} + +static void shadow_put(PoolRec *r, uint64_t a, uint64_t b); + +// shadow_lock held: the words of guest [a, b), inside pending range p, that differ from p's base go into the shadow. +static void shadow_merge(PoolRec *r, uint64_t a, uint64_t b, const SRange *p) { + const uint8_t *g = (const uint8_t *)(uintptr_t)a; + uint64_t x = 0, y; + BaseSet bs; + bases_over(a, b, &bs); + while (diff_run(g, p->base + (a - p->lo), b - a, &x, &y)) { + base_set(&bs, a + x, a + y, g + x); + shadow_put(r, a + x, a + y); + x = y; + } +} + +// shadow_lock held: copies guest [a, b) into the shadow; the bytes of GPU writes not copied back yet (spend from index +// i on) only where the CPU changed them. static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b, int i) { for (; i < nspend; i++) if (spend[i].lo < b && spend[i].hi > a) { if (spend[i].lo > a) shadow_copy(r, a, spend[i].lo, i + 1); if (spend[i].hi < b) shadow_copy(r, spend[i].hi, b, i + 1); + if (spend[i].base) + shadow_merge(r, a > spend[i].lo ? a : spend[i].lo, b < spend[i].hi ? b : spend[i].hi, &spend[i]); return; } + shadow_put(r, a, b); +} + +// shadow_lock held: guest [a, b) into the shadow. +static void shadow_put(PoolRec *r, uint64_t a, uint64_t b) { if (sdefer_now) { // the GPU copies it later: the bytes are read from guest memory when the prologue is recorded if (ndq == cdq) grow((void **)&dq, &cdq, sizeof *dq); dq[ndq++] = (DRange){r, a, b - a}; @@ -1240,12 +1334,35 @@ void vk_mem_shadow_sync(uint64_t fence) { int vk_mem_shadow_deferred(void) { return ndq; } +// Record time: the range's base is the shadow's bytes, except where an overlapping pending range has one: the GPU may +// have written those shadow bytes since. void vk_mem_gpu_pending(uint64_t addr, uint64_t size) { if (!shadow_on || !size) return; + uint64_t lo = addr, hi = addr + size; + plat_mutex_lock_shared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, addr, size); + PoolRec *r = e ? e->buf : NULL; plat_mutex_lock(&shadow_lock); - if (nspend == cspend) grow((void **)&spend, &cspend, sizeof *spend); - spend[nspend++] = (SRange){addr, addr + size}; + int i = 0; + while (i < nspend && (spend[i].lo != lo || spend[i].hi != hi)) i++; + if (i < nspend) { + spend[i].refs++; + } else { + uint8_t *base = NULL; + if (r && r->smap && lo >= r->import_lo && hi <= r->import_hi && merge_wanted() && (base = malloc(size))) { + memcpy(base, r->smap + (lo - r->import_lo), size); + for (int k = 0; k < nspend; k++) { + const SRange *p = &spend[k]; + if (!p->base || p->lo >= hi || p->hi <= lo) continue; + uint64_t x = lo > p->lo ? lo : p->lo, y = hi < p->hi ? hi : p->hi; + memcpy(base + (x - lo), p->base + (x - p->lo), y - x); + } + } + if (nspend == cspend) grow((void **)&spend, &cspend, sizeof *spend); + spend[nspend++] = (SRange){lo, hi, base, 1}; + } plat_mutex_unlock(&shadow_lock); + plat_mutex_unlock_shared(&pool_lock); } void vk_gpu_written(uint64_t addr, uint64_t size) { @@ -1254,13 +1371,27 @@ void vk_gpu_written(uint64_t addr, uint64_t size) { const pidx_ent *e = pidx_find(&pool_index, addr, size); PoolRec *r = e ? e->buf : NULL; plat_mutex_lock(&shadow_lock); + int k = 0; + while (k < nspend && (spend[k].lo != addr || spend[k].hi != addr + size)) k++; if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi) { mem_thaw(addr, size); - memcpy((void *)(uintptr_t)addr, r->smap + (addr - r->import_lo), size); - s_wb_bytes += size; + const uint8_t *s = r->smap + (addr - r->import_lo); + if (k < nspend && spend[k].base) { + uint64_t x = 0, y; + BaseSet bs; + bases_over(addr, addr + size, &bs); + while (diff_run(s, spend[k].base, size, &x, &y)) { + memcpy((void *)(uintptr_t)(addr + x), s + x, y - x); + base_set(&bs, addr + x, addr + y, (const uint8_t *)(uintptr_t)(addr + x)); + s_wb_bytes += y - x; + x = y; + } + } else { + memcpy((void *)(uintptr_t)addr, s, size); + s_wb_bytes += size; + } } - for (int i = 0; i < nspend; i++) - if (spend[i].lo == addr && spend[i].hi == addr + size) { spend[i] = spend[--nspend]; break; } + if (k < nspend && !--spend[k].refs) free(spend[k].base), spend[k] = spend[--nspend]; plat_mutex_unlock(&shadow_lock); plat_mutex_unlock_shared(&pool_lock); } @@ -1802,7 +1933,6 @@ static PlatMutex snap_pool_lock = PLAT_MUTEX_INIT; static SnapChunk snap_pool[SNAP_POOL_N]; static int nsnap_pool; static uint64_t snap_pool_bytes; -static int merge_on = -1; static struct { int tried, ok; PFN_vkCmdPushDescriptorSetKHR push; @@ -1845,11 +1975,7 @@ fail: } static int merging(void) { - if (merge_on < 0) { - merge_on = cfg_bool("TOTK_WB_MERGE"); - rt_log("mem: write-back merge %s (TOTK_WB_MERGE)", merge_on ? "on" : "off"); - } - if (!merge_on) return 0; + if (!merge_wanted()) return 0; if (!WM.tried) merge_init(vk_ctx()); return WM.ok; } @@ -2988,6 +3114,7 @@ void vk_mem_shutdown(void) { nsnap_pool = 0, snap_pool_bytes = 0; plat_mutex_unlock(&snap_pool_lock); plat_mutex_lock(&shadow_lock); + for (int i = 0; i < nspend; i++) free(spend[i].base); nstouch = nspend = 0; // shadows too s_live_n = s_live_bytes = 0; plat_mutex_unlock(&shadow_lock);