From cabbfc5c8656ce22e33281afa49244add2dc927d Mon Sep 17 00:00:00 2001 From: ipavlin98 Date: Mon, 16 Nov 2026 03:46:43 +0300 Subject: [PATCH 1/4] Shadow storage merges pending GPU writes word by word in both directions, so falling rain draws with shadow = 1: the CPU copy into the shadow skipped every range a recorded GPU write covered, and the particle buffer the rain shaders update is always one, so new drops never reached the GPU, and the copy back at the fence put the shadow's whole range over them --- docs/agents/symptoms.md | 1 + runtime/vk_memory.c | 116 +++++++++++++++++++++++++++++++++++----- 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 9c262928..d6a48643 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -11,6 +11,7 @@ GPU semantics behind the graphics rows: [maxwell.md](../maxwell.md). The method | Symptom | Usual causes | First check | |---|---|---| | magenta, brown or stale textures; caves or sky wrong | the CPU filled the texture and the GPU never got the upload (no flush on the console) | frame dump: the texture's upload count and mirror state in `passes.txt` | +| rain shows only its splashes on the ground, no falling drops; waterfall splashes and mist blink | a GPU write-back copies a whole range over particle records the CPU spawned in it (`vk_memory.c`: the write-back merge, and with `shadow = 1` the shadow's pending ranges) | a run with `TOTK_WB_MERGE=0` or `TOTK_SHADOW=0` against one without | | effects that never happen: cut grass stays up, fire does not spread | the CPU reads a GPU-rendered texture that was never written back | which texture the game reads after its fence; write-back of that target | | black scene, black dialog background or loading map | deferred binds read at record time; a 3D target rejected; a texture compare ignoring the channel mapping | frame dump: the first pass whose target goes black | | lime silhouettes, rectangles in the sky | a target outside the shader's output map overwritten instead of kept | dump: the target's previous contents versus after the pass | diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 9b959184..21c3f7b7 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -989,14 +989,20 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { // - CPU -> shadow: right before each submit (vk_mem_shadow_sync), the pools the replay resolved since the last one are // queried for written pages; pages stamped after a pool's sgen are copied in by the CPU. // - GPU -> guest: the replay notes each GPU write into CPU-visible memory at record time (vk_mem_gpu_pending) and -// reports it at the fence (vk_gpu_written), which copies those bytes back into guest memory. Until then the CPU -// copies skip them: a page the CPU and the GPU both write keeps the GPU's bytes. +// reports it at the fence (vk_gpu_written), which copies the bytes the GPU changed back into guest memory. +// Both directions go word by word while a write is pending, as the write-back merge does for imported pools: TotK's +// particle emitters write a new particle's whole record into a free slot of a buffer the simulation shaders update +// every frame, so that buffer is always pending. A whole-range copy back put the shadow's stale words over the fresh +// spawns, and a CPU copy that skipped pending ranges never let them reach the GPU: falling rain showed only its +// splashes. Each pending range keeps a base, the words guest memory and the shadow last agreed on (the shadow's bytes +// when the range became pending). The CPU copy takes the words where guest memory differs from the base; the fence +// copy back takes the words where the shadow differs from it. A word both changed takes the GPU's. // TOTK_SHADOW=1/0 (shadow in totk.ini) forces the mode on or off; the default is on for Linux and Wine. -typedef struct { uint64_t lo, hi; } SRange; +typedef struct { uint64_t lo, hi; uint8_t *base; int refs; } SRange; static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock static PoolRec **stouch; // shadow_lock: pools resolved since the last sync static int nstouch, cstouch; -static SRange *spend; // shadow_lock: GPU writes recorded and not copied back yet +static SRange *spend; // shadow_lock: distinct GPU writes recorded and not copied back yet static int nspend, cspend; static int64_t s_live_n, s_live_bytes; // shadow_lock static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks; // shadow_lock: since the last log @@ -1062,15 +1068,65 @@ static void shadow_forget(PoolRec *r) { plat_mutex_unlock(&shadow_lock); } -// shadow_lock held: copies guest [a, b) into the shadow, leaving out the bytes of GPU writes not copied back yet -// (spend from index i on). +// A run of differing words from *at on, as byte offsets [*at, *end) of x and y (n bytes, the last word may be shorter); +// 0 when none is left. +static int diff_run(const uint8_t *x, const uint8_t *y, uint64_t n, uint64_t *at, uint64_t *end) { + uint64_t i = *at; + for (; i < n; i += 4) { + uint32_t u = 0, v = 0, w = n - i < 4 ? (uint32_t)(n - i) : 4; + memcpy(&u, x + i, w), memcpy(&v, y + i, w); + if (u != v) break; + } + if (i >= n) return 0; + uint64_t j = i + 4; + for (; j < n; j += 4) { + uint32_t u = 0, v = 0, w = n - j < 4 ? (uint32_t)(n - j) : 4; + memcpy(&u, x + j, w), memcpy(&v, y + j, w); + if (u == v) break; + } + *at = i, *end = j < n ? j : n; + return 1; +} + +// shadow_lock held: src's bytes become guest [a, b) in the base of every pending range over them. +static void base_set(uint64_t a, uint64_t b, const uint8_t *src) { + for (int i = 0; i < nspend; i++) { + SRange *p = &spend[i]; + if (!p->base || p->lo >= b || p->hi <= a) continue; + uint64_t x = a > p->lo ? a : p->lo, y = b < p->hi ? b : p->hi; + memcpy(p->base + (x - p->lo), src + (x - a), y - x); + } +} + +static void shadow_put(PoolRec *r, uint64_t a, uint64_t b); + +// shadow_lock held: the words of guest [a, b), inside pending range p, that differ from p's base go into the shadow. +static void shadow_merge(PoolRec *r, uint64_t a, uint64_t b, const SRange *p) { + const uint8_t *g = (const uint8_t *)(uintptr_t)a; + uint64_t x = 0, y; + while (diff_run(g, p->base + (a - p->lo), b - a, &x, &y)) { + base_set(a + x, a + y, g + x); + shadow_put(r, a + x, a + y); + x = y; + } +} + +// shadow_lock held: copies guest [a, b) into the shadow; the bytes of GPU writes not copied back yet (spend from index +// i on) only where the CPU changed them. static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b, int i) { for (; i < nspend; i++) if (spend[i].lo < b && spend[i].hi > a) { if (spend[i].lo > a) shadow_copy(r, a, spend[i].lo, i + 1); if (spend[i].hi < b) shadow_copy(r, spend[i].hi, b, i + 1); + if (spend[i].base) + shadow_merge(r, a > spend[i].lo ? a : spend[i].lo, b < spend[i].hi ? b : spend[i].hi, &spend[i]); return; } + shadow_put(r, a, b); +} + +// shadow_lock held: guest [a, b) into the shadow. +static void shadow_put(PoolRec *r, uint64_t a, uint64_t b) { if (sdefer_now) { // the GPU copies it later: the bytes are read from guest memory when the prologue is recorded if (ndq == cdq) grow((void **)&dq, &cdq, sizeof *dq); dq[ndq++] = (DRange){r, a, b - a}; @@ -1240,12 +1296,35 @@ void vk_mem_shadow_sync(uint64_t fence) { int vk_mem_shadow_deferred(void) { return ndq; } +// Record time: the range's base is the shadow's bytes, except where an overlapping pending range has one: the GPU may +// have written those shadow bytes since. void vk_mem_gpu_pending(uint64_t addr, uint64_t size) { if (!shadow_on || !size) return; + uint64_t lo = addr, hi = addr + size; + plat_mutex_lock_shared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, addr, size); + PoolRec *r = e ? e->buf : NULL; plat_mutex_lock(&shadow_lock); - if (nspend == cspend) grow((void **)&spend, &cspend, sizeof *spend); - spend[nspend++] = (SRange){addr, addr + size}; + int i = 0; + while (i < nspend && (spend[i].lo != lo || spend[i].hi != hi)) i++; + if (i < nspend) { + spend[i].refs++; + } else { + uint8_t *base = NULL; + if (r && r->smap && lo >= r->import_lo && hi <= r->import_hi && (base = malloc(size))) { + memcpy(base, r->smap + (lo - r->import_lo), size); + for (int k = 0; k < nspend; k++) { + const SRange *p = &spend[k]; + if (!p->base || p->lo >= hi || p->hi <= lo) continue; + uint64_t x = lo > p->lo ? lo : p->lo, y = hi < p->hi ? hi : p->hi; + memcpy(base + (x - lo), p->base + (x - p->lo), y - x); + } + } + if (nspend == cspend) grow((void **)&spend, &cspend, sizeof *spend); + spend[nspend++] = (SRange){lo, hi, base, 1}; + } plat_mutex_unlock(&shadow_lock); + plat_mutex_unlock_shared(&pool_lock); } void vk_gpu_written(uint64_t addr, uint64_t size) { @@ -1254,13 +1333,25 @@ void vk_gpu_written(uint64_t addr, uint64_t size) { const pidx_ent *e = pidx_find(&pool_index, addr, size); PoolRec *r = e ? e->buf : NULL; plat_mutex_lock(&shadow_lock); + int k = 0; + while (k < nspend && (spend[k].lo != addr || spend[k].hi != addr + size)) k++; if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi) { mem_thaw(addr, size); - memcpy((void *)(uintptr_t)addr, r->smap + (addr - r->import_lo), size); - s_wb_bytes += size; + const uint8_t *s = r->smap + (addr - r->import_lo); + if (k < nspend && spend[k].base) { + uint64_t x = 0, y; + while (diff_run(s, spend[k].base, size, &x, &y)) { + memcpy((void *)(uintptr_t)(addr + x), s + x, y - x); + base_set(addr + x, addr + y, (const uint8_t *)(uintptr_t)(addr + x)); + s_wb_bytes += y - x; + x = y; + } + } else { + memcpy((void *)(uintptr_t)addr, s, size); + s_wb_bytes += size; + } } - for (int i = 0; i < nspend; i++) - if (spend[i].lo == addr && spend[i].hi == addr + size) { spend[i] = spend[--nspend]; break; } + if (k < nspend && !--spend[k].refs) free(spend[k].base), spend[k] = spend[--nspend]; plat_mutex_unlock(&shadow_lock); plat_mutex_unlock_shared(&pool_lock); } @@ -2988,6 +3079,7 @@ void vk_mem_shutdown(void) { nsnap_pool = 0, snap_pool_bytes = 0; plat_mutex_unlock(&snap_pool_lock); plat_mutex_lock(&shadow_lock); + for (int i = 0; i < nspend; i++) free(spend[i].base); nstouch = nspend = 0; // shadows too s_live_n = s_live_bytes = 0; plat_mutex_unlock(&shadow_lock); From 77a52e457719c23dd9921839170f01e120f2afeb Mon Sep 17 00:00:00 2001 From: ipavlin98 Date: Mon, 16 Nov 2026 03:55:56 +0300 Subject: [PATCH 2/4] TOTK_WB_MERGE=0 also turns off the shadow storage's word merge, so the two can be compared --- docs/settings.md | 2 +- runtime/vk_memory.c | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 1b0193fe..36318fbc 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -153,7 +153,7 @@ Many rows below are developer tools that only act together with another setting | `vram_headroom` (default `1`) | `0` turns off the spare half-block of video memory that a background thread keeps per memory type. | Texture allocations on the render thread can wait for a new block again. | Caution | | `vs_outs` | `all` declares all 32 vertex shader output locations; otherwise only the union of the paired fragment inputs. | Larger vertex shaders and new pipelines; unwritten outputs read the same as before. | Caution | | `wb_exact` (default `1`) | `0` decides write-back of GPU-written textures by page only. | A target that shares a page with data the CPU writes can go unwritten, so a name label next to the reticle can blink. | Caution | -| `wb_merge` (default `1`) | `0` copies GPU-written ranges back as plain copies instead of writing word by word the words that changed. | In mirror memory mode the GPU's copy can overwrite words the CPU wrote, which breaks effects such as falling rain and waterfall splashes. | Caution | +| `wb_merge` (default `1`) | `0` copies GPU-written ranges back as plain copies instead of writing word by word the words that changed; with `shadow = 1` it also leaves ranges the GPU has yet to write back out of the copy into the shadow. | In mirror memory mode, and with `shadow = 1`, the GPU's copy can overwrite words the CPU wrote, which breaks effects such as falling rain and waterfall splashes. | Caution | ## Motion vectors and DLSS diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 21c3f7b7..8329abb0 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -996,7 +996,8 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { // spawns, and a CPU copy that skipped pending ranges never let them reach the GPU: falling rain showed only its // splashes. Each pending range keeps a base, the words guest memory and the shadow last agreed on (the shadow's bytes // when the range became pending). The CPU copy takes the words where guest memory differs from the base; the fence -// copy back takes the words where the shadow differs from it. A word both changed takes the GPU's. +// copy back takes the words where the shadow differs from it. A word both changed takes the GPU's. TOTK_WB_MERGE=0 +// skips the pending ranges in the CPU copy and copies them back whole, as before. // TOTK_SHADOW=1/0 (shadow in totk.ini) forces the mode on or off; the default is on for Linux and Wine. typedef struct { uint64_t lo, hi; uint8_t *base; int refs; } SRange; static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock @@ -1068,6 +1069,16 @@ static void shadow_forget(PoolRec *r) { plat_mutex_unlock(&shadow_lock); } +// TOTK_WB_MERGE: the write-back merge of mirrors (wb_merge_record) and the word merge of the shadow's pending ranges. +static int merge_on = -1; +static int merge_wanted(void) { + if (merge_on < 0) { + merge_on = cfg_bool("TOTK_WB_MERGE"); + rt_log("mem: write-back merge %s (TOTK_WB_MERGE)", merge_on ? "on" : "off"); + } + return merge_on; +} + // A run of differing words from *at on, as byte offsets [*at, *end) of x and y (n bytes, the last word may be shorter); // 0 when none is left. static int diff_run(const uint8_t *x, const uint8_t *y, uint64_t n, uint64_t *at, uint64_t *end) { @@ -1311,7 +1322,7 @@ void vk_mem_gpu_pending(uint64_t addr, uint64_t size) { spend[i].refs++; } else { uint8_t *base = NULL; - if (r && r->smap && lo >= r->import_lo && hi <= r->import_hi && (base = malloc(size))) { + if (r && r->smap && lo >= r->import_lo && hi <= r->import_hi && merge_wanted() && (base = malloc(size))) { memcpy(base, r->smap + (lo - r->import_lo), size); for (int k = 0; k < nspend; k++) { const SRange *p = &spend[k]; @@ -1893,7 +1904,6 @@ static PlatMutex snap_pool_lock = PLAT_MUTEX_INIT; static SnapChunk snap_pool[SNAP_POOL_N]; static int nsnap_pool; static uint64_t snap_pool_bytes; -static int merge_on = -1; static struct { int tried, ok; PFN_vkCmdPushDescriptorSetKHR push; @@ -1936,11 +1946,7 @@ fail: } static int merging(void) { - if (merge_on < 0) { - merge_on = cfg_bool("TOTK_WB_MERGE"); - rt_log("mem: write-back merge %s (TOTK_WB_MERGE)", merge_on ? "on" : "off"); - } - if (!merge_on) return 0; + if (!merge_wanted()) return 0; if (!WM.tried) merge_init(vk_ctx()); return WM.ok; } From f8e0a7b37419e02565eb8291c2a0c963ec21cd2a Mon Sep 17 00:00:00 2001 From: ipavlin98 Date: Mon, 16 Nov 2026 04:01:22 +0300 Subject: [PATCH 3/4] The shadow's word merge skips equal words 64 bytes at a time --- runtime/vk_memory.c | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 8329abb0..8e6ebf05 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1083,6 +1083,7 @@ static int merge_wanted(void) { // 0 when none is left. static int diff_run(const uint8_t *x, const uint8_t *y, uint64_t n, uint64_t *at, uint64_t *end) { uint64_t i = *at; + while (i + 64 <= n && !memcmp(x + i, y + i, 64)) i += 64; // most words are equal: skip them a block at a time for (; i < n; i += 4) { uint32_t u = 0, v = 0, w = n - i < 4 ? (uint32_t)(n - i) : 4; memcpy(&u, x + i, w), memcpy(&v, y + i, w); From 547350167b1d9e4bb92136bb1644d0b5374e99df Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 16 Nov 2026 18:12:35 +0300 Subject: [PATCH 4/4] The shadow's word merge loads whole words and updates only the bases over the range diff_run read each word with a memcpy of variable size, a library call per word, and base_set walked every pending range (about a hundred) for each run of differing words. word_at loads 4 bytes directly and the bases over the merged range are found once per merge. Rain at night, shadow = 1 and sync_ahead = 1: the fence write-back took 11-12 ms per 60 frames instead of 20-22 (the plain copy with TOTK_WB_MERGE=0: 7-8 ms). No change in what is merged. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_memory.c | 64 ++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 8e6ebf05..1e73828e 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1079,45 +1079,71 @@ static int merge_wanted(void) { return merge_on; } +// The word at byte offset i of p's n bytes; a last word shorter than 4 bytes reads as zero-extended. The common case +// is a fixed 4-byte load: a memcpy of variable size is a library call per word. +static uint32_t word_at(const uint8_t *p, uint64_t i, uint64_t n) { + uint32_t v = 0; + if (n - i >= 4) memcpy(&v, p + i, 4); + else memcpy(&v, p + i, n - i); + return v; +} + // A run of differing words from *at on, as byte offsets [*at, *end) of x and y (n bytes, the last word may be shorter); // 0 when none is left. static int diff_run(const uint8_t *x, const uint8_t *y, uint64_t n, uint64_t *at, uint64_t *end) { uint64_t i = *at; while (i + 64 <= n && !memcmp(x + i, y + i, 64)) i += 64; // most words are equal: skip them a block at a time - for (; i < n; i += 4) { - uint32_t u = 0, v = 0, w = n - i < 4 ? (uint32_t)(n - i) : 4; - memcpy(&u, x + i, w), memcpy(&v, y + i, w); - if (u != v) break; - } + while (i < n && word_at(x, i, n) == word_at(y, i, n)) i += 4; if (i >= n) return 0; uint64_t j = i + 4; - for (; j < n; j += 4) { - uint32_t u = 0, v = 0, w = n - j < 4 ? (uint32_t)(n - j) : 4; - memcpy(&u, x + j, w), memcpy(&v, y + j, w); - if (u == v) break; - } + while (j < n && word_at(x, j, n) != word_at(y, j, n)) j += 4; *at = i, *end = j < n ? j : n; return 1; } -// shadow_lock held: src's bytes become guest [a, b) in the base of every pending range over them. -static void base_set(uint64_t a, uint64_t b, const uint8_t *src) { +// The pending ranges with a base over a guest range, found once per merge rather than once per differing run: the +// game's particle buffers differ in many short runs, and about a hundred ranges are pending at a time. n < 0: more than +// BASES_NEAR, every range is checked. +#define BASES_NEAR 16 +typedef struct { int n, at[BASES_NEAR]; } BaseSet; + +// shadow_lock held: the pending ranges with a base over guest [a, b). Valid while spend is unchanged. +static void bases_over(uint64_t a, uint64_t b, BaseSet *s) { + s->n = 0; for (int i = 0; i < nspend; i++) { - SRange *p = &spend[i]; - if (!p->base || p->lo >= b || p->hi <= a) continue; - uint64_t x = a > p->lo ? a : p->lo, y = b < p->hi ? b : p->hi; - memcpy(p->base + (x - p->lo), src + (x - a), y - x); + if (!spend[i].base || spend[i].lo >= b || spend[i].hi <= a) continue; + if (s->n == BASES_NEAR) { s->n = -1; return; } + s->at[s->n++] = i; } } +// shadow_lock held: src's bytes become guest [a, b) in p's base, where they overlap. +static void base_put(const SRange *p, uint64_t a, uint64_t b, const uint8_t *src) { + if (!p->base || p->lo >= b || p->hi <= a) return; + uint64_t x = a > p->lo ? a : p->lo, y = b < p->hi ? b : p->hi; + memcpy(p->base + (x - p->lo), src + (x - a), y - x); +} + +// shadow_lock held: src's bytes become guest [a, b) in the base of every pending range over them; s holds those +// ranges for a span that contains [a, b). +static void base_set(const BaseSet *s, uint64_t a, uint64_t b, const uint8_t *src) { + if (s->n < 0) { + for (int i = 0; i < nspend; i++) base_put(&spend[i], a, b, src); + return; + } + for (int k = 0; k < s->n; k++) base_put(&spend[s->at[k]], a, b, src); +} + static void shadow_put(PoolRec *r, uint64_t a, uint64_t b); // shadow_lock held: the words of guest [a, b), inside pending range p, that differ from p's base go into the shadow. static void shadow_merge(PoolRec *r, uint64_t a, uint64_t b, const SRange *p) { const uint8_t *g = (const uint8_t *)(uintptr_t)a; uint64_t x = 0, y; + BaseSet bs; + bases_over(a, b, &bs); while (diff_run(g, p->base + (a - p->lo), b - a, &x, &y)) { - base_set(a + x, a + y, g + x); + base_set(&bs, a + x, a + y, g + x); shadow_put(r, a + x, a + y); x = y; } @@ -1352,9 +1378,11 @@ void vk_gpu_written(uint64_t addr, uint64_t size) { const uint8_t *s = r->smap + (addr - r->import_lo); if (k < nspend && spend[k].base) { uint64_t x = 0, y; + BaseSet bs; + bases_over(addr, addr + size, &bs); while (diff_run(s, spend[k].base, size, &x, &y)) { memcpy((void *)(uintptr_t)(addr + x), s + x, y - x); - base_set(addr + x, addr + y, (const uint8_t *)(uintptr_t)(addr + x)); + base_set(&bs, addr + x, addr + y, (const uint8_t *)(uintptr_t)(addr + x)); s_wb_bytes += y - x; x = y; }