A mirror's copies from the shadow wait for the shadow sync: the prologue seals the mirror queues before it syncs the shadow and copies only those, and a mirror sync touches its shadow pool, so with replay_workers and shadow_defer a recording thread no longer queues a copy that reads the shadow before the CPU's write reached it, which left terrain in the Depths as stretched slabs over the ground for the rest of the session

This commit is contained in:
ipavlin98 committed 2026-11-18 02:03:53 +03:00
1 parent 6b0974f08d
commit 0f2cc1071a
3 files changed
+40 -18

No files matched your search

+1
View File
@@ -291,6 +291,7 @@ int vk_mvprev_find(int index, uint64_t addr, uint64_t size, uint64_t *prev, int
void vk_mvprev_frame(void); // each present, beside vk_dlaa_frame
void vk_mem_mirror_begin(void);
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence);
void vk_mem_mirror_seal(void); // before vk_mem_shadow_sync: the copies the next vk_mem_mirror_prologue takes
int vk_mem_mirror_pending(void);
void vk_mem_mirror_prologue(VkCommandBuffer cb);
// Shadow storage (vk_memory.c): vk_sync.c calls vk_mem_shadow_sync right before each submit; the replay notes every GPU
+36 -17
View File
@@ -1576,6 +1576,7 @@ static uint32_t mepoch; // replay: its write-watch snapshot epoch
typedef struct { PoolRec *r; uint64_t off, size; } MRange;
static MRange *pend, *wbl; // replay: copies for the next prologue; GPU writes into mirrors this submission
static int npend, cpend, nwbl, cwbl;
static int pend_seal, snl_seal; // mirror_lock: the copies the next prologue takes (vk_mem_mirror_seal)
static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions
static int cregions;
static PlatMutex flush_lock = PLAT_MUTEX_INIT;
@@ -1640,8 +1641,9 @@ static void mirror_mode(int discrete) {
"slower than mirror (the auto default on a discrete GPU); use memory = auto unless testing");
}
static void range_add(MRange **a, int *n, int *cap, PoolRec *r, uint64_t off, uint64_t size) {
if (*n && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) {
// keep: the entries before it are sealed and do not grow.
static void range_add(MRange **a, int *n, int *cap, int keep, PoolRec *r, uint64_t off, uint64_t size) {
if (*n > keep && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) {
(*a)[*n - 1].size += size;
return;
}
@@ -1762,7 +1764,8 @@ static void mirror_create(PoolRec *r) {
mlist[at] = r;
nm++;
plat_mutex_unlock(&pool_lock);
range_add(&pend, &npend, &cpend, r, 0, n);
range_add(&pend, &npend, &cpend, pend_seal, r, 0, n);
if (r->smap) shadow_touch(r); // the copy reads the shadow: see vk_mem_mirror_seal
__atomic_store_n(&r->acc_sub, msub, __ATOMIC_RELEASE);
__atomic_store_n(&r->mstate, MS_ON, __ATOMIC_RELEASE);
mrep.first_n++, mrep.first_bytes += n;
@@ -1826,7 +1829,7 @@ static void mirror_sync(PoolRec *r) {
if (!ch) continue;
for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT)
if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since)
range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT), mrep.page_bytes += 1ull << GEN_PAGE_SHIFT;
range_add(&pend, &npend, &cpend, pend_seal, r, x - lo, 1ull << GEN_PAGE_SHIFT), mrep.page_bytes += 1ull << GEN_PAGE_SHIFT;
}
r->mgen = gen_now;
if (excl) plat_mutex_unlock(&gen_lock);
@@ -1843,7 +1846,7 @@ static void mirror_sync(PoolRec *r) {
for (int i = 0; i < n; i++) {
uint64_t a = fl[i].lo > lo ? fl[i].lo : lo, b = fl[i].hi < hi ? fl[i].hi : hi;
if (a >= b) continue;
range_add(&pend, &npend, &cpend, r, a - lo, b - a);
range_add(&pend, &npend, &cpend, pend_seal, r, a - lo, b - a);
mrep.flush_n++, mrep.flush_bytes += b - a;
if ((rt_trace & RT_TRACE_PERF) && b - a >= 16u << 20)
rt_log("mem: mirror frame %llu: flush range %.1f MiB at +%.1f MiB of a %.1f MiB pool (%d ranges from %llu "
@@ -1851,6 +1854,7 @@ static void mirror_sync(PoolRec *r) {
(hi - lo) / 1048576.0, n, (unsigned long long)calls, sum / 1048576.0);
}
}
if (r->smap) shadow_touch(r); // the copies read the shadow: see vk_mem_mirror_seal
mrep.syncs++;
mrep.sync_t += qpc() - t0;
__atomic_store_n(&r->acc_sub, msub, __ATOMIC_RELEASE);
@@ -2029,12 +2033,12 @@ static int snap_add(PoolRec *r, uint64_t off, uint64_t size) {
// Prologue: the snapshot copies of the ranges added since the last prologue.
static void snap_copies(VkCommandBuffer cb) {
for (int i = snl_copied; i < nsnl; i++) {
for (int i = snl_copied; i < snl_seal; i++) {
VkBufferCopy cp = {snl[i].off, snl[i].soff, snl[i].size};
vkCmdCopyBuffer(cb, snl[i].r->mbuf, chunks[snl[i].chunk].buf, 1, &cp);
mrep.snap_bytes += snl[i].size;
}
snl_copied = nsnl;
snl_copied = snl_seal;
}
static int snap_cmp(const void *x, const void *y) {
@@ -2151,7 +2155,7 @@ static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) {
plat_mutex_unlock(&mirror_lock);
return;
}
range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size);
range_add(&wbl, &nwbl, &cwbl, 0, r, sp->offset, size);
plat_mutex_unlock(&mirror_lock);
}
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 0); }
@@ -2252,7 +2256,7 @@ void vk_mem_mirror_begin(void) {
return;
}
uint64_t t0 = qpc();
nsnl = snl_copied = nchunks = 0; // the previous submission handed its chunks to snap_free
nsnl = snl_seal = snl_copied = nchunks = 0; // the previous submission handed its chunks to snap_free
uint32_t prev = msub;
msub = msub + 1 ? msub + 1 : 1;
__atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE);
@@ -2284,30 +2288,45 @@ void vk_mem_mirror_begin(void) {
mrep.presnap_t += qpc() - t0;
}
// With shadow storage a mirror copies from the shadow, which vk_mem_shadow_sync brings up to date only for the pools
// touched before it runs; mirror_sync and mirror_create touch theirs after queuing the copies. In a chunked replay the
// recording threads queue copies while the replay thread runs a part's prologue: a copy queued after that prologue's
// shadow sync read the shadow before the CPU's write reached it (with TOTK_SHADOW_DEFER the write is a GPU copy of
// the next prologue, so always), and the mirror, marked synced, kept the old bytes until the CPU wrote the page again.
// In the Depths, terrain loaded with those pages stood as stretched slabs over the ground for the rest of the session.
// So the prologue seals the queues before the shadow sync and copies only what was sealed; the rest waits for the next
// prologue, whose shadow sync comes first.
void vk_mem_mirror_seal(void) {
plat_mutex_lock(&mirror_lock);
pend_seal = npend, snl_seal = nsnl;
plat_mutex_unlock(&mirror_lock);
}
int vk_mem_mirror_pending(void) {
plat_mutex_lock(&mirror_lock);
int n = npend || snl_copied < nsnl;
int n = pend_seal || snl_copied < snl_seal;
plat_mutex_unlock(&mirror_lock);
return n;
}
void vk_mem_mirror_prologue(VkCommandBuffer cb) {
plat_mutex_lock(&mirror_lock);
if (!npend && snl_copied == nsnl) {
if (!pend_seal && snl_copied == snl_seal) {
plat_mutex_unlock(&mirror_lock);
return;
}
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies);
if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote
if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
copy_ranges(cb, pend, pend_seal, 1, &mrep.upload_bytes, &mrep.upload_copies);
if (snl_copied < snl_seal) { // the snapshots read what the uploads above wrote
if (pend_seal) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT);
snap_copies(cb);
}
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
npend = 0;
memmove(pend, pend + pend_seal, sizeof *pend * (size_t)(npend - pend_seal));
npend -= pend_seal, pend_seal = 0;
mrep.prologues++;
plat_mutex_unlock(&mirror_lock);
}
@@ -2410,7 +2429,7 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
int m = wb_merge_record(cb);
if (!m) { // no range list: copy them plainly
for (int i = 0; i < nsnl; i++) range_add(&wbl, &nwbl, &cwbl, snl[i].r, snl[i].off, snl[i].size);
for (int i = 0; i < nsnl; i++) range_add(&wbl, &nwbl, &cwbl, 0, snl[i].r, snl[i].off, snl[i].size);
}
rec += m + 1;
mbarrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
@@ -3114,7 +3133,7 @@ void vk_mem_shutdown(void) {
plat_mutex_lock(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
pool_index.n = 0;
nm = npend = nwbl = nsnl = snl_copied = ndq = 0; // mirrors went with their pools
nm = npend = pend_seal = nwbl = nsnl = snl_seal = snl_copied = ndq = 0; // mirrors went with their pools
plat_mutex_lock(&snap_pool_lock);
for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc);
nsnap_pool = 0, snap_pool_bytes = 0;
+3 -1
View File
@@ -369,7 +369,9 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
// (vk_sync_part, which the caller made the recorder idle for).
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence, int direct) {
// Shadow storage: the CPU's writes go in before the part is submitted (TOTK_SYNC_AHEAD: queued for the GPU or
// made after the GPU finished the readers the game may think done, vk_memory.c).
// made after the GPU finished the readers the game may think done, vk_memory.c). The mirror copies queued before it
// are the ones this prologue makes, as they read the shadow (vk_mem_mirror_seal).
vk_mem_mirror_seal();
vk_mem_shadow_sync(fence);
if (!vk_mem_mirror_pending() && !vk_mem_shadow_deferred()) return VK_NULL_HANDLE;
int i = claim_slot(c, fence); // before reading slots: the claim may move it