diff --git a/runtime/vk.h b/runtime/vk.h index 39cf6ccf..c3c1e8cb 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -291,6 +291,7 @@ int vk_mvprev_find(int index, uint64_t addr, uint64_t size, uint64_t *prev, int void vk_mvprev_frame(void); // each present, beside vk_dlaa_frame void vk_mem_mirror_begin(void); int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); +void vk_mem_mirror_seal(void); // before vk_mem_shadow_sync: the copies the next vk_mem_mirror_prologue takes int vk_mem_mirror_pending(void); void vk_mem_mirror_prologue(VkCommandBuffer cb); // Shadow storage (vk_memory.c): vk_sync.c calls vk_mem_shadow_sync right before each submit; the replay notes every GPU diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 3a5ad963..8704dc60 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1576,6 +1576,7 @@ static uint32_t mepoch; // replay: its write-watch snapshot epoch typedef struct { PoolRec *r; uint64_t off, size; } MRange; static MRange *pend, *wbl; // replay: copies for the next prologue; GPU writes into mirrors this submission static int npend, cpend, nwbl, cwbl; +static int pend_seal, snl_seal; // mirror_lock: the copies the next prologue takes (vk_mem_mirror_seal) static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions static int cregions; static PlatMutex flush_lock = PLAT_MUTEX_INIT; @@ -1640,8 +1641,9 @@ static void mirror_mode(int discrete) { "slower than mirror (the auto default on a discrete GPU); use memory = auto unless testing"); } -static void range_add(MRange **a, int *n, int *cap, PoolRec *r, uint64_t off, uint64_t size) { - if (*n && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) { +// keep: the entries before it are sealed and do not grow. +static void range_add(MRange **a, int *n, int *cap, int keep, PoolRec *r, uint64_t off, uint64_t size) { + if (*n > keep && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) { (*a)[*n - 1].size += size; return; } @@ -1762,7 +1764,8 @@ static void mirror_create(PoolRec *r) { mlist[at] = r; nm++; plat_mutex_unlock(&pool_lock); - range_add(&pend, &npend, &cpend, r, 0, n); + range_add(&pend, &npend, &cpend, pend_seal, r, 0, n); + if (r->smap) shadow_touch(r); // the copy reads the shadow: see vk_mem_mirror_seal __atomic_store_n(&r->acc_sub, msub, __ATOMIC_RELEASE); __atomic_store_n(&r->mstate, MS_ON, __ATOMIC_RELEASE); mrep.first_n++, mrep.first_bytes += n; @@ -1826,7 +1829,7 @@ static void mirror_sync(PoolRec *r) { if (!ch) continue; for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since) - range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT), mrep.page_bytes += 1ull << GEN_PAGE_SHIFT; + range_add(&pend, &npend, &cpend, pend_seal, r, x - lo, 1ull << GEN_PAGE_SHIFT), mrep.page_bytes += 1ull << GEN_PAGE_SHIFT; } r->mgen = gen_now; if (excl) plat_mutex_unlock(&gen_lock); @@ -1843,7 +1846,7 @@ static void mirror_sync(PoolRec *r) { for (int i = 0; i < n; i++) { uint64_t a = fl[i].lo > lo ? fl[i].lo : lo, b = fl[i].hi < hi ? fl[i].hi : hi; if (a >= b) continue; - range_add(&pend, &npend, &cpend, r, a - lo, b - a); + range_add(&pend, &npend, &cpend, pend_seal, r, a - lo, b - a); mrep.flush_n++, mrep.flush_bytes += b - a; if ((rt_trace & RT_TRACE_PERF) && b - a >= 16u << 20) rt_log("mem: mirror frame %llu: flush range %.1f MiB at +%.1f MiB of a %.1f MiB pool (%d ranges from %llu " @@ -1851,6 +1854,7 @@ static void mirror_sync(PoolRec *r) { (hi - lo) / 1048576.0, n, (unsigned long long)calls, sum / 1048576.0); } } + if (r->smap) shadow_touch(r); // the copies read the shadow: see vk_mem_mirror_seal mrep.syncs++; mrep.sync_t += qpc() - t0; __atomic_store_n(&r->acc_sub, msub, __ATOMIC_RELEASE); @@ -2029,12 +2033,12 @@ static int snap_add(PoolRec *r, uint64_t off, uint64_t size) { // Prologue: the snapshot copies of the ranges added since the last prologue. static void snap_copies(VkCommandBuffer cb) { - for (int i = snl_copied; i < nsnl; i++) { + for (int i = snl_copied; i < snl_seal; i++) { VkBufferCopy cp = {snl[i].off, snl[i].soff, snl[i].size}; vkCmdCopyBuffer(cb, snl[i].r->mbuf, chunks[snl[i].chunk].buf, 1, &cp); mrep.snap_bytes += snl[i].size; } - snl_copied = nsnl; + snl_copied = snl_seal; } static int snap_cmp(const void *x, const void *y) { @@ -2151,7 +2155,7 @@ static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { plat_mutex_unlock(&mirror_lock); return; } - range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size); + range_add(&wbl, &nwbl, &cwbl, 0, r, sp->offset, size); plat_mutex_unlock(&mirror_lock); } void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 0); } @@ -2252,7 +2256,7 @@ void vk_mem_mirror_begin(void) { return; } uint64_t t0 = qpc(); - nsnl = snl_copied = nchunks = 0; // the previous submission handed its chunks to snap_free + nsnl = snl_seal = snl_copied = nchunks = 0; // the previous submission handed its chunks to snap_free uint32_t prev = msub; msub = msub + 1 ? msub + 1 : 1; __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); @@ -2284,30 +2288,45 @@ void vk_mem_mirror_begin(void) { mrep.presnap_t += qpc() - t0; } +// With shadow storage a mirror copies from the shadow, which vk_mem_shadow_sync brings up to date only for the pools +// touched before it runs; mirror_sync and mirror_create touch theirs after queuing the copies. In a chunked replay the +// recording threads queue copies while the replay thread runs a part's prologue: a copy queued after that prologue's +// shadow sync read the shadow before the CPU's write reached it (with TOTK_SHADOW_DEFER the write is a GPU copy of +// the next prologue, so always), and the mirror, marked synced, kept the old bytes until the CPU wrote the page again. +// In the Depths, terrain loaded with those pages stood as stretched slabs over the ground for the rest of the session. +// So the prologue seals the queues before the shadow sync and copies only what was sealed; the rest waits for the next +// prologue, whose shadow sync comes first. +void vk_mem_mirror_seal(void) { + plat_mutex_lock(&mirror_lock); + pend_seal = npend, snl_seal = nsnl; + plat_mutex_unlock(&mirror_lock); +} + int vk_mem_mirror_pending(void) { plat_mutex_lock(&mirror_lock); - int n = npend || snl_copied < nsnl; + int n = pend_seal || snl_copied < snl_seal; plat_mutex_unlock(&mirror_lock); return n; } void vk_mem_mirror_prologue(VkCommandBuffer cb) { plat_mutex_lock(&mirror_lock); - if (!npend && snl_copied == nsnl) { + if (!pend_seal && snl_copied == snl_seal) { plat_mutex_unlock(&mirror_lock); return; } mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); - copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies); - if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote - if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + copy_ranges(cb, pend, pend_seal, 1, &mrep.upload_bytes, &mrep.upload_copies); + if (snl_copied < snl_seal) { // the snapshots read what the uploads above wrote + if (pend_seal) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); snap_copies(cb); } mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); - npend = 0; + memmove(pend, pend + pend_seal, sizeof *pend * (size_t)(npend - pend_seal)); + npend -= pend_seal, pend_seal = 0; mrep.prologues++; plat_mutex_unlock(&mirror_lock); } @@ -2410,7 +2429,7 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT); int m = wb_merge_record(cb); if (!m) { // no range list: copy them plainly - for (int i = 0; i < nsnl; i++) range_add(&wbl, &nwbl, &cwbl, snl[i].r, snl[i].off, snl[i].size); + for (int i = 0; i < nsnl; i++) range_add(&wbl, &nwbl, &cwbl, 0, snl[i].r, snl[i].off, snl[i].size); } rec += m + 1; mbarrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, @@ -3114,7 +3133,7 @@ void vk_mem_shutdown(void) { plat_mutex_lock(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); pool_index.n = 0; - nm = npend = nwbl = nsnl = snl_copied = ndq = 0; // mirrors went with their pools + nm = npend = pend_seal = nwbl = nsnl = snl_seal = snl_copied = ndq = 0; // mirrors went with their pools plat_mutex_lock(&snap_pool_lock); for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc); nsnap_pool = 0, snap_pool_bytes = 0; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index ce168fe4..f48e237f 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -369,7 +369,9 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { // (vk_sync_part, which the caller made the recorder idle for). static VkCommandBuffer prologue(VkCtx *c, uint64_t fence, int direct) { // Shadow storage: the CPU's writes go in before the part is submitted (TOTK_SYNC_AHEAD: queued for the GPU or - // made after the GPU finished the readers the game may think done, vk_memory.c). + // made after the GPU finished the readers the game may think done, vk_memory.c). The mirror copies queued before it + // are the ones this prologue makes, as they read the shadow (vk_mem_mirror_seal). + vk_mem_mirror_seal(); vk_mem_shadow_sync(fence); if (!vk_mem_mirror_pending() && !vk_mem_shadow_deferred()) return VK_NULL_HANDLE; int i = claim_slot(c, fence); // before reading slots: the claim may move it