From e2fdb5b226cf67fa094541177803cd313546ed2c Mon Sep 17 00:00:00 2001 From: Michael Anthony Bitoon Date: Mon, 2 Nov 2026 17:32:13 +0800 Subject: [PATCH] Merge a shader's storage-buffer writes back into guest memory word by word In mirror mode the replay copied every range a shader wrote from the VRAM mirror back to guest memory at the end of the submission. TotK's particle emitters write a new particle's whole record into a free slot of a buffer the simulation shaders update, so the copy put the mirror's stale words over fresh spawns: the waterfall splashes and mist showed about once a second and the falling rain drifted. Now each written range gets a snapshot in the prologue of the part that first writes it, after the CPU-to-mirror copies, and at the end of the submission shaders/wb_merge.comp writes only the 32-bit words that differ from the snapshot. Words the GPU left alone keep what the CPU wrote. About 0.1 MiB and three or four dispatches a frame in rain. TOTK_WB_MERGE=0 restores the copies. The merge's push descriptors go through rec_push so they keep their place in the recorder's call order. --- runtime/cfg.c | 6 +- runtime/shaders/wb_merge.comp | 27 ++++ runtime/vk.h | 3 + runtime/vk_exec_draw.c | 2 +- runtime/vk_memory.c | 242 +++++++++++++++++++++++++++++++++- runtime/wb_merge_spv.h | 72 ++++++++++ 6 files changed, 342 insertions(+), 10 deletions(-) create mode 100644 runtime/shaders/wb_merge.comp create mode 100644 runtime/wb_merge_spv.h diff --git a/runtime/cfg.c b/runtime/cfg.c index 53f2bb98..5292ea86 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -55,7 +55,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", }; @@ -96,7 +96,7 @@ static const struct { const char *key, *section; } sections[] = { {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_EXACT", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -166,7 +166,7 @@ static const CfgMeta metas[] = { {"VK_DYN_A2C", "", "0|1", RESTART}, {"VK_SUBGROUP_PIN", "1", "0|1", RESTART}, // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/shaders/wb_merge.comp b/runtime/shaders/wb_merge.comp new file mode 100644 index 00000000..493a0325 --- /dev/null +++ b/runtime/shaders/wb_merge.comp @@ -0,0 +1,27 @@ +#version 450 +// Write-back merge of a VRAM mirror (vk_memory.c, wb_merge_record): copies a 32-bit word of what the GPU left in the +// mirror into the pool's imported buffer (guest memory) only where it differs from the snapshot taken before the +// submission's GPU work, so words the GPU did not change keep what the CPU wrote meanwhile. +// One invocation per word of the listed ranges; ranges are sorted by their first invocation (x: word offset in the +// mirror and imported bindings, y: word offset in the snapshot binding, z: words, w: first invocation). +layout(local_size_x = 64) in; +layout(std430, set = 0, binding = 0) readonly buffer Post { uint post[]; }; +layout(std430, set = 0, binding = 1) readonly buffer Pre { uint pre[]; }; +layout(std430, set = 0, binding = 2) buffer Dst { uint dst[]; }; +layout(std430, set = 0, binding = 3) readonly buffer Ranges { uvec4 rng[]; }; +layout(push_constant) uniform Push { uint nranges, total; } pc; + +void main() { + uint i = gl_GlobalInvocationID.x; + if (i >= pc.total) return; + uint lo = 0, hi = pc.nranges; + while (hi - lo > 1) { + uint m = (lo + hi) >> 1; + if (rng[m].w <= i) lo = m; + else hi = m; + } + uvec4 r = rng[lo]; + uint k = i - r.w; + uint v = post[r.x + k]; + if (v != pre[r.y + k]) dst[r.x + k] = v; +} diff --git a/runtime/vk.h b/runtime/vk.h index 8119ca59..4029fe43 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -250,6 +250,9 @@ void vk_mem_pick(VkMemSpan *sp); #define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0) #define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0) void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size); +// A shader's storage-buffer write: goes back through the write-back merge (vk_memory.c), which keeps the words the GPU +// did not change. +void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size); // Uniform blocks as of the submit call (vk_usnap.c): vk_exec_record copies the uniform ranges a submission binds before // it records, ubo bindings take the copy (0: not copied, bind memory), GPU writes of the replay drop the copies they touch. void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 58f03905..bd494363 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -807,7 +807,7 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, if (sp.offset % align == 0) { VK_MEM_USE(sp); *out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n}; - if (written) vk_mem_gpu_wrote(&sp, n), vk_usnap_drop(addr, n); + if (written) vk_mem_gpu_wrote_shader(&sp, n), vk_usnap_drop(addr, n); count_use(sp.cls, use, n); VK_MEM_BOUND(sp); return 1; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index d4242a80..2e16c4e4 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -21,6 +21,7 @@ #include "nvn_pool_index.h" #include "vk.h" #include "vk_exec.h" +#include "wb_merge_spv.h" #include "platform/platform.h" // Timestamps for the tracking cost, in ns; qpc_ms converts. @@ -1164,6 +1165,7 @@ static PlatMutex flush_lock = PLAT_MUTEX_INIT; typedef struct { uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges; uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes; + uint64_t snap_bytes, merge_bytes, merge_dispatches; // write-back merge } MirrorAcc; static MirrorAcc mrep; // replay: this submission static MirrorAcc macc; // mirror_stat_lock: since the last summary @@ -1426,11 +1428,211 @@ int vk_mem_mirror_probe(uint64_t addr) { return ((PoolRec *)sp.pool)->mstate == MS_ON; } -void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { +// ---- write-back merge +// A storage buffer a shader writes whole records of does not +// go back to guest memory as a plain copy. The CPU may write fields of the same records while the GPU runs: TotK's +// particle emitters write a new particle's whole record into a free slot of a buffer that the simulation shaders update +// (the waterfall splashes and mist, 2026-11-02), and a plain copy puts the mirror's stale copy of those words back over +// them. Instead: +// - record time (gpu_wrote from vk_mem_gpu_wrote_shader): each distinct range of the submission gets room in a snapshot +// chunk (snap_add); +// - each part's prologue, after the CPU -> mirror copies: the mirror's words of the ranges the part added are copied +// into their snapshots (snap_copies), the state before the part's GPU work. Words of an earlier part's writes that a +// later range overlaps are covered by the earlier range's own snapshot; +// - vk_mem_mirror_end: shaders/wb_merge.comp writes each 32-bit word that differs between the mirror and the snapshot +// into the pool's imported buffer (wb_merge_record), one dispatch per pool and group of ranges. +// Words the GPU left unchanged keep the CPU's value; a word both changed takes the GPU's, as a plain copy would. +// TOTK_WB_MERGE=0 restores the plain copies; so does a pipeline or allocation failure. +#define SNAP_CHUNK (1u << 20) +#define MERGE_SPAN (64ull << 20) // the bytes of one pool a merge dispatch binds at most +#define MERGE_WORDS (65535u * 64u) // the words one merge dispatch covers at most +#define MERGE_MAX (4ull << 20) // larger written ranges stay plain copies +typedef struct { PoolRec *r; uint64_t off, size, soff; int chunk; } SnapRange; +typedef struct { VkBuffer buf; VmaAllocation alloc; uint64_t size, used; } SnapChunk; +typedef struct { int n; SnapChunk c[]; } SnapFree; // a submission's chunks and range lists, freed at its fence +static SnapRange *snl; // replay: this submission's merged ranges; [0, snl_copied) have their snapshot copy +static int nsnl, csnl, snl_copied; +static SnapChunk *chunks; // replay: this submission's snapshot chunks and merge range lists +static int nchunks, cchunks; +static int merge_on = -1; +static struct { + int tried, ok; + PFN_vkCmdPushDescriptorSetKHR push; + VkDescriptorSetLayout dsl; + VkPipelineLayout layout; + VkPipeline pipe; +} WM; + +static int merge_init(VkCtx *c) { + WM.tried = 1; + WM.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR"); + VkDescriptorSetLayoutBinding b[4]; + for (int i = 0; i < 4; i++) + b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}; + VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, + .bindingCount = 4, .pBindings = b}; + VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 8}; + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof wb_merge_spv, + .pCode = wb_merge_spv}; + VkShaderModule m = VK_NULL_HANDLE; + if (!WM.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &WM.dsl) != VK_SUCCESS) goto fail; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, + .pSetLayouts = &WM.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr}; + if (vkCreatePipelineLayout(c->device, &li, NULL, &WM.layout) != VK_SUCCESS || + vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) + goto fail; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = WM.layout}; + VkResult res = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &WM.pipe); + vkDestroyShaderModule(c->device, m, NULL); + if (res != VK_SUCCESS) goto fail; + return WM.ok = 1; +fail: + rt_log("mem: write-back merge off: its pipeline could not be created"); + return 0; +} + +static int merging(void) { + if (merge_on < 0) { + merge_on = cfg_bool("TOTK_WB_MERGE"); + rt_log("mem: write-back merge %s (TOTK_WB_MERGE)", merge_on ? "on" : "off"); + } + if (!merge_on) return 0; + if (!WM.tried) merge_init(vk_ctx()); + return WM.ok; +} + +// A chunk of at least size bytes in chunks[]; -1 when the allocation fails. +static int chunk_new(uint64_t size, int host) { + VkCtx *c = vk_ctx(); + if (nchunks == cchunks) grow((void **)&chunks, &cchunks, sizeof *chunks); + SnapChunk *ch = &chunks[nchunks]; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, + .flags = host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | + VMA_ALLOCATION_CREATE_MAPPED_BIT : 0}; + if (vmaCreateBuffer(c->vma, &bi, &ai, &ch->buf, &ch->alloc, NULL) != VK_SUCCESS) return -1; + ch->size = size, ch->used = 0; + return nchunks++; +} + +// Record time: room in a snapshot chunk for [off, off+size) of r's mirror, once per submission. 0: plain copy instead. +static int snap_add(PoolRec *r, uint64_t off, uint64_t size) { + for (int i = nsnl; i-- > 0;) + if (snl[i].r == r && snl[i].off == off && snl[i].size == size) return 1; + int k = -1; + for (int i = nchunks; i-- > 0;) // the newest snapshot chunk, if the range fits + if (chunks[i].used != UINT64_MAX && chunks[i].size - chunks[i].used >= size) { k = i; break; } + if (k < 0 && (k = chunk_new(size > SNAP_CHUNK ? size : SNAP_CHUNK, 0)) < 0) return 0; + if (nsnl == csnl) grow((void **)&snl, &csnl, sizeof *snl); + snl[nsnl++] = (SnapRange){r, off, size, chunks[k].used, k}; + chunks[k].used += (size + 15) & ~15ull; + return 1; +} + +// Prologue: the snapshot copies of the ranges added since the last prologue. +static void snap_copies(VkCommandBuffer cb) { + for (int i = snl_copied; i < nsnl; i++) { + VkBufferCopy cp = {snl[i].off, snl[i].soff, snl[i].size}; + vkCmdCopyBuffer(cb, snl[i].r->mbuf, chunks[snl[i].chunk].buf, 1, &cp); + mrep.snap_bytes += snl[i].size; + } + snl_copied = nsnl; +} + +static int snap_cmp(const void *x, const void *y) { + const SnapRange *a = x, *b = y; + if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1; + if (a->chunk != b->chunk) return a->chunk < b->chunk ? -1 : 1; + return a->off < b->off ? -1 : a->off > b->off; +} + +// One dispatch: ranges s[0..n) share a pool and a chunk and lie in [base, base + span) of the pool. +static void merge_dispatch(VkCommandBuffer cb, const SnapRange *s, int n, uint64_t base, uint64_t span, int list, + uint64_t list_off, uint32_t total) { + VkDescriptorBufferInfo bi[4] = {{s->r->mbuf, base, span}, {chunks[s->chunk].buf, 0, chunks[s->chunk].size}, + {s->r->buf, base, span}, {chunks[list].buf, list_off, (uint64_t)n * 16}}; + VkWriteDescriptorSet w[4]; + for (int i = 0; i < 4; i++) + w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i, + .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .pBufferInfo = &bi[i]}; + rec_push(WM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.layout, 0, 4, w); // in call order (vk_rec.h) + uint32_t pc[2] = {(uint32_t)n, total}; + vkCmdPushConstants(cb, WM.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); + vkCmdDispatch(cb, (total + 63) / 64, 1, 1); +} + +// vk_mem_mirror_end: the merge dispatches of this submission's ranges. Returns the commands recorded; 0 with nsnl +// left as it was when the range list cannot be allocated (the caller copies them plainly). +static int wb_merge_record(VkCommandBuffer cb) { + VkCtx *c = vk_ctx(); + uint64_t align = c->props.limits.minStorageBufferOffsetAlignment; + if (align < 16) align = 16; + SnapRange *s = malloc(sizeof *s * (size_t)nsnl); + if (!s) rt_fatal("vk: out of memory"); + memcpy(s, snl, sizeof *s * (size_t)nsnl); + qsort(s, (size_t)nsnl, sizeof *s, snap_cmp); + // the range lists: at most one 16-byte entry per range, each dispatch's list at an aligned offset + int list = chunk_new((uint64_t)nsnl * 16 + (uint64_t)nsnl * align, 1); + if (list < 0) { free(s); return 0; } + chunks[list].used = UINT64_MAX; // not a snapshot chunk + VmaAllocationInfo li; + vmaGetAllocationInfo(c->vma, chunks[list].alloc, &li); + uint32_t *lm = li.pMappedData; + uint64_t lo = 0; + int rec = 0; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.pipe); + for (int i = 0; i < nsnl;) { + uint64_t base = s[i].off & ~(align - 1), end = s[i].off + s[i].size; + uint32_t total = 0; + int j = i; + lo = (lo + align - 1) & ~(align - 1); + uint64_t l0 = lo; + for (; j < nsnl && s[j].r == s[i].r && s[j].chunk == s[i].chunk; j++) { + uint64_t e = s[j].off + s[j].size > end ? s[j].off + s[j].size : end; + if (j > i && (e - base > MERGE_SPAN || total + s[j].size / 4 > MERGE_WORDS)) break; + end = e; + uint32_t *q = lm + lo / 4; + q[0] = (uint32_t)((s[j].off - base) / 4), q[1] = (uint32_t)(s[j].soff / 4), q[2] = (uint32_t)(s[j].size / 4); + q[3] = total; + total += (uint32_t)(s[j].size / 4); + lo += 16; + mrep.merge_bytes += s[j].size; + } + merge_dispatch(cb, s + i, j - i, base, end - base, list, l0, total); + mrep.merge_dispatches++; + rec += 3; + i = j; + } + vmaFlushAllocation(c->vma, chunks[list].alloc, 0, VK_WHOLE_SIZE); + for (int i = 0; i < nsnl; i++) snl[i].r->wb_sub = msub; + free(s); + return rec; +} + +static void snap_free(void *arg) { // completion thread: the submission is done with its chunks + SnapFree *f = arg; + for (int i = 0; i < f->n; i++) vmaDestroyBuffer(vk_ctx()->vma, f->c[i].buf, f->c[i].alloc); + free(f); +} + +static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; + // A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies. + if (shader && !(sp->offset & 3) && !(size & 3) && size <= MERGE_MAX && merging() && snap_add(r, sp->offset, size)) + return; range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size); } +void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 0); } +void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 1); } void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { if (!vk_mirror_on || !size || !p->mem) return; @@ -1474,6 +1676,7 @@ void vk_mem_mirror_begin(void) { return; } uint64_t t0 = qpc(); + nsnl = snl_copied = nchunks = 0; // the previous submission handed its chunks to snap_free uint32_t prev = msub; msub = msub + 1 ? msub + 1 : 1; __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); @@ -1505,13 +1708,18 @@ void vk_mem_mirror_begin(void) { mrep.presnap_t += qpc() - t0; } -int vk_mem_mirror_pending(void) { return npend; } +int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; } void vk_mem_mirror_prologue(VkCommandBuffer cb) { - if (!npend) return; + if (!npend && snl_copied == nsnl) return; mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies); + if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote + if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT); + snap_copies(cb); + } mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); npend = 0; @@ -1610,6 +1818,26 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) { int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { if (!vk_mirror_on) return 0; int rec = 0; + if (nsnl) { + // the merge reads the mirror the submission's shaders wrote and writes the imported buffers + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, + VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT); + int m = wb_merge_record(cb); + if (!m) { // no range list: copy them plainly + for (int i = 0; i < nsnl; i++) range_add(&wbl, &nwbl, &cwbl, snl[i].r, snl[i].off, snl[i].size); + } + rec += m + 1; + mbarrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, + VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + } + if (nchunks) { // the last part's prologue still records snapshot copies into them; freed at the fence + SnapFree *f = malloc(sizeof *f + sizeof(SnapChunk) * (size_t)nchunks); + if (!f) rt_fatal("vk: out of memory"); + f->n = nchunks; + memcpy(f->c, chunks, sizeof(SnapChunk) * (size_t)nchunks); + vk_completion_add(fence, snap_free, f); + } if (nwbl) { mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); @@ -1664,12 +1892,14 @@ static void mirror_log(uint64_t frame) { (long long)plat_xchg64(&v_mismatch, 0)); rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked " "%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, " - "first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries " + "first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, merge %.3f MiB in %.1f dispatches " + "(snapshots %.3f MiB), flush calls %.1f (%.3f MiB), queries " "%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f)%s", (unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET], m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP], (long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f, - a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M, + a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, a.merge_bytes / f / M, + a.merge_dispatches / f, a.snap_bytes / f / M, fc / f, fb / f / M, a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f, qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver); } @@ -2033,7 +2263,7 @@ void vk_mem_shutdown(void) { plat_mutex_lock(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); pool_index.n = 0; - nm = npend = nwbl = 0; // mirrors went with their pools + nm = npend = nwbl = nsnl = snl_copied = 0; // mirrors went with their pools plat_mutex_lock(&shadow_lock); nstouch = nspend = 0; // shadows too s_live_n = s_live_bytes = 0; diff --git a/runtime/wb_merge_spv.h b/runtime/wb_merge_spv.h new file mode 100644 index 00000000..ca47057e --- /dev/null +++ b/runtime/wb_merge_spv.h @@ -0,0 +1,72 @@ +// Generated by tools/build_shaders.py from runtime/shaders/wb_merge.comp; do not edit. +// source sha256 a2d55900390e1cca3b742068d5ee21c4cb13a7eb5cb1d5a687f1001e001316b0 +#pragma once +#include +static const uint32_t wb_merge_spv[] = { + 0x07230203, 0x00010300, 0x000d000b, 0x00000083, 0x00000000, 0x00020011, 0x00000001, 0x0006000b, + 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001, + 0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000b, 0x00060010, 0x00000004, + 0x00000011, 0x00000040, 0x00000001, 0x00000001, 0x00040047, 0x0000000b, 0x0000000b, 0x0000001c, + 0x00030047, 0x00000011, 0x00000002, 0x00050048, 0x00000011, 0x00000000, 0x00000023, 0x00000000, + 0x00050048, 0x00000011, 0x00000001, 0x00000023, 0x00000004, 0x00040047, 0x00000033, 0x00000006, + 0x00000010, 0x00030047, 0x00000034, 0x00000002, 0x00040048, 0x00000034, 0x00000000, 0x00000018, + 0x00050048, 0x00000034, 0x00000000, 0x00000023, 0x00000000, 0x00030047, 0x00000036, 0x00000018, + 0x00040047, 0x00000036, 0x00000021, 0x00000003, 0x00040047, 0x00000036, 0x00000022, 0x00000000, + 0x00040047, 0x0000004f, 0x00000006, 0x00000004, 0x00030047, 0x00000050, 0x00000002, 0x00040048, + 0x00000050, 0x00000000, 0x00000018, 0x00050048, 0x00000050, 0x00000000, 0x00000023, 0x00000000, + 0x00030047, 0x00000052, 0x00000018, 0x00040047, 0x00000052, 0x00000021, 0x00000000, 0x00040047, + 0x00000052, 0x00000022, 0x00000000, 0x00040047, 0x0000005a, 0x00000006, 0x00000004, 0x00030047, + 0x0000005b, 0x00000002, 0x00040048, 0x0000005b, 0x00000000, 0x00000018, 0x00050048, 0x0000005b, + 0x00000000, 0x00000023, 0x00000000, 0x00030047, 0x0000005d, 0x00000018, 0x00040047, 0x0000005d, + 0x00000021, 0x00000001, 0x00040047, 0x0000005d, 0x00000022, 0x00000000, 0x00040047, 0x00000067, + 0x00000006, 0x00000004, 0x00030047, 0x00000068, 0x00000002, 0x00050048, 0x00000068, 0x00000000, + 0x00000023, 0x00000000, 0x00040047, 0x0000006a, 0x00000021, 0x00000002, 0x00040047, 0x0000006a, + 0x00000022, 0x00000000, 0x00040047, 0x00000072, 0x0000000b, 0x00000019, 0x00020013, 0x00000002, + 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006, 0x00000020, 0x00000000, 0x00040017, + 0x00000009, 0x00000006, 0x00000003, 0x00040020, 0x0000000a, 0x00000001, 0x00000009, 0x0004003b, + 0x0000000a, 0x0000000b, 0x00000001, 0x0004002b, 0x00000006, 0x0000000c, 0x00000000, 0x00040020, + 0x0000000d, 0x00000001, 0x00000006, 0x0004001e, 0x00000011, 0x00000006, 0x00000006, 0x00040020, + 0x00000012, 0x00000009, 0x00000011, 0x0004003b, 0x00000012, 0x00000013, 0x00000009, 0x00040015, + 0x00000014, 0x00000020, 0x00000001, 0x0004002b, 0x00000014, 0x00000015, 0x00000001, 0x00040020, + 0x00000016, 0x00000009, 0x00000006, 0x00020014, 0x00000019, 0x0004002b, 0x00000014, 0x00000020, + 0x00000000, 0x0004002b, 0x00000006, 0x0000002b, 0x00000001, 0x00040017, 0x00000032, 0x00000006, + 0x00000004, 0x0003001d, 0x00000033, 0x00000032, 0x0003001e, 0x00000034, 0x00000033, 0x00040020, + 0x00000035, 0x0000000c, 0x00000034, 0x0004003b, 0x00000035, 0x00000036, 0x0000000c, 0x0004002b, + 0x00000006, 0x00000038, 0x00000003, 0x00040020, 0x00000039, 0x0000000c, 0x00000006, 0x00040020, + 0x00000046, 0x0000000c, 0x00000032, 0x0003001d, 0x0000004f, 0x00000006, 0x0003001e, 0x00000050, + 0x0000004f, 0x00040020, 0x00000051, 0x0000000c, 0x00000050, 0x0004003b, 0x00000051, 0x00000052, + 0x0000000c, 0x0003001d, 0x0000005a, 0x00000006, 0x0003001e, 0x0000005b, 0x0000005a, 0x00040020, + 0x0000005c, 0x0000000c, 0x0000005b, 0x0004003b, 0x0000005c, 0x0000005d, 0x0000000c, 0x0003001d, + 0x00000067, 0x00000006, 0x0003001e, 0x00000068, 0x00000067, 0x00040020, 0x00000069, 0x0000000c, + 0x00000068, 0x0004003b, 0x00000069, 0x0000006a, 0x0000000c, 0x0004002b, 0x00000006, 0x00000071, + 0x00000040, 0x0006002c, 0x00000009, 0x00000072, 0x00000071, 0x0000002b, 0x0000002b, 0x00050036, + 0x00000002, 0x00000004, 0x00000000, 0x00000003, 0x000200f8, 0x00000005, 0x000300f7, 0x00000073, + 0x00000000, 0x000300fb, 0x0000000c, 0x00000074, 0x000200f8, 0x00000074, 0x00050041, 0x0000000d, + 0x0000000e, 0x0000000b, 0x0000000c, 0x0004003d, 0x00000006, 0x0000000f, 0x0000000e, 0x00050041, + 0x00000016, 0x00000017, 0x00000013, 0x00000015, 0x0004003d, 0x00000006, 0x00000018, 0x00000017, + 0x000500ae, 0x00000019, 0x0000001a, 0x0000000f, 0x00000018, 0x000300f7, 0x0000001c, 0x00000000, + 0x000400fa, 0x0000001a, 0x0000001b, 0x0000001c, 0x000200f8, 0x0000001b, 0x000200f9, 0x00000073, + 0x000200f8, 0x0000001c, 0x00050041, 0x00000016, 0x00000021, 0x00000013, 0x00000020, 0x0004003d, + 0x00000006, 0x00000022, 0x00000021, 0x000200f9, 0x00000023, 0x000200f8, 0x00000023, 0x000700f5, + 0x00000006, 0x0000007e, 0x0000000c, 0x0000001c, 0x00000081, 0x00000024, 0x000700f5, 0x00000006, + 0x0000007d, 0x00000022, 0x0000001c, 0x00000082, 0x00000024, 0x00050082, 0x00000006, 0x0000002a, + 0x0000007d, 0x0000007e, 0x000500ac, 0x00000019, 0x0000002c, 0x0000002a, 0x0000002b, 0x000400f6, + 0x00000025, 0x00000024, 0x00000000, 0x000400fa, 0x0000002c, 0x00000024, 0x00000025, 0x000200f8, + 0x00000024, 0x00050080, 0x00000006, 0x00000030, 0x0000007e, 0x0000007d, 0x000500c2, 0x00000006, + 0x00000031, 0x00000030, 0x00000015, 0x00070041, 0x00000039, 0x0000003a, 0x00000036, 0x00000020, + 0x00000031, 0x00000038, 0x0004003d, 0x00000006, 0x0000003b, 0x0000003a, 0x000500b2, 0x00000019, + 0x0000003d, 0x0000003b, 0x0000000f, 0x000600a9, 0x00000006, 0x00000081, 0x0000003d, 0x00000031, + 0x0000007e, 0x000600a9, 0x00000006, 0x00000082, 0x0000003d, 0x0000007d, 0x00000031, 0x000200f9, + 0x00000023, 0x000200f8, 0x00000025, 0x00060041, 0x00000046, 0x00000047, 0x00000036, 0x00000020, + 0x0000007e, 0x0004003d, 0x00000032, 0x00000048, 0x00000047, 0x00050051, 0x00000006, 0x0000004c, + 0x00000048, 0x00000003, 0x00050082, 0x00000006, 0x0000004d, 0x0000000f, 0x0000004c, 0x00050051, + 0x00000006, 0x00000054, 0x00000048, 0x00000000, 0x00050080, 0x00000006, 0x00000056, 0x00000054, + 0x0000004d, 0x00060041, 0x00000039, 0x00000057, 0x00000052, 0x00000020, 0x00000056, 0x0004003d, + 0x00000006, 0x00000058, 0x00000057, 0x00050051, 0x00000006, 0x0000005f, 0x00000048, 0x00000001, + 0x00050080, 0x00000006, 0x00000061, 0x0000005f, 0x0000004d, 0x00060041, 0x00000039, 0x00000062, + 0x0000005d, 0x00000020, 0x00000061, 0x0004003d, 0x00000006, 0x00000063, 0x00000062, 0x000500ab, + 0x00000019, 0x00000064, 0x00000058, 0x00000063, 0x000300f7, 0x00000066, 0x00000000, 0x000400fa, + 0x00000064, 0x00000065, 0x00000066, 0x000200f8, 0x00000065, 0x00060041, 0x00000039, 0x00000070, + 0x0000006a, 0x00000020, 0x00000056, 0x0003003e, 0x00000070, 0x00000058, 0x000200f9, 0x00000066, + 0x000200f8, 0x00000066, 0x000200f9, 0x00000073, 0x000200f8, 0x00000073, 0x000100fd, 0x00010038, +};