Merge branch 'pr/528' into task/wbmerge

This commit is contained in:
Valiantsin Dzerakh committed 2026-11-03 15:56:15 +03:00
commit 1cd3e3a584
6 files changed
+342 -10

No files matched your search

+3 -3
View File
@@ -55,7 +55,7 @@ static const char *const known[] = {
"SWKBD_ANSWER",
"SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS",
"ZSTD_CHECK", "ZSTD_NATIVE",
};
@@ -96,7 +96,7 @@ static const struct { const char *key, *section; } sections[] = {
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_EXACT", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"},
};
// The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a
@@ -166,7 +166,7 @@ static const CfgMeta metas[] = {
{"VK_DYN_A2C", "", "0|1", RESTART}, {"VK_SUBGROUP_PIN", "1", "0|1", RESTART},
// diagnostics
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_EXACT", "1", "0|1", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART},
{"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only
{"INPUT_RECORD", "", NULL, RESTART},
};
+27
View File
@@ -0,0 +1,27 @@
#version 450
// Write-back merge of a VRAM mirror (vk_memory.c, wb_merge_record): copies a 32-bit word of what the GPU left in the
// mirror into the pool's imported buffer (guest memory) only where it differs from the snapshot taken before the
// submission's GPU work, so words the GPU did not change keep what the CPU wrote meanwhile.
// One invocation per word of the listed ranges; ranges are sorted by their first invocation (x: word offset in the
// mirror and imported bindings, y: word offset in the snapshot binding, z: words, w: first invocation).
layout(local_size_x = 64) in;
layout(std430, set = 0, binding = 0) readonly buffer Post { uint post[]; };
layout(std430, set = 0, binding = 1) readonly buffer Pre { uint pre[]; };
layout(std430, set = 0, binding = 2) buffer Dst { uint dst[]; };
layout(std430, set = 0, binding = 3) readonly buffer Ranges { uvec4 rng[]; };
layout(push_constant) uniform Push { uint nranges, total; } pc;
void main() {
uint i = gl_GlobalInvocationID.x;
if (i >= pc.total) return;
uint lo = 0, hi = pc.nranges;
while (hi - lo > 1) {
uint m = (lo + hi) >> 1;
if (rng[m].w <= i) lo = m;
else hi = m;
}
uvec4 r = rng[lo];
uint k = i - r.w;
uint v = post[r.x + k];
if (v != pre[r.y + k]) dst[r.x + k] = v;
}
+3
View File
@@ -250,6 +250,9 @@ void vk_mem_pick(VkMemSpan *sp);
#define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0)
#define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0)
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size);
// A shader's storage-buffer write: goes back through the write-back merge (vk_memory.c), which keeps the words the GPU
// did not change.
void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size);
// Uniform blocks as of the submit call (vk_usnap.c): vk_exec_record copies the uniform ranges a submission binds before
// it records, ubo bindings take the copy (0: not copied, bind memory), GPU writes of the replay drop the copies they touch.
void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence);
+1 -1
View File
@@ -807,7 +807,7 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align,
if (sp.offset % align == 0) {
VK_MEM_USE(sp);
*out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n};
if (written) vk_mem_gpu_wrote(&sp, n), vk_usnap_drop(addr, n);
if (written) vk_mem_gpu_wrote_shader(&sp, n), vk_usnap_drop(addr, n);
count_use(sp.cls, use, n);
VK_MEM_BOUND(sp);
return 1;
+236 -6
View File
@@ -21,6 +21,7 @@
#include "nvn_pool_index.h"
#include "vk.h"
#include "vk_exec.h"
#include "wb_merge_spv.h"
#include "platform/platform.h"
// Timestamps for the tracking cost, in ns; qpc_ms converts.
@@ -1164,6 +1165,7 @@ static PlatMutex flush_lock = PLAT_MUTEX_INIT;
typedef struct {
uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges;
uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes;
uint64_t snap_bytes, merge_bytes, merge_dispatches; // write-back merge
} MirrorAcc;
static MirrorAcc mrep; // replay: this submission
static MirrorAcc macc; // mirror_stat_lock: since the last summary
@@ -1431,11 +1433,211 @@ int vk_mem_mirror_probe(uint64_t addr) {
return ((PoolRec *)sp.pool)->mstate == MS_ON;
}
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) {
// ---- write-back merge
// A storage buffer a shader writes whole records of does not
// go back to guest memory as a plain copy. The CPU may write fields of the same records while the GPU runs: TotK's
// particle emitters write a new particle's whole record into a free slot of a buffer that the simulation shaders update
// (the waterfall splashes and mist, 2026-11-02), and a plain copy puts the mirror's stale copy of those words back over
// them. Instead:
// - record time (gpu_wrote from vk_mem_gpu_wrote_shader): each distinct range of the submission gets room in a snapshot
// chunk (snap_add);
// - each part's prologue, after the CPU -> mirror copies: the mirror's words of the ranges the part added are copied
// into their snapshots (snap_copies), the state before the part's GPU work. Words of an earlier part's writes that a
// later range overlaps are covered by the earlier range's own snapshot;
// - vk_mem_mirror_end: shaders/wb_merge.comp writes each 32-bit word that differs between the mirror and the snapshot
// into the pool's imported buffer (wb_merge_record), one dispatch per pool and group of ranges.
// Words the GPU left unchanged keep the CPU's value; a word both changed takes the GPU's, as a plain copy would.
// TOTK_WB_MERGE=0 restores the plain copies; so does a pipeline or allocation failure.
#define SNAP_CHUNK (1u << 20)
#define MERGE_SPAN (64ull << 20) // the bytes of one pool a merge dispatch binds at most
#define MERGE_WORDS (65535u * 64u) // the words one merge dispatch covers at most
#define MERGE_MAX (4ull << 20) // larger written ranges stay plain copies
typedef struct { PoolRec *r; uint64_t off, size, soff; int chunk; } SnapRange;
typedef struct { VkBuffer buf; VmaAllocation alloc; uint64_t size, used; } SnapChunk;
typedef struct { int n; SnapChunk c[]; } SnapFree; // a submission's chunks and range lists, freed at its fence
static SnapRange *snl; // replay: this submission's merged ranges; [0, snl_copied) have their snapshot copy
static int nsnl, csnl, snl_copied;
static SnapChunk *chunks; // replay: this submission's snapshot chunks and merge range lists
static int nchunks, cchunks;
static int merge_on = -1;
static struct {
int tried, ok;
PFN_vkCmdPushDescriptorSetKHR push;
VkDescriptorSetLayout dsl;
VkPipelineLayout layout;
VkPipeline pipe;
} WM;
static int merge_init(VkCtx *c) {
WM.tried = 1;
WM.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR");
VkDescriptorSetLayoutBinding b[4];
for (int i = 0; i < 4; i++)
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
.bindingCount = 4, .pBindings = b};
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 8};
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof wb_merge_spv,
.pCode = wb_merge_spv};
VkShaderModule m = VK_NULL_HANDLE;
if (!WM.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &WM.dsl) != VK_SUCCESS) goto fail;
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
.pSetLayouts = &WM.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
if (vkCreatePipelineLayout(c->device, &li, NULL, &WM.layout) != VK_SUCCESS ||
vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS)
goto fail;
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
.layout = WM.layout};
VkResult res = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &WM.pipe);
vkDestroyShaderModule(c->device, m, NULL);
if (res != VK_SUCCESS) goto fail;
return WM.ok = 1;
fail:
rt_log("mem: write-back merge off: its pipeline could not be created");
return 0;
}
static int merging(void) {
if (merge_on < 0) {
merge_on = cfg_bool("TOTK_WB_MERGE");
rt_log("mem: write-back merge %s (TOTK_WB_MERGE)", merge_on ? "on" : "off");
}
if (!merge_on) return 0;
if (!WM.tried) merge_init(vk_ctx());
return WM.ok;
}
// A chunk of at least size bytes in chunks[]; -1 when the allocation fails.
static int chunk_new(uint64_t size, int host) {
VkCtx *c = vk_ctx();
if (nchunks == cchunks) grow((void **)&chunks, &cchunks, sizeof *chunks);
SnapChunk *ch = &chunks[nchunks];
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.flags = host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
VMA_ALLOCATION_CREATE_MAPPED_BIT : 0};
if (vmaCreateBuffer(c->vma, &bi, &ai, &ch->buf, &ch->alloc, NULL) != VK_SUCCESS) return -1;
ch->size = size, ch->used = 0;
return nchunks++;
}
// Record time: room in a snapshot chunk for [off, off+size) of r's mirror, once per submission. 0: plain copy instead.
static int snap_add(PoolRec *r, uint64_t off, uint64_t size) {
for (int i = nsnl; i-- > 0;)
if (snl[i].r == r && snl[i].off == off && snl[i].size == size) return 1;
int k = -1;
for (int i = nchunks; i-- > 0;) // the newest snapshot chunk, if the range fits
if (chunks[i].used != UINT64_MAX && chunks[i].size - chunks[i].used >= size) { k = i; break; }
if (k < 0 && (k = chunk_new(size > SNAP_CHUNK ? size : SNAP_CHUNK, 0)) < 0) return 0;
if (nsnl == csnl) grow((void **)&snl, &csnl, sizeof *snl);
snl[nsnl++] = (SnapRange){r, off, size, chunks[k].used, k};
chunks[k].used += (size + 15) & ~15ull;
return 1;
}
// Prologue: the snapshot copies of the ranges added since the last prologue.
static void snap_copies(VkCommandBuffer cb) {
for (int i = snl_copied; i < nsnl; i++) {
VkBufferCopy cp = {snl[i].off, snl[i].soff, snl[i].size};
vkCmdCopyBuffer(cb, snl[i].r->mbuf, chunks[snl[i].chunk].buf, 1, &cp);
mrep.snap_bytes += snl[i].size;
}
snl_copied = nsnl;
}
static int snap_cmp(const void *x, const void *y) {
const SnapRange *a = x, *b = y;
if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1;
if (a->chunk != b->chunk) return a->chunk < b->chunk ? -1 : 1;
return a->off < b->off ? -1 : a->off > b->off;
}
// One dispatch: ranges s[0..n) share a pool and a chunk and lie in [base, base + span) of the pool.
static void merge_dispatch(VkCommandBuffer cb, const SnapRange *s, int n, uint64_t base, uint64_t span, int list,
uint64_t list_off, uint32_t total) {
VkDescriptorBufferInfo bi[4] = {{s->r->mbuf, base, span}, {chunks[s->chunk].buf, 0, chunks[s->chunk].size},
{s->r->buf, base, span}, {chunks[list].buf, list_off, (uint64_t)n * 16}};
VkWriteDescriptorSet w[4];
for (int i = 0; i < 4; i++)
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &bi[i]};
rec_push(WM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.layout, 0, 4, w); // in call order (vk_rec.h)
uint32_t pc[2] = {(uint32_t)n, total};
vkCmdPushConstants(cb, WM.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc);
vkCmdDispatch(cb, (total + 63) / 64, 1, 1);
}
// vk_mem_mirror_end: the merge dispatches of this submission's ranges. Returns the commands recorded; 0 with nsnl
// left as it was when the range list cannot be allocated (the caller copies them plainly).
static int wb_merge_record(VkCommandBuffer cb) {
VkCtx *c = vk_ctx();
uint64_t align = c->props.limits.minStorageBufferOffsetAlignment;
if (align < 16) align = 16;
SnapRange *s = malloc(sizeof *s * (size_t)nsnl);
if (!s) rt_fatal("vk: out of memory");
memcpy(s, snl, sizeof *s * (size_t)nsnl);
qsort(s, (size_t)nsnl, sizeof *s, snap_cmp);
// the range lists: at most one 16-byte entry per range, each dispatch's list at an aligned offset
int list = chunk_new((uint64_t)nsnl * 16 + (uint64_t)nsnl * align, 1);
if (list < 0) { free(s); return 0; }
chunks[list].used = UINT64_MAX; // not a snapshot chunk
VmaAllocationInfo li;
vmaGetAllocationInfo(c->vma, chunks[list].alloc, &li);
uint32_t *lm = li.pMappedData;
uint64_t lo = 0;
int rec = 0;
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.pipe);
for (int i = 0; i < nsnl;) {
uint64_t base = s[i].off & ~(align - 1), end = s[i].off + s[i].size;
uint32_t total = 0;
int j = i;
lo = (lo + align - 1) & ~(align - 1);
uint64_t l0 = lo;
for (; j < nsnl && s[j].r == s[i].r && s[j].chunk == s[i].chunk; j++) {
uint64_t e = s[j].off + s[j].size > end ? s[j].off + s[j].size : end;
if (j > i && (e - base > MERGE_SPAN || total + s[j].size / 4 > MERGE_WORDS)) break;
end = e;
uint32_t *q = lm + lo / 4;
q[0] = (uint32_t)((s[j].off - base) / 4), q[1] = (uint32_t)(s[j].soff / 4), q[2] = (uint32_t)(s[j].size / 4);
q[3] = total;
total += (uint32_t)(s[j].size / 4);
lo += 16;
mrep.merge_bytes += s[j].size;
}
merge_dispatch(cb, s + i, j - i, base, end - base, list, l0, total);
mrep.merge_dispatches++;
rec += 3;
i = j;
}
vmaFlushAllocation(c->vma, chunks[list].alloc, 0, VK_WHOLE_SIZE);
for (int i = 0; i < nsnl; i++) snl[i].r->wb_sub = msub;
free(s);
return rec;
}
static void snap_free(void *arg) { // completion thread: the submission is done with its chunks
SnapFree *f = arg;
for (int i = 0; i < f->n; i++) vmaDestroyBuffer(vk_ctx()->vma, f->c[i].buf, f->c[i].alloc);
free(f);
}
static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return;
// A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies.
if (shader && !(sp->offset & 3) && !(size & 3) && size <= MERGE_MAX && merging() && snap_add(r, sp->offset, size))
return;
range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size);
}
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 0); }
void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 1); }
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) {
if (!vk_mirror_on || !size || !p->mem) return;
@@ -1479,6 +1681,7 @@ void vk_mem_mirror_begin(void) {
return;
}
uint64_t t0 = qpc();
nsnl = snl_copied = nchunks = 0; // the previous submission handed its chunks to snap_free
uint32_t prev = msub;
msub = msub + 1 ? msub + 1 : 1;
__atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE);
@@ -1510,13 +1713,18 @@ void vk_mem_mirror_begin(void) {
mrep.presnap_t += qpc() - t0;
}
int vk_mem_mirror_pending(void) { return npend; }
int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; }
void vk_mem_mirror_prologue(VkCommandBuffer cb) {
if (!npend) return;
if (!npend && snl_copied == nsnl) return;
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies);
if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote
if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT);
snap_copies(cb);
}
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
npend = 0;
@@ -1615,6 +1823,26 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) {
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
if (!vk_mirror_on) return 0;
int rec = 0;
if (nsnl) {
// the merge reads the mirror the submission's shaders wrote and writes the imported buffers
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT,
VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
int m = wb_merge_record(cb);
if (!m) { // no range list: copy them plainly
for (int i = 0; i < nsnl; i++) range_add(&wbl, &nwbl, &cwbl, snl[i].r, snl[i].off, snl[i].size);
}
rec += m + 1;
mbarrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
}
if (nchunks) { // the last part's prologue still records snapshot copies into them; freed at the fence
SnapFree *f = malloc(sizeof *f + sizeof(SnapChunk) * (size_t)nchunks);
if (!f) rt_fatal("vk: out of memory");
f->n = nchunks;
memcpy(f->c, chunks, sizeof(SnapChunk) * (size_t)nchunks);
vk_completion_add(fence, snap_free, f);
}
if (nwbl) {
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT);
@@ -1669,12 +1897,14 @@ static void mirror_log(uint64_t frame) {
(long long)plat_xchg64(&v_mismatch, 0));
rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked "
"%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, "
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries "
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, merge %.3f MiB in %.1f dispatches "
"(snapshots %.3f MiB), flush calls %.1f (%.3f MiB), queries "
"%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f)%s",
(unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET],
m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP],
(long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f,
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M,
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, a.merge_bytes / f / M,
a.merge_dispatches / f, a.snap_bytes / f / M, fc / f, fb / f / M,
a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f,
qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver);
}
@@ -2038,7 +2268,7 @@ void vk_mem_shutdown(void) {
plat_mutex_lock(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
pool_index.n = 0;
nm = npend = nwbl = 0; // mirrors went with their pools
nm = npend = nwbl = nsnl = snl_copied = 0; // mirrors went with their pools
plat_mutex_lock(&shadow_lock);
nstouch = nspend = 0; // shadows too
s_live_n = s_live_bytes = 0;
+72
View File
@@ -0,0 +1,72 @@
// Generated by tools/build_shaders.py from runtime/shaders/wb_merge.comp; do not edit.
// source sha256 a2d55900390e1cca3b742068d5ee21c4cb13a7eb5cb1d5a687f1001e001316b0
#pragma once
#include <stdint.h>
static const uint32_t wb_merge_spv[] = {
0x07230203, 0x00010300, 0x000d000b, 0x00000083, 0x00000000, 0x00020011, 0x00000001, 0x0006000b,
0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001,
0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000b, 0x00060010, 0x00000004,
0x00000011, 0x00000040, 0x00000001, 0x00000001, 0x00040047, 0x0000000b, 0x0000000b, 0x0000001c,
0x00030047, 0x00000011, 0x00000002, 0x00050048, 0x00000011, 0x00000000, 0x00000023, 0x00000000,
0x00050048, 0x00000011, 0x00000001, 0x00000023, 0x00000004, 0x00040047, 0x00000033, 0x00000006,
0x00000010, 0x00030047, 0x00000034, 0x00000002, 0x00040048, 0x00000034, 0x00000000, 0x00000018,
0x00050048, 0x00000034, 0x00000000, 0x00000023, 0x00000000, 0x00030047, 0x00000036, 0x00000018,
0x00040047, 0x00000036, 0x00000021, 0x00000003, 0x00040047, 0x00000036, 0x00000022, 0x00000000,
0x00040047, 0x0000004f, 0x00000006, 0x00000004, 0x00030047, 0x00000050, 0x00000002, 0x00040048,
0x00000050, 0x00000000, 0x00000018, 0x00050048, 0x00000050, 0x00000000, 0x00000023, 0x00000000,
0x00030047, 0x00000052, 0x00000018, 0x00040047, 0x00000052, 0x00000021, 0x00000000, 0x00040047,
0x00000052, 0x00000022, 0x00000000, 0x00040047, 0x0000005a, 0x00000006, 0x00000004, 0x00030047,
0x0000005b, 0x00000002, 0x00040048, 0x0000005b, 0x00000000, 0x00000018, 0x00050048, 0x0000005b,
0x00000000, 0x00000023, 0x00000000, 0x00030047, 0x0000005d, 0x00000018, 0x00040047, 0x0000005d,
0x00000021, 0x00000001, 0x00040047, 0x0000005d, 0x00000022, 0x00000000, 0x00040047, 0x00000067,
0x00000006, 0x00000004, 0x00030047, 0x00000068, 0x00000002, 0x00050048, 0x00000068, 0x00000000,
0x00000023, 0x00000000, 0x00040047, 0x0000006a, 0x00000021, 0x00000002, 0x00040047, 0x0000006a,
0x00000022, 0x00000000, 0x00040047, 0x00000072, 0x0000000b, 0x00000019, 0x00020013, 0x00000002,
0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006, 0x00000020, 0x00000000, 0x00040017,
0x00000009, 0x00000006, 0x00000003, 0x00040020, 0x0000000a, 0x00000001, 0x00000009, 0x0004003b,
0x0000000a, 0x0000000b, 0x00000001, 0x0004002b, 0x00000006, 0x0000000c, 0x00000000, 0x00040020,
0x0000000d, 0x00000001, 0x00000006, 0x0004001e, 0x00000011, 0x00000006, 0x00000006, 0x00040020,
0x00000012, 0x00000009, 0x00000011, 0x0004003b, 0x00000012, 0x00000013, 0x00000009, 0x00040015,
0x00000014, 0x00000020, 0x00000001, 0x0004002b, 0x00000014, 0x00000015, 0x00000001, 0x00040020,
0x00000016, 0x00000009, 0x00000006, 0x00020014, 0x00000019, 0x0004002b, 0x00000014, 0x00000020,
0x00000000, 0x0004002b, 0x00000006, 0x0000002b, 0x00000001, 0x00040017, 0x00000032, 0x00000006,
0x00000004, 0x0003001d, 0x00000033, 0x00000032, 0x0003001e, 0x00000034, 0x00000033, 0x00040020,
0x00000035, 0x0000000c, 0x00000034, 0x0004003b, 0x00000035, 0x00000036, 0x0000000c, 0x0004002b,
0x00000006, 0x00000038, 0x00000003, 0x00040020, 0x00000039, 0x0000000c, 0x00000006, 0x00040020,
0x00000046, 0x0000000c, 0x00000032, 0x0003001d, 0x0000004f, 0x00000006, 0x0003001e, 0x00000050,
0x0000004f, 0x00040020, 0x00000051, 0x0000000c, 0x00000050, 0x0004003b, 0x00000051, 0x00000052,
0x0000000c, 0x0003001d, 0x0000005a, 0x00000006, 0x0003001e, 0x0000005b, 0x0000005a, 0x00040020,
0x0000005c, 0x0000000c, 0x0000005b, 0x0004003b, 0x0000005c, 0x0000005d, 0x0000000c, 0x0003001d,
0x00000067, 0x00000006, 0x0003001e, 0x00000068, 0x00000067, 0x00040020, 0x00000069, 0x0000000c,
0x00000068, 0x0004003b, 0x00000069, 0x0000006a, 0x0000000c, 0x0004002b, 0x00000006, 0x00000071,
0x00000040, 0x0006002c, 0x00000009, 0x00000072, 0x00000071, 0x0000002b, 0x0000002b, 0x00050036,
0x00000002, 0x00000004, 0x00000000, 0x00000003, 0x000200f8, 0x00000005, 0x000300f7, 0x00000073,
0x00000000, 0x000300fb, 0x0000000c, 0x00000074, 0x000200f8, 0x00000074, 0x00050041, 0x0000000d,
0x0000000e, 0x0000000b, 0x0000000c, 0x0004003d, 0x00000006, 0x0000000f, 0x0000000e, 0x00050041,
0x00000016, 0x00000017, 0x00000013, 0x00000015, 0x0004003d, 0x00000006, 0x00000018, 0x00000017,
0x000500ae, 0x00000019, 0x0000001a, 0x0000000f, 0x00000018, 0x000300f7, 0x0000001c, 0x00000000,
0x000400fa, 0x0000001a, 0x0000001b, 0x0000001c, 0x000200f8, 0x0000001b, 0x000200f9, 0x00000073,
0x000200f8, 0x0000001c, 0x00050041, 0x00000016, 0x00000021, 0x00000013, 0x00000020, 0x0004003d,
0x00000006, 0x00000022, 0x00000021, 0x000200f9, 0x00000023, 0x000200f8, 0x00000023, 0x000700f5,
0x00000006, 0x0000007e, 0x0000000c, 0x0000001c, 0x00000081, 0x00000024, 0x000700f5, 0x00000006,
0x0000007d, 0x00000022, 0x0000001c, 0x00000082, 0x00000024, 0x00050082, 0x00000006, 0x0000002a,
0x0000007d, 0x0000007e, 0x000500ac, 0x00000019, 0x0000002c, 0x0000002a, 0x0000002b, 0x000400f6,
0x00000025, 0x00000024, 0x00000000, 0x000400fa, 0x0000002c, 0x00000024, 0x00000025, 0x000200f8,
0x00000024, 0x00050080, 0x00000006, 0x00000030, 0x0000007e, 0x0000007d, 0x000500c2, 0x00000006,
0x00000031, 0x00000030, 0x00000015, 0x00070041, 0x00000039, 0x0000003a, 0x00000036, 0x00000020,
0x00000031, 0x00000038, 0x0004003d, 0x00000006, 0x0000003b, 0x0000003a, 0x000500b2, 0x00000019,
0x0000003d, 0x0000003b, 0x0000000f, 0x000600a9, 0x00000006, 0x00000081, 0x0000003d, 0x00000031,
0x0000007e, 0x000600a9, 0x00000006, 0x00000082, 0x0000003d, 0x0000007d, 0x00000031, 0x000200f9,
0x00000023, 0x000200f8, 0x00000025, 0x00060041, 0x00000046, 0x00000047, 0x00000036, 0x00000020,
0x0000007e, 0x0004003d, 0x00000032, 0x00000048, 0x00000047, 0x00050051, 0x00000006, 0x0000004c,
0x00000048, 0x00000003, 0x00050082, 0x00000006, 0x0000004d, 0x0000000f, 0x0000004c, 0x00050051,
0x00000006, 0x00000054, 0x00000048, 0x00000000, 0x00050080, 0x00000006, 0x00000056, 0x00000054,
0x0000004d, 0x00060041, 0x00000039, 0x00000057, 0x00000052, 0x00000020, 0x00000056, 0x0004003d,
0x00000006, 0x00000058, 0x00000057, 0x00050051, 0x00000006, 0x0000005f, 0x00000048, 0x00000001,
0x00050080, 0x00000006, 0x00000061, 0x0000005f, 0x0000004d, 0x00060041, 0x00000039, 0x00000062,
0x0000005d, 0x00000020, 0x00000061, 0x0004003d, 0x00000006, 0x00000063, 0x00000062, 0x000500ab,
0x00000019, 0x00000064, 0x00000058, 0x00000063, 0x000300f7, 0x00000066, 0x00000000, 0x000400fa,
0x00000064, 0x00000065, 0x00000066, 0x000200f8, 0x00000065, 0x00060041, 0x00000039, 0x00000070,
0x0000006a, 0x00000020, 0x00000056, 0x0003003e, 0x00000070, 0x00000058, 0x000200f9, 0x00000066,
0x000200f8, 0x00000066, 0x000200f9, 0x00000073, 0x000200f8, 0x00000073, 0x000100fd, 0x00010038,
};