mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 04:35:57 +00:00
Merge branch 'pr/528' into task/wbmerge
This commit is contained in:
commit
1cd3e3a584
6 files changed
+342
-10
No files matched your search
+3
-3
@@ -55,7 +55,7 @@ static const char *const known[] = {
|
||||
"SWKBD_ANSWER",
|
||||
"SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
|
||||
"VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
|
||||
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS",
|
||||
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS",
|
||||
"ZSTD_CHECK", "ZSTD_NATIVE",
|
||||
};
|
||||
|
||||
@@ -96,7 +96,7 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
|
||||
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"},
|
||||
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
|
||||
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_EXACT", "diagnostics"},
|
||||
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"},
|
||||
};
|
||||
|
||||
// The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a
|
||||
@@ -166,7 +166,7 @@ static const CfgMeta metas[] = {
|
||||
{"VK_DYN_A2C", "", "0|1", RESTART}, {"VK_SUBGROUP_PIN", "1", "0|1", RESTART},
|
||||
// diagnostics
|
||||
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
|
||||
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_EXACT", "1", "0|1", RESTART},
|
||||
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART},
|
||||
{"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only
|
||||
{"INPUT_RECORD", "", NULL, RESTART},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
#version 450
|
||||
// Write-back merge of a VRAM mirror (vk_memory.c, wb_merge_record): copies a 32-bit word of what the GPU left in the
|
||||
// mirror into the pool's imported buffer (guest memory) only where it differs from the snapshot taken before the
|
||||
// submission's GPU work, so words the GPU did not change keep what the CPU wrote meanwhile.
|
||||
// One invocation per word of the listed ranges; ranges are sorted by their first invocation (x: word offset in the
|
||||
// mirror and imported bindings, y: word offset in the snapshot binding, z: words, w: first invocation).
|
||||
layout(local_size_x = 64) in;
|
||||
layout(std430, set = 0, binding = 0) readonly buffer Post { uint post[]; };
|
||||
layout(std430, set = 0, binding = 1) readonly buffer Pre { uint pre[]; };
|
||||
layout(std430, set = 0, binding = 2) buffer Dst { uint dst[]; };
|
||||
layout(std430, set = 0, binding = 3) readonly buffer Ranges { uvec4 rng[]; };
|
||||
layout(push_constant) uniform Push { uint nranges, total; } pc;
|
||||
|
||||
void main() {
|
||||
uint i = gl_GlobalInvocationID.x;
|
||||
if (i >= pc.total) return;
|
||||
uint lo = 0, hi = pc.nranges;
|
||||
while (hi - lo > 1) {
|
||||
uint m = (lo + hi) >> 1;
|
||||
if (rng[m].w <= i) lo = m;
|
||||
else hi = m;
|
||||
}
|
||||
uvec4 r = rng[lo];
|
||||
uint k = i - r.w;
|
||||
uint v = post[r.x + k];
|
||||
if (v != pre[r.y + k]) dst[r.x + k] = v;
|
||||
}
|
||||
@@ -250,6 +250,9 @@ void vk_mem_pick(VkMemSpan *sp);
|
||||
#define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0)
|
||||
#define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0)
|
||||
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size);
|
||||
// A shader's storage-buffer write: goes back through the write-back merge (vk_memory.c), which keeps the words the GPU
|
||||
// did not change.
|
||||
void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size);
|
||||
// Uniform blocks as of the submit call (vk_usnap.c): vk_exec_record copies the uniform ranges a submission binds before
|
||||
// it records, ubo bindings take the copy (0: not copied, bind memory), GPU writes of the replay drop the copies they touch.
|
||||
void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence);
|
||||
|
||||
@@ -807,7 +807,7 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align,
|
||||
if (sp.offset % align == 0) {
|
||||
VK_MEM_USE(sp);
|
||||
*out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n};
|
||||
if (written) vk_mem_gpu_wrote(&sp, n), vk_usnap_drop(addr, n);
|
||||
if (written) vk_mem_gpu_wrote_shader(&sp, n), vk_usnap_drop(addr, n);
|
||||
count_use(sp.cls, use, n);
|
||||
VK_MEM_BOUND(sp);
|
||||
return 1;
|
||||
|
||||
+236
-6
@@ -21,6 +21,7 @@
|
||||
#include "nvn_pool_index.h"
|
||||
#include "vk.h"
|
||||
#include "vk_exec.h"
|
||||
#include "wb_merge_spv.h"
|
||||
#include "platform/platform.h"
|
||||
|
||||
// Timestamps for the tracking cost, in ns; qpc_ms converts.
|
||||
@@ -1164,6 +1165,7 @@ static PlatMutex flush_lock = PLAT_MUTEX_INIT;
|
||||
typedef struct {
|
||||
uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges;
|
||||
uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes;
|
||||
uint64_t snap_bytes, merge_bytes, merge_dispatches; // write-back merge
|
||||
} MirrorAcc;
|
||||
static MirrorAcc mrep; // replay: this submission
|
||||
static MirrorAcc macc; // mirror_stat_lock: since the last summary
|
||||
@@ -1431,11 +1433,211 @@ int vk_mem_mirror_probe(uint64_t addr) {
|
||||
return ((PoolRec *)sp.pool)->mstate == MS_ON;
|
||||
}
|
||||
|
||||
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) {
|
||||
// ---- write-back merge
|
||||
// A storage buffer a shader writes whole records of does not
|
||||
// go back to guest memory as a plain copy. The CPU may write fields of the same records while the GPU runs: TotK's
|
||||
// particle emitters write a new particle's whole record into a free slot of a buffer that the simulation shaders update
|
||||
// (the waterfall splashes and mist, 2026-11-02), and a plain copy puts the mirror's stale copy of those words back over
|
||||
// them. Instead:
|
||||
// - record time (gpu_wrote from vk_mem_gpu_wrote_shader): each distinct range of the submission gets room in a snapshot
|
||||
// chunk (snap_add);
|
||||
// - each part's prologue, after the CPU -> mirror copies: the mirror's words of the ranges the part added are copied
|
||||
// into their snapshots (snap_copies), the state before the part's GPU work. Words of an earlier part's writes that a
|
||||
// later range overlaps are covered by the earlier range's own snapshot;
|
||||
// - vk_mem_mirror_end: shaders/wb_merge.comp writes each 32-bit word that differs between the mirror and the snapshot
|
||||
// into the pool's imported buffer (wb_merge_record), one dispatch per pool and group of ranges.
|
||||
// Words the GPU left unchanged keep the CPU's value; a word both changed takes the GPU's, as a plain copy would.
|
||||
// TOTK_WB_MERGE=0 restores the plain copies; so does a pipeline or allocation failure.
|
||||
#define SNAP_CHUNK (1u << 20)
|
||||
#define MERGE_SPAN (64ull << 20) // the bytes of one pool a merge dispatch binds at most
|
||||
#define MERGE_WORDS (65535u * 64u) // the words one merge dispatch covers at most
|
||||
#define MERGE_MAX (4ull << 20) // larger written ranges stay plain copies
|
||||
typedef struct { PoolRec *r; uint64_t off, size, soff; int chunk; } SnapRange;
|
||||
typedef struct { VkBuffer buf; VmaAllocation alloc; uint64_t size, used; } SnapChunk;
|
||||
typedef struct { int n; SnapChunk c[]; } SnapFree; // a submission's chunks and range lists, freed at its fence
|
||||
static SnapRange *snl; // replay: this submission's merged ranges; [0, snl_copied) have their snapshot copy
|
||||
static int nsnl, csnl, snl_copied;
|
||||
static SnapChunk *chunks; // replay: this submission's snapshot chunks and merge range lists
|
||||
static int nchunks, cchunks;
|
||||
static int merge_on = -1;
|
||||
static struct {
|
||||
int tried, ok;
|
||||
PFN_vkCmdPushDescriptorSetKHR push;
|
||||
VkDescriptorSetLayout dsl;
|
||||
VkPipelineLayout layout;
|
||||
VkPipeline pipe;
|
||||
} WM;
|
||||
|
||||
static int merge_init(VkCtx *c) {
|
||||
WM.tried = 1;
|
||||
WM.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR");
|
||||
VkDescriptorSetLayoutBinding b[4];
|
||||
for (int i = 0; i < 4; i++)
|
||||
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
||||
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
|
||||
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
||||
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
|
||||
.bindingCount = 4, .pBindings = b};
|
||||
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 8};
|
||||
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof wb_merge_spv,
|
||||
.pCode = wb_merge_spv};
|
||||
VkShaderModule m = VK_NULL_HANDLE;
|
||||
if (!WM.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &WM.dsl) != VK_SUCCESS) goto fail;
|
||||
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
|
||||
.pSetLayouts = &WM.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
|
||||
if (vkCreatePipelineLayout(c->device, &li, NULL, &WM.layout) != VK_SUCCESS ||
|
||||
vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS)
|
||||
goto fail;
|
||||
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
||||
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
||||
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
||||
.layout = WM.layout};
|
||||
VkResult res = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &WM.pipe);
|
||||
vkDestroyShaderModule(c->device, m, NULL);
|
||||
if (res != VK_SUCCESS) goto fail;
|
||||
return WM.ok = 1;
|
||||
fail:
|
||||
rt_log("mem: write-back merge off: its pipeline could not be created");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int merging(void) {
|
||||
if (merge_on < 0) {
|
||||
merge_on = cfg_bool("TOTK_WB_MERGE");
|
||||
rt_log("mem: write-back merge %s (TOTK_WB_MERGE)", merge_on ? "on" : "off");
|
||||
}
|
||||
if (!merge_on) return 0;
|
||||
if (!WM.tried) merge_init(vk_ctx());
|
||||
return WM.ok;
|
||||
}
|
||||
|
||||
// A chunk of at least size bytes in chunks[]; -1 when the allocation fails.
|
||||
static int chunk_new(uint64_t size, int host) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (nchunks == cchunks) grow((void **)&chunks, &cchunks, sizeof *chunks);
|
||||
SnapChunk *ch = &chunks[nchunks];
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size,
|
||||
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
|
||||
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
||||
VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
||||
.flags = host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
|
||||
VMA_ALLOCATION_CREATE_MAPPED_BIT : 0};
|
||||
if (vmaCreateBuffer(c->vma, &bi, &ai, &ch->buf, &ch->alloc, NULL) != VK_SUCCESS) return -1;
|
||||
ch->size = size, ch->used = 0;
|
||||
return nchunks++;
|
||||
}
|
||||
|
||||
// Record time: room in a snapshot chunk for [off, off+size) of r's mirror, once per submission. 0: plain copy instead.
|
||||
static int snap_add(PoolRec *r, uint64_t off, uint64_t size) {
|
||||
for (int i = nsnl; i-- > 0;)
|
||||
if (snl[i].r == r && snl[i].off == off && snl[i].size == size) return 1;
|
||||
int k = -1;
|
||||
for (int i = nchunks; i-- > 0;) // the newest snapshot chunk, if the range fits
|
||||
if (chunks[i].used != UINT64_MAX && chunks[i].size - chunks[i].used >= size) { k = i; break; }
|
||||
if (k < 0 && (k = chunk_new(size > SNAP_CHUNK ? size : SNAP_CHUNK, 0)) < 0) return 0;
|
||||
if (nsnl == csnl) grow((void **)&snl, &csnl, sizeof *snl);
|
||||
snl[nsnl++] = (SnapRange){r, off, size, chunks[k].used, k};
|
||||
chunks[k].used += (size + 15) & ~15ull;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Prologue: the snapshot copies of the ranges added since the last prologue.
|
||||
static void snap_copies(VkCommandBuffer cb) {
|
||||
for (int i = snl_copied; i < nsnl; i++) {
|
||||
VkBufferCopy cp = {snl[i].off, snl[i].soff, snl[i].size};
|
||||
vkCmdCopyBuffer(cb, snl[i].r->mbuf, chunks[snl[i].chunk].buf, 1, &cp);
|
||||
mrep.snap_bytes += snl[i].size;
|
||||
}
|
||||
snl_copied = nsnl;
|
||||
}
|
||||
|
||||
static int snap_cmp(const void *x, const void *y) {
|
||||
const SnapRange *a = x, *b = y;
|
||||
if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1;
|
||||
if (a->chunk != b->chunk) return a->chunk < b->chunk ? -1 : 1;
|
||||
return a->off < b->off ? -1 : a->off > b->off;
|
||||
}
|
||||
|
||||
// One dispatch: ranges s[0..n) share a pool and a chunk and lie in [base, base + span) of the pool.
|
||||
static void merge_dispatch(VkCommandBuffer cb, const SnapRange *s, int n, uint64_t base, uint64_t span, int list,
|
||||
uint64_t list_off, uint32_t total) {
|
||||
VkDescriptorBufferInfo bi[4] = {{s->r->mbuf, base, span}, {chunks[s->chunk].buf, 0, chunks[s->chunk].size},
|
||||
{s->r->buf, base, span}, {chunks[list].buf, list_off, (uint64_t)n * 16}};
|
||||
VkWriteDescriptorSet w[4];
|
||||
for (int i = 0; i < 4; i++)
|
||||
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
|
||||
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
||||
.pBufferInfo = &bi[i]};
|
||||
rec_push(WM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.layout, 0, 4, w); // in call order (vk_rec.h)
|
||||
uint32_t pc[2] = {(uint32_t)n, total};
|
||||
vkCmdPushConstants(cb, WM.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc);
|
||||
vkCmdDispatch(cb, (total + 63) / 64, 1, 1);
|
||||
}
|
||||
|
||||
// vk_mem_mirror_end: the merge dispatches of this submission's ranges. Returns the commands recorded; 0 with nsnl
|
||||
// left as it was when the range list cannot be allocated (the caller copies them plainly).
|
||||
static int wb_merge_record(VkCommandBuffer cb) {
|
||||
VkCtx *c = vk_ctx();
|
||||
uint64_t align = c->props.limits.minStorageBufferOffsetAlignment;
|
||||
if (align < 16) align = 16;
|
||||
SnapRange *s = malloc(sizeof *s * (size_t)nsnl);
|
||||
if (!s) rt_fatal("vk: out of memory");
|
||||
memcpy(s, snl, sizeof *s * (size_t)nsnl);
|
||||
qsort(s, (size_t)nsnl, sizeof *s, snap_cmp);
|
||||
// the range lists: at most one 16-byte entry per range, each dispatch's list at an aligned offset
|
||||
int list = chunk_new((uint64_t)nsnl * 16 + (uint64_t)nsnl * align, 1);
|
||||
if (list < 0) { free(s); return 0; }
|
||||
chunks[list].used = UINT64_MAX; // not a snapshot chunk
|
||||
VmaAllocationInfo li;
|
||||
vmaGetAllocationInfo(c->vma, chunks[list].alloc, &li);
|
||||
uint32_t *lm = li.pMappedData;
|
||||
uint64_t lo = 0;
|
||||
int rec = 0;
|
||||
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.pipe);
|
||||
for (int i = 0; i < nsnl;) {
|
||||
uint64_t base = s[i].off & ~(align - 1), end = s[i].off + s[i].size;
|
||||
uint32_t total = 0;
|
||||
int j = i;
|
||||
lo = (lo + align - 1) & ~(align - 1);
|
||||
uint64_t l0 = lo;
|
||||
for (; j < nsnl && s[j].r == s[i].r && s[j].chunk == s[i].chunk; j++) {
|
||||
uint64_t e = s[j].off + s[j].size > end ? s[j].off + s[j].size : end;
|
||||
if (j > i && (e - base > MERGE_SPAN || total + s[j].size / 4 > MERGE_WORDS)) break;
|
||||
end = e;
|
||||
uint32_t *q = lm + lo / 4;
|
||||
q[0] = (uint32_t)((s[j].off - base) / 4), q[1] = (uint32_t)(s[j].soff / 4), q[2] = (uint32_t)(s[j].size / 4);
|
||||
q[3] = total;
|
||||
total += (uint32_t)(s[j].size / 4);
|
||||
lo += 16;
|
||||
mrep.merge_bytes += s[j].size;
|
||||
}
|
||||
merge_dispatch(cb, s + i, j - i, base, end - base, list, l0, total);
|
||||
mrep.merge_dispatches++;
|
||||
rec += 3;
|
||||
i = j;
|
||||
}
|
||||
vmaFlushAllocation(c->vma, chunks[list].alloc, 0, VK_WHOLE_SIZE);
|
||||
for (int i = 0; i < nsnl; i++) snl[i].r->wb_sub = msub;
|
||||
free(s);
|
||||
return rec;
|
||||
}
|
||||
|
||||
static void snap_free(void *arg) { // completion thread: the submission is done with its chunks
|
||||
SnapFree *f = arg;
|
||||
for (int i = 0; i < f->n; i++) vmaDestroyBuffer(vk_ctx()->vma, f->c[i].buf, f->c[i].alloc);
|
||||
free(f);
|
||||
}
|
||||
|
||||
static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) {
|
||||
PoolRec *r = sp->pool;
|
||||
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return;
|
||||
// A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies.
|
||||
if (shader && !(sp->offset & 3) && !(size & 3) && size <= MERGE_MAX && merging() && snap_add(r, sp->offset, size))
|
||||
return;
|
||||
range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size);
|
||||
}
|
||||
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 0); }
|
||||
void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 1); }
|
||||
|
||||
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) {
|
||||
if (!vk_mirror_on || !size || !p->mem) return;
|
||||
@@ -1479,6 +1681,7 @@ void vk_mem_mirror_begin(void) {
|
||||
return;
|
||||
}
|
||||
uint64_t t0 = qpc();
|
||||
nsnl = snl_copied = nchunks = 0; // the previous submission handed its chunks to snap_free
|
||||
uint32_t prev = msub;
|
||||
msub = msub + 1 ? msub + 1 : 1;
|
||||
__atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE);
|
||||
@@ -1510,13 +1713,18 @@ void vk_mem_mirror_begin(void) {
|
||||
mrep.presnap_t += qpc() - t0;
|
||||
}
|
||||
|
||||
int vk_mem_mirror_pending(void) { return npend; }
|
||||
int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; }
|
||||
|
||||
void vk_mem_mirror_prologue(VkCommandBuffer cb) {
|
||||
if (!npend) return;
|
||||
if (!npend && snl_copied == nsnl) return;
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
||||
copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies);
|
||||
if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote
|
||||
if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_TRANSFER_READ_BIT);
|
||||
snap_copies(cb);
|
||||
}
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
||||
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
||||
npend = 0;
|
||||
@@ -1615,6 +1823,26 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) {
|
||||
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
|
||||
if (!vk_mirror_on) return 0;
|
||||
int rec = 0;
|
||||
if (nsnl) {
|
||||
// the merge reads the mirror the submission's shaders wrote and writes the imported buffers
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT,
|
||||
VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
|
||||
int m = wb_merge_record(cb);
|
||||
if (!m) { // no range list: copy them plainly
|
||||
for (int i = 0; i < nsnl; i++) range_add(&wbl, &nwbl, &cwbl, snl[i].r, snl[i].off, snl[i].size);
|
||||
}
|
||||
rec += m + 1;
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
|
||||
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
||||
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
||||
}
|
||||
if (nchunks) { // the last part's prologue still records snapshot copies into them; freed at the fence
|
||||
SnapFree *f = malloc(sizeof *f + sizeof(SnapChunk) * (size_t)nchunks);
|
||||
if (!f) rt_fatal("vk: out of memory");
|
||||
f->n = nchunks;
|
||||
memcpy(f->c, chunks, sizeof(SnapChunk) * (size_t)nchunks);
|
||||
vk_completion_add(fence, snap_free, f);
|
||||
}
|
||||
if (nwbl) {
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_TRANSFER_READ_BIT);
|
||||
@@ -1669,12 +1897,14 @@ static void mirror_log(uint64_t frame) {
|
||||
(long long)plat_xchg64(&v_mismatch, 0));
|
||||
rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked "
|
||||
"%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, "
|
||||
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries "
|
||||
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, merge %.3f MiB in %.1f dispatches "
|
||||
"(snapshots %.3f MiB), flush calls %.1f (%.3f MiB), queries "
|
||||
"%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f)%s",
|
||||
(unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET],
|
||||
m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP],
|
||||
(long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f,
|
||||
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M,
|
||||
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, a.merge_bytes / f / M,
|
||||
a.merge_dispatches / f, a.snap_bytes / f / M, fc / f, fb / f / M,
|
||||
a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f,
|
||||
qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver);
|
||||
}
|
||||
@@ -2038,7 +2268,7 @@ void vk_mem_shutdown(void) {
|
||||
plat_mutex_lock(&pool_lock);
|
||||
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
|
||||
pool_index.n = 0;
|
||||
nm = npend = nwbl = 0; // mirrors went with their pools
|
||||
nm = npend = nwbl = nsnl = snl_copied = 0; // mirrors went with their pools
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
nstouch = nspend = 0; // shadows too
|
||||
s_live_n = s_live_bytes = 0;
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
// Generated by tools/build_shaders.py from runtime/shaders/wb_merge.comp; do not edit.
|
||||
// source sha256 a2d55900390e1cca3b742068d5ee21c4cb13a7eb5cb1d5a687f1001e001316b0
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
static const uint32_t wb_merge_spv[] = {
|
||||
0x07230203, 0x00010300, 0x000d000b, 0x00000083, 0x00000000, 0x00020011, 0x00000001, 0x0006000b,
|
||||
0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001,
|
||||
0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000b, 0x00060010, 0x00000004,
|
||||
0x00000011, 0x00000040, 0x00000001, 0x00000001, 0x00040047, 0x0000000b, 0x0000000b, 0x0000001c,
|
||||
0x00030047, 0x00000011, 0x00000002, 0x00050048, 0x00000011, 0x00000000, 0x00000023, 0x00000000,
|
||||
0x00050048, 0x00000011, 0x00000001, 0x00000023, 0x00000004, 0x00040047, 0x00000033, 0x00000006,
|
||||
0x00000010, 0x00030047, 0x00000034, 0x00000002, 0x00040048, 0x00000034, 0x00000000, 0x00000018,
|
||||
0x00050048, 0x00000034, 0x00000000, 0x00000023, 0x00000000, 0x00030047, 0x00000036, 0x00000018,
|
||||
0x00040047, 0x00000036, 0x00000021, 0x00000003, 0x00040047, 0x00000036, 0x00000022, 0x00000000,
|
||||
0x00040047, 0x0000004f, 0x00000006, 0x00000004, 0x00030047, 0x00000050, 0x00000002, 0x00040048,
|
||||
0x00000050, 0x00000000, 0x00000018, 0x00050048, 0x00000050, 0x00000000, 0x00000023, 0x00000000,
|
||||
0x00030047, 0x00000052, 0x00000018, 0x00040047, 0x00000052, 0x00000021, 0x00000000, 0x00040047,
|
||||
0x00000052, 0x00000022, 0x00000000, 0x00040047, 0x0000005a, 0x00000006, 0x00000004, 0x00030047,
|
||||
0x0000005b, 0x00000002, 0x00040048, 0x0000005b, 0x00000000, 0x00000018, 0x00050048, 0x0000005b,
|
||||
0x00000000, 0x00000023, 0x00000000, 0x00030047, 0x0000005d, 0x00000018, 0x00040047, 0x0000005d,
|
||||
0x00000021, 0x00000001, 0x00040047, 0x0000005d, 0x00000022, 0x00000000, 0x00040047, 0x00000067,
|
||||
0x00000006, 0x00000004, 0x00030047, 0x00000068, 0x00000002, 0x00050048, 0x00000068, 0x00000000,
|
||||
0x00000023, 0x00000000, 0x00040047, 0x0000006a, 0x00000021, 0x00000002, 0x00040047, 0x0000006a,
|
||||
0x00000022, 0x00000000, 0x00040047, 0x00000072, 0x0000000b, 0x00000019, 0x00020013, 0x00000002,
|
||||
0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006, 0x00000020, 0x00000000, 0x00040017,
|
||||
0x00000009, 0x00000006, 0x00000003, 0x00040020, 0x0000000a, 0x00000001, 0x00000009, 0x0004003b,
|
||||
0x0000000a, 0x0000000b, 0x00000001, 0x0004002b, 0x00000006, 0x0000000c, 0x00000000, 0x00040020,
|
||||
0x0000000d, 0x00000001, 0x00000006, 0x0004001e, 0x00000011, 0x00000006, 0x00000006, 0x00040020,
|
||||
0x00000012, 0x00000009, 0x00000011, 0x0004003b, 0x00000012, 0x00000013, 0x00000009, 0x00040015,
|
||||
0x00000014, 0x00000020, 0x00000001, 0x0004002b, 0x00000014, 0x00000015, 0x00000001, 0x00040020,
|
||||
0x00000016, 0x00000009, 0x00000006, 0x00020014, 0x00000019, 0x0004002b, 0x00000014, 0x00000020,
|
||||
0x00000000, 0x0004002b, 0x00000006, 0x0000002b, 0x00000001, 0x00040017, 0x00000032, 0x00000006,
|
||||
0x00000004, 0x0003001d, 0x00000033, 0x00000032, 0x0003001e, 0x00000034, 0x00000033, 0x00040020,
|
||||
0x00000035, 0x0000000c, 0x00000034, 0x0004003b, 0x00000035, 0x00000036, 0x0000000c, 0x0004002b,
|
||||
0x00000006, 0x00000038, 0x00000003, 0x00040020, 0x00000039, 0x0000000c, 0x00000006, 0x00040020,
|
||||
0x00000046, 0x0000000c, 0x00000032, 0x0003001d, 0x0000004f, 0x00000006, 0x0003001e, 0x00000050,
|
||||
0x0000004f, 0x00040020, 0x00000051, 0x0000000c, 0x00000050, 0x0004003b, 0x00000051, 0x00000052,
|
||||
0x0000000c, 0x0003001d, 0x0000005a, 0x00000006, 0x0003001e, 0x0000005b, 0x0000005a, 0x00040020,
|
||||
0x0000005c, 0x0000000c, 0x0000005b, 0x0004003b, 0x0000005c, 0x0000005d, 0x0000000c, 0x0003001d,
|
||||
0x00000067, 0x00000006, 0x0003001e, 0x00000068, 0x00000067, 0x00040020, 0x00000069, 0x0000000c,
|
||||
0x00000068, 0x0004003b, 0x00000069, 0x0000006a, 0x0000000c, 0x0004002b, 0x00000006, 0x00000071,
|
||||
0x00000040, 0x0006002c, 0x00000009, 0x00000072, 0x00000071, 0x0000002b, 0x0000002b, 0x00050036,
|
||||
0x00000002, 0x00000004, 0x00000000, 0x00000003, 0x000200f8, 0x00000005, 0x000300f7, 0x00000073,
|
||||
0x00000000, 0x000300fb, 0x0000000c, 0x00000074, 0x000200f8, 0x00000074, 0x00050041, 0x0000000d,
|
||||
0x0000000e, 0x0000000b, 0x0000000c, 0x0004003d, 0x00000006, 0x0000000f, 0x0000000e, 0x00050041,
|
||||
0x00000016, 0x00000017, 0x00000013, 0x00000015, 0x0004003d, 0x00000006, 0x00000018, 0x00000017,
|
||||
0x000500ae, 0x00000019, 0x0000001a, 0x0000000f, 0x00000018, 0x000300f7, 0x0000001c, 0x00000000,
|
||||
0x000400fa, 0x0000001a, 0x0000001b, 0x0000001c, 0x000200f8, 0x0000001b, 0x000200f9, 0x00000073,
|
||||
0x000200f8, 0x0000001c, 0x00050041, 0x00000016, 0x00000021, 0x00000013, 0x00000020, 0x0004003d,
|
||||
0x00000006, 0x00000022, 0x00000021, 0x000200f9, 0x00000023, 0x000200f8, 0x00000023, 0x000700f5,
|
||||
0x00000006, 0x0000007e, 0x0000000c, 0x0000001c, 0x00000081, 0x00000024, 0x000700f5, 0x00000006,
|
||||
0x0000007d, 0x00000022, 0x0000001c, 0x00000082, 0x00000024, 0x00050082, 0x00000006, 0x0000002a,
|
||||
0x0000007d, 0x0000007e, 0x000500ac, 0x00000019, 0x0000002c, 0x0000002a, 0x0000002b, 0x000400f6,
|
||||
0x00000025, 0x00000024, 0x00000000, 0x000400fa, 0x0000002c, 0x00000024, 0x00000025, 0x000200f8,
|
||||
0x00000024, 0x00050080, 0x00000006, 0x00000030, 0x0000007e, 0x0000007d, 0x000500c2, 0x00000006,
|
||||
0x00000031, 0x00000030, 0x00000015, 0x00070041, 0x00000039, 0x0000003a, 0x00000036, 0x00000020,
|
||||
0x00000031, 0x00000038, 0x0004003d, 0x00000006, 0x0000003b, 0x0000003a, 0x000500b2, 0x00000019,
|
||||
0x0000003d, 0x0000003b, 0x0000000f, 0x000600a9, 0x00000006, 0x00000081, 0x0000003d, 0x00000031,
|
||||
0x0000007e, 0x000600a9, 0x00000006, 0x00000082, 0x0000003d, 0x0000007d, 0x00000031, 0x000200f9,
|
||||
0x00000023, 0x000200f8, 0x00000025, 0x00060041, 0x00000046, 0x00000047, 0x00000036, 0x00000020,
|
||||
0x0000007e, 0x0004003d, 0x00000032, 0x00000048, 0x00000047, 0x00050051, 0x00000006, 0x0000004c,
|
||||
0x00000048, 0x00000003, 0x00050082, 0x00000006, 0x0000004d, 0x0000000f, 0x0000004c, 0x00050051,
|
||||
0x00000006, 0x00000054, 0x00000048, 0x00000000, 0x00050080, 0x00000006, 0x00000056, 0x00000054,
|
||||
0x0000004d, 0x00060041, 0x00000039, 0x00000057, 0x00000052, 0x00000020, 0x00000056, 0x0004003d,
|
||||
0x00000006, 0x00000058, 0x00000057, 0x00050051, 0x00000006, 0x0000005f, 0x00000048, 0x00000001,
|
||||
0x00050080, 0x00000006, 0x00000061, 0x0000005f, 0x0000004d, 0x00060041, 0x00000039, 0x00000062,
|
||||
0x0000005d, 0x00000020, 0x00000061, 0x0004003d, 0x00000006, 0x00000063, 0x00000062, 0x000500ab,
|
||||
0x00000019, 0x00000064, 0x00000058, 0x00000063, 0x000300f7, 0x00000066, 0x00000000, 0x000400fa,
|
||||
0x00000064, 0x00000065, 0x00000066, 0x000200f8, 0x00000065, 0x00060041, 0x00000039, 0x00000070,
|
||||
0x0000006a, 0x00000020, 0x00000056, 0x0003003e, 0x00000070, 0x00000058, 0x000200f9, 0x00000066,
|
||||
0x000200f8, 0x00000066, 0x000200f9, 0x00000073, 0x000200f8, 0x00000073, 0x000100fd, 0x00010038,
|
||||
};
|
||||
Reference in new issue
Block a user