diff --git a/runtime/host.h b/runtime/host.h index 57a1f7c2..8603b6a1 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -128,6 +128,9 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64 // past max stays reported next time). The range must lie // inside the stack/heap block or the alias region // (write watch); otherwise returns UINT64_MAX +uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max); + // the same without the reset: the bits stay for the next + // mem_written_pages (verify checks only) void mem_kernel_free(void *p, uint64_t size); // TLR/TLS reuse after thread exit // ---- modules (loader.c, T1) diff --git a/runtime/memory.c b/runtime/memory.c index 84bf8646..7f8c6494 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -148,7 +148,7 @@ const RtGuestLayout *mem_layout(void) { return &L; } // Adapted from LAS Recomp runtime/platform/windows.c:40-63 @ da35b83 (MIT, Copyright (c) 2026 Peiikai): GetWriteWatch // with the reset flag. A range inside one watched reservation (stack and heap block, or alias region) only. -uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { +static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, DWORD flags) { if (!max || !size) return 0; const RtGuestLayout *g = &L; int low_block = addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size; @@ -157,11 +157,18 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64 ULONG_PTR n = (ULONG_PTR)max; DWORD gran; _Static_assert(sizeof(PVOID) == sizeof(uint64_t), "page addresses are written straight into a uint64_t array"); - if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran)) - return UINT64_MAX; + if (GetWriteWatch(flags, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran)) return UINT64_MAX; return n; } +uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { + return written_pages(addr, size, pages, max, WRITE_WATCH_FLAG_RESET); +} + +uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { + return written_pages(addr, size, pages, max, 0); +} + static uint64_t image_bytes(void) { uint64_t n = 0; for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size; diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index 4cfe2b3b..fb5b31d3 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -99,3 +99,6 @@ const char *nvn_op_name(uint32_t op); // "DRAW_IDX", ...; " // MemoryPoolInvalidateMappedRange: the CPU is about to read pool storage the GPU wrote. Weak no-op in nvn_ext.c; a back // end that renders into host-side copies defines it and writes them back. void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size); +// MemoryPoolFlushMappedRange: the CPU wrote [mem + off, mem + off + size). Weak no-op in nvn_ext.c; a back end that keeps +// device-side copies of pool memory (vk_memory.c's VRAM mirror) defines it and copies the range again. +void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size); diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 7978ed9e..0010d3b6 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -303,9 +303,14 @@ void nvn_flush_stats(uint64_t out[2][3][2]) { for (int j = 0; j < 2; j++) out[i][k][j] = __atomic_exchange_n(&flush_stat[i][k][j], 0, __ATOMIC_RELAXED); } -// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent +// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent; a +// back end with device-side copies hears of it (nvn_pool_flushed) NVN_API(nvnMemoryPoolFlushMappedRange) { - if (rt_trace & RT_TRACE_PERF) flush_note(0, arg_p(c, 0), arg_x(c, 2)); + const pool_t *p = arg_p(c, 0); + int64_t off = (int64_t)arg_x(c, 1); + uint64_t size = arg_x(c, 2); + if (rt_trace & RT_TRACE_PERF) flush_note(0, p, size); + if (p && off >= 0 && (uint64_t)off < p->size) nvn_pool_flushed(p, (uint64_t)off, size < p->size - (uint64_t)off ? size : p->size - (uint64_t)off); } // NVNmemoryPoolFlags nvnMemoryPoolGetFlags(const NVNmemoryPool *p) NVN_API(nvnMemoryPoolGetFlags) { ret_x(c, ((const pool_t *)arg_p(c, 0))->flags); } diff --git a/runtime/nvn_ext.c b/runtime/nvn_ext.c index 3e31ad0d..11346faf 100644 --- a/runtime/nvn_ext.c +++ b/runtime/nvn_ext.c @@ -20,6 +20,8 @@ void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size) { gfx_backend()->guest_read((uintptr_t)p->mem + off, size); } +__attribute__((weak)) void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { (void)p, (void)off, (void)size; } + // ---------- debug labels, markers, groups, checkpoints: meaningful only to NVN debugging tools // void nvnSetDebugLabel(NVN *o, const char *label) NVN_API(nvnDeviceSetDebugLabel) { (void)c; } diff --git a/runtime/vk.h b/runtime/vk.h index 69dc6449..a2df85ed 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -138,6 +138,30 @@ void vk_mem_count(int cls, int use, uint64_t bytes); void vk_mem_bound_note(void *pool); #define VK_MEM_BOUND(sp) do { if ((rt_trace & RT_TRACE_PERF) && (sp).pool) vk_mem_bound_note((sp).pool); } while (0) void vk_mem_dirty_submit(void); // vk_sync_submit, under its lock, before the replay +// VRAM mirror of buffer-bound pools (vk_memory.c, build/mem-plan.md). vk_mirror_on: TOTK_MEMORY=mirror|verify, or the +// device class is discrete and TOTK_MEMORY is unset or auto; set at device selection. The replay (submitting thread +// under vk_sync_submit's lock) calls: +// - VK_MEM_USE at a vertex, index, uniform, storage or indirect binding of a span vk_mem_resolve returned: the pool's +// mirror is created on first use and sp.buffer becomes it (same offsets); a pool that stays imported keeps its buffer. +// - VK_MEM_PICK where a copy, fill or update reads or writes a span: the mirror if the pool has one, never a new one. +// - vk_mem_gpu_wrote after recording a GPU write of [sp.offset, sp.offset + size) into sp.buffer: a mirror's bytes go +// back to guest memory at the end of the submission. No-op for imported buffers. +// - vk_mem_mirror_begin at the start of a submission's recording, vk_mem_mirror_end (records the write-backs; returns +// the commands recorded) after its last barrier. +// vk_sync.c records the copies the mirrors' first accesses queued (vk_mem_mirror_pending) into a prologue command +// buffer with vk_mem_mirror_prologue and submits it right before each part. +extern int vk_mirror_on; +void vk_mem_use(VkMemSpan *sp); +void vk_mem_pick(VkMemSpan *sp); +#define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0) +#define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0) +void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size); +void vk_mem_mirror_begin(void); +int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); +int vk_mem_mirror_pending(void); +void vk_mem_mirror_prologue(VkCommandBuffer cb); +int vk_mem_mirror_probe(uint64_t addr); // --probe-vkexec: the pool at addr gets its mirror as on a first buffer use; + // 1 when it has one void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m); // once int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked) void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and, @@ -223,6 +247,7 @@ void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); / void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads); void vk_texture_shutdown(void); // vk_mem_shutdown calls it void vk_texture_new_submit(void); // a queue submit starts: draw bindings check storage again +uint32_t vk_texture_epoch(void); // the write-watch snapshot epoch vk_texture_new_submit started VkSampler vk_sampler_by_id(int id); // VK_NULL_HANDLE for an empty slot VkSampler vk_sampler_for(const samp_par *s); // cached by content int vk_sampler_params(int id, samp_par *out); // parameters of a sampler slot; 0 for an empty slot @@ -237,6 +262,7 @@ void vk_sync_start(void); // vk_init (fatal without a device) a uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted uint64_t vk_sync_fence_done(void); // highest done value +uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1) void vk_sync_finish(void); // every submitted value done void vk_sync_shutdown(void); // finish, stop the completion thread, free the command buffers // The op-stream replay (vk_exec.c: T6c): records the command handles of one submission into cb, which the caller has diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index 3b41a46a..d3e61202 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -399,6 +399,7 @@ static void buf_snap(VkExec *ex, int pass) { if (!bd_size) return; VkMemSpan s; if (!vk_mem_resolve(bd_addr, bd_size, &s) || s.avail < bd_size) return; + VK_MEM_PICK(s); // the GPU's view of the buffer BufJob *j = calloc(1, sizeof *j); if (!j) rt_fatal("vk_dump: out of memory"); VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bd_size, diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 7cdbe4ad..ae6cd861 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -688,6 +688,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_frame.submits++; split_draws = part_start = 0; x_here = 1; + if (n) vk_mem_mirror_begin(); for (int i = 0; i < n; i++) { if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i); walk(ex, nvn_op_first(handles[i]), 0); @@ -695,6 +696,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_pass_end(ex); x_here = 0; if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes + if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence); // VRAM mirror bytes the GPU wrote go back to guest memory if (ex->nviews) { ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); if (!l) rt_fatal("vk: out of memory"); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 8fd5ca0b..aca46658 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -569,16 +569,20 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) { return (VkDescriptorBufferInfo){Z.zbuf, 0, max < ZERO_BYTES ? max : ZERO_BYTES}; } -// A buffer descriptor for [addr, addr+size) of guest memory: its backing when aligned, a copy in transient memory when -// the backing is host memory with an unaligned offset. 0 when neither works. -static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, VkDescriptorBufferInfo *out) { +// A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when +// aligned, a copy in transient memory when the backing is host memory with an unaligned offset. 0 when neither works. +// written: the shader stores into it (a mirror's range goes back to guest memory at the end of the submission). +static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, int written, + VkDescriptorBufferInfo *out) { VkMemSpan sp; if (!addr || !vk_mem_resolve(addr, 1, &sp)) return 0; uint64_t n = size && size < sp.avail ? size : sp.avail; if (n > max) n = max; if (!n) return 0; if (sp.offset % align == 0) { + VK_MEM_USE(sp); *out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n}; + if (written) vk_mem_gpu_wrote(&sp, n); vk_mem_count(sp.cls, use, n); VK_MEM_BOUND(sp); return 1; @@ -665,11 +669,12 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { uint32_t len, c1; if (find_stage(ex->prog, st, &ctl, &code, &len)) { memcpy(&c1, ctl + 0x700, 4); - if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, &bi)) return bi; + if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, 0, &bi)) + return bi; } } else if (slot >= 3 && slot - 3 < 18) { const p_bind_buf *b = &ex->ubo[st][slot - 3]; - if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, &bi)) return bi; + if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, 0, &bi)) return bi; } return zero_info(max); } @@ -699,7 +704,8 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * const p_bind_buf *b = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot]; // vk_caps.c: shaders translated for sbuf_align address the buffer from its address rounded down to it. uint64_t lead = vk_caps.sbuf_align ? b->addr & (vk_caps.sbuf_align - 1) : 0; - if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO, &bi)) { + if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO, + written, &bi)) { if (written && bi.buffer != VK_NULL_HANDLE) note_written(ex, b->addr - lead, bi.range); return bi; } @@ -1095,6 +1101,7 @@ static void bind_vertex_buffers(VkExec *ex) { const NVNbufferRange *r = &ex->vbuf[b]; VkMemSpan sp; if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue; + VK_MEM_USE(sp); uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail; vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool; } @@ -1271,6 +1278,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { } else if (sp.offset % isize[d->itype]) { return warn_once(W_INDEX, "index buffer %#llx is not aligned to its index size: draw skipped", (unsigned long long)d->ibuf); } else { + VK_MEM_USE(sp); vkCmdBindIndexBuffer(ex->cb, sp.buffer, sp.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); ex->recorded++; vk_mem_count(sp.cls, VK_MUSE_IB, bytes); @@ -1295,6 +1303,8 @@ static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) { return warn_once(W_INDEX, "index buffer %#llx is not backed or not aligned: indirect draws skipped", (unsigned long long)d->ibuf); if (!prepare_draw(ex, d->prim, 0x7fffffff, restart_for(ex, d->prim, d->itype))) return; vk_exec_frame.draws++; + VK_MEM_USE(ip); + VK_MEM_USE(ib); vkCmdBindIndexBuffer(ex->cb, ib.buffer, ib.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); vkCmdDrawIndexedIndirect(ex->cb, ip.buffer, ip.offset, 1, sizeof(VkDrawIndexedIndirectCommand)); ex->recorded += 2; @@ -1341,6 +1351,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { VkPipeline p = vk_pipeline_compute(cs, &info); if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped"); vk_exec_barrier(ex); + VK_MEM_USE(ip); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); c0_dim[0] = c0_dim[1] = c0_dim[2] = 0; B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads @@ -1368,8 +1379,10 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { VkBuffer b; VkDeviceSize o; memcpy(scratch(ex, u->size, &b, &o), u->data, u->size); + VK_MEM_PICK(sp); VkBufferCopy cp = {o, sp.offset, u->size}; vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp); + vk_mem_gpu_wrote(&sp, u->size); ex->recorded++; vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size); note_written(ex, u->dst, u->size); diff --git a/runtime/vk_exec_probe.c b/runtime/vk_exec_probe.c index 7b83d323..0396501f 100644 --- a/runtime/vk_exec_probe.c +++ b/runtime/vk_exec_probe.c @@ -802,6 +802,54 @@ static int test_nvnx(const GfxBackend *b, uint8_t *guest, int *bytes) { return 0; } +// VRAM mirror (vk_memory.c; skipped in import mode): a pool given its mirror as on a first buffer use, after which +// buffer copies and fills read and write the mirror. The first use copies the whole pool; CPU writes between +// submissions reach the mirror through write watch; GPU writes reach guest memory by the fence, and only the bytes +// written: CPU bytes beside a GPU fill keep their values. +static int test_mirror(const GfxBackend *b, uint8_t *guest, int *bytes) { + *bytes = 0; + if (!vk_mirror_on) return 0; + pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x22, .mem = guest + 0xC00000, .size = 0x100000, + .game_mem = guest + 0xC00000}; + CHECK(b->pool_init(&pool), "pool_init of a CPU-visible pool for the mirror part"); + uint8_t *m = pool.mem; + for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 13 + 1); + memset(m + 0x1000, 0xEE, 0x5000); + CHECK(vk_mem_mirror_probe((uint64_t)(uintptr_t)m), "the pool got no VRAM mirror"); + static _Alignas(8) uint8_t stream[1024]; + uint64_t a = (uint64_t)(uintptr_t)m; + // 1: B2B in the mirror after its first-use copy, and a fill; both written back. + uint8_t *p = put(stream, OP_BEGIN, NULL, 0); + const p_copy_b2b c1 = {a, a + 0x1000, 4096}; + const p_clear_buf f1 = {a + 0x2000, 0x800, 0x5a5a5a5au, 0}; + p = put(p, OP_COPY_B2B, &c1, sizeof c1); + p = put(p, OP_CLEAR_BUF, &f1, sizeof f1); + CHECK(xf_submit(b, stream, p), "fence_wait of the first mirror submission timed out"); + for (int i = 0; i < 4096; i++) CHECK(m[0x1000 + i] == m[i], "mirror B2B byte %d is %02x, expected %02x", i, m[0x1000 + i], m[i]); + for (int i = 0; i < 0x800; i++) CHECK(m[0x2000 + i] == 0x5a, "mirror fill byte %d is %02x", i, m[0x2000 + i]); + for (int i = 0x800; i < 0x1000; i++) CHECK(m[0x2000 + i] == 0xEE, "byte %#x past the mirror fill is %02x", i, m[0x2000 + i]); + // 2: the CPU rewrites the source; the next submission copies the new bytes. + for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 7 + 5); + p = put(stream, OP_BEGIN, NULL, 0); + const p_copy_b2b c2 = {a, a + 0x3000, 4096}; + p = put(p, OP_COPY_B2B, &c2, sizeof c2); + CHECK(xf_submit(b, stream, p), "fence_wait of the second mirror submission timed out"); + for (int i = 0; i < 4096; i++) + CHECK(m[0x3000 + i] == m[i], "B2B byte %d after a CPU write is %02x, expected %02x (write not mirrored)", i, + m[0x3000 + i], m[i]); + // 3: CPU bytes in the page of a GPU fill, written before the submission, survive the write-back. + memset(m + 0x4000, 0x11, 0x800); + p = put(stream, OP_BEGIN, NULL, 0); + const p_clear_buf f3 = {a + 0x4800, 0x800, 0x77777777u, 0}; + p = put(p, OP_CLEAR_BUF, &f3, sizeof f3); + CHECK(xf_submit(b, stream, p), "fence_wait of the third mirror submission timed out"); + for (int i = 0; i < 0x800; i++) CHECK(m[0x4000 + i] == 0x11, "CPU byte %#x beside a fill is %02x", i, m[0x4000 + i]); + for (int i = 0; i < 0x800; i++) CHECK(m[0x4800 + i] == 0x77, "fill byte %#x is %02x", i, m[0x4800 + i]); + b->pool_fini(&pool); + *bytes = 4096 + 0x1000 + 4096 + 0x1000; + return 0; +} + int probe_vkexec(void) { layouts_equal = 0; main_tid = GetCurrentThreadId(); @@ -819,8 +867,9 @@ int probe_vkexec(void) { if (!bad) bad = test_xfer(b, guest, &xfer_bytes); int zero_bytes = 0; if (!bad) bad = test_c0() || test_zero(b, guest, &zero_bytes); - int nvnx_bytes = 0; + int nvnx_bytes = 0, mirror_bytes = 0; if (!bad) bad = test_nvnx(b, guest, &nvnx_bytes); + if (!bad) bad = test_mirror(b, guest, &mirror_bytes); b->finish(); vk_pipeline_shutdown(); // the pipelines and the draw part's stand-ins go before the device vk_exec_draw_shutdown(); @@ -860,5 +909,10 @@ int probe_vkexec(void) { printf("probe-vkexec: c0 and zero parts passed (c0 tables, dispatch size, point size and base instance at their offsets, " "unbound slots zero; %d bytes of an unwritten physical-pool texture read zero, TOTK_FIX_ZERO=%s)\n", zero_bytes, cfg_get("TOTK_FIX_ZERO") ? cfg_get("TOTK_FIX_ZERO") : "1"); + if (vk_mirror_on) + printf("probe-vkexec: mirror part passed (B2B and fill in a VRAM mirror written back by the fence, a CPU write " + "between submissions mirrored, CPU bytes beside a fill kept, %d bytes compared)\n", mirror_bytes); + else + printf("probe-vkexec: mirror part skipped (memory mode import)\n"); return 0; } diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index f33ecf60..de6ac5b8 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -379,9 +379,10 @@ static VkExtent3D sub_extent(const Sub *s) { return (VkExtent3D){s->w, s->h, s->img->type == VK_IMAGE_TYPE_3D ? s->nz : 1}; } -// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span. +// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span (a VRAM mirror when the pool +// has one). static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *size, VkBufferImageCopy *c, - VkBuffer *buf, int *cls) { + VkMemSpan *span) { uint32_t wb = (s->w + (uint32_t)s->bw - 1) / (uint32_t)s->bw, hb = (s->h + (uint32_t)s->bh - 1) / (uint32_t)s->bh; uint64_t row = ex->copy_stride > 0 ? (uint64_t)ex->copy_stride : (uint64_t)wb * (uint64_t)s->bpb; if (row % (uint64_t)s->bpb || row < (uint64_t)wb * (uint64_t)s->bpb) { @@ -397,8 +398,8 @@ static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t * (unsigned long long)addr, (unsigned long long)*size); return 0; } - *buf = m.buffer; - *cls = m.cls; + VK_MEM_PICK(m); + *span = m; *c = (VkBufferImageCopy){.bufferOffset = m.offset, .bufferRowLength = (uint32_t)(row / (uint64_t)s->bpb) * (uint32_t)s->bw, .bufferImageHeight = hb * (uint32_t)s->bh, .imageSubresource = sub_layers(s), .imageOffset = sub_offset(s), .imageExtent = sub_extent(s)}; @@ -410,17 +411,16 @@ static void copy_buffer_image(VkExec *ex, const p_copy_b2t *p, int to_image) { if (!sub_resolve(p->tex, &p->view, p->has_view, &p->r, 1, &s)) return; uint64_t size; VkBufferImageCopy c; - VkBuffer buf; - int cls; - if (!buffer_side(ex, &s, p->src, &size, &c, &buf, &cls)) return; - vk_mem_count(cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size); + VkMemSpan m; + if (!buffer_side(ex, &s, p->src, &size, &c, &m)) return; + vk_mem_count(m.cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size); vk_exec_pass_end(ex); vk_exec_barrier(ex); - if (to_image) vkCmdCopyBufferToImage(ex->cb, buf, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c); - else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, buf, 1, &c); + if (to_image) vkCmdCopyBufferToImage(ex->cb, m.buffer, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c); + else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, m.buffer, 1, &c); ex->recorded++; if (to_image) vk_texture_gpu_written(p->tex); - else note_gpu_write(ex, p->src, size); + else vk_mem_gpu_wrote(&m, size), note_gpu_write(ex, p->src, size); } static void copy_t2t(VkExec *ex, const p_copy_t2t *p) { @@ -476,6 +476,8 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) { } uint64_t n = left < s.avail ? left : s.avail; if (n > d.avail) n = d.avail; + VK_MEM_PICK(s); + VK_MEM_PICK(d); if (s.buffer == d.buffer && s.offset < d.offset + n && d.offset < s.offset + n) { warn_once(W_OVERLAP, "buffer copy %#llx -> %#llx overlaps within one buffer: skipped", (unsigned long long)p->src, (unsigned long long)p->dst); @@ -484,6 +486,7 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) { if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1; VkBufferCopy bc = {s.offset, d.offset, n}; vkCmdCopyBuffer(ex->cb, s.buffer, d.buffer, 1, &bc); + vk_mem_gpu_wrote(&d, n); ex->recorded++; vk_mem_count(s.cls, VK_MUSE_COPY_SRC, n); vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n); @@ -514,7 +517,9 @@ static void clear_buf(VkExec *ex, const p_clear_buf *p) { uint64_t n = left < d.avail ? left : d.avail & ~3ull; if (!n) break; if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1; + VK_MEM_PICK(d); vkCmdFillBuffer(ex->cb, d.buffer, d.offset, n, p->value); + vk_mem_gpu_wrote(&d, n); ex->recorded++; vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n); dst += n, left -= n; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ebda951c..58b5e4bc 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -24,6 +24,7 @@ // ---------------------------------------------------------------- pools enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT }; +enum { MS_NONE, MS_ON, MS_REFUSED }; typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page -> physical pool and offset @@ -44,8 +45,22 @@ typedef struct PoolRec { uint32_t synced_gen; // page generation its lazy-mirror count covers; DIRTY_UNSYNCED until scanned uint8_t bufbound; // ever bound as a vertex, index, uniform, storage or indirect buffer uint8_t untracked; // the last dirty scan found no write watch over it + // VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only. + uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED + VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf + VmaAllocation malloc_; + uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync) + uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory + uint32_t mgen; // gen_lock: every page write stamped up to this generation is in the mirror + // (or in a copy queued before its next use) + uint64_t fl_lo, fl_hi; // flush_lock: nvnMemoryPoolFlushMappedRange bytes not copied yet (fl_hi 0: none) } PoolRec; +static void mirror_forget(PoolRec *r); // pool_lock exclusive: off the mirrored list +static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown) +static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record +static void grow(void **arr, int *cap, size_t elem); + static SRWLOCK pool_lock = SRWLOCK_INIT; // Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits // stay valid while it is unchanged. @@ -197,6 +212,7 @@ static PoolRec *phys_find(const pool_t *key) { } static void destroy_rec(VkCtx *c, PoolRec *r) { + if (r->mbuf) vmaDestroyBuffer(c->vma, r->mbuf, r->malloc_); if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL); if (r->mem) vkFreeMemory(c->device, r->mem, NULL); if (r->alloc) vmaFreeMemory(c->vma, r->alloc); @@ -259,6 +275,7 @@ void vk_pool_fini(pool_t *p) { int had = pidx_remove(&pool_index, p, &e); __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); PoolRec *r = had ? e.buf : NULL; + if (r && r->mstate == MS_ON) mirror_forget(r); if (r && r->kind == POOL_PHYS) { for (int i = 0; i < nphys; i++) if (phys_list[i] == r) { phys_list[i] = phys_list[--nphys]; break; } @@ -281,7 +298,10 @@ void vk_pool_fini(pool_t *p) { vk_queue_unlock(); } vk_mem_collect(); - destroy_rec(c, r); + // A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may + // name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued). + if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r); + else destroy_rec(c, r); } // Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c:54-67); apply them in order. @@ -414,9 +434,12 @@ static void flag_names(uint32_t f, char *buf, size_t n) { // run-game saves), at frame 600, and with TOTK_TRACE=perf every 1800 frames. static void dirty_frame(uint64_t frame); +static void mirror_log(uint64_t frame); + void vk_mem_census(uint64_t frame) { static uint64_t window_start; - if (rt_trace & RT_TRACE_PERF) dirty_frame(frame); + if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); + if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame); if (frame == 1200) { // loading is over: the first window starts here for (int c = 0; c < VK_MCLS_N; c++) for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0); @@ -497,6 +520,10 @@ static uint32_t *gen_chunk[GEN_CHUNKS]; static uint32_t gen_now; // last generation handed out static SRWLOCK gen_lock = SRWLOCK_INIT; static uint64_t gen_pages[1024]; +// The newest generation stamped on any page of each 2 MiB region (the write-watch snapshot regions below): the mirror +// sync skips regions with nothing newer than its pool's mirror. +#define REG_SHIFT 21 +static uint32_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT]; static uint32_t *gen_slot(uint64_t page_addr, int create) { uint64_t ci = page_addr >> GEN_CHUNK_SHIFT; @@ -509,11 +536,16 @@ static uint32_t *gen_slot(uint64_t page_addr, int create) { return &gen_chunk[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)]; } +// gen_lock held exclusively; g is the newest generation handed out. +static void stamp_page(uint64_t a, uint32_t g) { + uint32_t *s = gen_slot(a, 1); + if (!s) return; + *s = g; + reg_max[a >> REG_SHIFT] = g; +} + static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) { - for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) { - uint32_t *s = gen_slot(a, 1); - if (s) *s = g; - } + for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page(a, g); } void vk_guest_written(uint64_t addr, uint64_t size) { @@ -533,10 +565,7 @@ int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now) for (;;) { uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages); if (n == UINT64_MAX) { tracked = 0; break; } - for (uint64_t k = 0; k < n; k++) { - uint32_t *s = gen_slot(gen_pages[k], 1); - if (s) *s = g; - } + for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g); if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call } int changed = 0; @@ -560,6 +589,31 @@ static uint32_t snap_at[SNAP_REGIONS]; // epoch of the region's last snapsh static uint8_t snap_untracked[SNAP_REGIONS]; static uint64_t n_snap_checks, n_snap_calls, n_snap_untracked; +// gen_lock held exclusively. Queries the regions r0..r1 that were not snapped in epoch, one query per run of them; +// returns the number of queries. +static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) { + uint32_t g = 0; + uint64_t calls = 0; + for (uint64_t r = r0; r <= r1;) { + if (snap_at[r] == epoch) { r++; continue; } + uint64_t k = r; // one query over a run of regions not seen in this epoch + while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++; + uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT; + int tracked = 1; + if (!g) g = ++gen_now; + for (;;) { + uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages); + if (n == UINT64_MAX) { tracked = 0; break; } + for (uint64_t j = 0; j < n; j++) stamp_page(gen_pages[j], g); + if (n < sizeof gen_pages / sizeof *gen_pages) break; + } + calls++; + for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked; + r = k + 1; + } + return calls; +} + int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) { if (!size) { *now = since; return 0; } uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; @@ -572,26 +626,7 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e if (excl) { ReleaseSRWLockShared(&gen_lock); AcquireSRWLockExclusive(&gen_lock); - uint32_t g = ++gen_now; - for (uint64_t r = r0; r <= r1;) { - if (snap_at[r] == epoch) { r++; continue; } - uint64_t k = r; // one query over a run of regions not seen in this epoch - while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++; - uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT; - int tracked = 1; - for (;;) { - uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages); - if (n == UINT64_MAX) { tracked = 0; break; } - for (uint64_t j = 0; j < n; j++) { - uint32_t *s = gen_slot(gen_pages[j], 1); - if (s) *s = g; - } - if (n < sizeof gen_pages / sizeof *gen_pages) break; - } - n_snap_calls++; - for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked; - r = k + 1; - } + n_snap_calls += snap_regions(r0, r1, epoch); } int untracked = 0, changed = 0; for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r]; @@ -663,16 +698,13 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { for (;;) { uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages); if (n == UINT64_MAX) return 0; - for (uint64_t k = 0; k < n; k++) { - uint32_t *s = gen_slot(gen_pages[k], 1); - if (s) *s = g; - } + for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g); if (n < sizeof gen_pages / sizeof *gen_pages) return 1; } } void vk_mem_dirty_submit(void) { - if (!(rt_trace & RT_TRACE_PERF)) return; + if (!(rt_trace & RT_TRACE_PERF) || vk_mirror_on) return; // the mirror's own line replaces the census uint64_t t0 = kern_now_ns(); uint32_t prev = dirty_sub; // the submit whose replay noted the bindings dirty_sub = prev + 1 ? prev + 1 : 1; @@ -793,8 +825,505 @@ static void dirty_frame(uint64_t frame) { rt_log("%s", line); } +// ---------------------------------------------------------------- VRAM mirror of buffer-bound pools +// build/mem-plan.md, "Design: VRAM mirror of buffer-bound pools". An imported pool keeps vertices, indices, constants +// and storage buffers in system memory, so a discrete GPU reads them over PCIe on every draw (GTX 1050 Ti and RTX +// 3090 Ti: about 110 ms of GPU time per frame in the field). In mirror mode (device class discrete, TOTK_MEMORY) an +// imported pool gets a device-local buffer over its imported range the first time the replay uses it as a vertex, +// index, uniform, storage or indirect buffer (vk_mem_use). From then on those bindings, and the buffer copies, fills +// and uniform updates that touch the pool (vk_mem_pick), use the mirror at the same offsets. Guest memory stays what +// the CPU reads and writes; textures and every host path that reads or writes pool bytes keep using it. +// - CPU -> mirror: the first access of a submission to a mirror syncs it (mirror_sync): the 2 MiB write-watch +// snapshots of its regions for this epoch (snap_regions, shared with the texture checks: the OS bits keep one +// consumer, the shared page generations) and every page stamped after the mirror's mgen goes into a copy; so do the +// nvnMemoryPoolFlushMappedRange ranges. The first use copies the whole range. The copies run in a prologue command +// buffer that vk_sync.c submits right before the part that accessed the mirror first, between full barriers. +// vk_mem_mirror_begin snaps the regions of the pools the previous submission accessed in a few merged queries. +// - GPU -> guest: what the replay writes into a mirror (storage buffers its shaders write, buffer copies, fills, +// uniform updates, texture-to-buffer copies: vk_mem_gpu_wrote) goes back into the imported buffer at the end of the +// submission, exact byte ranges, before the fence signals (vk_mem_mirror_end). +// - TOTK_MEMORY=verify: every submission also copies up to VERIFY_BYTES of mirror pages, round robin, into host memory; +// at its fence the pages the CPU has not written since their sync are compared with guest memory. +// A pool stays imported when its mirror would not fit the VRAM budget, its range is not write-watched, it overlaps +// another pool or the allocation fails. +#define VERIFY_BYTES (4u << 20) +#define VERIFY_SAMPLES 64 +#define VERIFY_LOG 16 +enum { MR_BUDGET, MR_UNTRACKED, MR_OVERLAP, MR_ALLOC, MR_N }; + +int vk_mirror_on; +static int mirror_verify; +static int mirror_gap = 2; // pre-snap: hot pools at most this many 2 MiB regions apart share one query +static PoolRec **mlist; // pool_lock: mirrored pools by import_lo +static int nm, mcap; +static uint32_t msub; // replay: number of the submission being recorded +static uint32_t mepoch; // replay: its write-watch snapshot epoch (vk_texture_epoch) +typedef struct { PoolRec *r; uint64_t off, size; } MRange; +static MRange *pend, *wbl; // replay: copies for the next prologue; GPU writes into mirrors this submission +static int npend, cpend, nwbl, cwbl; +static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions +static int cregions; +static SRWLOCK flush_lock = SRWLOCK_INIT; + +typedef struct { + uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges; + uint64_t presnap_queries, sync_queries, cpu_ns, prologues; +} MirrorAcc; +static MirrorAcc mrep; // replay: this submission +static MirrorAcc macc; // mirror_stat_lock: since the last summary +static SRWLOCK mirror_stat_lock = SRWLOCK_INIT; +static volatile LONG64 m_live_n, m_live_bytes, m_refused_n[MR_N], m_refused_bytes[MR_N]; +static volatile LONG64 m_flush_calls, m_flush_bytes, v_pages, v_skipped, v_mismatch; + +static void mirror_mode(int discrete) { + const char *e = cfg_get("TOTK_MEMORY"); + int mode = discrete ? 1 : 0; // 0 import, 1 mirror, 2 verify + const char *why = discrete ? "auto, discrete gpu" : "auto, uma"; + if (e && *e && strcmp(e, "auto")) { + if (!strcmp(e, "import")) mode = 0, why = "TOTK_MEMORY=import"; + else if (!strcmp(e, "mirror")) mode = 1, why = "TOTK_MEMORY=mirror"; + else if (!strcmp(e, "verify")) mode = 2, why = "TOTK_MEMORY=verify"; + else rt_log("mem: TOTK_MEMORY=%s is not import, mirror, verify or auto: auto", e); + } + vk_mirror_on = mode > 0; + mirror_verify = mode == 2; + const char *g = cfg_get("TOTK_MIRROR_GAP"); + mirror_gap = g && *g ? atoi(g) : 2; + if (mirror_gap < 0) mirror_gap = 0; + rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why, + mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory"); +} + +static void range_add(MRange **a, int *n, int *cap, PoolRec *r, uint64_t off, uint64_t size) { + if (*n && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) { + (*a)[*n - 1].size += size; + return; + } + if (*n == *cap) grow((void **)a, cap, sizeof **a); + (*a)[(*n)++] = (MRange){r, off, size}; +} + +static int range_cmp(const void *x, const void *y) { + const MRange *a = x, *b = y; + if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1; + return a->off < b->off ? -1 : a->off > b->off; +} + +// Records the ranges as one vkCmdCopyBuffer per pool, overlapping and adjacent ranges merged; to_mirror: buf -> mbuf, +// else mbuf -> buf. Returns the commands recorded; adds what they move to *bytes and *copies. +static int copy_ranges(VkCommandBuffer cb, MRange *a, int n, int to_mirror, uint64_t *bytes, uint64_t *copies) { + qsort(a, (size_t)n, sizeof *a, range_cmp); + int rec = 0; + for (int i = 0; i < n;) { + PoolRec *r = a[i].r; + uint64_t lim = r->import_hi - r->import_lo; + int k = 0; + for (; i < n && a[i].r == r; i++) { + uint64_t lo = a[i].off, hi = a[i].off + a[i].size; + if (hi > lim) hi = lim; + if (lo >= hi) continue; + if (k && lo <= mregions[k - 1].srcOffset + mregions[k - 1].size) { + uint64_t end = mregions[k - 1].srcOffset + mregions[k - 1].size; + if (hi > end) mregions[k - 1].size += hi - end; + continue; + } + if (k == cregions) grow((void **)&mregions, &cregions, sizeof *mregions); + mregions[k++] = (VkBufferCopy){lo, lo, hi - lo}; + } + if (!k) continue; + if (to_mirror) vkCmdCopyBuffer(cb, r->buf, r->mbuf, (uint32_t)k, mregions); + else vkCmdCopyBuffer(cb, r->mbuf, r->buf, (uint32_t)k, mregions); + rec++; + for (int j = 0; j < k; j++) *bytes += mregions[j].size; + *copies += (uint64_t)k; + } + return rec; +} + +static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, + VkAccessFlags2 da) { + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = ss, .srcAccessMask = sa, + .dstStageMask = ds, .dstAccessMask = da}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); +} + +static void refuse(PoolRec *r, int why, const char *what) { + r->mstate = MS_REFUSED; + uint64_t n = r->import_hi - r->import_lo; + if (InterlockedIncrement64(&m_refused_n[why]) <= 4) { + char f[160]; + flag_names(r->flags, f, sizeof f); + rt_log("mem: pool %#llx+%#llx (%s) stays imported: %s", (unsigned long long)r->lo, (unsigned long long)r->size, + f, what); + } + InterlockedAdd64(&m_refused_bytes[why], (LONG64)n); +} + +// Replay thread: the pool's mirror on its first buffer use. +static void mirror_create(PoolRec *r) { + VkCtx *c = vk_ctx(); + uint64_t n = r->import_hi - r->import_lo; + r->mstate = MS_REFUSED; + if (r->kind != POOL_IMPORT || !r->buf || !n) return; + AcquireSRWLockShared(&pool_lock); + int i = pidx_upper(&pool_index, r->lo) - 1; + while (i >= 0 && pool_index.e[i].buf != r) i--; + int alone = i >= 0 && (i == 0 || pool_index.maxhi[i - 1] <= pool_index.e[i].lo) && + (i + 1 == pool_index.n || pool_index.e[i + 1].lo >= pool_index.e[i].hi); + ReleaseSRWLockShared(&pool_lock); + if (i < 0) return; // finalized meanwhile + if (!alone) return refuse(r, MR_OVERLAP, "it overlaps another pool"); + uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT; + if (r1 >= SNAP_REGIONS) return refuse(r, MR_UNTRACKED, "outside the write-watched address space"); + uint64_t used, budget; + int exact; + if (vk_mem_vram(&used, &budget, &exact) && used + n + budget / 8 > budget) { + char why[128]; + snprintf(why, sizeof why, "its %.1f MiB mirror does not fit the VRAM budget (%.0f of %.0f MiB in use)", + n / 1048576.0, used / 1048576.0, budget / 1048576.0); + return refuse(r, MR_BUDGET, why); + } + AcquireSRWLockExclusive(&gen_lock); + mrep.sync_queries += snap_regions(r0, r1, mepoch); + int untracked = 0; + for (uint64_t k = r0; k <= r1; k++) untracked |= snap_untracked[k]; + r->mgen = gen_now; // the whole-range copy below holds every write observed so far + ReleaseSRWLockExclusive(&gen_lock); + if (untracked) return refuse(r, MR_UNTRACKED, "its range is not write-watched"); + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = n, .usage = pool_usage(c), + .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, + .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + VkResult res = vmaCreateBuffer(c->vma, &bi, &ai, &r->mbuf, &r->malloc_, NULL); + if (res != VK_SUCCESS) { + r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL; + char why[96]; + snprintf(why, sizeof why, "device-local allocation of %.1f MiB failed (%d)", n / 1048576.0, (int)res); + return refuse(r, MR_ALLOC, why); + } + AcquireSRWLockExclusive(&pool_lock); + if (nm == mcap) grow((void **)&mlist, &mcap, sizeof *mlist); + int at = nm; + while (at > 0 && mlist[at - 1]->import_lo > r->import_lo) at--; + memmove(mlist + at + 1, mlist + at, (size_t)(nm - at) * sizeof *mlist); + mlist[at] = r; + nm++; + ReleaseSRWLockExclusive(&pool_lock); + r->acc_sub = msub; + r->mstate = MS_ON; + range_add(&pend, &npend, &cpend, r, 0, n); + mrep.first_n++, mrep.first_bytes += n; + InterlockedIncrement64(&m_live_n); + InterlockedAdd64(&m_live_bytes, (LONG64)n); +} + +static void mirror_forget(PoolRec *r) { + for (int i = 0; i < nm; i++) + if (mlist[i] == r) { + memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist); + nm--; + InterlockedDecrement64(&m_live_n); + InterlockedAdd64(&m_live_bytes, -(LONG64)(r->import_hi - r->import_lo)); + break; + } +} + +// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran. +static PoolRec **retiring; +static int nretiring, cretiring; +static SRWLOCK retire_lock = SRWLOCK_INIT; + +static void mirror_retiring(PoolRec *r) { + AcquireSRWLockExclusive(&retire_lock); + if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring); + retiring[nretiring++] = r; + ReleaseSRWLockExclusive(&retire_lock); +} + +static void mirror_retire(void *arg) { + AcquireSRWLockExclusive(&retire_lock); + for (int i = 0; i < nretiring; i++) + if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; } + ReleaseSRWLockExclusive(&retire_lock); + destroy_rec(vk_ctx(), arg); +} + +// Replay thread, the first access of this submission: queues the copies of the pages written since the last sync. +static void mirror_sync(PoolRec *r) { + uint64_t t0 = kern_now_ns(); + r->acc_sub = msub; + uint64_t lo = r->import_lo, hi = r->import_hi; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; + int excl = 0; + AcquireSRWLockShared(&gen_lock); + for (uint64_t k = r0; k <= r1; k++) + if (snap_at[k] != mepoch) { excl = 1; break; } + if (excl) { + ReleaseSRWLockShared(&gen_lock); + AcquireSRWLockExclusive(&gen_lock); + mrep.sync_queries += snap_regions(r0, r1, mepoch); + } + uint32_t since = r->mgen; + for (uint64_t k = r0; k <= r1; k++) { + if (reg_max[k] <= since) continue; + uint64_t a = k << SNAP_SHIFT, b = a + (1ull << SNAP_SHIFT); + if (a < lo) a = lo; + if (b > hi) b = hi; + const uint32_t *ch = gen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk + if (!ch) continue; + for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) + if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since) + range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT); + } + r->mgen = gen_now; + if (excl) ReleaseSRWLockExclusive(&gen_lock); + else ReleaseSRWLockShared(&gen_lock); + if (__atomic_load_n(&r->fl_hi, __ATOMIC_ACQUIRE)) { + AcquireSRWLockExclusive(&flush_lock); + uint64_t a = r->fl_lo, b = r->fl_hi; + r->fl_hi = 0; + ReleaseSRWLockExclusive(&flush_lock); + a = a > lo ? a & ~0xfffull : lo; + b = b < hi ? (b + 0xfff) & ~0xfffull : hi; + if (a < b) range_add(&pend, &npend, &cpend, r, a - lo, b - a); + } + mrep.syncs++; + mrep.cpu_ns += kern_now_ns() - t0; +} + +void vk_mem_use(VkMemSpan *sp) { + PoolRec *r = sp->pool; + if (r->mstate == MS_NONE) mirror_create(r); + if (r->mstate != MS_ON) return; + if (r->acc_sub != msub) mirror_sync(r); + sp->buffer = r->mbuf; +} + +void vk_mem_pick(VkMemSpan *sp) { + PoolRec *r = sp->pool; + if (!r || r->mstate != MS_ON) return; + if (r->acc_sub != msub) mirror_sync(r); + sp->buffer = r->mbuf; +} + +int vk_mem_mirror_probe(uint64_t addr) { + VkMemSpan sp; + if (!vk_mirror_on || !vk_mem_resolve(addr, 1, &sp) || !sp.pool) return 0; + mepoch = vk_texture_epoch(); + vk_mem_use(&sp); + return ((PoolRec *)sp.pool)->mstate == MS_ON; +} + +void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { + PoolRec *r = sp->pool; + if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; + range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size); +} + +void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { + if (!vk_mirror_on || !size || !p->mem) return; + InterlockedIncrement64(&m_flush_calls); + InterlockedAdd64(&m_flush_bytes, (LONG64)size); + AcquireSRWLockShared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)p->mem, 0); + PoolRec *r = e && e->key == p ? e->buf : NULL; + if (r && r->mstate == MS_ON) { + uint64_t a = (uintptr_t)p->mem + off, b = a + size; + AcquireSRWLockExclusive(&flush_lock); + if (!r->fl_hi) r->fl_lo = a, r->fl_hi = b; + else { + if (a < r->fl_lo) r->fl_lo = a; + if (b > r->fl_hi) r->fl_hi = b; + } + ReleaseSRWLockExclusive(&flush_lock); + } + ReleaseSRWLockShared(&pool_lock); +} + +void vk_mem_mirror_begin(void) { + if (!vk_mirror_on) return; + uint64_t t0 = kern_now_ns(); + uint32_t prev = msub; + msub = msub + 1 ? msub + 1 : 1; + mepoch = vk_texture_epoch(); + uint64_t q = 0, run0 = 0, run1 = 0; + int have = 0; + AcquireSRWLockShared(&pool_lock); + AcquireSRWLockExclusive(&gen_lock); + for (int i = 0; i < nm; i++) { + const PoolRec *r = mlist[i]; + if (r->acc_sub != prev) continue; + uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT; + if (have && a <= run1 + 1 + (uint64_t)mirror_gap) { + if (b > run1) run1 = b; + continue; + } + if (have) q += snap_regions(run0, run1, mepoch); + run0 = a, run1 = b, have = 1; + } + if (have) q += snap_regions(run0, run1, mepoch); + ReleaseSRWLockExclusive(&gen_lock); + ReleaseSRWLockShared(&pool_lock); + mrep.subs++; + mrep.presnap_queries += q; + mrep.cpu_ns += kern_now_ns() - t0; +} + +int vk_mem_mirror_pending(void) { return npend; } + +void vk_mem_mirror_prologue(VkCommandBuffer cb) { + if (!npend) return; + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies); + mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + npend = 0; + mrep.prologues++; +} + +// ---- verify mode +typedef struct { PoolRec *r; uint64_t off, len, dst; uint32_t mgen; } VSample; +typedef struct { uint32_t sub; int n; VkBuffer buf; VmaAllocation alloc; const uint8_t *map; VSample s[VERIFY_SAMPLES]; } VJob; +static int ver_i; +static uint64_t ver_off; + +// The CPU wrote the page since the mirror's sync that mgen records: stamped later, or its write-watch bit is set. +static int page_dirty(uint64_t a, uint32_t mgen) { + uint32_t *s = gen_slot(a, 0); + if (s && __atomic_load_n(s, __ATOMIC_RELAXED) > mgen) return 1; + uint64_t p[1]; + return mem_written_peek(a, 1ull << GEN_PAGE_SHIFT, p, 1) != 0; // UINT64_MAX (not watched) counts as written +} + +static void verify_cb(void *arg) { // completion thread + VJob *j = arg; + static volatile LONG logged; + uint64_t pages = 0, skipped = 0, bad = 0; + for (int i = 0; i < j->n; i++) { + const VSample *s = &j->s[i]; + PoolRec *r = s->r; + uint32_t wb = __atomic_load_n(&r->wb_sub, __ATOMIC_RELAXED); + if (wb != j->sub && wb - j->sub < 0x80000000u) { // a later submission wrote back into it + skipped += s->len >> GEN_PAGE_SHIFT; + continue; + } + for (uint64_t o = 0; o < s->len; o += 1ull << GEN_PAGE_SHIFT) { + uint64_t a = r->import_lo + s->off + o; + const uint8_t *m = j->map + s->dst + o, *g = (const uint8_t *)(uintptr_t)a; + pages++; + if (!memcmp(m, g, 1ull << GEN_PAGE_SHIFT)) continue; + if (page_dirty(a, s->mgen)) { skipped++; continue; } + bad++; + if (InterlockedIncrement(&logged) <= VERIFY_LOG) { + int k = 0, e = (1 << GEN_PAGE_SHIFT) - 1; + while (k < e && m[k] == g[k]) k++; + while (e > k && m[e] == g[e]) e--; + char f[160]; + flag_names(r->flags, f, sizeof f); + rt_log("mem: verify mismatch: pool %#llx+%#llx (%s) offset %#llx: bytes %#x..%#x of the page differ, " + "mirror %02x guest %02x at the first (submission %u)", (unsigned long long)r->lo, + (unsigned long long)r->size, f, (unsigned long long)(a - r->lo), k, e, m[k], g[k], j->sub); + } + } + } + InterlockedAdd64(&v_pages, (LONG64)pages); + InterlockedAdd64(&v_skipped, (LONG64)skipped); + InterlockedAdd64(&v_mismatch, (LONG64)bad); + vmaDestroyBuffer(vk_ctx()->vma, j->buf, j->alloc); + free(j); +} + +static int verify_record(VkCommandBuffer cb, uint64_t fence) { + VJob *j = calloc(1, sizeof *j); + if (!j) rt_fatal("vk: out of memory"); + uint64_t left = VERIFY_BYTES, dst = 0; + AcquireSRWLockShared(&pool_lock); + for (int tries = 0; nm && left && j->n < VERIFY_SAMPLES && tries <= nm; tries++) { + if (ver_i >= nm) ver_i = 0, ver_off = 0; + PoolRec *r = mlist[ver_i]; + uint64_t size = r->import_hi - r->import_lo; + if (ver_off >= size) { ver_i++, ver_off = 0; continue; } + uint64_t len = size - ver_off < left ? size - ver_off : left; + j->s[j->n++] = (VSample){r, ver_off, len, dst, r->mgen}; + dst += len, left -= len, ver_off += len; + if (ver_off >= size) ver_i++, ver_off = 0; + } + ReleaseSRWLockShared(&pool_lock); + if (!j->n) { free(j); return 0; } + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = dst, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; + VmaAllocationInfo info; + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &j->buf, &j->alloc, &info) != VK_SUCCESS) { free(j); return 0; } + j->map = info.pMappedData; + j->sub = msub; + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT); + for (int i = 0; i < j->n; i++) { + VkBufferCopy c = {j->s[i].off, j->s[i].dst, j->s[i].len}; + vkCmdCopyBuffer(cb, j->s[i].r->mbuf, j->buf, 1, &c); + } + mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT, + VK_ACCESS_2_HOST_READ_BIT); + vk_completion_add(fence, verify_cb, j); + return j->n + 2; +} + +int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { + if (!vk_mirror_on) return 0; + int rec = 0; + if (nwbl) { + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT); + for (int i = 0; i < nwbl; i++) wbl[i].r->wb_sub = msub; + mrep.wb_ranges += (uint64_t)nwbl; + rec += copy_ranges(cb, wbl, nwbl, 0, &mrep.wb_bytes, &mrep.wb_copies) + 2; + mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + nwbl = 0; + } + if (mirror_verify) rec += verify_record(cb, fence); + AcquireSRWLockExclusive(&mirror_stat_lock); + uint64_t *d = (uint64_t *)&macc, *s = (uint64_t *)&mrep; + for (size_t i = 0; i < sizeof macc / sizeof(uint64_t); i++) d[i] += s[i]; + ReleaseSRWLockExclusive(&mirror_stat_lock); + memset(&mrep, 0, sizeof mrep); + return rec; +} + +// One summary line: with TOTK_TRACE=perf every 60 frames, else at frames 600 and 1800. +static void mirror_log(uint64_t frame) { + static uint64_t last; + if (!vk_mirror_on) return; + AcquireSRWLockExclusive(&mirror_stat_lock); + MirrorAcc a = macc; + memset(&macc, 0, sizeof macc); + ReleaseSRWLockExclusive(&mirror_stat_lock); + double f = frame > last ? (double)(frame - last) : 1.0, M = 1048576.0; + last = frame; + uint64_t fc = (uint64_t)InterlockedExchange64(&m_flush_calls, 0), fb = (uint64_t)InterlockedExchange64(&m_flush_bytes, 0); + char ver[160] = ""; + if (mirror_verify) + snprintf(ver, sizeof ver, "; verify: pages compared %lld, skipped %lld (written since), mismatches %lld", + (long long)InterlockedExchange64(&v_pages, 0), (long long)InterlockedExchange64(&v_skipped, 0), + (long long)InterlockedExchange64(&v_mismatch, 0)); + rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked " + "%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, " + "first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries " + "%.1f pre-snap + %.1f sync, prologues %.2f, tracking cpu %.3f ms%s", + (unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET], + m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP], + (long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f, + a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M, + a.presnap_queries / f, a.sync_queries / f, a.prologues / f, a.cpu_ns / f / 1e6, ver); +} + // Device class (build/mem-plan.md): INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a -// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Only logged for now. +// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Sets the mirror mode. void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m) { int big_local = -1, host_heap = -1; for (uint32_t h = 0; h < m->memoryHeapCount; h++) { @@ -811,6 +1340,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic else if (big_local < 0) cls = "uma", why = "no device-local heap of 1 GiB or more"; else if (host_heap < 0) cls = "uma", why = "every heap is device-local"; else cls = "discrete", why = "device-local heap separate from host memory"; + int discrete = cls[0] == 'd'; char line[2048]; size_t k = (size_t)snprintf(line, sizeof line, "mem: class %s (%s; device type %d %s, %s)", cls, why, t, tn, p->deviceName); for (uint32_t h = 0; h < m->memoryHeapCount; h++) @@ -830,6 +1360,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic f & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT ? " LAZY" : ""); } rt_log("%s", line); + mirror_mode(discrete); } // ---------------------------------------------------------------- upload batches and deferred destruction @@ -1069,6 +1600,9 @@ void vk_mem_shutdown(void) { AcquireSRWLockExclusive(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); pool_index.n = 0; + nm = npend = nwbl = 0; // mirrors went with their pools + for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran + nretiring = 0; __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); nphys = 0; ReleaseSRWLockExclusive(&pool_lock); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 63a082f1..61234cef 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -107,27 +107,33 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order free(t); } -// At the start of a part's command buffer (sub_lock held). -static void ts_begin(VkCommandBuffer cb) { - ts_cur = -1; - if (ts_state != 1) return; +// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed. +static int ts_open(VkCommandBuffer cb) { + if (ts_state != 1) return -1; uint32_t p = ts_next % TS_PAIRS; - if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return; } + if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return -1; } ts_next++; ts_busy[p] = 1; vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2); vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p); - ts_cur = (int)p; + return (int)p; } -// At the end of the part, before vkEndCommandBuffer; fence: the value whose completion covers the part. -static void ts_end(VkCommandBuffer cb, uint64_t fence) { - if (ts_cur < 0) return; - vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)ts_cur + 1); +// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it. +static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence) { + if (pair < 0) return; + vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1); TsRead *t = malloc(sizeof *t); if (!t) rt_fatal("vk: out of memory"); - t->pair = (uint32_t)ts_cur; + t->pair = (uint32_t)pair; vk_completion_add(fence, ts_read_cb, t); +} + +// The pair of the part being recorded. +static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); } + +static void ts_end(VkCommandBuffer cb, uint64_t fence) { + ts_close(cb, ts_cur, fence); ts_cur = -1; } @@ -227,27 +233,47 @@ void vk_sync_start(void) { ReleaseSRWLockExclusive(&start_lock); } -// A command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held. -static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { +// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move +// slots[]. +static int claim_slot(VkCtx *c, uint64_t fence) { uint64_t gpu = 0; vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu); for (int i = 0; i < nslots; i++) if (slots[i].fence <= gpu) { slots[i].fence = fence; - cur_slot = i; - return &slots[i]; + return i; } CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1)); if (!n) rt_fatal("vk: out of memory (command buffers)"); slots = n; CmdSlot *s = &slots[nslots]; - cur_slot = nslots; VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation"); s->fence = fence; - nslots++; - return s; + return nslots++; +} + +static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { + cur_slot = claim_slot(c, fence); + return &slots[cur_slot]; +} + +// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is +// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the +// part was recorded, sub_lock held; may move slots[]. +static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { + if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE; + int i = claim_slot(c, fence); // before reading slots: the claim may move it + VkCommandBuffer cb = slots[i].cb; + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(cb, &bi); + int pair = ts_open(cb); + vk_mem_mirror_prologue(cb); + ts_close(cb, pair, fence); + vkEndCommandBuffer(cb); + return cb; } VkCommandBuffer vk_sync_split(void) { @@ -256,8 +282,11 @@ VkCommandBuffer vk_sync_split(void) { ts_end(cb, cur_fence); vkEndCommandBuffer(cb); vk_upload_flush(0); // the uploads this part asked for precede it in queue order - VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = 1, .pCommandBufferInfos = &cbi}; + VkCommandBuffer pro = prologue(c, cur_fence); + VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, + {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}}; + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1, + .pCommandBufferInfos = pro ? cbi : cbi + 1}; vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); @@ -292,11 +321,16 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { perf_replay(t0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. vk_upload_flush(0); - VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}; + VkCommandBuffer pro = prologue(c, fence); + s = &slots[cur_slot]; + VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, + {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}}; VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = recorded ? 1 : 0, - .pCommandBufferInfos = &cbi, .signalSemaphoreInfoCount = 1, .pSignalSemaphoreInfos = &sig}; + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, + .commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u), + .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1, + .pSignalSemaphoreInfos = &sig}; vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); @@ -365,6 +399,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); } +uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); } + void vk_sync_finish(void) { vk_sync_start(); vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index f8484921..68f10f22 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -888,6 +888,8 @@ int vk_texture_slot_swz(int id, uint8_t swz[4]) { return ok; } +uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); } + void vk_texture_new_submit(void) { lock(); if (!++submit_epoch) submit_epoch = 1;