From 1fe552ae0a78422b8497572f85f424e7312fa1ff Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 25 Oct 2026 00:45:56 +0300 Subject: [PATCH 1/2] Mirror buffer-bound pools in device-local memory on discrete GPUs Imported pools put vertices, indices, constants and storage buffers in system memory, so discrete GPUs read them over PCIe on every draw. A pool now gets a device-local mirror on its first use as a vertex, index, uniform, storage or indirect buffer; bindings, buffer copies, fills and uniform updates use it at the same offsets. The first access of a submission syncs the pages written since (write-watch snapshots shared with the texture checks, plus nvnMemoryPoolFlushMappedRange ranges) in a prologue command buffer before the part; what the GPU writes into a mirror goes back to guest memory at the end of the submission. TOTK_MEMORY=import|mirror|verify overrides the device class; verify compares sampled mirror pages with guest memory at the fence. The vkexec probe gains a mirror part. Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 3 + runtime/memory.c | 13 +- runtime/nvn_api.h | 3 + runtime/nvn_api_core.c | 9 +- runtime/nvn_ext.c | 2 + runtime/vk.h | 26 ++ runtime/vk_dump.c | 1 + runtime/vk_exec.c | 2 + runtime/vk_exec_draw.c | 25 +- runtime/vk_exec_probe.c | 56 +++- runtime/vk_exec_xfer.c | 27 +- runtime/vk_memory.c | 606 +++++++++++++++++++++++++++++++++++++--- runtime/vk_sync.c | 82 ++++-- runtime/vk_texture.c | 2 + 14 files changed, 775 insertions(+), 82 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index 57a1f7c2..8603b6a1 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -128,6 +128,9 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64 // past max stays reported next time). The range must lie // inside the stack/heap block or the alias region // (write watch); otherwise returns UINT64_MAX +uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max); + // the same without the reset: the bits stay for the next + // mem_written_pages (verify checks only) void mem_kernel_free(void *p, uint64_t size); // TLR/TLS reuse after thread exit // ---- modules (loader.c, T1) diff --git a/runtime/memory.c b/runtime/memory.c index 84bf8646..7f8c6494 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -148,7 +148,7 @@ const RtGuestLayout *mem_layout(void) { return &L; } // Adapted from LAS Recomp runtime/platform/windows.c:40-63 @ da35b83 (MIT, Copyright (c) 2026 Peiikai): GetWriteWatch // with the reset flag. A range inside one watched reservation (stack and heap block, or alias region) only. -uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { +static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, DWORD flags) { if (!max || !size) return 0; const RtGuestLayout *g = &L; int low_block = addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size; @@ -157,11 +157,18 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64 ULONG_PTR n = (ULONG_PTR)max; DWORD gran; _Static_assert(sizeof(PVOID) == sizeof(uint64_t), "page addresses are written straight into a uint64_t array"); - if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran)) - return UINT64_MAX; + if (GetWriteWatch(flags, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran)) return UINT64_MAX; return n; } +uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { + return written_pages(addr, size, pages, max, WRITE_WATCH_FLAG_RESET); +} + +uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { + return written_pages(addr, size, pages, max, 0); +} + static uint64_t image_bytes(void) { uint64_t n = 0; for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size; diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index 4cfe2b3b..fb5b31d3 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -99,3 +99,6 @@ const char *nvn_op_name(uint32_t op); // "DRAW_IDX", ...; " // MemoryPoolInvalidateMappedRange: the CPU is about to read pool storage the GPU wrote. Weak no-op in nvn_ext.c; a back // end that renders into host-side copies defines it and writes them back. void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size); +// MemoryPoolFlushMappedRange: the CPU wrote [mem + off, mem + off + size). Weak no-op in nvn_ext.c; a back end that keeps +// device-side copies of pool memory (vk_memory.c's VRAM mirror) defines it and copies the range again. +void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size); diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 7978ed9e..0010d3b6 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -303,9 +303,14 @@ void nvn_flush_stats(uint64_t out[2][3][2]) { for (int j = 0; j < 2; j++) out[i][k][j] = __atomic_exchange_n(&flush_stat[i][k][j], 0, __ATOMIC_RELAXED); } -// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent +// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent; a +// back end with device-side copies hears of it (nvn_pool_flushed) NVN_API(nvnMemoryPoolFlushMappedRange) { - if (rt_trace & RT_TRACE_PERF) flush_note(0, arg_p(c, 0), arg_x(c, 2)); + const pool_t *p = arg_p(c, 0); + int64_t off = (int64_t)arg_x(c, 1); + uint64_t size = arg_x(c, 2); + if (rt_trace & RT_TRACE_PERF) flush_note(0, p, size); + if (p && off >= 0 && (uint64_t)off < p->size) nvn_pool_flushed(p, (uint64_t)off, size < p->size - (uint64_t)off ? size : p->size - (uint64_t)off); } // NVNmemoryPoolFlags nvnMemoryPoolGetFlags(const NVNmemoryPool *p) NVN_API(nvnMemoryPoolGetFlags) { ret_x(c, ((const pool_t *)arg_p(c, 0))->flags); } diff --git a/runtime/nvn_ext.c b/runtime/nvn_ext.c index 3e31ad0d..11346faf 100644 --- a/runtime/nvn_ext.c +++ b/runtime/nvn_ext.c @@ -20,6 +20,8 @@ void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size) { gfx_backend()->guest_read((uintptr_t)p->mem + off, size); } +__attribute__((weak)) void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { (void)p, (void)off, (void)size; } + // ---------- debug labels, markers, groups, checkpoints: meaningful only to NVN debugging tools // void nvnSetDebugLabel(NVN *o, const char *label) NVN_API(nvnDeviceSetDebugLabel) { (void)c; } diff --git a/runtime/vk.h b/runtime/vk.h index 69dc6449..a2df85ed 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -138,6 +138,30 @@ void vk_mem_count(int cls, int use, uint64_t bytes); void vk_mem_bound_note(void *pool); #define VK_MEM_BOUND(sp) do { if ((rt_trace & RT_TRACE_PERF) && (sp).pool) vk_mem_bound_note((sp).pool); } while (0) void vk_mem_dirty_submit(void); // vk_sync_submit, under its lock, before the replay +// VRAM mirror of buffer-bound pools (vk_memory.c, build/mem-plan.md). vk_mirror_on: TOTK_MEMORY=mirror|verify, or the +// device class is discrete and TOTK_MEMORY is unset or auto; set at device selection. The replay (submitting thread +// under vk_sync_submit's lock) calls: +// - VK_MEM_USE at a vertex, index, uniform, storage or indirect binding of a span vk_mem_resolve returned: the pool's +// mirror is created on first use and sp.buffer becomes it (same offsets); a pool that stays imported keeps its buffer. +// - VK_MEM_PICK where a copy, fill or update reads or writes a span: the mirror if the pool has one, never a new one. +// - vk_mem_gpu_wrote after recording a GPU write of [sp.offset, sp.offset + size) into sp.buffer: a mirror's bytes go +// back to guest memory at the end of the submission. No-op for imported buffers. +// - vk_mem_mirror_begin at the start of a submission's recording, vk_mem_mirror_end (records the write-backs; returns +// the commands recorded) after its last barrier. +// vk_sync.c records the copies the mirrors' first accesses queued (vk_mem_mirror_pending) into a prologue command +// buffer with vk_mem_mirror_prologue and submits it right before each part. +extern int vk_mirror_on; +void vk_mem_use(VkMemSpan *sp); +void vk_mem_pick(VkMemSpan *sp); +#define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0) +#define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0) +void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size); +void vk_mem_mirror_begin(void); +int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); +int vk_mem_mirror_pending(void); +void vk_mem_mirror_prologue(VkCommandBuffer cb); +int vk_mem_mirror_probe(uint64_t addr); // --probe-vkexec: the pool at addr gets its mirror as on a first buffer use; + // 1 when it has one void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m); // once int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked) void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and, @@ -223,6 +247,7 @@ void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); / void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads); void vk_texture_shutdown(void); // vk_mem_shutdown calls it void vk_texture_new_submit(void); // a queue submit starts: draw bindings check storage again +uint32_t vk_texture_epoch(void); // the write-watch snapshot epoch vk_texture_new_submit started VkSampler vk_sampler_by_id(int id); // VK_NULL_HANDLE for an empty slot VkSampler vk_sampler_for(const samp_par *s); // cached by content int vk_sampler_params(int id, samp_par *out); // parameters of a sampler slot; 0 for an empty slot @@ -237,6 +262,7 @@ void vk_sync_start(void); // vk_init (fatal without a device) a uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted uint64_t vk_sync_fence_done(void); // highest done value +uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1) void vk_sync_finish(void); // every submitted value done void vk_sync_shutdown(void); // finish, stop the completion thread, free the command buffers // The op-stream replay (vk_exec.c: T6c): records the command handles of one submission into cb, which the caller has diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index 3b41a46a..d3e61202 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -399,6 +399,7 @@ static void buf_snap(VkExec *ex, int pass) { if (!bd_size) return; VkMemSpan s; if (!vk_mem_resolve(bd_addr, bd_size, &s) || s.avail < bd_size) return; + VK_MEM_PICK(s); // the GPU's view of the buffer BufJob *j = calloc(1, sizeof *j); if (!j) rt_fatal("vk_dump: out of memory"); VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bd_size, diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 7cdbe4ad..ae6cd861 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -688,6 +688,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_frame.submits++; split_draws = part_start = 0; x_here = 1; + if (n) vk_mem_mirror_begin(); for (int i = 0; i < n; i++) { if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i); walk(ex, nvn_op_first(handles[i]), 0); @@ -695,6 +696,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_pass_end(ex); x_here = 0; if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes + if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence); // VRAM mirror bytes the GPU wrote go back to guest memory if (ex->nviews) { ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); if (!l) rt_fatal("vk: out of memory"); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 8fd5ca0b..aca46658 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -569,16 +569,20 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) { return (VkDescriptorBufferInfo){Z.zbuf, 0, max < ZERO_BYTES ? max : ZERO_BYTES}; } -// A buffer descriptor for [addr, addr+size) of guest memory: its backing when aligned, a copy in transient memory when -// the backing is host memory with an unaligned offset. 0 when neither works. -static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, VkDescriptorBufferInfo *out) { +// A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when +// aligned, a copy in transient memory when the backing is host memory with an unaligned offset. 0 when neither works. +// written: the shader stores into it (a mirror's range goes back to guest memory at the end of the submission). +static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, int written, + VkDescriptorBufferInfo *out) { VkMemSpan sp; if (!addr || !vk_mem_resolve(addr, 1, &sp)) return 0; uint64_t n = size && size < sp.avail ? size : sp.avail; if (n > max) n = max; if (!n) return 0; if (sp.offset % align == 0) { + VK_MEM_USE(sp); *out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n}; + if (written) vk_mem_gpu_wrote(&sp, n); vk_mem_count(sp.cls, use, n); VK_MEM_BOUND(sp); return 1; @@ -665,11 +669,12 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { uint32_t len, c1; if (find_stage(ex->prog, st, &ctl, &code, &len)) { memcpy(&c1, ctl + 0x700, 4); - if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, &bi)) return bi; + if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, 0, &bi)) + return bi; } } else if (slot >= 3 && slot - 3 < 18) { const p_bind_buf *b = &ex->ubo[st][slot - 3]; - if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, &bi)) return bi; + if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, 0, &bi)) return bi; } return zero_info(max); } @@ -699,7 +704,8 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * const p_bind_buf *b = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot]; // vk_caps.c: shaders translated for sbuf_align address the buffer from its address rounded down to it. uint64_t lead = vk_caps.sbuf_align ? b->addr & (vk_caps.sbuf_align - 1) : 0; - if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO, &bi)) { + if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO, + written, &bi)) { if (written && bi.buffer != VK_NULL_HANDLE) note_written(ex, b->addr - lead, bi.range); return bi; } @@ -1095,6 +1101,7 @@ static void bind_vertex_buffers(VkExec *ex) { const NVNbufferRange *r = &ex->vbuf[b]; VkMemSpan sp; if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue; + VK_MEM_USE(sp); uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail; vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool; } @@ -1271,6 +1278,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { } else if (sp.offset % isize[d->itype]) { return warn_once(W_INDEX, "index buffer %#llx is not aligned to its index size: draw skipped", (unsigned long long)d->ibuf); } else { + VK_MEM_USE(sp); vkCmdBindIndexBuffer(ex->cb, sp.buffer, sp.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); ex->recorded++; vk_mem_count(sp.cls, VK_MUSE_IB, bytes); @@ -1295,6 +1303,8 @@ static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) { return warn_once(W_INDEX, "index buffer %#llx is not backed or not aligned: indirect draws skipped", (unsigned long long)d->ibuf); if (!prepare_draw(ex, d->prim, 0x7fffffff, restart_for(ex, d->prim, d->itype))) return; vk_exec_frame.draws++; + VK_MEM_USE(ip); + VK_MEM_USE(ib); vkCmdBindIndexBuffer(ex->cb, ib.buffer, ib.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); vkCmdDrawIndexedIndirect(ex->cb, ip.buffer, ip.offset, 1, sizeof(VkDrawIndexedIndirectCommand)); ex->recorded += 2; @@ -1341,6 +1351,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { VkPipeline p = vk_pipeline_compute(cs, &info); if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped"); vk_exec_barrier(ex); + VK_MEM_USE(ip); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); c0_dim[0] = c0_dim[1] = c0_dim[2] = 0; B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads @@ -1368,8 +1379,10 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { VkBuffer b; VkDeviceSize o; memcpy(scratch(ex, u->size, &b, &o), u->data, u->size); + VK_MEM_PICK(sp); VkBufferCopy cp = {o, sp.offset, u->size}; vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp); + vk_mem_gpu_wrote(&sp, u->size); ex->recorded++; vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size); note_written(ex, u->dst, u->size); diff --git a/runtime/vk_exec_probe.c b/runtime/vk_exec_probe.c index 7b83d323..0396501f 100644 --- a/runtime/vk_exec_probe.c +++ b/runtime/vk_exec_probe.c @@ -802,6 +802,54 @@ static int test_nvnx(const GfxBackend *b, uint8_t *guest, int *bytes) { return 0; } +// VRAM mirror (vk_memory.c; skipped in import mode): a pool given its mirror as on a first buffer use, after which +// buffer copies and fills read and write the mirror. The first use copies the whole pool; CPU writes between +// submissions reach the mirror through write watch; GPU writes reach guest memory by the fence, and only the bytes +// written: CPU bytes beside a GPU fill keep their values. +static int test_mirror(const GfxBackend *b, uint8_t *guest, int *bytes) { + *bytes = 0; + if (!vk_mirror_on) return 0; + pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x22, .mem = guest + 0xC00000, .size = 0x100000, + .game_mem = guest + 0xC00000}; + CHECK(b->pool_init(&pool), "pool_init of a CPU-visible pool for the mirror part"); + uint8_t *m = pool.mem; + for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 13 + 1); + memset(m + 0x1000, 0xEE, 0x5000); + CHECK(vk_mem_mirror_probe((uint64_t)(uintptr_t)m), "the pool got no VRAM mirror"); + static _Alignas(8) uint8_t stream[1024]; + uint64_t a = (uint64_t)(uintptr_t)m; + // 1: B2B in the mirror after its first-use copy, and a fill; both written back. + uint8_t *p = put(stream, OP_BEGIN, NULL, 0); + const p_copy_b2b c1 = {a, a + 0x1000, 4096}; + const p_clear_buf f1 = {a + 0x2000, 0x800, 0x5a5a5a5au, 0}; + p = put(p, OP_COPY_B2B, &c1, sizeof c1); + p = put(p, OP_CLEAR_BUF, &f1, sizeof f1); + CHECK(xf_submit(b, stream, p), "fence_wait of the first mirror submission timed out"); + for (int i = 0; i < 4096; i++) CHECK(m[0x1000 + i] == m[i], "mirror B2B byte %d is %02x, expected %02x", i, m[0x1000 + i], m[i]); + for (int i = 0; i < 0x800; i++) CHECK(m[0x2000 + i] == 0x5a, "mirror fill byte %d is %02x", i, m[0x2000 + i]); + for (int i = 0x800; i < 0x1000; i++) CHECK(m[0x2000 + i] == 0xEE, "byte %#x past the mirror fill is %02x", i, m[0x2000 + i]); + // 2: the CPU rewrites the source; the next submission copies the new bytes. + for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 7 + 5); + p = put(stream, OP_BEGIN, NULL, 0); + const p_copy_b2b c2 = {a, a + 0x3000, 4096}; + p = put(p, OP_COPY_B2B, &c2, sizeof c2); + CHECK(xf_submit(b, stream, p), "fence_wait of the second mirror submission timed out"); + for (int i = 0; i < 4096; i++) + CHECK(m[0x3000 + i] == m[i], "B2B byte %d after a CPU write is %02x, expected %02x (write not mirrored)", i, + m[0x3000 + i], m[i]); + // 3: CPU bytes in the page of a GPU fill, written before the submission, survive the write-back. + memset(m + 0x4000, 0x11, 0x800); + p = put(stream, OP_BEGIN, NULL, 0); + const p_clear_buf f3 = {a + 0x4800, 0x800, 0x77777777u, 0}; + p = put(p, OP_CLEAR_BUF, &f3, sizeof f3); + CHECK(xf_submit(b, stream, p), "fence_wait of the third mirror submission timed out"); + for (int i = 0; i < 0x800; i++) CHECK(m[0x4000 + i] == 0x11, "CPU byte %#x beside a fill is %02x", i, m[0x4000 + i]); + for (int i = 0; i < 0x800; i++) CHECK(m[0x4800 + i] == 0x77, "fill byte %#x is %02x", i, m[0x4800 + i]); + b->pool_fini(&pool); + *bytes = 4096 + 0x1000 + 4096 + 0x1000; + return 0; +} + int probe_vkexec(void) { layouts_equal = 0; main_tid = GetCurrentThreadId(); @@ -819,8 +867,9 @@ int probe_vkexec(void) { if (!bad) bad = test_xfer(b, guest, &xfer_bytes); int zero_bytes = 0; if (!bad) bad = test_c0() || test_zero(b, guest, &zero_bytes); - int nvnx_bytes = 0; + int nvnx_bytes = 0, mirror_bytes = 0; if (!bad) bad = test_nvnx(b, guest, &nvnx_bytes); + if (!bad) bad = test_mirror(b, guest, &mirror_bytes); b->finish(); vk_pipeline_shutdown(); // the pipelines and the draw part's stand-ins go before the device vk_exec_draw_shutdown(); @@ -860,5 +909,10 @@ int probe_vkexec(void) { printf("probe-vkexec: c0 and zero parts passed (c0 tables, dispatch size, point size and base instance at their offsets, " "unbound slots zero; %d bytes of an unwritten physical-pool texture read zero, TOTK_FIX_ZERO=%s)\n", zero_bytes, cfg_get("TOTK_FIX_ZERO") ? cfg_get("TOTK_FIX_ZERO") : "1"); + if (vk_mirror_on) + printf("probe-vkexec: mirror part passed (B2B and fill in a VRAM mirror written back by the fence, a CPU write " + "between submissions mirrored, CPU bytes beside a fill kept, %d bytes compared)\n", mirror_bytes); + else + printf("probe-vkexec: mirror part skipped (memory mode import)\n"); return 0; } diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index f33ecf60..de6ac5b8 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -379,9 +379,10 @@ static VkExtent3D sub_extent(const Sub *s) { return (VkExtent3D){s->w, s->h, s->img->type == VK_IMAGE_TYPE_3D ? s->nz : 1}; } -// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span. +// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span (a VRAM mirror when the pool +// has one). static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *size, VkBufferImageCopy *c, - VkBuffer *buf, int *cls) { + VkMemSpan *span) { uint32_t wb = (s->w + (uint32_t)s->bw - 1) / (uint32_t)s->bw, hb = (s->h + (uint32_t)s->bh - 1) / (uint32_t)s->bh; uint64_t row = ex->copy_stride > 0 ? (uint64_t)ex->copy_stride : (uint64_t)wb * (uint64_t)s->bpb; if (row % (uint64_t)s->bpb || row < (uint64_t)wb * (uint64_t)s->bpb) { @@ -397,8 +398,8 @@ static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t * (unsigned long long)addr, (unsigned long long)*size); return 0; } - *buf = m.buffer; - *cls = m.cls; + VK_MEM_PICK(m); + *span = m; *c = (VkBufferImageCopy){.bufferOffset = m.offset, .bufferRowLength = (uint32_t)(row / (uint64_t)s->bpb) * (uint32_t)s->bw, .bufferImageHeight = hb * (uint32_t)s->bh, .imageSubresource = sub_layers(s), .imageOffset = sub_offset(s), .imageExtent = sub_extent(s)}; @@ -410,17 +411,16 @@ static void copy_buffer_image(VkExec *ex, const p_copy_b2t *p, int to_image) { if (!sub_resolve(p->tex, &p->view, p->has_view, &p->r, 1, &s)) return; uint64_t size; VkBufferImageCopy c; - VkBuffer buf; - int cls; - if (!buffer_side(ex, &s, p->src, &size, &c, &buf, &cls)) return; - vk_mem_count(cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size); + VkMemSpan m; + if (!buffer_side(ex, &s, p->src, &size, &c, &m)) return; + vk_mem_count(m.cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size); vk_exec_pass_end(ex); vk_exec_barrier(ex); - if (to_image) vkCmdCopyBufferToImage(ex->cb, buf, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c); - else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, buf, 1, &c); + if (to_image) vkCmdCopyBufferToImage(ex->cb, m.buffer, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c); + else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, m.buffer, 1, &c); ex->recorded++; if (to_image) vk_texture_gpu_written(p->tex); - else note_gpu_write(ex, p->src, size); + else vk_mem_gpu_wrote(&m, size), note_gpu_write(ex, p->src, size); } static void copy_t2t(VkExec *ex, const p_copy_t2t *p) { @@ -476,6 +476,8 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) { } uint64_t n = left < s.avail ? left : s.avail; if (n > d.avail) n = d.avail; + VK_MEM_PICK(s); + VK_MEM_PICK(d); if (s.buffer == d.buffer && s.offset < d.offset + n && d.offset < s.offset + n) { warn_once(W_OVERLAP, "buffer copy %#llx -> %#llx overlaps within one buffer: skipped", (unsigned long long)p->src, (unsigned long long)p->dst); @@ -484,6 +486,7 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) { if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1; VkBufferCopy bc = {s.offset, d.offset, n}; vkCmdCopyBuffer(ex->cb, s.buffer, d.buffer, 1, &bc); + vk_mem_gpu_wrote(&d, n); ex->recorded++; vk_mem_count(s.cls, VK_MUSE_COPY_SRC, n); vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n); @@ -514,7 +517,9 @@ static void clear_buf(VkExec *ex, const p_clear_buf *p) { uint64_t n = left < d.avail ? left : d.avail & ~3ull; if (!n) break; if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1; + VK_MEM_PICK(d); vkCmdFillBuffer(ex->cb, d.buffer, d.offset, n, p->value); + vk_mem_gpu_wrote(&d, n); ex->recorded++; vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n); dst += n, left -= n; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ebda951c..58b5e4bc 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -24,6 +24,7 @@ // ---------------------------------------------------------------- pools enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT }; +enum { MS_NONE, MS_ON, MS_REFUSED }; typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page -> physical pool and offset @@ -44,8 +45,22 @@ typedef struct PoolRec { uint32_t synced_gen; // page generation its lazy-mirror count covers; DIRTY_UNSYNCED until scanned uint8_t bufbound; // ever bound as a vertex, index, uniform, storage or indirect buffer uint8_t untracked; // the last dirty scan found no write watch over it + // VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only. + uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED + VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf + VmaAllocation malloc_; + uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync) + uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory + uint32_t mgen; // gen_lock: every page write stamped up to this generation is in the mirror + // (or in a copy queued before its next use) + uint64_t fl_lo, fl_hi; // flush_lock: nvnMemoryPoolFlushMappedRange bytes not copied yet (fl_hi 0: none) } PoolRec; +static void mirror_forget(PoolRec *r); // pool_lock exclusive: off the mirrored list +static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown) +static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record +static void grow(void **arr, int *cap, size_t elem); + static SRWLOCK pool_lock = SRWLOCK_INIT; // Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits // stay valid while it is unchanged. @@ -197,6 +212,7 @@ static PoolRec *phys_find(const pool_t *key) { } static void destroy_rec(VkCtx *c, PoolRec *r) { + if (r->mbuf) vmaDestroyBuffer(c->vma, r->mbuf, r->malloc_); if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL); if (r->mem) vkFreeMemory(c->device, r->mem, NULL); if (r->alloc) vmaFreeMemory(c->vma, r->alloc); @@ -259,6 +275,7 @@ void vk_pool_fini(pool_t *p) { int had = pidx_remove(&pool_index, p, &e); __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); PoolRec *r = had ? e.buf : NULL; + if (r && r->mstate == MS_ON) mirror_forget(r); if (r && r->kind == POOL_PHYS) { for (int i = 0; i < nphys; i++) if (phys_list[i] == r) { phys_list[i] = phys_list[--nphys]; break; } @@ -281,7 +298,10 @@ void vk_pool_fini(pool_t *p) { vk_queue_unlock(); } vk_mem_collect(); - destroy_rec(c, r); + // A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may + // name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued). + if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r); + else destroy_rec(c, r); } // Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c:54-67); apply them in order. @@ -414,9 +434,12 @@ static void flag_names(uint32_t f, char *buf, size_t n) { // run-game saves), at frame 600, and with TOTK_TRACE=perf every 1800 frames. static void dirty_frame(uint64_t frame); +static void mirror_log(uint64_t frame); + void vk_mem_census(uint64_t frame) { static uint64_t window_start; - if (rt_trace & RT_TRACE_PERF) dirty_frame(frame); + if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); + if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame); if (frame == 1200) { // loading is over: the first window starts here for (int c = 0; c < VK_MCLS_N; c++) for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0); @@ -497,6 +520,10 @@ static uint32_t *gen_chunk[GEN_CHUNKS]; static uint32_t gen_now; // last generation handed out static SRWLOCK gen_lock = SRWLOCK_INIT; static uint64_t gen_pages[1024]; +// The newest generation stamped on any page of each 2 MiB region (the write-watch snapshot regions below): the mirror +// sync skips regions with nothing newer than its pool's mirror. +#define REG_SHIFT 21 +static uint32_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT]; static uint32_t *gen_slot(uint64_t page_addr, int create) { uint64_t ci = page_addr >> GEN_CHUNK_SHIFT; @@ -509,11 +536,16 @@ static uint32_t *gen_slot(uint64_t page_addr, int create) { return &gen_chunk[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)]; } +// gen_lock held exclusively; g is the newest generation handed out. +static void stamp_page(uint64_t a, uint32_t g) { + uint32_t *s = gen_slot(a, 1); + if (!s) return; + *s = g; + reg_max[a >> REG_SHIFT] = g; +} + static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) { - for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) { - uint32_t *s = gen_slot(a, 1); - if (s) *s = g; - } + for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page(a, g); } void vk_guest_written(uint64_t addr, uint64_t size) { @@ -533,10 +565,7 @@ int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now) for (;;) { uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages); if (n == UINT64_MAX) { tracked = 0; break; } - for (uint64_t k = 0; k < n; k++) { - uint32_t *s = gen_slot(gen_pages[k], 1); - if (s) *s = g; - } + for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g); if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call } int changed = 0; @@ -560,6 +589,31 @@ static uint32_t snap_at[SNAP_REGIONS]; // epoch of the region's last snapsh static uint8_t snap_untracked[SNAP_REGIONS]; static uint64_t n_snap_checks, n_snap_calls, n_snap_untracked; +// gen_lock held exclusively. Queries the regions r0..r1 that were not snapped in epoch, one query per run of them; +// returns the number of queries. +static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) { + uint32_t g = 0; + uint64_t calls = 0; + for (uint64_t r = r0; r <= r1;) { + if (snap_at[r] == epoch) { r++; continue; } + uint64_t k = r; // one query over a run of regions not seen in this epoch + while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++; + uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT; + int tracked = 1; + if (!g) g = ++gen_now; + for (;;) { + uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages); + if (n == UINT64_MAX) { tracked = 0; break; } + for (uint64_t j = 0; j < n; j++) stamp_page(gen_pages[j], g); + if (n < sizeof gen_pages / sizeof *gen_pages) break; + } + calls++; + for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked; + r = k + 1; + } + return calls; +} + int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) { if (!size) { *now = since; return 0; } uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; @@ -572,26 +626,7 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e if (excl) { ReleaseSRWLockShared(&gen_lock); AcquireSRWLockExclusive(&gen_lock); - uint32_t g = ++gen_now; - for (uint64_t r = r0; r <= r1;) { - if (snap_at[r] == epoch) { r++; continue; } - uint64_t k = r; // one query over a run of regions not seen in this epoch - while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++; - uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT; - int tracked = 1; - for (;;) { - uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages); - if (n == UINT64_MAX) { tracked = 0; break; } - for (uint64_t j = 0; j < n; j++) { - uint32_t *s = gen_slot(gen_pages[j], 1); - if (s) *s = g; - } - if (n < sizeof gen_pages / sizeof *gen_pages) break; - } - n_snap_calls++; - for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked; - r = k + 1; - } + n_snap_calls += snap_regions(r0, r1, epoch); } int untracked = 0, changed = 0; for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r]; @@ -663,16 +698,13 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { for (;;) { uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages); if (n == UINT64_MAX) return 0; - for (uint64_t k = 0; k < n; k++) { - uint32_t *s = gen_slot(gen_pages[k], 1); - if (s) *s = g; - } + for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g); if (n < sizeof gen_pages / sizeof *gen_pages) return 1; } } void vk_mem_dirty_submit(void) { - if (!(rt_trace & RT_TRACE_PERF)) return; + if (!(rt_trace & RT_TRACE_PERF) || vk_mirror_on) return; // the mirror's own line replaces the census uint64_t t0 = kern_now_ns(); uint32_t prev = dirty_sub; // the submit whose replay noted the bindings dirty_sub = prev + 1 ? prev + 1 : 1; @@ -793,8 +825,505 @@ static void dirty_frame(uint64_t frame) { rt_log("%s", line); } +// ---------------------------------------------------------------- VRAM mirror of buffer-bound pools +// build/mem-plan.md, "Design: VRAM mirror of buffer-bound pools". An imported pool keeps vertices, indices, constants +// and storage buffers in system memory, so a discrete GPU reads them over PCIe on every draw (GTX 1050 Ti and RTX +// 3090 Ti: about 110 ms of GPU time per frame in the field). In mirror mode (device class discrete, TOTK_MEMORY) an +// imported pool gets a device-local buffer over its imported range the first time the replay uses it as a vertex, +// index, uniform, storage or indirect buffer (vk_mem_use). From then on those bindings, and the buffer copies, fills +// and uniform updates that touch the pool (vk_mem_pick), use the mirror at the same offsets. Guest memory stays what +// the CPU reads and writes; textures and every host path that reads or writes pool bytes keep using it. +// - CPU -> mirror: the first access of a submission to a mirror syncs it (mirror_sync): the 2 MiB write-watch +// snapshots of its regions for this epoch (snap_regions, shared with the texture checks: the OS bits keep one +// consumer, the shared page generations) and every page stamped after the mirror's mgen goes into a copy; so do the +// nvnMemoryPoolFlushMappedRange ranges. The first use copies the whole range. The copies run in a prologue command +// buffer that vk_sync.c submits right before the part that accessed the mirror first, between full barriers. +// vk_mem_mirror_begin snaps the regions of the pools the previous submission accessed in a few merged queries. +// - GPU -> guest: what the replay writes into a mirror (storage buffers its shaders write, buffer copies, fills, +// uniform updates, texture-to-buffer copies: vk_mem_gpu_wrote) goes back into the imported buffer at the end of the +// submission, exact byte ranges, before the fence signals (vk_mem_mirror_end). +// - TOTK_MEMORY=verify: every submission also copies up to VERIFY_BYTES of mirror pages, round robin, into host memory; +// at its fence the pages the CPU has not written since their sync are compared with guest memory. +// A pool stays imported when its mirror would not fit the VRAM budget, its range is not write-watched, it overlaps +// another pool or the allocation fails. +#define VERIFY_BYTES (4u << 20) +#define VERIFY_SAMPLES 64 +#define VERIFY_LOG 16 +enum { MR_BUDGET, MR_UNTRACKED, MR_OVERLAP, MR_ALLOC, MR_N }; + +int vk_mirror_on; +static int mirror_verify; +static int mirror_gap = 2; // pre-snap: hot pools at most this many 2 MiB regions apart share one query +static PoolRec **mlist; // pool_lock: mirrored pools by import_lo +static int nm, mcap; +static uint32_t msub; // replay: number of the submission being recorded +static uint32_t mepoch; // replay: its write-watch snapshot epoch (vk_texture_epoch) +typedef struct { PoolRec *r; uint64_t off, size; } MRange; +static MRange *pend, *wbl; // replay: copies for the next prologue; GPU writes into mirrors this submission +static int npend, cpend, nwbl, cwbl; +static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions +static int cregions; +static SRWLOCK flush_lock = SRWLOCK_INIT; + +typedef struct { + uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges; + uint64_t presnap_queries, sync_queries, cpu_ns, prologues; +} MirrorAcc; +static MirrorAcc mrep; // replay: this submission +static MirrorAcc macc; // mirror_stat_lock: since the last summary +static SRWLOCK mirror_stat_lock = SRWLOCK_INIT; +static volatile LONG64 m_live_n, m_live_bytes, m_refused_n[MR_N], m_refused_bytes[MR_N]; +static volatile LONG64 m_flush_calls, m_flush_bytes, v_pages, v_skipped, v_mismatch; + +static void mirror_mode(int discrete) { + const char *e = cfg_get("TOTK_MEMORY"); + int mode = discrete ? 1 : 0; // 0 import, 1 mirror, 2 verify + const char *why = discrete ? "auto, discrete gpu" : "auto, uma"; + if (e && *e && strcmp(e, "auto")) { + if (!strcmp(e, "import")) mode = 0, why = "TOTK_MEMORY=import"; + else if (!strcmp(e, "mirror")) mode = 1, why = "TOTK_MEMORY=mirror"; + else if (!strcmp(e, "verify")) mode = 2, why = "TOTK_MEMORY=verify"; + else rt_log("mem: TOTK_MEMORY=%s is not import, mirror, verify or auto: auto", e); + } + vk_mirror_on = mode > 0; + mirror_verify = mode == 2; + const char *g = cfg_get("TOTK_MIRROR_GAP"); + mirror_gap = g && *g ? atoi(g) : 2; + if (mirror_gap < 0) mirror_gap = 0; + rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why, + mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory"); +} + +static void range_add(MRange **a, int *n, int *cap, PoolRec *r, uint64_t off, uint64_t size) { + if (*n && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) { + (*a)[*n - 1].size += size; + return; + } + if (*n == *cap) grow((void **)a, cap, sizeof **a); + (*a)[(*n)++] = (MRange){r, off, size}; +} + +static int range_cmp(const void *x, const void *y) { + const MRange *a = x, *b = y; + if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1; + return a->off < b->off ? -1 : a->off > b->off; +} + +// Records the ranges as one vkCmdCopyBuffer per pool, overlapping and adjacent ranges merged; to_mirror: buf -> mbuf, +// else mbuf -> buf. Returns the commands recorded; adds what they move to *bytes and *copies. +static int copy_ranges(VkCommandBuffer cb, MRange *a, int n, int to_mirror, uint64_t *bytes, uint64_t *copies) { + qsort(a, (size_t)n, sizeof *a, range_cmp); + int rec = 0; + for (int i = 0; i < n;) { + PoolRec *r = a[i].r; + uint64_t lim = r->import_hi - r->import_lo; + int k = 0; + for (; i < n && a[i].r == r; i++) { + uint64_t lo = a[i].off, hi = a[i].off + a[i].size; + if (hi > lim) hi = lim; + if (lo >= hi) continue; + if (k && lo <= mregions[k - 1].srcOffset + mregions[k - 1].size) { + uint64_t end = mregions[k - 1].srcOffset + mregions[k - 1].size; + if (hi > end) mregions[k - 1].size += hi - end; + continue; + } + if (k == cregions) grow((void **)&mregions, &cregions, sizeof *mregions); + mregions[k++] = (VkBufferCopy){lo, lo, hi - lo}; + } + if (!k) continue; + if (to_mirror) vkCmdCopyBuffer(cb, r->buf, r->mbuf, (uint32_t)k, mregions); + else vkCmdCopyBuffer(cb, r->mbuf, r->buf, (uint32_t)k, mregions); + rec++; + for (int j = 0; j < k; j++) *bytes += mregions[j].size; + *copies += (uint64_t)k; + } + return rec; +} + +static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, + VkAccessFlags2 da) { + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = ss, .srcAccessMask = sa, + .dstStageMask = ds, .dstAccessMask = da}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); +} + +static void refuse(PoolRec *r, int why, const char *what) { + r->mstate = MS_REFUSED; + uint64_t n = r->import_hi - r->import_lo; + if (InterlockedIncrement64(&m_refused_n[why]) <= 4) { + char f[160]; + flag_names(r->flags, f, sizeof f); + rt_log("mem: pool %#llx+%#llx (%s) stays imported: %s", (unsigned long long)r->lo, (unsigned long long)r->size, + f, what); + } + InterlockedAdd64(&m_refused_bytes[why], (LONG64)n); +} + +// Replay thread: the pool's mirror on its first buffer use. +static void mirror_create(PoolRec *r) { + VkCtx *c = vk_ctx(); + uint64_t n = r->import_hi - r->import_lo; + r->mstate = MS_REFUSED; + if (r->kind != POOL_IMPORT || !r->buf || !n) return; + AcquireSRWLockShared(&pool_lock); + int i = pidx_upper(&pool_index, r->lo) - 1; + while (i >= 0 && pool_index.e[i].buf != r) i--; + int alone = i >= 0 && (i == 0 || pool_index.maxhi[i - 1] <= pool_index.e[i].lo) && + (i + 1 == pool_index.n || pool_index.e[i + 1].lo >= pool_index.e[i].hi); + ReleaseSRWLockShared(&pool_lock); + if (i < 0) return; // finalized meanwhile + if (!alone) return refuse(r, MR_OVERLAP, "it overlaps another pool"); + uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT; + if (r1 >= SNAP_REGIONS) return refuse(r, MR_UNTRACKED, "outside the write-watched address space"); + uint64_t used, budget; + int exact; + if (vk_mem_vram(&used, &budget, &exact) && used + n + budget / 8 > budget) { + char why[128]; + snprintf(why, sizeof why, "its %.1f MiB mirror does not fit the VRAM budget (%.0f of %.0f MiB in use)", + n / 1048576.0, used / 1048576.0, budget / 1048576.0); + return refuse(r, MR_BUDGET, why); + } + AcquireSRWLockExclusive(&gen_lock); + mrep.sync_queries += snap_regions(r0, r1, mepoch); + int untracked = 0; + for (uint64_t k = r0; k <= r1; k++) untracked |= snap_untracked[k]; + r->mgen = gen_now; // the whole-range copy below holds every write observed so far + ReleaseSRWLockExclusive(&gen_lock); + if (untracked) return refuse(r, MR_UNTRACKED, "its range is not write-watched"); + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = n, .usage = pool_usage(c), + .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, + .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + VkResult res = vmaCreateBuffer(c->vma, &bi, &ai, &r->mbuf, &r->malloc_, NULL); + if (res != VK_SUCCESS) { + r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL; + char why[96]; + snprintf(why, sizeof why, "device-local allocation of %.1f MiB failed (%d)", n / 1048576.0, (int)res); + return refuse(r, MR_ALLOC, why); + } + AcquireSRWLockExclusive(&pool_lock); + if (nm == mcap) grow((void **)&mlist, &mcap, sizeof *mlist); + int at = nm; + while (at > 0 && mlist[at - 1]->import_lo > r->import_lo) at--; + memmove(mlist + at + 1, mlist + at, (size_t)(nm - at) * sizeof *mlist); + mlist[at] = r; + nm++; + ReleaseSRWLockExclusive(&pool_lock); + r->acc_sub = msub; + r->mstate = MS_ON; + range_add(&pend, &npend, &cpend, r, 0, n); + mrep.first_n++, mrep.first_bytes += n; + InterlockedIncrement64(&m_live_n); + InterlockedAdd64(&m_live_bytes, (LONG64)n); +} + +static void mirror_forget(PoolRec *r) { + for (int i = 0; i < nm; i++) + if (mlist[i] == r) { + memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist); + nm--; + InterlockedDecrement64(&m_live_n); + InterlockedAdd64(&m_live_bytes, -(LONG64)(r->import_hi - r->import_lo)); + break; + } +} + +// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran. +static PoolRec **retiring; +static int nretiring, cretiring; +static SRWLOCK retire_lock = SRWLOCK_INIT; + +static void mirror_retiring(PoolRec *r) { + AcquireSRWLockExclusive(&retire_lock); + if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring); + retiring[nretiring++] = r; + ReleaseSRWLockExclusive(&retire_lock); +} + +static void mirror_retire(void *arg) { + AcquireSRWLockExclusive(&retire_lock); + for (int i = 0; i < nretiring; i++) + if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; } + ReleaseSRWLockExclusive(&retire_lock); + destroy_rec(vk_ctx(), arg); +} + +// Replay thread, the first access of this submission: queues the copies of the pages written since the last sync. +static void mirror_sync(PoolRec *r) { + uint64_t t0 = kern_now_ns(); + r->acc_sub = msub; + uint64_t lo = r->import_lo, hi = r->import_hi; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; + int excl = 0; + AcquireSRWLockShared(&gen_lock); + for (uint64_t k = r0; k <= r1; k++) + if (snap_at[k] != mepoch) { excl = 1; break; } + if (excl) { + ReleaseSRWLockShared(&gen_lock); + AcquireSRWLockExclusive(&gen_lock); + mrep.sync_queries += snap_regions(r0, r1, mepoch); + } + uint32_t since = r->mgen; + for (uint64_t k = r0; k <= r1; k++) { + if (reg_max[k] <= since) continue; + uint64_t a = k << SNAP_SHIFT, b = a + (1ull << SNAP_SHIFT); + if (a < lo) a = lo; + if (b > hi) b = hi; + const uint32_t *ch = gen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk + if (!ch) continue; + for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) + if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since) + range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT); + } + r->mgen = gen_now; + if (excl) ReleaseSRWLockExclusive(&gen_lock); + else ReleaseSRWLockShared(&gen_lock); + if (__atomic_load_n(&r->fl_hi, __ATOMIC_ACQUIRE)) { + AcquireSRWLockExclusive(&flush_lock); + uint64_t a = r->fl_lo, b = r->fl_hi; + r->fl_hi = 0; + ReleaseSRWLockExclusive(&flush_lock); + a = a > lo ? a & ~0xfffull : lo; + b = b < hi ? (b + 0xfff) & ~0xfffull : hi; + if (a < b) range_add(&pend, &npend, &cpend, r, a - lo, b - a); + } + mrep.syncs++; + mrep.cpu_ns += kern_now_ns() - t0; +} + +void vk_mem_use(VkMemSpan *sp) { + PoolRec *r = sp->pool; + if (r->mstate == MS_NONE) mirror_create(r); + if (r->mstate != MS_ON) return; + if (r->acc_sub != msub) mirror_sync(r); + sp->buffer = r->mbuf; +} + +void vk_mem_pick(VkMemSpan *sp) { + PoolRec *r = sp->pool; + if (!r || r->mstate != MS_ON) return; + if (r->acc_sub != msub) mirror_sync(r); + sp->buffer = r->mbuf; +} + +int vk_mem_mirror_probe(uint64_t addr) { + VkMemSpan sp; + if (!vk_mirror_on || !vk_mem_resolve(addr, 1, &sp) || !sp.pool) return 0; + mepoch = vk_texture_epoch(); + vk_mem_use(&sp); + return ((PoolRec *)sp.pool)->mstate == MS_ON; +} + +void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { + PoolRec *r = sp->pool; + if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; + range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size); +} + +void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { + if (!vk_mirror_on || !size || !p->mem) return; + InterlockedIncrement64(&m_flush_calls); + InterlockedAdd64(&m_flush_bytes, (LONG64)size); + AcquireSRWLockShared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)p->mem, 0); + PoolRec *r = e && e->key == p ? e->buf : NULL; + if (r && r->mstate == MS_ON) { + uint64_t a = (uintptr_t)p->mem + off, b = a + size; + AcquireSRWLockExclusive(&flush_lock); + if (!r->fl_hi) r->fl_lo = a, r->fl_hi = b; + else { + if (a < r->fl_lo) r->fl_lo = a; + if (b > r->fl_hi) r->fl_hi = b; + } + ReleaseSRWLockExclusive(&flush_lock); + } + ReleaseSRWLockShared(&pool_lock); +} + +void vk_mem_mirror_begin(void) { + if (!vk_mirror_on) return; + uint64_t t0 = kern_now_ns(); + uint32_t prev = msub; + msub = msub + 1 ? msub + 1 : 1; + mepoch = vk_texture_epoch(); + uint64_t q = 0, run0 = 0, run1 = 0; + int have = 0; + AcquireSRWLockShared(&pool_lock); + AcquireSRWLockExclusive(&gen_lock); + for (int i = 0; i < nm; i++) { + const PoolRec *r = mlist[i]; + if (r->acc_sub != prev) continue; + uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT; + if (have && a <= run1 + 1 + (uint64_t)mirror_gap) { + if (b > run1) run1 = b; + continue; + } + if (have) q += snap_regions(run0, run1, mepoch); + run0 = a, run1 = b, have = 1; + } + if (have) q += snap_regions(run0, run1, mepoch); + ReleaseSRWLockExclusive(&gen_lock); + ReleaseSRWLockShared(&pool_lock); + mrep.subs++; + mrep.presnap_queries += q; + mrep.cpu_ns += kern_now_ns() - t0; +} + +int vk_mem_mirror_pending(void) { return npend; } + +void vk_mem_mirror_prologue(VkCommandBuffer cb) { + if (!npend) return; + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies); + mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + npend = 0; + mrep.prologues++; +} + +// ---- verify mode +typedef struct { PoolRec *r; uint64_t off, len, dst; uint32_t mgen; } VSample; +typedef struct { uint32_t sub; int n; VkBuffer buf; VmaAllocation alloc; const uint8_t *map; VSample s[VERIFY_SAMPLES]; } VJob; +static int ver_i; +static uint64_t ver_off; + +// The CPU wrote the page since the mirror's sync that mgen records: stamped later, or its write-watch bit is set. +static int page_dirty(uint64_t a, uint32_t mgen) { + uint32_t *s = gen_slot(a, 0); + if (s && __atomic_load_n(s, __ATOMIC_RELAXED) > mgen) return 1; + uint64_t p[1]; + return mem_written_peek(a, 1ull << GEN_PAGE_SHIFT, p, 1) != 0; // UINT64_MAX (not watched) counts as written +} + +static void verify_cb(void *arg) { // completion thread + VJob *j = arg; + static volatile LONG logged; + uint64_t pages = 0, skipped = 0, bad = 0; + for (int i = 0; i < j->n; i++) { + const VSample *s = &j->s[i]; + PoolRec *r = s->r; + uint32_t wb = __atomic_load_n(&r->wb_sub, __ATOMIC_RELAXED); + if (wb != j->sub && wb - j->sub < 0x80000000u) { // a later submission wrote back into it + skipped += s->len >> GEN_PAGE_SHIFT; + continue; + } + for (uint64_t o = 0; o < s->len; o += 1ull << GEN_PAGE_SHIFT) { + uint64_t a = r->import_lo + s->off + o; + const uint8_t *m = j->map + s->dst + o, *g = (const uint8_t *)(uintptr_t)a; + pages++; + if (!memcmp(m, g, 1ull << GEN_PAGE_SHIFT)) continue; + if (page_dirty(a, s->mgen)) { skipped++; continue; } + bad++; + if (InterlockedIncrement(&logged) <= VERIFY_LOG) { + int k = 0, e = (1 << GEN_PAGE_SHIFT) - 1; + while (k < e && m[k] == g[k]) k++; + while (e > k && m[e] == g[e]) e--; + char f[160]; + flag_names(r->flags, f, sizeof f); + rt_log("mem: verify mismatch: pool %#llx+%#llx (%s) offset %#llx: bytes %#x..%#x of the page differ, " + "mirror %02x guest %02x at the first (submission %u)", (unsigned long long)r->lo, + (unsigned long long)r->size, f, (unsigned long long)(a - r->lo), k, e, m[k], g[k], j->sub); + } + } + } + InterlockedAdd64(&v_pages, (LONG64)pages); + InterlockedAdd64(&v_skipped, (LONG64)skipped); + InterlockedAdd64(&v_mismatch, (LONG64)bad); + vmaDestroyBuffer(vk_ctx()->vma, j->buf, j->alloc); + free(j); +} + +static int verify_record(VkCommandBuffer cb, uint64_t fence) { + VJob *j = calloc(1, sizeof *j); + if (!j) rt_fatal("vk: out of memory"); + uint64_t left = VERIFY_BYTES, dst = 0; + AcquireSRWLockShared(&pool_lock); + for (int tries = 0; nm && left && j->n < VERIFY_SAMPLES && tries <= nm; tries++) { + if (ver_i >= nm) ver_i = 0, ver_off = 0; + PoolRec *r = mlist[ver_i]; + uint64_t size = r->import_hi - r->import_lo; + if (ver_off >= size) { ver_i++, ver_off = 0; continue; } + uint64_t len = size - ver_off < left ? size - ver_off : left; + j->s[j->n++] = (VSample){r, ver_off, len, dst, r->mgen}; + dst += len, left -= len, ver_off += len; + if (ver_off >= size) ver_i++, ver_off = 0; + } + ReleaseSRWLockShared(&pool_lock); + if (!j->n) { free(j); return 0; } + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = dst, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; + VmaAllocationInfo info; + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &j->buf, &j->alloc, &info) != VK_SUCCESS) { free(j); return 0; } + j->map = info.pMappedData; + j->sub = msub; + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT); + for (int i = 0; i < j->n; i++) { + VkBufferCopy c = {j->s[i].off, j->s[i].dst, j->s[i].len}; + vkCmdCopyBuffer(cb, j->s[i].r->mbuf, j->buf, 1, &c); + } + mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT, + VK_ACCESS_2_HOST_READ_BIT); + vk_completion_add(fence, verify_cb, j); + return j->n + 2; +} + +int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { + if (!vk_mirror_on) return 0; + int rec = 0; + if (nwbl) { + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT); + for (int i = 0; i < nwbl; i++) wbl[i].r->wb_sub = msub; + mrep.wb_ranges += (uint64_t)nwbl; + rec += copy_ranges(cb, wbl, nwbl, 0, &mrep.wb_bytes, &mrep.wb_copies) + 2; + mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + nwbl = 0; + } + if (mirror_verify) rec += verify_record(cb, fence); + AcquireSRWLockExclusive(&mirror_stat_lock); + uint64_t *d = (uint64_t *)&macc, *s = (uint64_t *)&mrep; + for (size_t i = 0; i < sizeof macc / sizeof(uint64_t); i++) d[i] += s[i]; + ReleaseSRWLockExclusive(&mirror_stat_lock); + memset(&mrep, 0, sizeof mrep); + return rec; +} + +// One summary line: with TOTK_TRACE=perf every 60 frames, else at frames 600 and 1800. +static void mirror_log(uint64_t frame) { + static uint64_t last; + if (!vk_mirror_on) return; + AcquireSRWLockExclusive(&mirror_stat_lock); + MirrorAcc a = macc; + memset(&macc, 0, sizeof macc); + ReleaseSRWLockExclusive(&mirror_stat_lock); + double f = frame > last ? (double)(frame - last) : 1.0, M = 1048576.0; + last = frame; + uint64_t fc = (uint64_t)InterlockedExchange64(&m_flush_calls, 0), fb = (uint64_t)InterlockedExchange64(&m_flush_bytes, 0); + char ver[160] = ""; + if (mirror_verify) + snprintf(ver, sizeof ver, "; verify: pages compared %lld, skipped %lld (written since), mismatches %lld", + (long long)InterlockedExchange64(&v_pages, 0), (long long)InterlockedExchange64(&v_skipped, 0), + (long long)InterlockedExchange64(&v_mismatch, 0)); + rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked " + "%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, " + "first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries " + "%.1f pre-snap + %.1f sync, prologues %.2f, tracking cpu %.3f ms%s", + (unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET], + m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP], + (long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f, + a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M, + a.presnap_queries / f, a.sync_queries / f, a.prologues / f, a.cpu_ns / f / 1e6, ver); +} + // Device class (build/mem-plan.md): INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a -// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Only logged for now. +// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Sets the mirror mode. void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m) { int big_local = -1, host_heap = -1; for (uint32_t h = 0; h < m->memoryHeapCount; h++) { @@ -811,6 +1340,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic else if (big_local < 0) cls = "uma", why = "no device-local heap of 1 GiB or more"; else if (host_heap < 0) cls = "uma", why = "every heap is device-local"; else cls = "discrete", why = "device-local heap separate from host memory"; + int discrete = cls[0] == 'd'; char line[2048]; size_t k = (size_t)snprintf(line, sizeof line, "mem: class %s (%s; device type %d %s, %s)", cls, why, t, tn, p->deviceName); for (uint32_t h = 0; h < m->memoryHeapCount; h++) @@ -830,6 +1360,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic f & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT ? " LAZY" : ""); } rt_log("%s", line); + mirror_mode(discrete); } // ---------------------------------------------------------------- upload batches and deferred destruction @@ -1069,6 +1600,9 @@ void vk_mem_shutdown(void) { AcquireSRWLockExclusive(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); pool_index.n = 0; + nm = npend = nwbl = 0; // mirrors went with their pools + for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran + nretiring = 0; __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); nphys = 0; ReleaseSRWLockExclusive(&pool_lock); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 63a082f1..61234cef 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -107,27 +107,33 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order free(t); } -// At the start of a part's command buffer (sub_lock held). -static void ts_begin(VkCommandBuffer cb) { - ts_cur = -1; - if (ts_state != 1) return; +// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed. +static int ts_open(VkCommandBuffer cb) { + if (ts_state != 1) return -1; uint32_t p = ts_next % TS_PAIRS; - if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return; } + if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return -1; } ts_next++; ts_busy[p] = 1; vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2); vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p); - ts_cur = (int)p; + return (int)p; } -// At the end of the part, before vkEndCommandBuffer; fence: the value whose completion covers the part. -static void ts_end(VkCommandBuffer cb, uint64_t fence) { - if (ts_cur < 0) return; - vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)ts_cur + 1); +// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it. +static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence) { + if (pair < 0) return; + vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1); TsRead *t = malloc(sizeof *t); if (!t) rt_fatal("vk: out of memory"); - t->pair = (uint32_t)ts_cur; + t->pair = (uint32_t)pair; vk_completion_add(fence, ts_read_cb, t); +} + +// The pair of the part being recorded. +static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); } + +static void ts_end(VkCommandBuffer cb, uint64_t fence) { + ts_close(cb, ts_cur, fence); ts_cur = -1; } @@ -227,27 +233,47 @@ void vk_sync_start(void) { ReleaseSRWLockExclusive(&start_lock); } -// A command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held. -static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { +// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move +// slots[]. +static int claim_slot(VkCtx *c, uint64_t fence) { uint64_t gpu = 0; vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu); for (int i = 0; i < nslots; i++) if (slots[i].fence <= gpu) { slots[i].fence = fence; - cur_slot = i; - return &slots[i]; + return i; } CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1)); if (!n) rt_fatal("vk: out of memory (command buffers)"); slots = n; CmdSlot *s = &slots[nslots]; - cur_slot = nslots; VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation"); s->fence = fence; - nslots++; - return s; + return nslots++; +} + +static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { + cur_slot = claim_slot(c, fence); + return &slots[cur_slot]; +} + +// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is +// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the +// part was recorded, sub_lock held; may move slots[]. +static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { + if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE; + int i = claim_slot(c, fence); // before reading slots: the claim may move it + VkCommandBuffer cb = slots[i].cb; + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(cb, &bi); + int pair = ts_open(cb); + vk_mem_mirror_prologue(cb); + ts_close(cb, pair, fence); + vkEndCommandBuffer(cb); + return cb; } VkCommandBuffer vk_sync_split(void) { @@ -256,8 +282,11 @@ VkCommandBuffer vk_sync_split(void) { ts_end(cb, cur_fence); vkEndCommandBuffer(cb); vk_upload_flush(0); // the uploads this part asked for precede it in queue order - VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = 1, .pCommandBufferInfos = &cbi}; + VkCommandBuffer pro = prologue(c, cur_fence); + VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, + {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}}; + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1, + .pCommandBufferInfos = pro ? cbi : cbi + 1}; vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); @@ -292,11 +321,16 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { perf_replay(t0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. vk_upload_flush(0); - VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}; + VkCommandBuffer pro = prologue(c, fence); + s = &slots[cur_slot]; + VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, + {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}}; VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = recorded ? 1 : 0, - .pCommandBufferInfos = &cbi, .signalSemaphoreInfoCount = 1, .pSignalSemaphoreInfos = &sig}; + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, + .commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u), + .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1, + .pSignalSemaphoreInfos = &sig}; vk_queue_lock(); VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); @@ -365,6 +399,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); } +uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); } + void vk_sync_finish(void) { vk_sync_start(); vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index f8484921..68f10f22 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -888,6 +888,8 @@ int vk_texture_slot_swz(int id, uint8_t swz[4]) { return ok; } +uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); } + void vk_texture_new_submit(void) { lock(); if (!++submit_epoch) submit_epoch = 1; From 51567ac32ac34a791ec20d4d30b08d0575ee73da Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 25 Oct 2026 08:39:47 +0300 Subject: [PATCH 2/2] Freeze idle pool regions read-only so write-watch queries skip them The mirror and the texture checks queried about 930 MiB of write-watched memory per frame (0.96 ms on the RX 9070 machine, about 3 ms on an 8th-gen i7), at about 1 us per MiB, and 770 MiB of it lay in 2 MiB regions nothing had written for two seconds. In mirror mode a region left unwritten for TOTK_FREEZE_IDLE submissions (300; 0 turns it off) and lying wholly in imported pools is now made read-only (memory.c mem_freeze), and the snapshot and change queries skip it. The first write faults and a vectored handler thaws the region before the write retries; file reads into guest memory, which the kernel would refuse on a read-only page, thaw their buffer first and log a refusal, as do pool init and finalize, and commits, decommits and protection changes thaw under the same lock. A region stays thawed for 5 s after a file read. Field: 66 queries over 380 MiB, 0.56 ms per frame; 0.38 ms of it are the queries that find writes. The perf line `mem: write-watch` gives queries, bytes, time, frozen MiB, freezes and thaws. The gfx-abi probe covers freeze, thaw by a write and the hold after mem_thaw. TOTK_MEMORY, TOTK_MIRROR_GAP and TOTK_FREEZE_IDLE are known settings now. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 4 +- runtime/fs_host.c | 8 ++- runtime/gfx_abi_probe.c | 17 +++++ runtime/host.h | 9 +++ runtime/memory.c | 133 +++++++++++++++++++++++++++++++++-- runtime/romfs.c | 9 ++- runtime/vk_memory.c | 151 +++++++++++++++++++++++++++++++++++----- 7 files changed, 303 insertions(+), 28 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index ef5edf4a..59bc2854 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -27,10 +27,10 @@ static const char *const known[] = { "BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "CLOTH_REALTIME", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", - "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "GPL", + "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "FREEZE_IDLE", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", - "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MINIMAL_LAYOUT", "MOTION", "MOUSE_SENS", "MOVIE_HOST", + "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SYNC", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", diff --git a/runtime/fs_host.c b/runtime/fs_host.c index abd04ce9..83577a31 100644 --- a/runtime/fs_host.c +++ b/runtime/fs_host.c @@ -691,7 +691,13 @@ static File *file_of(uint64_t h, HANDLE *fh, int *mode) { static int64_t host_read_at(HANDLE f, void *buf, uint32_t n, uint64_t off) { OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32)}; DWORD got = 0; - if (!ReadFile(f, buf, n, &got, &ov)) return GetLastError() == ERROR_HANDLE_EOF ? 0 : -1; + mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting + if (!ReadFile(f, buf, n, &got, &ov)) { + DWORD e = GetLastError(); + if (e == ERROR_NOACCESS) // a read-only page in the buffer: a frozen region missed its thaw + rt_log("fs: read of %u bytes into %p refused: the buffer is not writable", n, buf); + return e == ERROR_HANDLE_EOF ? 0 : -1; + } return got; } diff --git a/runtime/gfx_abi_probe.c b/runtime/gfx_abi_probe.c index edc93dfa..e84bc62c 100644 --- a/runtime/gfx_abi_probe.c +++ b/runtime/gfx_abi_probe.c @@ -5,6 +5,7 @@ // Needs no game files. #include #include +#include #include "gfx.h" @@ -60,6 +61,22 @@ static void check_write_watch(void) { printf("write-watch: three pages written, max 2: %llu then %llu\n", (unsigned long long)n, (unsigned long long)m); CHECK(n == 2 && m == 1, "max 2 reported %llu, then %llu (want 2, 1)", (unsigned long long)n, (unsigned long long)m); + // Frozen region: read-only until the first write thaws it (the write lands and is reported); mem_thaw (a file + // read) thaws it and holds it thawed; SetHeapSize below decommits it. + mem_written_pages(heap, 0x200000, pages, 8); + CHECK(mem_freeze(heap) == 1 && mem_frozen(heap), "freeze of an idle committed 2 MiB region"); + MEMORY_BASIC_INFORMATION mbi; + CHECK(VirtualQuery((void *)(uintptr_t)heap, &mbi, sizeof mbi) && mbi.Protect == PAGE_READONLY, "frozen region is not read-only"); + p[0x3000] = 7; + CHECK(!mem_frozen(heap) && p[0x3000] == 7, "a write into a frozen region did not thaw it"); + n = mem_written_pages(heap, 0x200000, pages, 8); + CHECK(n == 1 && pages[0] == heap + 0x3000, "write after a thaw: %llu pages reported (want 1)", (unsigned long long)n); + CHECK(mem_freeze(heap) == 1, "refreeze after a write thaw"); + mem_thaw(heap + 0x6000, 0x1000); + CHECK(!mem_frozen(heap) && mem_freeze(heap) == 0, "mem_thaw did not thaw and hold the region"); + p[0x6000] = 8; + printf("write-watch: frozen region thawed by a write (reported once) and by mem_thaw (held)\n"); + uint64_t a = g->alias.base + 0x10000; CHECK(mem_map_physical(a, 0x4000) == 0, "MapPhysicalMemory"); mem_written_pages(a, 0x4000, pages, 8); diff --git a/runtime/host.h b/runtime/host.h index 8603b6a1..96626a6e 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -131,6 +131,15 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64 uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max); // the same without the reset: the bits stay for the next // mem_written_pages (verify checks only) +int mem_freeze(uint64_t a); // makes the 2 MiB region at a (2 MiB aligned, watched, + // committed PAGE_READWRITE) read-only until a write or + // mem_thaw thaws it; 1 frozen (or already), 0 refused +int mem_frozen(uint64_t a); // the 2 MiB region of a is frozen: no page in it changed + // since its freeze +void mem_thaw(uint64_t a, uint64_t n); // makes the frozen regions over [a, a+n) writable again; + // before a kernel write (file read) into guest memory +void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws); + // regions frozen now; freezes and thaws since the last call void mem_kernel_free(void *p, uint64_t size); // TLR/TLS reuse after thread exit // ---- modules (loader.c, T1) diff --git a/runtime/memory.c b/runtime/memory.c index 7f8c6494..6d0770b8 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -67,21 +67,142 @@ static void *reserve_at(uint64_t at, uint64_t size, DWORD extra) { return p; } +// ---- frozen regions +// A 2 MiB region of the watched reservations that nothing wrote for a while can be made read-only (mem_freeze), so the +// back end's write-watch queries skip it: no page of a frozen region changes before a thaw. The first write faults and +// thaw_handler makes the region writable again before the write retries. A kernel write into guest memory (a file +// read) fails on a read-only page instead of faulting, so file reads thaw their buffer first (mem_thaw), and so does +// every commit, decommit and protection change here. A region stays thawed for FRZ_HOLD_MS after a mem_thaw: a file +// read still in flight writes its buffer later. frz_lock is a leaf lock: nothing writes guest memory under it. +#define FRZ_SHIFT 21 +#define FRZ_SIZE (1ull << FRZ_SHIFT) +#define FRZ_HOLD_MS 5000u +static volatile uint8_t frz[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; +static uint32_t thaw_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: GetTickCount of the last thaw request, 0 never +static volatile LONG nfrz; +static SRWLOCK frz_lock = SRWLOCK_INIT; +static volatile LONG64 frz_count[4]; // freezes; thaws by a write fault, by mem_thaw, by a mapping change + +static int watched(uint64_t addr, uint64_t size) { + const RtGuestLayout *g = &L; + return (addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size) || + (addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size); +} + +// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. +static void thaw_locked(uint64_t r, int why) { + if (why == 2) thaw_ms[r] = (uint32_t)GetTickCount64() | 1; + if (!frz[r]) return; + DWORD old; + uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE; + if (!VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READWRITE, &old)) { + // Not all committed (cannot happen while mapping changes thaw under frz_lock): the read-only runs only. + MEMORY_BASIC_INFORMATION mbi; + while (a < end && VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi)) { + uint64_t b = (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize; + if (b > end) b = end; + if (mbi.State == MEM_COMMIT && mbi.Protect == PAGE_READONLY && + !VirtualProtect((void *)(uintptr_t)a, b - a, PAGE_READWRITE, &old)) + rt_fatal("memory: thawing %#llx+%#llx failed (error %lu)", (unsigned long long)a, + (unsigned long long)(b - a), GetLastError()); + a = b; + } + } + __atomic_store_n(&frz[r], 0, __ATOMIC_RELEASE); + InterlockedDecrement(&nfrz); + InterlockedIncrement64(&frz_count[why]); +} + +// frz_lock held exclusively. +static void thaw_range_locked(uint64_t a, uint64_t n, int why) { + if (!n || a >= RT_ADDRESS_LIMIT) return; + uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT; + for (uint64_t r = r0; r <= r1; r++) thaw_locked(r, why); +} + +void mem_thaw(uint64_t a, uint64_t n) { + AcquireSRWLockExclusive(&frz_lock); // also when nothing is frozen: the hold keeps a read's buffer writable + thaw_range_locked(a, n, 2); + ReleaseSRWLockExclusive(&frz_lock); +} + +int mem_freeze(uint64_t a) { + if ((a & (FRZ_SIZE - 1)) || !watched(a, FRZ_SIZE)) return 0; + uint64_t r = a >> FRZ_SHIFT; + MEMORY_BASIC_INFORMATION mbi; + DWORD old; + int ok = 0; + AcquireSRWLockExclusive(&frz_lock); + if (frz[r]) ok = 1; + else if (thaw_ms[r] && (int32_t)((uint32_t)GetTickCount64() - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) ok = 0; // |1 may be ahead + else if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT && + mbi.Protect == PAGE_READWRITE && (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize >= a + FRZ_SIZE && + VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READONLY, &old)) { + __atomic_store_n(&frz[r], 1, __ATOMIC_RELEASE); + InterlockedIncrement(&nfrz); + InterlockedIncrement64(&frz_count[0]); + ok = 1; + } + ReleaseSRWLockExclusive(&frz_lock); + return ok; +} + +int mem_frozen(uint64_t a) { return a < RT_ADDRESS_LIMIT && __atomic_load_n(&frz[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE); } + +void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws) { + *frozen = (uint64_t)__atomic_load_n(&nfrz, __ATOMIC_RELAXED); + *freezes = (uint64_t)InterlockedExchange64(&frz_count[0], 0); + *fault_thaws = (uint64_t)InterlockedExchange64(&frz_count[1], 0); + *other_thaws = (uint64_t)InterlockedExchange64(&frz_count[2], 0) + (uint64_t)InterlockedExchange64(&frz_count[3], 0); +} + +// The first write into a frozen region thaws it and retries. A fault in a region another thread thawed meanwhile +// retries too; any other fault goes on to the crash handler. +static LONG CALLBACK thaw_handler(EXCEPTION_POINTERS *ep) { + const EXCEPTION_RECORD *er = ep->ExceptionRecord; + if (er->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || er->NumberParameters < 2 || er->ExceptionInformation[0] != 1) + return EXCEPTION_CONTINUE_SEARCH; + uint64_t a = er->ExceptionInformation[1]; + if (a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return EXCEPTION_CONTINUE_SEARCH; + AcquireSRWLockExclusive(&frz_lock); + int was = frz[a >> FRZ_SHIFT]; + thaw_locked(a >> FRZ_SHIFT, 1); + ReleaseSRWLockExclusive(&frz_lock); + if (was) return EXCEPTION_CONTINUE_EXECUTION; + MEMORY_BASIC_INFORMATION mbi; + if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT && mbi.Protect == PAGE_READWRITE) + return EXCEPTION_CONTINUE_EXECUTION; + return EXCEPTION_CONTINUE_SEARCH; +} + +// Mapping changes thaw the regions they touch and run under frz_lock, so no freeze sees them half done. static void commit(uint64_t a, uint64_t n, const char *what) { - if (n && !VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE)) + AcquireSRWLockExclusive(&frz_lock); + thaw_range_locked(a, n, 3); + int ok = !n || VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE); + ReleaseSRWLockExclusive(&frz_lock); + if (!ok) rt_fatal("memory: commit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a, (unsigned long long)n, GetLastError()); } static void decommit(uint64_t a, uint64_t n, const char *what) { - if (n && !VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT)) + AcquireSRWLockExclusive(&frz_lock); + thaw_range_locked(a, n, 3); + int ok = !n || VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT); + ReleaseSRWLockExclusive(&frz_lock); + if (!ok) rt_fatal("memory: decommit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a, (unsigned long long)n, GetLastError()); } static void protect(uint64_t a, uint64_t n, DWORD prot, const char *what) { DWORD old; - if (n && !VirtualProtect((void *)(uintptr_t)a, n, prot, &old)) + AcquireSRWLockExclusive(&frz_lock); + thaw_range_locked(a, n, 3); + int ok = !n || VirtualProtect((void *)(uintptr_t)a, n, prot, &old); + ReleaseSRWLockExclusive(&frz_lock); + if (!ok) rt_fatal("memory: protect %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a, (unsigned long long)n, GetLastError()); } @@ -129,6 +250,7 @@ void mem_init(void) { if (reserve_at(at, ALIAS_SIZE, MEM_WRITE_WATCH)) alias = at; if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39"); + AddVectoredExceptionHandler(1, thaw_handler); // before the crash handler alias_pages = VirtualAlloc(NULL, ALIAS_PAGES, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); if (!alias_pages) rt_fatal("memory: cannot allocate the alias page map"); const char *pool = cfg_get("TOTK_APP_POOL"); @@ -150,10 +272,7 @@ const RtGuestLayout *mem_layout(void) { return &L; } // with the reset flag. A range inside one watched reservation (stack and heap block, or alias region) only. static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, DWORD flags) { if (!max || !size) return 0; - const RtGuestLayout *g = &L; - int low_block = addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size; - int alias = addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size; - if (!low_block && !alias) return UINT64_MAX; + if (!watched(addr, size)) return UINT64_MAX; ULONG_PTR n = (ULONG_PTR)max; DWORD gran; _Static_assert(sizeof(PVOID) == sizeof(uint64_t), "page addresses are written straight into a uint64_t array"); diff --git a/runtime/romfs.c b/runtime/romfs.c index b1c14f15..46b50f88 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -84,11 +84,16 @@ static int64_t read_at(HANDLE f, void *buf, uint32_t n, uint64_t off) { rt_fatal("RomFS: CreateEvent failed (%lu)", GetLastError()); OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32), .hEvent = ev}; DWORD got = 0; + mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting if (!ReadFile(f, buf, n, &got, &ov)) { DWORD e = GetLastError(); if (e == ERROR_HANDLE_EOF) return 0; - if (e != ERROR_IO_PENDING) return -1; - if (!GetOverlappedResult(f, &ov, &got, TRUE)) return GetLastError() == ERROR_HANDLE_EOF ? 0 : -1; + if (e == ERROR_IO_PENDING && !GetOverlappedResult(f, &ov, &got, TRUE)) e = GetLastError(); + else if (e == ERROR_IO_PENDING) return got; + if (e == ERROR_HANDLE_EOF) return 0; + if (e == ERROR_NOACCESS) // a read-only page in the buffer: a frozen region missed its thaw + rt_log("RomFS: read of %u bytes into %p refused: the buffer is not writable", n, buf); + return -1; } return got; } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 58b5e4bc..f424a3a3 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -22,6 +22,23 @@ #include "nvn_pool_index.h" #include "vk.h" +// Raw performance counter ticks for the tracking cost (no division on the replay's path); qpc_ms converts. +static uint64_t qpc(void) { + LARGE_INTEGER t; + QueryPerformanceCounter(&t); + return (uint64_t)t.QuadPart; +} + +static double qpc_ms(uint64_t ticks) { + static double per_ms; + if (!per_ms) { + LARGE_INTEGER f; + QueryPerformanceFrequency(&f); + per_ms = (double)f.QuadPart / 1000.0; + } + return (double)ticks / per_ms; +} + // ---------------------------------------------------------------- pools enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT }; enum { MS_NONE, MS_ON, MS_REFUSED }; @@ -242,6 +259,7 @@ int vk_pool_init(pool_t *p) { ok = alloc_physical(c, r); } else { r->kind = POOL_IMPORT; + mem_thaw((uintptr_t)p->mem, p->size); // the import may need write access to the pages ok = import_pool(c, r, (uintptr_t)p->mem, p->size); if (!ok) { // Not fatal for the game: the pool works for the CPU, and vk_mem_resolve reports its addresses as unbacked. @@ -288,6 +306,7 @@ void vk_pool_fini(pool_t *p) { } ReleaseSRWLockExclusive(&pool_lock); if (!r) return; + if (r->kind == POOL_IMPORT) mem_thaw(r->lo, r->size); // the storage goes back to the guest writable // Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the // guest gets the storage back when this returns. vk_texture_range_gone(r->lo, r->size, r->alloc); @@ -439,7 +458,8 @@ static void mirror_log(uint64_t frame); void vk_mem_census(uint64_t frame) { static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); - if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame); + if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame); // and the + // write-watch line if (frame == 1200) { // loading is over: the first window starts here for (int c = 0; c < VK_MCLS_N; c++) for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0); @@ -525,6 +545,32 @@ static uint64_t gen_pages[1024]; #define REG_SHIFT 21 static uint32_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT]; +// Every write-watch query of the generation table (gen_lock held exclusively): calls, bytes and time since the last +// mirror summary, all consumers together; the queries that found written pages (they also flush TLBs) apart. +static uint64_t ww_calls, ww_bytes, ww_ticks, ww_dirty_calls, ww_dirty_ticks, ww_pages; + +static uint64_t ww_query(uint64_t addr, uint64_t size) { + uint64_t t0 = qpc(); + uint64_t n = mem_written_pages(addr, size, gen_pages, sizeof gen_pages / sizeof *gen_pages); + uint64_t dt = qpc() - t0; + ww_ticks += dt, ww_calls++, ww_bytes += size; + if (n && n != UINT64_MAX) ww_dirty_calls++, ww_dirty_ticks += dt, ww_pages += n; + return n; +} + +// Frozen regions (mirror mode): a 2 MiB region whose pages nothing wrote for freeze_idle submissions, and which lies +// wholly in imported pools, is made read-only (memory.c mem_freeze), and write-watch queries skip it until a write or a +// file read thaws it. Most of the pool memory the mirror and the texture checks cover is static (meshes, textures), and +// a query costs about 1 us per MiB, so this keeps the queries to the regions the CPU writes. +#define FREEZE_PER_SUB 32 +static int freeze_idle; // 0: off +static uint32_t ww_ep = 1; // gen_lock: the submission being recorded (vk_mem_mirror_begin) +static uint32_t reg_last[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of the region's last stamp +static uint32_t reg_tried[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of its last freeze candidacy +static uint32_t *fcand; // gen_lock: regions to try at the next freeze pass +static int nfcand, cfcand; +static uint64_t n_frz_nopool, n_frz_refused; // gen_lock: candidates not wholly in pools, refused by mem_freeze + static uint32_t *gen_slot(uint64_t page_addr, int create) { uint64_t ci = page_addr >> GEN_CHUNK_SHIFT; if (ci >= GEN_CHUNKS) return NULL; @@ -542,6 +588,7 @@ static void stamp_page(uint64_t a, uint32_t g) { if (!s) return; *s = g; reg_max[a >> REG_SHIFT] = g; + reg_last[a >> REG_SHIFT] = ww_ep; } static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) { @@ -561,9 +608,10 @@ int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now) uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; AcquireSRWLockExclusive(&gen_lock); uint32_t g = ++gen_now; - int tracked = 1; - for (;;) { - uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages); + int tracked = 1, frozen = 1; + for (uint64_t a = lo & ~((1ull << REG_SHIFT) - 1); a < hi && frozen; a += 1ull << REG_SHIFT) frozen = mem_frozen(a); + while (!frozen) { + uint64_t n = ww_query(lo, hi - lo); if (n == UINT64_MAX) { tracked = 0; break; } for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g); if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call @@ -596,19 +644,32 @@ static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) { uint64_t calls = 0; for (uint64_t r = r0; r <= r1;) { if (snap_at[r] == epoch) { r++; continue; } + if (mem_frozen(r << SNAP_SHIFT)) { // nothing in it changed since its freeze + snap_at[r] = epoch, snap_untracked[r] = 0; + r++; + continue; + } uint64_t k = r; // one query over a run of regions not seen in this epoch - while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++; + while (k + 1 <= r1 && snap_at[k + 1] != epoch && !mem_frozen((k + 1) << SNAP_SHIFT)) k++; uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT; int tracked = 1; if (!g) g = ++gen_now; for (;;) { - uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages); + uint64_t n = ww_query(a, b - a); if (n == UINT64_MAX) { tracked = 0; break; } for (uint64_t j = 0; j < n; j++) stamp_page(gen_pages[j], g); if (n < sizeof gen_pages / sizeof *gen_pages) break; } calls++; - for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked; + for (uint64_t j = r; j <= k; j++) { + snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked; + if (tracked && freeze_idle && ww_ep - reg_last[j] >= (uint32_t)freeze_idle && + ww_ep - reg_tried[j] >= (uint32_t)freeze_idle) { + reg_tried[j] = ww_ep; + if (nfcand == cfcand) grow((void **)&fcand, &cfcand, sizeof *fcand); + fcand[nfcand++] = (uint32_t)j; + } + } r = k + 1; } return calls; @@ -696,7 +757,7 @@ static uint64_t page_hi(const PoolRec *r) { return (r->lo + r->size + 0xfff) & ~ // gen_lock held exclusively. 0: the range is not write-watched. static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { for (;;) { - uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages); + uint64_t n = ww_query(lo, hi - lo); if (n == UINT64_MAX) return 0; for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g); if (n < sizeof gen_pages / sizeof *gen_pages) return 1; @@ -867,7 +928,7 @@ static SRWLOCK flush_lock = SRWLOCK_INIT; typedef struct { uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges; - uint64_t presnap_queries, sync_queries, cpu_ns, prologues; + uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes; } MirrorAcc; static MirrorAcc mrep; // replay: this submission static MirrorAcc macc; // mirror_stat_lock: since the last summary @@ -890,6 +951,9 @@ static void mirror_mode(int discrete) { const char *g = cfg_get("TOTK_MIRROR_GAP"); mirror_gap = g && *g ? atoi(g) : 2; if (mirror_gap < 0) mirror_gap = 0; + const char *fi = cfg_get("TOTK_FREEZE_IDLE"); // submissions a region stays unwritten before it is frozen + freeze_idle = !vk_mirror_on ? 0 : fi && *fi ? atoi(fi) : 300; + if (freeze_idle < 0) freeze_idle = 0; rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why, mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory"); } @@ -1051,7 +1115,7 @@ static void mirror_retire(void *arg) { // Replay thread, the first access of this submission: queues the copies of the pages written since the last sync. static void mirror_sync(PoolRec *r) { - uint64_t t0 = kern_now_ns(); + uint64_t t0 = qpc(); r->acc_sub = msub; uint64_t lo = r->import_lo, hi = r->import_hi; uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; @@ -1089,7 +1153,7 @@ static void mirror_sync(PoolRec *r) { if (a < b) range_add(&pend, &npend, &cpend, r, a - lo, b - a); } mrep.syncs++; - mrep.cpu_ns += kern_now_ns() - t0; + mrep.sync_t += qpc() - t0; } void vk_mem_use(VkMemSpan *sp) { @@ -1141,9 +1205,42 @@ void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { ReleaseSRWLockShared(&pool_lock); } +// pool_lock shared: [a, b) lies wholly in the host ranges of imported pools with a GPU buffer. +static int pools_cover(uint64_t a, uint64_t b) { + uint64_t lo[64], hi[64]; + int n = 0; + for (int i = pidx_upper(&pool_index, b - 1) - 1; i >= 0 && pool_index.maxhi[i] > a; i--) { + const PoolRec *r = pool_index.e[i].buf; + if (r->kind != POOL_IMPORT || !r->buf || r->import_hi <= a || r->import_lo >= b) continue; + if (n == 64) return 0; + int at = n++; + for (; at > 0 && lo[at - 1] > r->import_lo; at--) lo[at] = lo[at - 1], hi[at] = hi[at - 1]; + lo[at] = r->import_lo, hi[at] = r->import_hi; + } + uint64_t end = a; + for (int i = 0; i < n && end < b; i++) { + if (lo[i] > end) return 0; + if (hi[i] > end) end = hi[i]; + } + return end >= b; +} + +// pool_lock shared, gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the +// protection collects the writes made before it took effect. +static void freeze_pass(void) { + for (int done = 0; nfcand && done < FREEZE_PER_SUB;) { + uint64_t j = fcand[--nfcand], a = j << REG_SHIFT, b = a + (1ull << REG_SHIFT); + if (ww_ep - reg_last[j] < (uint32_t)freeze_idle || mem_frozen(a)) continue; + if (!pools_cover(a, b)) { n_frz_nopool++; continue; } + done++; + if (!mem_freeze(a)) { n_frz_refused++; continue; } + query_stamp(a, b, ++gen_now); + } +} + void vk_mem_mirror_begin(void) { if (!vk_mirror_on) return; - uint64_t t0 = kern_now_ns(); + uint64_t t0 = qpc(); uint32_t prev = msub; msub = msub + 1 ? msub + 1 : 1; mepoch = vk_texture_epoch(); @@ -1151,6 +1248,8 @@ void vk_mem_mirror_begin(void) { int have = 0; AcquireSRWLockShared(&pool_lock); AcquireSRWLockExclusive(&gen_lock); + ww_ep++; + uint64_t b0 = ww_bytes; for (int i = 0; i < nm; i++) { const PoolRec *r = mlist[i]; if (r->acc_sub != prev) continue; @@ -1163,11 +1262,13 @@ void vk_mem_mirror_begin(void) { run0 = a, run1 = b, have = 1; } if (have) q += snap_regions(run0, run1, mepoch); + mrep.presnap_bytes += ww_bytes - b0; + if (freeze_idle) freeze_pass(); ReleaseSRWLockExclusive(&gen_lock); ReleaseSRWLockShared(&pool_lock); mrep.subs++; mrep.presnap_queries += q; - mrep.cpu_ns += kern_now_ns() - t0; + mrep.presnap_t += qpc() - t0; } int vk_mem_mirror_pending(void) { return npend; } @@ -1297,7 +1398,24 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { // One summary line: with TOTK_TRACE=perf every 60 frames, else at frames 600 and 1800. static void mirror_log(uint64_t frame) { - static uint64_t last; + static uint64_t last, ww_last; + AcquireSRWLockExclusive(&gen_lock); + uint64_t wc = ww_calls, wb = ww_bytes, wt = ww_ticks, dc = ww_dirty_calls, dt = ww_dirty_ticks, wp = ww_pages; + ww_pages = 0; + uint64_t np = n_frz_nopool, nr = n_frz_refused; + ww_calls = ww_bytes = ww_ticks = ww_dirty_calls = ww_dirty_ticks = n_frz_nopool = n_frz_refused = 0; + ReleaseSRWLockExclusive(&gen_lock); + double wf = frame > ww_last ? (double)(frame - ww_last) : 1.0; + ww_last = frame; + uint64_t fz, ff, ft, fo; + mem_freeze_stats(&fz, &ff, &ft, &fo); + if (rt_trace & RT_TRACE_PERF) + rt_log("mem: write-watch frame %llu: per frame, every consumer: %.1f queries over %.1f MiB, %.3f ms (%.1f found " + "%.0f written pages, %.3f ms); frozen %.0f MiB; since the last line: freezes %llu, thaws by a write %llu, by a file read " + "or mapping change %llu, candidates not wholly in pools %llu, refused %llu", + (unsigned long long)frame, wc / wf, wb / wf / 1048576.0, qpc_ms(wt) / wf, dc / wf, wp / wf, qpc_ms(dt) / wf, + fz * 2.0, (unsigned long long)ff, (unsigned long long)ft, (unsigned long long)fo, (unsigned long long)np, + (unsigned long long)nr); if (!vk_mirror_on) return; AcquireSRWLockExclusive(&mirror_stat_lock); MirrorAcc a = macc; @@ -1314,12 +1432,13 @@ static void mirror_log(uint64_t frame) { rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked " "%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, " "first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries " - "%.1f pre-snap + %.1f sync, prologues %.2f, tracking cpu %.3f ms%s", + "%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f)%s", (unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET], m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP], (long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f, a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M, - a.presnap_queries / f, a.sync_queries / f, a.prologues / f, a.cpu_ns / f / 1e6, ver); + a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f, + qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver); } // Device class (build/mem-plan.md): INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a