diff --git a/runtime/cfg.c b/runtime/cfg.c index 0ed1b184..4a071fcf 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -46,7 +46,7 @@ static const char *const known[] = { "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", }; diff --git a/runtime/vk.h b/runtime/vk.h index 597e0e02..8bcfe00d 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -162,6 +162,14 @@ void vk_mem_pick(VkMemSpan *sp); #define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0) #define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0) void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size); +// Uniform blocks as of the submit call (vk_usnap.c): vk_exec_record copies the uniform ranges a submission binds before +// it records, ubo bindings take the copy (0: not copied, bind memory), GPU writes of the replay drop the copies they touch. +void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence); +void vk_usnap_end(uint64_t fence); +int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out); +void vk_usnap_drop(uint64_t addr, uint64_t n); // the replay makes the GPU write [addr, addr+n) of guest memory +uint64_t vk_usnap_gen(void); // changes whenever a binding may resolve differently +void vk_usnap_shutdown(void); // vk_exec_draw_shutdown: frees the arenas void vk_mem_mirror_begin(void); int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); int vk_mem_mirror_pending(void); diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index f0dbc146..c1540295 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -745,7 +745,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t split_draws = part_start = 0; part_t0 = kern_now_ns(); x_here = 1; - if (n) vk_mem_mirror_begin(); + if (n) vk_mem_mirror_begin(), vk_usnap_begin(handles, n, fence); for (int i = 0; i < n; i++) { if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i); walk(ex, nvn_op_first(handles[i]), 0); @@ -753,7 +753,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_pass_end(ex); x_here = 0; if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes - if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence); // VRAM mirror bytes the GPU wrote go back to guest memory + if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory if (ex->nviews) { ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); if (!l) rt_fatal("vk: out of memory"); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 63d4ee5d..b39a87a8 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -386,6 +386,7 @@ void vk_exec_draw_shutdown(void) { for (int i = 0; i < nspare_pool; i++) vkDestroyDescriptorPool(c->device, spare_pool[i], NULL); for (int i = 0; i < nspare_chunk; i++) vmaDestroyBuffer(c->vma, spare_chunk[i].buf, spare_chunk[i].alloc); nspare_pool = nspare_chunk = 0; + vk_usnap_shutdown(); vkDestroyBufferView(c->device, Z.zview, NULL); vmaDestroyBuffer(c->vma, Z.zbuf, Z.zalloc); memset(&Z, 0, sizeof Z); @@ -775,7 +776,7 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, if (sp.offset % align == 0) { VK_MEM_USE(sp); *out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n}; - if (written) vk_mem_gpu_wrote(&sp, n); + if (written) vk_mem_gpu_wrote(&sp, n), vk_usnap_drop(addr, n); count_use(sp.cls, use, n); VK_MEM_BOUND(sp); return 1; @@ -878,6 +879,10 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { } } else if (slot >= 3 && slot - 3 < 18) { const p_bind_buf *b = &ex->ubo[st][slot - 3]; + if (b->size && vk_usnap_find(b->addr, b->size < max ? b->size : max, align, &bi)) { // as of the submit call + count_use(VK_MCLS_SCRATCH, VK_MUSE_UBO, bi.range); + return bi; + } if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, 0, &bi)) return bi; } return zero_info(max); @@ -1420,7 +1425,7 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { PUT(ex->prog->serial); PUT((uint64_t)vk_texture_slots_gen() << 32 | vk_texture_epoch()); PUT((uint64_t)vk_sampler_gen() << 32 | (uint32_t)ex->nfbl); - PUT(vk_mem_gen()); + PUT(vk_mem_gen() + vk_usnap_gen()); // both only grow for (int i = 0; i < ex->nfbl && i < 9; i++) PUT((uintptr_t)ex->fbl_img[i]); if (l->sup) { float in[3]; @@ -2125,6 +2130,7 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { VkBufferCopy cp = {o, sp.offset, u->size}; vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp); vk_mem_gpu_wrote(&sp, u->size); + vk_usnap_drop(u->dst, u->size); ex->recorded++; vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size); note_written(ex, u->dst, u->size); diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index ed6a90f3..8c63e030 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -71,6 +71,7 @@ static void guest_written_cb(void *arg) { // The GPU wrote [addr, addr+size) of CPU-visible memory: tell the texture cache once the fence is reached. static void note_gpu_write(VkExec *ex, uint64_t addr, uint64_t size) { + vk_usnap_drop(addr, size); if (nvn_gpu_only(addr)) return; Range *r = malloc(sizeof *r); if (!r) rt_fatal("vk: out of memory"); diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c new file mode 100644 index 00000000..82815823 --- /dev/null +++ b/runtime/vk_usnap.c @@ -0,0 +1,309 @@ +// Uniform blocks as of the submit call. NVN starts the GPU when nvnQueueSubmitCommands returns and the GPU reads the +// constants soon after; here the replay records for several milliseconds, and the GPU reads a pool's bytes when it +// runs the part that uses them (the VRAM mirror's sync copies run then, vk_memory.c "VRAM mirror"). Meanwhile the game, +// having handed the frame over, rewrites per-object uniform blocks on other threads with no fence: late draws read +// another object's matrix (phantom rocks, a flat far field; TotK 1.2.1's producer f_70008c6460, 14-24 ms after the +// submit). So at the start of a submission's recording every uniform range it binds is copied into an arena of its own, +// and the uniform bindings of the replay use the copy. +// - Only ranges of imported pools (memory the CPU writes) are copied, aligned out to 256 bytes (the largest +// minUniformBufferOffsetAlignment) and at most 64 KiB from the bound address, overlapping ones merged. +// - A range the GPU wrote in a submission that has not completed is left out: its bytes reach guest memory only with +// that submission's write-back, the mirror holds them until then. +// - A GPU write within the submission (UpdateUniformBuffer, a storage buffer a shader writes, a buffer copy or fill, +// an image-to-buffer copy: vk_usnap_drop) drops the copies it touches from then on, so later draws read the result. +// - The arena goes back to a free list once the submission's fence is done (completion thread). +// The design follows MacNativeGames' TOTK_UBO_SNAPSHOT (games/totk/third_party/d2r_gfx/gfx/nvn/nvn_mtl.m usnap_take, +// usnap_drop, @ ffd8d10, MIT, Copyright (c) 2026 jhaizhou); the code is ours. +// The copy is taken where vk_exec_record starts: at the submit call when the replay runs on the submitting thread, later +// with TOTK_REPLAY_ASYNC. TOTK_UBO_SNAPSHOT=0 binds live memory as before; =ab alternates (TOTK_AB). +#include +#include +#include + +#include "host.h" +#include "nvn_api.h" +#include "vk.h" + +#define ALIGN 256u +#define MAX_RANGE (64u << 10) +#define ARENA_MIN (4u << 20) +#define CALL_DEPTH 16 + +typedef struct Arena { + struct Arena *next; + VkBuffer buf; + VmaAllocation alloc; + uint8_t *map; + uint64_t size; +} Arena; + +typedef struct { uint64_t lo, hi; } Rng; +typedef struct { uint64_t lo, hi, off; int dropped; } Reg; +typedef struct { uint64_t lo, hi, fence; } Wrote; + +static struct { + int mode; // rt_knob: 0 off, 1 on, 2 ab + int on; // this submission has a snapshot + uint64_t fence, gen; + Arena *arena; + Rng *col; // collected ranges + int ncol, ccol; + Reg *reg; // merged, sorted by lo, disjoint + int nreg, creg; + Wrote *wr; // GPU writes of submissions not known to be complete + int nwr, cwr; + // since the last summary + uint64_t subs, regions, bytes, copy_ns, hits, drops, inflight, refused, off_subs; +} U; + +static Arena *free_arenas; +static SRWLOCK free_lock = SRWLOCK_INIT; + +static void *grow(void *p, int *cap, size_t elem) { + int n = *cap ? *cap * 2 : 1024; + void *q = realloc(p, (size_t)n * elem); + if (!q) rt_fatal("vk_usnap: out of memory"); + *cap = n; + return q; +} + +static Arena *arena_get(uint64_t need) { + AcquireSRWLockExclusive(&free_lock); + Arena **pp = &free_arenas, *a = NULL; + for (; *pp; pp = &(*pp)->next) + if ((*pp)->size >= need) { + a = *pp, *pp = a->next; + break; + } + ReleaseSRWLockExclusive(&free_lock); + if (a) return a; + a = calloc(1, sizeof *a); + if (!a) rt_fatal("vk_usnap: out of memory"); + a->size = need < ARENA_MIN ? ARENA_MIN : (need + (1u << 20) - 1) & ~(uint64_t)((1u << 20) - 1); + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = a->size, + .usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT}; + // Device-local through the BAR when the device has it (the GPU reads constants on every draw), else host memory. + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | + VMA_ALLOCATION_CREATE_MAPPED_BIT}; + VmaAllocationInfo info; + VkResult res = vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &a->buf, &a->alloc, &info); + if (res != VK_SUCCESS) { + rt_log("vk_usnap: arena of %llu bytes failed (%d): this submission binds live memory", + (unsigned long long)a->size, (int)res); + free(a); + return NULL; + } + a->map = info.pMappedData; + rt_log("vk_usnap: new arena of %.1f MiB", a->size / 1048576.0); + return a; +} + +static void arena_done(void *arg) { // completion thread: the GPU has read it + Arena *a = arg; + AcquireSRWLockExclusive(&free_lock); + a->next = free_arenas, free_arenas = a; + ReleaseSRWLockExclusive(&free_lock); +} + +static void collect(const op_hdr *h, int depth) { + for (;;) { + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) + rt_fatal("vk_usnap: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); + const void *a = nvn_op_payload(h); + if (h->op == OP_END) return; + if (h->op == OP_JUMP) { + h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; + continue; + } + if (h->op == OP_CALL && depth < CALL_DEPTH) collect(nvn_op_first(((const p_jump *)a)->target), depth + 1); + if (h->op == OP_BIND_UBO) { + const p_bind_buf *b = a; + if (b->addr && b->size) { + if (U.ncol == U.ccol) U.col = grow(U.col, &U.ccol, sizeof *U.col); + uint64_t n = b->size < MAX_RANGE ? b->size : MAX_RANGE; + U.col[U.ncol++] = (Rng){b->addr & ~(uint64_t)(ALIGN - 1), b->addr + n}; + } + } + h = nvn_op_next(h); + } +} + +static int by_lo(const void *a, const void *b) { + uint64_t x = ((const Rng *)a)->lo, y = ((const Rng *)b)->lo; + return x < y ? -1 : x > y; +} +static int wr_by_lo(const void *a, const void *b) { + uint64_t x = ((const Wrote *)a)->lo, y = ((const Wrote *)b)->lo; + return x < y ? -1 : x > y; +} + +// Sorts the collected ranges by lo. A town frame binds about 10,000 uniform ranges and qsort took half a millisecond +// of the copy's 0.96 ms on the game thread, so they go through an LSD radix sort on (lo - lowest) / ALIGN, three +// passes of 11 bits; qsort is left for a span that does not fit in 33 bits (2 TiB). +#define RADIX_BITS 11 +static void sort_col(void) { + static Rng *tmp; + static int ctmp; + int n = U.ncol; + if (n < 2) return; + uint64_t lo = UINT64_MAX, hi = 0; + for (int i = 0; i < n; i++) { + if (U.col[i].lo < lo) lo = U.col[i].lo; + if (U.col[i].lo > hi) hi = U.col[i].lo; + } + if ((hi - lo) / ALIGN >> (3 * RADIX_BITS)) { + qsort(U.col, (size_t)n, sizeof *U.col, by_lo); + return; + } + if (ctmp < n) { + free(tmp); + ctmp = U.ccol; + if (!(tmp = malloc((size_t)ctmp * sizeof *tmp))) rt_fatal("vk_usnap: out of memory"); + } + Rng *src = U.col, *dst = tmp; + for (int pass = 0; pass < 3; pass++) { + int shift = pass * RADIX_BITS, count[1 << RADIX_BITS] = {0}; + for (int i = 0; i < n; i++) count[(src[i].lo - lo) / ALIGN >> shift & ((1 << RADIX_BITS) - 1)]++; + for (int k = 0, sum = 0; k < 1 << RADIX_BITS; k++) { + int c = count[k]; + count[k] = sum, sum += c; + } + for (int i = 0; i < n; i++) dst[count[(src[i].lo - lo) / ALIGN >> shift & ((1 << RADIX_BITS) - 1)]++] = src[i]; + Rng *t = src; + src = dst, dst = t; + } + if (src != U.col) memcpy(U.col, src, (size_t)n * sizeof *src); // three passes end in tmp +} + +// Keeps the GPU writes of submissions that are not done, merged into disjoint ranges sorted by lo (the terrain rewrites +// one block before every draw: hundreds of equal entries), each with the newest fence among its parts. +static void wr_prune(void) { + uint64_t done = vk_sync_fence_done(); + int n = 0; + for (int i = 0; i < U.nwr; i++) + if (U.wr[i].fence > done) U.wr[n++] = U.wr[i]; + qsort(U.wr, (size_t)n, sizeof *U.wr, wr_by_lo); + int m = 0; + for (int i = 0; i < n; i++) { + if (m && U.wr[i].lo <= U.wr[m - 1].hi) { + if (U.wr[i].hi > U.wr[m - 1].hi) U.wr[m - 1].hi = U.wr[i].hi; + if (U.wr[i].fence > U.wr[m - 1].fence) U.wr[m - 1].fence = U.wr[i].fence; + } else { + U.wr[m++] = U.wr[i]; + } + } + U.nwr = m; +} +static int wr_hits(uint64_t lo, uint64_t hi) { // wr_prune ran: disjoint, sorted + int a = 0, b = U.nwr; + while (a < b) { + int k = (a + b) / 2; + if (U.wr[k].hi <= lo) a = k + 1; + else b = k; + } + return a < U.nwr && U.wr[a].lo < hi; +} + +static void summary(void) { + if (U.subs + U.off_subs < 3600) return; + double s = U.subs ? (double)U.subs : 1.0; + rt_log("vk_usnap: %llu submissions with a snapshot, %llu without; per submission %.0f ranges, %.1f KiB copied in " + "%.3f ms, %.0f uniform bindings from the copy, %.1f dropped by GPU writes, %.1f left out (GPU writes in " + "flight), %.1f refused (not an imported pool)", + (unsigned long long)U.subs, (unsigned long long)U.off_subs, U.regions / s, U.bytes / s / 1024.0, + U.copy_ns / s / 1e6, U.hits / s, U.drops / s, U.inflight / s, U.refused / s); + U.subs = U.off_subs = U.regions = U.bytes = U.copy_ns = U.hits = U.drops = U.inflight = U.refused = 0; +} + +void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) { + if (!U.mode) U.mode = rt_knob("UBO_SNAPSHOT", "1") + 1; // stored +1: 0 is "not read yet" + U.on = 0, U.fence = fence, U.gen++; + summary(); + if (!RT_KNOB_ON(U.mode - 1)) { + U.off_subs++; + return; + } + uint64_t t0 = kern_now_ns(); + U.ncol = 0; + for (int i = 0; i < n; i++) + if (handles[i]) collect(nvn_op_first(handles[i]), 0); + sort_col(); + wr_prune(); + // merge, keeping only what lies wholly in one imported pool and no GPU write in flight touches + U.nreg = 0; + uint64_t total = 0; + for (int i = 0; i < U.ncol;) { + uint64_t lo = U.col[i].lo, hi = U.col[i].hi; + for (i++; i < U.ncol && U.col[i].lo <= hi; i++) + if (U.col[i].hi > hi) hi = U.col[i].hi; + VkMemSpan sp; + if (!vk_mem_resolve(lo, hi - lo, &sp) || sp.alloc || !sp.pool || sp.avail < hi - lo || + (sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) { + U.refused++; + continue; + } + if (wr_hits(lo, hi)) { + U.inflight++; + continue; + } + if (U.nreg == U.creg) U.reg = grow(U.reg, &U.creg, sizeof *U.reg); + U.reg[U.nreg++] = (Reg){lo, hi, total, 0}; + total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1); + } + if (!U.nreg) return; + if (!(U.arena = arena_get(total))) return; + for (int i = 0; i < U.nreg; i++) + memcpy(U.arena->map + U.reg[i].off, (const void *)(uintptr_t)U.reg[i].lo, U.reg[i].hi - U.reg[i].lo); + vmaFlushAllocation(vk_ctx()->vma, U.arena->alloc, 0, total); // a no-op on coherent memory + U.on = 1; + U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += total, U.copy_ns += kern_now_ns() - t0; +} + +void vk_usnap_end(uint64_t fence) { + if (U.arena) vk_completion_add(fence, arena_done, U.arena), U.arena = NULL; + U.on = 0, U.gen++; +} + +// First region whose end is above a. +static int first_above(uint64_t a) { + int lo = 0, hi = U.nreg; + while (lo < hi) { + int m = (lo + hi) / 2; + if (U.reg[m].hi <= a) lo = m + 1; + else hi = m; + } + return lo; +} + +int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out) { + if (!U.on || !n) return 0; + int i = first_above(addr); + if (i >= U.nreg) return 0; + const Reg *r = &U.reg[i]; + if (r->dropped || addr < r->lo || addr + n > r->hi) return 0; + uint64_t off = r->off + (addr - r->lo); + if (off % align) return 0; + *out = (VkDescriptorBufferInfo){U.arena->buf, off, n}; + U.hits++; + return 1; +} + +void vk_usnap_drop(uint64_t addr, uint64_t n) { + if (!U.mode || !n) return; + if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr); + U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence}; + if (!U.on) return; + for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++) + if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, U.gen++; +} + +uint64_t vk_usnap_gen(void) { return U.gen; } + +// The arenas back on the free list (vk_exec_draw_shutdown: the device is idle, every completion ran). +void vk_usnap_shutdown(void) { + AcquireSRWLockExclusive(&free_lock); + for (Arena *a = free_arenas, *next; a; a = next) next = a->next, vmaDestroyBuffer(vk_ctx()->vma, a->buf, a->alloc), free(a); + free_arenas = NULL; + ReleaseSRWLockExclusive(&free_lock); +}