diff --git a/docs/architecture.md b/docs/architecture.md index fb1afd42..0c123786 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,6 +90,10 @@ Every guest thread is a host thread. TotK assumes Horizon's scheduler: three app that reads them, while the replay still records it. At the start of a submission's replay every uniform range it binds from CPU-written pools is copied, and the draws bind the copy (`vk_usnap.c`, after MacNativeGames' `TOTK_UBO_SNAPSHOT`); without it, late draws read another object's matrix. +- Uniform updates. `nvnCommandBufferUpdateUniformBuffer` between the draws of a pass is replayed as an overlay: the + bytes go to scratch memory, later bindings of the range bind them, and the copies into the game's buffers are + recorded behind one barrier when the pass ends (`vk_exec_draw.c` `op_update_ub`, `TOTK_UB_OVERLAY`). A copy at the + update itself would end the pass with a full barrier, which the terrain paid before most of its draws. - Driver data. The NVN driver keeps a per-stage constant bank (c0) that shaders read for things like point size, dispatch size and base instance. The back end fills it the way the driver does. - Shaders: the Ryujinx translator 1.3.2 with MacNativeGames' patches and ours (`tools/xlate/patches/`), built with .NET 10 NativeAOT as `totk_xlate.exe` and run as a separate process. Its SPIR-V goes to Vulkan directly. `tools/prepare_shaders.py` and `totk.exe --prepare-shaders` translate and compile the variants of MacNativeGames' shared list ahead of play (29,587 for 1.4.3); the rest are translated on first use and cached on disk. - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it. Shader stages are compiled ahead into libraries; a draw that meets a new state links them at once (fast link), and an optimised pipeline replaces the linked one when a background thread finishes it. Pipeline keys seen in play are saved and built at boot. A draw whose pipeline is not ready is skipped when one of its pass's targets was rendered in the previous frame, and waits otherwise, so one-shot passes are never lost; a full-screen draw is skipped as well rather than waiting (`TOTK_PIPE_SKIP`); the stage libraries a waiting draw needs are built on idle workers in parallel (`TOTK_PIPE_WAIT`). Without the extension, a missing pipeline is built on a worker and the draw waits or is skipped by the same rule. diff --git a/runtime/cfg.c b/runtime/cfg.c index 36795a01..9325fc5d 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -57,7 +57,7 @@ static const char *const known[] = { "SHADER_MEMO", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 6eda5c0b..b35831c8 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -86,6 +86,10 @@ void vk_exec_frame_log(const tex_t *p, uint64_t frame) { nfw = 0; } +WEAK void vk_exec_ub_flush(VkExec *ex) { (void)ex; } +WEAK void vk_exec_ub_foreign(VkExec *ex, uint64_t addr, uint64_t n) { (void)ex, (void)addr, (void)n; } +WEAK void vk_exec_ub_perf(void) {} + WEAK void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) { (void)ex, (void)payload; rt_fatal("vk: op %s is not replayed yet", nvn_op_name(op)); @@ -557,6 +561,7 @@ static VkImageMemoryBarrier2 layout_barrier(VkImage img, VkImageAspectFlags aspe // go back to GENERAL and the new ones leave it, in the same command. static void barrier_to(VkExec *ex, const VkImage *want, const VkImageAspectFlags *wasp, int nwant) { vk_exec_pass_end(ex); + vk_exec_ub_flush(ex); // uniform updates overlaid in the pass reach their buffers before the barrier below VkImageMemoryBarrier2 ib[18]; uint32_t n = 0; for (int i = 0; i < ex->nfbl; i++) @@ -967,6 +972,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t } ex->cb = VK_NULL_HANDLE; if (n) n_subs++; + if (n && (rt_trace & RT_TRACE_PERF)) vk_exec_ub_perf(); if ((rt_trace & RT_TRACE_PERF) && n_subs >= 240) { rt_log("perf: replay command buffer parts per submission %.2f", (double)(n_subs + n_parts) / (double)n_subs); n_subs = n_parts = 0; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 506305c2..e3692d48 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -122,6 +122,12 @@ void vk_exec_pass_end(VkExec *ex); // Full memory barrier (all commands, all memory); ends the open pass first. Copies and dispatches order themselves with // it. Skipped when nothing was recorded since the last one. void vk_exec_barrier(VkExec *ex); +// UpdateUniformBuffer overlays (vk_exec_draw.c): the copies into the real buffers that updates inside a pass left for +// the next barrier (barrier_to records them first), and a GPU write of another kind into [addr, addr+n) of this +// submission (a copy, a fill, a storage write), which retires the overlays it touches. +void vk_exec_ub_flush(VkExec *ex); +void vk_exec_ub_foreign(VkExec *ex, uint64_t addr, uint64_t n); +void vk_exec_ub_perf(void); // vk_texture.c (TOTK_TEX_ALIAS): commands that must run at the replay's current position. vk_exec_inline_begin returns // the command buffer being recorded after a full barrier (an open pass is ended; *reopen says so), or VK_NULL_HANDLE // when this thread is not recording a replay. vk_exec_inline_end begins the pass again when it was open. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 58f18308..20417204 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -158,6 +158,35 @@ static struct { uint8_t ck[DS_SIG_BYTES]; } RS = {-1}; +// UpdateUniformBuffer overlays of the submission being replayed (op_update_ub, TOTK_UB_OVERLAY). Here because +// bind_sig reads them (ov_at). +#define OV_N 2048 +#define OV_FOREIGN 4096 +static struct { + int mode; // rt_knob: 0 off, 1 on, 2 ab; -1 unread + uint64_t fence; // the submission the lists belong to + uint64_t serial; // numbers the overlays, so bind_sig tells one from the next at the same range + uint64_t lo, hi; // the span of the overlays: most bindings lie outside it + uint32_t n, np, nf; + int f_over; // more foreign writes than f holds: no more overlays in this submission + struct { uint64_t lo, hi, serial; VkBuffer buf; VkDeviceSize off; } o[OV_N]; // oldest first, no two equal + struct { uint64_t dst; VkBuffer src; VkDeviceSize soff; VkMemSpan sp; uint64_t size; } p[OV_N]; // to copy + struct { uint64_t lo, hi; } f[OV_FOREIGN]; // GPU writes of other kinds + // since the last perf line (vk_exec_ub_perf) + uint64_t c_update, c_overlaid, c_flush, c_bind, c_merged, c_mixed, c_inpass, c_outside, c_early; +} OV = {.mode = -1}; + +// A binding without a size reaches to the end of its backing: as far as the overlays go, for ov_at. +static uint64_t ov_rest(uint64_t addr) { return addr < OV.hi ? OV.hi - addr : 1; } + +// The serial of the newest overlay a binding of [addr, addr+n) reads from, 0 for none (bind_sig). +static uint64_t ov_at(const VkExec *ex, uint64_t addr, uint64_t n) { + if (!OV.n || OV.fence != ex->fence || addr >= OV.hi || addr + n <= OV.lo) return 0; + for (uint32_t i = OV.n; i-- > 0;) + if (addr < OV.o[i].hi && addr + n > OV.o[i].lo) return OV.o[i].serial; + return 0; +} + // The bound-slot bytes of VkExec a pipeline's descriptors read, as ranges merged where they touch, built once per // pipeline info: c0's storage table, the uniform and storage ranges of the bound slots and the texture and image // handles. sup and c0: the support inputs and the base instance count too. p: storage buffers whose address the shader @@ -269,6 +298,9 @@ static uint8_t *scratch(VkExec *ex, uint64_t size, VkBuffer *buf, VkDeviceSize * return p; } +static int ub_bind(VkExec *ex, uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out); +static void ub_written(VkExec *ex, uint64_t addr, uint64_t n); + static void note_written(VkExec *ex, uint64_t addr, uint64_t size) { CbRes *r = cb_res(ex); for (int i = 0; i < r->nwritten; i++) @@ -807,10 +839,15 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t n = size && size < sp.avail ? size : sp.avail; if (n > max) n = max; if (!n) return 0; + if (!written && (use == VK_MUSE_UBO || use == VK_MUSE_SSBO) && ub_bind(ex, addr, n, align, out)) { + count_use(VK_MCLS_SCRATCH, use, n); + return 1; + } if (sp.offset % align == 0) { VK_MEM_USE(sp); + if (written) ub_written(ex, addr, n); *out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n}; - if (written) vk_mem_gpu_wrote_shader(&sp, n), vk_usnap_drop(addr, n); + if (written) vk_mem_gpu_wrote_shader(&sp, n), vk_usnap_drop(addr, n), vk_exec_ub_foreign(ex, addr, n); count_use(sp.cls, use, n); VK_MEM_BOUND(sp); return 1; @@ -920,6 +957,10 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { } } else if (slot >= 3 && slot - 3 < 18) { const p_bind_buf *b = &ex->ubo[st][slot - 3]; + if (b->size && ub_bind(ex, b->addr, b->size < max ? b->size : max, align, &bi)) { // updated in this pass + count_use(VK_MCLS_SCRATCH, VK_MUSE_UBO, bi.range); + return bi; + } if (b->size && vk_usnap_find(b->addr, b->size < max ? b->size : max, align, &bi)) { // as of the submit call count_use(VK_MCLS_SCRATCH, VK_MUSE_UBO, bi.range); return bi; @@ -983,6 +1024,16 @@ static int upd_read(const VkExec *ex, uint64_t a, void *out, size_t n) { return 0; } +// Applies the updates of this submission that touch [a, a+n) onto out, oldest first (ub_bind's merged copies). +static void upd_apply(const VkExec *ex, uint64_t a, uint8_t *out, size_t n) { + if (UPD.fence != ex->fence) return; + for (uint32_t i = UPD.n > UPD_N ? UPD.n - UPD_N : 0; i < UPD.n; i++) { + const typeof(UPD.u[0]) *e = &UPD.u[i % UPD_N]; + uint64_t lo = e->lo > a ? e->lo : a, hi = e->hi < a + n ? e->hi : a + n; + if (lo < hi) memcpy(out + (lo - a), e->data + (lo - e->lo), hi - lo); + } +} + // That address as the bound block holds it at this point of the replay; 0 when the block is unbound or too short. The // block has no size field. static uint64_t ubo_ptr(const VkExec *ex, int st, const GfxShaderRes *r) { @@ -1698,6 +1749,30 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { } if (l->c0) PUT(c0_binst); for (int i = 0; i < l->np; i++) PUT(ubo_ptr(ex, l->p[i].st, l->p[i].res)); + // The uniform overlays the draw's uniform and storage bindings read (op_update_ub): a draw of an updated block binds + // other bytes after each update, every other draw is left to skip + if (OV.n) { + uint64_t ov = 0; + const size_t lo = offsetof(VkExec, ubo), hi = offsetof(VkExec, ssbo) + sizeof ex->ssbo; + for (uint32_t i = 0; i < l->n; i++) { + const RsRange *r = &l->r[i]; + if (r->off + r->len <= lo || r->off >= hi) continue; + for (uint32_t o = 0; o + sizeof(p_bind_buf) <= r->len; o += sizeof(p_bind_buf)) { + size_t at = r->off + o; + if (at < lo || at + sizeof(p_bind_buf) > hi) continue; + if ((at < offsetof(VkExec, ssbo) ? at - lo : at - offsetof(VkExec, ssbo)) % sizeof(p_bind_buf)) continue; + const p_bind_buf *b = (const p_bind_buf *)((const uint8_t *)ex + at); + if (b->addr) ov = ov * 1099511628211ull ^ ov_at(ex, b->addr, b->size ? b->size : ov_rest(b->addr)); + } + } + for (int i = 0; i < l->np; i++) { // storage read through an address in a uniform block: no size either + uint64_t a = ubo_ptr(ex, l->p[i].st, l->p[i].res); + if (a) ov = ov * 1099511628211ull ^ ov_at(ex, a, ov_rest(a)); + } + PUT(ov); + } else { + PUT(0); + } #undef PUT diff |= n != RS.hn; RS.hn = n; @@ -3159,14 +3234,208 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { if (vk_dump_armed) vk_dump_dispatch(ex, cs, p, info, &(p_dispatch){0, 0, 0, d->caller}); } +// ---- UpdateUniformBuffer inside a pass (TOTK_UB_OVERLAY) +// nvnCommandBufferUpdateUniformBuffer writes a few hundred bytes of constants between draws: the draws recorded after +// it read the new bytes, those before it the old ones. Replayed as a buffer copy at that point it needs a barrier +// against the earlier draws, and a barrier ends the open pass; the terrain and the sky islands' meshes update before +// most of their draws, about 200 updates inside passes in a field submission, each a pass end, a full barrier and a +// pass begin. Here an update inside a pass goes into the scratch buffer as an overlay: the later bindings of a range it +// covers bind the scratch bytes (ub_bind, from ubo_info and bind_range), and the copies into the real buffers wait for +// the next full barrier, where the pass has ended anyway (vk_exec_ub_flush from vk_exec.c barrier_to: a pass end, a +// dispatch, a transfer, the end of the submission), behind one barrier of their own. Each draw reads the bytes it would +// have read with the copy in between. +// - A GPU write of another kind (a copy, a fill, a shader's storage write) is ordered with the copies only by the +// command buffer. vk_exec_ub_foreign notes it and retires the overlays it touches; an update over such bytes takes +// the old path. A shader that writes a range with copies pending gets them first (ub_written: the pass restarts). +// - An update that takes the old path (outside a pass, a full list) retires the overlays it covers (ov_retire). +// - A binding an overlay covers only in part, or at an offset the binding's alignment refuses, gets a merged scratch +// copy: the snapshot's bytes (vk_usnap.c) or guest memory, with every update of the submission so far (UPD) on top. +// - The descriptor build skip (bind_sig) hashes the overlay each uniform and storage binding of the draw reads +// (ov_at), so a draw of an updated block binds the newest bytes and the other draws still skip. The lists are per submission and keep one entry per range (ov_add), and a binding outside their span is +// answered in one comparison; when one is full, updates take the old path. + +static int ov_on(void) { + if (OV.mode < 0) OV.mode = rt_knob("UB_OVERLAY", "1"); + return RT_KNOB_ON(OV.mode); +} + +static void ov_sub(const VkExec *ex) { // the submission being replayed + if (OV.fence == ex->fence) return; + if (OV.np) // barrier_to records them at the latest at the submission's closing barrier + rt_log("vk: %u overlaid uniform updates of submission %llu were never copied into their buffers", OV.np, + (unsigned long long)OV.fence); + OV.fence = ex->fence, OV.n = OV.np = OV.nf = 0, OV.f_over = 0, OV.lo = UINT64_MAX, OV.hi = 0; +} + +static void ov_span(void) { + OV.lo = UINT64_MAX, OV.hi = 0; + for (uint32_t i = 0; i < OV.n; i++) { + if (OV.o[i].lo < OV.lo) OV.lo = OV.o[i].lo; + if (OV.o[i].hi > OV.hi) OV.hi = OV.o[i].hi; + } +} + +// A new overlay of [lo, hi), and its copy. An older overlay or pending copy of exactly that range is dropped: the new +// one covers it, and the terrain rewrites one block before each draw, so the lists stay as short as the blocks. +static void ov_add(uint64_t lo, uint64_t hi, VkBuffer b, VkDeviceSize o, const VkMemSpan *sp) { + for (uint32_t i = 0; i < OV.n; i++) + if (OV.o[i].lo == lo && OV.o[i].hi == hi) { + memmove(&OV.o[i], &OV.o[i + 1], sizeof OV.o[0] * (OV.n - i - 1)); + OV.n--; + break; + } + for (uint32_t i = 0; i < OV.np; i++) + if (OV.p[i].dst == lo && OV.p[i].size == hi - lo) { + memmove(&OV.p[i], &OV.p[i + 1], sizeof OV.p[0] * (OV.np - i - 1)); + OV.np--; + break; + } + OV.o[OV.n++] = (typeof(OV.o[0])){lo, hi, ++OV.serial, b, o}; + OV.p[OV.np++] = (typeof(OV.p[0])){lo, b, o, *sp, hi - lo}; + if (lo < OV.lo) OV.lo = lo; + if (hi > OV.hi) OV.hi = hi; +} + +static int ov_foreign_hits(uint64_t lo, uint64_t hi) { + if (OV.f_over) return 1; + for (uint32_t i = 0; i < OV.nf; i++) + if (lo < OV.f[i].hi && hi > OV.f[i].lo) return 1; + return 0; +} + +// The overlays over [lo, hi) no longer hold the newest bytes: the real buffer does. Their copies were recorded already +// (the write came after a barrier, or ub_written brought them forward), so they only stop serving bindings. +static void ov_retire(uint64_t lo, uint64_t hi) { + if (!OV.n || lo >= OV.hi || hi <= OV.lo) return; + uint32_t k = 0; + for (uint32_t i = 0; i < OV.n; i++) + if (!(lo < OV.o[i].hi && hi > OV.o[i].lo)) OV.o[k++] = OV.o[i]; + if (k == OV.n) return; + OV.n = k; + ov_span(); +} + +// A GPU write of another kind into [addr, addr+n) in this submission (vk_exec_xfer.c note_gpu_write, storage bindings). +void vk_exec_ub_foreign(VkExec *ex, uint64_t addr, uint64_t n) { + if (OV.mode <= 0 || !n) return; + ov_sub(ex); + if (OV.nf == OV_FOREIGN) OV.f_over = 1; + else OV.f[OV.nf].lo = addr, OV.f[OV.nf].hi = addr + n, OV.nf++; + ov_retire(addr, addr + n); +} + +// Records the copies of the updates overlaid so far, behind a barrier against the draws that read the old bytes. +void vk_exec_ub_flush(VkExec *ex) { + if (!OV.np || OV.fence != ex->fence) return; + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(ex->cb, &di); + ex->recorded++; + for (uint32_t i = 0; i < OV.np; i++) { + VkBufferCopy cp = {OV.p[i].soff, OV.p[i].sp.offset, OV.p[i].size}; + vkCmdCopyBuffer(ex->cb, OV.p[i].src, OV.p[i].sp.buffer, 1, &cp); + ex->recorded++; + } + OV.np = 0; + OV.c_flush++; +} + +// TOTK_TRACE=perf, every 600 submissions: per submission, the updates and how many stayed in their pass. +void vk_exec_ub_perf(void) { + static unsigned subs; + if (OV.mode <= 0 || ++subs % 600) return; + const double k = 1 / 600.0; + rt_log("perf: uniform updates per submission %.1f: %.1f overlaid in a pass (copied in %.2f batches), %.1f copied " + "in a pass, %.1f outside one; bindings from an overlay %.1f (%.1f merged, %.1f over other GPU writes), %.2f " + "batches brought forward for a storage write", + OV.c_update * k, OV.c_overlaid * k, OV.c_flush * k, OV.c_inpass * k, OV.c_outside * k, OV.c_bind * k, + OV.c_merged * k, OV.c_mixed * k, OV.c_early * k); + OV.c_update = OV.c_overlaid = OV.c_flush = OV.c_bind = OV.c_merged = OV.c_mixed = 0; + OV.c_inpass = OV.c_outside = OV.c_early = 0; +} + +// The scratch bytes of a uniform or storage binding [addr, addr+n) an overlay of this submission touches: 1 with *out +// set, 0 when none does. +static int ub_bind(VkExec *ex, uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out) { + if (OV.mode <= 0 || OV.fence != ex->fence || !OV.n || addr >= OV.hi || addr + n <= OV.lo) return 0; + uint64_t hi = addr + n; + int newest = -1; + for (int i = (int)OV.n; i-- > 0;) + if (addr < OV.o[i].hi && hi > OV.o[i].lo) { newest = i; break; } + if (newest < 0) return 0; + OV.c_bind++; + if (OV.o[newest].lo <= addr && OV.o[newest].hi >= hi) { + VkDeviceSize off = OV.o[newest].off + (addr - OV.o[newest].lo); + if (off % align == 0) { + *out = (VkDescriptorBufferInfo){OV.o[newest].buf, off, n}; + return 1; + } + } + // Merged: the bytes as of the submit call (the snapshot, else guest memory), then this submission's updates in + // order. A transient copy, so the descriptor build skip leaves the draw after it alone (RS.volatile_). Bytes a copy + // or a storage write of this submission produced are not in either source: counted. + VkBuffer b; + VkDeviceSize o; + uint8_t *dst = scratch(ex, n, &b, &o); + RS.volatile_ = 1; + const uint8_t *base = vk_usnap_bytes(addr, n); + VkMemSpan sp; + if (base) memcpy(dst, base, n); + else if (vk_mem_resolve(addr, n, &sp)) memcpy(dst, (const void *)(uintptr_t)addr, n); + else memset(dst, 0, n); + if (ov_foreign_hits(addr, hi)) OV.c_mixed++; + upd_apply(ex, addr, dst, n); + *out = (VkDescriptorBufferInfo){b, o, n}; + OV.c_merged++; + return 1; +} + +// Before a binding the shader writes: overlaid updates whose copies are still pending go to the buffer first, behind a +// barrier, and the pass restarts, so the shader's writes land after them as they did when every update was copied at +// once. +static void ub_written(VkExec *ex, uint64_t addr, uint64_t n) { + if (OV.mode <= 0 || OV.fence != ex->fence || !OV.np) return; + for (uint32_t i = 0; i < OV.np; i++) + if (addr < OV.p[i].dst + OV.p[i].size && OV.p[i].dst < addr + n) { + int reopen = ex->in_pass; + vk_exec_barrier(ex); // records the pending copies (vk_exec_ub_flush) before its barrier + if (reopen) vk_exec_pass_begin(ex); + OV.c_early++; + return; + } +} + static void op_update_ub(VkExec *ex, const p_update_ub *u) { if (!u->size) return; VkMemSpan sp; if (!vk_mem_resolve(u->dst, u->size, &sp)) return warn_once(W_UB, "UpdateUniformBuffer to %#llx is not backed: dropped", (unsigned long long)u->dst); cb_res(ex); - vk_exec_barrier(ex); // earlier draws finished with the old contents VkBuffer b; VkDeviceSize o; + if (ov_on()) { + ov_sub(ex); + OV.c_update++; + if (ex->in_pass && OV.n < OV_N && OV.np < OV_N && !ov_foreign_hits(u->dst, u->dst + u->size)) { + uint8_t *d = scratch(ex, u->size, &b, &o); + memcpy(d, u->data, u->size); + VK_MEM_PICK(sp); + ov_add(u->dst, u->dst + u->size, b, o, &sp); + vk_mem_gpu_wrote(&sp, u->size); + vk_usnap_drop(u->dst, u->size); + vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size); + note_written(ex, u->dst, u->size); + upd_note(ex, u); + OV.c_overlaid++; + return; + } + if (ex->in_pass) OV.c_inpass++; + else OV.c_outside++; + ov_retire(u->dst, u->dst + u->size); + } + vk_exec_barrier(ex); // earlier draws finished with the old contents; overlaid updates' copies go first memcpy(scratch(ex, u->size, &b, &o), u->data, u->size); VK_MEM_PICK(sp); VkBufferCopy cp = {o, sp.offset, u->size}; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 4c9db592..9cca6405 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -73,6 +73,7 @@ static void guest_written_cb(void *arg) { // The GPU wrote [addr, addr+size) of CPU-visible memory: tell the texture cache once the fence is reached. static void note_gpu_write(VkExec *ex, uint64_t addr, uint64_t size) { vk_usnap_drop(addr, size); + vk_exec_ub_foreign(ex, addr, size); if (nvn_gpu_only(addr)) return; Range *r = malloc(sizeof *r); if (!r) rt_fatal("vk: out of memory");