Mirror buffer-bound pools in device-local memory on discrete GPUs

Imported pools put vertices, indices, constants and storage buffers in system memory, so discrete GPUs read them over
PCIe on every draw. A pool now gets a device-local mirror on its first use as a vertex, index, uniform, storage or
indirect buffer; bindings, buffer copies, fills and uniform updates use it at the same offsets. The first access of a
submission syncs the pages written since (write-watch snapshots shared with the texture checks, plus
nvnMemoryPoolFlushMappedRange ranges) in a prologue command buffer before the part; what the GPU writes into a mirror
goes back to guest memory at the end of the submission. TOTK_MEMORY=import|mirror|verify overrides the device class;
verify compares sampled mirror pages with guest memory at the fence. The vkexec probe gains a mirror part.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-25 00:45:56 +03:00
1 parent c8a7b889d2
commit 1fe552ae0a
14 files changed
+775 -82

No files matched your search

+3
View File
@@ -128,6 +128,9 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64
// past max stays reported next time). The range must lie
// inside the stack/heap block or the alias region
// (write watch); otherwise returns UINT64_MAX
uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max);
// the same without the reset: the bits stay for the next
// mem_written_pages (verify checks only)
void mem_kernel_free(void *p, uint64_t size); // TLR/TLS reuse after thread exit
// ---- modules (loader.c, T1)
+10 -3
View File
@@ -148,7 +148,7 @@ const RtGuestLayout *mem_layout(void) { return &L; }
// Adapted from LAS Recomp runtime/platform/windows.c:40-63 @ da35b83 (MIT, Copyright (c) 2026 Peiikai): GetWriteWatch
// with the reset flag. A range inside one watched reservation (stack and heap block, or alias region) only.
uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, DWORD flags) {
if (!max || !size) return 0;
const RtGuestLayout *g = &L;
int low_block = addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size;
@@ -157,11 +157,18 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64
ULONG_PTR n = (ULONG_PTR)max;
DWORD gran;
_Static_assert(sizeof(PVOID) == sizeof(uint64_t), "page addresses are written straight into a uint64_t array");
if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran))
return UINT64_MAX;
if (GetWriteWatch(flags, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran)) return UINT64_MAX;
return n;
}
uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
return written_pages(addr, size, pages, max, WRITE_WATCH_FLAG_RESET);
}
uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
return written_pages(addr, size, pages, max, 0);
}
static uint64_t image_bytes(void) {
uint64_t n = 0;
for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size;
+3
View File
@@ -99,3 +99,6 @@ const char *nvn_op_name(uint32_t op); // "DRAW_IDX", ...; "
// MemoryPoolInvalidateMappedRange: the CPU is about to read pool storage the GPU wrote. Weak no-op in nvn_ext.c; a back
// end that renders into host-side copies defines it and writes them back.
void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size);
// MemoryPoolFlushMappedRange: the CPU wrote [mem + off, mem + off + size). Weak no-op in nvn_ext.c; a back end that keeps
// device-side copies of pool memory (vk_memory.c's VRAM mirror) defines it and copies the range again.
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size);
+7 -2
View File
@@ -303,9 +303,14 @@ void nvn_flush_stats(uint64_t out[2][3][2]) {
for (int j = 0; j < 2; j++) out[i][k][j] = __atomic_exchange_n(&flush_stat[i][k][j], 0, __ATOMIC_RELAXED);
}
// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent
// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent; a
// back end with device-side copies hears of it (nvn_pool_flushed)
NVN_API(nvnMemoryPoolFlushMappedRange) {
if (rt_trace & RT_TRACE_PERF) flush_note(0, arg_p(c, 0), arg_x(c, 2));
const pool_t *p = arg_p(c, 0);
int64_t off = (int64_t)arg_x(c, 1);
uint64_t size = arg_x(c, 2);
if (rt_trace & RT_TRACE_PERF) flush_note(0, p, size);
if (p && off >= 0 && (uint64_t)off < p->size) nvn_pool_flushed(p, (uint64_t)off, size < p->size - (uint64_t)off ? size : p->size - (uint64_t)off);
}
// NVNmemoryPoolFlags nvnMemoryPoolGetFlags(const NVNmemoryPool *p)
NVN_API(nvnMemoryPoolGetFlags) { ret_x(c, ((const pool_t *)arg_p(c, 0))->flags); }
+2
View File
@@ -20,6 +20,8 @@ void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size) {
gfx_backend()->guest_read((uintptr_t)p->mem + off, size);
}
__attribute__((weak)) void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { (void)p, (void)off, (void)size; }
// ---------- debug labels, markers, groups, checkpoints: meaningful only to NVN debugging tools
// void nvn<Object>SetDebugLabel(NVN<object> *o, const char *label)
NVN_API(nvnDeviceSetDebugLabel) { (void)c; }
+26
View File
@@ -138,6 +138,30 @@ void vk_mem_count(int cls, int use, uint64_t bytes);
void vk_mem_bound_note(void *pool);
#define VK_MEM_BOUND(sp) do { if ((rt_trace & RT_TRACE_PERF) && (sp).pool) vk_mem_bound_note((sp).pool); } while (0)
void vk_mem_dirty_submit(void); // vk_sync_submit, under its lock, before the replay
// VRAM mirror of buffer-bound pools (vk_memory.c, build/mem-plan.md). vk_mirror_on: TOTK_MEMORY=mirror|verify, or the
// device class is discrete and TOTK_MEMORY is unset or auto; set at device selection. The replay (submitting thread
// under vk_sync_submit's lock) calls:
// - VK_MEM_USE at a vertex, index, uniform, storage or indirect binding of a span vk_mem_resolve returned: the pool's
// mirror is created on first use and sp.buffer becomes it (same offsets); a pool that stays imported keeps its buffer.
// - VK_MEM_PICK where a copy, fill or update reads or writes a span: the mirror if the pool has one, never a new one.
// - vk_mem_gpu_wrote after recording a GPU write of [sp.offset, sp.offset + size) into sp.buffer: a mirror's bytes go
// back to guest memory at the end of the submission. No-op for imported buffers.
// - vk_mem_mirror_begin at the start of a submission's recording, vk_mem_mirror_end (records the write-backs; returns
// the commands recorded) after its last barrier.
// vk_sync.c records the copies the mirrors' first accesses queued (vk_mem_mirror_pending) into a prologue command
// buffer with vk_mem_mirror_prologue and submits it right before each part.
extern int vk_mirror_on;
void vk_mem_use(VkMemSpan *sp);
void vk_mem_pick(VkMemSpan *sp);
#define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0)
#define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0)
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size);
void vk_mem_mirror_begin(void);
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence);
int vk_mem_mirror_pending(void);
void vk_mem_mirror_prologue(VkCommandBuffer cb);
int vk_mem_mirror_probe(uint64_t addr); // --probe-vkexec: the pool at addr gets its mirror as on a first buffer use;
// 1 when it has one
void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m); // once
int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked)
void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and,
@@ -223,6 +247,7 @@ void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); /
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads);
void vk_texture_shutdown(void); // vk_mem_shutdown calls it
void vk_texture_new_submit(void); // a queue submit starts: draw bindings check storage again
uint32_t vk_texture_epoch(void); // the write-watch snapshot epoch vk_texture_new_submit started
VkSampler vk_sampler_by_id(int id); // VK_NULL_HANDLE for an empty slot
VkSampler vk_sampler_for(const samp_par *s); // cached by content
int vk_sampler_params(int id, samp_par *out); // parameters of a sampler slot; 0 for an empty slot
@@ -237,6 +262,7 @@ void vk_sync_start(void); // vk_init (fatal without a device) a
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
uint64_t vk_sync_fence_done(void); // highest done value
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
void vk_sync_finish(void); // every submitted value done
void vk_sync_shutdown(void); // finish, stop the completion thread, free the command buffers
// The op-stream replay (vk_exec.c: T6c): records the command handles of one submission into cb, which the caller has
+1
View File
@@ -399,6 +399,7 @@ static void buf_snap(VkExec *ex, int pass) {
if (!bd_size) return;
VkMemSpan s;
if (!vk_mem_resolve(bd_addr, bd_size, &s) || s.avail < bd_size) return;
VK_MEM_PICK(s); // the GPU's view of the buffer
BufJob *j = calloc(1, sizeof *j);
if (!j) rt_fatal("vk_dump: out of memory");
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bd_size,
+2
View File
@@ -688,6 +688,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
vk_exec_frame.submits++;
split_draws = part_start = 0;
x_here = 1;
if (n) vk_mem_mirror_begin();
for (int i = 0; i < n; i++) {
if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i);
walk(ex, nvn_op_first(handles[i]), 0);
@@ -695,6 +696,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
vk_exec_pass_end(ex);
x_here = 0;
if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes
if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence); // VRAM mirror bytes the GPU wrote go back to guest memory
if (ex->nviews) {
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
if (!l) rt_fatal("vk: out of memory");
+19 -6
View File
@@ -569,16 +569,20 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) {
return (VkDescriptorBufferInfo){Z.zbuf, 0, max < ZERO_BYTES ? max : ZERO_BYTES};
}
// A buffer descriptor for [addr, addr+size) of guest memory: its backing when aligned, a copy in transient memory when
// the backing is host memory with an unaligned offset. 0 when neither works.
static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, VkDescriptorBufferInfo *out) {
// A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when
// aligned, a copy in transient memory when the backing is host memory with an unaligned offset. 0 when neither works.
// written: the shader stores into it (a mirror's range goes back to guest memory at the end of the submission).
static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, int written,
VkDescriptorBufferInfo *out) {
VkMemSpan sp;
if (!addr || !vk_mem_resolve(addr, 1, &sp)) return 0;
uint64_t n = size && size < sp.avail ? size : sp.avail;
if (n > max) n = max;
if (!n) return 0;
if (sp.offset % align == 0) {
VK_MEM_USE(sp);
*out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n};
if (written) vk_mem_gpu_wrote(&sp, n);
vk_mem_count(sp.cls, use, n);
VK_MEM_BOUND(sp);
return 1;
@@ -665,11 +669,12 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) {
uint32_t len, c1;
if (find_stage(ex->prog, st, &ctl, &code, &len)) {
memcpy(&c1, ctl + 0x700, 4);
if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, &bi)) return bi;
if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, 0, &bi))
return bi;
}
} else if (slot >= 3 && slot - 3 < 18) {
const p_bind_buf *b = &ex->ubo[st][slot - 3];
if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, &bi)) return bi;
if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, 0, &bi)) return bi;
}
return zero_info(max);
}
@@ -699,7 +704,8 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *
const p_bind_buf *b = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot];
// vk_caps.c: shaders translated for sbuf_align address the buffer from its address rounded down to it.
uint64_t lead = vk_caps.sbuf_align ? b->addr & (vk_caps.sbuf_align - 1) : 0;
if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO, &bi)) {
if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO,
written, &bi)) {
if (written && bi.buffer != VK_NULL_HANDLE) note_written(ex, b->addr - lead, bi.range);
return bi;
}
@@ -1095,6 +1101,7 @@ static void bind_vertex_buffers(VkExec *ex) {
const NVNbufferRange *r = &ex->vbuf[b];
VkMemSpan sp;
if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue;
VK_MEM_USE(sp);
uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail;
vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool;
}
@@ -1271,6 +1278,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) {
} else if (sp.offset % isize[d->itype]) {
return warn_once(W_INDEX, "index buffer %#llx is not aligned to its index size: draw skipped", (unsigned long long)d->ibuf);
} else {
VK_MEM_USE(sp);
vkCmdBindIndexBuffer(ex->cb, sp.buffer, sp.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
ex->recorded++;
vk_mem_count(sp.cls, VK_MUSE_IB, bytes);
@@ -1295,6 +1303,8 @@ static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) {
return warn_once(W_INDEX, "index buffer %#llx is not backed or not aligned: indirect draws skipped", (unsigned long long)d->ibuf);
if (!prepare_draw(ex, d->prim, 0x7fffffff, restart_for(ex, d->prim, d->itype))) return;
vk_exec_frame.draws++;
VK_MEM_USE(ip);
VK_MEM_USE(ib);
vkCmdBindIndexBuffer(ex->cb, ib.buffer, ib.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
vkCmdDrawIndexedIndirect(ex->cb, ip.buffer, ip.offset, 1, sizeof(VkDrawIndexedIndirectCommand));
ex->recorded += 2;
@@ -1341,6 +1351,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) {
VkPipeline p = vk_pipeline_compute(cs, &info);
if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped");
vk_exec_barrier(ex);
VK_MEM_USE(ip);
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
c0_dim[0] = c0_dim[1] = c0_dim[2] = 0;
B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads
@@ -1368,8 +1379,10 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) {
VkBuffer b;
VkDeviceSize o;
memcpy(scratch(ex, u->size, &b, &o), u->data, u->size);
VK_MEM_PICK(sp);
VkBufferCopy cp = {o, sp.offset, u->size};
vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp);
vk_mem_gpu_wrote(&sp, u->size);
ex->recorded++;
vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size);
note_written(ex, u->dst, u->size);
+55 -1
View File
@@ -802,6 +802,54 @@ static int test_nvnx(const GfxBackend *b, uint8_t *guest, int *bytes) {
return 0;
}
// VRAM mirror (vk_memory.c; skipped in import mode): a pool given its mirror as on a first buffer use, after which
// buffer copies and fills read and write the mirror. The first use copies the whole pool; CPU writes between
// submissions reach the mirror through write watch; GPU writes reach guest memory by the fence, and only the bytes
// written: CPU bytes beside a GPU fill keep their values.
static int test_mirror(const GfxBackend *b, uint8_t *guest, int *bytes) {
*bytes = 0;
if (!vk_mirror_on) return 0;
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x22, .mem = guest + 0xC00000, .size = 0x100000,
.game_mem = guest + 0xC00000};
CHECK(b->pool_init(&pool), "pool_init of a CPU-visible pool for the mirror part");
uint8_t *m = pool.mem;
for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 13 + 1);
memset(m + 0x1000, 0xEE, 0x5000);
CHECK(vk_mem_mirror_probe((uint64_t)(uintptr_t)m), "the pool got no VRAM mirror");
static _Alignas(8) uint8_t stream[1024];
uint64_t a = (uint64_t)(uintptr_t)m;
// 1: B2B in the mirror after its first-use copy, and a fill; both written back.
uint8_t *p = put(stream, OP_BEGIN, NULL, 0);
const p_copy_b2b c1 = {a, a + 0x1000, 4096};
const p_clear_buf f1 = {a + 0x2000, 0x800, 0x5a5a5a5au, 0};
p = put(p, OP_COPY_B2B, &c1, sizeof c1);
p = put(p, OP_CLEAR_BUF, &f1, sizeof f1);
CHECK(xf_submit(b, stream, p), "fence_wait of the first mirror submission timed out");
for (int i = 0; i < 4096; i++) CHECK(m[0x1000 + i] == m[i], "mirror B2B byte %d is %02x, expected %02x", i, m[0x1000 + i], m[i]);
for (int i = 0; i < 0x800; i++) CHECK(m[0x2000 + i] == 0x5a, "mirror fill byte %d is %02x", i, m[0x2000 + i]);
for (int i = 0x800; i < 0x1000; i++) CHECK(m[0x2000 + i] == 0xEE, "byte %#x past the mirror fill is %02x", i, m[0x2000 + i]);
// 2: the CPU rewrites the source; the next submission copies the new bytes.
for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 7 + 5);
p = put(stream, OP_BEGIN, NULL, 0);
const p_copy_b2b c2 = {a, a + 0x3000, 4096};
p = put(p, OP_COPY_B2B, &c2, sizeof c2);
CHECK(xf_submit(b, stream, p), "fence_wait of the second mirror submission timed out");
for (int i = 0; i < 4096; i++)
CHECK(m[0x3000 + i] == m[i], "B2B byte %d after a CPU write is %02x, expected %02x (write not mirrored)", i,
m[0x3000 + i], m[i]);
// 3: CPU bytes in the page of a GPU fill, written before the submission, survive the write-back.
memset(m + 0x4000, 0x11, 0x800);
p = put(stream, OP_BEGIN, NULL, 0);
const p_clear_buf f3 = {a + 0x4800, 0x800, 0x77777777u, 0};
p = put(p, OP_CLEAR_BUF, &f3, sizeof f3);
CHECK(xf_submit(b, stream, p), "fence_wait of the third mirror submission timed out");
for (int i = 0; i < 0x800; i++) CHECK(m[0x4000 + i] == 0x11, "CPU byte %#x beside a fill is %02x", i, m[0x4000 + i]);
for (int i = 0; i < 0x800; i++) CHECK(m[0x4800 + i] == 0x77, "fill byte %#x is %02x", i, m[0x4800 + i]);
b->pool_fini(&pool);
*bytes = 4096 + 0x1000 + 4096 + 0x1000;
return 0;
}
int probe_vkexec(void) {
layouts_equal = 0;
main_tid = GetCurrentThreadId();
@@ -819,8 +867,9 @@ int probe_vkexec(void) {
if (!bad) bad = test_xfer(b, guest, &xfer_bytes);
int zero_bytes = 0;
if (!bad) bad = test_c0() || test_zero(b, guest, &zero_bytes);
int nvnx_bytes = 0;
int nvnx_bytes = 0, mirror_bytes = 0;
if (!bad) bad = test_nvnx(b, guest, &nvnx_bytes);
if (!bad) bad = test_mirror(b, guest, &mirror_bytes);
b->finish();
vk_pipeline_shutdown(); // the pipelines and the draw part's stand-ins go before the device
vk_exec_draw_shutdown();
@@ -860,5 +909,10 @@ int probe_vkexec(void) {
printf("probe-vkexec: c0 and zero parts passed (c0 tables, dispatch size, point size and base instance at their offsets, "
"unbound slots zero; %d bytes of an unwritten physical-pool texture read zero, TOTK_FIX_ZERO=%s)\n",
zero_bytes, cfg_get("TOTK_FIX_ZERO") ? cfg_get("TOTK_FIX_ZERO") : "1");
if (vk_mirror_on)
printf("probe-vkexec: mirror part passed (B2B and fill in a VRAM mirror written back by the fence, a CPU write "
"between submissions mirrored, CPU bytes beside a fill kept, %d bytes compared)\n", mirror_bytes);
else
printf("probe-vkexec: mirror part skipped (memory mode import)\n");
return 0;
}
+16 -11
View File
@@ -379,9 +379,10 @@ static VkExtent3D sub_extent(const Sub *s) {
return (VkExtent3D){s->w, s->h, s->img->type == VK_IMAGE_TYPE_3D ? s->nz : 1};
}
// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span.
// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span (a VRAM mirror when the pool
// has one).
static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *size, VkBufferImageCopy *c,
VkBuffer *buf, int *cls) {
VkMemSpan *span) {
uint32_t wb = (s->w + (uint32_t)s->bw - 1) / (uint32_t)s->bw, hb = (s->h + (uint32_t)s->bh - 1) / (uint32_t)s->bh;
uint64_t row = ex->copy_stride > 0 ? (uint64_t)ex->copy_stride : (uint64_t)wb * (uint64_t)s->bpb;
if (row % (uint64_t)s->bpb || row < (uint64_t)wb * (uint64_t)s->bpb) {
@@ -397,8 +398,8 @@ static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *
(unsigned long long)addr, (unsigned long long)*size);
return 0;
}
*buf = m.buffer;
*cls = m.cls;
VK_MEM_PICK(m);
*span = m;
*c = (VkBufferImageCopy){.bufferOffset = m.offset, .bufferRowLength = (uint32_t)(row / (uint64_t)s->bpb) * (uint32_t)s->bw,
.bufferImageHeight = hb * (uint32_t)s->bh, .imageSubresource = sub_layers(s),
.imageOffset = sub_offset(s), .imageExtent = sub_extent(s)};
@@ -410,17 +411,16 @@ static void copy_buffer_image(VkExec *ex, const p_copy_b2t *p, int to_image) {
if (!sub_resolve(p->tex, &p->view, p->has_view, &p->r, 1, &s)) return;
uint64_t size;
VkBufferImageCopy c;
VkBuffer buf;
int cls;
if (!buffer_side(ex, &s, p->src, &size, &c, &buf, &cls)) return;
vk_mem_count(cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size);
VkMemSpan m;
if (!buffer_side(ex, &s, p->src, &size, &c, &m)) return;
vk_mem_count(m.cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size);
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
if (to_image) vkCmdCopyBufferToImage(ex->cb, buf, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c);
else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, buf, 1, &c);
if (to_image) vkCmdCopyBufferToImage(ex->cb, m.buffer, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c);
else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, m.buffer, 1, &c);
ex->recorded++;
if (to_image) vk_texture_gpu_written(p->tex);
else note_gpu_write(ex, p->src, size);
else vk_mem_gpu_wrote(&m, size), note_gpu_write(ex, p->src, size);
}
static void copy_t2t(VkExec *ex, const p_copy_t2t *p) {
@@ -476,6 +476,8 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) {
}
uint64_t n = left < s.avail ? left : s.avail;
if (n > d.avail) n = d.avail;
VK_MEM_PICK(s);
VK_MEM_PICK(d);
if (s.buffer == d.buffer && s.offset < d.offset + n && d.offset < s.offset + n) {
warn_once(W_OVERLAP, "buffer copy %#llx -> %#llx overlaps within one buffer: skipped",
(unsigned long long)p->src, (unsigned long long)p->dst);
@@ -484,6 +486,7 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) {
if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1;
VkBufferCopy bc = {s.offset, d.offset, n};
vkCmdCopyBuffer(ex->cb, s.buffer, d.buffer, 1, &bc);
vk_mem_gpu_wrote(&d, n);
ex->recorded++;
vk_mem_count(s.cls, VK_MUSE_COPY_SRC, n);
vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n);
@@ -514,7 +517,9 @@ static void clear_buf(VkExec *ex, const p_clear_buf *p) {
uint64_t n = left < d.avail ? left : d.avail & ~3ull;
if (!n) break;
if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1;
VK_MEM_PICK(d);
vkCmdFillBuffer(ex->cb, d.buffer, d.offset, n, p->value);
vk_mem_gpu_wrote(&d, n);
ex->recorded++;
vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n);
dst += n, left -= n;
+570 -36
View File
@@ -24,6 +24,7 @@
// ---------------------------------------------------------------- pools
enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT };
enum { MS_NONE, MS_ON, MS_REFUSED };
typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page -> physical pool and offset
@@ -44,8 +45,22 @@ typedef struct PoolRec {
uint32_t synced_gen; // page generation its lazy-mirror count covers; DIRTY_UNSYNCED until scanned
uint8_t bufbound; // ever bound as a vertex, index, uniform, storage or indirect buffer
uint8_t untracked; // the last dirty scan found no write watch over it
// VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only.
uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED
VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf
VmaAllocation malloc_;
uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync)
uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory
uint32_t mgen; // gen_lock: every page write stamped up to this generation is in the mirror
// (or in a copy queued before its next use)
uint64_t fl_lo, fl_hi; // flush_lock: nvnMemoryPoolFlushMappedRange bytes not copied yet (fl_hi 0: none)
} PoolRec;
static void mirror_forget(PoolRec *r); // pool_lock exclusive: off the mirrored list
static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown)
static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record
static void grow(void **arr, int *cap, size_t elem);
static SRWLOCK pool_lock = SRWLOCK_INIT;
// Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits
// stay valid while it is unchanged.
@@ -197,6 +212,7 @@ static PoolRec *phys_find(const pool_t *key) {
}
static void destroy_rec(VkCtx *c, PoolRec *r) {
if (r->mbuf) vmaDestroyBuffer(c->vma, r->mbuf, r->malloc_);
if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL);
if (r->mem) vkFreeMemory(c->device, r->mem, NULL);
if (r->alloc) vmaFreeMemory(c->vma, r->alloc);
@@ -259,6 +275,7 @@ void vk_pool_fini(pool_t *p) {
int had = pidx_remove(&pool_index, p, &e);
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
PoolRec *r = had ? e.buf : NULL;
if (r && r->mstate == MS_ON) mirror_forget(r);
if (r && r->kind == POOL_PHYS) {
for (int i = 0; i < nphys; i++)
if (phys_list[i] == r) { phys_list[i] = phys_list[--nphys]; break; }
@@ -281,7 +298,10 @@ void vk_pool_fini(pool_t *p) {
vk_queue_unlock();
}
vk_mem_collect();
destroy_rec(c, r);
// A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may
// name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued).
if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r);
else destroy_rec(c, r);
}
// Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c:54-67); apply them in order.
@@ -414,9 +434,12 @@ static void flag_names(uint32_t f, char *buf, size_t n) {
// run-game saves), at frame 600, and with TOTK_TRACE=perf every 1800 frames.
static void dirty_frame(uint64_t frame);
static void mirror_log(uint64_t frame);
void vk_mem_census(uint64_t frame) {
static uint64_t window_start;
if (rt_trace & RT_TRACE_PERF) dirty_frame(frame);
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame);
if (frame == 1200) { // loading is over: the first window starts here
for (int c = 0; c < VK_MCLS_N; c++)
for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0);
@@ -497,6 +520,10 @@ static uint32_t *gen_chunk[GEN_CHUNKS];
static uint32_t gen_now; // last generation handed out
static SRWLOCK gen_lock = SRWLOCK_INIT;
static uint64_t gen_pages[1024];
// The newest generation stamped on any page of each 2 MiB region (the write-watch snapshot regions below): the mirror
// sync skips regions with nothing newer than its pool's mirror.
#define REG_SHIFT 21
static uint32_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT];
static uint32_t *gen_slot(uint64_t page_addr, int create) {
uint64_t ci = page_addr >> GEN_CHUNK_SHIFT;
@@ -509,11 +536,16 @@ static uint32_t *gen_slot(uint64_t page_addr, int create) {
return &gen_chunk[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)];
}
// gen_lock held exclusively; g is the newest generation handed out.
static void stamp_page(uint64_t a, uint32_t g) {
uint32_t *s = gen_slot(a, 1);
if (!s) return;
*s = g;
reg_max[a >> REG_SHIFT] = g;
}
static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) {
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) {
uint32_t *s = gen_slot(a, 1);
if (s) *s = g;
}
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page(a, g);
}
void vk_guest_written(uint64_t addr, uint64_t size) {
@@ -533,10 +565,7 @@ int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now)
for (;;) {
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
if (n == UINT64_MAX) { tracked = 0; break; }
for (uint64_t k = 0; k < n; k++) {
uint32_t *s = gen_slot(gen_pages[k], 1);
if (s) *s = g;
}
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call
}
int changed = 0;
@@ -560,6 +589,31 @@ static uint32_t snap_at[SNAP_REGIONS]; // epoch of the region's last snapsh
static uint8_t snap_untracked[SNAP_REGIONS];
static uint64_t n_snap_checks, n_snap_calls, n_snap_untracked;
// gen_lock held exclusively. Queries the regions r0..r1 that were not snapped in epoch, one query per run of them;
// returns the number of queries.
static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) {
uint32_t g = 0;
uint64_t calls = 0;
for (uint64_t r = r0; r <= r1;) {
if (snap_at[r] == epoch) { r++; continue; }
uint64_t k = r; // one query over a run of regions not seen in this epoch
while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++;
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
int tracked = 1;
if (!g) g = ++gen_now;
for (;;) {
uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages);
if (n == UINT64_MAX) { tracked = 0; break; }
for (uint64_t j = 0; j < n; j++) stamp_page(gen_pages[j], g);
if (n < sizeof gen_pages / sizeof *gen_pages) break;
}
calls++;
for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
r = k + 1;
}
return calls;
}
int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) {
if (!size) { *now = since; return 0; }
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
@@ -572,26 +626,7 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e
if (excl) {
ReleaseSRWLockShared(&gen_lock);
AcquireSRWLockExclusive(&gen_lock);
uint32_t g = ++gen_now;
for (uint64_t r = r0; r <= r1;) {
if (snap_at[r] == epoch) { r++; continue; }
uint64_t k = r; // one query over a run of regions not seen in this epoch
while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++;
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
int tracked = 1;
for (;;) {
uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages);
if (n == UINT64_MAX) { tracked = 0; break; }
for (uint64_t j = 0; j < n; j++) {
uint32_t *s = gen_slot(gen_pages[j], 1);
if (s) *s = g;
}
if (n < sizeof gen_pages / sizeof *gen_pages) break;
}
n_snap_calls++;
for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
r = k + 1;
}
n_snap_calls += snap_regions(r0, r1, epoch);
}
int untracked = 0, changed = 0;
for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r];
@@ -663,16 +698,13 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
for (;;) {
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
if (n == UINT64_MAX) return 0;
for (uint64_t k = 0; k < n; k++) {
uint32_t *s = gen_slot(gen_pages[k], 1);
if (s) *s = g;
}
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
if (n < sizeof gen_pages / sizeof *gen_pages) return 1;
}
}
void vk_mem_dirty_submit(void) {
if (!(rt_trace & RT_TRACE_PERF)) return;
if (!(rt_trace & RT_TRACE_PERF) || vk_mirror_on) return; // the mirror's own line replaces the census
uint64_t t0 = kern_now_ns();
uint32_t prev = dirty_sub; // the submit whose replay noted the bindings
dirty_sub = prev + 1 ? prev + 1 : 1;
@@ -793,8 +825,505 @@ static void dirty_frame(uint64_t frame) {
rt_log("%s", line);
}
// ---------------------------------------------------------------- VRAM mirror of buffer-bound pools
// build/mem-plan.md, "Design: VRAM mirror of buffer-bound pools". An imported pool keeps vertices, indices, constants
// and storage buffers in system memory, so a discrete GPU reads them over PCIe on every draw (GTX 1050 Ti and RTX
// 3090 Ti: about 110 ms of GPU time per frame in the field). In mirror mode (device class discrete, TOTK_MEMORY) an
// imported pool gets a device-local buffer over its imported range the first time the replay uses it as a vertex,
// index, uniform, storage or indirect buffer (vk_mem_use). From then on those bindings, and the buffer copies, fills
// and uniform updates that touch the pool (vk_mem_pick), use the mirror at the same offsets. Guest memory stays what
// the CPU reads and writes; textures and every host path that reads or writes pool bytes keep using it.
// - CPU -> mirror: the first access of a submission to a mirror syncs it (mirror_sync): the 2 MiB write-watch
// snapshots of its regions for this epoch (snap_regions, shared with the texture checks: the OS bits keep one
// consumer, the shared page generations) and every page stamped after the mirror's mgen goes into a copy; so do the
// nvnMemoryPoolFlushMappedRange ranges. The first use copies the whole range. The copies run in a prologue command
// buffer that vk_sync.c submits right before the part that accessed the mirror first, between full barriers.
// vk_mem_mirror_begin snaps the regions of the pools the previous submission accessed in a few merged queries.
// - GPU -> guest: what the replay writes into a mirror (storage buffers its shaders write, buffer copies, fills,
// uniform updates, texture-to-buffer copies: vk_mem_gpu_wrote) goes back into the imported buffer at the end of the
// submission, exact byte ranges, before the fence signals (vk_mem_mirror_end).
// - TOTK_MEMORY=verify: every submission also copies up to VERIFY_BYTES of mirror pages, round robin, into host memory;
// at its fence the pages the CPU has not written since their sync are compared with guest memory.
// A pool stays imported when its mirror would not fit the VRAM budget, its range is not write-watched, it overlaps
// another pool or the allocation fails.
#define VERIFY_BYTES (4u << 20)
#define VERIFY_SAMPLES 64
#define VERIFY_LOG 16
enum { MR_BUDGET, MR_UNTRACKED, MR_OVERLAP, MR_ALLOC, MR_N };
int vk_mirror_on;
static int mirror_verify;
static int mirror_gap = 2; // pre-snap: hot pools at most this many 2 MiB regions apart share one query
static PoolRec **mlist; // pool_lock: mirrored pools by import_lo
static int nm, mcap;
static uint32_t msub; // replay: number of the submission being recorded
static uint32_t mepoch; // replay: its write-watch snapshot epoch (vk_texture_epoch)
typedef struct { PoolRec *r; uint64_t off, size; } MRange;
static MRange *pend, *wbl; // replay: copies for the next prologue; GPU writes into mirrors this submission
static int npend, cpend, nwbl, cwbl;
static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions
static int cregions;
static SRWLOCK flush_lock = SRWLOCK_INIT;
typedef struct {
uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges;
uint64_t presnap_queries, sync_queries, cpu_ns, prologues;
} MirrorAcc;
static MirrorAcc mrep; // replay: this submission
static MirrorAcc macc; // mirror_stat_lock: since the last summary
static SRWLOCK mirror_stat_lock = SRWLOCK_INIT;
static volatile LONG64 m_live_n, m_live_bytes, m_refused_n[MR_N], m_refused_bytes[MR_N];
static volatile LONG64 m_flush_calls, m_flush_bytes, v_pages, v_skipped, v_mismatch;
static void mirror_mode(int discrete) {
const char *e = cfg_get("TOTK_MEMORY");
int mode = discrete ? 1 : 0; // 0 import, 1 mirror, 2 verify
const char *why = discrete ? "auto, discrete gpu" : "auto, uma";
if (e && *e && strcmp(e, "auto")) {
if (!strcmp(e, "import")) mode = 0, why = "TOTK_MEMORY=import";
else if (!strcmp(e, "mirror")) mode = 1, why = "TOTK_MEMORY=mirror";
else if (!strcmp(e, "verify")) mode = 2, why = "TOTK_MEMORY=verify";
else rt_log("mem: TOTK_MEMORY=%s is not import, mirror, verify or auto: auto", e);
}
vk_mirror_on = mode > 0;
mirror_verify = mode == 2;
const char *g = cfg_get("TOTK_MIRROR_GAP");
mirror_gap = g && *g ? atoi(g) : 2;
if (mirror_gap < 0) mirror_gap = 0;
rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why,
mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory");
}
static void range_add(MRange **a, int *n, int *cap, PoolRec *r, uint64_t off, uint64_t size) {
if (*n && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) {
(*a)[*n - 1].size += size;
return;
}
if (*n == *cap) grow((void **)a, cap, sizeof **a);
(*a)[(*n)++] = (MRange){r, off, size};
}
static int range_cmp(const void *x, const void *y) {
const MRange *a = x, *b = y;
if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1;
return a->off < b->off ? -1 : a->off > b->off;
}
// Records the ranges as one vkCmdCopyBuffer per pool, overlapping and adjacent ranges merged; to_mirror: buf -> mbuf,
// else mbuf -> buf. Returns the commands recorded; adds what they move to *bytes and *copies.
static int copy_ranges(VkCommandBuffer cb, MRange *a, int n, int to_mirror, uint64_t *bytes, uint64_t *copies) {
qsort(a, (size_t)n, sizeof *a, range_cmp);
int rec = 0;
for (int i = 0; i < n;) {
PoolRec *r = a[i].r;
uint64_t lim = r->import_hi - r->import_lo;
int k = 0;
for (; i < n && a[i].r == r; i++) {
uint64_t lo = a[i].off, hi = a[i].off + a[i].size;
if (hi > lim) hi = lim;
if (lo >= hi) continue;
if (k && lo <= mregions[k - 1].srcOffset + mregions[k - 1].size) {
uint64_t end = mregions[k - 1].srcOffset + mregions[k - 1].size;
if (hi > end) mregions[k - 1].size += hi - end;
continue;
}
if (k == cregions) grow((void **)&mregions, &cregions, sizeof *mregions);
mregions[k++] = (VkBufferCopy){lo, lo, hi - lo};
}
if (!k) continue;
if (to_mirror) vkCmdCopyBuffer(cb, r->buf, r->mbuf, (uint32_t)k, mregions);
else vkCmdCopyBuffer(cb, r->mbuf, r->buf, (uint32_t)k, mregions);
rec++;
for (int j = 0; j < k; j++) *bytes += mregions[j].size;
*copies += (uint64_t)k;
}
return rec;
}
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
VkAccessFlags2 da) {
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = ss, .srcAccessMask = sa,
.dstStageMask = ds, .dstAccessMask = da};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(cb, &di);
}
static void refuse(PoolRec *r, int why, const char *what) {
r->mstate = MS_REFUSED;
uint64_t n = r->import_hi - r->import_lo;
if (InterlockedIncrement64(&m_refused_n[why]) <= 4) {
char f[160];
flag_names(r->flags, f, sizeof f);
rt_log("mem: pool %#llx+%#llx (%s) stays imported: %s", (unsigned long long)r->lo, (unsigned long long)r->size,
f, what);
}
InterlockedAdd64(&m_refused_bytes[why], (LONG64)n);
}
// Replay thread: the pool's mirror on its first buffer use.
static void mirror_create(PoolRec *r) {
VkCtx *c = vk_ctx();
uint64_t n = r->import_hi - r->import_lo;
r->mstate = MS_REFUSED;
if (r->kind != POOL_IMPORT || !r->buf || !n) return;
AcquireSRWLockShared(&pool_lock);
int i = pidx_upper(&pool_index, r->lo) - 1;
while (i >= 0 && pool_index.e[i].buf != r) i--;
int alone = i >= 0 && (i == 0 || pool_index.maxhi[i - 1] <= pool_index.e[i].lo) &&
(i + 1 == pool_index.n || pool_index.e[i + 1].lo >= pool_index.e[i].hi);
ReleaseSRWLockShared(&pool_lock);
if (i < 0) return; // finalized meanwhile
if (!alone) return refuse(r, MR_OVERLAP, "it overlaps another pool");
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
if (r1 >= SNAP_REGIONS) return refuse(r, MR_UNTRACKED, "outside the write-watched address space");
uint64_t used, budget;
int exact;
if (vk_mem_vram(&used, &budget, &exact) && used + n + budget / 8 > budget) {
char why[128];
snprintf(why, sizeof why, "its %.1f MiB mirror does not fit the VRAM budget (%.0f of %.0f MiB in use)",
n / 1048576.0, used / 1048576.0, budget / 1048576.0);
return refuse(r, MR_BUDGET, why);
}
AcquireSRWLockExclusive(&gen_lock);
mrep.sync_queries += snap_regions(r0, r1, mepoch);
int untracked = 0;
for (uint64_t k = r0; k <= r1; k++) untracked |= snap_untracked[k];
r->mgen = gen_now; // the whole-range copy below holds every write observed so far
ReleaseSRWLockExclusive(&gen_lock);
if (untracked) return refuse(r, MR_UNTRACKED, "its range is not write-watched");
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = n, .usage = pool_usage(c),
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
VkResult res = vmaCreateBuffer(c->vma, &bi, &ai, &r->mbuf, &r->malloc_, NULL);
if (res != VK_SUCCESS) {
r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL;
char why[96];
snprintf(why, sizeof why, "device-local allocation of %.1f MiB failed (%d)", n / 1048576.0, (int)res);
return refuse(r, MR_ALLOC, why);
}
AcquireSRWLockExclusive(&pool_lock);
if (nm == mcap) grow((void **)&mlist, &mcap, sizeof *mlist);
int at = nm;
while (at > 0 && mlist[at - 1]->import_lo > r->import_lo) at--;
memmove(mlist + at + 1, mlist + at, (size_t)(nm - at) * sizeof *mlist);
mlist[at] = r;
nm++;
ReleaseSRWLockExclusive(&pool_lock);
r->acc_sub = msub;
r->mstate = MS_ON;
range_add(&pend, &npend, &cpend, r, 0, n);
mrep.first_n++, mrep.first_bytes += n;
InterlockedIncrement64(&m_live_n);
InterlockedAdd64(&m_live_bytes, (LONG64)n);
}
static void mirror_forget(PoolRec *r) {
for (int i = 0; i < nm; i++)
if (mlist[i] == r) {
memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist);
nm--;
InterlockedDecrement64(&m_live_n);
InterlockedAdd64(&m_live_bytes, -(LONG64)(r->import_hi - r->import_lo));
break;
}
}
// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran.
static PoolRec **retiring;
static int nretiring, cretiring;
static SRWLOCK retire_lock = SRWLOCK_INIT;
static void mirror_retiring(PoolRec *r) {
AcquireSRWLockExclusive(&retire_lock);
if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring);
retiring[nretiring++] = r;
ReleaseSRWLockExclusive(&retire_lock);
}
static void mirror_retire(void *arg) {
AcquireSRWLockExclusive(&retire_lock);
for (int i = 0; i < nretiring; i++)
if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; }
ReleaseSRWLockExclusive(&retire_lock);
destroy_rec(vk_ctx(), arg);
}
// Replay thread, the first access of this submission: queues the copies of the pages written since the last sync.
static void mirror_sync(PoolRec *r) {
uint64_t t0 = kern_now_ns();
r->acc_sub = msub;
uint64_t lo = r->import_lo, hi = r->import_hi;
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
int excl = 0;
AcquireSRWLockShared(&gen_lock);
for (uint64_t k = r0; k <= r1; k++)
if (snap_at[k] != mepoch) { excl = 1; break; }
if (excl) {
ReleaseSRWLockShared(&gen_lock);
AcquireSRWLockExclusive(&gen_lock);
mrep.sync_queries += snap_regions(r0, r1, mepoch);
}
uint32_t since = r->mgen;
for (uint64_t k = r0; k <= r1; k++) {
if (reg_max[k] <= since) continue;
uint64_t a = k << SNAP_SHIFT, b = a + (1ull << SNAP_SHIFT);
if (a < lo) a = lo;
if (b > hi) b = hi;
const uint32_t *ch = gen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk
if (!ch) continue;
for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT)
if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since)
range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT);
}
r->mgen = gen_now;
if (excl) ReleaseSRWLockExclusive(&gen_lock);
else ReleaseSRWLockShared(&gen_lock);
if (__atomic_load_n(&r->fl_hi, __ATOMIC_ACQUIRE)) {
AcquireSRWLockExclusive(&flush_lock);
uint64_t a = r->fl_lo, b = r->fl_hi;
r->fl_hi = 0;
ReleaseSRWLockExclusive(&flush_lock);
a = a > lo ? a & ~0xfffull : lo;
b = b < hi ? (b + 0xfff) & ~0xfffull : hi;
if (a < b) range_add(&pend, &npend, &cpend, r, a - lo, b - a);
}
mrep.syncs++;
mrep.cpu_ns += kern_now_ns() - t0;
}
void vk_mem_use(VkMemSpan *sp) {
PoolRec *r = sp->pool;
if (r->mstate == MS_NONE) mirror_create(r);
if (r->mstate != MS_ON) return;
if (r->acc_sub != msub) mirror_sync(r);
sp->buffer = r->mbuf;
}
void vk_mem_pick(VkMemSpan *sp) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON) return;
if (r->acc_sub != msub) mirror_sync(r);
sp->buffer = r->mbuf;
}
int vk_mem_mirror_probe(uint64_t addr) {
VkMemSpan sp;
if (!vk_mirror_on || !vk_mem_resolve(addr, 1, &sp) || !sp.pool) return 0;
mepoch = vk_texture_epoch();
vk_mem_use(&sp);
return ((PoolRec *)sp.pool)->mstate == MS_ON;
}
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return;
range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size);
}
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) {
if (!vk_mirror_on || !size || !p->mem) return;
InterlockedIncrement64(&m_flush_calls);
InterlockedAdd64(&m_flush_bytes, (LONG64)size);
AcquireSRWLockShared(&pool_lock);
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)p->mem, 0);
PoolRec *r = e && e->key == p ? e->buf : NULL;
if (r && r->mstate == MS_ON) {
uint64_t a = (uintptr_t)p->mem + off, b = a + size;
AcquireSRWLockExclusive(&flush_lock);
if (!r->fl_hi) r->fl_lo = a, r->fl_hi = b;
else {
if (a < r->fl_lo) r->fl_lo = a;
if (b > r->fl_hi) r->fl_hi = b;
}
ReleaseSRWLockExclusive(&flush_lock);
}
ReleaseSRWLockShared(&pool_lock);
}
void vk_mem_mirror_begin(void) {
if (!vk_mirror_on) return;
uint64_t t0 = kern_now_ns();
uint32_t prev = msub;
msub = msub + 1 ? msub + 1 : 1;
mepoch = vk_texture_epoch();
uint64_t q = 0, run0 = 0, run1 = 0;
int have = 0;
AcquireSRWLockShared(&pool_lock);
AcquireSRWLockExclusive(&gen_lock);
for (int i = 0; i < nm; i++) {
const PoolRec *r = mlist[i];
if (r->acc_sub != prev) continue;
uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT;
if (have && a <= run1 + 1 + (uint64_t)mirror_gap) {
if (b > run1) run1 = b;
continue;
}
if (have) q += snap_regions(run0, run1, mepoch);
run0 = a, run1 = b, have = 1;
}
if (have) q += snap_regions(run0, run1, mepoch);
ReleaseSRWLockExclusive(&gen_lock);
ReleaseSRWLockShared(&pool_lock);
mrep.subs++;
mrep.presnap_queries += q;
mrep.cpu_ns += kern_now_ns() - t0;
}
int vk_mem_mirror_pending(void) { return npend; }
void vk_mem_mirror_prologue(VkCommandBuffer cb) {
if (!npend) return;
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies);
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
npend = 0;
mrep.prologues++;
}
// ---- verify mode
typedef struct { PoolRec *r; uint64_t off, len, dst; uint32_t mgen; } VSample;
typedef struct { uint32_t sub; int n; VkBuffer buf; VmaAllocation alloc; const uint8_t *map; VSample s[VERIFY_SAMPLES]; } VJob;
static int ver_i;
static uint64_t ver_off;
// The CPU wrote the page since the mirror's sync that mgen records: stamped later, or its write-watch bit is set.
static int page_dirty(uint64_t a, uint32_t mgen) {
uint32_t *s = gen_slot(a, 0);
if (s && __atomic_load_n(s, __ATOMIC_RELAXED) > mgen) return 1;
uint64_t p[1];
return mem_written_peek(a, 1ull << GEN_PAGE_SHIFT, p, 1) != 0; // UINT64_MAX (not watched) counts as written
}
static void verify_cb(void *arg) { // completion thread
VJob *j = arg;
static volatile LONG logged;
uint64_t pages = 0, skipped = 0, bad = 0;
for (int i = 0; i < j->n; i++) {
const VSample *s = &j->s[i];
PoolRec *r = s->r;
uint32_t wb = __atomic_load_n(&r->wb_sub, __ATOMIC_RELAXED);
if (wb != j->sub && wb - j->sub < 0x80000000u) { // a later submission wrote back into it
skipped += s->len >> GEN_PAGE_SHIFT;
continue;
}
for (uint64_t o = 0; o < s->len; o += 1ull << GEN_PAGE_SHIFT) {
uint64_t a = r->import_lo + s->off + o;
const uint8_t *m = j->map + s->dst + o, *g = (const uint8_t *)(uintptr_t)a;
pages++;
if (!memcmp(m, g, 1ull << GEN_PAGE_SHIFT)) continue;
if (page_dirty(a, s->mgen)) { skipped++; continue; }
bad++;
if (InterlockedIncrement(&logged) <= VERIFY_LOG) {
int k = 0, e = (1 << GEN_PAGE_SHIFT) - 1;
while (k < e && m[k] == g[k]) k++;
while (e > k && m[e] == g[e]) e--;
char f[160];
flag_names(r->flags, f, sizeof f);
rt_log("mem: verify mismatch: pool %#llx+%#llx (%s) offset %#llx: bytes %#x..%#x of the page differ, "
"mirror %02x guest %02x at the first (submission %u)", (unsigned long long)r->lo,
(unsigned long long)r->size, f, (unsigned long long)(a - r->lo), k, e, m[k], g[k], j->sub);
}
}
}
InterlockedAdd64(&v_pages, (LONG64)pages);
InterlockedAdd64(&v_skipped, (LONG64)skipped);
InterlockedAdd64(&v_mismatch, (LONG64)bad);
vmaDestroyBuffer(vk_ctx()->vma, j->buf, j->alloc);
free(j);
}
static int verify_record(VkCommandBuffer cb, uint64_t fence) {
VJob *j = calloc(1, sizeof *j);
if (!j) rt_fatal("vk: out of memory");
uint64_t left = VERIFY_BYTES, dst = 0;
AcquireSRWLockShared(&pool_lock);
for (int tries = 0; nm && left && j->n < VERIFY_SAMPLES && tries <= nm; tries++) {
if (ver_i >= nm) ver_i = 0, ver_off = 0;
PoolRec *r = mlist[ver_i];
uint64_t size = r->import_hi - r->import_lo;
if (ver_off >= size) { ver_i++, ver_off = 0; continue; }
uint64_t len = size - ver_off < left ? size - ver_off : left;
j->s[j->n++] = (VSample){r, ver_off, len, dst, r->mgen};
dst += len, left -= len, ver_off += len;
if (ver_off >= size) ver_i++, ver_off = 0;
}
ReleaseSRWLockShared(&pool_lock);
if (!j->n) { free(j); return 0; }
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = dst,
.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
VmaAllocationInfo info;
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &j->buf, &j->alloc, &info) != VK_SUCCESS) { free(j); return 0; }
j->map = info.pMappedData;
j->sub = msub;
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT);
for (int i = 0; i < j->n; i++) {
VkBufferCopy c = {j->s[i].off, j->s[i].dst, j->s[i].len};
vkCmdCopyBuffer(cb, j->s[i].r->mbuf, j->buf, 1, &c);
}
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
VK_ACCESS_2_HOST_READ_BIT);
vk_completion_add(fence, verify_cb, j);
return j->n + 2;
}
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
if (!vk_mirror_on) return 0;
int rec = 0;
if (nwbl) {
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT);
for (int i = 0; i < nwbl; i++) wbl[i].r->wb_sub = msub;
mrep.wb_ranges += (uint64_t)nwbl;
rec += copy_ranges(cb, wbl, nwbl, 0, &mrep.wb_bytes, &mrep.wb_copies) + 2;
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
nwbl = 0;
}
if (mirror_verify) rec += verify_record(cb, fence);
AcquireSRWLockExclusive(&mirror_stat_lock);
uint64_t *d = (uint64_t *)&macc, *s = (uint64_t *)&mrep;
for (size_t i = 0; i < sizeof macc / sizeof(uint64_t); i++) d[i] += s[i];
ReleaseSRWLockExclusive(&mirror_stat_lock);
memset(&mrep, 0, sizeof mrep);
return rec;
}
// One summary line: with TOTK_TRACE=perf every 60 frames, else at frames 600 and 1800.
static void mirror_log(uint64_t frame) {
static uint64_t last;
if (!vk_mirror_on) return;
AcquireSRWLockExclusive(&mirror_stat_lock);
MirrorAcc a = macc;
memset(&macc, 0, sizeof macc);
ReleaseSRWLockExclusive(&mirror_stat_lock);
double f = frame > last ? (double)(frame - last) : 1.0, M = 1048576.0;
last = frame;
uint64_t fc = (uint64_t)InterlockedExchange64(&m_flush_calls, 0), fb = (uint64_t)InterlockedExchange64(&m_flush_bytes, 0);
char ver[160] = "";
if (mirror_verify)
snprintf(ver, sizeof ver, "; verify: pages compared %lld, skipped %lld (written since), mismatches %lld",
(long long)InterlockedExchange64(&v_pages, 0), (long long)InterlockedExchange64(&v_skipped, 0),
(long long)InterlockedExchange64(&v_mismatch, 0));
rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked "
"%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, "
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries "
"%.1f pre-snap + %.1f sync, prologues %.2f, tracking cpu %.3f ms%s",
(unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET],
m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP],
(long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f,
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M,
a.presnap_queries / f, a.sync_queries / f, a.prologues / f, a.cpu_ns / f / 1e6, ver);
}
// Device class (build/mem-plan.md): INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a
// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Only logged for now.
// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Sets the mirror mode.
void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m) {
int big_local = -1, host_heap = -1;
for (uint32_t h = 0; h < m->memoryHeapCount; h++) {
@@ -811,6 +1340,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic
else if (big_local < 0) cls = "uma", why = "no device-local heap of 1 GiB or more";
else if (host_heap < 0) cls = "uma", why = "every heap is device-local";
else cls = "discrete", why = "device-local heap separate from host memory";
int discrete = cls[0] == 'd';
char line[2048];
size_t k = (size_t)snprintf(line, sizeof line, "mem: class %s (%s; device type %d %s, %s)", cls, why, t, tn, p->deviceName);
for (uint32_t h = 0; h < m->memoryHeapCount; h++)
@@ -830,6 +1360,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic
f & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT ? " LAZY" : "");
}
rt_log("%s", line);
mirror_mode(discrete);
}
// ---------------------------------------------------------------- upload batches and deferred destruction
@@ -1069,6 +1600,9 @@ void vk_mem_shutdown(void) {
AcquireSRWLockExclusive(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
pool_index.n = 0;
nm = npend = nwbl = 0; // mirrors went with their pools
for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran
nretiring = 0;
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
nphys = 0;
ReleaseSRWLockExclusive(&pool_lock);
+59 -23
View File
@@ -107,27 +107,33 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order
free(t);
}
// At the start of a part's command buffer (sub_lock held).
static void ts_begin(VkCommandBuffer cb) {
ts_cur = -1;
if (ts_state != 1) return;
// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed.
static int ts_open(VkCommandBuffer cb) {
if (ts_state != 1) return -1;
uint32_t p = ts_next % TS_PAIRS;
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return; }
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return -1; }
ts_next++;
ts_busy[p] = 1;
vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2);
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p);
ts_cur = (int)p;
return (int)p;
}
// At the end of the part, before vkEndCommandBuffer; fence: the value whose completion covers the part.
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
if (ts_cur < 0) return;
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)ts_cur + 1);
// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it.
static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence) {
if (pair < 0) return;
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1);
TsRead *t = malloc(sizeof *t);
if (!t) rt_fatal("vk: out of memory");
t->pair = (uint32_t)ts_cur;
t->pair = (uint32_t)pair;
vk_completion_add(fence, ts_read_cb, t);
}
// The pair of the part being recorded.
static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); }
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
ts_close(cb, ts_cur, fence);
ts_cur = -1;
}
@@ -227,27 +233,47 @@ void vk_sync_start(void) {
ReleaseSRWLockExclusive(&start_lock);
}
// A command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held.
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move
// slots[].
static int claim_slot(VkCtx *c, uint64_t fence) {
uint64_t gpu = 0;
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
for (int i = 0; i < nslots; i++)
if (slots[i].fence <= gpu) {
slots[i].fence = fence;
cur_slot = i;
return &slots[i];
return i;
}
CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1));
if (!n) rt_fatal("vk: out of memory (command buffers)");
slots = n;
CmdSlot *s = &slots[nslots];
cur_slot = nslots;
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
s->fence = fence;
nslots++;
return s;
return nslots++;
}
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
cur_slot = claim_slot(c, fence);
return &slots[cur_slot];
}
// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is
// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the
// part was recorded, sub_lock held; may move slots[].
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
int i = claim_slot(c, fence); // before reading slots: the claim may move it
VkCommandBuffer cb = slots[i].cb;
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
vkBeginCommandBuffer(cb, &bi);
int pair = ts_open(cb);
vk_mem_mirror_prologue(cb);
ts_close(cb, pair, fence);
vkEndCommandBuffer(cb);
return cb;
}
VkCommandBuffer vk_sync_split(void) {
@@ -256,8 +282,11 @@ VkCommandBuffer vk_sync_split(void) {
ts_end(cb, cur_fence);
vkEndCommandBuffer(cb);
vk_upload_flush(0); // the uploads this part asked for precede it in queue order
VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = 1, .pCommandBufferInfos = &cbi};
VkCommandBuffer pro = prologue(c, cur_fence);
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
.pCommandBufferInfos = pro ? cbi : cbi + 1};
vk_queue_lock();
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
@@ -292,11 +321,16 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
perf_replay(t0);
// After the recording: the uploads it asked for (and every earlier one) precede it in queue order.
vk_upload_flush(0);
VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb};
VkCommandBuffer pro = prologue(c, fence);
s = &slots[cur_slot];
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}};
VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = recorded ? 1 : 0,
.pCommandBufferInfos = &cbi, .signalSemaphoreInfoCount = 1, .pSignalSemaphoreInfos = &sig};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u),
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1,
.pSignalSemaphoreInfos = &sig};
vk_queue_lock();
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
@@ -365,6 +399,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
void vk_sync_finish(void) {
vk_sync_start();
vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX);
+2
View File
@@ -888,6 +888,8 @@ int vk_texture_slot_swz(int id, uint8_t swz[4]) {
return ok;
}
uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); }
void vk_texture_new_submit(void) {
lock();
if (!++submit_epoch) submit_epoch = 1;