diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index 58a1fabd..4cfe2b3b 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -47,6 +47,9 @@ uint64_t nvn_new_serial(void); // next tex_t / prog_ uint64_t nvn_fence_last(void); // fence value of the newest submit (0 before the first) int nvn_depth_minus_w(void); // 1 after DeviceSetDepthMode(NEAR_IS_MINUS_W), else 0 void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the game wrote pool storage: back end re-reads +// TOTK_TRACE=perf: MemoryPoolFlushMappedRange [0] and BufferFlushMappedRange [1] calls [0] and bytes [1] by the pool's +// CPU class (0 CPU_CACHED, 1 CPU_UNCACHED, 2 other) since the previous call. +void nvn_flush_stats(uint64_t out[2][3][2]); void nvn_submit_prepare(const uint64_t *handles, int n); // weak no-op; the command-buffer slice reads deferred binds #define NVN_HANDLE_TEX(h) ((int)((h) & 0xfffff)) // combined texture handle = texture id | sampler id << 20 #define NVN_HANDLE_SAMP(h) ((int)(((h) >> 20) & 0xfff)) diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 665fcc24..7978ed9e 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -287,8 +287,26 @@ NVN_API(nvnMemoryPoolFinalize) { o->span = 0; o->magic = 0; } +static uint64_t flush_stat[2][3][2]; // nvn_flush_stats + +static void flush_note(int kind, const pool_t *p, uint64_t size) { + uint32_t f = p ? p->flags : 0; + int k = f & NVN_MEMORY_POOL_FLAGS_CPU_NO_ACCESS ? 2 : f & NVN_MEMORY_POOL_FLAGS_CPU_CACHED ? 0 + : f & NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED ? 1 : 2; + __atomic_fetch_add(&flush_stat[kind][k][0], 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&flush_stat[kind][k][1], size, __ATOMIC_RELAXED); +} + +void nvn_flush_stats(uint64_t out[2][3][2]) { + for (int i = 0; i < 2; i++) + for (int k = 0; k < 3; k++) + for (int j = 0; j < 2; j++) out[i][k][j] = __atomic_exchange_n(&flush_stat[i][k][j], 0, __ATOMIC_RELAXED); +} + // void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent -NVN_API(nvnMemoryPoolFlushMappedRange) { (void)c; } +NVN_API(nvnMemoryPoolFlushMappedRange) { + if (rt_trace & RT_TRACE_PERF) flush_note(0, arg_p(c, 0), arg_x(c, 2)); +} // NVNmemoryPoolFlags nvnMemoryPoolGetFlags(const NVNmemoryPool *p) NVN_API(nvnMemoryPoolGetFlags) { ret_x(c, ((const pool_t *)arg_p(c, 0))->flags); } // void *nvnMemoryPoolMap(const NVNmemoryPool *p): NULL for CPU_NO_ACCESS @@ -402,6 +420,7 @@ NVN_API(nvnBufferFlushMappedRange) { if (off < 0 || (uint64_t)off > o->size || size > o->size - (uint64_t)off) rt_fatal("nvn: BufferFlushMappedRange %#llx+%#llx outside the buffer (%#llx bytes)", (unsigned long long)off, (unsigned long long)size, (unsigned long long)o->size); + if (rt_trace & RT_TRACE_PERF) flush_note(1, o->pool, size); uint64_t a = buf_address(o); if (a && !nvn_gpu_only(a)) nvn_guest_written(a + (uint64_t)off, size); } diff --git a/runtime/vk.h b/runtime/vk.h index a1da6708..69dc6449 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -121,6 +121,7 @@ typedef struct { VmaAllocation alloc; VkDeviceSize alloc_offset; int cls; + void *pool; // the imported pool record behind the span (NULL for device memory), for VK_MEM_BOUND } VkMemSpan; // Traffic census (vk_memory.c vk_mem_census): bytes the replay binds or copies, by pool class and use. Imported pools by // their CPU access flag; DEVICE is physical memory (directly or through a virtual pool); SCRATCH the replay's transient @@ -130,6 +131,14 @@ enum { VK_MCLS_NONE, VK_MCLS_HOST_NOACC, VK_MCLS_HOST_UNC, VK_MCLS_HOST_CACHED, enum { VK_MUSE_VB, VK_MUSE_IB, VK_MUSE_UBO, VK_MUSE_SSBO, VK_MUSE_IND, VK_MUSE_COPY_SRC, VK_MUSE_GPU_WRITE, VK_MUSE_CPU_READ, VK_MUSE_N }; void vk_mem_count(int cls, int use, uint64_t bytes); +// TOTK_TRACE=perf dirty census (vk_memory.c): the replay notes each imported pool it binds as a vertex, index, uniform, +// storage or indirect buffer; at the start of every submit the pages the CPU wrote in such pools since the previous +// submit are counted from the shared page generations and logged as `mem: dirty` every 60 frames. Nothing when tracing +// is off: VK_MEM_BOUND is one flag test at the call site. +void vk_mem_bound_note(void *pool); +#define VK_MEM_BOUND(sp) do { if ((rt_trace & RT_TRACE_PERF) && (sp).pool) vk_mem_bound_note((sp).pool); } while (0) +void vk_mem_dirty_submit(void); // vk_sync_submit, under its lock, before the replay +void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m); // once int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked) void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and, // with TOTK_TRACE=perf, every 1800 frames diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 80a20b1d..2ab28c51 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -165,6 +165,7 @@ static int pick_device(void) { } free(qp); rt_log("vk: using gpu %d: %s", best, g_vk.props.deviceName); + vk_mem_class_log(&g_vk.props, &g_vk.mem_props); return 1; } diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 38b7c1fe..8fd5ca0b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -580,6 +580,7 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, if (sp.offset % align == 0) { *out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n}; vk_mem_count(sp.cls, use, n); + VK_MEM_BOUND(sp); return 1; } if (sp.alloc) return 0; @@ -1086,6 +1087,7 @@ static void bind_vertex_buffers(VkExec *ex) { VkBuffer vb[17]; VkDeviceSize vo[17], vsz[17], vstr[17]; int vcls[17] = {0}; + void *vpool[17] = {0}; int dyn = vk_pipeline_vertex_input_dynamic(); for (int b = 0; b < 17; b++) { vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0; @@ -1094,7 +1096,7 @@ static void bind_vertex_buffers(VkExec *ex) { VkMemSpan sp; if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue; uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail; - vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls; + vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool; } if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) && (dyn || !memcmp(B.vstr, vstr, sizeof vstr))) @@ -1103,6 +1105,9 @@ static void bind_vertex_buffers(VkExec *ex) { ex->recorded++; for (int b = 0; b < 17; b++) if (vcls[b]) vk_mem_count(vcls[b], VK_MUSE_VB, vsz[b]); + if (rt_trace & RT_TRACE_PERF) + for (int b = 0; b < 17; b++) + if (vpool[b]) vk_mem_bound_note(vpool[b]); memcpy(B.vb, vb, sizeof vb), memcpy(B.vo, vo, sizeof vo), memcpy(B.vsz, vsz, sizeof vsz), memcpy(B.vstr, vstr, sizeof vstr); B.vb_valid = 1; } @@ -1269,6 +1274,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { vkCmdBindIndexBuffer(ex->cb, sp.buffer, sp.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); ex->recorded++; vk_mem_count(sp.cls, VK_MUSE_IB, bytes); + VK_MEM_BOUND(sp); } vkCmdDrawIndexed(ex->cb, (uint32_t)d->count, d->instances > 0 ? (uint32_t)d->instances : 1, 0, d->base_vertex, (uint32_t)d->base_instance); @@ -1293,6 +1299,8 @@ static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) { vkCmdDrawIndexedIndirect(ex->cb, ip.buffer, ip.offset, 1, sizeof(VkDrawIndexedIndirectCommand)); ex->recorded += 2; vk_mem_count(ip.cls, VK_MUSE_IND, sizeof(VkDrawIndexedIndirectCommand)); + VK_MEM_BOUND(ip); + VK_MEM_BOUND(ib); } // ---- dispatch and uniform buffer updates @@ -1347,6 +1355,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vkCmdDispatchIndirect(ex->cb, ip.buffer, ip.offset); ex->recorded++; vk_mem_count(ip.cls, VK_MUSE_IND, sizeof(VkDispatchIndirectCommand)); + VK_MEM_BOUND(ip); if (vk_dump_armed) vk_dump_dispatch(ex, cs, p, info, &(p_dispatch){0, 0, 0, d->caller}); } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 328b04b5..ebda951c 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -39,6 +39,11 @@ typedef struct PoolRec { VPage *pt; // POOL_VIRT: size / 64 KiB entries uint64_t npages; volatile long used; // resolved at least once: the GPU may still read it at finalize + // TOTK_TRACE=perf dirty census (vk_mem_dirty_submit), replay thread under the submit lock only: + uint32_t bound_sub; // last submit that bound it as a buffer (dirty_sub), 0 never + uint32_t synced_gen; // page generation its lazy-mirror count covers; DIRTY_UNSYNCED until scanned + uint8_t bufbound; // ever bound as a vertex, index, uniform, storage or indirect buffer + uint8_t untracked; // the last dirty scan found no write watch over it } PoolRec; static SRWLOCK pool_lock = SRWLOCK_INIT; @@ -313,7 +318,7 @@ static int rec_class(const PoolRec *r) { : r->flags & NVN_MEMORY_POOL_FLAGS_CPU_CACHED ? VK_MCLS_HOST_CACHED : VK_MCLS_HOST_UNC; } -typedef struct { uint64_t gen, lo, hi, base, end; VkBuffer buf; VmaAllocation alloc; int cls; } ResolveHit; +typedef struct { uint64_t gen, lo, hi, base, end; VkBuffer buf; VmaAllocation alloc; int cls; void *pool; } ResolveHit; static _Thread_local ResolveHit rhit[4]; static _Thread_local unsigned rhit_next; @@ -329,6 +334,7 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { out->avail = h->end - addr; out->alloc = h->alloc; out->cls = h->cls; + out->pool = h->pool; if (h->alloc) out->alloc_offset = out->offset; return 1; } @@ -342,12 +348,13 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { ResolveHit *h = &rhit[rhit_next++ & 3]; h->gen = __atomic_load_n(&pool_gen, __ATOMIC_RELAXED); // stable: changed only under the exclusive lock h->lo = e->lo, h->hi = e->hi, h->buf = r->buf, h->cls = rec_class(r); - if (r->kind == POOL_IMPORT) h->base = r->import_lo, h->end = r->import_hi, h->alloc = NULL; - else h->base = r->lo, h->end = r->lo + r->size, h->alloc = r->alloc; + if (r->kind == POOL_IMPORT) h->base = r->import_lo, h->end = r->import_hi, h->alloc = NULL, h->pool = r; + else h->base = r->lo, h->end = r->lo + r->size, h->alloc = r->alloc, h->pool = NULL; } } if (r && r->kind == POOL_IMPORT && r->buf) { out->cls = rec_class(r); + out->pool = r; out->buffer = r->buf; out->offset = addr - r->import_lo; out->avail = r->import_hi - addr; @@ -404,16 +411,20 @@ static void flag_names(uint32_t f, char *buf, size_t n) { // One line per (kind, flags) group of the live pools, then the bytes the replay bound or copied per frame by pool // class since the previous census (the first one: since frame 1200). Logged at frame 1800 (in the world from the -// run-game saves) and with TOTK_TRACE=perf every 1800 frames. +// run-game saves), at frame 600, and with TOTK_TRACE=perf every 1800 frames. +static void dirty_frame(uint64_t frame); + void vk_mem_census(uint64_t frame) { static uint64_t window_start; + if (rt_trace & RT_TRACE_PERF) dirty_frame(frame); if (frame == 1200) { // loading is over: the first window starts here for (int c = 0; c < VK_MCLS_N; c++) for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0); window_start = frame; return; } - if (frame % 1800 || !(frame == 1800 || (rt_trace & RT_TRACE_PERF))) return; + // Frame 600 as well: runs on slow machines end before 1800. + if (frame != 600 && (frame % 1800 || !(frame == 1800 || (rt_trace & RT_TRACE_PERF)))) return; typedef struct { int kind; uint32_t flags; uint64_t n, bytes; } Group; Group g[64]; int ng = 0; @@ -602,6 +613,225 @@ void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) { *untracked = __atomic_exchange_n(&n_snap_untracked, 0, __ATOMIC_RELAXED); } +// ---------------------------------------------------------------- dirty census (TOTK_TRACE=perf) +// What a device-local mirror of the buffer-bound pools would upload (build/mem-plan.md step 2). The replay notes every +// imported pool it binds as a vertex, index, uniform, storage or indirect buffer (vk_mem_bound_note). At the start of +// each submit one write-watch query per run of such pools stamps the pages written since with a new generation, in the +// shared generation table (no second consumer of the OS bits: texture checks see the stamps), and the pages of those +// pools are counted: +// - eager: pages stamped after the previous submit's scan, in every buffer-bound pool (upload at every submit); +// - lazy: pages stamped after the pool's previous count, in pools the previous submit bound (upload when bound); +// - first use: the size of a pool the first time it is bound (a mirror copies the whole pool then). +// Classes: CPU_CACHED, CPU_UNCACHED, other (CPU_NO_ACCESS). Logged as `mem: dirty` every 60 frames. +#define DIRTY_UNSYNCED UINT32_MAX +#define DIRTY_GAP 0x100000ull // pools closer than this share one query (the gap's pages are stamped too) +#define DIRTY_WIN 60 +enum { DC_CACHED, DC_UNC, DC_OTHER, DC_N }; +typedef struct { + uint64_t eager[DC_N], lazy[DC_N], first[DC_N]; + uint64_t submits, queries, scan_ns, sub_max, frames; +} DirtyAcc; +static uint32_t dirty_sub = 1, dirty_gen; // submit lock: the current submit number, generation of the last scan +static uint64_t first_pending[DC_N], ever_n[DC_N], ever_bytes[DC_N]; // submit lock +static PoolRec **dlist; // submit lock: the buffer-bound pools of a scan +static int dcap; +static SRWLOCK dirty_lock = SRWLOCK_INIT; // everything below +static DirtyAcc dfr, dwin; // this frame, this window +static uint32_t dwin_kib[DIRTY_WIN][DC_N]; // eager KiB of each frame of the window +static uint64_t live_n[DC_N], live_bytes[DC_N], live_untracked, ever_snap_n[DC_N], ever_snap_bytes[DC_N]; + +static int dc_class(const PoolRec *r) { + int c = rec_class(r); + return c == VK_MCLS_HOST_CACHED ? DC_CACHED : c == VK_MCLS_HOST_UNC ? DC_UNC : DC_OTHER; +} + +void vk_mem_bound_note(void *pool) { + PoolRec *r = pool; + if (r->bound_sub == dirty_sub) return; + r->bound_sub = dirty_sub; + if (r->bufbound) return; + r->bufbound = 1, r->synced_gen = DIRTY_UNSYNCED; + int k = dc_class(r); + ever_n[k]++, ever_bytes[k] += r->size, first_pending[k] += r->size; +} + +static uint64_t page_lo(const PoolRec *r) { return r->lo & ~0xfffull; } +static uint64_t page_hi(const PoolRec *r) { return (r->lo + r->size + 0xfff) & ~0xfffull; } + +// gen_lock held exclusively. 0: the range is not write-watched. +static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { + for (;;) { + uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages); + if (n == UINT64_MAX) return 0; + for (uint64_t k = 0; k < n; k++) { + uint32_t *s = gen_slot(gen_pages[k], 1); + if (s) *s = g; + } + if (n < sizeof gen_pages / sizeof *gen_pages) return 1; + } +} + +void vk_mem_dirty_submit(void) { + if (!(rt_trace & RT_TRACE_PERF)) return; + uint64_t t0 = kern_now_ns(); + uint32_t prev = dirty_sub; // the submit whose replay noted the bindings + dirty_sub = prev + 1 ? prev + 1 : 1; + uint64_t eager[DC_N] = {0}, lazy[DC_N] = {0}, ln[DC_N] = {0}, lb[DC_N] = {0}, untracked = 0, queries = 0; + AcquireSRWLockShared(&pool_lock); + if (dcap < pool_index.n) { + dcap = pool_index.n * 2; + dlist = realloc(dlist, (size_t)dcap * sizeof *dlist); + if (!dlist) rt_fatal("vk: out of memory for the dirty census"); + } + int n = 0; + for (int i = 0; i < pool_index.n; i++) { + PoolRec *r = pool_index.e[i].buf; + if (r->kind == POOL_IMPORT && r->bufbound) dlist[n++] = r; + } + AcquireSRWLockExclusive(&gen_lock); + uint32_t g = ++gen_now, since = dirty_gen; + dirty_gen = g; + for (int i = 0; i < n;) { // one query per run of pools (sorted by address) + uint64_t lo = page_lo(dlist[i]), hi = page_hi(dlist[i]); + int j = i + 1; + for (; j < n && page_lo(dlist[j]) <= hi + DIRTY_GAP; j++) + if (page_hi(dlist[j]) > hi) hi = page_hi(dlist[j]); + queries++; + int ok = query_stamp(lo, hi, g); + for (int k = i; k < j; k++) { + dlist[k]->untracked = 0; + if (!ok) queries++, dlist[k]->untracked = (uint8_t)!query_stamp(page_lo(dlist[k]), page_hi(dlist[k]), g); + } + i = j; + } + uint64_t covered = 0; + for (int i = 0; i < n; i++) { + PoolRec *r = dlist[i]; + int k = dc_class(r); + ln[k]++, lb[k] += r->size; + int fresh = r->synced_gen == DIRTY_UNSYNCED, lz = !fresh && r->bound_sub == prev; + uint32_t ls = r->synced_gen; + if (fresh || r->bound_sub == prev) r->synced_gen = g; + if (r->untracked) { untracked += r->size; continue; } + if (fresh) continue; // counted as first use + uint64_t lo = page_lo(r), hi = page_hi(r), ne = 0, nl = 0; + if (lo < covered) lo = covered; // overlapping pools: each page once + if (lo >= hi) continue; + covered = hi; + for (uint64_t a = lo; a < hi;) { + uint64_t ci = a >> GEN_CHUNK_SHIFT, ce = (ci + 1) << GEN_CHUNK_SHIFT; + if (ce > hi) ce = hi; + const uint32_t *ch = ci < GEN_CHUNKS ? gen_chunk[ci] : NULL; + if (ch) + for (uint64_t x = a; x < ce; x += 1ull << GEN_PAGE_SHIFT) { + uint32_t v = ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)]; + ne += v > since; + nl += lz && v > ls; + } + a = ce; + } + eager[k] += ne << GEN_PAGE_SHIFT, lazy[k] += nl << GEN_PAGE_SHIFT; + } + ReleaseSRWLockExclusive(&gen_lock); + ReleaseSRWLockShared(&pool_lock); + uint64_t tot = 0; + for (int k = 0; k < DC_N; k++) tot += eager[k]; + AcquireSRWLockExclusive(&dirty_lock); + for (int k = 0; k < DC_N; k++) { + dfr.eager[k] += eager[k], dfr.lazy[k] += lazy[k], dfr.first[k] += first_pending[k]; + first_pending[k] = 0; + live_n[k] = ln[k], live_bytes[k] = lb[k], ever_snap_n[k] = ever_n[k], ever_snap_bytes[k] = ever_bytes[k]; + } + live_untracked = untracked; + dfr.submits++, dfr.queries += queries; + if (tot > dfr.sub_max) dfr.sub_max = tot; + dfr.scan_ns += kern_now_ns() - t0; + ReleaseSRWLockExclusive(&dirty_lock); +} + +static void dirty_frame(uint64_t frame) { + AcquireSRWLockExclusive(&dirty_lock); + if (dwin.frames < DIRTY_WIN) + for (int k = 0; k < DC_N; k++) dwin_kib[dwin.frames][k] = (uint32_t)(dfr.eager[k] >> 10); + for (int k = 0; k < DC_N; k++) dwin.eager[k] += dfr.eager[k], dwin.lazy[k] += dfr.lazy[k], dwin.first[k] += dfr.first[k]; + dwin.submits += dfr.submits, dwin.queries += dfr.queries, dwin.scan_ns += dfr.scan_ns, dwin.frames++; + if (dfr.sub_max > dwin.sub_max) dwin.sub_max = dfr.sub_max; + memset(&dfr, 0, sizeof dfr); + if (frame % DIRTY_WIN) { ReleaseSRWLockExclusive(&dirty_lock); return; } + DirtyAcc w = dwin; + uint32_t kib[DIRTY_WIN][DC_N]; + memcpy(kib, dwin_kib, sizeof kib); + uint64_t lv[DC_N], lvb[DC_N], ev[DC_N], evb[DC_N], unt = live_untracked; + for (int k = 0; k < DC_N; k++) lv[k] = live_n[k], lvb[k] = live_bytes[k], ev[k] = ever_snap_n[k], evb[k] = ever_snap_bytes[k]; + memset(&dwin, 0, sizeof dwin); + ReleaseSRWLockExclusive(&dirty_lock); + if (!w.frames) return; + uint64_t fl[2][3][2]; + nvn_flush_stats(fl); + double f = (double)w.frames, M = 1048576.0; + uint64_t sub_tot = 0; + for (int k = 0; k < DC_N; k++) sub_tot += w.eager[k]; +#define C3(a, d) (a)[0] / (d), (a)[1] / (d), (a)[2] / (d) + rt_log("mem: dirty frame %llu: frames %llu submits_per_frame %.2f; per frame, cached/uncached/other: eager_mib " + "%.3f/%.3f/%.3f lazy_mib %.3f/%.3f/%.3f first_mib %.3f/%.3f/%.3f flush_pool_calls %.1f/%.1f/%.1f " + "flush_pool_mib %.3f/%.3f/%.3f flush_buf_calls %.1f/%.1f/%.1f flush_buf_mib %.3f/%.3f/%.3f; per submit: " + "avg_mib %.3f max_mib %.3f; mirror: live_pools %llu/%llu/%llu live_mib %.1f/%.1f/%.1f ever_pools %llu/%llu/%llu " + "ever_mib %.1f/%.1f/%.1f untracked_mib %.1f; cost: queries_per_frame %.1f scan_ms_per_frame %.3f", + (unsigned long long)frame, (unsigned long long)w.frames, w.submits / f, + C3(w.eager, f * M), C3(w.lazy, f * M), C3(w.first, f * M), + fl[0][0][0] / f, fl[0][1][0] / f, fl[0][2][0] / f, fl[0][0][1] / f / M, fl[0][1][1] / f / M, fl[0][2][1] / f / M, + fl[1][0][0] / f, fl[1][1][0] / f, fl[1][2][0] / f, fl[1][0][1] / f / M, fl[1][1][1] / f / M, fl[1][2][1] / f / M, + w.submits ? sub_tot / (double)w.submits / M : 0.0, w.sub_max / M, + (unsigned long long)lv[0], (unsigned long long)lv[1], (unsigned long long)lv[2], C3(lvb, M), + (unsigned long long)ev[0], (unsigned long long)ev[1], (unsigned long long)ev[2], C3(evb, M), unt / M, + w.queries / f, w.scan_ns / 1e6 / f); +#undef C3 + char line[DIRTY_WIN * 36 + 64]; + size_t k = (size_t)snprintf(line, sizeof line, "mem: dirty frames KiB cached/uncached/other:"); + for (uint64_t i = 0; i < w.frames && i < DIRTY_WIN; i++) + k += (size_t)snprintf(line + k, k < sizeof line ? sizeof line - k : 0, " %u/%u/%u", kib[i][0], kib[i][1], kib[i][2]); + rt_log("%s", line); +} + +// Device class (build/mem-plan.md): INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a +// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Only logged for now. +void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m) { + int big_local = -1, host_heap = -1; + for (uint32_t h = 0; h < m->memoryHeapCount; h++) { + if (m->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) { + if (m->memoryHeaps[h].size >= (1ull << 30) && big_local < 0) big_local = (int)h; + } else if (host_heap < 0) host_heap = (int)h; + } + static const char *const types[] = {"other", "integrated", "discrete", "virtual", "cpu"}; + int t = (int)p->deviceType; + const char *tn = t >= 0 && t <= 4 ? types[t] : "unknown"; + const char *cls, *why; + if (p->deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) cls = "uma", why = "integrated gpu"; + else if (p->deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) cls = "uma", why = "not a discrete gpu"; + else if (big_local < 0) cls = "uma", why = "no device-local heap of 1 GiB or more"; + else if (host_heap < 0) cls = "uma", why = "every heap is device-local"; + else cls = "discrete", why = "device-local heap separate from host memory"; + char line[2048]; + size_t k = (size_t)snprintf(line, sizeof line, "mem: class %s (%s; device type %d %s, %s)", cls, why, t, tn, p->deviceName); + for (uint32_t h = 0; h < m->memoryHeapCount; h++) + k += (size_t)snprintf(line + k, k < sizeof line ? sizeof line - k : 0, "; heap %u %.0f MiB flags %#x%s%s", h, + m->memoryHeaps[h].size / 1048576.0, m->memoryHeaps[h].flags, + m->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? " DEVICE_LOCAL" : "", + m->memoryHeaps[h].flags & VK_MEMORY_HEAP_MULTI_INSTANCE_BIT ? " MULTI_INSTANCE" : ""); + rt_log("%s", line); + k = (size_t)snprintf(line, sizeof line, "mem: types"); + for (uint32_t i = 0; i < m->memoryTypeCount; i++) { + VkMemoryPropertyFlags f = m->memoryTypes[i].propertyFlags; + k += (size_t)snprintf(line + k, k < sizeof line ? sizeof line - k : 0, "; %u heap %u flags %#x%s%s%s%s%s", i, + m->memoryTypes[i].heapIndex, f, f & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT ? " DEVICE_LOCAL" : "", + f & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT ? " HOST_VISIBLE" : "", + f & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT ? " HOST_COHERENT" : "", + f & VK_MEMORY_PROPERTY_HOST_CACHED_BIT ? " HOST_CACHED" : "", + f & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT ? " LAZY" : ""); + } + rt_log("%s", line); +} + // ---------------------------------------------------------------- upload batches and deferred destruction typedef struct { VkBuffer buf; VmaAllocation alloc; } Staging; typedef struct { diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 5023002f..63a082f1 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -275,6 +275,7 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_sync_start(); VkCtx *c = vk_ctx(); AcquireSRWLockExclusive(&sub_lock); + vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1; cur_fence = fence; CmdSlot *s = take_slot(c, fence); diff --git a/tools/perf_summary.py b/tools/perf_summary.py index ff12ac86..68725a7e 100644 --- a/tools/perf_summary.py +++ b/tools/perf_summary.py @@ -1,6 +1,9 @@ """Condense the TOTK_TRACE=perf and TOTK_TRACE=sched lines of a game log over a frame window. python tools/perf_summary.py build/v121/logs/fight.log 1560 2600 + +`mem: dirty` lines (vk_memory.c dirty census) are summarised per pool class (CPU_CACHED, CPU_UNCACHED, other): what +a device-local mirror of buffer-bound pools would upload per frame and per submit, FlushMappedRange use, mirror size. """ import re import statistics @@ -15,6 +18,49 @@ SCHED = re.compile(r'sched: frame (\d+): (.*)') THREAD = re.compile(r'sched: (render )?(0x[0-9a-f]+) (p\d+ m\d+) run (\d+)% core (\d+)% host (\d+)% sleep (\d+)% ' r'sync (\d+)% cv (\d+)% lock (\d+)% addr (\d+)% cpu (\d+)% blocks (\d+) preempted (\d+)') RWAIT = re.compile(r'sched: render wait (\d): (.*)') +DIRTY = re.compile(r'mem: dirty frame (\d+): (.*)') +DIRTY_FRAMES = re.compile(r'mem: dirty frames KiB [\w/]+:(.*)') +MEMCLASS = re.compile(r'mem: (class .*)') +CLASSES = ('cached', 'uncached', 'other') + + +def dirty_fields(text): + """`key a/b/c` and `key x` pairs of a `mem: dirty frame` line -> {key: [floats]}.""" + return {k: [float(x) for x in v.split('/')] for k, v in re.findall(r'(\w+) ([\d.]+(?:/[\d.]+)*)(?=[ ;]|$)', text)} + + +def p95(v): + v = sorted(v) + return v[min(len(v) - 1, int(0.95 * len(v)))] if v else 0.0 + + +def dirty_summary(lines, frames): + """lines: [(frame, fields)] of the window; frames: per-frame eager KiB triples.""" + if not lines: + return + n = sum(f['frames'][0] for _, f in lines) + + def avg(key, i): # frame-weighted mean of a per-frame value + return sum(f[key][i] * f['frames'][0] for _, f in lines if key in f) / n + + for i, c in enumerate(CLASSES): + per = [t[i] / 1024 for t in frames] + print(f'SUMMARY dirty {c}: eager MiB/frame avg {avg("eager_mib", i):.3f} p95 {p95(per):.3f} ' + f'max {max(per, default=0):.3f}; lazy avg {avg("lazy_mib", i):.3f}; first use avg {avg("first_mib", i):.3f}; ' + f'FlushMappedRange pool {avg("flush_pool_calls", i):.1f} calls {avg("flush_pool_mib", i):.3f} MiB, ' + f'buffer {avg("flush_buf_calls", i):.1f} calls {avg("flush_buf_mib", i):.3f} MiB per frame') + tot = [sum(t) / 1024 for t in frames] + subs = avg('submits_per_frame', 0) + last = lines[-1][1] + print(f'SUMMARY dirty all: eager MiB/frame avg {sum(avg("eager_mib", i) for i in range(3)):.3f} p95 {p95(tot):.3f} ' + f'max {max(tot, default=0):.3f}; submits/frame {subs:.2f}; per submit avg ' + f'{sum(f["avg_mib"][0] * f["frames"][0] for _, f in lines) / n:.3f} MiB max ' + f'{max(f["max_mib"][0] for _, f in lines):.3f} MiB') + mirror = ', '.join(f'{c} {int(last["live_pools"][i])} pools {last["live_mib"][i]:.1f} MiB (ever ' + f'{int(last["ever_pools"][i])} / {last["ever_mib"][i]:.1f} MiB)' for i, c in enumerate(CLASSES)) + print(f'SUMMARY mirror at frame {lines[-1][0]}: {mirror}; untracked {last["untracked_mib"][0]:.1f} MiB') + print(f'SUMMARY dirty census cost: {avg("queries_per_frame", 0):.1f} write-watch queries, ' + f'{avg("scan_ms_per_frame", 0):.3f} ms per frame') def main(): @@ -23,9 +69,19 @@ def main(): lo = int(sys.argv[2]) if len(sys.argv) > 2 else 1560 hi = int(sys.argv[3]) if len(sys.argv) > 3 else 10**9 fr, fps, mx, gpu, vram, ram, bound = None, [], [], [], [], [], {} + dirty, dframes, dlast = [], [], None for line in open(sys.argv[1], encoding='utf-8', errors='replace'): if line.startswith('run_game:'): print(line.rstrip()) + elif m := MEMCLASS.search(line): + print(m[1]) + elif m := DIRTY.search(line): + dlast = int(m[1]) + if lo < dlast <= hi: + dirty.append((dlast, dirty_fields(m[2]))) + elif m := DIRTY_FRAMES.search(line): + if dlast is not None and lo < dlast <= hi: + dframes += [tuple(float(x) for x in t.split('/')) for t in m[1].split()] elif m := PERF.search(line): f = int(m[1]) if lo < f <= hi: @@ -61,6 +117,7 @@ def main(): b = ', '.join(f'{k} {n}' for k, n in sorted(bound.items())) print(f'SUMMARY gpu: {g}, {v}, ram ws max {max(w for w, _ in ram):.0f} MiB, ' f'private max {max(p for _, p in ram):.0f} MiB; bound windows: {b}') + dirty_summary(dirty, dframes) if __name__ == '__main__':