diff --git a/docs/agent-playbook.md b/docs/agent-playbook.md index 5caf902e..9781fa2a 100644 --- a/docs/agent-playbook.md +++ b/docs/agent-playbook.md @@ -25,6 +25,12 @@ other `totk.exe` or `ninja.exe` runs. The log's first and last lines record the - `--env TOTK_TRACE=perf,sched`, then `python tools/perf_summary.py `: fps, max frame and the render-thread buckets per 60 frames. Per-thread run, core, host, lock and addr shares come per 120 presents. + A second `perf: frame N: gpu avg` line per window has the GPU busy time per frame (timestamp queries per submission + part, `vk_sync.c`; upload batches not included), fence waits of the render thread and of other threads, VRAM use and + budget (`VK_EXT_memory_budget`), process RAM and `bound: gpu|cpu|pace` (pace: the frame waited for pacing 15% of the + time or more; gpu: the GPU was busy 80% of the interval or more). +- The pool census (`mem: census`, `vk_memory.c` `vk_mem_census`) is logged at frame 1800, with `perf` every 1800 + frames: pools by kind and flags, and the bytes bound or copied per frame by pool class since frame 1200. - `TOTK_PROFILE=1 TOTK_PROFILE_FRAMES=lo-hi`: sampling profiler (`runtime/prof_win.c`). Resolve `exe+0x...` RVAs with the build's `totk.map`. - `tools/regress.py --base build/v121-m4 [--routes fight,rock,island,field] [--repeat 3] [--cold] [--validation] diff --git a/runtime/host.h b/runtime/host.h index bd7cff84..57a1f7c2 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -41,6 +41,15 @@ enum { PERF_CNT_PIPE_ASYNC, PERF_CNT_PIPE_SKIP, PERF_CNT_TEX_UP, PERF_CNT_TEX_RE void perf_count(int kind); void perf_present(uint64_t frame); void perf_summary_flush(void); // prints the frame-time summary of the running period (exit); the 60 s lines come from perf_present +void perf_gpu_wait(uint64_t t0); // a wait for a GPU fence ended (any thread; the render thread's also counts as PERF_FENCE) +// GPU side of the 60-frame perf line, filled by the graphics back end when it set perf_gpu_stats. gpu_ns: GPU busy time +// since the previous call (timestamp queries, overlaps counted once), have_gpu 0 when the device has no timestamps. +typedef struct { + int have_gpu, have_vram, vram_exact; + uint64_t gpu_ns, gpu_parts, gpu_missed; // busy time, submission parts timed, parts left untimed + uint64_t vram_used, vram_budget; +} PerfGpu; +extern void (*perf_gpu_stats)(PerfGpu *out); #define RT_RESULT(module, desc) ((uint32_t)(module) | ((uint32_t)(desc) << 9)) void rt_sha256(const void *p, size_t n, uint8_t out[32]); // BCrypt; fatal if the provider fails diff --git a/runtime/prof_win.c b/runtime/prof_win.c index a4ddd154..fea5af3a 100644 --- a/runtime/prof_win.c +++ b/runtime/prof_win.c @@ -187,6 +187,16 @@ void perf_end(int kind, uint64_t t0) { frame_ns[kind] += d; } +static volatile LONG64 other_wait_ns; // perf_gpu_wait on threads other than the render thread, over the window +static volatile LONG64 render_wait_ns; // the same on the render thread +void (*perf_gpu_stats)(PerfGpu *out); + +void perf_gpu_wait(uint64_t t0) { + if (!t0 || !perf_on) return; + LONG64 d = (LONG64)(kern_now_ns() - t0); + InterlockedAdd64(GetCurrentThreadId() == render_tid ? &render_wait_ns : &other_wait_ns, d); +} + static volatile LONG64 replay_ns; // perf_replay: summed over the window, any thread static volatile LONG64 frame_replay_ns; // the same since the previous present (frame-time summary) @@ -273,6 +283,35 @@ static void ft_frame(uint64_t frame, uint64_t now, uint64_t d) { } } +// The second line of the 60-frame window: GPU busy time (timestamps), the time threads waited for GPU fences, VRAM and +// process memory, and which side holds the frame back. pace: the frame waited for pacing (vsync, the frame limiter) at +// least 15% of the window; gpu: the GPU was busy for 80% of the frame interval or more (without timestamps: threads +// waited for fences for half the interval or more); cpu otherwise. +static void perf_gpu_line(uint64_t frame, double interval_ms, double pace_share, uint64_t n) { + PerfGpu g; + memset(&g, 0, sizeof g); + if (perf_gpu_stats) perf_gpu_stats(&g); + double rw = InterlockedExchange64(&render_wait_ns, 0) / 1e6 / n, ow = InterlockedExchange64(&other_wait_ns, 0) / 1e6 / n; + double gpu_ms = g.have_gpu ? g.gpu_ns / 1e6 / n : 0; + char gs[96], vs[96]; + if (g.have_gpu) snprintf(gs, sizeof gs, "gpu avg %.2f ms (%llu parts timed, %llu not)", gpu_ms, + (unsigned long long)g.gpu_parts, (unsigned long long)g.gpu_missed); + else snprintf(gs, sizeof gs, "gpu avg n/a"); + if (g.have_vram) snprintf(vs, sizeof vs, "vram %.0f of %.0f MiB%s", g.vram_used / 1048576.0, g.vram_budget / 1048576.0, + g.vram_exact ? "" : " (own allocations of the heap size)"); + else snprintf(vs, sizeof vs, "vram n/a"); + PROCESS_MEMORY_COUNTERS_EX pm = {.cb = sizeof pm}; + double ws = 0, priv = 0; + if (K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS *)&pm, sizeof pm)) + ws = pm.WorkingSetSize / 1048576.0, priv = pm.PrivateUsage / 1048576.0; + const char *bound = pace_share >= 0.15 ? "pace" + : g.have_gpu ? (gpu_ms >= 0.8 * interval_ms ? "gpu" : "cpu") + : (rw + ow >= 0.5 * interval_ms ? "gpu" : "cpu"); + rt_log("perf: frame %llu: %s; fence wait per frame: render thread %.2f ms, other threads %.2f ms; %s; ram working set " + "%.0f MiB, private %.0f MiB; bound: %s", + (unsigned long long)frame, gs, rw, ow, vs, ws, priv, bound); +} + void perf_present(uint64_t frame) { static uint64_t last, sum, max, n, win_start; static LONG64 win_prev[PERF_CNT_N]; @@ -329,6 +368,7 @@ void perf_present(uint64_t frame) { (unsigned long long)shader_keys, (unsigned long long)shader_xlated, (span > wait_ns[PERF_PACE] ? span - wait_ns[PERF_PACE] : 0) / 1e6 / n, InterlockedExchange64(&replay_ns, 0) / 1e6 / n); + if (n && span) perf_gpu_line(frame, sum / 1e6 / n, (double)wait_ns[PERF_PACE] / span, n); sum = max = n = 0; memset(wait_ns, 0, sizeof wait_ns); win_start = now; diff --git a/runtime/vk.h b/runtime/vk.h index b1101d12..a1da6708 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -114,12 +114,28 @@ void vk_guest_written(uint64_t addr, uint64_t size); // a host write write-wat // GPU address -> backing (section 3.1). avail: bytes from offset that stay contiguous in buffer. alloc/alloc_offset: // the device-local memory behind physical and (through the page table) virtual pool addresses, NULL for imported pools. +// cls: the pool class of the backing (VK_MCLS_*), for the traffic census. typedef struct { VkBuffer buffer; VkDeviceSize offset, avail; VmaAllocation alloc; VkDeviceSize alloc_offset; + int cls; } VkMemSpan; +// Traffic census (vk_memory.c vk_mem_census): bytes the replay binds or copies, by pool class and use. Imported pools by +// their CPU access flag; DEVICE is physical memory (directly or through a virtual pool); SCRATCH the replay's transient +// copies. CPU_READ counts bytes host code reads from pool storage in place of the GPU (misaligned binds, byte indices, +// counter words); GPU_WRITE what copies, fills, counter reports and uniform updates write. +enum { VK_MCLS_NONE, VK_MCLS_HOST_NOACC, VK_MCLS_HOST_UNC, VK_MCLS_HOST_CACHED, VK_MCLS_DEVICE, VK_MCLS_SCRATCH, VK_MCLS_N }; +enum { VK_MUSE_VB, VK_MUSE_IB, VK_MUSE_UBO, VK_MUSE_SSBO, VK_MUSE_IND, VK_MUSE_COPY_SRC, VK_MUSE_GPU_WRITE, + VK_MUSE_CPU_READ, VK_MUSE_N }; +void vk_mem_count(int cls, int use, uint64_t bytes); +int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked) +void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and, + // with TOTK_TRACE=perf, every 1800 frames +// VRAM in use and its budget, summed over the DEVICE_LOCAL heaps. exact 1: VK_EXT_memory_budget (the whole process), +// 0: this allocator's own allocations against the heap sizes. 0 when unknown. +int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact); int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed // Page write generations over guest memory (mem_written_pages plus vk_guest_written). Returns 1 when a page of the // range was written after generation `since`, 0 when not, -1 when the range is not write-watched (treat as written); diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index c11181ef..8193f0d6 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -90,6 +90,7 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads vk_exec_frame_log(t, frame); + vk_mem_census(frame); if (vk_dump_armed) vk_dump_present(frame); __atomic_store_n(&gfx_counters.validation_errors, (uint64_t)vk_validation_errors_total(), __ATOMIC_RELAXED); const VkTexImage *img = vk_texture_get(t); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 1144f2af..38b7c1fe 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -571,7 +571,7 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) { // A buffer descriptor for [addr, addr+size) of guest memory: its backing when aligned, a copy in transient memory when // the backing is host memory with an unaligned offset. 0 when neither works. -static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, VkDescriptorBufferInfo *out) { +static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, VkDescriptorBufferInfo *out) { VkMemSpan sp; if (!addr || !vk_mem_resolve(addr, 1, &sp)) return 0; uint64_t n = size && size < sp.avail ? size : sp.avail; @@ -579,12 +579,15 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, if (!n) return 0; if (sp.offset % align == 0) { *out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n}; + vk_mem_count(sp.cls, use, n); return 1; } if (sp.alloc) return 0; VkBuffer b; VkDeviceSize o; memcpy(scratch(ex, n, &b, &o), (const void *)(uintptr_t)addr, n); + vk_mem_count(sp.cls, VK_MUSE_CPU_READ, n); + vk_mem_count(VK_MCLS_SCRATCH, use, n); *out = (VkDescriptorBufferInfo){b, o, n}; return 1; } @@ -661,11 +664,11 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { uint32_t len, c1; if (find_stage(ex->prog, st, &ctl, &code, &len)) { memcpy(&c1, ctl + 0x700, 4); - if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, &bi)) return bi; + if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, &bi)) return bi; } } else if (slot >= 3 && slot - 3 < 18) { const p_bind_buf *b = &ex->ubo[st][slot - 3]; - if (bind_range(ex, b->addr, b->size, align, max, &bi)) return bi; + if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, &bi)) return bi; } return zero_info(max); } @@ -695,7 +698,7 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * const p_bind_buf *b = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot]; // vk_caps.c: shaders translated for sbuf_align address the buffer from its address rounded down to it. uint64_t lead = vk_caps.sbuf_align ? b->addr & (vk_caps.sbuf_align - 1) : 0; - if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, &bi)) { + if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO, &bi)) { if (written && bi.buffer != VK_NULL_HANDLE) note_written(ex, b->addr - lead, bi.range); return bi; } @@ -1082,6 +1085,7 @@ static void record_fixed(VkExec *ex) { static void bind_vertex_buffers(VkExec *ex) { VkBuffer vb[17]; VkDeviceSize vo[17], vsz[17], vstr[17]; + int vcls[17] = {0}; int dyn = vk_pipeline_vertex_input_dynamic(); for (int b = 0; b < 17; b++) { vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0; @@ -1090,13 +1094,15 @@ static void bind_vertex_buffers(VkExec *ex) { VkMemSpan sp; if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue; uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail; - vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n; + vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls; } if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) && (dyn || !memcmp(B.vstr, vstr, sizeof vstr))) return; vkCmdBindVertexBuffers2(ex->cb, 0, 17, vb, vo, vsz, dyn ? NULL : vstr); ex->recorded++; + for (int b = 0; b < 17; b++) + if (vcls[b]) vk_mem_count(vcls[b], VK_MUSE_VB, vsz[b]); memcpy(B.vb, vb, sizeof vb), memcpy(B.vo, vo, sizeof vo), memcpy(B.vsz, vsz, sizeof vsz), memcpy(B.vstr, vstr, sizeof vstr); B.vb_valid = 1; } @@ -1247,6 +1253,8 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { if (!vk_mem_resolve(d->ibuf, bytes, &sp)) return warn_once(W_INDEX, "index buffer %#llx is not backed: draws with it are skipped", (unsigned long long)d->ibuf); if (d->itype == NVN_INDEX_TYPE_UNSIGNED_BYTE) { // no uint8 index type here: widen (host-readable backing only) if (sp.alloc) return warn_once(W_INDEX, "byte indices in device-local memory: draw skipped"); + vk_mem_count(sp.cls, VK_MUSE_CPU_READ, bytes); + vk_mem_count(VK_MCLS_SCRATCH, VK_MUSE_IB, 2 * bytes); uint16_t *ix = malloc(sizeof *ix * (size_t)d->count); if (!ix) rt_fatal("vk: out of memory"); for (int i = 0; i < d->count; i++) { @@ -1260,6 +1268,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { } else { vkCmdBindIndexBuffer(ex->cb, sp.buffer, sp.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); ex->recorded++; + vk_mem_count(sp.cls, VK_MUSE_IB, bytes); } vkCmdDrawIndexed(ex->cb, (uint32_t)d->count, d->instances > 0 ? (uint32_t)d->instances : 1, 0, d->base_vertex, (uint32_t)d->base_instance); @@ -1283,6 +1292,7 @@ static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) { vkCmdBindIndexBuffer(ex->cb, ib.buffer, ib.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); vkCmdDrawIndexedIndirect(ex->cb, ip.buffer, ip.offset, 1, sizeof(VkDrawIndexedIndirectCommand)); ex->recorded += 2; + vk_mem_count(ip.cls, VK_MUSE_IND, sizeof(VkDrawIndexedIndirectCommand)); } // ---- dispatch and uniform buffer updates @@ -1336,6 +1346,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { } vkCmdDispatchIndirect(ex->cb, ip.buffer, ip.offset); ex->recorded++; + vk_mem_count(ip.cls, VK_MUSE_IND, sizeof(VkDispatchIndirectCommand)); if (vk_dump_armed) vk_dump_dispatch(ex, cs, p, info, &(p_dispatch){0, 0, 0, d->caller}); } @@ -1351,6 +1362,7 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { VkBufferCopy cp = {o, sp.offset, u->size}; vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp); ex->recorded++; + vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size); note_written(ex, u->dst, u->size); } diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index bba80837..f33ecf60 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -275,6 +275,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { VkMemSpan m; if (!vk_mem_resolve(p->addr, sizeof(NVNcounterData), &m)) rt_fatal("vk: counter report at %#llx is in no memory pool", (unsigned long long)p->addr); + vk_mem_count(m.cls, VK_MUSE_GPU_WRITE, sizeof(NVNcounterData)); if (samples) oq_report(ex, p->addr); else { uint64_t *a = malloc(sizeof *a); @@ -297,6 +298,7 @@ static int counter_word(uint64_t addr, uint64_t *v) { VkMemSpan m; if (nvn_gpu_only(addr) || !vk_mem_resolve(addr, sizeof *v, &m)) return 0; memcpy(v, (const void *)(uintptr_t)addr, sizeof *v); + vk_mem_count(m.cls, VK_MUSE_CPU_READ, sizeof *v); return 1; } @@ -379,7 +381,7 @@ static VkExtent3D sub_extent(const Sub *s) { // Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span. static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *size, VkBufferImageCopy *c, - VkBuffer *buf) { + VkBuffer *buf, int *cls) { uint32_t wb = (s->w + (uint32_t)s->bw - 1) / (uint32_t)s->bw, hb = (s->h + (uint32_t)s->bh - 1) / (uint32_t)s->bh; uint64_t row = ex->copy_stride > 0 ? (uint64_t)ex->copy_stride : (uint64_t)wb * (uint64_t)s->bpb; if (row % (uint64_t)s->bpb || row < (uint64_t)wb * (uint64_t)s->bpb) { @@ -396,6 +398,7 @@ static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t * return 0; } *buf = m.buffer; + *cls = m.cls; *c = (VkBufferImageCopy){.bufferOffset = m.offset, .bufferRowLength = (uint32_t)(row / (uint64_t)s->bpb) * (uint32_t)s->bw, .bufferImageHeight = hb * (uint32_t)s->bh, .imageSubresource = sub_layers(s), .imageOffset = sub_offset(s), .imageExtent = sub_extent(s)}; @@ -408,7 +411,9 @@ static void copy_buffer_image(VkExec *ex, const p_copy_b2t *p, int to_image) { uint64_t size; VkBufferImageCopy c; VkBuffer buf; - if (!buffer_side(ex, &s, p->src, &size, &c, &buf)) return; + int cls; + if (!buffer_side(ex, &s, p->src, &size, &c, &buf, &cls)) return; + vk_mem_count(cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size); vk_exec_pass_end(ex); vk_exec_barrier(ex); if (to_image) vkCmdCopyBufferToImage(ex->cb, buf, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c); @@ -480,6 +485,8 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) { VkBufferCopy bc = {s.offset, d.offset, n}; vkCmdCopyBuffer(ex->cb, s.buffer, d.buffer, 1, &bc); ex->recorded++; + vk_mem_count(s.cls, VK_MUSE_COPY_SRC, n); + vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n); src += n, dst += n, left -= n; } if (started) note_gpu_write(ex, p->dst, p->size - left); @@ -509,6 +516,7 @@ static void clear_buf(VkExec *ex, const p_clear_buf *p) { if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1; vkCmdFillBuffer(ex->cb, d.buffer, d.offset, n, p->value); ex->recorded++; + vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n); dst += n, left -= n; } if (started) note_gpu_write(ex, p->addr, p->size - left); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index d1845d12..328b04b5 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -307,7 +307,13 @@ int vk_pool_map_virtual(pool_t *v, int n, const void *requests) { // Per-thread copies of resolved import and physical pools that overlap no other index entry (so pidx_find picks them for // any range inside), valid while pool_gen is unchanged: the replay resolves a few pools thousands of times per frame. -typedef struct { uint64_t gen, lo, hi, base, end; VkBuffer buf; VmaAllocation alloc; } ResolveHit; +static int rec_class(const PoolRec *r) { + if (r->kind != POOL_IMPORT) return VK_MCLS_DEVICE; + return r->flags & NVN_MEMORY_POOL_FLAGS_CPU_NO_ACCESS ? VK_MCLS_HOST_NOACC + : r->flags & NVN_MEMORY_POOL_FLAGS_CPU_CACHED ? VK_MCLS_HOST_CACHED : VK_MCLS_HOST_UNC; +} + +typedef struct { uint64_t gen, lo, hi, base, end; VkBuffer buf; VmaAllocation alloc; int cls; } ResolveHit; static _Thread_local ResolveHit rhit[4]; static _Thread_local unsigned rhit_next; @@ -322,6 +328,7 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { out->offset = addr - h->base; out->avail = h->end - addr; out->alloc = h->alloc; + out->cls = h->cls; if (h->alloc) out->alloc_offset = out->offset; return 1; } @@ -334,12 +341,13 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { if ((i == 0 || pool_index.maxhi[i - 1] <= e->lo) && (i + 1 == pool_index.n || pool_index.e[i + 1].lo >= e->hi)) { ResolveHit *h = &rhit[rhit_next++ & 3]; h->gen = __atomic_load_n(&pool_gen, __ATOMIC_RELAXED); // stable: changed only under the exclusive lock - h->lo = e->lo, h->hi = e->hi, h->buf = r->buf; + h->lo = e->lo, h->hi = e->hi, h->buf = r->buf, h->cls = rec_class(r); if (r->kind == POOL_IMPORT) h->base = r->import_lo, h->end = r->import_hi, h->alloc = NULL; else h->base = r->lo, h->end = r->lo + r->size, h->alloc = r->alloc; } } if (r && r->kind == POOL_IMPORT && r->buf) { + out->cls = rec_class(r); out->buffer = r->buf; out->offset = addr - r->import_lo; out->avail = r->import_hi - addr; @@ -349,6 +357,7 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { out->offset = out->alloc_offset = addr - r->lo; out->avail = r->size - out->offset; out->alloc = r->alloc; + out->cls = VK_MCLS_DEVICE; ok = 1; } else if (r && r->kind == POOL_VIRT) { uint64_t rel = addr - r->lo, k = rel / NVN_VPOOL_PAGE; @@ -360,6 +369,7 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { out->offset = out->alloc_offset = off; out->avail = end - off; out->alloc = ph->alloc; + out->cls = VK_MCLS_DEVICE; ok = out->avail >= size; if (ok && !ph->used) InterlockedExchange(&ph->used, 1); // read first: the flag is only ever set } @@ -370,6 +380,101 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { return ok; } +int vk_mem_class_of(uint64_t addr) { + VkMemSpan m; + return vk_mem_resolve(addr, 1, &m) ? m.cls : VK_MCLS_NONE; +} + +// ---------------------------------------------------------------- census +static volatile LONG64 traffic[VK_MCLS_N][VK_MUSE_N]; + +void vk_mem_count(int cls, int use, uint64_t bytes) { + if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N) InterlockedAdd64(&traffic[cls][use], (LONG64)bytes); +} + +static void flag_names(uint32_t f, char *buf, size_t n) { + static const char *const names[] = {"CPU_NO_ACCESS", "CPU_UNCACHED", "CPU_CACHED", "GPU_NO_ACCESS", "GPU_UNCACHED", + "GPU_CACHED", "SHADER_CODE", "COMPRESSIBLE", "PHYSICAL", "VIRTUAL"}; + size_t k = 0; + buf[0] = 0; + for (int i = 0; i < 10; i++) + if (f >> i & 1) k += (size_t)snprintf(buf + k, k < n ? n - k : 0, "%s%s", k ? "|" : "", names[i]); + if (!k) snprintf(buf, n, "none"); +} + +// One line per (kind, flags) group of the live pools, then the bytes the replay bound or copied per frame by pool +// class since the previous census (the first one: since frame 1200). Logged at frame 1800 (in the world from the +// run-game saves) and with TOTK_TRACE=perf every 1800 frames. +void vk_mem_census(uint64_t frame) { + static uint64_t window_start; + if (frame == 1200) { // loading is over: the first window starts here + for (int c = 0; c < VK_MCLS_N; c++) + for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0); + window_start = frame; + return; + } + if (frame % 1800 || !(frame == 1800 || (rt_trace & RT_TRACE_PERF))) return; + typedef struct { int kind; uint32_t flags; uint64_t n, bytes; } Group; + Group g[64]; + int ng = 0; + uint64_t total[4] = {0}, count[4] = {0}; + AcquireSRWLockShared(&pool_lock); + for (int i = 0; i < pool_index.n; i++) { + const PoolRec *r = pool_index.e[i].buf; + int j = 0; + while (j < ng && (g[j].kind != r->kind || g[j].flags != r->flags)) j++; + if (j == ng && ng < 64) g[ng++] = (Group){r->kind, r->flags, 0, 0}; + if (j < ng) g[j].n++, g[j].bytes += r->size; + total[r->kind] += r->size, count[r->kind]++; + } + ReleaseSRWLockShared(&pool_lock); + rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%ld without a GPU buffer), physical %llu pools %.1f MiB, " + "virtual %llu pools %.1f MiB of address space", + (unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0, + n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0, + (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0); + static const char *const kinds[] = {"", "import", "physical", "virtual"}; + for (int j = 0; j < ng; j++) { + char f[160]; + flag_names(g[j].flags, f, sizeof f); + rt_log("mem: %s %#x %s: %llu pools, %.1f MiB", kinds[g[j].kind], g[j].flags, f, (unsigned long long)g[j].n, + g[j].bytes / 1048576.0); + } + uint64_t frames = window_start && frame > window_start ? frame - window_start : frame; + static const char *const cls[] = {"", "host CPU_NO_ACCESS", "host CPU_UNCACHED", "host CPU_CACHED", "device", "scratch"}; + rt_log("mem: traffic per frame over %llu frames, MiB (bound ranges, not bytes read): class: vertex index uniform " + "storage indirect copy-src gpu-write cpu-read", (unsigned long long)frames); + for (int c = 1; c < VK_MCLS_N; c++) { + double v[VK_MUSE_N]; + for (int u = 0; u < VK_MUSE_N; u++) v[u] = InterlockedExchange64(&traffic[c][u], 0) / 1048576.0 / (double)(frames ? frames : 1); + rt_log("mem: %s: %.2f %.2f %.2f %.2f %.3f %.2f %.2f %.3f", cls[c], v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7]); + } + window_start = frame; +} + +int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact) { + VkCtx *c = vk_ctx(); + *used = *budget = 0, *exact = 0; + if (!c) return 0; + const VkPhysicalDeviceMemoryProperties *mp = &c->mem_props; + if (c->has_memory_budget) { + VkPhysicalDeviceMemoryBudgetPropertiesEXT b = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &b}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + for (uint32_t h = 0; h < p2.memoryProperties.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (p2.memoryProperties.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + *used += b.heapUsage[h], *budget += b.heapBudget[h]; + *exact = 1; + return *budget != 0; + } + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + for (uint32_t h = 0; h < mp->memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (mp->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + *used += vb[h].statistics.blockBytes, *budget += mp->memoryHeaps[h].size; + return *budget != 0; +} + // ---------------------------------------------------------------- page write generations // Two-level table over the guest address space (below 2^39): 256 MiB chunks of 65,536 page generations, allocated // when a page of the chunk is first stamped. 0 = never stamped. diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index a2809e49..5023002f 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -46,6 +46,106 @@ static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; +// ---- GPU time (TOTK_TRACE=perf): each submission part (vk_sync_split) writes a timestamp when it starts and when it +// ends, into a ring of query pairs. The completion thread reads a pair once the part's value is reached and adds the +// part's interval to the busy time, counting overlap with the previous part once. Off (one log line) when the queue +// family has no timestamp bits or the period is 0. Upload batches (vk_memory.c) are not timed. +#define TS_PAIRS 1024 +static VkQueryPool ts_pool; +static int ts_state; // 0 not set up, 1 on, -1 off +static double ts_period; // ns per tick +static uint64_t ts_mask; // valid bits of a timestamp +static uint32_t ts_next; // submitting thread (sub_lock): next pair +static volatile uint8_t ts_busy[TS_PAIRS]; // written until the completion thread read the pair +static int ts_cur = -1; // the pair of the part being recorded, -1 untimed +static uint64_t ts_last_end; // completion thread: end of the latest part read (ticks) +static volatile LONG64 ts_busy_ns, ts_parts, ts_missed; + +static void ts_setup(VkCtx *c) { + ts_state = -1; + if (!(rt_trace & RT_TRACE_PERF)) return; + uint32_t nq = 0; + vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL); + VkQueueFamilyProperties *qp = calloc(nq ? nq : 1, sizeof *qp); + if (!qp) return; + vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp); + uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0; + free(qp); + float period = c->props.limits.timestampPeriod; + if (!bits || period <= 0) { + rt_log("vk: no GPU timestamps on this queue (valid bits %u, period %.3f ns): the perf line shows no GPU time", bits, period); + return; + } + VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = 2 * TS_PAIRS}; + VkResult r = vkCreateQueryPool(c->device, &qi, NULL, &ts_pool); + if (r != VK_SUCCESS) { + rt_log("vk: timestamp query pool failed (%d): the perf line shows no GPU time", (int)r); + return; + } + ts_period = period; + ts_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1; + ts_state = 1; +} + +typedef struct { uint32_t pair; } TsRead; + +static void ts_read_cb(void *arg) { // completion thread, in submission order + TsRead *t = arg; + VkCtx *c = vk_ctx(); + uint64_t v[2]; + if (vkGetQueryPoolResults(c->device, ts_pool, 2 * t->pair, 2, sizeof v, v, sizeof v[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) { + uint64_t s = v[0] & ts_mask, e = v[1] & ts_mask; + if (e >= s) { + uint64_t from = s > ts_last_end ? s : ts_last_end; + if (e > from) InterlockedAdd64(&ts_busy_ns, (LONG64)((double)(e - from) * ts_period)); + if (e > ts_last_end) ts_last_end = e; + InterlockedIncrement64(&ts_parts); + } else InterlockedIncrement64(&ts_missed); // the counter wrapped + } else InterlockedIncrement64(&ts_missed); + __atomic_store_n(&ts_busy[t->pair], 0, __ATOMIC_RELEASE); + free(t); +} + +// At the start of a part's command buffer (sub_lock held). +static void ts_begin(VkCommandBuffer cb) { + ts_cur = -1; + if (ts_state != 1) return; + uint32_t p = ts_next % TS_PAIRS; + if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return; } + ts_next++; + ts_busy[p] = 1; + vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2); + vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p); + ts_cur = (int)p; +} + +// At the end of the part, before vkEndCommandBuffer; fence: the value whose completion covers the part. +static void ts_end(VkCommandBuffer cb, uint64_t fence) { + if (ts_cur < 0) return; + vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)ts_cur + 1); + TsRead *t = malloc(sizeof *t); + if (!t) rt_fatal("vk: out of memory"); + t->pair = (uint32_t)ts_cur; + vk_completion_add(fence, ts_read_cb, t); + ts_cur = -1; +} + +// An empty submission records no command buffer: its pair is never written. +static void ts_drop(void) { + if (ts_cur < 0) return; + __atomic_store_n(&ts_busy[ts_cur], 0, __ATOMIC_RELEASE); + ts_cur = -1; +} + +static void gpu_stats(PerfGpu *g) { + g->have_gpu = ts_state == 1; + g->gpu_ns = (uint64_t)InterlockedExchange64(&ts_busy_ns, 0); + g->gpu_parts = (uint64_t)InterlockedExchange64(&ts_parts, 0); + g->gpu_missed = (uint64_t)InterlockedExchange64(&ts_missed, 0); + g->have_vram = vk_mem_vram(&g->vram_used, &g->vram_budget, &g->vram_exact); +} + // Without the replay (T6c) only empty recordings ({OP_BEGIN, OP_END}) can be submitted. WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { (void)cb, (void)fence; @@ -105,6 +205,8 @@ void vk_sync_start(void) { .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = c->gfx_family}; if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: submission command pool"); + if (!ts_state) ts_setup(c); + perf_gpu_stats = gpu_stats; uint64_t now = 0; vkGetSemaphoreCounterValue(c->device, c->timeline, &now); if (issued > now) { // a new device after vk_shutdown: values keep increasing for the front end @@ -151,6 +253,7 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { VkCommandBuffer vk_sync_split(void) { VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; + ts_end(cb, cur_fence); vkEndCommandBuffer(cb); vk_upload_flush(0); // the uploads this part asked for precede it in queue order VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}; @@ -164,6 +267,7 @@ VkCommandBuffer vk_sync_split(void) { VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(s->cb, &bi); + ts_begin(s->cb); return s->cb; } @@ -178,8 +282,11 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; uint64_t t0 = perf_begin(); vkBeginCommandBuffer(s->cb, &bi); + ts_begin(s->cb); int recorded = vk_exec_record(s->cb, handles, n, fence); s = &slots[cur_slot]; // the last part (vk_sync_split) + if (recorded) ts_end(s->cb, fence); + else ts_drop(); vkEndCommandBuffer(s->cb); perf_replay(t0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. @@ -227,6 +334,7 @@ int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) { uint64_t p0 = perf_begin(); int ok = fence_wait_impl(value, timeout_ns); perf_end(PERF_FENCE, p0); + perf_gpu_wait(p0); return ok; } @@ -279,6 +387,12 @@ void vk_sync_shutdown(void) { AcquireSRWLockExclusive(&sub_lock); VkCtx *c = vk_ctx(); if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL); + perf_gpu_stats = NULL; + if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL); + ts_pool = VK_NULL_HANDLE; + ts_state = 0; + ts_last_end = 0; + memset((void *)ts_busy, 0, sizeof ts_busy); cmd_pool = VK_NULL_HANDLE; free(slots); slots = NULL; diff --git a/tools/perf_summary.py b/tools/perf_summary.py index fb344e1e..ff12ac86 100644 --- a/tools/perf_summary.py +++ b/tools/perf_summary.py @@ -8,6 +8,9 @@ import sys PERF = re.compile(r'perf: frame (\d+): interval avg ([\d.]+) ms max ([\d.]+) ms \(([\d.]+) fps\); render thread blocked: ' r'pacing (\d+)%.*?pipeline build (\d+)%.*?texture (\d+)%') +GPU = re.compile(r'perf: frame (\d+): gpu avg (?:([\d.]+) ms|n/a).*?fence wait per frame: render thread ([\d.]+) ms, ' + r'other threads ([\d.]+) ms; vram (?:([\d.]+) of ([\d.]+) MiB|n/a).*?ram working set ([\d.]+) MiB, ' + r'private ([\d.]+) MiB; bound: (\w+)') SCHED = re.compile(r'sched: frame (\d+): (.*)') THREAD = re.compile(r'sched: (render )?(0x[0-9a-f]+) (p\d+ m\d+) run (\d+)% core (\d+)% host (\d+)% sleep (\d+)% ' r'sync (\d+)% cv (\d+)% lock (\d+)% addr (\d+)% cpu (\d+)% blocks (\d+) preempted (\d+)') @@ -19,7 +22,7 @@ def main(): sys.exit(__doc__) lo = int(sys.argv[2]) if len(sys.argv) > 2 else 1560 hi = int(sys.argv[3]) if len(sys.argv) > 3 else 10**9 - fr, fps, mx = None, [], [] + fr, fps, mx, gpu, vram, ram, bound = None, [], [], [], [], [], {} for line in open(sys.argv[1], encoding='utf-8', errors='replace'): if line.startswith('run_game:'): print(line.rstrip()) @@ -29,6 +32,17 @@ def main(): fps.append(float(m[4])) mx.append(float(m[3])) print(f'perf {f}: {m[4]} fps max {m[3]} pacing {m[5]}% pipe {m[6]}% tex {m[7]}%') + elif m := GPU.search(line): + f = int(m[1]) + if lo < f <= hi: + if m[2]: + gpu.append(float(m[2])) + if m[5]: + vram.append((float(m[5]), float(m[6]))) + ram.append((float(m[7]), float(m[8]))) + bound[m[9]] = bound.get(m[9], 0) + 1 + print(f' gpu {f}: {m[2] or "n/a"} ms, fence wait render {m[3]} other {m[4]} ms, ' + f'vram {m[5] or "n/a"} MiB, ram ws {m[7]} private {m[8]} MiB, bound {m[9]}') elif m := SCHED.search(line): fr = int(m[1]) if lo < fr <= hi: @@ -41,6 +55,12 @@ def main(): if fps: print(f'SUMMARY frames {lo}-{hi}: mean fps {statistics.mean(fps):.1f}, min window fps {min(fps):.1f}, ' f'max frame {max(mx):.1f} ms') + if ram: + g = f'gpu mean {statistics.mean(gpu):.2f} ms' if gpu else 'gpu n/a' + v = f'vram max {max(u for u, _ in vram):.0f} of {vram[-1][1]:.0f} MiB' if vram else 'vram n/a' + b = ', '.join(f'{k} {n}' for k, n in sorted(bound.items())) + print(f'SUMMARY gpu: {g}, {v}, ram ws max {max(w for w, _ in ram):.0f} MiB, ' + f'private max {max(p for _, p in ram):.0f} MiB; bound windows: {b}') if __name__ == '__main__':