diff --git a/docs/backlog.md b/docs/backlog.md index 1724ca56..32ea98b3 100644 --- a/docs/backlog.md +++ b/docs/backlog.md @@ -89,9 +89,24 @@ machine A runs scripted routes; B and C send logs from the user's play. 20. **GPU memory over long sessions** (port-lessons P3), never measured. MacNativeGames' TotK reached its 5000 MB GPU budget after 2 h 43 min (their issue #72) and grew from 4.5 to 6.25 GiB resident; their dead-serial hash sets reached about 1 GiB after 3 h. Our textures are destroyed when their pool range goes or a new texture takes the - address (`vk_texture.c` `entry_of`, `vk_texture_range_gone`), pools at Finalize, Vulkan objects deferred. Next: a - memory line every 1800 frames (VMA heaps, textures, pools, pipelines), a list of growing containers, then a 2-3 h - session. 6-8 GB cards are the risk. + address (`vk_texture.c` `entry_of`, `vk_texture_range_gone`), pools at Finalize, Vulkan objects deferred. + `mem: vram` (every 600 frames with `TOTK_TRACE=perf`, `vk_memory.c` `vram_log`) splits device-local memory. A 16,000 + frame uncapped fight on A (`build/v121-vram/NOTES.md`): pools flat (import 1160 MiB host, physical 111 MiB), mirrors + 440 MiB, live textures 1635 MiB, released textures up to 825 MiB (1686 entries); 3.25 GB in all and flat from frame + 9000, no leak. The released entries are the grower: TextureFinalize keeps the image (`vk_texture_release`) for the + finalize-and-init-again cycle of transient textures and for finalized slots that still name the storage, and + `sweep_released` drops all of them at once only past 4096 entries, a count, not bytes (about 2 GB at this mix). + The GTX 1050 Ti of machine B (budget 3620 MiB) reaches 3300-3386 MiB within 1-2 min of play, and from then on up to + 394 pools stay imported (`mem: mirror` "stay imported: budget"), so the GPU reads them over PCIe. + Next: + - Count the frames from release to reuse of each released entry, in `mem: vram`, on the fight and the field: where + transient textures end and the tail that can go begins. + - Drop released entries oldest first, one by one, never all at once, and only under pressure (device-local use past + about 85% of the budget, before a mirror is refused); unchanged on large cards. A fixed cap below the transient + working set would recreate images every frame and lose GPU-rendered content (flicker, empty tile maps). + - Check on B: mirrors refused for the budget and fps before and after; then a 2-3 h session with travel. + - 2-4 GB cards: ASTC kept as RGBA8 is 4x its size; BC1/BC3 recompression (yuzu, `maxwell.md` 2.13) is the larger + saving. 21. **Later**: recording on several threads; descriptor buffers or bindless (touches the translator); CPU/GPU frame overlap as LAS Recomp 13398126d (only after item 9, and it must keep the guard of item 3); an x86-64-v2 build for CPUs without AVX2/FMA (SavageWindRecomp ae42014b4); render passes ended by uniform updates and clear-only work diff --git a/runtime/vk.h b/runtime/vk.h index 17e579f2..6d459a68 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -284,6 +284,10 @@ uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size); // newest recor void vk_texture_write_back(uint64_t addr, uint64_t size); // writes those images back into their storage (waits) void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); // vk_pool_fini: drop the images there void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads); +// Entries now, live and released (TextureFinalize, kept until sweep_released), with the bytes of their own memory +// (images not bound into pool memory). +typedef struct { uint64_t live, released, own_live, own_released, bytes_live, bytes_released; } VkTexCensus; +void vk_texture_census(VkTexCensus *s); void vk_texture_shutdown(void); // vk_mem_shutdown calls it void vk_texture_new_submit(void); // a queue submit starts: draw bindings check storage again uint32_t vk_texture_epoch(void); // the write-watch snapshot epoch vk_texture_new_submit started diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index b6b2e669..7dbd2f87 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -491,9 +491,13 @@ static void flag_names(uint32_t f, char *buf, size_t n) { static void dirty_frame(uint64_t frame); static void mirror_log(uint64_t frame); +static int dead_pending(void); + +static void vram_log(uint64_t frame); void vk_mem_census(uint64_t frame) { static uint64_t window_start; + if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame); // and the // write-watch line @@ -1487,6 +1491,30 @@ static void mirror_log(uint64_t frame) { qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver); } +// With TOTK_TRACE=perf every 600 frames: what holds device-local memory (backlog: GPU memory over long sessions). +// VMA's blocks and allocations on the device-local heaps, texture entries with their own memory, the VRAM mirrors and +// the objects waiting for their last submission to retire. +static void vram_log(uint64_t frame) { + VkCtx *c = vk_ctx(); + if (!c) return; + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + uint64_t blocks = 0, block_bytes = 0, allocs = 0, alloc_bytes = 0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + blocks += vb[h].statistics.blockCount, block_bytes += vb[h].statistics.blockBytes, + allocs += vb[h].statistics.allocationCount, alloc_bytes += vb[h].statistics.allocationBytes; + VkTexCensus t; + vk_texture_census(&t); + const double M = 1048576.0; + rt_log("mem: vram frame %llu: vma %llu blocks %.0f MiB, %llu allocations %.0f MiB; textures live %llu (own memory %llu, " + "%.0f MiB), released %llu (own memory %llu, %.0f MiB); mirrors %lld %.0f MiB; releases pending %d", + (unsigned long long)frame, (unsigned long long)blocks, block_bytes / M, (unsigned long long)allocs, + alloc_bytes / M, (unsigned long long)t.live, (unsigned long long)t.own_live, t.bytes_live / M, + (unsigned long long)t.released, (unsigned long long)t.own_released, t.bytes_released / M, (long long)m_live_n, + m_live_bytes / M, dead_pending()); +} + // Device class: INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a // heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Sets the mirror mode. void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m) { @@ -1701,6 +1729,14 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) { LeaveCriticalSection(&cmd_lock); } +static int dead_pending(void) { + InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL); + EnterCriticalSection(&cmd_lock); + int n = ndead; + LeaveCriticalSection(&cmd_lock); + return n; +} + void vk_defer_image(VkImage img, VmaAllocation alloc) { if (img) defer(alloc ? DEAD_IMAGE_VMA : DEAD_IMAGE, (uint64_t)(uintptr_t)img, alloc); } diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index f381e912..c127235a 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1576,6 +1576,25 @@ void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) unlock(); } +void vk_texture_census(VkTexCensus *s) { + VkCtx *c = vk_ctx(); + memset(s, 0, sizeof *s); + lock(); + for (uint32_t i = 0; i < by_key.cap; i++) { + const Tex *e = by_key.vals[i]; + if (!e) continue; + uint64_t bytes = 0; + if (e->alloc && c) { + VmaAllocationInfo ai; + vmaGetAllocationInfo(c->vma, e->alloc, &ai); + bytes = ai.size; + } + if (e->released) s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes; + else s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes; + } + unlock(); +} + void vk_texture_shutdown(void) { VkCtx *c = vk_ctx(); lock();