From d1149e506046392debef2206adfba99baef53d3d Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 16:05:24 +0300 Subject: [PATCH] Allocate the next VMA block of each memory type on a background thread: a new block made by another thread wakes vk_headroom.c, which allocates half a block every 500 ms and keeps it as the anchor of a block it had to create, so the render thread finds room instead of calling vkAllocateMemory for 8-19 ms; no headroom above 85% of the VRAM budget (TOTK_VRAM_HEADROOM) Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 +- runtime/vk.h | 5 ++ runtime/vk_device.c | 6 +- runtime/vk_headroom.c | 139 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 runtime/vk_headroom.c diff --git a/runtime/cfg.c b/runtime/cfg.c index f18b45b1..a53e4994 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -47,7 +47,7 @@ static const char *const known[] = { "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", - "VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VS_OUTS", + "VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", }; diff --git a/runtime/vk.h b/runtime/vk.h index 9428f56d..74cccc6a 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -175,6 +175,11 @@ void vk_mem_census(uint64_t frame); // at each present: pool censu // VRAM in use and its budget, summed over the DEVICE_LOCAL heaps. exact 1: VK_EXT_memory_budget (the whole process), // 0: this allocator's own allocations against the heap sizes. 0 when unknown. int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact); +// VRAM headroom (vk_headroom.c): the VMA callbacks that see new device memory, and the thread that allocates the next +// block of each memory type in use ahead of the render thread. +const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void); +void vk_headroom_start(void); +void vk_headroom_stop(void); // before vmaDestroyAllocator int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed // Changes whenever vk_mem_resolve plus VK_MEM_USE may give another span for an address: a pool added, finalized or // mapped, a mirror created, refused or forgotten, a submission started. diff --git a/runtime/vk_device.c b/runtime/vk_device.c index aecedc36..01573a38 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -308,7 +308,8 @@ static int create_device(void) { VmaAllocatorCreateInfo ai = {.physicalDevice = g_vk.phys, .device = g_vk.device, .instance = g_vk.instance, .vulkanApiVersion = VK_API_VERSION_1_3, .flags = (g_vk.has_buffer_device_address ? VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT : 0) | - (g_vk.has_memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0)}; + (g_vk.has_memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0), + .pDeviceMemoryCallbacks = vk_headroom_callbacks()}; if ((r = vmaCreateAllocator(&ai, &g_vk.vma)) != VK_SUCCESS) { rt_log("vk: vmaCreateAllocator failed (%d)", (int)r); return 0; } VkSemaphoreTypeCreateInfo st = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO, @@ -328,6 +329,7 @@ static void destroy_all(void) { if (g_vk.device) vkDeviceWaitIdle(g_vk.device); if (g_vk.device) vk_astc_release(); if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL); + vk_headroom_stop(); if (g_vk.vma) vmaDestroyAllocator(g_vk.vma); if (g_vk.device) vkDestroyDevice(g_vk.device, NULL); if (g_vk.messenger) { @@ -346,7 +348,7 @@ int vk_init(void) { AcquireSRWLockExclusive(&g_lock); if (!g_ready) { memset(&g_vk, 0, sizeof g_vk); - if (create_instance() && pick_device() && create_device()) g_ready = 1; + if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_headroom_start(); else destroy_all(); } int ok = g_ready; diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c new file mode 100644 index 00000000..c34b552b --- /dev/null +++ b/runtime/vk_headroom.c @@ -0,0 +1,139 @@ +// VRAM headroom: the next VMA block of a memory type is allocated on a background thread, not on the render thread. +// +// VMA places an allocation in an existing block of its memory type and allocates a new block (vkAllocateMemory, 32 to +// 256 MiB) only when none has room. On the render thread that took 8-19 ms per block, several times in the first +// minute of play while VRAM grows (textures from create_tex, mirrors). A new VkDeviceMemory made by any other thread +// (VMA's pfnAllocate callback) wakes this thread for its memory type, which it then checks every 500 ms. A check +// allocates half a default block of that type (the largest size VMA still places in blocks, not in dedicated memory). +// When a block has that much room the allocation lands there and is freed at once. When none has, VMA allocates a new +// block here, and the allocation stays as that block's anchor: VMA deletes an empty block at the next free of its +// type (VmaBlockVector::Free, "we have another empty block"), so a spare block left empty lasted until the next +// texture went. The previous anchor is freed then, which gives its half block to the render thread, so at most half +// a block per memory type is held back. +// Under VRAM pressure no headroom is kept: the spare block counts against the budget that decides whether a pool gets +// its VRAM mirror (vk_memory.c), so while use plus two blocks passes 85% of the budget the anchor goes and nothing new +// is allocated. +// TOTK_VRAM_HEADROOM=0 turns it off. +#include + +#include "cfg.h" +#include "host.h" +#include "vk.h" + +#define HR_PERIOD_MS 500 +#define HR_SMALL_HEAP (1ull << 30) // VMA_SMALL_HEAP_MAX_SIZE: smaller heaps use blocks of a heap eighth + +static struct { + int on, stop; + SRWLOCK lock; // held around each top-up; vk_headroom_stop takes it before the allocator goes + HANDLE ev; + DWORD tid; + volatile LONG64 types; // memory types a thread other than this one allocated device memory of + volatile LONG64 made; // blocks this thread brought in + volatile LONG64 other; // device memory of 32 MiB or more other threads allocated (VMA blocks, large dedicated) + VmaAllocation anchor[64]; // per memory type: the half block that keeps the newest spare block from being deleted +} H; + +static void VKAPI_PTR on_allocate(VmaAllocator a, uint32_t type, VkDeviceMemory m, VkDeviceSize size, void *user) { + (void)a, (void)m, (void)size, (void)user; + if (!H.on || type >= 64) return; + if (GetCurrentThreadId() == H.tid) { + InterlockedIncrement64(&H.made); + return; + } + if (size >= 32ull << 20) InterlockedIncrement64(&H.other); + InterlockedOr64(&H.types, (LONG64)(1ull << type)); + SetEvent(H.ev); +} + +const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void) { + static const VmaDeviceMemoryCallbacks cb = {.pfnAllocate = on_allocate}; + H.on = !cfg_get("TOTK_VRAM_HEADROOM") || cfg_bool("TOTK_VRAM_HEADROOM"); + return H.on ? &cb : NULL; +} + +static VkDeviceSize block_size(const VkCtx *c, uint32_t type) { + VkDeviceSize heap = c->mem_props.memoryHeaps[c->mem_props.memoryTypes[type].heapIndex].size; + return heap <= HR_SMALL_HEAP ? heap / 8 : 256ull << 20; // VMA's default preferredLargeHeapBlockSize +} + +static int room(const VkCtx *c, uint32_t type, VkDeviceSize block) { + VmaBudget b[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, b); + const VmaBudget *h = &b[c->mem_props.memoryTypes[type].heapIndex]; + return h->budget && h->usage + 2 * block <= h->budget / 100 * 85; +} + +static void drop_anchor(const VkCtx *c, uint32_t type) { + if (H.anchor[type]) vmaFreeMemory(c->vma, H.anchor[type]); + H.anchor[type] = NULL; +} + +static void top_up(const VkCtx *c, uint32_t type) { + VkDeviceSize block = block_size(c, type); + if (!room(c, type, block)) { + drop_anchor(c, type); + return; + } + VkMemoryRequirements req = {.size = block / 2, .alignment = 256, .memoryTypeBits = 1u << type}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_UNKNOWN}; + VmaAllocation a; + LONG64 before = H.made; + if (vmaAllocateMemory(c->vma, &req, &ai, &a, NULL) != VK_SUCCESS) return; + if (H.made == before) { // there was room + vmaFreeMemory(c->vma, a); + return; + } + drop_anchor(c, type); + H.anchor[type] = a; + static volatile LONG logged; + if (InterlockedIncrement(&logged) <= 32) + rt_log("vk: headroom: a %.0f MiB block of memory type %u allocated ahead (%lld so far; %lld allocations of 32 MiB " + "or more on other threads)", (double)block / 1048576.0, type, (long long)H.made, (long long)H.other); +} + +static DWORD WINAPI headroom_main(void *arg) { + (void)arg; + uint64_t watch = 0; // memory types seen so far + for (;;) { + DWORD w = WaitForSingleObject(H.ev, watch ? HR_PERIOD_MS : INFINITE); + if (w == WAIT_FAILED) return 0; + watch |= (uint64_t)InterlockedExchange64(&H.types, 0); + AcquireSRWLockExclusive(&H.lock); + const VkCtx *c = H.stop ? NULL : vk_ctx(); + for (uint32_t t = 0; c && t < c->mem_props.memoryTypeCount && t < 64; t++) + if (watch >> t & 1) top_up(c, t); + int stop = H.stop; + if (stop && (c = vk_ctx())) + for (uint32_t t = 0; t < 64; t++) drop_anchor(c, t); + ReleaseSRWLockExclusive(&H.lock); + if (stop) return 0; + } +} + +// Before the allocator is destroyed: waits for a running top-up, frees the anchors, and the thread allocates no more. +void vk_headroom_stop(void) { + if (!H.ev) return; + AcquireSRWLockExclusive(&H.lock); + H.stop = 1; + const VkCtx *c = vk_ctx(); + for (uint32_t t = 0; c && t < 64; t++) drop_anchor(c, t); + ReleaseSRWLockExclusive(&H.lock); + SetEvent(H.ev); +} + +void vk_headroom_start(void) { + if (!H.on || H.ev) return; + InitializeSRWLock(&H.lock); + H.ev = CreateEventW(NULL, FALSE, FALSE, NULL); + HANDLE t = H.ev ? CreateThread(NULL, 0, headroom_main, NULL, 0, &H.tid) : NULL; + if (!t) { + H.on = 0; + rt_log("vk: headroom: no thread, new VMA blocks stay on the allocating thread"); + return; + } + SetThreadPriority(t, THREAD_PRIORITY_BELOW_NORMAL); + CloseHandle(t); + rt_log("vk: headroom: the next VMA block of each memory type in use is allocated on a background thread " + "(TOTK_VRAM_HEADROOM=0 turns it off)"); +}