Merge the VRAM headroom: a background thread allocates the next VMA block of each memory type before the render thread needs it (vk_headroom.c, TOTK_VRAM_HEADROOM)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-27 16:09:38 +03:00
commit acd550e822
4 files changed
+149 -3

No files matched your search

+1 -1
View File
@@ -47,7 +47,7 @@ static const char *const known[] = {
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
"SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SPLIT_US", "SWKBD_ANSWER",
"SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VS_OUTS",
"VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS",
};
+5
View File
@@ -175,6 +175,11 @@ void vk_mem_census(uint64_t frame); // at each present: pool censu
// VRAM in use and its budget, summed over the DEVICE_LOCAL heaps. exact 1: VK_EXT_memory_budget (the whole process),
// 0: this allocator's own allocations against the heap sizes. 0 when unknown.
int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact);
// VRAM headroom (vk_headroom.c): the VMA callbacks that see new device memory, and the thread that allocates the next
// block of each memory type in use ahead of the render thread.
const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void);
void vk_headroom_start(void);
void vk_headroom_stop(void); // before vmaDestroyAllocator
int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed
// Changes whenever vk_mem_resolve plus VK_MEM_USE may give another span for an address: a pool added, finalized or
// mapped, a mirror created, refused or forgotten, a submission started.
+4 -2
View File
@@ -308,7 +308,8 @@ static int create_device(void) {
VmaAllocatorCreateInfo ai = {.physicalDevice = g_vk.phys, .device = g_vk.device, .instance = g_vk.instance,
.vulkanApiVersion = VK_API_VERSION_1_3,
.flags = (g_vk.has_buffer_device_address ? VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT : 0) |
(g_vk.has_memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0)};
(g_vk.has_memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0),
.pDeviceMemoryCallbacks = vk_headroom_callbacks()};
if ((r = vmaCreateAllocator(&ai, &g_vk.vma)) != VK_SUCCESS) { rt_log("vk: vmaCreateAllocator failed (%d)", (int)r); return 0; }
VkSemaphoreTypeCreateInfo st = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
@@ -328,6 +329,7 @@ static void destroy_all(void) {
if (g_vk.device) vkDeviceWaitIdle(g_vk.device);
if (g_vk.device) vk_astc_release();
if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL);
vk_headroom_stop();
if (g_vk.vma) vmaDestroyAllocator(g_vk.vma);
if (g_vk.device) vkDestroyDevice(g_vk.device, NULL);
if (g_vk.messenger) {
@@ -346,7 +348,7 @@ int vk_init(void) {
AcquireSRWLockExclusive(&g_lock);
if (!g_ready) {
memset(&g_vk, 0, sizeof g_vk);
if (create_instance() && pick_device() && create_device()) g_ready = 1;
if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_headroom_start();
else destroy_all();
}
int ok = g_ready;
+139
View File
@@ -0,0 +1,139 @@
// VRAM headroom: the next VMA block of a memory type is allocated on a background thread, not on the render thread.
//
// VMA places an allocation in an existing block of its memory type and allocates a new block (vkAllocateMemory, 32 to
// 256 MiB) only when none has room. On the render thread that took 8-19 ms per block, several times in the first
// minute of play while VRAM grows (textures from create_tex, mirrors). A new VkDeviceMemory made by any other thread
// (VMA's pfnAllocate callback) wakes this thread for its memory type, which it then checks every 500 ms. A check
// allocates half a default block of that type (the largest size VMA still places in blocks, not in dedicated memory).
// When a block has that much room the allocation lands there and is freed at once. When none has, VMA allocates a new
// block here, and the allocation stays as that block's anchor: VMA deletes an empty block at the next free of its
// type (VmaBlockVector::Free, "we have another empty block"), so a spare block left empty lasted until the next
// texture went. The previous anchor is freed then, which gives its half block to the render thread, so at most half
// a block per memory type is held back.
// Under VRAM pressure no headroom is kept: the spare block counts against the budget that decides whether a pool gets
// its VRAM mirror (vk_memory.c), so while use plus two blocks passes 85% of the budget the anchor goes and nothing new
// is allocated.
// TOTK_VRAM_HEADROOM=0 turns it off.
#include <windows.h>
#include "cfg.h"
#include "host.h"
#include "vk.h"
#define HR_PERIOD_MS 500
#define HR_SMALL_HEAP (1ull << 30) // VMA_SMALL_HEAP_MAX_SIZE: smaller heaps use blocks of a heap eighth
static struct {
int on, stop;
SRWLOCK lock; // held around each top-up; vk_headroom_stop takes it before the allocator goes
HANDLE ev;
DWORD tid;
volatile LONG64 types; // memory types a thread other than this one allocated device memory of
volatile LONG64 made; // blocks this thread brought in
volatile LONG64 other; // device memory of 32 MiB or more other threads allocated (VMA blocks, large dedicated)
VmaAllocation anchor[64]; // per memory type: the half block that keeps the newest spare block from being deleted
} H;
static void VKAPI_PTR on_allocate(VmaAllocator a, uint32_t type, VkDeviceMemory m, VkDeviceSize size, void *user) {
(void)a, (void)m, (void)size, (void)user;
if (!H.on || type >= 64) return;
if (GetCurrentThreadId() == H.tid) {
InterlockedIncrement64(&H.made);
return;
}
if (size >= 32ull << 20) InterlockedIncrement64(&H.other);
InterlockedOr64(&H.types, (LONG64)(1ull << type));
SetEvent(H.ev);
}
const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void) {
static const VmaDeviceMemoryCallbacks cb = {.pfnAllocate = on_allocate};
H.on = !cfg_get("TOTK_VRAM_HEADROOM") || cfg_bool("TOTK_VRAM_HEADROOM");
return H.on ? &cb : NULL;
}
static VkDeviceSize block_size(const VkCtx *c, uint32_t type) {
VkDeviceSize heap = c->mem_props.memoryHeaps[c->mem_props.memoryTypes[type].heapIndex].size;
return heap <= HR_SMALL_HEAP ? heap / 8 : 256ull << 20; // VMA's default preferredLargeHeapBlockSize
}
static int room(const VkCtx *c, uint32_t type, VkDeviceSize block) {
VmaBudget b[VK_MAX_MEMORY_HEAPS];
vmaGetHeapBudgets(c->vma, b);
const VmaBudget *h = &b[c->mem_props.memoryTypes[type].heapIndex];
return h->budget && h->usage + 2 * block <= h->budget / 100 * 85;
}
static void drop_anchor(const VkCtx *c, uint32_t type) {
if (H.anchor[type]) vmaFreeMemory(c->vma, H.anchor[type]);
H.anchor[type] = NULL;
}
static void top_up(const VkCtx *c, uint32_t type) {
VkDeviceSize block = block_size(c, type);
if (!room(c, type, block)) {
drop_anchor(c, type);
return;
}
VkMemoryRequirements req = {.size = block / 2, .alignment = 256, .memoryTypeBits = 1u << type};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_UNKNOWN};
VmaAllocation a;
LONG64 before = H.made;
if (vmaAllocateMemory(c->vma, &req, &ai, &a, NULL) != VK_SUCCESS) return;
if (H.made == before) { // there was room
vmaFreeMemory(c->vma, a);
return;
}
drop_anchor(c, type);
H.anchor[type] = a;
static volatile LONG logged;
if (InterlockedIncrement(&logged) <= 32)
rt_log("vk: headroom: a %.0f MiB block of memory type %u allocated ahead (%lld so far; %lld allocations of 32 MiB "
"or more on other threads)", (double)block / 1048576.0, type, (long long)H.made, (long long)H.other);
}
static DWORD WINAPI headroom_main(void *arg) {
(void)arg;
uint64_t watch = 0; // memory types seen so far
for (;;) {
DWORD w = WaitForSingleObject(H.ev, watch ? HR_PERIOD_MS : INFINITE);
if (w == WAIT_FAILED) return 0;
watch |= (uint64_t)InterlockedExchange64(&H.types, 0);
AcquireSRWLockExclusive(&H.lock);
const VkCtx *c = H.stop ? NULL : vk_ctx();
for (uint32_t t = 0; c && t < c->mem_props.memoryTypeCount && t < 64; t++)
if (watch >> t & 1) top_up(c, t);
int stop = H.stop;
if (stop && (c = vk_ctx()))
for (uint32_t t = 0; t < 64; t++) drop_anchor(c, t);
ReleaseSRWLockExclusive(&H.lock);
if (stop) return 0;
}
}
// Before the allocator is destroyed: waits for a running top-up, frees the anchors, and the thread allocates no more.
void vk_headroom_stop(void) {
if (!H.ev) return;
AcquireSRWLockExclusive(&H.lock);
H.stop = 1;
const VkCtx *c = vk_ctx();
for (uint32_t t = 0; c && t < 64; t++) drop_anchor(c, t);
ReleaseSRWLockExclusive(&H.lock);
SetEvent(H.ev);
}
void vk_headroom_start(void) {
if (!H.on || H.ev) return;
InitializeSRWLock(&H.lock);
H.ev = CreateEventW(NULL, FALSE, FALSE, NULL);
HANDLE t = H.ev ? CreateThread(NULL, 0, headroom_main, NULL, 0, &H.tid) : NULL;
if (!t) {
H.on = 0;
rt_log("vk: headroom: no thread, new VMA blocks stay on the allocating thread");
return;
}
SetThreadPriority(t, THREAD_PRIORITY_BELOW_NORMAL);
CloseHandle(t);
rt_log("vk: headroom: the next VMA block of each memory type in use is allocated on a background thread "
"(TOTK_VRAM_HEADROOM=0 turns it off)");
}