From 8e05d2b409b35ab81ebd0af7fb250c0be5e058bd Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 9 Nov 2026 10:11:44 +0300 Subject: [PATCH] Give textures a pool of their own with 64 MiB blocks Textures shared the default 256 MiB blocks with the mirrors and other buffers, and a block went back to the driver only once everything in it was gone. On the RX 9070 after a few minutes of roaming the field, 3 of 9 device-local blocks were 13-27% full, held by a hundred small textures and buffers each, and blocks held 0.8-1.4 GB more than their allocations; where device memory takes process commit, that commit backs nothing. Own-memory textures now go to a VMA pool with 64 MiB blocks (TOTK_TEX_BLOCK_MIB, 0 = the default pools). Images of a quarter block or more, images the driver wants a dedicated allocation for, and images the pool's memory type does not take get memory of their own; the requirements are read before the image is made, since VMA checks neither for a custom pool. The headroom thread keeps the pool's next block ahead like the default pools'. Same 10-minute route: 807 MiB of blocks beyond their allocations instead of 1037, most pool blocks 75-100% full, slow frames not up. Probes pass on v143. Co-Authored-By: Claude Opus 5.5 --- docs/settings.md | 1 + runtime/cfg.c | 6 ++-- runtime/vk.h | 4 +++ runtime/vk_headroom.c | 51 ++++++++++++++++++++++++---------- runtime/vk_memory.c | 64 +++++++++++++++++++++++++++++++++++++++++++ runtime/vk_texture.c | 3 +- 6 files changed, 109 insertions(+), 20 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 2be80094..daefcc60 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -191,6 +191,7 @@ Many rows below are developer tools that only act together with another setting | `restart_wait` | Set by the runtime when it restarts itself; names the old process by id and creation time so the new process waits for it to exit. | Meant for the runtime's own use; the code does not describe a manual value. | Unknown | | `spin` (default `1`) | `0` makes a guest thread that polls memory keep its virtual core. | Polling threads can starve the thread whose item they wait for, so frames can stall. | Caution | | `spin_handoff_us` (default `50`) | Microseconds a polling wait lasts before it gives its virtual core to a waiting thread. | A shorter value hands cores over sooner; a longer one keeps them longer. | Caution | +| `tex_block_mib` (default `64`) | Block size in MiB of the pool that holds the game's textures; images of a quarter block or more get memory of their own. `0` puts textures in the default pools with the buffers. | Larger blocks hold more memory that no texture uses; smaller ones mean more allocations from the driver. | Caution | | `tex_released_mib` (default `256`) | MiB of textures the game released that are kept for reuse; older ones beyond it are dropped. `0` keeps up to 2048 of them, whatever their size. | A larger value keeps more device memory and process commit; a smaller one makes the game create more textures again. | Caution | ## Generated-code patches diff --git a/runtime/cfg.c b/runtime/cfg.c index 73660ba0..f86ad083 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -59,7 +59,7 @@ static const char *const known[] = { "SHADER_MEMO", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHADOW_SNAP", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_RELEASED_MIB", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UB_OVERLAY_OUT", "UPSCALER", "VB_USED", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_BLOCK_MIB", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_RELEASED_MIB", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UB_OVERLAY_OUT", "UPSCALER", "VB_USED", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -99,7 +99,7 @@ static const struct { const char *key, *section; } sections[] = { {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, - {"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_RELEASED_MIB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, + {"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_RELEASED_MIB", "performance"}, {"TEX_BLOCK_MIB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, @@ -174,7 +174,7 @@ static const CfgMeta metas[] = { // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, - {"TEX_RELEASED_MIB", "256", "0..65536", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_RELEASED_MIB", "256", "0..65536", RESTART}, {"TEX_BLOCK_MIB", "64", "0..1024", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, // render fixes {"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART}, {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART}, diff --git a/runtime/vk.h b/runtime/vk.h index 9cac3e54..1a41e73e 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -314,6 +314,8 @@ int vk_mem_uma(void); // 1: the device class is uma (vk_ const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void); void vk_headroom_start(void); void vk_headroom_stop(void); // before vmaDestroyAllocator +// The texture pool (vk_memory.c vk_texture_image) gets its next block ahead too; pool NULL frees the anchor in it. +void vk_headroom_pool(VmaPool pool, uint32_t type, VkDeviceSize block); int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed // Changes whenever vk_mem_resolve plus VK_MEM_USE may give another span for an address: a pool added, finalized or // mapped, a mirror created, refused or forgotten, a submission started. @@ -349,6 +351,8 @@ void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc); void vk_defer_free(void *p); // Device-local scratch for the upload batch (storage, transfer src and dst), from a pool of its own (vk_memory.c). VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc); +// A game texture's image with memory of its own: the texture pool, or a dedicated allocation (vk_memory.c). +VkResult vk_texture_image(const VkImageCreateInfo *ci, VkImage *img, VmaAllocation *alloc); // Staging and scratch bytes the open upload batch holds until it completes. Large texture loads submit the batch early // (vk_texture.c UPLOAD_HELD_MAX) so the GPU starts on it and its buffers come back before the pools need new blocks. VkDeviceSize vk_upload_held(void); diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c index 88083025..baa6ab16 100644 --- a/runtime/vk_headroom.c +++ b/runtime/vk_headroom.c @@ -13,6 +13,7 @@ // Under VRAM pressure no headroom is kept: the spare block counts against the budget that decides whether a pool gets // its VRAM mirror (vk_memory.c), so while use plus two blocks passes 85% of the budget the anchor goes and nothing new // is allocated. The same holds when the system runs short of memory (room). +// The texture pool (vk_memory.c vk_texture_image) is kept ahead the same way, with its own anchor of half its block. // TOTK_VRAM_HEADROOM=0 turns it off. #include "cfg.h" @@ -33,6 +34,10 @@ static struct { volatile int64_t made; // blocks this thread brought in volatile int64_t other; // device memory of 32 MiB or more other threads allocated (VMA blocks, large dedicated) VmaAllocation anchor[64]; // per memory type: the half block that keeps the newest spare block from being deleted + VmaPool pool; // the texture pool, its memory type and block size, and its anchor + uint32_t pool_type; + VkDeviceSize pool_block; + VmaAllocation pool_anchor; } H; static void VKAPI_PTR on_allocate(VmaAllocator a, uint32_t type, VkDeviceMemory m, VkDeviceSize size, void *user) { @@ -68,23 +73,23 @@ static int room(const VkCtx *c, uint32_t type, VkDeviceSize block) { return h->budget && h->usage + 2 * block <= h->budget / 100 * 85 && vk_mem_host_room() >= 4 * block; } -static void drop_anchor(const VkCtx *c, uint32_t type) { - if (H.anchor[type]) vmaFreeMemory(c->vma, H.anchor[type]); - H.anchor[type] = NULL; +static void drop_anchor(const VkCtx *c, VmaAllocation *anchor) { + if (*anchor) vmaFreeMemory(c->vma, *anchor); + *anchor = NULL; } -static void top_up(const VkCtx *c, uint32_t type) { - VkDeviceSize block = block_size(c, type); +// pool NULL: the default pool of the type +static void top_up(const VkCtx *c, uint32_t type, VmaPool pool, VkDeviceSize block, VmaAllocation *anchor) { // Device-local types only: textures and mirrors grow there. Host memory types are woken by the staging pool's own // blocks (vk_memory.c vk_staging), while their default pool held 4 MiB: on the RX 9070 the spare blocks of types 1 // and 3 held 512 MiB of commit for nothing. if (!(c->mem_props.memoryTypes[type].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) || !room(c, type, block)) { - drop_anchor(c, type); + drop_anchor(c, anchor); return; } VkMemoryRequirements req = {.size = block / 2, .alignment = 256, .memoryTypeBits = 1u << type}; - VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_UNKNOWN}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_UNKNOWN, .pool = pool}; VmaAllocation a; int64_t before = H.made; if (vmaAllocateMemory(c->vma, &req, &ai, &a, NULL) != VK_SUCCESS) return; @@ -92,12 +97,13 @@ static void top_up(const VkCtx *c, uint32_t type) { vmaFreeMemory(c->vma, a); return; } - drop_anchor(c, type); - H.anchor[type] = a; + drop_anchor(c, anchor); + *anchor = a; static volatile int32_t logged; if (plat_inc32(&logged) <= 32) - rt_log("vk: headroom: a %.0f MiB block of memory type %u allocated ahead (%lld so far; %lld allocations of 32 MiB " - "or more on other threads)", (double)block / 1048576.0, type, (long long)H.made, (long long)H.other); + rt_log("vk: headroom: a %.0f MiB block of memory type %u%s allocated ahead (%lld so far; %lld allocations of " + "32 MiB or more on other threads)", (double)block / 1048576.0, type, pool ? " (texture pool)" : "", + (long long)H.made, (long long)H.other); } static void headroom_main(void *arg) { @@ -112,10 +118,15 @@ static void headroom_main(void *arg) { plat_mutex_lock(&H.lock); const VkCtx *c = H.stop ? NULL : vk_ctx(); for (uint32_t t = 0; c && t < c->mem_props.memoryTypeCount && t < 64; t++) - if (watch >> t & 1) top_up(c, t); + if (watch >> t & 1) { + top_up(c, t, NULL, block_size(c, t), &H.anchor[t]); + if (H.pool && t == H.pool_type) top_up(c, t, H.pool, H.pool_block, &H.pool_anchor); + } int stop = H.stop; - if (stop && (c = vk_ctx())) - for (uint32_t t = 0; t < 64; t++) drop_anchor(c, t); + if (stop && (c = vk_ctx())) { + for (uint32_t t = 0; t < 64; t++) drop_anchor(c, &H.anchor[t]); + drop_anchor(c, &H.pool_anchor); + } plat_mutex_unlock(&H.lock); if (stop) return; } @@ -127,11 +138,21 @@ void vk_headroom_stop(void) { plat_mutex_lock(&H.lock); H.stop = 1; const VkCtx *c = vk_ctx(); - for (uint32_t t = 0; c && t < 64; t++) drop_anchor(c, t); + for (uint32_t t = 0; c && t < 64; t++) drop_anchor(c, &H.anchor[t]); + if (c) drop_anchor(c, &H.pool_anchor); plat_mutex_unlock(&H.lock); plat_event_set(&H.ev); } +void vk_headroom_pool(VmaPool pool, uint32_t type, VkDeviceSize block) { + if (!H.started) return; + plat_mutex_lock(&H.lock); + const VkCtx *c = vk_ctx(); + if (c) drop_anchor(c, &H.pool_anchor); + H.pool = pool, H.pool_type = type, H.pool_block = block; + plat_mutex_unlock(&H.lock); +} + void vk_headroom_start(void) { if (!H.on || H.started) return; plat_mutex_init(&H.lock); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index c887a560..4b862c19 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2564,6 +2564,66 @@ VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *allo VkDeviceSize vk_upload_held(void) { return open_active ? open_b.held : 0; } +// Own memory of the game's textures (vk_texture.c create_tex): a pool of its own with blocks of TOTK_TEX_BLOCK_MIB +// (default 64; 0 puts textures back in the default pools). In the default pools textures shared 256 MiB blocks with the +// mirrors and other buffers, and a block went back to the driver only once all of them were gone: on the RX 9070, +// after 4.5 minutes of roaming the field, 3 of the 9 device-local blocks were 13-27% full, held by 87-223 small +// textures and buffers each, and the full ones had hundreds of free gaps under 1 MiB, so blocks held 800 MiB more than +// their allocations. Where device memory takes process commit (vk_mem_host_probe) that commit backs nothing. A smaller +// block, holding textures only, empties sooner and wastes less when it does not. With the pool the same 10-minute route +// ended with 807 MiB of blocks beyond their allocations instead of 1037, the texture pool's 22 blocks mostly 75-100% +// full, and the slow frames did not grow. +// An image of a quarter block or more gets memory of its own (a dedicated allocation) that goes back to the driver +// with the image; so do images the driver prefers that for, and images the pool's memory type does not take. VMA +// checks neither for a custom pool (VmaAllocator_T::AllocateMemory takes the pool's type as it is), so the image's +// requirements are read here first. +static VmaPool tex_pool; +static uint32_t tex_type; +static VkDeviceSize tex_block; +static int tex_pool_tried; + +static void tex_pool_init(VkCtx *c, const VkImageCreateInfo *ci) { + plat_once(&cmd_once, cmd_init, NULL); + plat_rmutex_lock(&cmd_lock); + if (!tex_pool_tried) { + tex_block = (VkDeviceSize)cfg_int("TOTK_TEX_BLOCK_MIB") << 20; // schema default 64 + VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + VmaPoolCreateInfo pi = {.blockSize = tex_block}; + if (tex_block && vmaFindMemoryTypeIndexForImageInfo(c->vma, ci, &aci, &tex_type) == VK_SUCCESS) { + pi.memoryTypeIndex = tex_type; + if (vmaCreatePool(c->vma, &pi, &tex_pool) != VK_SUCCESS) tex_pool = NULL; + } + if (tex_pool) { + vk_headroom_pool(tex_pool, tex_type, tex_block); + rt_log("vk: texture pool: blocks of %llu MiB in memory type %u; images of %llu MiB or more get memory of " + "their own (TOTK_TEX_BLOCK_MIB)", (unsigned long long)(tex_block >> 20), tex_type, + (unsigned long long)(tex_block >> 22)); + } else { + rt_log("vk: texture pool %s: textures go to the default pools (TOTK_TEX_BLOCK_MIB)", + tex_block ? "unavailable" : "off"); + } + __atomic_store_n(&tex_pool_tried, 1, __ATOMIC_RELEASE); + } + plat_rmutex_unlock(&cmd_lock); +} + +VkResult vk_texture_image(const VkImageCreateInfo *ci, VkImage *img, VmaAllocation *alloc) { + VkCtx *c = vk_ctx(); + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (!__atomic_load_n(&tex_pool_tried, __ATOMIC_ACQUIRE)) tex_pool_init(c, ci); + if (tex_pool) { + VkDeviceImageMemoryRequirements q = {.sType = VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS, .pCreateInfo = ci}; + VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; + VkMemoryRequirements2 mr = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr}; + vkGetDeviceImageMemoryRequirements(c->device, &q, &mr); + if (dr.requiresDedicatedAllocation || dr.prefersDedicatedAllocation || mr.memoryRequirements.size >= tex_block / 4) + ai.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + else if (mr.memoryRequirements.memoryTypeBits >> tex_type & 1) + ai = (VmaAllocationCreateInfo){.usage = VMA_MEMORY_USAGE_UNKNOWN, .pool = tex_pool}; // the pool's type + } + return vmaCreateImage(c->vma, ci, &ai, img, alloc, NULL); +} + // At device creation: the staging and scratch pools with their kept blocks, made through a first small buffer each // (freed at once), so the first world frame does not allocate them (staging was first used there once the GPU took // the early texture uploads). @@ -2730,6 +2790,10 @@ void vk_mem_shutdown(void) { if (scr_pool) vmaDestroyPool(c->vma, scr_pool); // deferred scratch buffers went with collect_locked above scr_pool = NULL; scr_pool_tried = 0; + vk_headroom_pool(NULL, 0, 0); // frees the headroom's anchor in the texture pool + if (tex_pool) vmaDestroyPool(c->vma, tex_pool); // textures went with vk_texture_shutdown, deferred ones above + tex_pool = NULL; + tex_pool_tried = 0; plat_rmutex_unlock(&cmd_lock); plat_mutex_lock(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 1dd7662d..fb3bcc55 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -459,8 +459,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { } } if (!img) { - VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; - VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); + VkResult r = vk_texture_image(&ci, &img, &e->alloc); // the texture pool or memory of its own if (r != VK_SUCCESS) { log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r, d->format, w, h, depth, d->target, levels, layers);