// Vulkan memory of the M4 graphics layer: NVN memory pools as Vulkan // buffers, the virtual-pool page table, GPU address resolution, page write generations for re-uploads, and the // upload batch (command buffer, staging buffers, deferred destruction) that vk_texture.c records into. // // Behaviour ported from MacNativeGames games/totk/third_party/d2r_gfx/gfx/nvn/totk_vpool.c (request checks, unmap // semantics, 64 KiB pages) and nvn_dirty.c (generation stamps: a texture is stale when a covered page carries a // generation newer than its upload), @ b75be8cf (MIT, Copyright (c) 2026 jhaizhou; origin d2r-mobile); and from LAS // Recomp runtime/nvn/vulkan/exec.c:2094-2131 (vk_pages_changed: every write-watch query resets the OS bits, so each // query stamps the pages it saw with a new generation shared by all users) and textures.c:240-270 (deferred // destruction) @ da35b83a (MIT, Copyright (c) 2026 Peiikai). MNG aliases host VM (mach_vm_remap) and protects pages // (fault handler); here virtual pools are a GPU-side page table over device-local memory and dirty pages come from // GetWriteWatch (mem_written_pages), so neither is needed. #include #include #include #include #include "cfg.h" #include "gfx.h" #include "host.h" #include "nvn_api.h" #include "nvn_pool_index.h" #include "vk.h" // ---------------------------------------------------------------- pools enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT }; typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page -> physical pool and offset typedef struct PoolRec { const pool_t *key; int kind; uint32_t flags; uint64_t lo, size; // GPU address range of the pool uint64_t import_lo, import_hi; // POOL_IMPORT: the imported host range (aligned outwards) VkBuffer buf; VkDeviceMemory mem; // POOL_IMPORT VmaAllocation alloc; // POOL_PHYS VPage *pt; // POOL_VIRT: size / 64 KiB entries uint64_t npages; volatile long used; // resolved at least once: the GPU may still read it at finalize } PoolRec; static SRWLOCK pool_lock = SRWLOCK_INIT; // Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits // stay valid while it is unchanged. static uint64_t pool_gen = 1; static pidx pool_index; // POOL_IMPORT, POOL_PHYS and POOL_VIRT records by GPU address static PoolRec **phys_list; // physical pools by key, for MapVirtual requests static int nphys, cap_phys; static long n_imports, n_import_failures; static PFN_vkGetMemoryHostPointerPropertiesEXT p_host_props; static VkBufferUsageFlags pool_usage(const VkCtx *c) { VkBufferUsageFlags u = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; if (c->has_transform_feedback) u |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT; if (c->has_conditional_rendering) u |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT; return u; } static int memory_type(const VkCtx *c, uint32_t bits, VkMemoryPropertyFlags want) { for (uint32_t i = 0; i < c->mem_props.memoryTypeCount; i++) if ((bits >> i & 1) && (c->mem_props.memoryTypes[i].propertyFlags & want) == want) return (int)i; for (uint32_t i = 0; i < c->mem_props.memoryTypeCount; i++) if (bits >> i & 1) return (int)i; return -1; } // CPU-visible storage: the guest pages themselves become the buffer's memory (VK_EXT_external_memory_host). static int import_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) { if (!p_host_props) p_host_props = (PFN_vkGetMemoryHostPointerPropertiesEXT)vkGetDeviceProcAddr(c->device, "vkGetMemoryHostPointerPropertiesEXT"); uint64_t al = c->min_imported_host_alignment ? c->min_imported_host_alignment : 0x1000; r->import_lo = addr & ~(al - 1); r->import_hi = (addr + size + al - 1) & ~(al - 1); uint64_t n = r->import_hi - r->import_lo; VkMemoryHostPointerPropertiesEXT hp = {.sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT}; VkResult res = p_host_props ? p_host_props(c->device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, (void *)(uintptr_t)r->import_lo, &hp) : VK_ERROR_EXTENSION_NOT_PRESENT; if (res != VK_SUCCESS || !hp.memoryTypeBits) { rt_log("vk: pool %#llx+%#llx: host pointer not importable (%d)", (unsigned long long)addr, (unsigned long long)size, (int)res); return 0; } VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT}; VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = n, .usage = pool_usage(c), .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; if ((res = vkCreateBuffer(c->device, &bi, NULL, &r->buf)) != VK_SUCCESS) { rt_log("vk: pool %#llx: vkCreateBuffer for the import failed (%d)", (unsigned long long)addr, (int)res); return 0; } VkMemoryRequirements req; vkGetBufferMemoryRequirements(c->device, r->buf, &req); int type = memory_type(c, req.memoryTypeBits & hp.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); if (type < 0 || req.size > n) { rt_log("vk: pool %#llx: no memory type for the import (buffer %#x, host %#x, size %#llx of %#llx)", (unsigned long long)addr, req.memoryTypeBits, hp.memoryTypeBits, (unsigned long long)req.size, (unsigned long long)n); vkDestroyBuffer(c->device, r->buf, NULL); r->buf = VK_NULL_HANDLE; return 0; } VkImportMemoryHostPointerInfoEXT im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, .pHostPointer = (void *)(uintptr_t)r->import_lo}; VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = &im, .allocationSize = n, .memoryTypeIndex = (uint32_t)type}; if ((res = vkAllocateMemory(c->device, &ai, NULL, &r->mem)) != VK_SUCCESS || (res = vkBindBufferMemory(c->device, r->buf, r->mem, 0)) != VK_SUCCESS) { rt_log("vk: pool %#llx+%#llx: host memory import failed (%d)", (unsigned long long)addr, (unsigned long long)size, (int)res); if (r->mem) vkFreeMemory(c->device, r->mem, NULL); vkDestroyBuffer(c->device, r->buf, NULL); r->buf = VK_NULL_HANDLE; r->mem = VK_NULL_HANDLE; return 0; } InterlockedIncrement(&n_imports); return 1; } // Physical pool: device-local memory of the pool size that both buffers and optimal-tiling images can be bound to. static int alloc_physical(VkCtx *c, PoolRec *r) { VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = r->size, .usage = pool_usage(c), .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; VkResult res = vkCreateBuffer(c->device, &bi, NULL, &r->buf); if (res != VK_SUCCESS) { rt_log("vk: physical pool: vkCreateBuffer failed (%d)", (int)res); return 0; } VkMemoryRequirements rb; vkGetBufferMemoryRequirements(c->device, r->buf, &rb); VkImageCreateInfo ii = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, .format = VK_FORMAT_R8G8B8A8_UNORM, .extent = {256, 256, 1}, .mipLevels = 1, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT}; VkDeviceImageMemoryRequirements di = {.sType = VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS, .pCreateInfo = &ii}; VkMemoryRequirements2 ri = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2}; vkGetDeviceImageMemoryRequirements(c->device, &di, &ri); uint32_t bits = rb.memoryTypeBits & ri.memoryRequirements.memoryTypeBits; if (!bits) bits = rb.memoryTypeBits; VkDeviceSize align = NVN_VPOOL_PAGE; if (rb.alignment > align) align = rb.alignment; if (ri.memoryRequirements.alignment > align) align = ri.memoryRequirements.alignment; VkMemoryRequirements req = {.size = (rb.size + align - 1) & ~(align - 1), .alignment = align, .memoryTypeBits = bits}; VmaAllocationCreateInfo aci = {.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; if ((res = vmaAllocateMemory(c->vma, &req, &aci, &r->alloc, NULL)) != VK_SUCCESS || (res = vmaBindBufferMemory(c->vma, r->alloc, r->buf)) != VK_SUCCESS) { rt_log("vk: physical pool of %#llx bytes: device-local allocation failed (%d)", (unsigned long long)r->size, (int)res); if (r->alloc) vmaFreeMemory(c->vma, r->alloc); vkDestroyBuffer(c->device, r->buf, NULL); r->buf = VK_NULL_HANDLE; r->alloc = NULL; return 0; } if (vk_fix_zero()) { // Switch pool memory starts zeroed; once per pool, not per image placed in it VkCommandBuffer cb = vk_upload_begin(); vkCmdFillBuffer(cb, r->buf, 0, VK_WHOLE_SIZE, 0); VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); vk_upload_end(); } return 1; } int vk_fix_zero(void) { static int on = -1; if (on < 0) { const char *e = cfg_get("TOTK_FIX_ZERO"); on = !(e && !strcmp(e, "0")); } return on; } static void index_put(PoolRec *r) { __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); pidx_ent e = {.lo = r->lo, .hi = r->lo + r->size, .base = r->lo, .buf = r, .gpu = r->lo, .key = r->key}; pidx_remove(&pool_index, r->key, NULL); // an object initialized again without Finalize if (!pidx_insert(&pool_index, &e)) rt_fatal("vk: out of memory for the pool index"); } static PoolRec *phys_find(const pool_t *key) { for (int i = 0; i < nphys; i++) if (phys_list[i]->key == key) return phys_list[i]; return NULL; } static void destroy_rec(VkCtx *c, PoolRec *r) { if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL); if (r->mem) vkFreeMemory(c->device, r->mem, NULL); if (r->alloc) vmaFreeMemory(c->vma, r->alloc); free(r->pt); free(r); } int vk_pool_init(pool_t *p) { VkCtx *c = vk_ctx(); if (!c) { rt_log("vk: MemoryPoolInitialize before vk_init"); return 0; } // The GPU never sees a GPU_NO_ACCESS pool, except a physical one (0x189) through the virtual pools that map it. if ((p->flags & NVN_MEMORY_POOL_FLAGS_GPU_NO_ACCESS) && !(p->flags & NVN_MEMORY_POOL_FLAGS_PHYSICAL)) return 1; PoolRec *r = calloc(1, sizeof *r); if (!r) rt_fatal("vk: out of memory for a pool record"); r->key = p; r->flags = p->flags; r->lo = (uintptr_t)p->mem; r->size = p->size; int ok; if (p->flags & NVN_MEMORY_POOL_FLAGS_VIRTUAL) { r->kind = POOL_VIRT; r->npages = p->size / NVN_VPOOL_PAGE; r->pt = calloc(r->npages ? r->npages : 1, sizeof *r->pt); ok = r->pt != NULL; } else if (p->flags & NVN_MEMORY_POOL_FLAGS_PHYSICAL) { r->kind = POOL_PHYS; ok = alloc_physical(c, r); } else { r->kind = POOL_IMPORT; ok = import_pool(c, r, (uintptr_t)p->mem, p->size); if (!ok) { // Not fatal for the game: the pool works for the CPU, and vk_mem_resolve reports its addresses as unbacked. long n = InterlockedIncrement(&n_import_failures); if (n <= 8) rt_log("vk: pool %p (flags %#x, %#llx bytes) has no GPU buffer (%ld so far)", (void *)p->mem, p->flags, (unsigned long long)p->size, n); r->buf = VK_NULL_HANDLE; ok = 1; } } if (!ok) { free(r->pt); free(r); return 0; } AcquireSRWLockExclusive(&pool_lock); index_put(r); if (r->kind == POOL_PHYS) { if (nphys == cap_phys) { cap_phys = cap_phys ? cap_phys * 2 : 16; phys_list = realloc(phys_list, (size_t)cap_phys * sizeof *phys_list); if (!phys_list) rt_fatal("vk: out of memory for the physical pool list"); } phys_list[nphys++] = r; } ReleaseSRWLockExclusive(&pool_lock); return 1; } void vk_pool_fini(pool_t *p) { VkCtx *c = vk_ctx(); if (!c) return; pidx_ent e; AcquireSRWLockExclusive(&pool_lock); int had = pidx_remove(&pool_index, p, &e); __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); PoolRec *r = had ? e.buf : NULL; if (r && r->kind == POOL_PHYS) { for (int i = 0; i < nphys; i++) if (phys_list[i] == r) { phys_list[i] = phys_list[--nphys]; break; } for (int i = 0; i < pool_index.n; i++) { // virtual pages that still map it become unmapped PoolRec *v = pool_index.e[i].buf; if (v->kind != POOL_VIRT) continue; for (uint64_t k = 0; k < v->npages; k++) if (v->pt[k].phys == r) v->pt[k].phys = NULL; } } ReleaseSRWLockExclusive(&pool_lock); if (!r) return; // Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the // guest gets the storage back when this returns. vk_texture_range_gone(r->lo, r->size, r->alloc); if (r->used && r->kind != POOL_VIRT) { vk_upload_flush(1); vk_queue_lock(); vkQueueWaitIdle(c->gfx_queue); vk_queue_unlock(); } vk_mem_collect(); destroy_rec(c, r); } // Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c:54-67); apply them in order. int vk_pool_map_virtual(pool_t *v, int n, const void *requests) { const nvn_mapping_t *rq = requests; AcquireSRWLockExclusive(&pool_lock); const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)v->mem, 0); PoolRec *r = e && e->key == v ? e->buf : NULL; int ok = r && r->kind == POOL_VIRT; __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); for (int i = 0; ok && i < n; i++) { PoolRec *ph = rq[i].physical_pool ? phys_find((const pool_t *)(const void *)rq[i].physical_pool) : NULL; if (rq[i].physical_pool && !ph) { rt_log("vk: MapVirtual request %d names physical pool %p that has no Vulkan memory", i, (const void *)rq[i].physical_pool); ok = 0; break; } uint64_t first = (uint64_t)rq[i].virtual_offset / NVN_VPOOL_PAGE, count = rq[i].size / NVN_VPOOL_PAGE; if (first + count > r->npages) { ok = 0; break; } for (uint64_t k = 0; k < count; k++) r->pt[first + k] = (VPage){ph, ph ? (uint64_t)rq[i].physical_offset + k * NVN_VPOOL_PAGE : 0}; } ReleaseSRWLockExclusive(&pool_lock); if (!ok) rt_log("vk: MapVirtual on pool %p failed", (void *)v->mem); return ok; } // Per-thread copies of resolved import and physical pools that overlap no other index entry (so pidx_find picks them for // any range inside), valid while pool_gen is unchanged: the replay resolves a few pools thousands of times per frame. typedef struct { uint64_t gen, lo, hi, base, end; VkBuffer buf; VmaAllocation alloc; } ResolveHit; static _Thread_local ResolveHit rhit[4]; static _Thread_local unsigned rhit_next; int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { memset(out, 0, sizeof *out); uint64_t g = __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE), end = addr + size; if (end >= addr) for (int i = 0; i < 4; i++) { const ResolveHit *h = &rhit[i]; if (h->gen != g || addr < h->lo || end > h->hi) continue; out->buffer = h->buf; out->offset = addr - h->base; out->avail = h->end - addr; out->alloc = h->alloc; if (h->alloc) out->alloc_offset = out->offset; return 1; } AcquireSRWLockShared(&pool_lock); const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)addr, size); PoolRec *r = e ? e->buf : NULL; int ok = 0; if (r && (r->kind == POOL_IMPORT ? r->buf != VK_NULL_HANDLE : r->kind == POOL_PHYS)) { int i = (int)(e - pool_index.e); if ((i == 0 || pool_index.maxhi[i - 1] <= e->lo) && (i + 1 == pool_index.n || pool_index.e[i + 1].lo >= e->hi)) { ResolveHit *h = &rhit[rhit_next++ & 3]; h->gen = __atomic_load_n(&pool_gen, __ATOMIC_RELAXED); // stable: changed only under the exclusive lock h->lo = e->lo, h->hi = e->hi, h->buf = r->buf; if (r->kind == POOL_IMPORT) h->base = r->import_lo, h->end = r->import_hi, h->alloc = NULL; else h->base = r->lo, h->end = r->lo + r->size, h->alloc = r->alloc; } } if (r && r->kind == POOL_IMPORT && r->buf) { out->buffer = r->buf; out->offset = addr - r->import_lo; out->avail = r->import_hi - addr; ok = 1; } else if (r && r->kind == POOL_PHYS) { out->buffer = r->buf; out->offset = out->alloc_offset = addr - r->lo; out->avail = r->size - out->offset; out->alloc = r->alloc; ok = 1; } else if (r && r->kind == POOL_VIRT) { uint64_t rel = addr - r->lo, k = rel / NVN_VPOOL_PAGE; if (k < r->npages && r->pt[k].phys) { PoolRec *ph = r->pt[k].phys; uint64_t off = r->pt[k].off + rel % NVN_VPOOL_PAGE, end = r->pt[k].off + NVN_VPOOL_PAGE; for (uint64_t j = k + 1; j < r->npages && r->pt[j].phys == ph && r->pt[j].off == end; j++) end += NVN_VPOOL_PAGE; out->buffer = ph->buf; out->offset = out->alloc_offset = off; out->avail = end - off; out->alloc = ph->alloc; ok = out->avail >= size; if (ok && !ph->used) InterlockedExchange(&ph->used, 1); // read first: the flag is only ever set } } if (ok && !r->used) InterlockedExchange(&r->used, 1); ReleaseSRWLockShared(&pool_lock); if (!ok) memset(out, 0, sizeof *out); return ok; } // ---------------------------------------------------------------- page write generations // Two-level table over the guest address space (below 2^39): 256 MiB chunks of 65,536 page generations, allocated // when a page of the chunk is first stamped. 0 = never stamped. #define GEN_CHUNK_SHIFT 28 #define GEN_PAGE_SHIFT 12 #define GEN_CHUNKS (RT_ADDRESS_LIMIT >> GEN_CHUNK_SHIFT) #define GEN_PER_CHUNK (1u << (GEN_CHUNK_SHIFT - GEN_PAGE_SHIFT)) static uint32_t *gen_chunk[GEN_CHUNKS]; static uint32_t gen_now; // last generation handed out static SRWLOCK gen_lock = SRWLOCK_INIT; static uint64_t gen_pages[1024]; static uint32_t *gen_slot(uint64_t page_addr, int create) { uint64_t ci = page_addr >> GEN_CHUNK_SHIFT; if (ci >= GEN_CHUNKS) return NULL; if (!gen_chunk[ci]) { if (!create) return NULL; gen_chunk[ci] = calloc(GEN_PER_CHUNK, sizeof(uint32_t)); if (!gen_chunk[ci]) rt_fatal("vk: out of memory for page generations"); } return &gen_chunk[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)]; } static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) { for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) { uint32_t *s = gen_slot(a, 1); if (s) *s = g; } } void vk_guest_written(uint64_t addr, uint64_t size) { if (!size || nvn_gpu_only(addr)) return; uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; AcquireSRWLockExclusive(&gen_lock); stamp_range(lo, hi, ++gen_now); ReleaseSRWLockExclusive(&gen_lock); } int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now) { if (!size) { *now = since; return 0; } uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; AcquireSRWLockExclusive(&gen_lock); uint32_t g = ++gen_now; int tracked = 1; for (;;) { uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages); if (n == UINT64_MAX) { tracked = 0; break; } for (uint64_t k = 0; k < n; k++) { uint32_t *s = gen_slot(gen_pages[k], 1); if (s) *s = g; } if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call } int changed = 0; for (uint64_t a = lo; tracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) { uint32_t *s = gen_slot(a, 0); if (s && *s > since) changed = 1; } ReleaseSRWLockExclusive(&gen_lock); *now = g; return tracked ? changed : -1; } // Write-watch snapshots per submission: the guest address space in 2 MiB regions, each queried at most once per epoch // (a guest submission), at the first check that covers it. A query costs about 1.3 us plus 0.55 us per MiB, so one per // region instead of one per texture. The snapshot only adds observations to the generation table: a page written // before it carries a generation newer than any check before it, and a page written after it keeps its OS bit until // the next query over it (at the latest the next epoch's snapshot). #define SNAP_SHIFT 21 #define SNAP_REGIONS (RT_ADDRESS_LIMIT >> SNAP_SHIFT) static uint32_t snap_at[SNAP_REGIONS]; // epoch of the region's last snapshot; 0 never static uint8_t snap_untracked[SNAP_REGIONS]; static uint64_t n_snap_checks, n_snap_calls, n_snap_untracked; int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) { if (!size) { *now = since; return 0; } uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; if (!epoch || r1 >= SNAP_REGIONS) return vk_mem_changed(addr, size, since, now); int excl = 0; AcquireSRWLockShared(&gen_lock); for (uint64_t r = r0; r <= r1; r++) if (snap_at[r] != epoch) { excl = 1; break; } if (excl) { ReleaseSRWLockShared(&gen_lock); AcquireSRWLockExclusive(&gen_lock); uint32_t g = ++gen_now; for (uint64_t r = r0; r <= r1;) { if (snap_at[r] == epoch) { r++; continue; } uint64_t k = r; // one query over a run of regions not seen in this epoch while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++; uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT; int tracked = 1; for (;;) { uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages); if (n == UINT64_MAX) { tracked = 0; break; } for (uint64_t j = 0; j < n; j++) { uint32_t *s = gen_slot(gen_pages[j], 1); if (s) *s = g; } if (n < sizeof gen_pages / sizeof *gen_pages) break; } n_snap_calls++; for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked; r = k + 1; } } int untracked = 0, changed = 0; for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r]; for (uint64_t a = lo; !untracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) { uint32_t *s = gen_slot(a, 0); if (s && *s > since) changed = 1; } *now = gen_now; // every observation so far is in the table __atomic_add_fetch(&n_snap_checks, 1, __ATOMIC_RELAXED); if (untracked) __atomic_add_fetch(&n_snap_untracked, 1, __ATOMIC_RELAXED); if (excl) ReleaseSRWLockExclusive(&gen_lock); else ReleaseSRWLockShared(&gen_lock); return untracked ? -1 : changed; } void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) { *checks = __atomic_exchange_n(&n_snap_checks, 0, __ATOMIC_RELAXED); *calls = __atomic_exchange_n(&n_snap_calls, 0, __ATOMIC_RELAXED); *untracked = __atomic_exchange_n(&n_snap_untracked, 0, __ATOMIC_RELAXED); } // ---------------------------------------------------------------- upload batches and deferred destruction typedef struct { VkBuffer buf; VmaAllocation alloc; } Staging; typedef struct { VkCommandBuffer cb; VkFence fence; uint64_t seq; Staging *stg; int nstg, cap; } Batch; enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA }; typedef struct { int kind; uint64_t handle; VmaAllocation alloc; uint64_t seq, tl; } Dead; static CRITICAL_SECTION cmd_lock; static INIT_ONCE cmd_once = INIT_ONCE_STATIC_INIT; static VkCommandPool cmd_pool; static Batch open_b; static int open_active; static Batch *inflight; static int ninflight, cap_inflight; static Batch *spare; // completed batches whose command buffer and fence are reused static int nspare, cap_spare; static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq) static Dead *dead; static int ndead, cap_dead; static uint64_t tl_noted; // newest timeline value T6 reported submitted static BOOL CALLBACK cmd_init(PINIT_ONCE o, PVOID p, PVOID *ctx) { (void)o, (void)p, (void)ctx; InitializeCriticalSection(&cmd_lock); return TRUE; } static void grow(void **arr, int *cap, size_t elem) { *cap = *cap ? *cap * 2 : 16; void *n = realloc(*arr, (size_t)*cap * elem); if (!n) rt_fatal("vk: out of memory (upload bookkeeping)"); *arr = n; } static void free_staging(VkCtx *c, Batch *b) { for (int i = 0; i < b->nstg; i++) vmaDestroyBuffer(c->vma, b->stg[i].buf, b->stg[i].alloc); b->nstg = 0; } // Batches complete in submission order (one queue): retire from the front while their fences are signalled. static void retire(VkCtx *c, int wait_all) { int k = 0; for (; k < ninflight; k++) { Batch *b = &inflight[k]; if (wait_all) vkWaitForFences(c->device, 1, &b->fence, VK_TRUE, UINT64_MAX); else if (vkGetFenceStatus(c->device, b->fence) != VK_SUCCESS) break; free_staging(c, b); done_seq = b->seq; if (nspare == cap_spare) grow((void **)&spare, &cap_spare, sizeof *spare); spare[nspare++] = *b; } memmove(inflight, inflight + k, (size_t)(ninflight - k) * sizeof *inflight); ninflight -= k; } static void collect_locked(VkCtx *c) { retire(c, 0); uint64_t tl = 0; if (c->timeline) vkGetSemaphoreCounterValue(c->device, c->timeline, &tl); int k = 0; for (int i = 0; i < ndead; i++) { Dead *d = &dead[i]; if (d->seq > done_seq || d->tl > tl) { dead[k++] = *d; continue; } switch (d->kind) { case DEAD_IMAGE: vkDestroyImage(c->device, (VkImage)(uintptr_t)d->handle, NULL); break; case DEAD_IMAGE_VMA: vmaDestroyImage(c->vma, (VkImage)(uintptr_t)d->handle, d->alloc); break; case DEAD_VIEW: vkDestroyImageView(c->device, (VkImageView)(uintptr_t)d->handle, NULL); break; case DEAD_BUFFER_VMA: vmaDestroyBuffer(c->vma, (VkBuffer)(uintptr_t)d->handle, d->alloc); break; } } ndead = k; } VkCommandBuffer vk_upload_begin(void) { InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL); EnterCriticalSection(&cmd_lock); VkCtx *c = vk_ctx(); if (!c) rt_fatal("vk: upload before vk_init"); if (open_active) return open_b.cb; if (!cmd_pool) { VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = c->gfx_family}; if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: upload command pool"); } retire(c, 0); memset(&open_b, 0, sizeof open_b); if (nspare) { Batch s = spare[--nspare]; open_b.cb = s.cb, open_b.fence = s.fence, open_b.stg = s.stg, open_b.cap = s.cap; vkResetCommandBuffer(open_b.cb, 0); vkResetFences(c->device, 1, &open_b.fence); } else { VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; if (vkAllocateCommandBuffers(c->device, &ai, &open_b.cb) != VK_SUCCESS || vkCreateFence(c->device, &fi, NULL, &open_b.fence) != VK_SUCCESS) rt_fatal("vk: upload command buffer"); } VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(open_b.cb, &bi); open_b.seq = ++next_seq; open_active = 1; return open_b.cb; } void vk_upload_end(void) { LeaveCriticalSection(&cmd_lock); } void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { VkCtx *c = vk_ctx(); VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16, .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; VmaAllocationInfo info; VkBuffer b; VmaAllocation a; VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info); if (r != VK_SUCCESS) rt_fatal("vk: staging buffer of %llu bytes failed (%d)", (unsigned long long)size, (int)r); *buf = b; if (alloc) *alloc = a; // the caller destroys it (vmaDestroyBuffer) after vk_upload_flush(1) else { if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg); open_b.stg[open_b.nstg++] = (Staging){b, a}; } return info.pMappedData; } uint64_t vk_upload_flush(int wait) { InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL); EnterCriticalSection(&cmd_lock); VkCtx *c = vk_ctx(); uint64_t seq = next_seq; if (c && open_active) { for (int i = 0; i < open_b.nstg; i++) vmaFlushAllocation(c->vma, open_b.stg[i].alloc, 0, VK_WHOLE_SIZE); vkEndCommandBuffer(open_b.cb); VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &open_b.cb}; vk_queue_lock(); VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); if (r != VK_SUCCESS) rt_fatal("vk: upload submit failed (%d)", (int)r); if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight); inflight[ninflight++] = open_b; open_active = 0; } if (c && wait) { while (ninflight && done_seq < seq) { vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX); retire(c, 0); } } if (c) collect_locked(c); LeaveCriticalSection(&cmd_lock); return seq; } static void defer(int kind, uint64_t handle, VmaAllocation alloc) { InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL); EnterCriticalSection(&cmd_lock); if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead); // The open batch may record into the object: it counts as the newest user. dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl_noted}; LeaveCriticalSection(&cmd_lock); } void vk_defer_image(VkImage img, VmaAllocation alloc) { if (img) defer(alloc ? DEAD_IMAGE_VMA : DEAD_IMAGE, (uint64_t)(uintptr_t)img, alloc); } void vk_defer_view(VkImageView view) { if (view) defer(DEAD_VIEW, (uint64_t)(uintptr_t)view, NULL); } void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) { if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc); } void vk_mem_note_submit(uint64_t timeline_value) { InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL); EnterCriticalSection(&cmd_lock); if (timeline_value > tl_noted) tl_noted = timeline_value; LeaveCriticalSection(&cmd_lock); } void vk_mem_collect(void) { VkCtx *c = vk_ctx(); if (!c) return; InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL); EnterCriticalSection(&cmd_lock); collect_locked(c); LeaveCriticalSection(&cmd_lock); } void vk_mem_stats(VkMemStats *s) { memset(s, 0, sizeof *s); AcquireSRWLockShared(&pool_lock); for (int i = 0; i < pool_index.n; i++) { const PoolRec *r = pool_index.e[i].buf; if (r->kind == POOL_IMPORT) s->imported_pools++; else if (r->kind == POOL_PHYS) s->physical_pools++; else s->virtual_pools++; } ReleaseSRWLockShared(&pool_lock); s->import_failures = (uint64_t)n_import_failures; s->upload_batches = next_seq; } void vk_mem_shutdown(void) { VkCtx *c = vk_ctx(); if (!c) return; vk_upload_flush(1); vk_queue_lock(); vkDeviceWaitIdle(c->device); vk_queue_unlock(); vk_texture_shutdown(); EnterCriticalSection(&cmd_lock); retire(c, 1); uint64_t keep = tl_noted; tl_noted = 0; for (int i = 0; i < ndead; i++) dead[i].seq = 0, dead[i].tl = 0; // the device is idle collect_locked(c); tl_noted = keep; for (int i = 0; i < nspare; i++) { vkDestroyFence(c->device, spare[i].fence, NULL); free(spare[i].stg); } nspare = 0; if (cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL); cmd_pool = VK_NULL_HANDLE; LeaveCriticalSection(&cmd_lock); AcquireSRWLockExclusive(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); pool_index.n = 0; __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); nphys = 0; ReleaseSRWLockExclusive(&pool_lock); AcquireSRWLockExclusive(&gen_lock); for (uint64_t i = 0; i < GEN_CHUNKS; i++) free(gen_chunk[i]), gen_chunk[i] = NULL; ReleaseSRWLockExclusive(&gen_lock); }