mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-18 20:15:56 +00:00
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
744 lines
34 KiB
C
744 lines
34 KiB
C
// Vulkan memory of the M4 graphics layer: NVN memory pools as Vulkan
|
|
// buffers, the virtual-pool page table, GPU address resolution, page write generations for re-uploads, and the
|
|
// upload batch (command buffer, staging buffers, deferred destruction) that vk_texture.c records into.
|
|
//
|
|
// Behaviour ported from MacNativeGames games/totk/third_party/d2r_gfx/gfx/nvn/totk_vpool.c (request checks, unmap
|
|
// semantics, 64 KiB pages) and nvn_dirty.c (generation stamps: a texture is stale when a covered page carries a
|
|
// generation newer than its upload), @ b75be8cf (MIT, Copyright (c) 2026 jhaizhou; origin d2r-mobile); and from LAS
|
|
// Recomp runtime/nvn/vulkan/exec.c:2094-2131 (vk_pages_changed: every write-watch query resets the OS bits, so each
|
|
// query stamps the pages it saw with a new generation shared by all users) and textures.c:240-270 (deferred
|
|
// destruction) @ da35b83a (MIT, Copyright (c) 2026 Peiikai). MNG aliases host VM (mach_vm_remap) and protects pages
|
|
// (fault handler); here virtual pools are a GPU-side page table over device-local memory and dirty pages come from
|
|
// GetWriteWatch (mem_written_pages), so neither is needed.
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <windows.h>
|
|
|
|
#include "cfg.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "nvn_api.h"
|
|
#include "nvn_pool_index.h"
|
|
#include "vk.h"
|
|
|
|
// ---------------------------------------------------------------- pools
|
|
enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT };
|
|
|
|
typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page -> physical pool and offset
|
|
|
|
typedef struct PoolRec {
|
|
const pool_t *key;
|
|
int kind;
|
|
uint32_t flags;
|
|
uint64_t lo, size; // GPU address range of the pool
|
|
uint64_t import_lo, import_hi; // POOL_IMPORT: the imported host range (aligned outwards)
|
|
VkBuffer buf;
|
|
VkDeviceMemory mem; // POOL_IMPORT
|
|
VmaAllocation alloc; // POOL_PHYS
|
|
VPage *pt; // POOL_VIRT: size / 64 KiB entries
|
|
uint64_t npages;
|
|
volatile long used; // resolved at least once: the GPU may still read it at finalize
|
|
} PoolRec;
|
|
|
|
static SRWLOCK pool_lock = SRWLOCK_INIT;
|
|
// Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits
|
|
// stay valid while it is unchanged.
|
|
static uint64_t pool_gen = 1;
|
|
static pidx pool_index; // POOL_IMPORT, POOL_PHYS and POOL_VIRT records by GPU address
|
|
static PoolRec **phys_list; // physical pools by key, for MapVirtual requests
|
|
static int nphys, cap_phys;
|
|
static long n_imports, n_import_failures;
|
|
static PFN_vkGetMemoryHostPointerPropertiesEXT p_host_props;
|
|
|
|
static VkBufferUsageFlags pool_usage(const VkCtx *c) {
|
|
VkBufferUsageFlags u = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
|
|
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
|
|
if (c->has_transform_feedback)
|
|
u |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT;
|
|
if (c->has_conditional_rendering) u |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT;
|
|
return u;
|
|
}
|
|
|
|
static int memory_type(const VkCtx *c, uint32_t bits, VkMemoryPropertyFlags want) {
|
|
for (uint32_t i = 0; i < c->mem_props.memoryTypeCount; i++)
|
|
if ((bits >> i & 1) && (c->mem_props.memoryTypes[i].propertyFlags & want) == want) return (int)i;
|
|
for (uint32_t i = 0; i < c->mem_props.memoryTypeCount; i++)
|
|
if (bits >> i & 1) return (int)i;
|
|
return -1;
|
|
}
|
|
|
|
// CPU-visible storage: the guest pages themselves become the buffer's memory (VK_EXT_external_memory_host).
|
|
static int import_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
|
|
if (!p_host_props)
|
|
p_host_props = (PFN_vkGetMemoryHostPointerPropertiesEXT)vkGetDeviceProcAddr(c->device, "vkGetMemoryHostPointerPropertiesEXT");
|
|
uint64_t al = c->min_imported_host_alignment ? c->min_imported_host_alignment : 0x1000;
|
|
r->import_lo = addr & ~(al - 1);
|
|
r->import_hi = (addr + size + al - 1) & ~(al - 1);
|
|
uint64_t n = r->import_hi - r->import_lo;
|
|
VkMemoryHostPointerPropertiesEXT hp = {.sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};
|
|
VkResult res = p_host_props ? p_host_props(c->device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
|
|
(void *)(uintptr_t)r->import_lo, &hp)
|
|
: VK_ERROR_EXTENSION_NOT_PRESENT;
|
|
if (res != VK_SUCCESS || !hp.memoryTypeBits) {
|
|
rt_log("vk: pool %#llx+%#llx: host pointer not importable (%d)", (unsigned long long)addr, (unsigned long long)size, (int)res);
|
|
return 0;
|
|
}
|
|
VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
|
|
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT};
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = n,
|
|
.usage = pool_usage(c), .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
if ((res = vkCreateBuffer(c->device, &bi, NULL, &r->buf)) != VK_SUCCESS) {
|
|
rt_log("vk: pool %#llx: vkCreateBuffer for the import failed (%d)", (unsigned long long)addr, (int)res);
|
|
return 0;
|
|
}
|
|
VkMemoryRequirements req;
|
|
vkGetBufferMemoryRequirements(c->device, r->buf, &req);
|
|
int type = memory_type(c, req.memoryTypeBits & hp.memoryTypeBits,
|
|
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
|
|
if (type < 0 || req.size > n) {
|
|
rt_log("vk: pool %#llx: no memory type for the import (buffer %#x, host %#x, size %#llx of %#llx)",
|
|
(unsigned long long)addr, req.memoryTypeBits, hp.memoryTypeBits, (unsigned long long)req.size,
|
|
(unsigned long long)n);
|
|
vkDestroyBuffer(c->device, r->buf, NULL);
|
|
r->buf = VK_NULL_HANDLE;
|
|
return 0;
|
|
}
|
|
VkImportMemoryHostPointerInfoEXT im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
|
|
.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
|
|
.pHostPointer = (void *)(uintptr_t)r->import_lo};
|
|
VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = &im, .allocationSize = n,
|
|
.memoryTypeIndex = (uint32_t)type};
|
|
if ((res = vkAllocateMemory(c->device, &ai, NULL, &r->mem)) != VK_SUCCESS ||
|
|
(res = vkBindBufferMemory(c->device, r->buf, r->mem, 0)) != VK_SUCCESS) {
|
|
rt_log("vk: pool %#llx+%#llx: host memory import failed (%d)", (unsigned long long)addr, (unsigned long long)size, (int)res);
|
|
if (r->mem) vkFreeMemory(c->device, r->mem, NULL);
|
|
vkDestroyBuffer(c->device, r->buf, NULL);
|
|
r->buf = VK_NULL_HANDLE;
|
|
r->mem = VK_NULL_HANDLE;
|
|
return 0;
|
|
}
|
|
InterlockedIncrement(&n_imports);
|
|
return 1;
|
|
}
|
|
|
|
// Physical pool: device-local memory of the pool size that both buffers and optimal-tiling images can be bound to.
|
|
static int alloc_physical(VkCtx *c, PoolRec *r) {
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = r->size, .usage = pool_usage(c),
|
|
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
|
VkResult res = vkCreateBuffer(c->device, &bi, NULL, &r->buf);
|
|
if (res != VK_SUCCESS) { rt_log("vk: physical pool: vkCreateBuffer failed (%d)", (int)res); return 0; }
|
|
VkMemoryRequirements rb;
|
|
vkGetBufferMemoryRequirements(c->device, r->buf, &rb);
|
|
VkImageCreateInfo ii = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D,
|
|
.format = VK_FORMAT_R8G8B8A8_UNORM, .extent = {256, 256, 1}, .mipLevels = 1, .arrayLayers = 1,
|
|
.samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL,
|
|
.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT};
|
|
VkDeviceImageMemoryRequirements di = {.sType = VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS, .pCreateInfo = &ii};
|
|
VkMemoryRequirements2 ri = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2};
|
|
vkGetDeviceImageMemoryRequirements(c->device, &di, &ri);
|
|
uint32_t bits = rb.memoryTypeBits & ri.memoryRequirements.memoryTypeBits;
|
|
if (!bits) bits = rb.memoryTypeBits;
|
|
VkDeviceSize align = NVN_VPOOL_PAGE;
|
|
if (rb.alignment > align) align = rb.alignment;
|
|
if (ri.memoryRequirements.alignment > align) align = ri.memoryRequirements.alignment;
|
|
VkMemoryRequirements req = {.size = (rb.size + align - 1) & ~(align - 1), .alignment = align, .memoryTypeBits = bits};
|
|
VmaAllocationCreateInfo aci = {.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
|
|
if ((res = vmaAllocateMemory(c->vma, &req, &aci, &r->alloc, NULL)) != VK_SUCCESS ||
|
|
(res = vmaBindBufferMemory(c->vma, r->alloc, r->buf)) != VK_SUCCESS) {
|
|
rt_log("vk: physical pool of %#llx bytes: device-local allocation failed (%d)", (unsigned long long)r->size, (int)res);
|
|
if (r->alloc) vmaFreeMemory(c->vma, r->alloc);
|
|
vkDestroyBuffer(c->device, r->buf, NULL);
|
|
r->buf = VK_NULL_HANDLE;
|
|
r->alloc = NULL;
|
|
return 0;
|
|
}
|
|
if (vk_fix_zero()) { // Switch pool memory starts zeroed; once per pool, not per image placed in it
|
|
VkCommandBuffer cb = vk_upload_begin();
|
|
vkCmdFillBuffer(cb, r->buf, 0, VK_WHOLE_SIZE, 0);
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(cb, &di);
|
|
vk_upload_end();
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
int vk_fix_zero(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
const char *e = cfg_get("TOTK_FIX_ZERO");
|
|
on = !(e && !strcmp(e, "0"));
|
|
}
|
|
return on;
|
|
}
|
|
|
|
static void index_put(PoolRec *r) {
|
|
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
|
pidx_ent e = {.lo = r->lo, .hi = r->lo + r->size, .base = r->lo, .buf = r, .gpu = r->lo, .key = r->key};
|
|
pidx_remove(&pool_index, r->key, NULL); // an object initialized again without Finalize
|
|
if (!pidx_insert(&pool_index, &e)) rt_fatal("vk: out of memory for the pool index");
|
|
}
|
|
|
|
static PoolRec *phys_find(const pool_t *key) {
|
|
for (int i = 0; i < nphys; i++) if (phys_list[i]->key == key) return phys_list[i];
|
|
return NULL;
|
|
}
|
|
|
|
static void destroy_rec(VkCtx *c, PoolRec *r) {
|
|
if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL);
|
|
if (r->mem) vkFreeMemory(c->device, r->mem, NULL);
|
|
if (r->alloc) vmaFreeMemory(c->vma, r->alloc);
|
|
free(r->pt);
|
|
free(r);
|
|
}
|
|
|
|
int vk_pool_init(pool_t *p) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) { rt_log("vk: MemoryPoolInitialize before vk_init"); return 0; }
|
|
// The GPU never sees a GPU_NO_ACCESS pool, except a physical one (0x189) through the virtual pools that map it.
|
|
if ((p->flags & NVN_MEMORY_POOL_FLAGS_GPU_NO_ACCESS) && !(p->flags & NVN_MEMORY_POOL_FLAGS_PHYSICAL)) return 1;
|
|
PoolRec *r = calloc(1, sizeof *r);
|
|
if (!r) rt_fatal("vk: out of memory for a pool record");
|
|
r->key = p;
|
|
r->flags = p->flags;
|
|
r->lo = (uintptr_t)p->mem;
|
|
r->size = p->size;
|
|
int ok;
|
|
if (p->flags & NVN_MEMORY_POOL_FLAGS_VIRTUAL) {
|
|
r->kind = POOL_VIRT;
|
|
r->npages = p->size / NVN_VPOOL_PAGE;
|
|
r->pt = calloc(r->npages ? r->npages : 1, sizeof *r->pt);
|
|
ok = r->pt != NULL;
|
|
} else if (p->flags & NVN_MEMORY_POOL_FLAGS_PHYSICAL) {
|
|
r->kind = POOL_PHYS;
|
|
ok = alloc_physical(c, r);
|
|
} else {
|
|
r->kind = POOL_IMPORT;
|
|
ok = import_pool(c, r, (uintptr_t)p->mem, p->size);
|
|
if (!ok) {
|
|
// Not fatal for the game: the pool works for the CPU, and vk_mem_resolve reports its addresses as unbacked.
|
|
long n = InterlockedIncrement(&n_import_failures);
|
|
if (n <= 8) rt_log("vk: pool %p (flags %#x, %#llx bytes) has no GPU buffer (%ld so far)", (void *)p->mem,
|
|
p->flags, (unsigned long long)p->size, n);
|
|
r->buf = VK_NULL_HANDLE;
|
|
ok = 1;
|
|
}
|
|
}
|
|
if (!ok) { free(r->pt); free(r); return 0; }
|
|
AcquireSRWLockExclusive(&pool_lock);
|
|
index_put(r);
|
|
if (r->kind == POOL_PHYS) {
|
|
if (nphys == cap_phys) {
|
|
cap_phys = cap_phys ? cap_phys * 2 : 16;
|
|
phys_list = realloc(phys_list, (size_t)cap_phys * sizeof *phys_list);
|
|
if (!phys_list) rt_fatal("vk: out of memory for the physical pool list");
|
|
}
|
|
phys_list[nphys++] = r;
|
|
}
|
|
ReleaseSRWLockExclusive(&pool_lock);
|
|
return 1;
|
|
}
|
|
|
|
void vk_pool_fini(pool_t *p) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
pidx_ent e;
|
|
AcquireSRWLockExclusive(&pool_lock);
|
|
int had = pidx_remove(&pool_index, p, &e);
|
|
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
|
PoolRec *r = had ? e.buf : NULL;
|
|
if (r && r->kind == POOL_PHYS) {
|
|
for (int i = 0; i < nphys; i++)
|
|
if (phys_list[i] == r) { phys_list[i] = phys_list[--nphys]; break; }
|
|
for (int i = 0; i < pool_index.n; i++) { // virtual pages that still map it become unmapped
|
|
PoolRec *v = pool_index.e[i].buf;
|
|
if (v->kind != POOL_VIRT) continue;
|
|
for (uint64_t k = 0; k < v->npages; k++)
|
|
if (v->pt[k].phys == r) v->pt[k].phys = NULL;
|
|
}
|
|
}
|
|
ReleaseSRWLockExclusive(&pool_lock);
|
|
if (!r) return;
|
|
// Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the
|
|
// guest gets the storage back when this returns.
|
|
vk_texture_range_gone(r->lo, r->size, r->alloc);
|
|
if (r->used && r->kind != POOL_VIRT) {
|
|
vk_upload_flush(1);
|
|
vk_queue_lock();
|
|
vkQueueWaitIdle(c->gfx_queue);
|
|
vk_queue_unlock();
|
|
}
|
|
vk_mem_collect();
|
|
destroy_rec(c, r);
|
|
}
|
|
|
|
// Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c:54-67); apply them in order.
|
|
int vk_pool_map_virtual(pool_t *v, int n, const void *requests) {
|
|
const nvn_mapping_t *rq = requests;
|
|
AcquireSRWLockExclusive(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)v->mem, 0);
|
|
PoolRec *r = e && e->key == v ? e->buf : NULL;
|
|
int ok = r && r->kind == POOL_VIRT;
|
|
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
|
for (int i = 0; ok && i < n; i++) {
|
|
PoolRec *ph = rq[i].physical_pool ? phys_find((const pool_t *)(const void *)rq[i].physical_pool) : NULL;
|
|
if (rq[i].physical_pool && !ph) {
|
|
rt_log("vk: MapVirtual request %d names physical pool %p that has no Vulkan memory", i,
|
|
(const void *)rq[i].physical_pool);
|
|
ok = 0;
|
|
break;
|
|
}
|
|
uint64_t first = (uint64_t)rq[i].virtual_offset / NVN_VPOOL_PAGE, count = rq[i].size / NVN_VPOOL_PAGE;
|
|
if (first + count > r->npages) { ok = 0; break; }
|
|
for (uint64_t k = 0; k < count; k++)
|
|
r->pt[first + k] = (VPage){ph, ph ? (uint64_t)rq[i].physical_offset + k * NVN_VPOOL_PAGE : 0};
|
|
}
|
|
ReleaseSRWLockExclusive(&pool_lock);
|
|
if (!ok) rt_log("vk: MapVirtual on pool %p failed", (void *)v->mem);
|
|
return ok;
|
|
}
|
|
|
|
// Per-thread copies of resolved import and physical pools that overlap no other index entry (so pidx_find picks them for
|
|
// any range inside), valid while pool_gen is unchanged: the replay resolves a few pools thousands of times per frame.
|
|
typedef struct { uint64_t gen, lo, hi, base, end; VkBuffer buf; VmaAllocation alloc; } ResolveHit;
|
|
static _Thread_local ResolveHit rhit[4];
|
|
static _Thread_local unsigned rhit_next;
|
|
|
|
int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) {
|
|
memset(out, 0, sizeof *out);
|
|
uint64_t g = __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE), end = addr + size;
|
|
if (end >= addr)
|
|
for (int i = 0; i < 4; i++) {
|
|
const ResolveHit *h = &rhit[i];
|
|
if (h->gen != g || addr < h->lo || end > h->hi) continue;
|
|
out->buffer = h->buf;
|
|
out->offset = addr - h->base;
|
|
out->avail = h->end - addr;
|
|
out->alloc = h->alloc;
|
|
if (h->alloc) out->alloc_offset = out->offset;
|
|
return 1;
|
|
}
|
|
AcquireSRWLockShared(&pool_lock);
|
|
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)addr, size);
|
|
PoolRec *r = e ? e->buf : NULL;
|
|
int ok = 0;
|
|
if (r && (r->kind == POOL_IMPORT ? r->buf != VK_NULL_HANDLE : r->kind == POOL_PHYS)) {
|
|
int i = (int)(e - pool_index.e);
|
|
if ((i == 0 || pool_index.maxhi[i - 1] <= e->lo) && (i + 1 == pool_index.n || pool_index.e[i + 1].lo >= e->hi)) {
|
|
ResolveHit *h = &rhit[rhit_next++ & 3];
|
|
h->gen = __atomic_load_n(&pool_gen, __ATOMIC_RELAXED); // stable: changed only under the exclusive lock
|
|
h->lo = e->lo, h->hi = e->hi, h->buf = r->buf;
|
|
if (r->kind == POOL_IMPORT) h->base = r->import_lo, h->end = r->import_hi, h->alloc = NULL;
|
|
else h->base = r->lo, h->end = r->lo + r->size, h->alloc = r->alloc;
|
|
}
|
|
}
|
|
if (r && r->kind == POOL_IMPORT && r->buf) {
|
|
out->buffer = r->buf;
|
|
out->offset = addr - r->import_lo;
|
|
out->avail = r->import_hi - addr;
|
|
ok = 1;
|
|
} else if (r && r->kind == POOL_PHYS) {
|
|
out->buffer = r->buf;
|
|
out->offset = out->alloc_offset = addr - r->lo;
|
|
out->avail = r->size - out->offset;
|
|
out->alloc = r->alloc;
|
|
ok = 1;
|
|
} else if (r && r->kind == POOL_VIRT) {
|
|
uint64_t rel = addr - r->lo, k = rel / NVN_VPOOL_PAGE;
|
|
if (k < r->npages && r->pt[k].phys) {
|
|
PoolRec *ph = r->pt[k].phys;
|
|
uint64_t off = r->pt[k].off + rel % NVN_VPOOL_PAGE, end = r->pt[k].off + NVN_VPOOL_PAGE;
|
|
for (uint64_t j = k + 1; j < r->npages && r->pt[j].phys == ph && r->pt[j].off == end; j++) end += NVN_VPOOL_PAGE;
|
|
out->buffer = ph->buf;
|
|
out->offset = out->alloc_offset = off;
|
|
out->avail = end - off;
|
|
out->alloc = ph->alloc;
|
|
ok = out->avail >= size;
|
|
if (ok && !ph->used) InterlockedExchange(&ph->used, 1); // read first: the flag is only ever set
|
|
}
|
|
}
|
|
if (ok && !r->used) InterlockedExchange(&r->used, 1);
|
|
ReleaseSRWLockShared(&pool_lock);
|
|
if (!ok) memset(out, 0, sizeof *out);
|
|
return ok;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- page write generations
|
|
// Two-level table over the guest address space (below 2^39): 256 MiB chunks of 65,536 page generations, allocated
|
|
// when a page of the chunk is first stamped. 0 = never stamped.
|
|
#define GEN_CHUNK_SHIFT 28
|
|
#define GEN_PAGE_SHIFT 12
|
|
#define GEN_CHUNKS (RT_ADDRESS_LIMIT >> GEN_CHUNK_SHIFT)
|
|
#define GEN_PER_CHUNK (1u << (GEN_CHUNK_SHIFT - GEN_PAGE_SHIFT))
|
|
static uint32_t *gen_chunk[GEN_CHUNKS];
|
|
static uint32_t gen_now; // last generation handed out
|
|
static SRWLOCK gen_lock = SRWLOCK_INIT;
|
|
static uint64_t gen_pages[1024];
|
|
|
|
static uint32_t *gen_slot(uint64_t page_addr, int create) {
|
|
uint64_t ci = page_addr >> GEN_CHUNK_SHIFT;
|
|
if (ci >= GEN_CHUNKS) return NULL;
|
|
if (!gen_chunk[ci]) {
|
|
if (!create) return NULL;
|
|
gen_chunk[ci] = calloc(GEN_PER_CHUNK, sizeof(uint32_t));
|
|
if (!gen_chunk[ci]) rt_fatal("vk: out of memory for page generations");
|
|
}
|
|
return &gen_chunk[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)];
|
|
}
|
|
|
|
static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) {
|
|
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) {
|
|
uint32_t *s = gen_slot(a, 1);
|
|
if (s) *s = g;
|
|
}
|
|
}
|
|
|
|
void vk_guest_written(uint64_t addr, uint64_t size) {
|
|
if (!size || nvn_gpu_only(addr)) return;
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
AcquireSRWLockExclusive(&gen_lock);
|
|
stamp_range(lo, hi, ++gen_now);
|
|
ReleaseSRWLockExclusive(&gen_lock);
|
|
}
|
|
|
|
int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now) {
|
|
if (!size) { *now = since; return 0; }
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
AcquireSRWLockExclusive(&gen_lock);
|
|
uint32_t g = ++gen_now;
|
|
int tracked = 1;
|
|
for (;;) {
|
|
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
|
if (n == UINT64_MAX) { tracked = 0; break; }
|
|
for (uint64_t k = 0; k < n; k++) {
|
|
uint32_t *s = gen_slot(gen_pages[k], 1);
|
|
if (s) *s = g;
|
|
}
|
|
if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call
|
|
}
|
|
int changed = 0;
|
|
for (uint64_t a = lo; tracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) {
|
|
uint32_t *s = gen_slot(a, 0);
|
|
if (s && *s > since) changed = 1;
|
|
}
|
|
ReleaseSRWLockExclusive(&gen_lock);
|
|
*now = g;
|
|
return tracked ? changed : -1;
|
|
}
|
|
|
|
// Write-watch snapshots per submission: the guest address space in 2 MiB regions, each queried at most once per epoch
|
|
// (a guest submission), at the first check that covers it. A query costs about 1.3 us plus 0.55 us per MiB, so one per
|
|
// region instead of one per texture. The snapshot only adds observations to the generation table: a page written
|
|
// before it carries a generation newer than any check before it, and a page written after it keeps its OS bit until
|
|
// the next query over it (at the latest the next epoch's snapshot).
|
|
#define SNAP_SHIFT 21
|
|
#define SNAP_REGIONS (RT_ADDRESS_LIMIT >> SNAP_SHIFT)
|
|
static uint32_t snap_at[SNAP_REGIONS]; // epoch of the region's last snapshot; 0 never
|
|
static uint8_t snap_untracked[SNAP_REGIONS];
|
|
static uint64_t n_snap_checks, n_snap_calls, n_snap_untracked;
|
|
|
|
int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) {
|
|
if (!size) { *now = since; return 0; }
|
|
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
|
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
|
if (!epoch || r1 >= SNAP_REGIONS) return vk_mem_changed(addr, size, since, now);
|
|
int excl = 0;
|
|
AcquireSRWLockShared(&gen_lock);
|
|
for (uint64_t r = r0; r <= r1; r++)
|
|
if (snap_at[r] != epoch) { excl = 1; break; }
|
|
if (excl) {
|
|
ReleaseSRWLockShared(&gen_lock);
|
|
AcquireSRWLockExclusive(&gen_lock);
|
|
uint32_t g = ++gen_now;
|
|
for (uint64_t r = r0; r <= r1;) {
|
|
if (snap_at[r] == epoch) { r++; continue; }
|
|
uint64_t k = r; // one query over a run of regions not seen in this epoch
|
|
while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++;
|
|
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
|
|
int tracked = 1;
|
|
for (;;) {
|
|
uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
|
if (n == UINT64_MAX) { tracked = 0; break; }
|
|
for (uint64_t j = 0; j < n; j++) {
|
|
uint32_t *s = gen_slot(gen_pages[j], 1);
|
|
if (s) *s = g;
|
|
}
|
|
if (n < sizeof gen_pages / sizeof *gen_pages) break;
|
|
}
|
|
n_snap_calls++;
|
|
for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
|
|
r = k + 1;
|
|
}
|
|
}
|
|
int untracked = 0, changed = 0;
|
|
for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r];
|
|
for (uint64_t a = lo; !untracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) {
|
|
uint32_t *s = gen_slot(a, 0);
|
|
if (s && *s > since) changed = 1;
|
|
}
|
|
*now = gen_now; // every observation so far is in the table
|
|
__atomic_add_fetch(&n_snap_checks, 1, __ATOMIC_RELAXED);
|
|
if (untracked) __atomic_add_fetch(&n_snap_untracked, 1, __ATOMIC_RELAXED);
|
|
if (excl) ReleaseSRWLockExclusive(&gen_lock);
|
|
else ReleaseSRWLockShared(&gen_lock);
|
|
return untracked ? -1 : changed;
|
|
}
|
|
|
|
void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) {
|
|
*checks = __atomic_exchange_n(&n_snap_checks, 0, __ATOMIC_RELAXED);
|
|
*calls = __atomic_exchange_n(&n_snap_calls, 0, __ATOMIC_RELAXED);
|
|
*untracked = __atomic_exchange_n(&n_snap_untracked, 0, __ATOMIC_RELAXED);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- upload batches and deferred destruction
|
|
typedef struct { VkBuffer buf; VmaAllocation alloc; } Staging;
|
|
typedef struct {
|
|
VkCommandBuffer cb;
|
|
VkFence fence;
|
|
uint64_t seq;
|
|
Staging *stg;
|
|
int nstg, cap;
|
|
} Batch;
|
|
|
|
enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA };
|
|
typedef struct { int kind; uint64_t handle; VmaAllocation alloc; uint64_t seq, tl; } Dead;
|
|
|
|
static CRITICAL_SECTION cmd_lock;
|
|
static INIT_ONCE cmd_once = INIT_ONCE_STATIC_INIT;
|
|
static VkCommandPool cmd_pool;
|
|
static Batch open_b;
|
|
static int open_active;
|
|
static Batch *inflight;
|
|
static int ninflight, cap_inflight;
|
|
static Batch *spare; // completed batches whose command buffer and fence are reused
|
|
static int nspare, cap_spare;
|
|
static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq)
|
|
static Dead *dead;
|
|
static int ndead, cap_dead;
|
|
static uint64_t tl_noted; // newest timeline value T6 reported submitted
|
|
|
|
static BOOL CALLBACK cmd_init(PINIT_ONCE o, PVOID p, PVOID *ctx) {
|
|
(void)o, (void)p, (void)ctx;
|
|
InitializeCriticalSection(&cmd_lock);
|
|
return TRUE;
|
|
}
|
|
|
|
static void grow(void **arr, int *cap, size_t elem) {
|
|
*cap = *cap ? *cap * 2 : 16;
|
|
void *n = realloc(*arr, (size_t)*cap * elem);
|
|
if (!n) rt_fatal("vk: out of memory (upload bookkeeping)");
|
|
*arr = n;
|
|
}
|
|
|
|
static void free_staging(VkCtx *c, Batch *b) {
|
|
for (int i = 0; i < b->nstg; i++) vmaDestroyBuffer(c->vma, b->stg[i].buf, b->stg[i].alloc);
|
|
b->nstg = 0;
|
|
}
|
|
|
|
// Batches complete in submission order (one queue): retire from the front while their fences are signalled.
|
|
static void retire(VkCtx *c, int wait_all) {
|
|
int k = 0;
|
|
for (; k < ninflight; k++) {
|
|
Batch *b = &inflight[k];
|
|
if (wait_all) vkWaitForFences(c->device, 1, &b->fence, VK_TRUE, UINT64_MAX);
|
|
else if (vkGetFenceStatus(c->device, b->fence) != VK_SUCCESS) break;
|
|
free_staging(c, b);
|
|
done_seq = b->seq;
|
|
if (nspare == cap_spare) grow((void **)&spare, &cap_spare, sizeof *spare);
|
|
spare[nspare++] = *b;
|
|
}
|
|
memmove(inflight, inflight + k, (size_t)(ninflight - k) * sizeof *inflight);
|
|
ninflight -= k;
|
|
}
|
|
|
|
static void collect_locked(VkCtx *c) {
|
|
retire(c, 0);
|
|
uint64_t tl = 0;
|
|
if (c->timeline) vkGetSemaphoreCounterValue(c->device, c->timeline, &tl);
|
|
int k = 0;
|
|
for (int i = 0; i < ndead; i++) {
|
|
Dead *d = &dead[i];
|
|
if (d->seq > done_seq || d->tl > tl) { dead[k++] = *d; continue; }
|
|
switch (d->kind) {
|
|
case DEAD_IMAGE: vkDestroyImage(c->device, (VkImage)(uintptr_t)d->handle, NULL); break;
|
|
case DEAD_IMAGE_VMA: vmaDestroyImage(c->vma, (VkImage)(uintptr_t)d->handle, d->alloc); break;
|
|
case DEAD_VIEW: vkDestroyImageView(c->device, (VkImageView)(uintptr_t)d->handle, NULL); break;
|
|
case DEAD_BUFFER_VMA: vmaDestroyBuffer(c->vma, (VkBuffer)(uintptr_t)d->handle, d->alloc); break;
|
|
}
|
|
}
|
|
ndead = k;
|
|
}
|
|
|
|
VkCommandBuffer vk_upload_begin(void) {
|
|
InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL);
|
|
EnterCriticalSection(&cmd_lock);
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) rt_fatal("vk: upload before vk_init");
|
|
if (open_active) return open_b.cb;
|
|
if (!cmd_pool) {
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
|
|
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
|
|
.queueFamilyIndex = c->gfx_family};
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: upload command pool");
|
|
}
|
|
retire(c, 0);
|
|
memset(&open_b, 0, sizeof open_b);
|
|
if (nspare) {
|
|
Batch s = spare[--nspare];
|
|
open_b.cb = s.cb, open_b.fence = s.fence, open_b.stg = s.stg, open_b.cap = s.cap;
|
|
vkResetCommandBuffer(open_b.cb, 0);
|
|
vkResetFences(c->device, 1, &open_b.fence);
|
|
} else {
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
if (vkAllocateCommandBuffers(c->device, &ai, &open_b.cb) != VK_SUCCESS ||
|
|
vkCreateFence(c->device, &fi, NULL, &open_b.fence) != VK_SUCCESS)
|
|
rt_fatal("vk: upload command buffer");
|
|
}
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(open_b.cb, &bi);
|
|
open_b.seq = ++next_seq;
|
|
open_active = 1;
|
|
return open_b.cb;
|
|
}
|
|
|
|
void vk_upload_end(void) { LeaveCriticalSection(&cmd_lock); }
|
|
|
|
void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
|
|
VkCtx *c = vk_ctx();
|
|
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16,
|
|
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
|
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
|
VmaAllocationInfo info;
|
|
VkBuffer b;
|
|
VmaAllocation a;
|
|
VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info);
|
|
if (r != VK_SUCCESS) rt_fatal("vk: staging buffer of %llu bytes failed (%d)", (unsigned long long)size, (int)r);
|
|
*buf = b;
|
|
if (alloc) *alloc = a; // the caller destroys it (vmaDestroyBuffer) after vk_upload_flush(1)
|
|
else {
|
|
if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg);
|
|
open_b.stg[open_b.nstg++] = (Staging){b, a};
|
|
}
|
|
return info.pMappedData;
|
|
}
|
|
|
|
uint64_t vk_upload_flush(int wait) {
|
|
InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL);
|
|
EnterCriticalSection(&cmd_lock);
|
|
VkCtx *c = vk_ctx();
|
|
uint64_t seq = next_seq;
|
|
if (c && open_active) {
|
|
for (int i = 0; i < open_b.nstg; i++) vmaFlushAllocation(c->vma, open_b.stg[i].alloc, 0, VK_WHOLE_SIZE);
|
|
vkEndCommandBuffer(open_b.cb);
|
|
VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &open_b.cb};
|
|
vk_queue_lock();
|
|
VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence);
|
|
vk_queue_unlock();
|
|
if (r != VK_SUCCESS) rt_fatal("vk: upload submit failed (%d)", (int)r);
|
|
if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight);
|
|
inflight[ninflight++] = open_b;
|
|
open_active = 0;
|
|
}
|
|
if (c && wait) {
|
|
while (ninflight && done_seq < seq) {
|
|
vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX);
|
|
retire(c, 0);
|
|
}
|
|
}
|
|
if (c) collect_locked(c);
|
|
LeaveCriticalSection(&cmd_lock);
|
|
return seq;
|
|
}
|
|
|
|
static void defer(int kind, uint64_t handle, VmaAllocation alloc) {
|
|
InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL);
|
|
EnterCriticalSection(&cmd_lock);
|
|
if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead);
|
|
// The open batch may record into the object: it counts as the newest user.
|
|
dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl_noted};
|
|
LeaveCriticalSection(&cmd_lock);
|
|
}
|
|
|
|
void vk_defer_image(VkImage img, VmaAllocation alloc) {
|
|
if (img) defer(alloc ? DEAD_IMAGE_VMA : DEAD_IMAGE, (uint64_t)(uintptr_t)img, alloc);
|
|
}
|
|
void vk_defer_view(VkImageView view) { if (view) defer(DEAD_VIEW, (uint64_t)(uintptr_t)view, NULL); }
|
|
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) {
|
|
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
|
|
}
|
|
|
|
void vk_mem_note_submit(uint64_t timeline_value) {
|
|
InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL);
|
|
EnterCriticalSection(&cmd_lock);
|
|
if (timeline_value > tl_noted) tl_noted = timeline_value;
|
|
LeaveCriticalSection(&cmd_lock);
|
|
}
|
|
|
|
void vk_mem_collect(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
InitOnceExecuteOnce(&cmd_once, cmd_init, NULL, NULL);
|
|
EnterCriticalSection(&cmd_lock);
|
|
collect_locked(c);
|
|
LeaveCriticalSection(&cmd_lock);
|
|
}
|
|
|
|
void vk_mem_stats(VkMemStats *s) {
|
|
memset(s, 0, sizeof *s);
|
|
AcquireSRWLockShared(&pool_lock);
|
|
for (int i = 0; i < pool_index.n; i++) {
|
|
const PoolRec *r = pool_index.e[i].buf;
|
|
if (r->kind == POOL_IMPORT) s->imported_pools++;
|
|
else if (r->kind == POOL_PHYS) s->physical_pools++;
|
|
else s->virtual_pools++;
|
|
}
|
|
ReleaseSRWLockShared(&pool_lock);
|
|
s->import_failures = (uint64_t)n_import_failures;
|
|
s->upload_batches = next_seq;
|
|
}
|
|
|
|
void vk_mem_shutdown(void) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return;
|
|
vk_upload_flush(1);
|
|
vk_queue_lock();
|
|
vkDeviceWaitIdle(c->device);
|
|
vk_queue_unlock();
|
|
vk_texture_shutdown();
|
|
EnterCriticalSection(&cmd_lock);
|
|
retire(c, 1);
|
|
uint64_t keep = tl_noted;
|
|
tl_noted = 0;
|
|
for (int i = 0; i < ndead; i++) dead[i].seq = 0, dead[i].tl = 0; // the device is idle
|
|
collect_locked(c);
|
|
tl_noted = keep;
|
|
for (int i = 0; i < nspare; i++) {
|
|
vkDestroyFence(c->device, spare[i].fence, NULL);
|
|
free(spare[i].stg);
|
|
}
|
|
nspare = 0;
|
|
if (cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
|
|
cmd_pool = VK_NULL_HANDLE;
|
|
LeaveCriticalSection(&cmd_lock);
|
|
AcquireSRWLockExclusive(&pool_lock);
|
|
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
|
|
pool_index.n = 0;
|
|
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
|
nphys = 0;
|
|
ReleaseSRWLockExclusive(&pool_lock);
|
|
AcquireSRWLockExclusive(&gen_lock);
|
|
for (uint64_t i = 0; i < GEN_CHUNKS; i++) free(gen_chunk[i]), gen_chunk[i] = NULL;
|
|
ReleaseSRWLockExclusive(&gen_lock);
|
|
}
|