mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 20:25:57 +00:00
Bind uniform blocks from a copy taken when the submission starts recording
TotK rewrites per-object uniform blocks on other threads soon after it submits the frame that reads them, and the replay records for milliseconds after the submit call, so late draws could read another object's matrix (phantom rocks, a flat far field). The guard in nvn_api_core.c obs_release_read only knows 1.2.1's producer. runtime/vk_usnap.c copies every uniform range of imported pools that the submission binds into an arena, after MacNativeGames' TOTK_UBO_SNAPSHOT, and ubo_info binds the copy; ranges a submission still in flight writes are left out, and GPU writes within the submission drop the copies they touch. TOTK_UBO_SNAPSHOT=0 binds live memory; =ab alternates. Town on an RX 9070 (in-launch A/B, 70 s): 2235 ranges, 1.9 MiB per submission in 0.49 ms; interval +0.02 ms, replay +0.19 ms. The sort of the ranges was half of the cost with qsort (0.53 ms for about 10,000 bindings) and is a radix sort now. fight2 interval within noise. The four probes passed on 1.4.3. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c2943970cc
commit
3af201b43b
6 files changed
+329
-5
No files matched your search
+1
-1
@@ -46,7 +46,7 @@ static const char *const known[] = {
|
||||
"REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
|
||||
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
|
||||
"SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "SWKBD_ANSWER",
|
||||
"SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
|
||||
"SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
|
||||
"VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
|
||||
"WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE",
|
||||
};
|
||||
|
||||
@@ -162,6 +162,14 @@ void vk_mem_pick(VkMemSpan *sp);
|
||||
#define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0)
|
||||
#define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0)
|
||||
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size);
|
||||
// Uniform blocks as of the submit call (vk_usnap.c): vk_exec_record copies the uniform ranges a submission binds before
|
||||
// it records, ubo bindings take the copy (0: not copied, bind memory), GPU writes of the replay drop the copies they touch.
|
||||
void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence);
|
||||
void vk_usnap_end(uint64_t fence);
|
||||
int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out);
|
||||
void vk_usnap_drop(uint64_t addr, uint64_t n); // the replay makes the GPU write [addr, addr+n) of guest memory
|
||||
uint64_t vk_usnap_gen(void); // changes whenever a binding may resolve differently
|
||||
void vk_usnap_shutdown(void); // vk_exec_draw_shutdown: frees the arenas
|
||||
void vk_mem_mirror_begin(void);
|
||||
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence);
|
||||
int vk_mem_mirror_pending(void);
|
||||
|
||||
+2
-2
@@ -745,7 +745,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
|
||||
split_draws = part_start = 0;
|
||||
part_t0 = kern_now_ns();
|
||||
x_here = 1;
|
||||
if (n) vk_mem_mirror_begin();
|
||||
if (n) vk_mem_mirror_begin(), vk_usnap_begin(handles, n, fence);
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i);
|
||||
walk(ex, nvn_op_first(handles[i]), 0);
|
||||
@@ -753,7 +753,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
|
||||
vk_exec_pass_end(ex);
|
||||
x_here = 0;
|
||||
if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes
|
||||
if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence); // VRAM mirror bytes the GPU wrote go back to guest memory
|
||||
if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory
|
||||
if (ex->nviews) {
|
||||
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
|
||||
if (!l) rt_fatal("vk: out of memory");
|
||||
|
||||
@@ -386,6 +386,7 @@ void vk_exec_draw_shutdown(void) {
|
||||
for (int i = 0; i < nspare_pool; i++) vkDestroyDescriptorPool(c->device, spare_pool[i], NULL);
|
||||
for (int i = 0; i < nspare_chunk; i++) vmaDestroyBuffer(c->vma, spare_chunk[i].buf, spare_chunk[i].alloc);
|
||||
nspare_pool = nspare_chunk = 0;
|
||||
vk_usnap_shutdown();
|
||||
vkDestroyBufferView(c->device, Z.zview, NULL);
|
||||
vmaDestroyBuffer(c->vma, Z.zbuf, Z.zalloc);
|
||||
memset(&Z, 0, sizeof Z);
|
||||
@@ -775,7 +776,7 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align,
|
||||
if (sp.offset % align == 0) {
|
||||
VK_MEM_USE(sp);
|
||||
*out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n};
|
||||
if (written) vk_mem_gpu_wrote(&sp, n);
|
||||
if (written) vk_mem_gpu_wrote(&sp, n), vk_usnap_drop(addr, n);
|
||||
count_use(sp.cls, use, n);
|
||||
VK_MEM_BOUND(sp);
|
||||
return 1;
|
||||
@@ -878,6 +879,10 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) {
|
||||
}
|
||||
} else if (slot >= 3 && slot - 3 < 18) {
|
||||
const p_bind_buf *b = &ex->ubo[st][slot - 3];
|
||||
if (b->size && vk_usnap_find(b->addr, b->size < max ? b->size : max, align, &bi)) { // as of the submit call
|
||||
count_use(VK_MCLS_SCRATCH, VK_MUSE_UBO, bi.range);
|
||||
return bi;
|
||||
}
|
||||
if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, 0, &bi)) return bi;
|
||||
}
|
||||
return zero_info(max);
|
||||
@@ -1420,7 +1425,7 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) {
|
||||
PUT(ex->prog->serial);
|
||||
PUT((uint64_t)vk_texture_slots_gen() << 32 | vk_texture_epoch());
|
||||
PUT((uint64_t)vk_sampler_gen() << 32 | (uint32_t)ex->nfbl);
|
||||
PUT(vk_mem_gen());
|
||||
PUT(vk_mem_gen() + vk_usnap_gen()); // both only grow
|
||||
for (int i = 0; i < ex->nfbl && i < 9; i++) PUT((uintptr_t)ex->fbl_img[i]);
|
||||
if (l->sup) {
|
||||
float in[3];
|
||||
@@ -2125,6 +2130,7 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) {
|
||||
VkBufferCopy cp = {o, sp.offset, u->size};
|
||||
vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp);
|
||||
vk_mem_gpu_wrote(&sp, u->size);
|
||||
vk_usnap_drop(u->dst, u->size);
|
||||
ex->recorded++;
|
||||
vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size);
|
||||
note_written(ex, u->dst, u->size);
|
||||
|
||||
@@ -71,6 +71,7 @@ static void guest_written_cb(void *arg) {
|
||||
|
||||
// The GPU wrote [addr, addr+size) of CPU-visible memory: tell the texture cache once the fence is reached.
|
||||
static void note_gpu_write(VkExec *ex, uint64_t addr, uint64_t size) {
|
||||
vk_usnap_drop(addr, size);
|
||||
if (nvn_gpu_only(addr)) return;
|
||||
Range *r = malloc(sizeof *r);
|
||||
if (!r) rt_fatal("vk: out of memory");
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
// Uniform blocks as of the submit call. NVN starts the GPU when nvnQueueSubmitCommands returns and the GPU reads the
|
||||
// constants soon after; here the replay records for several milliseconds, and the GPU reads a pool's bytes when it
|
||||
// runs the part that uses them (the VRAM mirror's sync copies run then, vk_memory.c "VRAM mirror"). Meanwhile the game,
|
||||
// having handed the frame over, rewrites per-object uniform blocks on other threads with no fence: late draws read
|
||||
// another object's matrix (phantom rocks, a flat far field; TotK 1.2.1's producer f_70008c6460, 14-24 ms after the
|
||||
// submit). So at the start of a submission's recording every uniform range it binds is copied into an arena of its own,
|
||||
// and the uniform bindings of the replay use the copy.
|
||||
// - Only ranges of imported pools (memory the CPU writes) are copied, aligned out to 256 bytes (the largest
|
||||
// minUniformBufferOffsetAlignment) and at most 64 KiB from the bound address, overlapping ones merged.
|
||||
// - A range the GPU wrote in a submission that has not completed is left out: its bytes reach guest memory only with
|
||||
// that submission's write-back, the mirror holds them until then.
|
||||
// - A GPU write within the submission (UpdateUniformBuffer, a storage buffer a shader writes, a buffer copy or fill,
|
||||
// an image-to-buffer copy: vk_usnap_drop) drops the copies it touches from then on, so later draws read the result.
|
||||
// - The arena goes back to a free list once the submission's fence is done (completion thread).
|
||||
// The design follows MacNativeGames' TOTK_UBO_SNAPSHOT (games/totk/third_party/d2r_gfx/gfx/nvn/nvn_mtl.m usnap_take,
|
||||
// usnap_drop, @ ffd8d10, MIT, Copyright (c) 2026 jhaizhou); the code is ours.
|
||||
// The copy is taken where vk_exec_record starts: at the submit call when the replay runs on the submitting thread, later
|
||||
// with TOTK_REPLAY_ASYNC. TOTK_UBO_SNAPSHOT=0 binds live memory as before; =ab alternates (TOTK_AB).
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <windows.h>
|
||||
|
||||
#include "host.h"
|
||||
#include "nvn_api.h"
|
||||
#include "vk.h"
|
||||
|
||||
#define ALIGN 256u
|
||||
#define MAX_RANGE (64u << 10)
|
||||
#define ARENA_MIN (4u << 20)
|
||||
#define CALL_DEPTH 16
|
||||
|
||||
typedef struct Arena {
|
||||
struct Arena *next;
|
||||
VkBuffer buf;
|
||||
VmaAllocation alloc;
|
||||
uint8_t *map;
|
||||
uint64_t size;
|
||||
} Arena;
|
||||
|
||||
typedef struct { uint64_t lo, hi; } Rng;
|
||||
typedef struct { uint64_t lo, hi, off; int dropped; } Reg;
|
||||
typedef struct { uint64_t lo, hi, fence; } Wrote;
|
||||
|
||||
static struct {
|
||||
int mode; // rt_knob: 0 off, 1 on, 2 ab
|
||||
int on; // this submission has a snapshot
|
||||
uint64_t fence, gen;
|
||||
Arena *arena;
|
||||
Rng *col; // collected ranges
|
||||
int ncol, ccol;
|
||||
Reg *reg; // merged, sorted by lo, disjoint
|
||||
int nreg, creg;
|
||||
Wrote *wr; // GPU writes of submissions not known to be complete
|
||||
int nwr, cwr;
|
||||
// since the last summary
|
||||
uint64_t subs, regions, bytes, copy_ns, hits, drops, inflight, refused, off_subs;
|
||||
} U;
|
||||
|
||||
static Arena *free_arenas;
|
||||
static SRWLOCK free_lock = SRWLOCK_INIT;
|
||||
|
||||
static void *grow(void *p, int *cap, size_t elem) {
|
||||
int n = *cap ? *cap * 2 : 1024;
|
||||
void *q = realloc(p, (size_t)n * elem);
|
||||
if (!q) rt_fatal("vk_usnap: out of memory");
|
||||
*cap = n;
|
||||
return q;
|
||||
}
|
||||
|
||||
static Arena *arena_get(uint64_t need) {
|
||||
AcquireSRWLockExclusive(&free_lock);
|
||||
Arena **pp = &free_arenas, *a = NULL;
|
||||
for (; *pp; pp = &(*pp)->next)
|
||||
if ((*pp)->size >= need) {
|
||||
a = *pp, *pp = a->next;
|
||||
break;
|
||||
}
|
||||
ReleaseSRWLockExclusive(&free_lock);
|
||||
if (a) return a;
|
||||
a = calloc(1, sizeof *a);
|
||||
if (!a) rt_fatal("vk_usnap: out of memory");
|
||||
a->size = need < ARENA_MIN ? ARENA_MIN : (need + (1u << 20) - 1) & ~(uint64_t)((1u << 20) - 1);
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = a->size,
|
||||
.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT};
|
||||
// Device-local through the BAR when the device has it (the GPU reads constants on every draw), else host memory.
|
||||
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
||||
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
|
||||
VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
||||
VmaAllocationInfo info;
|
||||
VkResult res = vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &a->buf, &a->alloc, &info);
|
||||
if (res != VK_SUCCESS) {
|
||||
rt_log("vk_usnap: arena of %llu bytes failed (%d): this submission binds live memory",
|
||||
(unsigned long long)a->size, (int)res);
|
||||
free(a);
|
||||
return NULL;
|
||||
}
|
||||
a->map = info.pMappedData;
|
||||
rt_log("vk_usnap: new arena of %.1f MiB", a->size / 1048576.0);
|
||||
return a;
|
||||
}
|
||||
|
||||
static void arena_done(void *arg) { // completion thread: the GPU has read it
|
||||
Arena *a = arg;
|
||||
AcquireSRWLockExclusive(&free_lock);
|
||||
a->next = free_arenas, free_arenas = a;
|
||||
ReleaseSRWLockExclusive(&free_lock);
|
||||
}
|
||||
|
||||
static void collect(const op_hdr *h, int depth) {
|
||||
for (;;) {
|
||||
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_)
|
||||
rt_fatal("vk_usnap: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size);
|
||||
const void *a = nvn_op_payload(h);
|
||||
if (h->op == OP_END) return;
|
||||
if (h->op == OP_JUMP) {
|
||||
h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target;
|
||||
continue;
|
||||
}
|
||||
if (h->op == OP_CALL && depth < CALL_DEPTH) collect(nvn_op_first(((const p_jump *)a)->target), depth + 1);
|
||||
if (h->op == OP_BIND_UBO) {
|
||||
const p_bind_buf *b = a;
|
||||
if (b->addr && b->size) {
|
||||
if (U.ncol == U.ccol) U.col = grow(U.col, &U.ccol, sizeof *U.col);
|
||||
uint64_t n = b->size < MAX_RANGE ? b->size : MAX_RANGE;
|
||||
U.col[U.ncol++] = (Rng){b->addr & ~(uint64_t)(ALIGN - 1), b->addr + n};
|
||||
}
|
||||
}
|
||||
h = nvn_op_next(h);
|
||||
}
|
||||
}
|
||||
|
||||
static int by_lo(const void *a, const void *b) {
|
||||
uint64_t x = ((const Rng *)a)->lo, y = ((const Rng *)b)->lo;
|
||||
return x < y ? -1 : x > y;
|
||||
}
|
||||
static int wr_by_lo(const void *a, const void *b) {
|
||||
uint64_t x = ((const Wrote *)a)->lo, y = ((const Wrote *)b)->lo;
|
||||
return x < y ? -1 : x > y;
|
||||
}
|
||||
|
||||
// Sorts the collected ranges by lo. A town frame binds about 10,000 uniform ranges and qsort took half a millisecond
|
||||
// of the copy's 0.96 ms on the game thread, so they go through an LSD radix sort on (lo - lowest) / ALIGN, three
|
||||
// passes of 11 bits; qsort is left for a span that does not fit in 33 bits (2 TiB).
|
||||
#define RADIX_BITS 11
|
||||
static void sort_col(void) {
|
||||
static Rng *tmp;
|
||||
static int ctmp;
|
||||
int n = U.ncol;
|
||||
if (n < 2) return;
|
||||
uint64_t lo = UINT64_MAX, hi = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (U.col[i].lo < lo) lo = U.col[i].lo;
|
||||
if (U.col[i].lo > hi) hi = U.col[i].lo;
|
||||
}
|
||||
if ((hi - lo) / ALIGN >> (3 * RADIX_BITS)) {
|
||||
qsort(U.col, (size_t)n, sizeof *U.col, by_lo);
|
||||
return;
|
||||
}
|
||||
if (ctmp < n) {
|
||||
free(tmp);
|
||||
ctmp = U.ccol;
|
||||
if (!(tmp = malloc((size_t)ctmp * sizeof *tmp))) rt_fatal("vk_usnap: out of memory");
|
||||
}
|
||||
Rng *src = U.col, *dst = tmp;
|
||||
for (int pass = 0; pass < 3; pass++) {
|
||||
int shift = pass * RADIX_BITS, count[1 << RADIX_BITS] = {0};
|
||||
for (int i = 0; i < n; i++) count[(src[i].lo - lo) / ALIGN >> shift & ((1 << RADIX_BITS) - 1)]++;
|
||||
for (int k = 0, sum = 0; k < 1 << RADIX_BITS; k++) {
|
||||
int c = count[k];
|
||||
count[k] = sum, sum += c;
|
||||
}
|
||||
for (int i = 0; i < n; i++) dst[count[(src[i].lo - lo) / ALIGN >> shift & ((1 << RADIX_BITS) - 1)]++] = src[i];
|
||||
Rng *t = src;
|
||||
src = dst, dst = t;
|
||||
}
|
||||
if (src != U.col) memcpy(U.col, src, (size_t)n * sizeof *src); // three passes end in tmp
|
||||
}
|
||||
|
||||
// Keeps the GPU writes of submissions that are not done, merged into disjoint ranges sorted by lo (the terrain rewrites
|
||||
// one block before every draw: hundreds of equal entries), each with the newest fence among its parts.
|
||||
static void wr_prune(void) {
|
||||
uint64_t done = vk_sync_fence_done();
|
||||
int n = 0;
|
||||
for (int i = 0; i < U.nwr; i++)
|
||||
if (U.wr[i].fence > done) U.wr[n++] = U.wr[i];
|
||||
qsort(U.wr, (size_t)n, sizeof *U.wr, wr_by_lo);
|
||||
int m = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (m && U.wr[i].lo <= U.wr[m - 1].hi) {
|
||||
if (U.wr[i].hi > U.wr[m - 1].hi) U.wr[m - 1].hi = U.wr[i].hi;
|
||||
if (U.wr[i].fence > U.wr[m - 1].fence) U.wr[m - 1].fence = U.wr[i].fence;
|
||||
} else {
|
||||
U.wr[m++] = U.wr[i];
|
||||
}
|
||||
}
|
||||
U.nwr = m;
|
||||
}
|
||||
static int wr_hits(uint64_t lo, uint64_t hi) { // wr_prune ran: disjoint, sorted
|
||||
int a = 0, b = U.nwr;
|
||||
while (a < b) {
|
||||
int k = (a + b) / 2;
|
||||
if (U.wr[k].hi <= lo) a = k + 1;
|
||||
else b = k;
|
||||
}
|
||||
return a < U.nwr && U.wr[a].lo < hi;
|
||||
}
|
||||
|
||||
static void summary(void) {
|
||||
if (U.subs + U.off_subs < 3600) return;
|
||||
double s = U.subs ? (double)U.subs : 1.0;
|
||||
rt_log("vk_usnap: %llu submissions with a snapshot, %llu without; per submission %.0f ranges, %.1f KiB copied in "
|
||||
"%.3f ms, %.0f uniform bindings from the copy, %.1f dropped by GPU writes, %.1f left out (GPU writes in "
|
||||
"flight), %.1f refused (not an imported pool)",
|
||||
(unsigned long long)U.subs, (unsigned long long)U.off_subs, U.regions / s, U.bytes / s / 1024.0,
|
||||
U.copy_ns / s / 1e6, U.hits / s, U.drops / s, U.inflight / s, U.refused / s);
|
||||
U.subs = U.off_subs = U.regions = U.bytes = U.copy_ns = U.hits = U.drops = U.inflight = U.refused = 0;
|
||||
}
|
||||
|
||||
void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) {
|
||||
if (!U.mode) U.mode = rt_knob("UBO_SNAPSHOT", "1") + 1; // stored +1: 0 is "not read yet"
|
||||
U.on = 0, U.fence = fence, U.gen++;
|
||||
summary();
|
||||
if (!RT_KNOB_ON(U.mode - 1)) {
|
||||
U.off_subs++;
|
||||
return;
|
||||
}
|
||||
uint64_t t0 = kern_now_ns();
|
||||
U.ncol = 0;
|
||||
for (int i = 0; i < n; i++)
|
||||
if (handles[i]) collect(nvn_op_first(handles[i]), 0);
|
||||
sort_col();
|
||||
wr_prune();
|
||||
// merge, keeping only what lies wholly in one imported pool and no GPU write in flight touches
|
||||
U.nreg = 0;
|
||||
uint64_t total = 0;
|
||||
for (int i = 0; i < U.ncol;) {
|
||||
uint64_t lo = U.col[i].lo, hi = U.col[i].hi;
|
||||
for (i++; i < U.ncol && U.col[i].lo <= hi; i++)
|
||||
if (U.col[i].hi > hi) hi = U.col[i].hi;
|
||||
VkMemSpan sp;
|
||||
if (!vk_mem_resolve(lo, hi - lo, &sp) || sp.alloc || !sp.pool || sp.avail < hi - lo ||
|
||||
(sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) {
|
||||
U.refused++;
|
||||
continue;
|
||||
}
|
||||
if (wr_hits(lo, hi)) {
|
||||
U.inflight++;
|
||||
continue;
|
||||
}
|
||||
if (U.nreg == U.creg) U.reg = grow(U.reg, &U.creg, sizeof *U.reg);
|
||||
U.reg[U.nreg++] = (Reg){lo, hi, total, 0};
|
||||
total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1);
|
||||
}
|
||||
if (!U.nreg) return;
|
||||
if (!(U.arena = arena_get(total))) return;
|
||||
for (int i = 0; i < U.nreg; i++)
|
||||
memcpy(U.arena->map + U.reg[i].off, (const void *)(uintptr_t)U.reg[i].lo, U.reg[i].hi - U.reg[i].lo);
|
||||
vmaFlushAllocation(vk_ctx()->vma, U.arena->alloc, 0, total); // a no-op on coherent memory
|
||||
U.on = 1;
|
||||
U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += total, U.copy_ns += kern_now_ns() - t0;
|
||||
}
|
||||
|
||||
void vk_usnap_end(uint64_t fence) {
|
||||
if (U.arena) vk_completion_add(fence, arena_done, U.arena), U.arena = NULL;
|
||||
U.on = 0, U.gen++;
|
||||
}
|
||||
|
||||
// First region whose end is above a.
|
||||
static int first_above(uint64_t a) {
|
||||
int lo = 0, hi = U.nreg;
|
||||
while (lo < hi) {
|
||||
int m = (lo + hi) / 2;
|
||||
if (U.reg[m].hi <= a) lo = m + 1;
|
||||
else hi = m;
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
|
||||
int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out) {
|
||||
if (!U.on || !n) return 0;
|
||||
int i = first_above(addr);
|
||||
if (i >= U.nreg) return 0;
|
||||
const Reg *r = &U.reg[i];
|
||||
if (r->dropped || addr < r->lo || addr + n > r->hi) return 0;
|
||||
uint64_t off = r->off + (addr - r->lo);
|
||||
if (off % align) return 0;
|
||||
*out = (VkDescriptorBufferInfo){U.arena->buf, off, n};
|
||||
U.hits++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
void vk_usnap_drop(uint64_t addr, uint64_t n) {
|
||||
if (!U.mode || !n) return;
|
||||
if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr);
|
||||
U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence};
|
||||
if (!U.on) return;
|
||||
for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++)
|
||||
if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, U.gen++;
|
||||
}
|
||||
|
||||
uint64_t vk_usnap_gen(void) { return U.gen; }
|
||||
|
||||
// The arenas back on the free list (vk_exec_draw_shutdown: the device is idle, every completion ran).
|
||||
void vk_usnap_shutdown(void) {
|
||||
AcquireSRWLockExclusive(&free_lock);
|
||||
for (Arena *a = free_arenas, *next; a; a = next) next = a->next, vmaDestroyBuffer(vk_ctx()->vma, a->buf, a->alloc), free(a);
|
||||
free_arenas = NULL;
|
||||
ReleaseSRWLockExclusive(&free_lock);
|
||||
}
|
||||
Reference in new issue
Block a user