Files
totk-pc-recomp/runtime/vk_usnap.c
T
Valiantsin DzerakhandClaude Opus 5.5 b8edbdd873 The per-object motion reads last frame's object and bone blocks from last frame's uniform snapshot, kept one more frame, instead of having every snapshot copy both neighbours of each such block: the town's snapshot was 2836 KiB a submission with the neighbours and 1916 KiB without, and each byte is copied twice with the motion on (into host memory for the CPU's reads, then into the device copy). vk_mvprev_find still picks the address; the block's bytes and the motion draw's descriptor come from vk_usnap_prev. A block bound at the same address both frames keeps this frame's copy, as before.
Lifetime: a scene submission's arena (vk_usnap_scene, from the scene draws vk_mvprev notes) is kept at its vk_usnap_end as the last-frame copy instead of going to the free list at its fence. The next scene submission's end, or any end once the motion is off, hands the kept arena to the completion thread with that submission's fence, then keeps its own. The motion draws that bind the kept arena are listed and recorded within the next scene submission (vk_mvobj_execute drops a list of another fence), so the GPU has read it before that fence; the fences of one queue complete in order, so its own submission's reads are done too. One arena is held beyond those in flight, whatever the session length; vk_usnap_shutdown frees it with the others. A frame whose scene spans two submissions finds only the last one's blocks; the others keep the camera's motion.

TOTK_MVPREV_CHECK=1 still copies the neighbours and compares them with the kept copy. Field still view, 28 s: the mv_prev lines intact 100 % in c5, c6, c7, c14, c15 and c16; over whole blocks 87 of 218320 last-frame copies differ from the neighbour copy (the game changed the other copy after last frame's submit; the kept copy is what last frame drew); mv_check 0 of 60 checks with over 0.5 % object pixels fit better by a variant, object end-point error median 0.98 px against the camera's 1.27; the c7 lookups split as before (nearer to side 1.87, reference 1.83).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 19:49:47 +03:00

422 lines
20 KiB
C

// Uniform blocks as of the submit call. NVN starts the GPU when nvnQueueSubmitCommands returns and the GPU reads the
// constants soon after; here the replay records for several milliseconds, and the GPU reads a pool's bytes when it
// runs the part that uses them (the VRAM mirror's sync copies run then, vk_memory.c "VRAM mirror"). Meanwhile the game,
// having handed the frame over, rewrites per-object uniform blocks on other threads with no fence: late draws read
// another object's matrix (phantom rocks, a flat far field; TotK 1.2.1's producer f_70008c6460, 14-24 ms after the
// submit). So at the start of a submission's recording every uniform range it binds is copied into an arena of its own,
// and the uniform bindings of the replay use the copy.
// - Only ranges of imported pools (memory the CPU writes) are copied, aligned out to 256 bytes (the largest
// minUniformBufferOffsetAlignment) and at most 64 KiB from the bound address, overlapping ones merged.
// - A range the GPU wrote in a submission that has not completed is left out: its bytes reach guest memory only with
// that submission's write-back, the mirror holds them until then.
// - A GPU write within the submission (UpdateUniformBuffer, a storage buffer a shader writes, a buffer copy or fill,
// an image-to-buffer copy: vk_usnap_drop) drops the copies it touches from then on, so later draws read the result.
// A chunked replay (vk_exec.c) records the ops out of order: its scan drops at the position of the op
// (vk_usnap_drop_at, scan_writes for the shaders' stores), and a chunk finds a copy only at a position before that.
// - The arena goes back to a free list once the submission's fence is done (completion thread).
// - With the per-object motion, a scene submission's arena (vk_usnap_scene: the replay drew into the scene depth) is
// kept past its fence as the last-frame copy (PV): the next scene submission's motion draws bind last frame's blocks
// from it (vk_usnap_prev). It goes back to the free list at the fence of the submission that replaces or drops it,
// which is the submission whose motion draws read it (or a later one): its lists are recorded and run within that
// submission (vk_exec_draw.c vk_mvobj_execute drops a list of another fence), and the GPU finishes a submission's
// work, and every earlier one's, before its fence. One arena held beyond those in flight, whatever the session length.
// The design follows MacNativeGames' TOTK_UBO_SNAPSHOT (games/totk/third_party/d2r_gfx/gfx/nvn/nvn_mtl.m usnap_take,
// usnap_drop; MIT, Copyright (c) 2026 jhaizhou); the code is ours.
// The copy is taken where vk_exec_record starts: at the submit call when the replay runs on the submitting thread, later
// with TOTK_REPLAY_ASYNC; a chunked replay takes it on a recording thread while its scan runs. TOTK_UBO_SNAPSHOT=0 binds live memory as before; =ab alternates (TOTK_AB).
#include <stdlib.h>
#include <string.h>
#include "host.h"
#include "nvn_api.h"
#include "vk.h"
#include "vk_dlaa.h"
#include "platform/platform.h"
#define ALIGN 256u
#define MAX_RANGE (64u << 10)
#define ARENA_MIN (4u << 20)
#define CALL_DEPTH 16
typedef struct Arena {
struct Arena *next;
VkBuffer buf;
VmaAllocation alloc;
uint8_t *map;
uint8_t *cpu; // with the per-object motion (vk_objmv_wanted: DLSS or FSR 3.1), the same bytes in host memory for
// the CPU's reads (vk_usnap_bytes): map is often device memory through the BAR, written fast but
// read back at about a microsecond a cache line. Without it nothing reads them back, and the bytes
// go into map in one copy.
uint64_t size;
} Arena;
typedef struct { uint64_t lo, hi; } Rng;
typedef struct { uint64_t lo, hi, off, drop_at; } Reg; // drop_at: dropped from this position on; UINT64_MAX not
typedef struct { uint64_t lo, hi, fence; } Wrote;
static struct {
int mode; // rt_knob: 0 off, 1 on, 2 ab
int on; // this submission has a snapshot
uint64_t fence, gen;
Arena *arena;
Rng *col; // collected ranges
int ncol, ccol;
Reg *reg; // merged, sorted by lo, disjoint
int nreg, creg;
Wrote *wr; // GPU writes of submissions not known to be complete
int nwr, cwr;
// since the last summary
uint64_t subs, regions, bytes, copy_ns, hits, drops, inflight, refused, off_subs;
} U;
static Arena *free_arenas;
// the last scene submission's arena and regions (vk_usnap_prev); its regions' drop_at final
static struct {
Arena *arena;
Reg *reg;
int nreg, creg;
} PV;
static int scene; // vk_usnap_scene since the last vk_usnap_begin
static PlatMutex free_lock = PLAT_MUTEX_INIT;
static PlatMutex drop_lock = PLAT_MUTEX_INIT; // threads recording chunks of one submission drop concurrently
static void *grow(void *p, int *cap, size_t elem) {
int n = *cap ? *cap * 2 : 1024;
void *q = realloc(p, (size_t)n * elem);
if (!q) rt_fatal("vk_usnap: out of memory");
*cap = n;
return q;
}
static Arena *arena_get(uint64_t need) {
plat_mutex_lock(&free_lock);
Arena **pp = &free_arenas, *a = NULL;
for (; *pp; pp = &(*pp)->next)
if ((*pp)->size >= need) {
a = *pp, *pp = a->next;
break;
}
plat_mutex_unlock(&free_lock);
if (a) return a;
a = calloc(1, sizeof *a);
if (!a) rt_fatal("vk_usnap: out of memory");
a->size = need < ARENA_MIN ? ARENA_MIN : (need + (1u << 20) - 1) & ~(uint64_t)((1u << 20) - 1);
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = a->size,
// transfer source: DLAA copies the camera block out (vk_dlaa.c mv_pass)
.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
// Device-local through the BAR when the device has it (the GPU reads constants on every draw), else host memory.
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
VMA_ALLOCATION_CREATE_MAPPED_BIT};
VmaAllocationInfo info;
VkResult res = vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &a->buf, &a->alloc, &info);
if (res != VK_SUCCESS) {
rt_log("vk_usnap: arena of %llu bytes failed (%d): this submission binds live memory",
(unsigned long long)a->size, (int)res);
free(a);
return NULL;
}
a->map = info.pMappedData;
rt_log("vk_usnap: new arena of %.1f MiB", a->size / 1048576.0);
return a;
}
static void arena_done(void *arg) { // completion thread: the GPU has read it
Arena *a = arg;
plat_mutex_lock(&free_lock);
a->next = free_arenas, free_arenas = a;
plat_mutex_unlock(&free_lock);
}
static void collect(const op_hdr *h, int depth) {
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_)
rt_fatal("vk_usnap: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size);
const void *a = nvn_op_payload(h);
if (h->op == OP_END) return;
if (h->op == OP_JUMP) {
h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target;
continue;
}
if (h->op == OP_CALL && depth < CALL_DEPTH) collect(nvn_op_first(((const p_jump *)a)->target), depth + 1);
if (h->op == OP_BIND_UBO) {
const p_bind_buf *b = a;
if (b->addr && b->size) {
uint64_t n = b->size < MAX_RANGE ? b->size : MAX_RANGE;
// TOTK_MVPREV_CHECK: the vertex stage's object and bone blocks, also the game's other copy on either
// side, which still holds last frame's (vk_mvprev.c compares it with the last-frame copy)
uint64_t step = vk_mvprev_step(b->stage, b->index, b->size);
// which neighbours lie in an imported pool, per block and vk_mem_gen: the same blocks are bound by
// many draws, and two resolves for each binding were a quarter of the snapshot's time
static struct { uint64_t addr, size, gen; int ok; } nc[1024];
uint32_t ci = (uint32_t)((b->addr ^ n << 48) * 0x9E3779B97F4A7C15ull >> 54);
uint64_t gen = vk_mem_gen();
int known = step && nc[ci].addr == b->addr && nc[ci].size == n && nc[ci].gen == gen, ok = 0;
for (int k = step ? -1 : 0; k <= (step ? 1 : 0); k++) {
uint64_t at = b->addr + (uint64_t)((int64_t)k * (int64_t)step);
if (k < 0 && b->addr < step) continue;
// a neighbour past its pool would make the whole merged region fail the pool check below, and
// the bindings in it read live memory again
VkMemSpan sp;
uint64_t lo = at & ~(uint64_t)(ALIGN - 1);
if (k && known) {
if (!(nc[ci].ok >> (k + 1) & 1)) continue;
} else if (k) {
if (!vk_mem_resolve(lo, at + n - lo, &sp) || sp.alloc || !sp.pool || sp.avail < at + n - lo ||
(sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC))
continue;
ok |= 1 << (k + 1);
}
if (U.ncol == U.ccol) U.col = grow(U.col, &U.ccol, sizeof *U.col);
U.col[U.ncol++] = (Rng){at & ~(uint64_t)(ALIGN - 1), at + n};
}
if (step && !known) nc[ci].addr = b->addr, nc[ci].size = n, nc[ci].gen = gen, nc[ci].ok = ok;
}
}
h = nvn_op_next(h);
}
}
static int by_lo(const void *a, const void *b) {
uint64_t x = ((const Rng *)a)->lo, y = ((const Rng *)b)->lo;
return x < y ? -1 : x > y;
}
static int wr_by_lo(const void *a, const void *b) {
uint64_t x = ((const Wrote *)a)->lo, y = ((const Wrote *)b)->lo;
return x < y ? -1 : x > y;
}
// Sorts the collected ranges by lo. A town frame binds about 10,000 uniform ranges and qsort took half a millisecond
// of the copy's 0.96 ms on the game thread, so they go through an LSD radix sort on (lo - lowest) / ALIGN, three
// passes of 11 bits; qsort is left for a span that does not fit in 33 bits (2 TiB).
#define RADIX_BITS 11
static void sort_col(void) {
static Rng *tmp;
static int ctmp;
int n = U.ncol;
if (n < 2) return;
uint64_t lo = UINT64_MAX, hi = 0;
for (int i = 0; i < n; i++) {
if (U.col[i].lo < lo) lo = U.col[i].lo;
if (U.col[i].lo > hi) hi = U.col[i].lo;
}
if ((hi - lo) / ALIGN >> (3 * RADIX_BITS)) {
qsort(U.col, (size_t)n, sizeof *U.col, by_lo);
return;
}
if (ctmp < n) {
free(tmp);
ctmp = U.ccol;
if (!(tmp = malloc((size_t)ctmp * sizeof *tmp))) rt_fatal("vk_usnap: out of memory");
}
Rng *src = U.col, *dst = tmp;
for (int pass = 0; pass < 3; pass++) {
int shift = pass * RADIX_BITS, count[1 << RADIX_BITS] = {0};
for (int i = 0; i < n; i++) count[(src[i].lo - lo) / ALIGN >> shift & ((1 << RADIX_BITS) - 1)]++;
for (int k = 0, sum = 0; k < 1 << RADIX_BITS; k++) {
int c = count[k];
count[k] = sum, sum += c;
}
for (int i = 0; i < n; i++) dst[count[(src[i].lo - lo) / ALIGN >> shift & ((1 << RADIX_BITS) - 1)]++] = src[i];
Rng *t = src;
src = dst, dst = t;
}
if (src != U.col) memcpy(U.col, src, (size_t)n * sizeof *src); // three passes end in tmp
}
// Keeps the GPU writes of submissions that are not done, merged into disjoint ranges sorted by lo (the terrain rewrites
// one block before every draw: hundreds of equal entries), each with the newest fence among its parts.
static void wr_prune(void) {
uint64_t done = vk_sync_fence_done();
int n = 0;
for (int i = 0; i < U.nwr; i++)
if (U.wr[i].fence > done) U.wr[n++] = U.wr[i];
qsort(U.wr, (size_t)n, sizeof *U.wr, wr_by_lo);
int m = 0;
for (int i = 0; i < n; i++) {
if (m && U.wr[i].lo <= U.wr[m - 1].hi) {
if (U.wr[i].hi > U.wr[m - 1].hi) U.wr[m - 1].hi = U.wr[i].hi;
if (U.wr[i].fence > U.wr[m - 1].fence) U.wr[m - 1].fence = U.wr[i].fence;
} else {
U.wr[m++] = U.wr[i];
}
}
U.nwr = m;
}
static int wr_hits(uint64_t lo, uint64_t hi) { // wr_prune ran: disjoint, sorted
int a = 0, b = U.nwr;
while (a < b) {
int k = (a + b) / 2;
if (U.wr[k].hi <= lo) a = k + 1;
else b = k;
}
return a < U.nwr && U.wr[a].lo < hi;
}
static void summary(void) {
if (U.subs + U.off_subs < 3600) return;
double s = U.subs ? (double)U.subs : 1.0;
rt_log("vk_usnap: %llu submissions with a snapshot, %llu without; per submission %.0f ranges, %.1f KiB copied in "
"%.3f ms, %.0f uniform bindings from the copy, %.1f dropped by GPU writes, %.1f left out (GPU writes in "
"flight), %.1f refused (not an imported pool)",
(unsigned long long)U.subs, (unsigned long long)U.off_subs, U.regions / s, U.bytes / s / 1024.0,
U.copy_ns / s / 1e6, U.hits / s, U.drops / s, U.inflight / s, U.refused / s);
U.subs = U.off_subs = U.regions = U.bytes = U.copy_ns = U.hits = U.drops = U.inflight = U.refused = 0;
}
void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) {
if (!U.mode) U.mode = rt_knob("UBO_SNAPSHOT", "1") + 1; // stored +1: 0 is "not read yet"
U.on = 0, U.fence = fence, U.gen++;
summary();
if (!RT_KNOB_ON(U.mode - 1)) {
U.off_subs++;
return;
}
uint64_t t0 = kern_now_ns();
U.ncol = 0;
for (int i = 0; i < n; i++)
if (handles[i]) collect(nvn_op_first(handles[i]), 0);
sort_col();
wr_prune();
// merge, keeping only what lies wholly in one imported pool and no GPU write in flight touches
U.nreg = 0;
uint64_t total = 0;
for (int i = 0; i < U.ncol;) {
uint64_t lo = U.col[i].lo, hi = U.col[i].hi;
for (i++; i < U.ncol && U.col[i].lo <= hi; i++)
if (U.col[i].hi > hi) hi = U.col[i].hi;
VkMemSpan sp;
if (!vk_mem_resolve(lo, hi - lo, &sp) || sp.alloc || !sp.pool || sp.avail < hi - lo ||
(sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) {
U.refused++;
continue;
}
if (wr_hits(lo, hi)) {
U.inflight++;
continue;
}
if (U.nreg == U.creg) U.reg = grow(U.reg, &U.creg, sizeof *U.reg);
U.reg[U.nreg++] = (Reg){lo, hi, total, UINT64_MAX};
total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1);
}
if (!U.nreg) return;
if (!(U.arena = arena_get(total))) return;
uint8_t *dst = U.arena->map;
if (vk_objmv_wanted()) {
if (!U.arena->cpu && !(U.arena->cpu = malloc(U.arena->size))) rt_fatal("vk_usnap: out of memory");
dst = U.arena->cpu;
}
for (int i = 0; i < U.nreg; i++)
memcpy(dst + U.reg[i].off, (const void *)(uintptr_t)U.reg[i].lo, U.reg[i].hi - U.reg[i].lo);
if (dst != U.arena->map) memcpy(U.arena->map, dst, total); // one sequential write into the device copy
vmaFlushAllocation(vk_ctx()->vma, U.arena->alloc, 0, total); // a no-op on coherent memory
U.on = 1;
U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += total, U.copy_ns += kern_now_ns() - t0;
}
void vk_usnap_end(uint64_t fence) {
int keep = __atomic_load_n(&scene, __ATOMIC_RELAXED) && U.arena && U.arena->cpu;
// a scene submission replaces the last-frame copy (or drops it, without a snapshot of its own); so does the motion
// going off, which no longer reads it
if (PV.arena && (__atomic_load_n(&scene, __ATOMIC_RELAXED) || !vk_mvprev_on()))
vk_completion_add(fence, arena_done, PV.arena), PV.arena = NULL, PV.nreg = 0;
if (keep) {
Reg *r = PV.reg;
int c = PV.creg;
PV.arena = U.arena, PV.reg = U.reg, PV.nreg = U.nreg, PV.creg = U.creg;
U.reg = r, U.creg = c, U.nreg = 0;
} else if (U.arena) {
vk_completion_add(fence, arena_done, U.arena);
}
U.arena = NULL;
__atomic_store_n(&scene, 0, __ATOMIC_RELAXED);
U.on = 0, U.gen++;
}
void vk_usnap_scene(void) {
if (!__atomic_load_n(&scene, __ATOMIC_RELAXED)) __atomic_store_n(&scene, 1, __ATOMIC_RELAXED);
}
int vk_usnap_prev(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out, const uint8_t **bytes) {
if (!PV.arena || !n) return 0;
int lo = 0, hi = PV.nreg;
while (lo < hi) { // first region whose end is above addr
int m = (lo + hi) / 2;
if (PV.reg[m].hi <= addr) lo = m + 1;
else hi = m;
}
if (lo >= PV.nreg) return 0;
const Reg *r = &PV.reg[lo];
// a region a GPU write dropped held other bytes for the draws after it: left out, as vk_usnap_bytes does
if (r->drop_at != UINT64_MAX || addr < r->lo || addr + n > r->hi) return 0;
uint64_t off = r->off + (addr - r->lo);
if (off % align) return 0;
*out = (VkDescriptorBufferInfo){PV.arena->buf, off, n};
*bytes = PV.arena->cpu + off;
return 1;
}
// First region whose end is above a.
static int first_above(uint64_t a) {
int lo = 0, hi = U.nreg;
while (lo < hi) {
int m = (lo + hi) / 2;
if (U.reg[m].hi <= a) lo = m + 1;
else hi = m;
}
return lo;
}
int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t at, uint64_t align, VkDescriptorBufferInfo *out) {
if (!U.on || !n) return 0;
int i = first_above(addr);
if (i >= U.nreg) return 0;
const Reg *r = &U.reg[i];
if (__atomic_load_n(&r->drop_at, __ATOMIC_RELAXED) <= at || addr < r->lo || addr + n > r->hi) return 0;
uint64_t off = r->off + (addr - r->lo);
if (off % align) return 0;
*out = (VkDescriptorBufferInfo){U.arena->buf, off, n};
__atomic_fetch_add(&U.hits, 1, __ATOMIC_RELAXED);
return 1;
}
static void drop(uint64_t addr, uint64_t n, uint64_t at) {
if (!U.mode || !n) return;
plat_mutex_lock(&drop_lock);
if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr);
U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence};
if (U.on)
for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++)
if (at < U.reg[i].drop_at) {
if (U.reg[i].drop_at == UINT64_MAX) U.drops++;
__atomic_store_n(&U.reg[i].drop_at, at, __ATOMIC_RELAXED);
__atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED);
}
plat_mutex_unlock(&drop_lock);
}
void vk_usnap_drop(uint64_t addr, uint64_t n) { drop(addr, n, 0); }
void vk_usnap_drop_at(uint64_t addr, uint64_t n, uint32_t at) { drop(addr, n, at); }
uint64_t vk_usnap_gen(void) { return __atomic_load_n(&U.gen, __ATOMIC_RELAXED); }
// the host copy where there is one (DLSS), else the device copy
static uint8_t *host_copy(void) { return U.arena->cpu ? U.arena->cpu : U.arena->map; }
const uint8_t *vk_usnap_map(void) { return U.arena ? host_copy() : NULL; }
const uint8_t *vk_usnap_bytes(uint64_t addr, uint64_t n) {
VkDescriptorBufferInfo bi;
return vk_usnap_find(addr, n, UINT32_MAX, 1, &bi) ? host_copy() + bi.offset : NULL;
}
// The arenas back on the free list (vk_exec_draw_shutdown: the device is idle, every completion ran).
void vk_usnap_shutdown(void) {
plat_mutex_lock(&free_lock);
if (PV.arena) PV.arena->next = free_arenas, free_arenas = PV.arena, PV.arena = NULL;
for (Arena *a = free_arenas, *next; a; a = next) next = a->next, vmaDestroyBuffer(vk_ctx()->vma, a->buf, a->alloc), free(a->cpu), free(a);
free_arenas = NULL;
plat_mutex_unlock(&free_lock);
}