Files
totk-pc-recomp/runtime/vk_usnap.c
T
Valiantsin DzerakhandClaude Opus 5.5 22794ab332 Merge task/fg into task/fsr31: frame generation's per-object motion vectors and its contexts kept across dynamic resolution, on one motion gate with FSR 3.1
vk_objmv_wanted is FSR 3.1's latched upscaler half (DLSS with NGX up, or FSR 3.1 picked at start) or frame generation following the scene with its objects (vk_fg_objects_on), and vk_mvprev_frame turns the tables on for the running upscaler or for frame generation. vk_usnap.c keeps this branch's host regions: a region's host flag says whether this submission wrote its host copy, which covers frame generation going on and off live without fg's separate U.host pointer. The jitter fix in mv_pass stays a single copy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 20:15:40 +03:00

438 lines
21 KiB
C

// Uniform blocks as of the submit call. NVN starts the GPU when nvnQueueSubmitCommands returns and the GPU reads the
// constants soon after; here the replay records for several milliseconds, and the GPU reads a pool's bytes when it
// runs the part that uses them (the VRAM mirror's sync copies run then, vk_memory.c "VRAM mirror"). Meanwhile the game,
// having handed the frame over, rewrites per-object uniform blocks on other threads with no fence: late draws read
// another object's matrix (phantom rocks, a flat far field; TotK 1.2.1's producer f_70008c6460, 14-24 ms after the
// submit). So at the start of a submission's recording every uniform range it binds is copied into an arena of its own,
// and the uniform bindings of the replay use the copy.
// - Only ranges of imported pools (memory the CPU writes) are copied, aligned out to 256 bytes (the largest
// minUniformBufferOffsetAlignment) and at most 64 KiB from the bound address, overlapping ones merged.
// - A range the GPU wrote in a submission that has not completed is left out: its bytes reach guest memory only with
// that submission's write-back, the mirror holds them until then.
// - A GPU write within the submission (UpdateUniformBuffer, a storage buffer a shader writes, a buffer copy or fill,
// an image-to-buffer copy: vk_usnap_drop) drops the copies it touches from then on, so later draws read the result.
// A chunked replay (vk_exec.c) records the ops out of order: its scan drops at the position of the op
// (vk_usnap_drop_at, scan_writes for the shaders' stores), and a chunk finds a copy only at a position before that.
// - The arena goes back to a free list once the submission's fence is done (completion thread).
// - With the per-object motion, a scene submission's arena (vk_usnap_scene: the replay drew into the scene depth) is
// kept past its fence as the last-frame copy (PV): the next scene submission's motion draws bind last frame's blocks
// from it (vk_usnap_prev). It goes back to the free list at the fence of the submission that replaces or drops it,
// which is the submission whose motion draws read it (or a later one): its lists are recorded and run within that
// submission (vk_exec_draw.c vk_mvobj_execute drops a list of another fence), and the GPU finishes a submission's
// work, and every earlier one's, before its fence. One arena held beyond those in flight, whatever the session length.
// The design follows MacNativeGames' TOTK_UBO_SNAPSHOT (games/totk/third_party/d2r_gfx/gfx/nvn/nvn_mtl.m usnap_take,
// usnap_drop; MIT, Copyright (c) 2026 jhaizhou); the code is ours.
// The copy is taken where vk_exec_record starts: at the submit call when the replay runs on the submitting thread, later
// with TOTK_REPLAY_ASYNC; a chunked replay takes it on a recording thread while its scan runs. TOTK_UBO_SNAPSHOT=0 binds live memory as before; =ab alternates (TOTK_AB).
#include <stdlib.h>
#include <string.h>
#include "host.h"
#include "nvn_api.h"
#include "vk.h"
#include "vk_dlaa.h"
#include "platform/platform.h"
#define ALIGN 256u
#define MAX_RANGE (64u << 10)
#define ARENA_MIN (4u << 20)
#define CALL_DEPTH 16
typedef struct Arena {
struct Arena *next;
VkBuffer buf;
VmaAllocation alloc;
uint8_t *map;
uint8_t *cpu; // with the per-object motion (vk_objmv_wanted: DLSS, FSR 3.1, or frame generation with its objects),
// the bytes of the regions the CPU reads back many times (Reg.host) in host memory too, at the same
// offsets (vk_usnap_bytes): map is often device memory through the BAR, written fast but read back
// at about a microsecond a cache line. The other regions go into map alone, as without the motion.
// Frame generation turns on and off live: a region's host flag says whether this submission wrote
// its host copy, so a stale copy from an earlier submission is never read.
uint64_t size;
} Arena;
// host: a vertex-stage binding of a bank the per-object motion compares (vk_mvprev_kept) or of the camera (c4), whose
// bytes the CPU reads (vk_exec_draw.c mo_prev_block, vk_mvprev_find, vk_dlaa.c): its region also gets a host copy
typedef struct { uint64_t lo, hi; uint32_t host; } Rng;
typedef struct { uint64_t lo, hi, off, drop_at; uint32_t host; } Reg; // drop_at: dropped from this position on; UINT64_MAX not
typedef struct { uint64_t lo, hi, fence; } Wrote;
static struct {
int mode; // rt_knob: 0 off, 1 on, 2 ab
int on; // this submission has a snapshot
uint64_t fence, gen;
Arena *arena;
Rng *col; // collected ranges
int ncol, ccol;
Reg *reg; // merged, sorted by lo, disjoint
int nreg, creg;
Wrote *wr; // GPU writes of submissions not known to be complete
int nwr, cwr;
// since the last summary
uint64_t subs, regions, bytes, hbytes, copy_ns, hits, drops, inflight, refused, off_subs;
} U;
static Arena *free_arenas;
// the last scene submission's arena and regions (vk_usnap_prev); its regions' drop_at final
static struct {
Arena *arena;
Reg *reg;
int nreg, creg;
} PV;
static int scene; // vk_usnap_scene since the last vk_usnap_begin
static PlatMutex free_lock = PLAT_MUTEX_INIT;
static PlatMutex drop_lock = PLAT_MUTEX_INIT; // threads recording chunks of one submission drop concurrently
static void *grow(void *p, int *cap, size_t elem) {
int n = *cap ? *cap * 2 : 1024;
void *q = realloc(p, (size_t)n * elem);
if (!q) rt_fatal("vk_usnap: out of memory");
*cap = n;
return q;
}
static Arena *arena_get(uint64_t need) {
plat_mutex_lock(&free_lock);
Arena **pp = &free_arenas, *a = NULL;
for (; *pp; pp = &(*pp)->next)
if ((*pp)->size >= need) {
a = *pp, *pp = a->next;
break;
}
plat_mutex_unlock(&free_lock);
if (a) return a;
a = calloc(1, sizeof *a);
if (!a) rt_fatal("vk_usnap: out of memory");
a->size = need < ARENA_MIN ? ARENA_MIN : (need + (1u << 20) - 1) & ~(uint64_t)((1u << 20) - 1);
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = a->size,
// transfer source: DLAA copies the camera block out (vk_dlaa.c mv_pass)
.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
// Device-local through the BAR when the device has it (the GPU reads constants on every draw), else host memory.
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
VMA_ALLOCATION_CREATE_MAPPED_BIT};
VmaAllocationInfo info;
VkResult res = vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &a->buf, &a->alloc, &info);
if (res != VK_SUCCESS) {
rt_log("vk_usnap: arena of %llu bytes failed (%d): this submission binds live memory",
(unsigned long long)a->size, (int)res);
free(a);
return NULL;
}
a->map = info.pMappedData;
rt_log("vk_usnap: new arena of %.1f MiB", a->size / 1048576.0);
return a;
}
static void arena_done(void *arg) { // completion thread: the GPU has read it
Arena *a = arg;
plat_mutex_lock(&free_lock);
a->next = free_arenas, free_arenas = a;
plat_mutex_unlock(&free_lock);
}
static void collect(const op_hdr *h, int depth) {
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_)
rt_fatal("vk_usnap: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size);
const void *a = nvn_op_payload(h);
if (h->op == OP_END) return;
if (h->op == OP_JUMP) {
h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target;
continue;
}
if (h->op == OP_CALL && depth < CALL_DEPTH) collect(nvn_op_first(((const p_jump *)a)->target), depth + 1);
if (h->op == OP_BIND_UBO) {
const p_bind_buf *b = a;
if (b->addr && b->size) {
uint64_t n = b->size < MAX_RANGE ? b->size : MAX_RANGE;
// TOTK_MVPREV_CHECK: the vertex stage's object and bone blocks, also the game's other copy on either
// side, which still holds last frame's (vk_mvprev.c compares it with the last-frame copy)
uint64_t step = vk_mvprev_step(b->stage, b->index, b->size);
uint32_t host = b->stage == 0 && (b->index == 1 || vk_mvprev_kept(b->index));
// which neighbours lie in an imported pool, per block and vk_mem_gen: the same blocks are bound by
// many draws, and two resolves for each binding were a quarter of the snapshot's time
static struct { uint64_t addr, size, gen; int ok; } nc[1024];
uint32_t ci = (uint32_t)((b->addr ^ n << 48) * 0x9E3779B97F4A7C15ull >> 54);
uint64_t gen = vk_mem_gen();
int known = step && nc[ci].addr == b->addr && nc[ci].size == n && nc[ci].gen == gen, ok = 0;
for (int k = step ? -1 : 0; k <= (step ? 1 : 0); k++) {
uint64_t at = b->addr + (uint64_t)((int64_t)k * (int64_t)step);
if (k < 0 && b->addr < step) continue;
// a neighbour past its pool would make the whole merged region fail the pool check below, and
// the bindings in it read live memory again
VkMemSpan sp;
uint64_t lo = at & ~(uint64_t)(ALIGN - 1);
if (k && known) {
if (!(nc[ci].ok >> (k + 1) & 1)) continue;
} else if (k) {
if (!vk_mem_resolve(lo, at + n - lo, &sp) || sp.alloc || !sp.pool || sp.avail < at + n - lo ||
(sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC))
continue;
ok |= 1 << (k + 1);
}
if (U.ncol == U.ccol) U.col = grow(U.col, &U.ccol, sizeof *U.col);
U.col[U.ncol++] = (Rng){at & ~(uint64_t)(ALIGN - 1), at + n, host};
}
if (step && !known) nc[ci].addr = b->addr, nc[ci].size = n, nc[ci].gen = gen, nc[ci].ok = ok;
}
}
h = nvn_op_next(h);
}
}
static int by_lo(const void *a, const void *b) {
uint64_t x = ((const Rng *)a)->lo, y = ((const Rng *)b)->lo;
return x < y ? -1 : x > y;
}
static int wr_by_lo(const void *a, const void *b) {
uint64_t x = ((const Wrote *)a)->lo, y = ((const Wrote *)b)->lo;
return x < y ? -1 : x > y;
}
// Sorts the collected ranges by lo. A town frame binds about 10,000 uniform ranges and qsort took half a millisecond
// of the copy's 0.96 ms on the game thread, so they go through an LSD radix sort on (lo - lowest) / ALIGN, three
// passes of 11 bits; qsort is left for a span that does not fit in 33 bits (2 TiB).
#define RADIX_BITS 11
static void sort_col(void) {
static Rng *tmp;
static int ctmp;
int n = U.ncol;
if (n < 2) return;
uint64_t lo = UINT64_MAX, hi = 0;
for (int i = 0; i < n; i++) {
if (U.col[i].lo < lo) lo = U.col[i].lo;
if (U.col[i].lo > hi) hi = U.col[i].lo;
}
if ((hi - lo) / ALIGN >> (3 * RADIX_BITS)) {
qsort(U.col, (size_t)n, sizeof *U.col, by_lo);
return;
}
if (ctmp < n) {
free(tmp);
ctmp = U.ccol;
if (!(tmp = malloc((size_t)ctmp * sizeof *tmp))) rt_fatal("vk_usnap: out of memory");
}
Rng *src = U.col, *dst = tmp;
for (int pass = 0; pass < 3; pass++) {
int shift = pass * RADIX_BITS, count[1 << RADIX_BITS] = {0};
for (int i = 0; i < n; i++) count[(src[i].lo - lo) / ALIGN >> shift & ((1 << RADIX_BITS) - 1)]++;
for (int k = 0, sum = 0; k < 1 << RADIX_BITS; k++) {
int c = count[k];
count[k] = sum, sum += c;
}
for (int i = 0; i < n; i++) dst[count[(src[i].lo - lo) / ALIGN >> shift & ((1 << RADIX_BITS) - 1)]++] = src[i];
Rng *t = src;
src = dst, dst = t;
}
if (src != U.col) memcpy(U.col, src, (size_t)n * sizeof *src); // three passes end in tmp
}
// Keeps the GPU writes of submissions that are not done, merged into disjoint ranges sorted by lo (the terrain rewrites
// one block before every draw: hundreds of equal entries), each with the newest fence among its parts.
static void wr_prune(void) {
uint64_t done = vk_sync_fence_done();
int n = 0;
for (int i = 0; i < U.nwr; i++)
if (U.wr[i].fence > done) U.wr[n++] = U.wr[i];
qsort(U.wr, (size_t)n, sizeof *U.wr, wr_by_lo);
int m = 0;
for (int i = 0; i < n; i++) {
if (m && U.wr[i].lo <= U.wr[m - 1].hi) {
if (U.wr[i].hi > U.wr[m - 1].hi) U.wr[m - 1].hi = U.wr[i].hi;
if (U.wr[i].fence > U.wr[m - 1].fence) U.wr[m - 1].fence = U.wr[i].fence;
} else {
U.wr[m++] = U.wr[i];
}
}
U.nwr = m;
}
static int wr_hits(uint64_t lo, uint64_t hi) { // wr_prune ran: disjoint, sorted
int a = 0, b = U.nwr;
while (a < b) {
int k = (a + b) / 2;
if (U.wr[k].hi <= lo) a = k + 1;
else b = k;
}
return a < U.nwr && U.wr[a].lo < hi;
}
static void summary(void) {
if (U.subs + U.off_subs < 3600) return;
double s = U.subs ? (double)U.subs : 1.0;
rt_log("vk_usnap: %llu submissions with a snapshot, %llu without; per submission %.0f ranges, %.1f KiB copied "
"(%.1f KiB of it into host memory too) in %.3f ms, %.0f uniform bindings from the copy, %.1f dropped by GPU "
"writes, %.1f left out (GPU writes in flight), %.1f refused (not an imported pool)",
(unsigned long long)U.subs, (unsigned long long)U.off_subs, U.regions / s, U.bytes / s / 1024.0,
U.hbytes / s / 1024.0, U.copy_ns / s / 1e6, U.hits / s, U.drops / s, U.inflight / s, U.refused / s);
U.subs = U.off_subs = U.regions = U.bytes = U.hbytes = U.copy_ns = U.hits = U.drops = U.inflight = U.refused = 0;
}
void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) {
if (!U.mode) U.mode = rt_knob("UBO_SNAPSHOT", "1") + 1; // stored +1: 0 is "not read yet"
U.on = 0, U.fence = fence, U.gen++;
summary();
if (!RT_KNOB_ON(U.mode - 1)) {
U.off_subs++;
return;
}
uint64_t t0 = kern_now_ns();
U.ncol = 0;
for (int i = 0; i < n; i++)
if (handles[i]) collect(nvn_op_first(handles[i]), 0);
sort_col();
wr_prune();
// merge, keeping only what lies wholly in one imported pool and no GPU write in flight touches
U.nreg = 0;
uint64_t total = 0;
for (int i = 0; i < U.ncol;) {
uint64_t lo = U.col[i].lo, hi = U.col[i].hi;
uint32_t host = U.col[i].host;
for (i++; i < U.ncol && U.col[i].lo <= hi; i++) {
if (U.col[i].hi > hi) hi = U.col[i].hi;
host |= U.col[i].host;
}
VkMemSpan sp;
if (!vk_mem_resolve(lo, hi - lo, &sp) || sp.alloc || !sp.pool || sp.avail < hi - lo ||
(sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) {
U.refused++;
continue;
}
if (wr_hits(lo, hi)) {
U.inflight++;
continue;
}
if (U.nreg == U.creg) U.reg = grow(U.reg, &U.creg, sizeof *U.reg);
U.reg[U.nreg++] = (Reg){lo, hi, total, UINT64_MAX, host};
total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1);
}
if (!U.nreg) return;
if (!(U.arena = arena_get(total))) return;
uint8_t *map = U.arena->map, *cpu = NULL;
if (vk_objmv_wanted()) {
if (!U.arena->cpu && !(U.arena->cpu = malloc(U.arena->size))) rt_fatal("vk_usnap: out of memory");
cpu = U.arena->cpu;
} else {
for (int i = 0; i < U.nreg; i++) U.reg[i].host = 0; // nothing reads them back: the device copy alone
}
// a host region into host memory first and from there into the device copy, so both hold the same bytes while
// the game writes guest memory on other threads
uint64_t hbytes = 0;
for (int i = 0; i < U.nreg; i++) {
const Reg *r = &U.reg[i];
uint64_t len = r->hi - r->lo;
if (r->host) memcpy(cpu + r->off, (const void *)(uintptr_t)r->lo, len), memcpy(map + r->off, cpu + r->off, len), hbytes += len;
else memcpy(map + r->off, (const void *)(uintptr_t)r->lo, len);
}
vmaFlushAllocation(vk_ctx()->vma, U.arena->alloc, 0, total); // a no-op on coherent memory
U.on = 1;
U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += total, U.hbytes += hbytes, U.copy_ns += kern_now_ns() - t0;
}
void vk_usnap_end(uint64_t fence) {
int keep = __atomic_load_n(&scene, __ATOMIC_RELAXED) && U.arena && U.arena->cpu;
// a scene submission replaces the last-frame copy (or drops it, without a snapshot of its own); so does the motion
// going off, which no longer reads it
if (PV.arena && (__atomic_load_n(&scene, __ATOMIC_RELAXED) || !vk_mvprev_on()))
vk_completion_add(fence, arena_done, PV.arena), PV.arena = NULL, PV.nreg = 0;
if (keep) {
Reg *r = PV.reg;
int c = PV.creg;
PV.arena = U.arena, PV.reg = U.reg, PV.nreg = U.nreg, PV.creg = U.creg;
U.reg = r, U.creg = c, U.nreg = 0;
} else if (U.arena) {
vk_completion_add(fence, arena_done, U.arena);
}
U.arena = NULL;
__atomic_store_n(&scene, 0, __ATOMIC_RELAXED);
U.on = 0, U.gen++;
}
void vk_usnap_scene(void) {
if (!__atomic_load_n(&scene, __ATOMIC_RELAXED)) __atomic_store_n(&scene, 1, __ATOMIC_RELAXED);
}
int vk_usnap_prev(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out, const uint8_t **bytes) {
if (!PV.arena || !n) return 0;
int lo = 0, hi = PV.nreg;
while (lo < hi) { // first region whose end is above addr
int m = (lo + hi) / 2;
if (PV.reg[m].hi <= addr) lo = m + 1;
else hi = m;
}
if (lo >= PV.nreg) return 0;
const Reg *r = &PV.reg[lo];
// a region a GPU write dropped held other bytes for the draws after it: left out, as vk_usnap_bytes does
if (r->drop_at != UINT64_MAX || !r->host || addr < r->lo || addr + n > r->hi) return 0;
uint64_t off = r->off + (addr - r->lo);
if (off % align) return 0;
*out = (VkDescriptorBufferInfo){PV.arena->buf, off, n};
*bytes = PV.arena->cpu + off;
return 1;
}
// First region whose end is above a.
static int first_above(uint64_t a) {
int lo = 0, hi = U.nreg;
while (lo < hi) {
int m = (lo + hi) / 2;
if (U.reg[m].hi <= a) lo = m + 1;
else hi = m;
}
return lo;
}
int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t at, uint64_t align, VkDescriptorBufferInfo *out) {
if (!U.on || !n) return 0;
int i = first_above(addr);
if (i >= U.nreg) return 0;
const Reg *r = &U.reg[i];
if (__atomic_load_n(&r->drop_at, __ATOMIC_RELAXED) <= at || addr < r->lo || addr + n > r->hi) return 0;
uint64_t off = r->off + (addr - r->lo);
if (off % align) return 0;
*out = (VkDescriptorBufferInfo){U.arena->buf, off, n};
__atomic_fetch_add(&U.hits, 1, __ATOMIC_RELAXED);
return 1;
}
static void drop(uint64_t addr, uint64_t n, uint64_t at) {
if (!U.mode || !n) return;
plat_mutex_lock(&drop_lock);
if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr);
U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence};
if (U.on)
for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++)
if (at < U.reg[i].drop_at) {
if (U.reg[i].drop_at == UINT64_MAX) U.drops++;
__atomic_store_n(&U.reg[i].drop_at, at, __ATOMIC_RELAXED);
__atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED);
}
plat_mutex_unlock(&drop_lock);
}
void vk_usnap_drop(uint64_t addr, uint64_t n) { drop(addr, n, 0); }
void vk_usnap_drop_at(uint64_t addr, uint64_t n, uint32_t at) { drop(addr, n, at); }
uint64_t vk_usnap_gen(void) { return __atomic_load_n(&U.gen, __ATOMIC_RELAXED); }
// the host copy of a host region, else the device copy (slow to read through the BAR, read rarely)
const uint8_t *vk_usnap_bytes(uint64_t addr, uint64_t n) {
if (!U.on || !n) return NULL;
int i = first_above(addr);
if (i >= U.nreg) return NULL;
const Reg *r = &U.reg[i];
if (__atomic_load_n(&r->drop_at, __ATOMIC_RELAXED) != UINT64_MAX || addr < r->lo || addr + n > r->hi) return NULL;
return (r->host && U.arena->cpu ? U.arena->cpu : U.arena->map) + r->off + (addr - r->lo);
}
// The arenas back on the free list (vk_exec_draw_shutdown: the device is idle, every completion ran).
void vk_usnap_shutdown(void) {
plat_mutex_lock(&free_lock);
if (PV.arena) PV.arena->next = free_arenas, free_arenas = PV.arena, PV.arena = NULL;
for (Arena *a = free_arenas, *next; a; a = next) next = a->next, vmaDestroyBuffer(vk_ctx()->vma, a->buf, a->alloc), free(a->cpu), free(a);
free_arenas = NULL;
plat_mutex_unlock(&free_lock);
}