diff --git a/runtime/vk.h b/runtime/vk.h index 2ad113f9..e96ae1ad 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -283,12 +283,19 @@ void vk_usnap_drop(uint64_t addr, uint64_t n); // the replay makes the GPU wri void vk_usnap_drop_at(uint64_t addr, uint64_t n, uint32_t at); uint64_t vk_usnap_gen(void); // changes whenever a binding may resolve differently void vk_usnap_shutdown(void); // vk_exec_draw_shutdown: frees the arenas +// The replay drew into the scene depth (vk_mvprev.c): with the per-object motion, this submission's arena is kept as +// the last-frame copy until the next scene submission's fence. +void vk_usnap_scene(void); +// [addr, addr+n) in the last scene submission's snapshot (last frame's blocks for the per-object motion): a descriptor +// (offset a multiple of align) and its bytes in host memory; 0: not in it. +int vk_usnap_prev(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out, const uint8_t **bytes); // vk_mvprev.c: last frame's copy of the scene's object (c7) and bone (c5) blocks, for per-object motion vectors. // vk_mvprev_step: the distance of the game's other copy of a vertex-stage block the snapshot also copies, 0: none. // vk_mvprev_find: last frame's block of that object (prev, its address; how, MVPREV_*), 0: none. enum { MVPREV_NONE, MVPREV_SELF, MVPREV_SIDE, MVPREV_NEARER }; uint64_t vk_mvprev_step(int stage, int index, uint64_t size); int vk_mvprev_on(void); +int vk_mvprev_checking(void); // TOTK_MVPREV_CHECK on: the snapshot also copies the blocks' neighbours void vk_mvprev_want(uint32_t index_mask); // also keep these binding indices' last-frame copies (bit n: index n) // DLAA on and TOTK_DLAA_OBJMV not 0, as of the last present int vk_mvprev_find(int index, uint64_t addr, uint64_t size, uint64_t *prev, int *how); void vk_mvprev_frame(void); // each present, beside vk_dlaa_frame diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 6af8a2e1..305d67a3 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -3575,6 +3575,7 @@ static struct { uint64_t no_out, storage, no_fs; // variant without the extra output, with storage resources; no fs uint64_t other_cam; // listed draws left out: their camera block is not the scene camera uint64_t lk[14][5]; // last-frame lookups per binding index: none, self, side, nearer, refused (another object) + uint64_t pv_cmp, pv_diff; // TOTK_MVPREV_CHECK: last-frame copies compared with this snapshot's neighbour, differing uint64_t e_draws, e_pipe, e_dyn, e_push, e_pushw, e_sets, e_vb, e_setw, e_setreuse; // emitted commands (summary) } MO; @@ -3962,13 +3963,19 @@ static int mo_prev_block(int index, const p_bind_buf *u, VkDescriptorBufferInfo uint64_t c1 = __builtin_ia32_rdtsc(); MO.cyc[0] += c1 - c0, MO.ncyc++, MO.big += n; if (index >= 0 && index < 14) MO.lk[index][found ? how : MVPREV_NONE]++; - if (!found || !vk_usnap_find(pa, n, UINT32_MAX, LIM.ubo_align, out)) return 0; - if (how == MVPREV_SELF) return 1; // bound at the same address last frame: not written again - const uint8_t *map = vk_usnap_map(), *cur = vk_usnap_bytes(u->addr, n); + if (!found) return 0; + // bound at the same address last frame: not written again, this frame's copy + if (how == MVPREV_SELF) return vk_usnap_find(pa, n, UINT32_MAX, LIM.ubo_align, out); + const uint8_t *old, *cur; // last frame's from last frame's snapshot (vk_usnap_prev), this frame's + if (!vk_usnap_prev(pa, n, LIM.ubo_align, out, &old)) return 0; + if (vk_mvprev_checking()) { // the neighbour copy the snapshot took before last frame's was kept: the same bytes? + const uint8_t *nb = vk_usnap_bytes(pa, n); + if (nb) MO.pv_cmp++, MO.pv_diff += memcmp(nb, old, n) != 0; + } + cur = vk_usnap_bytes(u->addr, n); uint64_t c2 = __builtin_ia32_rdtsc(); MO.cyc[1] += c2 - c1; if (!cur) return 0; - const uint8_t *old = map + out->offset; static long trust = -2; // TOTK_MVPREV_TRUST= (test): the copy found is taken without the check if (trust == -2) { const char *e = cfg_get("TOTK_MVPREV_TRUST"); @@ -4248,6 +4255,9 @@ int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth (unsigned long long)MO.lk[i][1], (unsigned long long)MO.lk[i][2], (unsigned long long)MO.lk[i][3], (unsigned long long)MO.lk[i][4]); if (o) rt_log("vk_exec: per-object motion last-frame lookups (none/self/side/nearer/refused):%s", l); + if (MO.pv_cmp) + rt_log("vk_exec: per-object motion last-frame copies against the neighbour copies: %llu compared, %llu differ", + (unsigned long long)MO.pv_cmp, (unsigned long long)MO.pv_diff); } if (frames % 60 == 0 && MO.ran) rt_log("vk_exec: per-object motion recorded a frame: %.0f draws, %.0f pipeline binds, %.0f dynamic states, %.0f " diff --git a/runtime/vk_mvprev.c b/runtime/vk_mvprev.c index c5e26f92..e38b9c94 100644 --- a/runtime/vk_mvprev.c +++ b/runtime/vk_mvprev.c @@ -5,22 +5,25 @@ // a neighbour object's copy only by knowing every block bound (noting only the listing's candidates turned a sixth of // the c7 "nearer" picks into "side" ones and doubled the copies mo_similar refuses). The banks the listing never looks // up (c9 and c11, which it leaves at this frame's block, and the banks of shaders it does not draw again) are left out: -// probing them for every scene draw, and copying their neighbours into the snapshot, bought nothing. +// probing them for every scene draw bought nothing. // // The scene's vertex shaders read an object's place from bank c7 (gsys_shape: one 3x4 matrix at the start of a // 0x100 block) and a skinned object's bones from c5 (gsys_skeleton: a run of 3x4 matrices), of this frame only. The // game writes both into two copies that take turns, the second right after the first: an object's block is at addr in // one frame and at addr +- step in the next, step its size rounded up to 256 bytes (vk_mvsurvey.c measured it: every -// c7 pair 0x100 apart, c5 pairs their rounded size apart, the pair's contents the same object moved a little). The -// copy this frame does not write still holds last frame's when the frame is submitted, so the uniform snapshot -// (vk_usnap.c) also copies both neighbours of these blocks, and vk_mvprev_find picks last frame's: -// self the block was bound at the same address last frame too (an object the game did not write again): itself +// c7 pair 0x100 apart, c5 pairs their rounded size apart, the pair's contents the same object moved a little). +// vk_mvprev_find picks last frame's address, and the bytes come from last frame's uniform snapshot (vk_usnap_prev: the +// scene submission's arena is kept one more frame), which holds what last frame's draws read there: +// self the block was bound at the same address last frame too (an object the game did not write again): itself, +// from this frame's snapshot // side one neighbour was bound in that bank last frame: that one // nearer both were (the copies of adjacent objects interleave): the one with more of its first 16 words equal to // this frame's // none neither: a new object, no motion of its own -// The pick is checked against what was snapshotted of that block last frame (intact: the same first CHECK bytes), and -// a summary goes to the log every SUMMARY frames: +// Until 2026-11-20 the snapshot copied both neighbours of every such block instead, as the copy this frame does not +// write still holds last frame's when the frame is submitted: a third of the snapshot's bytes. TOTK_MVPREV_CHECK=1 +// still copies them and checks that the neighbour picked holds what was snapshotted of that block last frame (intact: +// the same first CHECK bytes), so the two sources agree; a summary goes to the log every SUMMARY frames: // mv_prev: c: blocks: self a %, side b %, nearer c %, none d %; intact e %; moved: still f %, under // one unit g %, more h % #include @@ -81,6 +84,7 @@ static Seen *find(Seen *t, uint64_t addr, int index, int add) { } int vk_mvprev_on(void) { return P.on > 0; } +int vk_mvprev_checking(void) { return P.on > 0 && P.bytes[0]; } void vk_mvprev_want(uint32_t index_mask) { static int fixed = -1; @@ -89,7 +93,7 @@ void vk_mvprev_want(uint32_t index_mask) { } uint64_t vk_mvprev_step(int stage, int index, uint64_t size) { - if (P.on <= 0 || stage != 0 || index < 0 || index >= BANKS || !(P.mask >> index & 1)) return 0; + if (P.on <= 0 || !P.bytes[0] || stage != 0 || index < 0 || index >= BANKS || !(P.mask >> index & 1)) return 0; return (size + 255) & ~(uint64_t)255; } @@ -113,9 +117,11 @@ int vk_mvprev_find(int index, uint64_t addr, uint64_t size, uint64_t *prev, int if (addr >= step && find(P.t[1], addr - step, index, 0)) cand[n++] = addr - step; if (!n) return 0; if (n == 2) { // the copies of adjacent objects interleave: the one with more words equal to this frame's - const uint32_t *c = (const uint32_t *)vk_usnap_bytes(addr, 64), *a = (const uint32_t *)vk_usnap_bytes(cand[0], 64), - *b = (const uint32_t *)vk_usnap_bytes(cand[1], 64); - if (!c || !a || !b) return 0; + VkDescriptorBufferInfo bi; + const uint8_t *pa, *pb; + const uint32_t *c = (const uint32_t *)vk_usnap_bytes(addr, 64); + if (!c || !vk_usnap_prev(cand[0], 64, 1, &bi, &pa) || !vk_usnap_prev(cand[1], 64, 1, &bi, &pb)) return 0; + const uint32_t *a = (const uint32_t *)pa, *b = (const uint32_t *)pb; int na = 0, nb = 0; for (int w = 0; w < 16; w++) na += c[w] == a[w], nb += c[w] == b[w]; if (nb > na) cand[0] = cand[1]; @@ -139,6 +145,7 @@ void vk_mvprev_draw(VkExec *ex) { static void mvprev_draw_(VkExec *ex) { const VkExecTarget *z = &ex->depth; if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return; + vk_usnap_scene(); // this submission's snapshot is next frame's last-frame copy int check = P.bytes[0] != NULL; // TOTK_MVPREV_CHECK=1: the summary's checks (3.6 ms a frame) for (uint32_t m = P.mask & ((1u << BANKS) - 1); m; m &= m - 1) { int index = __builtin_ctz(m); diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index f360e130..6e5e1ad9 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -14,6 +14,12 @@ // A chunked replay (vk_exec.c) records the ops out of order: its scan drops at the position of the op // (vk_usnap_drop_at, scan_writes for the shaders' stores), and a chunk finds a copy only at a position before that. // - The arena goes back to a free list once the submission's fence is done (completion thread). +// - With the per-object motion, a scene submission's arena (vk_usnap_scene: the replay drew into the scene depth) is +// kept past its fence as the last-frame copy (PV): the next scene submission's motion draws bind last frame's blocks +// from it (vk_usnap_prev). It goes back to the free list at the fence of the submission that replaces or drops it, +// which is the submission whose motion draws read it (or a later one): its lists are recorded and run within that +// submission (vk_exec_draw.c vk_mvobj_execute drops a list of another fence), and the GPU finishes a submission's +// work, and every earlier one's, before its fence. One arena held beyond those in flight, whatever the session length. // The design follows MacNativeGames' TOTK_UBO_SNAPSHOT (games/totk/third_party/d2r_gfx/gfx/nvn/nvn_mtl.m usnap_take, // usnap_drop; MIT, Copyright (c) 2026 jhaizhou); the code is ours. // The copy is taken where vk_exec_record starts: at the submit call when the replay runs on the submitting thread, later @@ -64,6 +70,13 @@ static struct { } U; static Arena *free_arenas; +// the last scene submission's arena and regions (vk_usnap_prev); its regions' drop_at final +static struct { + Arena *arena; + Reg *reg; + int nreg, creg; +} PV; +static int scene; // vk_usnap_scene since the last vk_usnap_begin static PlatMutex free_lock = PLAT_MUTEX_INIT; static PlatMutex drop_lock = PLAT_MUTEX_INIT; // threads recording chunks of one submission drop concurrently @@ -131,8 +144,8 @@ static void collect(const op_hdr *h, int depth) { const p_bind_buf *b = a; if (b->addr && b->size) { uint64_t n = b->size < MAX_RANGE ? b->size : MAX_RANGE; - // the vertex stage's object and bone blocks: also the game's other copy on either side, which still - // holds last frame's (vk_mvprev.c) + // TOTK_MVPREV_CHECK: the vertex stage's object and bone blocks, also the game's other copy on either + // side, which still holds last frame's (vk_mvprev.c compares it with the last-frame copy) uint64_t step = vk_mvprev_step(b->stage, b->index, b->size); // which neighbours lie in an imported pool, per block and vk_mem_gen: the same blocks are bound by // many draws, and two resolves for each binding were a quarter of the snapshot's time @@ -303,10 +316,47 @@ void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) { } void vk_usnap_end(uint64_t fence) { - if (U.arena) vk_completion_add(fence, arena_done, U.arena), U.arena = NULL; + int keep = __atomic_load_n(&scene, __ATOMIC_RELAXED) && U.arena && U.arena->cpu; + // a scene submission replaces the last-frame copy (or drops it, without a snapshot of its own); so does the motion + // going off, which no longer reads it + if (PV.arena && (__atomic_load_n(&scene, __ATOMIC_RELAXED) || !vk_mvprev_on())) + vk_completion_add(fence, arena_done, PV.arena), PV.arena = NULL, PV.nreg = 0; + if (keep) { + Reg *r = PV.reg; + int c = PV.creg; + PV.arena = U.arena, PV.reg = U.reg, PV.nreg = U.nreg, PV.creg = U.creg; + U.reg = r, U.creg = c, U.nreg = 0; + } else if (U.arena) { + vk_completion_add(fence, arena_done, U.arena); + } + U.arena = NULL; + __atomic_store_n(&scene, 0, __ATOMIC_RELAXED); U.on = 0, U.gen++; } +void vk_usnap_scene(void) { + if (!__atomic_load_n(&scene, __ATOMIC_RELAXED)) __atomic_store_n(&scene, 1, __ATOMIC_RELAXED); +} + +int vk_usnap_prev(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out, const uint8_t **bytes) { + if (!PV.arena || !n) return 0; + int lo = 0, hi = PV.nreg; + while (lo < hi) { // first region whose end is above addr + int m = (lo + hi) / 2; + if (PV.reg[m].hi <= addr) lo = m + 1; + else hi = m; + } + if (lo >= PV.nreg) return 0; + const Reg *r = &PV.reg[lo]; + // a region a GPU write dropped held other bytes for the draws after it: left out, as vk_usnap_bytes does + if (r->drop_at != UINT64_MAX || addr < r->lo || addr + n > r->hi) return 0; + uint64_t off = r->off + (addr - r->lo); + if (off % align) return 0; + *out = (VkDescriptorBufferInfo){PV.arena->buf, off, n}; + *bytes = PV.arena->cpu + off; + return 1; +} + // First region whose end is above a. static int first_above(uint64_t a) { int lo = 0, hi = U.nreg; @@ -364,6 +414,7 @@ const uint8_t *vk_usnap_bytes(uint64_t addr, uint64_t n) { // The arenas back on the free list (vk_exec_draw_shutdown: the device is idle, every completion ran). void vk_usnap_shutdown(void) { plat_mutex_lock(&free_lock); + if (PV.arena) PV.arena->next = free_arenas, free_arenas = PV.arena, PV.arena = NULL; for (Arena *a = free_arenas, *next; a; a = next) next = a->next, vmaDestroyBuffer(vk_ctx()->vma, a->buf, a->alloc), free(a->cpu), free(a); free_arenas = NULL; plat_mutex_unlock(&free_lock);