Merge the replay of NVN submissions on a queued thread, with reads waiting for the newest writer instead of a drain, behind TOTK_REPLAY_ASYNC

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-26 08:22:35 +03:00
commit 84abfaaf2c
12 files changed
+558 -37

No files matched your search

+3
View File
@@ -14,6 +14,7 @@
#include "cfg.h"
#include "gfx.h"
#include "nvn_api.h"
#include "vk.h"
#define SYM_INIT_START "_ZN2nn4init5StartEmmPFvvES2_S2_"
@@ -200,6 +201,7 @@ static void save_shot(const tex_t *t, const char *path) {
view_t view;
memset(&view, 0, sizeof view);
NVNcopyRegion r = {.width = (int)w, .height = (int)h, .depth = 1};
if (gfx_replay_async()) gfx_backend()->finish(); // the presented frame may still be queued for the replay
gfx_backend()->read_texels(t, &view, &r, px, 4ull * w, 4ull * w * h);
if (bgra)
for (uint64_t i = 0; i < 4ull * w * h; i += 4) {
@@ -238,6 +240,7 @@ static void series_shot(const tex_t *t, uint64_t n) {
view_t view;
memset(&view, 0, sizeof view);
NVNcopyRegion r = {.width = (int)w, .height = (int)h, .depth = 1};
if (gfx_replay_async()) gfx_backend()->finish(); // the presented frame may still be queued for the replay
gfx_backend()->read_texels(t, &view, &r, px, 4ull * w, 4ull * w * h);
for (uint32_t y = 0; y < oh; y++)
for (uint32_t x = 0; x < ow; x++) {
+1 -1
View File
@@ -34,7 +34,7 @@ static const char *const known[] = {
"MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
"PREWARM_XLATE",
"PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC",
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS",
"SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL",
"SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS",
+2
View File
@@ -28,6 +28,7 @@ void prof_write(void); // idempotent; boot.c calls it b
uint64_t perf_begin(void); // start time of a wait; pair with perf_end
void perf_end(int kind, uint64_t t0); // counts render-thread time only
void perf_replay(uint64_t t0); // replay (NVN -> Vulkan recording) time on any thread, waits inside it included
void perf_replay_thread(void); // the calling thread replays for the render thread: perf_end counts its waits
// Events per 60-frame window: graphics pipelines finished by the background workers, draws skipped while pending,
// first texture uploads and re-uploads after CPU writes, draws skipped while a shader translates in the background,
// draws that waited for their pending pipeline and got it, draws into a target the previous frame did not render that
@@ -41,6 +42,7 @@ enum { PERF_CNT_PIPE_ASYNC, PERF_CNT_PIPE_SKIP, PERF_CNT_TEX_UP, PERF_CNT_TEX_RE
void perf_count(int kind);
void perf_present(uint64_t frame);
void perf_summary_flush(void); // prints the frame-time summary of the running period (exit); the 60 s lines come from perf_present
void gfx_exit_log(void); // back end's lines for the exit summary (weak no-op in prof_win.c)
void perf_gpu_wait(uint64_t t0); // a wait for a GPU fence ended (any thread; the render thread's also counts as PERF_FENCE)
// GPU side of the 60-frame perf line, filled by the graphics back end when it set perf_gpu_stats. gpu_ns: GPU busy time
// since the previous call (timestamp queries, overlaps counted once), have_gpu 0 when the device has no timestamps.
+5
View File
@@ -51,6 +51,11 @@ void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the ga
// CPU class (0 CPU_CACHED, 1 CPU_UNCACHED, 2 other) since the previous call.
void nvn_flush_stats(uint64_t out[2][3][2]);
void nvn_submit_prepare(const uint64_t *handles, int n); // weak no-op; the command-buffer slice reads deferred binds
// Asynchronous replay: as nvn_submit_prepare, writing the handles to replay to out (a clone where a still queued
// submission uses the same recording with other deferred binds)
void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n);
int gfx_replay_async(void); // 1: the back end replays submissions on its own thread
uint64_t gfx_replayed_submits(void); // newest submit number (nvn_submit_count) replayed
#define NVN_HANDLE_TEX(h) ((int)((h) & 0xfffff)) // combined texture handle = texture id | sampler id << 20
#define NVN_HANDLE_SAMP(h) ((int)(((h) >> 20) & 0xfff))
extern GfxStats gfx_counters; // behind gfx_stats(); add with __atomic_fetch_add. frames:
+74 -6
View File
@@ -160,7 +160,7 @@ static const tex_t *tex_pin(const NVNtexture *nt) {
// ---------- recording copies
// A handle points just past a copy_prefix. Copies are indexed by the 4 KiB page of their control-memory position, so
// adding control memory again releases the copies recorded there without walking every live copy.
typedef struct { uint64_t magic, bytes, made_at, used_at; } copy_prefix;
typedef struct { uint64_t magic, bytes, made_at, used_at, deferred; } copy_prefix; // deferred: 0 unknown, 1 none, 2 some
_Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned");
typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy;
typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket;
@@ -168,6 +168,30 @@ static SRWLOCK copy_lock = SRWLOCK_INIT; // guards the two maps, the counters
static nvn_hmap copy_pages = {.klen = 8}; // page -> copy_bucket
static nvn_hmap copy_live = {.klen = 8}; // handle -> copy_prefix of a live copy
static uint64_t ncopies, copy_bytes, copies_made, nsubmits;
// Asynchronous replay: copies released by the game, and per-submission clones, that a queued submission still reads.
// Freed once gfx_replayed_submits() reaches their used_at. Empty with synchronous replay.
static copy_prefix **zombies;
static uint32_t nzombies, cap_zombies;
static void zombie_add(copy_prefix *pf) { // copy_lock held
if (nzombies == cap_zombies) {
cap_zombies = cap_zombies ? cap_zombies * 2 : 64;
zombies = realloc(zombies, sizeof *zombies * cap_zombies);
if (!zombies) rt_fatal("nvn: out of memory (released recording copies)");
}
zombies[nzombies++] = pf;
}
static void zombie_sweep(uint64_t replayed) { // copy_lock held
for (uint32_t i = 0; i < nzombies;) {
if (zombies[i]->used_at <= replayed) {
free(zombies[i]);
zombies[i] = zombies[--nzombies];
} else {
i++;
}
}
}
static copy_prefix *copy_find(uint64_t handle) { // copy_lock held
return nvn_hm_get(&copy_live, &handle);
@@ -258,7 +282,9 @@ static void copy_add(uintptr_t lo, uintptr_t hi, uint8_t *copy) {
// Adding control memory again retires every recording copy made in it (the driver's handle lived there).
static void release_copies(uintptr_t lo, uintptr_t hi) {
if (hi <= lo) return;
uint64_t replayed = gfx_replayed_submits();
AcquireSRWLockExclusive(&copy_lock);
if (nzombies) zombie_sweep(replayed);
for (uintptr_t pg = lo >> COPY_PAGE_SHIFT; pg <= (hi - 1) >> COPY_PAGE_SHIFT; pg++) {
copy_bucket *b = nvn_hm_get(&copy_pages, &pg);
if (!b) continue;
@@ -271,7 +297,8 @@ static void release_copies(uintptr_t lo, uintptr_t hi) {
ncopies--;
copy_bytes -= pf->bytes;
pf->magic = 0;
free(pf);
if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it
else free(pf);
b->v[i] = b->v[--b->n];
}
if (!b->n) {
@@ -836,16 +863,17 @@ NVN_API(nvnCommandBufferSetPatchSize) { (void)c; }
// What the console reads at the submit call is read here, on the game thread: every handle must be a live recording,
// and deferred binds take the handles the game's arrays hold now (the game fills its tables for the next frame as soon
// as the submit returns). The result is written into the recording copy.
static uint64_t def_reads, def_changed;
static uint64_t def_reads, def_changed, n_clones;
static void resolve_deferred(const uint8_t *p, int depth) {
static int resolve_deferred(const uint8_t *p, int depth) { // returns the number of deferred binds
int nd = 0;
for (;;) {
op_hdr h;
memcpy(&h, p, sizeof h);
if (h.size < 8 || (h.size & 7) || h.op >= OP_COUNT_)
rt_fatal("nvn: corrupt recording at %p while resolving deferred binds (op %u size %u)", (const void *)p,
h.op, h.size);
if (h.op == OP_END) return;
if (h.op == OP_END) return nd;
if (h.op == OP_JUMP) {
memcpy(&p, p + 8, 8);
continue;
@@ -854,10 +882,11 @@ static void resolve_deferred(const uint8_t *p, int depth) {
uint64_t t;
memcpy(&t, p + 8, 8);
if (depth >= 16) rt_fatal("nvn: deferred bind resolution: call depth over 16");
resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1);
nd += resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1);
} else if (h.op == OP_BIND_TEX || h.op == OP_BIND_IMG || h.op == OP_BIND_SAMP) {
p_bind_handles *b = (p_bind_handles *)(uintptr_t)(p + 8);
if (b->src && b->n > 0 && b->n <= 32) {
nd++;
uint64_t cur[32];
memcpy(cur, (const void *)(uintptr_t)b->src, 8 * (size_t)b->n);
__atomic_fetch_add(&def_reads, 1, __ATOMIC_RELAXED);
@@ -879,6 +908,45 @@ void nvn_submit_prepare(const uint64_t *handles, int n) {
for (int i = 0; i < n; i++) resolve_deferred((const uint8_t *)(uintptr_t)handles[i], 0);
}
// Asynchronous replay. A copy a queued submission still reads keeps that submission's deferred binds: when it holds
// deferred binds (or is not known to hold none), this submission gets a clone, resolved here and freed after its replay.
void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) {
uint64_t now = __atomic_add_fetch(&nsubmits, 1, __ATOMIC_RELAXED);
uint64_t replayed = gfx_replayed_submits();
AcquireSRWLockExclusive(&copy_lock);
if (nzombies) zombie_sweep(replayed);
for (int i = 0; i < n; i++) {
copy_prefix *pf = copy_find(handles[i]);
if (!pf) {
ReleaseSRWLockExclusive(&copy_lock);
rt_fatal("nvn: QueueSubmitCommands: command handle %#llx is not a live recording (released or never recorded)",
(unsigned long long)handles[i]);
}
out[i] = handles[i];
if (pf->used_at > replayed && pf->used_at != now && pf->deferred != 1) {
size_t sz = sizeof *pf + pf->bytes;
copy_prefix *cl = malloc(sz);
if (!cl) rt_fatal("nvn: out of memory (recording clone, %zu bytes)", sz);
memcpy(cl, pf, sz);
cl->magic = 0; // not a live handle: the game never sees it
cl->used_at = now;
zombie_add(cl);
out[i] = (uintptr_t)(cl + 1);
__atomic_fetch_add(&n_clones, 1, __ATOMIC_RELAXED);
} else {
pf->used_at = now;
}
}
ReleaseSRWLockExclusive(&copy_lock);
for (int i = 0; i < n; i++) {
int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0);
if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1;
}
if (now % 7200 == 0)
rt_log("nvn: asynchronous replay at submit %llu: %llu recording clones so far, %u copies waiting for the replay",
(unsigned long long)now, (unsigned long long)__atomic_load_n(&n_clones, __ATOMIC_RELAXED), nzombies);
}
uint64_t nvn_submit_count(void) { return __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); }
const char *nvn_op_name(uint32_t op) {
+24 -3
View File
@@ -58,6 +58,11 @@ static void ov_memcpy(Cpu *c) {
HOST_OVERRIDE("memcpy", ov_memcpy);
__attribute__((weak)) void nvn_submit_prepare(const uint64_t *handles, int n) { (void)handles, (void)n; }
__attribute__((weak)) void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) {
memcpy(out, handles, sizeof *out * (size_t)n);
}
__attribute__((weak)) int gfx_replay_async(void) { return 0; }
__attribute__((weak)) uint64_t gfx_replayed_submits(void) { return nvn_submit_count(); }
// ---------- pools: GPU window (section 3.1) and the address index
static SRWLOCK pool_lock = SRWLOCK_INIT; // guards gva and index
@@ -560,7 +565,9 @@ NVN_API(nvnQueueFinish) {
static uint64_t lod_entered, lod_issued, lod_fence[LOD_RING];
// void nvnQueueSubmitCommands(NVNqueue *q, int n, const NVNcommandHandle *handles): replayed on this thread with its
// core given up (section 2.2); the newest fence value is what QueueFenceSync records
// core given up (section 2.2), or queued for the replay thread (TOTK_REPLAY_ASYNC, vk_backend.c); the newest fence
// value is what QueueFenceSync records. With the queue the fence covers replay and GPU, so the LOD guard below waits
// for both.
NVN_API(nvnQueueSubmitCommands) {
queue_t *q = arg_p(c, 0);
int n = (int)arg_x(c, 1);
@@ -569,8 +576,22 @@ NVN_API(nvnQueueSubmitCommands) {
NVN_BLOCKING();
uint64_t no = __atomic_add_fetch(&lod_entered, 1, __ATOMIC_SEQ_CST);
q->submits++;
nvn_submit_prepare(h, n);
uint64_t f = gfx_backend()->submit(h, n), prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED);
uint64_t f;
if (gfx_replay_async()) {
// Prepare and queue under one lock: submit numbers follow queue order (gfx_replayed_submits).
static SRWLOCK submit_lock = SRWLOCK_INIT;
uint64_t local[32], *hs = n <= 32 ? local : malloc(sizeof *hs * (size_t)n);
if (!hs) rt_fatal("nvn: out of memory (submit of %d handles)", n);
AcquireSRWLockExclusive(&submit_lock);
nvn_submit_prepare_async(h, hs, n);
f = gfx_backend()->submit(hs, n);
ReleaseSRWLockExclusive(&submit_lock);
if (hs != local) free(hs);
} else {
nvn_submit_prepare(h, n);
f = gfx_backend()->submit(h, n);
}
uint64_t prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED);
while (f > prev && !__atomic_compare_exchange_n(&last_fence, &prev, f, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {}
__atomic_store_n(&lod_fence[no % LOD_RING], f, __ATOMIC_RELAXED);
__atomic_store_n(&lod_issued, no, __ATOMIC_SEQ_CST);
+20 -4
View File
@@ -269,11 +269,22 @@ static uint64_t wait_ns[PERF_N], frame_ns[PERF_N], worst_ns[PERF_N]; // window
uint64_t perf_begin(void) { return kern_now_ns(); }
static volatile DWORD replay_tid; // asynchronous replay (vk_backend.c): its waits hold up the frame too
void perf_replay_thread(void) { replay_tid = GetCurrentThreadId(); }
void perf_end(int kind, uint64_t t0) {
if (!t0 || GetCurrentThreadId() != render_tid) return;
if (!t0) return;
DWORD tid = GetCurrentThreadId();
if (tid != render_tid && tid != replay_tid) return;
uint64_t d = kern_now_ns() - t0;
wait_ns[kind] += d;
frame_ns[kind] += d;
if (tid == render_tid && !replay_tid) { // one thread adds: no atomics on the synchronous path
wait_ns[kind] += d;
frame_ns[kind] += d;
return;
}
__atomic_fetch_add(&wait_ns[kind], d, __ATOMIC_RELAXED);
__atomic_fetch_add(&frame_ns[kind], d, __ATOMIC_RELAXED);
}
static volatile LONG64 other_wait_ns; // perf_gpu_wait on threads other than the render thread, over the window
@@ -339,8 +350,13 @@ static void ft_report(uint64_t now, const char *tag) {
ft.t0 = now;
}
__attribute__((weak)) void gfx_exit_log(void) {}
void perf_summary_flush(void) { // at exit (prof_write)
if (summary_t0) ft_report(kern_now_ns(), "exit");
if (summary_t0) {
ft_report(kern_now_ns(), "exit");
gfx_exit_log();
}
summary_t0 = 0;
}
+6
View File
@@ -261,6 +261,9 @@ VkSampler vk_sampler_by_id_locked(int id);
uint32_t vk_texture_slots_gen(void);
uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered
void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage
void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission
uint64_t vk_texture_last_write(const tex_t *t); // timeline value of the newest submission recorded to write it; 0: none
size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img); // bytes ReadTexels spans; 0 for a bad region
int vk_texture_small_dirty(void); // a small mirrored target holds GPU texels its storage lacks
void vk_texture_write_back_small(uint64_t fence); // writes those rendered by work up to fence back to storage
void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written); // vk_dump
@@ -268,6 +271,7 @@ int vk_texture_slot_swz(int id, uint8_t swz[4]); // vk_dump: swizzle of the d
int vk_texture_read_image(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row,
uint64_t img); // always from the image (GPU copy, waited for); 0 on failure
int vk_texture_cpu_stale(uint64_t addr, uint64_t size); // a GPU-written mirrored texture there has stale storage
uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size); // newest recorded writer of such a texture; 0: none
void vk_texture_write_back(uint64_t addr, uint64_t size); // writes those images back into their storage (waits)
void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); // vk_pool_fini: drop the images there
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads);
@@ -286,6 +290,7 @@ void rt_write_png(const char *path, const uint8_t *rgba, uint32_t w, uint32_t h)
// once the GPU reached it and the completion thread ran its side effects. Guest or host threads.
void vk_sync_start(void); // vk_init (fatal without a device) and the completion thread; idempotent
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
uint64_t vk_sync_fence_done(void); // highest done value
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
@@ -308,6 +313,7 @@ void rdoc_init(void); // runtime/rdoc_win.c: TOTK_RDOC debug
int rdoc_active(void);
void rdoc_after_present(uint64_t frame);
void vk_present_drain(void); // returns once every queued present was submitted to gfx_queue
int vk_replay_async(void); // TOTK_REPLAY_ASYNC (default 1): submissions replay on the replay thread
void vk_program_release(const prog_t *p); // GfxBackend.program_release: pipelines (T6b); weak no-op in vk_backend.c
void vk_backend_shutdown(void); // presents, vk_sync_shutdown, vk_mem_shutdown, vk_shutdown in that order
+366 -21
View File
@@ -10,8 +10,19 @@
// Back-edge pacing: the present thread submits the blit at once, then holds the host present (vkQueuePresentKHR) until
// the frame's display time from the pacer (nvn_api_obj.c), so present-to-present intervals stay even whatever the
// replay took. The blit is already on the queue, so the wait blocks no submission.
//
// Asynchronous replay (TOTK_REPLAY_ASYNC, default 1; 0 keeps the synchronous path unchanged): nvnQueueSubmitCommands
// and nvnQueuePresentTexture only queue. Each item gets its timeline value at enqueue (vk_sync_reserve), in queue
// order; the replay thread runs the synchronous submit and present bodies in that order, so every value is signalled
// by the submission it was handed out for. The game runs at most one frame ahead: queueing present N waits until the
// replay thread handled present N-1. Calls that read or free GPU state that queued work may use drain the queue first
// (replay_drain), each cause counted. Design and the orderings it keeps: build/v121-replayq/NOTES.md.
#include <windows.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cfg.h"
#include "flight.h"
#include "gfx.h"
#include "host.h"
@@ -21,7 +32,7 @@
#define WEAK __attribute__((weak))
typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued; } PresentJob;
typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued, value; } PresentJob;
static SRWLOCK pl = SRWLOCK_INIT;
static CONDITION_VARIABLE pcv = CONDITION_VARIABLE_INIT; // job queued, job taken, job submitted
@@ -52,7 +63,10 @@ static DWORD WINAPI present_main(void *arg) {
ReleaseSRWLockExclusive(&pl);
if (flight_on) vk_thumbs_frame(j.frame);
vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.w, j.h, j.has_crop ? j.crop : NULL);
vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too
uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too
if (j.value && f != j.value)
rt_fatal("vk: the present of frame %llu signalled %llu, queued as %llu", (unsigned long long)j.frame,
(unsigned long long)f, (unsigned long long)j.value);
AcquireSRWLockExclusive(&pl);
busy = 0;
WakeAllConditionVariable(&pcv);
@@ -84,11 +98,11 @@ void vk_present_drain(void) {
perf_end(PERF_DRAIN, p0);
}
static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
// The present after the frame count: resolves the image and hands the blit to the present thread. queued: when the game
// presented; value: the timeline value queued for this present (asynchronous replay), 0 for the next one (synchronous).
static void present_image(const tex_t *t, const int crop[4], uint64_t at_ns, uint64_t queued, uint64_t frame,
uint64_t value) {
static volatile long n_skipped;
vk_sync_start();
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads
vk_exec_frame_log(t, frame);
vk_mem_census(frame);
if (vk_dump_armed) vk_dump_present(frame);
@@ -97,10 +111,15 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
if (!img || !(img->aspect & VK_IMAGE_ASPECT_COLOR_BIT)) {
if (InterlockedIncrement(&n_skipped) <= 8)
rt_log("vk: present of texture %p (format %#x) skipped: no colour image", (const void *)t, t ? t->d.format : 0);
if (value) { // the queued value is still signalled, in order
vk_present_drain();
uint64_t f = vk_sync_submit(NULL, 0);
if (f != value) rt_fatal("vk: skipped present signalled %llu, queued as %llu", (unsigned long long)f, (unsigned long long)value);
}
return;
}
PresentJob j = {.image = img->image, .format = img->format, .w = (int)img->width, .h = (int)img->height, .frame = frame,
.at = at_ns, .queued = kern_now_ns()};
.at = at_ns, .queued = queued, .value = value};
if (crop) {
j.has_crop = 1;
for (int i = 0; i < 4; i++) j.crop[i] = crop[i];
@@ -121,45 +140,369 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
if (rdoc_active()) { vk_present_drain(); rdoc_after_present(frame); }
}
// Pipelines (T6b) drop a released program's pipelines here.
WEAK void vk_program_release(const prog_t *p) { (void)p; }
// ---- Asynchronous replay
enum { RI_SUBMIT, RI_PRESENT, RI_RELEASE };
typedef struct {
int kind, n, has_crop, crop[4];
uint64_t value, submit_no, at, queued;
uint64_t *h;
const tex_t *tex; // RI_RELEASE: a copy the item owns (the game may reuse the object's memory)
} ReplayItem;
#define RQ_CAP 1024 // a frame queues 1 submit and 1 present, and up to a few hundred releases
enum { DR_POOL_FINI, DR_MAP_VIRTUAL, DR_TEX_RELEASE, DR_PROG_RELEASE, DR_FINISH, DR_READ_TEXELS, DR_WRITE_TEXELS,
DR_GUEST_READ, DR_FRAME, DR_FULL, DR_CHECK, DR_N };
static const char *const dr_name[DR_N] = {
"pool_fini", "map_virtual", "texture_release (queued)", "program_release", "finish", "read_texels (writer fence)",
"write_texels", "guest_read (writer fence)", "frame bound", "queue full", "check drain"};
static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); }
static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); vk_present_drain(); vk_pool_fini(p); }
static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) {
vk_sync_start();
vk_present_drain();
return vk_pool_map_virtual(v, n, requests);
static SRWLOCK rl = SRWLOCK_INIT;
static CONDITION_VARIABLE rcv = CONDITION_VARIABLE_INIT; // item queued, item taken, item done
static ReplayItem rq[RQ_CAP];
static int rq_head, rq_n, r_busy, r_started;
static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames;
// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run.
static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N];
// TOTK_REPLAY_CHECK: ReadTexels compared with a read after a full drain (under rl): checked, differing, differing
// although no submission recorded a write to the texture in between, differing with the queue idle at the first read.
static uint64_t ck_reads, ck_diff, ck_diff_unexplained, ck_diff_idle;
int vk_replay_async(void) {
static int on = -1;
if (on < 0) {
const char *e = cfg_get("TOTK_REPLAY_ASYNC");
on = !(e && !strcmp(e, "0"));
rt_log("vk: replay %s (TOTK_REPLAY_ASYNC=%d)",
on ? "on its own thread, at most one frame behind the game" : "on the submitting thread", on);
}
return on;
}
static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); }
static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); }
static void vk_be_texture_release(const tex_t *t) { vk_sync_start(); vk_present_drain(); vk_texture_release(t); }
static void vk_be_program_release(const prog_t *p) { vk_program_release(p); }
static uint64_t vk_be_submit(const uint64_t *handles, int n) {
// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end.
int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); }
// The newest submit number (nvn_submit_count) whose replay is done: recording copies it used may be freed.
uint64_t gfx_replayed_submits(void) {
if (!gfx_replay_async()) return nvn_submit_count();
AcquireSRWLockShared(&rl);
uint64_t v = r_submit_done;
ReleaseSRWLockShared(&rl);
return v;
}
static uint64_t submit_sync(const uint64_t *handles, int n) {
vk_sync_start();
vk_present_drain();
vk_texture_new_submit();
return vk_sync_submit(handles, n);
}
static void vk_be_finish(void) { vk_sync_start(); vk_present_drain(); vk_sync_finish(); }
static DWORD WINAPI replay_main(void *arg) {
(void)arg;
perf_replay_thread();
AcquireSRWLockExclusive(&rl);
for (;;) {
while (!rq_n) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
ReplayItem it = rq[rq_head];
rq_head = (rq_head + 1) % RQ_CAP;
rq_n--;
r_busy = 1;
WakeAllConditionVariable(&rcv);
ReleaseSRWLockExclusive(&rl);
if (it.kind == RI_SUBMIT) {
uint64_t f = submit_sync(it.h, it.n);
if (f != it.value)
rt_fatal("vk: replayed submission signalled %llu, queued as %llu", (unsigned long long)f,
(unsigned long long)it.value);
free(it.h);
} else if (it.kind == RI_RELEASE) {
vk_present_drain();
vk_texture_release(it.tex);
free((void *)it.tex);
} else {
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value);
}
AcquireSRWLockExclusive(&rl);
if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no;
else r_presents_done++;
r_busy = 0;
WakeAllConditionVariable(&rcv);
}
return 0;
}
static void wait_counted(int why, uint64_t t0) { // rl held
uint64_t d = kern_now_ns() - t0;
dr_waits[why]++, dr_twaits[why]++;
dr_ns[why] += d, dr_tns[why] += d;
perf_end(PERF_DRAIN, t0);
}
// A guest thread waiting for the replay gives its virtual core up, as around any blocking host wait (NVN_BLOCKING):
// guest_read (memcpy, pool invalidate), pool and texture finalize run without it, and a drain lasts up to a replay.
// Called and returning with rl held; rl is not held while the core is taken back.
static int core_away_unlocked(void) {
ReleaseSRWLockExclusive(&rl);
int away = kern_core_away();
AcquireSRWLockExclusive(&rl);
return away;
}
static void core_back_unlocked(int *away) {
if (!*away) return;
ReleaseSRWLockExclusive(&rl);
kern_core_back(away);
AcquireSRWLockExclusive(&rl);
}
// Queues an item (waits while the queue is full) and returns its value, reserved under rl so values follow queue order.
static uint64_t replay_enqueue(ReplayItem *it) {
AcquireSRWLockExclusive(&rl);
if (!r_started) {
HANDLE h = CreateThread(NULL, 0, replay_main, NULL, 0, NULL);
if (!h) rt_fatal("vk: cannot start the replay thread (error %lu)", GetLastError());
CloseHandle(h);
r_started = 1;
}
// rl is dropped while the core is taken back, and another thread may fill the queue again meanwhile: check again.
while (rq_n == RQ_CAP) {
uint64_t p0 = perf_begin();
dr_calls[DR_FULL]++, dr_tcalls[DR_FULL]++;
int away = core_away_unlocked();
while (rq_n == RQ_CAP) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
wait_counted(DR_FULL, p0);
core_back_unlocked(&away);
}
it->value = it->kind == RI_RELEASE ? 0 : vk_sync_reserve(); // a release signals nothing
rq[(rq_head + rq_n) % RQ_CAP] = *it;
rq_n++;
WakeAllConditionVariable(&rcv);
ReleaseSRWLockExclusive(&rl);
return it->value;
}
// Returns once every queued item was replayed and submitted. Back-end entries only: the replay thread never calls them.
static void replay_drain(int why) {
if (!gfx_replay_async()) return;
AcquireSRWLockExclusive(&rl);
dr_calls[why]++, dr_tcalls[why]++;
if (rq_n || r_busy) {
uint64_t p0 = perf_begin();
int away = core_away_unlocked();
while (rq_n || r_busy) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
wait_counted(why, p0);
core_back_unlocked(&away);
}
ReleaseSRWLockExclusive(&rl);
}
// What made the game wait for the replay: since the last line (every 1800 game frames), or over the run (at exit).
static void replay_log(uint64_t frame, int total) {
char buf[1024];
int k = 0;
AcquireSRWLockExclusive(&rl);
const uint64_t *c = total ? dr_tcalls : dr_calls, *w = total ? dr_twaits : dr_waits, *t = total ? dr_tns : dr_ns;
for (int i = 0; i < DR_N && k < (int)sizeof buf - 96; i++)
if (c[i] || w[i])
k += snprintf(buf + k, sizeof buf - (size_t)k, "%s %s %llu calls, %llu waited %.1f ms", k ? ";" : "", dr_name[i],
(unsigned long long)c[i], (unsigned long long)w[i], t[i] / 1e6);
if (ck_reads && k < (int)sizeof buf - 128)
k += snprintf(buf + k, sizeof buf - (size_t)k, "; check (run): %llu reads, %llu differ, %llu without a recorded write, %llu with the queue idle",
(unsigned long long)ck_reads, (unsigned long long)ck_diff, (unsigned long long)ck_diff_unexplained,
(unsigned long long)ck_diff_idle);
memset(dr_calls, 0, sizeof dr_calls);
memset(dr_waits, 0, sizeof dr_waits);
memset(dr_ns, 0, sizeof dr_ns);
ReleaseSRWLockExclusive(&rl);
rt_log("vk: replay queue %s frame %llu:%s", total ? "over the run to" : "at", (unsigned long long)frame,
k ? buf : " no waits");
}
// The run total, once: the game exits through prof_write (boot.c _exit, the window close), probes through
// vk_backend_shutdown.
void gfx_exit_log(void) {
static LONG done;
if (gfx_replay_async() && r_started && !InterlockedExchange(&done, 1)) replay_log(game_frames, 1);
}
static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
vk_sync_start();
if (!gfx_replay_async()) {
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads
present_image(t, crop, at_ns, kern_now_ns(), frame, 0);
return;
}
// Frame bound: the replay thread handled the previous present before this one is queued.
AcquireSRWLockExclusive(&rl);
dr_calls[DR_FRAME]++, dr_tcalls[DR_FRAME]++;
if (r_presents_done < r_presents_queued) {
uint64_t p0 = perf_begin();
while (r_presents_done < r_presents_queued) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
wait_counted(DR_FRAME, p0);
}
r_presents_queued++;
ReleaseSRWLockExclusive(&rl);
uint64_t frame = ++game_frames;
perf_present(frame);
if (frame % 1800 == 0) replay_log(frame, 0);
ReplayItem it = {.kind = RI_PRESENT, .tex = t, .at = at_ns, .queued = kern_now_ns()};
if (crop) {
it.has_crop = 1;
for (int i = 0; i < 4; i++) it.crop[i] = crop[i];
}
replay_enqueue(&it);
}
// Pipelines (T6b) drop a released program's pipelines here.
WEAK void vk_program_release(const prog_t *p) { (void)p; }
static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); }
static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); replay_drain(DR_POOL_FINI); vk_present_drain(); vk_pool_fini(p); }
static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) {
vk_sync_start();
replay_drain(DR_MAP_VIRTUAL);
vk_present_drain();
return vk_pool_map_virtual(v, n, requests);
}
static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); }
static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); }
// Asynchronous: the release is queued behind the submissions before it, so their replay still resolves the texture's
// slots to its image (a released slot follows another live texture at the storage), and the ones after it see it
// released, the order of the synchronous replay. vk_texture_release reads only the object's key fields: a copy.
static void vk_be_texture_release(const tex_t *t) {
vk_sync_start();
if (!gfx_replay_async()) {
vk_present_drain();
vk_texture_release(t);
return;
}
if (!t || !t->mem) return;
tex_t *copy = malloc(sizeof *copy);
if (!copy) rt_fatal("vk: out of memory (replay queue)");
*copy = *t;
ReplayItem it = {.kind = RI_RELEASE, .tex = copy};
AcquireSRWLockExclusive(&rl);
dr_calls[DR_TEX_RELEASE]++, dr_tcalls[DR_TEX_RELEASE]++;
ReleaseSRWLockExclusive(&rl);
replay_enqueue(&it);
}
static void vk_be_program_release(const prog_t *p) { replay_drain(DR_PROG_RELEASE); vk_program_release(p); }
// Asynchronous: the handles (nvn_submit_prepare_async ran under the front end's submit lock) are copied and queued;
// the value returned is the one the replay will signal.
static uint64_t vk_be_submit(const uint64_t *handles, int n) {
if (!gfx_replay_async()) return submit_sync(handles, n);
vk_sync_start();
ReplayItem it = {.kind = RI_SUBMIT, .n = n, .submit_no = nvn_submit_count()};
if (n > 0) {
it.h = malloc(sizeof *it.h * (size_t)n);
if (!it.h) rt_fatal("vk: out of memory (replay queue)");
memcpy(it.h, handles, sizeof *it.h * (size_t)n);
}
return replay_enqueue(&it);
}
static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); }
static int replay_check(void) {
static int on = -1;
if (on < 0) {
const char *e = cfg_get("TOTK_REPLAY_CHECK");
on = e && strcmp(e, "0");
if (on) rt_log("vk: TOTK_REPLAY_CHECK: every ReadTexels is read again after a full replay drain and compared");
}
return on;
}
// ReadTexels with the replay queued: waits for the newest submission recorded to write the texture (replayed and
// completed), not for the whole queue. The readback copy goes through the upload batch on the same queue, after every
// submission the replay has submitted. Submissions still queued or being recorded run after the copy, as on hardware
// when the GPU has not reached them yet: the game only sees their texels after waiting on a sync, and every sync
// value covers replay and GPU, so a read the game synchronised sees them. TOTK_REPLAY_CHECK compares with a full drain.
static void vk_be_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row,
uint64_t img) {
vk_sync_start();
if (!gfx_replay_async()) {
vk_read_texels(t, v, r, dst, row, img);
return;
}
uint64_t f = vk_texture_last_write(t);
AcquireSRWLockExclusive(&rl);
dr_calls[DR_READ_TEXELS]++, dr_tcalls[DR_READ_TEXELS]++;
int idle = !rq_n && !r_busy;
ReleaseSRWLockExclusive(&rl);
if (f && vk_sync_fence_done() < f) {
uint64_t t0 = kern_now_ns();
vk_sync_fence_wait(f, UINT64_MAX); // counts as a fence wait in perf
uint64_t d = kern_now_ns() - t0;
AcquireSRWLockExclusive(&rl);
dr_waits[DR_READ_TEXELS]++, dr_twaits[DR_READ_TEXELS]++;
dr_ns[DR_READ_TEXELS] += d, dr_tns[DR_READ_TEXELS] += d;
ReleaseSRWLockExclusive(&rl);
}
vk_read_texels(t, v, r, dst, row, img);
if (!replay_check()) return;
size_t n = vk_texture_texel_span(t, v, r, row, img);
uint8_t *again = n ? malloc(n) : NULL;
if (!again) return;
memcpy(again, dst, n); // bytes between the rows stay the caller's
uint64_t f0 = vk_texture_last_write(t);
replay_drain(DR_CHECK);
uint64_t f1 = vk_texture_last_write(t);
vk_read_texels(t, v, r, again, row, img);
int differ = memcmp(again, dst, n) != 0;
static volatile long n_logged;
if (differ && InterlockedIncrement(&n_logged) <= 40) {
size_t at = 0;
while (at < n && again[at] == dst[at]) at++;
rt_log("vk: replay check: ReadTexels of %p (format %#x, %dx%dx%d at %d,%d,%d) differs at byte %zu of %zu; writer "
"fence %llu at the read, %llu after the drain, done %llu; queue %s at the read",
(void *)t->mem, t->d.format, r->width, r->height, r->depth, r->xoffset, r->yoffset, r->zoffset, at, n,
(unsigned long long)f0, (unsigned long long)f1, (unsigned long long)vk_sync_fence_done(), idle ? "idle" : "busy");
}
AcquireSRWLockExclusive(&rl);
ck_reads++;
if (differ) {
ck_diff++;
if (f1 == f0) ck_diff_unexplained++;
if (idle) ck_diff_idle++;
}
ReleaseSRWLockExclusive(&rl);
free(again);
}
static void vk_be_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row,
uint64_t img) {
vk_sync_start();
replay_drain(DR_WRITE_TEXELS);
vk_present_drain();
vk_write_texels(t, v, r, src, row, img);
}
static void vk_be_guest_written(uint64_t addr, uint64_t size) { vk_sync_start(); vk_guest_written(addr, size); }
// A CPU read of GPU-written mirrored storage (memcpy, pool invalidate). With the replay queued it waits for the newest
// submission recorded to write the textures there, not for the whole queue: the write-back copy then runs after their
// renders, and clears gpu_dirty only for content that is on the queue. Submissions not replayed yet render after the
// copy, as on hardware when the GPU has not reached them; they mark the texture dirty again for a later read.
static void vk_be_guest_read(uint64_t addr, uint64_t size) {
if (!vk_texture_cpu_stale(addr, size)) return;
vk_sync_start();
if (gfx_replay_async()) {
uint64_t f = vk_texture_dirty_fence(addr, size);
AcquireSRWLockExclusive(&rl);
dr_calls[DR_GUEST_READ]++, dr_tcalls[DR_GUEST_READ]++;
ReleaseSRWLockExclusive(&rl);
if (f && vk_sync_fence_done() < f) {
uint64_t t0 = kern_now_ns();
NVN_BLOCKING();
vk_sync_fence_wait(f, UINT64_MAX);
uint64_t d = kern_now_ns() - t0;
AcquireSRWLockExclusive(&rl);
dr_waits[DR_GUEST_READ]++, dr_twaits[DR_GUEST_READ]++;
dr_ns[DR_GUEST_READ] += d, dr_tns[DR_GUEST_READ] += d;
ReleaseSRWLockExclusive(&rl);
}
}
vk_present_drain();
vk_texture_write_back(addr, size);
}
// After a fence wait: the value is done, so its replay is too; a later submission the replay records meanwhile marks
// its targets with a later fence, which write_back_small leaves alone. No drain.
static void vk_be_cpu_synced(uint64_t fence) {
if (!vk_texture_small_dirty()) return;
vk_sync_start();
@@ -189,6 +532,8 @@ const GfxBackend gfx_vulkan_backend = {
};
void vk_backend_shutdown(void) {
replay_drain(DR_FINISH);
gfx_exit_log();
vk_present_drain();
vk_sync_shutdown();
vk_mem_shutdown();
+1
View File
@@ -922,6 +922,7 @@ static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage,
int hv;
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
if (img) view = get_view(ex, img, &v, hv, r->type, storage);
if (img && storage) vk_texture_storage_bound_locked(img);
if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT;
if (!storage && !separate) samp = vk_sampler_by_id_locked((int)(h >> 20 & 0xfff));
}
+15 -2
View File
@@ -42,6 +42,7 @@ static SRWLOCK st_lock = SRWLOCK_INIT;
static CONDITION_VARIABLE cv_work = CONDITION_VARIABLE_INIT; // a submission, a callback or stop
static CONDITION_VARIABLE cv_done = CONDITION_VARIABLE_INIT; // done advanced
static uint64_t issued, done; // newest submitted value; newest value whose side effects ran
static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued)
static int stop;
static Pending *queue; // side effects in the order they were added
static int qhead, qn, qcap;
@@ -223,6 +224,7 @@ void vk_sync_start(void) {
}
AcquireSRWLockExclusive(&st_lock);
issued = done = now;
if (reserved < now) reserved = now;
stop = 0;
ReleaseSRWLockExclusive(&st_lock);
vk_mem_note_submit(now);
@@ -301,6 +303,17 @@ VkCommandBuffer vk_sync_split(void) {
return s->cb;
}
// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed
// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value.
uint64_t vk_sync_reserve(void) {
vk_sync_start();
AcquireSRWLockExclusive(&st_lock);
if (reserved < issued) reserved = issued;
uint64_t v = ++reserved;
ReleaseSRWLockExclusive(&st_lock);
return v;
}
uint64_t vk_sync_submit(const uint64_t *handles, int n) {
vk_sync_start();
VkCtx *c = vk_ctx();
@@ -378,8 +391,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
vk_sync_start();
AcquireSRWLockExclusive(&st_lock);
int ok;
if (value > issued) {
ok = 0; // never submitted: never reached (as in the null back end)
if (value > issued && value > reserved) {
ok = 0; // never submitted nor queued: never reached (as in the null back end)
} else {
uint64_t t0 = kern_now_ns();
while (done < value) {
+41
View File
@@ -52,6 +52,8 @@ typedef struct Tex {
uint32_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded
uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes)
uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one)
uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear,
// storage binding); ReadTexels waits for it instead of draining the replay queue
uint8_t listed; // in owned[]
uint32_t wt_n; // WriteTexels into the storage since the last upload
} Tex;
@@ -819,6 +821,8 @@ void vk_texture_gpu_written(const tex_t *t) {
if (e) {
make_fresh(e); // a partial render keeps what the storage held before
e->pub.gpu_owned = 1;
uint64_t f = vk_exec_recording_fence();
if (f > e->wr_fence) e->wr_fence = f;
if (e->pub.mirrored) {
e->gpu_dirty = 1;
storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render
@@ -1240,6 +1244,28 @@ static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) {
return n;
}
// A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c
// tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels.
void vk_texture_storage_bound_locked(const VkTexImage *img) {
Tex *e = (Tex *)img; // pub is the first member
uint64_t f = vk_exec_recording_fence();
if (f > e->wr_fence) e->wr_fence = f;
}
uint64_t vk_texture_last_write(const tex_t *t) {
lock();
Tex *e = entry_of(t);
uint64_t f = e ? e->wr_fence : 0;
unlock();
return f;
}
size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) {
Region g;
if (!region_of(t, v, r, row, img, &g)) return 0;
return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb;
}
int vk_texture_cpu_stale(uint64_t addr, uint64_t size) {
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
AcquireSRWLockShared(&owned_lock);
@@ -1248,6 +1274,21 @@ int vk_texture_cpu_stale(uint64_t addr, uint64_t size) {
return dirty;
}
// The newest submission recorded to write a GPU-written mirrored texture overlapping [addr, addr+size); 0: none.
uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size) {
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
lock();
uint64_t f = 0;
for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) {
const Tex *e = owned[i];
if (!e->gpu_dirty || e->pub.addr + e->t.L.size <= addr) continue;
if (e->gpu_fence > f) f = e->gpu_fence;
if (e->wr_fence > f) f = e->wr_fence;
}
unlock();
return f;
}
// The image is the newest content only while nothing wrote the storage since the GPU render: TotK finalizes grass maps,
// reuses their storage for new data and memcpys that data on, and writing the old image back first handed the copy
// a stale map or stale resource bytes (wrong textures in the distance). Every hit is checked before any is written