Replay NVN submissions on a queued replay thread behind TOTK_REPLAY_ASYNC, with drains for calls that read or free GPU state

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-25 19:24:39 +03:00
1 parent dc3a0e906a
commit 14af7e7347
8 files changed
+368 -35

No files matched your search

+1
View File
@@ -28,6 +28,7 @@ void prof_write(void); // idempotent; boot.c calls it b
uint64_t perf_begin(void); // start time of a wait; pair with perf_end
void perf_end(int kind, uint64_t t0); // counts render-thread time only
void perf_replay(uint64_t t0); // replay (NVN -> Vulkan recording) time on any thread, waits inside it included
void perf_replay_thread(void); // the calling thread replays for the render thread: perf_end counts its waits
// Events per 60-frame window: graphics pipelines finished by the background workers, draws skipped while pending,
// first texture uploads and re-uploads after CPU writes, draws skipped while a shader translates in the background,
// draws that waited for their pending pipeline and got it, draws into a target the previous frame did not render that
+5
View File
@@ -51,6 +51,11 @@ void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the ga
// CPU class (0 CPU_CACHED, 1 CPU_UNCACHED, 2 other) since the previous call.
void nvn_flush_stats(uint64_t out[2][3][2]);
void nvn_submit_prepare(const uint64_t *handles, int n); // weak no-op; the command-buffer slice reads deferred binds
// Asynchronous replay: as nvn_submit_prepare, writing the handles to replay to out (a clone where a still queued
// submission uses the same recording with other deferred binds)
void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n);
int gfx_replay_async(void); // 1: the back end replays submissions on its own thread
uint64_t gfx_replayed_submits(void); // newest submit number (nvn_submit_count) replayed
#define NVN_HANDLE_TEX(h) ((int)((h) & 0xfffff)) // combined texture handle = texture id | sampler id << 20
#define NVN_HANDLE_SAMP(h) ((int)(((h) >> 20) & 0xfff))
extern GfxStats gfx_counters; // behind gfx_stats(); add with __atomic_fetch_add. frames:
+74 -6
View File
@@ -160,7 +160,7 @@ static const tex_t *tex_pin(const NVNtexture *nt) {
// ---------- recording copies
// A handle points just past a copy_prefix. Copies are indexed by the 4 KiB page of their control-memory position, so
// adding control memory again releases the copies recorded there without walking every live copy.
typedef struct { uint64_t magic, bytes, made_at, used_at; } copy_prefix;
typedef struct { uint64_t magic, bytes, made_at, used_at, deferred; } copy_prefix; // deferred: 0 unknown, 1 none, 2 some
_Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned");
typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy;
typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket;
@@ -168,6 +168,30 @@ static SRWLOCK copy_lock = SRWLOCK_INIT; // guards the two maps, the counters
static nvn_hmap copy_pages = {.klen = 8}; // page -> copy_bucket
static nvn_hmap copy_live = {.klen = 8}; // handle -> copy_prefix of a live copy
static uint64_t ncopies, copy_bytes, copies_made, nsubmits;
// Asynchronous replay: copies released by the game, and per-submission clones, that a queued submission still reads.
// Freed once gfx_replayed_submits() reaches their used_at. Empty with synchronous replay.
static copy_prefix **zombies;
static uint32_t nzombies, cap_zombies;
static void zombie_add(copy_prefix *pf) { // copy_lock held
if (nzombies == cap_zombies) {
cap_zombies = cap_zombies ? cap_zombies * 2 : 64;
zombies = realloc(zombies, sizeof *zombies * cap_zombies);
if (!zombies) rt_fatal("nvn: out of memory (released recording copies)");
}
zombies[nzombies++] = pf;
}
static void zombie_sweep(uint64_t replayed) { // copy_lock held
for (uint32_t i = 0; i < nzombies;) {
if (zombies[i]->used_at <= replayed) {
free(zombies[i]);
zombies[i] = zombies[--nzombies];
} else {
i++;
}
}
}
static copy_prefix *copy_find(uint64_t handle) { // copy_lock held
return nvn_hm_get(&copy_live, &handle);
@@ -258,7 +282,9 @@ static void copy_add(uintptr_t lo, uintptr_t hi, uint8_t *copy) {
// Adding control memory again retires every recording copy made in it (the driver's handle lived there).
static void release_copies(uintptr_t lo, uintptr_t hi) {
if (hi <= lo) return;
uint64_t replayed = gfx_replayed_submits();
AcquireSRWLockExclusive(&copy_lock);
if (nzombies) zombie_sweep(replayed);
for (uintptr_t pg = lo >> COPY_PAGE_SHIFT; pg <= (hi - 1) >> COPY_PAGE_SHIFT; pg++) {
copy_bucket *b = nvn_hm_get(&copy_pages, &pg);
if (!b) continue;
@@ -271,7 +297,8 @@ static void release_copies(uintptr_t lo, uintptr_t hi) {
ncopies--;
copy_bytes -= pf->bytes;
pf->magic = 0;
free(pf);
if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it
else free(pf);
b->v[i] = b->v[--b->n];
}
if (!b->n) {
@@ -836,16 +863,17 @@ NVN_API(nvnCommandBufferSetPatchSize) { (void)c; }
// What the console reads at the submit call is read here, on the game thread: every handle must be a live recording,
// and deferred binds take the handles the game's arrays hold now (the game fills its tables for the next frame as soon
// as the submit returns). The result is written into the recording copy.
static uint64_t def_reads, def_changed;
static uint64_t def_reads, def_changed, n_clones;
static void resolve_deferred(const uint8_t *p, int depth) {
static int resolve_deferred(const uint8_t *p, int depth) { // returns the number of deferred binds
int nd = 0;
for (;;) {
op_hdr h;
memcpy(&h, p, sizeof h);
if (h.size < 8 || (h.size & 7) || h.op >= OP_COUNT_)
rt_fatal("nvn: corrupt recording at %p while resolving deferred binds (op %u size %u)", (const void *)p,
h.op, h.size);
if (h.op == OP_END) return;
if (h.op == OP_END) return nd;
if (h.op == OP_JUMP) {
memcpy(&p, p + 8, 8);
continue;
@@ -854,10 +882,11 @@ static void resolve_deferred(const uint8_t *p, int depth) {
uint64_t t;
memcpy(&t, p + 8, 8);
if (depth >= 16) rt_fatal("nvn: deferred bind resolution: call depth over 16");
resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1);
nd += resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1);
} else if (h.op == OP_BIND_TEX || h.op == OP_BIND_IMG || h.op == OP_BIND_SAMP) {
p_bind_handles *b = (p_bind_handles *)(uintptr_t)(p + 8);
if (b->src && b->n > 0 && b->n <= 32) {
nd++;
uint64_t cur[32];
memcpy(cur, (const void *)(uintptr_t)b->src, 8 * (size_t)b->n);
__atomic_fetch_add(&def_reads, 1, __ATOMIC_RELAXED);
@@ -879,6 +908,45 @@ void nvn_submit_prepare(const uint64_t *handles, int n) {
for (int i = 0; i < n; i++) resolve_deferred((const uint8_t *)(uintptr_t)handles[i], 0);
}
// Asynchronous replay. A copy a queued submission still reads keeps that submission's deferred binds: when it holds
// deferred binds (or is not known to hold none), this submission gets a clone, resolved here and freed after its replay.
void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) {
uint64_t now = __atomic_add_fetch(&nsubmits, 1, __ATOMIC_RELAXED);
uint64_t replayed = gfx_replayed_submits();
AcquireSRWLockExclusive(&copy_lock);
if (nzombies) zombie_sweep(replayed);
for (int i = 0; i < n; i++) {
copy_prefix *pf = copy_find(handles[i]);
if (!pf) {
ReleaseSRWLockExclusive(&copy_lock);
rt_fatal("nvn: QueueSubmitCommands: command handle %#llx is not a live recording (released or never recorded)",
(unsigned long long)handles[i]);
}
out[i] = handles[i];
if (pf->used_at > replayed && pf->used_at != now && pf->deferred != 1) {
size_t sz = sizeof *pf + pf->bytes;
copy_prefix *cl = malloc(sz);
if (!cl) rt_fatal("nvn: out of memory (recording clone, %zu bytes)", sz);
memcpy(cl, pf, sz);
cl->magic = 0; // not a live handle: the game never sees it
cl->used_at = now;
zombie_add(cl);
out[i] = (uintptr_t)(cl + 1);
__atomic_fetch_add(&n_clones, 1, __ATOMIC_RELAXED);
} else {
pf->used_at = now;
}
}
ReleaseSRWLockExclusive(&copy_lock);
for (int i = 0; i < n; i++) {
int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0);
if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1;
}
if (now % 7200 == 0)
rt_log("nvn: asynchronous replay at submit %llu: %llu recording clones so far, %u copies waiting for the replay",
(unsigned long long)now, (unsigned long long)__atomic_load_n(&n_clones, __ATOMIC_RELAXED), nzombies);
}
uint64_t nvn_submit_count(void) { return __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); }
const char *nvn_op_name(uint32_t op) {
+24 -3
View File
@@ -58,6 +58,11 @@ static void ov_memcpy(Cpu *c) {
HOST_OVERRIDE("memcpy", ov_memcpy);
__attribute__((weak)) void nvn_submit_prepare(const uint64_t *handles, int n) { (void)handles, (void)n; }
__attribute__((weak)) void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) {
memcpy(out, handles, sizeof *out * (size_t)n);
}
__attribute__((weak)) int gfx_replay_async(void) { return 0; }
__attribute__((weak)) uint64_t gfx_replayed_submits(void) { return nvn_submit_count(); }
// ---------- pools: GPU window (section 3.1) and the address index
static SRWLOCK pool_lock = SRWLOCK_INIT; // guards gva and index
@@ -560,7 +565,9 @@ NVN_API(nvnQueueFinish) {
static uint64_t lod_entered, lod_issued, lod_fence[LOD_RING];
// void nvnQueueSubmitCommands(NVNqueue *q, int n, const NVNcommandHandle *handles): replayed on this thread with its
// core given up (section 2.2); the newest fence value is what QueueFenceSync records
// core given up (section 2.2), or queued for the replay thread (TOTK_REPLAY_ASYNC, vk_backend.c); the newest fence
// value is what QueueFenceSync records. With the queue the fence covers replay and GPU, so the LOD guard below waits
// for both.
NVN_API(nvnQueueSubmitCommands) {
queue_t *q = arg_p(c, 0);
int n = (int)arg_x(c, 1);
@@ -569,8 +576,22 @@ NVN_API(nvnQueueSubmitCommands) {
NVN_BLOCKING();
uint64_t no = __atomic_add_fetch(&lod_entered, 1, __ATOMIC_SEQ_CST);
q->submits++;
nvn_submit_prepare(h, n);
uint64_t f = gfx_backend()->submit(h, n), prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED);
uint64_t f;
if (gfx_replay_async()) {
// Prepare and queue under one lock: submit numbers follow queue order (gfx_replayed_submits).
static SRWLOCK submit_lock = SRWLOCK_INIT;
uint64_t local[32], *hs = n <= 32 ? local : malloc(sizeof *hs * (size_t)n);
if (!hs) rt_fatal("nvn: out of memory (submit of %d handles)", n);
AcquireSRWLockExclusive(&submit_lock);
nvn_submit_prepare_async(h, hs, n);
f = gfx_backend()->submit(hs, n);
ReleaseSRWLockExclusive(&submit_lock);
if (hs != local) free(hs);
} else {
nvn_submit_prepare(h, n);
f = gfx_backend()->submit(h, n);
}
uint64_t prev = __atomic_load_n(&last_fence, __ATOMIC_RELAXED);
while (f > prev && !__atomic_compare_exchange_n(&last_fence, &prev, f, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {}
__atomic_store_n(&lod_fence[no % LOD_RING], f, __ATOMIC_RELAXED);
__atomic_store_n(&lod_issued, no, __ATOMIC_SEQ_CST);
+14 -3
View File
@@ -269,11 +269,22 @@ static uint64_t wait_ns[PERF_N], frame_ns[PERF_N], worst_ns[PERF_N]; // window
uint64_t perf_begin(void) { return kern_now_ns(); }
static volatile DWORD replay_tid; // asynchronous replay (vk_backend.c): its waits hold up the frame too
void perf_replay_thread(void) { replay_tid = GetCurrentThreadId(); }
void perf_end(int kind, uint64_t t0) {
if (!t0 || GetCurrentThreadId() != render_tid) return;
if (!t0) return;
DWORD tid = GetCurrentThreadId();
if (tid != render_tid && tid != replay_tid) return;
uint64_t d = kern_now_ns() - t0;
wait_ns[kind] += d;
frame_ns[kind] += d;
if (tid == render_tid && !replay_tid) { // one thread adds: no atomics on the synchronous path
wait_ns[kind] += d;
frame_ns[kind] += d;
return;
}
__atomic_fetch_add(&wait_ns[kind], d, __ATOMIC_RELAXED);
__atomic_fetch_add(&frame_ns[kind], d, __ATOMIC_RELAXED);
}
static volatile LONG64 other_wait_ns; // perf_gpu_wait on threads other than the render thread, over the window
+2
View File
@@ -286,6 +286,7 @@ void rt_write_png(const char *path, const uint8_t *rgba, uint32_t w, uint32_t h)
// once the GPU reached it and the completion thread ran its side effects. Guest or host threads.
void vk_sync_start(void); // vk_init (fatal without a device) and the completion thread; idempotent
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
uint64_t vk_sync_fence_done(void); // highest done value
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
@@ -308,6 +309,7 @@ void rdoc_init(void); // runtime/rdoc_win.c: TOTK_RDOC debug
int rdoc_active(void);
void rdoc_after_present(uint64_t frame);
void vk_present_drain(void); // returns once every queued present was submitted to gfx_queue
int vk_replay_async(void); // TOTK_REPLAY_ASYNC (default 1): submissions replay on the replay thread
void vk_program_release(const prog_t *p); // GfxBackend.program_release: pipelines (T6b); weak no-op in vk_backend.c
void vk_backend_shutdown(void); // presents, vk_sync_shutdown, vk_mem_shutdown, vk_shutdown in that order
+233 -21
View File
@@ -10,8 +10,19 @@
// Back-edge pacing: the present thread submits the blit at once, then holds the host present (vkQueuePresentKHR) until
// the frame's display time from the pacer (nvn_api_obj.c), so present-to-present intervals stay even whatever the
// replay took. The blit is already on the queue, so the wait blocks no submission.
//
// Asynchronous replay (TOTK_REPLAY_ASYNC, default 1; 0 keeps the synchronous path unchanged): nvnQueueSubmitCommands
// and nvnQueuePresentTexture only queue. Each item gets its timeline value at enqueue (vk_sync_reserve), in queue
// order; the replay thread runs the synchronous submit and present bodies in that order, so every value is signalled
// by the submission it was handed out for. The game runs at most one frame ahead: queueing present N waits until the
// replay thread handled present N-1. Calls that read or free GPU state that queued work may use drain the queue first
// (replay_drain), each cause counted. Design and the orderings it keeps: build/v121-replayq/NOTES.md.
#include <windows.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cfg.h"
#include "flight.h"
#include "gfx.h"
#include "host.h"
@@ -21,7 +32,7 @@
#define WEAK __attribute__((weak))
typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued; } PresentJob;
typedef struct { VkImage image; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued, value; } PresentJob;
static SRWLOCK pl = SRWLOCK_INIT;
static CONDITION_VARIABLE pcv = CONDITION_VARIABLE_INIT; // job queued, job taken, job submitted
@@ -52,7 +63,10 @@ static DWORD WINAPI present_main(void *arg) {
ReleaseSRWLockExclusive(&pl);
if (flight_on) vk_thumbs_frame(j.frame);
vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.w, j.h, j.has_crop ? j.crop : NULL);
vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too
uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too
if (j.value && f != j.value)
rt_fatal("vk: the present of frame %llu signalled %llu, queued as %llu", (unsigned long long)j.frame,
(unsigned long long)f, (unsigned long long)j.value);
AcquireSRWLockExclusive(&pl);
busy = 0;
WakeAllConditionVariable(&pcv);
@@ -84,11 +98,11 @@ void vk_present_drain(void) {
perf_end(PERF_DRAIN, p0);
}
static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
// The present after the frame count: resolves the image and hands the blit to the present thread. queued: when the game
// presented; value: the timeline value queued for this present (asynchronous replay), 0 for the next one (synchronous).
static void present_image(const tex_t *t, const int crop[4], uint64_t at_ns, uint64_t queued, uint64_t frame,
uint64_t value) {
static volatile long n_skipped;
vk_sync_start();
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads
vk_exec_frame_log(t, frame);
vk_mem_census(frame);
if (vk_dump_armed) vk_dump_present(frame);
@@ -97,10 +111,15 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
if (!img || !(img->aspect & VK_IMAGE_ASPECT_COLOR_BIT)) {
if (InterlockedIncrement(&n_skipped) <= 8)
rt_log("vk: present of texture %p (format %#x) skipped: no colour image", (const void *)t, t ? t->d.format : 0);
if (value) { // the queued value is still signalled, in order
vk_present_drain();
uint64_t f = vk_sync_submit(NULL, 0);
if (f != value) rt_fatal("vk: skipped present signalled %llu, queued as %llu", (unsigned long long)f, (unsigned long long)value);
}
return;
}
PresentJob j = {.image = img->image, .format = img->format, .w = (int)img->width, .h = (int)img->height, .frame = frame,
.at = at_ns, .queued = kern_now_ns()};
.at = at_ns, .queued = queued, .value = value};
if (crop) {
j.has_crop = 1;
for (int i = 0; i < 4; i++) j.crop[i] = crop[i];
@@ -121,35 +140,223 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
if (rdoc_active()) { vk_present_drain(); rdoc_after_present(frame); }
}
// Pipelines (T6b) drop a released program's pipelines here.
WEAK void vk_program_release(const prog_t *p) { (void)p; }
// ---- Asynchronous replay
enum { RI_SUBMIT, RI_PRESENT };
typedef struct {
int kind, n, has_crop, crop[4];
uint64_t value, submit_no, at, queued;
uint64_t *h;
const tex_t *tex;
} ReplayItem;
#define RQ_CAP 64
enum { DR_POOL_FINI, DR_MAP_VIRTUAL, DR_TEX_RELEASE, DR_PROG_RELEASE, DR_FINISH, DR_READ_TEXELS, DR_WRITE_TEXELS,
DR_GUEST_READ, DR_FRAME, DR_FULL, DR_N };
static const char *const dr_name[DR_N] = {"pool_fini", "map_virtual", "texture_release", "program_release", "finish",
"read_texels", "write_texels", "guest_read", "frame bound", "queue full"};
static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); }
static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); vk_present_drain(); vk_pool_fini(p); }
static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) {
vk_sync_start();
vk_present_drain();
return vk_pool_map_virtual(v, n, requests);
static SRWLOCK rl = SRWLOCK_INIT;
static CONDITION_VARIABLE rcv = CONDITION_VARIABLE_INIT; // item queued, item taken, item done
static ReplayItem rq[RQ_CAP];
static int rq_head, rq_n, r_busy, r_started;
static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames;
// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run.
static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N];
int vk_replay_async(void) {
static int on = -1;
if (on < 0) {
const char *e = cfg_get("TOTK_REPLAY_ASYNC");
on = !(e && !strcmp(e, "0"));
rt_log("vk: replay %s (TOTK_REPLAY_ASYNC=%d)",
on ? "on its own thread, at most one frame behind the game" : "on the submitting thread", on);
}
return on;
}
static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); }
static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); }
static void vk_be_texture_release(const tex_t *t) { vk_sync_start(); vk_present_drain(); vk_texture_release(t); }
static void vk_be_program_release(const prog_t *p) { vk_program_release(p); }
static uint64_t vk_be_submit(const uint64_t *handles, int n) {
// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end.
int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); }
// The newest submit number (nvn_submit_count) whose replay is done: recording copies it used may be freed.
uint64_t gfx_replayed_submits(void) {
if (!gfx_replay_async()) return nvn_submit_count();
AcquireSRWLockShared(&rl);
uint64_t v = r_submit_done;
ReleaseSRWLockShared(&rl);
return v;
}
static uint64_t submit_sync(const uint64_t *handles, int n) {
vk_sync_start();
vk_present_drain();
vk_texture_new_submit();
return vk_sync_submit(handles, n);
}
static void vk_be_finish(void) { vk_sync_start(); vk_present_drain(); vk_sync_finish(); }
static DWORD WINAPI replay_main(void *arg) {
(void)arg;
perf_replay_thread();
AcquireSRWLockExclusive(&rl);
for (;;) {
while (!rq_n) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
ReplayItem it = rq[rq_head];
rq_head = (rq_head + 1) % RQ_CAP;
rq_n--;
r_busy = 1;
WakeAllConditionVariable(&rcv);
ReleaseSRWLockExclusive(&rl);
if (it.kind == RI_SUBMIT) {
uint64_t f = submit_sync(it.h, it.n);
if (f != it.value)
rt_fatal("vk: replayed submission signalled %llu, queued as %llu", (unsigned long long)f,
(unsigned long long)it.value);
free(it.h);
} else {
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value);
}
AcquireSRWLockExclusive(&rl);
if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no;
else r_presents_done++;
r_busy = 0;
WakeAllConditionVariable(&rcv);
}
return 0;
}
static void wait_counted(int why, uint64_t t0) { // rl held
uint64_t d = kern_now_ns() - t0;
dr_waits[why]++, dr_twaits[why]++;
dr_ns[why] += d, dr_tns[why] += d;
perf_end(PERF_DRAIN, t0);
}
// Queues an item (waits while the queue is full) and returns its value, reserved under rl so values follow queue order.
static uint64_t replay_enqueue(ReplayItem *it) {
AcquireSRWLockExclusive(&rl);
if (!r_started) {
HANDLE h = CreateThread(NULL, 0, replay_main, NULL, 0, NULL);
if (!h) rt_fatal("vk: cannot start the replay thread (error %lu)", GetLastError());
CloseHandle(h);
r_started = 1;
}
if (rq_n == RQ_CAP) {
uint64_t p0 = perf_begin();
dr_calls[DR_FULL]++, dr_tcalls[DR_FULL]++;
while (rq_n == RQ_CAP) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
wait_counted(DR_FULL, p0);
}
it->value = vk_sync_reserve();
rq[(rq_head + rq_n) % RQ_CAP] = *it;
rq_n++;
WakeAllConditionVariable(&rcv);
ReleaseSRWLockExclusive(&rl);
return it->value;
}
// Returns once every queued item was replayed and submitted. Back-end entries only: the replay thread never calls them.
static void replay_drain(int why) {
if (!gfx_replay_async()) return;
AcquireSRWLockExclusive(&rl);
dr_calls[why]++, dr_tcalls[why]++;
if (rq_n || r_busy) {
uint64_t p0 = perf_begin();
while (rq_n || r_busy) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
wait_counted(why, p0);
}
ReleaseSRWLockExclusive(&rl);
}
// What made the game wait for the replay: since the last line (every 1800 game frames), or over the run (at exit).
static void replay_log(uint64_t frame, int total) {
char buf[1024];
int k = 0;
AcquireSRWLockExclusive(&rl);
const uint64_t *c = total ? dr_tcalls : dr_calls, *w = total ? dr_twaits : dr_waits, *t = total ? dr_tns : dr_ns;
for (int i = 0; i < DR_N && k < (int)sizeof buf - 96; i++)
if (c[i] || w[i])
k += snprintf(buf + k, sizeof buf - (size_t)k, "%s %s %llu calls, %llu waited %.1f ms", k ? ";" : "", dr_name[i],
(unsigned long long)c[i], (unsigned long long)w[i], t[i] / 1e6);
memset(dr_calls, 0, sizeof dr_calls);
memset(dr_waits, 0, sizeof dr_waits);
memset(dr_ns, 0, sizeof dr_ns);
ReleaseSRWLockExclusive(&rl);
rt_log("vk: replay queue %s frame %llu:%s", total ? "over the run to" : "at", (unsigned long long)frame,
k ? buf : " no waits");
}
static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
vk_sync_start();
if (!gfx_replay_async()) {
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads
present_image(t, crop, at_ns, kern_now_ns(), frame, 0);
return;
}
// Frame bound: the replay thread handled the previous present before this one is queued.
AcquireSRWLockExclusive(&rl);
dr_calls[DR_FRAME]++, dr_tcalls[DR_FRAME]++;
if (r_presents_done < r_presents_queued) {
uint64_t p0 = perf_begin();
while (r_presents_done < r_presents_queued) SleepConditionVariableSRW(&rcv, &rl, INFINITE, 0);
wait_counted(DR_FRAME, p0);
}
r_presents_queued++;
ReleaseSRWLockExclusive(&rl);
uint64_t frame = ++game_frames;
perf_present(frame);
if (frame % 1800 == 0) replay_log(frame, 0);
ReplayItem it = {.kind = RI_PRESENT, .tex = t, .at = at_ns, .queued = kern_now_ns()};
if (crop) {
it.has_crop = 1;
for (int i = 0; i < 4; i++) it.crop[i] = crop[i];
}
replay_enqueue(&it);
}
// Pipelines (T6b) drop a released program's pipelines here.
WEAK void vk_program_release(const prog_t *p) { (void)p; }
static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); }
static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); replay_drain(DR_POOL_FINI); vk_present_drain(); vk_pool_fini(p); }
static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) {
vk_sync_start();
replay_drain(DR_MAP_VIRTUAL);
vk_present_drain();
return vk_pool_map_virtual(v, n, requests);
}
static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); }
static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); }
static void vk_be_texture_release(const tex_t *t) {
vk_sync_start();
replay_drain(DR_TEX_RELEASE);
vk_present_drain();
vk_texture_release(t);
}
static void vk_be_program_release(const prog_t *p) { replay_drain(DR_PROG_RELEASE); vk_program_release(p); }
// Asynchronous: the handles (nvn_submit_prepare_async ran under the front end's submit lock) are copied and queued;
// the value returned is the one the replay will signal.
static uint64_t vk_be_submit(const uint64_t *handles, int n) {
if (!gfx_replay_async()) return submit_sync(handles, n);
vk_sync_start();
ReplayItem it = {.kind = RI_SUBMIT, .n = n, .submit_no = nvn_submit_count()};
if (n > 0) {
it.h = malloc(sizeof *it.h * (size_t)n);
if (!it.h) rt_fatal("vk: out of memory (replay queue)");
memcpy(it.h, handles, sizeof *it.h * (size_t)n);
}
return replay_enqueue(&it);
}
static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); }
static void vk_be_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row,
uint64_t img) {
vk_sync_start();
replay_drain(DR_READ_TEXELS);
vk_read_texels(t, v, r, dst, row, img);
}
static void vk_be_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row,
uint64_t img) {
vk_sync_start();
replay_drain(DR_WRITE_TEXELS);
vk_present_drain();
vk_write_texels(t, v, r, src, row, img);
}
@@ -157,9 +364,12 @@ static void vk_be_guest_written(uint64_t addr, uint64_t size) { vk_sync_start();
static void vk_be_guest_read(uint64_t addr, uint64_t size) {
if (!vk_texture_cpu_stale(addr, size)) return;
vk_sync_start();
replay_drain(DR_GUEST_READ);
vk_present_drain();
vk_texture_write_back(addr, size);
}
// After a fence wait: the value is done, so its replay is too; a later submission the replay records meanwhile marks
// its targets with a later fence, which write_back_small leaves alone. No drain.
static void vk_be_cpu_synced(uint64_t fence) {
if (!vk_texture_small_dirty()) return;
vk_sync_start();
@@ -189,6 +399,8 @@ const GfxBackend gfx_vulkan_backend = {
};
void vk_backend_shutdown(void) {
replay_drain(DR_FINISH);
if (gfx_replay_async() && r_started) replay_log(game_frames, 1);
vk_present_drain();
vk_sync_shutdown();
vk_mem_shutdown();
+15 -2
View File
@@ -42,6 +42,7 @@ static SRWLOCK st_lock = SRWLOCK_INIT;
static CONDITION_VARIABLE cv_work = CONDITION_VARIABLE_INIT; // a submission, a callback or stop
static CONDITION_VARIABLE cv_done = CONDITION_VARIABLE_INIT; // done advanced
static uint64_t issued, done; // newest submitted value; newest value whose side effects ran
static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued)
static int stop;
static Pending *queue; // side effects in the order they were added
static int qhead, qn, qcap;
@@ -223,6 +224,7 @@ void vk_sync_start(void) {
}
AcquireSRWLockExclusive(&st_lock);
issued = done = now;
if (reserved < now) reserved = now;
stop = 0;
ReleaseSRWLockExclusive(&st_lock);
vk_mem_note_submit(now);
@@ -301,6 +303,17 @@ VkCommandBuffer vk_sync_split(void) {
return s->cb;
}
// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed
// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value.
uint64_t vk_sync_reserve(void) {
vk_sync_start();
AcquireSRWLockExclusive(&st_lock);
if (reserved < issued) reserved = issued;
uint64_t v = ++reserved;
ReleaseSRWLockExclusive(&st_lock);
return v;
}
uint64_t vk_sync_submit(const uint64_t *handles, int n) {
vk_sync_start();
VkCtx *c = vk_ctx();
@@ -378,8 +391,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
vk_sync_start();
AcquireSRWLockExclusive(&st_lock);
int ok;
if (value > issued) {
ok = 0; // never submitted: never reached (as in the null back end)
if (value > issued && value > reserved) {
ok = 0; // never submitted nor queued: never reached (as in the null back end)
} else {
uint64_t t0 = kern_now_ns();
while (done < value) {