Merge pull request #917: let the game run one frame further ahead of the GPU, sync_ahead, off by default (#676 by ProSep, carried by itaskaev2)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-10 17:55:23 +03:00
commit b2db7809b4
11 files changed
+275 -17

No files matched your search

+3 -2
View File
@@ -60,7 +60,7 @@ static const char *const known[] = {
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "STICK_SENS",
"SWKBD_ANSWER",
"SWKBD_TEST", "TEST_OOM", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_BLOCK_MIB", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_RELEASED_MIB", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UB_OVERLAY_OUT", "UPSCALER", "VB_USED", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VMA_BLOCK_MIB", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS",
"VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VMA_BLOCK_MIB", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS", "SHADOW_DEFER", "SYNC_AHEAD",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"ZSTD_CHECK", "ZSTD_NATIVE",
};
@@ -101,7 +101,7 @@ static const struct { const char *key, *section; } sections[] = {
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
{"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_RELEASED_MIB", "performance"}, {"TEX_BLOCK_MIB", "performance"}, {"VMA_BLOCK_MIB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
{"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"},
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
{"SYNC_AHEAD", "performance"}, {"SHADOW_DEFER", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"},
@@ -166,6 +166,7 @@ static const CfgMeta metas[] = {
{"REPLAY_WORKERS", "0", "0..16", RESTART}, {"REPLAY_CHUNKS", "0", "0..64", RESTART},
{"REPLAY_CHUNK_DRAWS", "128", "1..100000", RESTART}, {"REPLAY_STATE_CHECK", "0", "0|1", RESTART},
{"REPLAY_SKEW_US", "0", "0..100000", RESTART},
{"SYNC_AHEAD", "0", "0|1", RESTART}, {"SHADOW_DEFER", "1", "0|1", RESTART},
{"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART},
// shaders and pipelines
{"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "fullscreen", "strict|any|fullscreen", RESTART},
+9
View File
@@ -43,6 +43,15 @@ _Static_assert(sizeof(nvn_mapping_t) == 0x28, "NVNmappingRequest stride");
const pool_t *nvn_pool_find(uint64_t addr, uint64_t len); // GPU-visible pool holding [addr, addr+len], NULL if none
uint64_t nvn_new_serial(void); // next tex_t / prog_t serial, from 1
// TOTK_SYNC_AHEAD (nvn_api_core.c): nvnQueuePresentTexture notes the newest fence at each present; the newest
// previous-frame value the game was told is done although it may not be (0: none). The back end's shadow copies
// wait for it or are queued for the GPU (vk_sync.c prologue), so the GPU never reads CPU writes the game made under
// that assumption too early.
void nvn_sync_ahead_present(void);
uint64_t nvn_sync_ahead_guard(void);
int vk_mem_shadow_on(void); // vk_memory.c: 1 with shadow storage on (valid once the Vulkan back end is up)
void vk_mem_shadow_fence(void); // vk_memory.c: returns once every shadow sync that read the old guard is done
void gfx_replay_wait_value(uint64_t v); // vk_backend.c: the replay thread submitted value v
uint64_t nvn_fence_last(void); // fence value of the newest submit (0 before the first)
int nvn_depth_minus_w(void); // 1 after DeviceSetDepthMode(NEAR_IS_MINUS_W), else 0
void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the game wrote pool storage: back end re-reads
+67 -3
View File
@@ -471,6 +471,49 @@ NVN_API(nvnQueueFenceSync) {
s->fenced = 1;
s->fence = nvn_fence_last();
}
// TOTK_SYNC_AHEAD=1: one more frame in flight. Each frame the game waits (one nvnSyncWait, frame_wait_lr) for
// the frame it presented last; on the Switch that work is long done by then, here it is still in the replay and on the
// GPU (about 11 ms from the submit), and the wait sets the frame time. With the switch on, that wait, and only that
// one, waits for the frame before it instead. What the game reads back of GPU results after the wait can then be a
// frame old. The game's CPU writes to memory the GPU still reads reach the GPU only through shadow copies, which are
// queued for the GPU or wait (nvn_sync_ahead_guard). The wait still lasts until the replay thread submitted the asked
// value, so no copy of that value itself picks up those writes. pa_fence: the newest fence at the latest present and
// the one before, present thread only for writes. pa_guard: the newest value the game was told is done early.
static uint64_t pa_fence[2], pa_guard;
// The return address of that wait. 1.2.1: 0x7000b23dfc. 1.4.3: 0x70002db99c, the second of the two nvnSyncWait calls
// in f_70002db8ac; like on 1.2.1 its sync comes from a two-entry ring indexed per frame, and the first call waits on
// one object's sync behind a mode check. 0: no site known, the switch stays off.
static uint64_t frame_wait_lr(void) {
return rt_game_version == 121 ? 0x7000b23dfcull : rt_game_version == 143 ? 0x70002db99cull : 0;
}
static int sync_ahead(void) {
static int on = -1;
if (on < 0) {
on = cfg_bool("TOTK_SYNC_AHEAD");
// Without shadow storage the GPU reads the game's own memory, which the game writes ahead: nothing guards
// that. The Vulkan back end sets the mode while it starts, before the game's first present or wait; the null
// back end never does.
if (on && !vk_mem_shadow_on()) {
rt_log("nvn: TOTK_SYNC_AHEAD needs shadow storage (TOTK_SHADOW=1 on native Windows); "
"the game waits for the GPU as before");
on = 0;
}
if (on && !frame_wait_lr()) {
rt_log("nvn: TOTK_SYNC_AHEAD: no frame wait site is known for this game version; the game waits for the "
"GPU as before");
on = 0;
}
if (on) rt_log("nvn: TOTK_SYNC_AHEAD: a wait for the frame presented last waits for the frame before it");
}
return on;
}
void nvn_sync_ahead_present(void) {
if (!sync_ahead()) return;
__atomic_store_n(&pa_fence[1], __atomic_load_n(&pa_fence[0], __ATOMIC_RELAXED), __ATOMIC_RELEASE);
__atomic_store_n(&pa_fence[0], nvn_fence_last(), __ATOMIC_RELEASE);
}
uint64_t nvn_sync_ahead_guard(void) { return __atomic_load_n(&pa_guard, __ATOMIC_ACQUIRE); }
// NVNsyncWaitResult nvnSyncWait(const NVNsync *s, uint64_t timeout_ns): a sync never fenced reports a timeout at once;
// a fenced one is released only after the back end completed that fence value
NVN_API(nvnSyncWait) {
@@ -478,13 +521,34 @@ NVN_API(nvnSyncWait) {
uint64_t timeout = arg_x(c, 1);
if (!s->fenced) { ret_x(c, NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED); return; }
const GfxBackend *b = gfx_backend();
if (b->fence_done() >= s->fence) { b->cpu_synced(s->fence); ret_x(c, NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED); return; }
uint64_t want = s->fence;
// Only the game's per-frame wait lags, and not a poll (timeout 0).
if (timeout && c->x[30] == frame_wait_lr() && sync_ahead() && b->fence_done() < want) {
uint64_t p0 = __atomic_load_n(&pa_fence[0], __ATOMIC_ACQUIRE);
uint64_t p1 = __atomic_load_n(&pa_fence[1], __ATOMIC_ACQUIRE);
if (want <= p0 && want > p1) want = p1;
}
if (want != s->fence) { // the replay of the asked value is done first: the guard then covers every copy of it
{
NVN_BLOCKING();
gfx_replay_wait_value(s->fence);
}
uint64_t g = __atomic_load_n(&pa_guard, __ATOMIC_RELAXED);
while (s->fence > g &&
!__atomic_compare_exchange_n(&pa_guard, &g, s->fence, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {}
vk_mem_shadow_fence(); // a shadow sync that read the old guard finishes before the game writes again
}
if (b->fence_done() >= want) {
b->cpu_synced(want == s->fence ? s->fence : b->fence_done());
ret_x(c, NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED);
return;
}
int ok;
{
NVN_BLOCKING();
ok = b->fence_wait(s->fence, timeout);
ok = b->fence_wait(want, timeout);
}
if (ok) b->cpu_synced(s->fence);
if (ok) b->cpu_synced(want == s->fence ? s->fence : b->fence_done());
ret_x(c, ok ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED);
}
// void nvnEventBuilderSetDefaults(NVNeventBuilder *b)
+1
View File
@@ -713,6 +713,7 @@ NVN_API(nvnQueuePresentTexture) {
const tex_t *t = (const tex_t *)o->tex[idx];
uint64_t period = present_period(o), now = kern_now_ns();
uint64_t slot = now + pace(&o->pace, now, period);
nvn_sync_ahead_present();
gfx_backend()->present(t, o->crop, slot);
o->ticket[idx] = o->presents;
o->avail[idx] = cfg_bool("TOTK_FRAME_QUEUE") ? slot : slot + period;
+1 -1
View File
@@ -355,7 +355,7 @@ static int test_import(pool_t *pool) {
GCHECK(n == 4 && pages[0] == a + 0x30000 && pages[3] == a + 0x33000, "CPU writes to 4 pages: %llu reported",
(unsigned long long)n);
vk_mem_resolve(a + 0x30000, 0x4000, &sp);
vk_mem_shadow_sync();
vk_mem_shadow_sync(0);
copy_wait(sp.buffer, sp.offset, sb, 0, 0x4000);
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
GCHECK(!memcmp(s, m + 0x30000, 0x4000), "GPU read of CPU-written heap differs");
+7 -1
View File
@@ -290,7 +290,13 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb);
// Shadow storage (vk_memory.c): vk_sync.c calls vk_mem_shadow_sync right before each submit; the replay notes every GPU
// write into CPU-visible memory at record time (vk_mem_gpu_pending) and reports it at the fence with vk_gpu_written,
// which copies the bytes back into guest memory before telling the texture cache (vk_guest_written).
void vk_mem_shadow_sync(void);
void vk_mem_shadow_sync(uint64_t fence); // fence: the value of the part about to be submitted (0 untracked)
// TOTK_SHADOW_DEFER: shadow updates that may not be written by the CPU while the GPU still reads the shadow are queued
// by vk_mem_shadow_sync as ranges of guest memory. vk_mem_shadow_deferred says whether there are any;
// vk_mem_shadow_prologue records the copies (staging -> shadow buffers) into the prologue command buffer, before
// vk_mem_mirror_prologue's copies, which read the shadow.
int vk_mem_shadow_deferred(void);
void vk_mem_shadow_prologue(VkCommandBuffer cb, uint64_t fence);
void vk_mem_gpu_pending(uint64_t addr, uint64_t size);
void vk_gpu_written(uint64_t addr, uint64_t size);
int vk_mem_flush_add_probe(void); // --probe-vkexec: the flush range list; 0 or the number of the failed check
+11
View File
@@ -160,6 +160,7 @@ static PlatCond rcv = PLAT_COND_INIT; // item queued, item taken, item done
static ReplayItem rq[RQ_CAP];
static int rq_head, rq_n, r_busy, r_started;
static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames;
static uint64_t r_value_done; // under rl: the newest timeline value the replay thread submitted
// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run.
static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N];
// TOTK_REPLAY_CHECK: ReadTexels compared with a read after a full drain (under rl): checked, differing, differing
@@ -192,6 +193,15 @@ uint64_t gfx_replayed_submits(void) {
return v;
}
// TOTK_SYNC_AHEAD (nvn_api_core.c nvnSyncWait): waits until the replay thread submitted value v, so every shadow copy
// of v was made before the game writes memory v reads. The caller has given its core up.
void gfx_replay_wait_value(uint64_t v) {
if (!gfx_replay_async()) return;
plat_mutex_lock(&rl);
while (r_value_done < v) plat_cond_wait(&rcv, &rl);
plat_mutex_unlock(&rl);
}
static uint64_t submit_sync(const uint64_t *handles, int n) {
vk_sync_start();
vk_present_drain();
@@ -227,6 +237,7 @@ static void replay_main(void *arg) {
present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value);
}
plat_mutex_lock(&rl);
if (it.kind != RI_RELEASE) r_value_done = it.value;
if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no;
else r_presents_done++;
r_busy = 0;
+156 -8
View File
@@ -71,6 +71,9 @@ typedef struct PoolRec {
uint8_t *smap; // its mapping; offsets as in buf (from import_lo)
uint32_t sgen; // gen_lock: every page write stamped up to this generation is in the shadow
volatile uint8_t stouched; // on the touched list: the GPU may use it in the submission being recorded
uint64_t suse; // shadow_lock: the newest timeline value whose submission reads the shadow
uint64_t sdefer; // shadow_lock: the newest value whose prologue carries a deferred copy into the
// shadow (TOTK_SHADOW_DEFER); 0 never
} PoolRec;
static void mirror_forget(PoolRec *r); // pool_lock exclusive: off the mirrored list
@@ -339,7 +342,8 @@ void vk_pool_fini(pool_t *p) {
// recording may already name this pool: a mirrored pool's sync copies and write-backs, an import pool's buffer
// through a texture upload or a released cache entry (vk_texture.c slot_resolve), a physical pool's memory under
// the images vk_texture_range_gone just deferred to that submission. The game frees a pool only once its own GPU
// work is done, so only the port's work can name it. Freed here, the submission ran with freed memory.
// work is done, so only the port's work can name it. Freed here, the submission ran with freed memory. A shadow
// with a deferred copy (TOTK_SHADOW_DEFER) is named by the queue (dq) until the prologue of that submission.
pool_retiring(r);
vk_completion_add(retire_value(), pool_retire, r);
}
@@ -997,6 +1001,18 @@ static int nspend, cspend;
static int64_t s_live_n, s_live_bytes; // shadow_lock
static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks; // shadow_lock: since the last log
// TOTK_SHADOW_DEFER (with TOTK_SYNC_AHEAD): a shadow update the CPU must not write yet is queued instead and executed
// by the GPU, in queue order, at the start of the prologue of the part submitted next (vk_mem_shadow_prologue).
// defer_on: the switch is on. sdefer_now: the pool being synced is deferred, and sdefer_fence the value its copies
// belong to. Replay thread (sub_lock): the queue.
typedef struct { PoolRec *r; uint64_t addr, len; } DRange;
static int defer_on;
static int sdefer_now;
static uint64_t sdefer_fence;
static DRange *dq; // guest ranges to copy into their pools' shadows
static int ndq, cdq;
static uint64_t s_defer_pools, s_defer_bytes, s_defer_copies, s_wait_ticks, s_waits; // shadow_lock: since the last log
static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
r->import_lo = addr & ~0xfffull;
r->import_hi = (addr + size + 0xfff) & ~0xfffull;
@@ -1055,6 +1071,13 @@ static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b, int i) {
if (spend[i].hi < b) shadow_copy(r, spend[i].hi, b, i + 1);
return;
}
if (sdefer_now) { // the GPU copies it later: the bytes are read from guest memory when the prologue is recorded
if (ndq == cdq) grow((void **)&dq, &cdq, sizeof *dq);
dq[ndq++] = (DRange){r, a, b - a};
r->sdefer = sdefer_fence;
s_defer_bytes += b - a;
return;
}
memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, b - a);
s_copied += b - a;
}
@@ -1137,13 +1160,76 @@ static void shadow_copy_newer(PoolRec *r, uint32_t since) {
if (run) shadow_copy(r, run, hi, 0);
}
void vk_mem_shadow_sync(void) {
if (!shadow_on) return;
uint64_t t0 = qpc();
// TOTK_SYNC_AHEAD (nvn_api_core.c): the game may have been told a value is done (at most the guard) that the GPU still
// runs, and then write memory that value reads. The CPU copies its writes into a shadow, so while the guard is not done
// a shadow with any reader in flight must not be written by the CPU: that reader may be the guarded value or an older
// one (the parts of one submission share a value), and a pool records only its newest reader. A shadow with no reader
// in flight is copied at once.
// With TOTK_SHADOW_DEFER the update of such a shadow is queued instead (this function). Without it the replay thread
// waits (shadow_wait_guard).
// shadow_lock held: whether a shadow this sync is about to update has a reader in flight.
static int shadow_reader_busy(uint64_t done) {
for (int i = 0; i < nstouch; i++)
if (stouch[i]->suse > done) return 1;
return 0;
}
// shadow_lock held (released while waiting), no TOTK_SHADOW_DEFER: waits until the guard is done when a shadow to
// update has a reader in flight, then re-reads the guard under the lock, since the game may have raised it meanwhile. The wait
// runs without shadow_lock: the completion thread needs it for the fence callbacks (vk_gpu_written). *g and *done are
// current on return.
static void shadow_wait_guard(uint64_t fence, uint64_t *g, uint64_t *done) {
while (*g > *done && (!fence || *g < fence) && shadow_reader_busy(*done)) {
uint64_t want = *g;
plat_mutex_unlock(&shadow_lock);
uint64_t t0 = qpc();
vk_sync_fence_wait(want, UINT64_MAX);
uint64_t dt = qpc() - t0;
plat_mutex_lock(&shadow_lock);
s_wait_ticks += dt, s_waits++;
*g = nvn_sync_ahead_guard();
*done = vk_sync_fence_done();
}
}
// shadow_lock held, TOTK_SHADOW_DEFER: the pool's update goes through the queue when a reader of it is in flight while
// the guard is (see above), or when an earlier deferred copy into it has not executed yet: a direct memcpy must never
// land before it, or the queued copy would overwrite newer bytes with older ones.
static int shadow_defers(const PoolRec *r, uint64_t g, uint64_t done) {
if (r->sdefer > done) return 1;
return g > done && r->suse > done;
}
// nvnSyncWait raised the guard: a sync that read the old guard holds shadow_lock until its copies are made, so taking
// the lock once here makes the game return from the wait only after such a sync finished. Later syncs see the new
// guard.
void vk_mem_shadow_fence(void) {
plat_mutex_lock(&shadow_lock);
plat_mutex_unlock(&shadow_lock);
}
int vk_mem_shadow_on(void) { return shadow_on; }
void vk_mem_shadow_sync(uint64_t fence) {
if (!shadow_on) return;
plat_mutex_lock(&shadow_lock);
// The guard is read under the lock: see vk_mem_shadow_fence.
uint64_t g = nvn_sync_ahead_guard(), done = g ? vk_sync_fence_done() : 0;
if (!defer_on && nstouch) shadow_wait_guard(fence, &g, &done);
uint64_t t0 = qpc();
if (nstouch) {
plat_mutex_lock(&gen_lock);
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i]), stouch[i]->stouched = 0;
for (int i = 0; i < nstouch; i++) {
PoolRec *r = stouch[i];
sdefer_now = defer_on && fence && shadow_defers(r, g, done);
sdefer_fence = fence;
int before = ndq;
shadow_sync_pool(r), r->stouched = 0;
if (ndq > before) s_defer_pools++;
sdefer_now = 0;
if (fence > r->suse) r->suse = fence;
}
plat_mutex_unlock(&gen_lock);
s_syncs += (uint64_t)nstouch;
nstouch = 0;
@@ -1152,6 +1238,8 @@ void vk_mem_shadow_sync(void) {
plat_mutex_unlock(&shadow_lock);
}
int vk_mem_shadow_deferred(void) { return ndq; }
void vk_mem_gpu_pending(uint64_t addr, uint64_t size) {
if (!shadow_on || !size) return;
plat_mutex_lock(&shadow_lock);
@@ -1183,11 +1271,14 @@ static void shadow_log(uint64_t frame) {
if (!shadow_on) return;
plat_mutex_lock(&shadow_lock);
rt_log("mem: shadow at frame %llu: %lld pools %.1f MiB; since the last: %llu pool syncs, %llu queries, %.1f MiB "
"copied in (%llu untracked), %.1f MiB written back, %.2f ms, %d GPU writes pending",
"copied in (%llu untracked), %.1f MiB written back, %.2f ms, %d GPU writes pending; deferred (%s): %llu pools, "
"%.1f MiB in %llu copies; replay waited for a reader %llu times, %.2f ms",
(unsigned long long)frame, (long long)s_live_n, s_live_bytes / 1048576.0, (unsigned long long)s_syncs,
(unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, s_wb_bytes / 1048576.0,
qpc_ms(s_ticks), nspend);
qpc_ms(s_ticks), nspend, defer_on ? "on" : "off", (unsigned long long)s_defer_pools,
s_defer_bytes / 1048576.0, (unsigned long long)s_defer_copies, (unsigned long long)s_waits, qpc_ms(s_wait_ticks));
s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = 0;
s_defer_pools = s_defer_bytes = s_defer_copies = s_wait_ticks = s_waits = 0;
plat_mutex_unlock(&shadow_lock);
}
@@ -1403,6 +1494,8 @@ static void mirror_mode(int discrete) {
#else
shadow_on = !sh || !*sh || strcmp(sh, "0");
#endif
defer_on = shadow_on && cfg_bool("TOTK_SYNC_AHEAD") && cfg_bool("TOTK_SHADOW_DEFER");
if (defer_on) rt_log("mem: TOTK_SHADOW_DEFER: shadow updates that wait for a reader are queued for the GPU instead");
if (shadow_on) rt_log("mem: shadow storage: CPU-visible pools are host-visible copies of guest memory, not imports%s",
sh && *sh ? " (TOTK_SHADOW)" : "");
rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why,
@@ -2545,6 +2638,61 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
return info.pMappedData;
}
typedef struct { VkBuffer b; VmaAllocation a; } StgFree;
static void free_shadow_stg(void *arg) {
StgFree *f = arg;
vmaDestroyBuffer(vk_ctx()->vma, f->b, f->a);
free(f);
}
// Records the deferred shadow copies (vk_mem_shadow_sync queued the ranges): the guest bytes go into staging buffers
// now, the GPU copies them into the pools' shadows. The barrier before waits for every earlier command, writes and
// reads alike (a read of the shadow must be over before the copy overwrites it); the one after makes the copies visible
// to the rest of the queue, including the mirror copies that follow in this command buffer. The staging buffers are
// freed when the value completes. Replay thread, sub_lock held.
void vk_mem_shadow_prologue(VkCommandBuffer cb, uint64_t fence) {
if (!ndq) return;
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT | VK_ACCESS_2_MEMORY_READ_BIT,
VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
const uint64_t lim = STG_BLOCK / 2;
uint64_t copies = 0;
int i = 0;
uint64_t at_i = 0; // bytes of dq[i] already recorded
while (i < ndq) {
uint64_t total = 0, jo = at_i; // how much the next staging buffer holds
for (int j = i; j < ndq && total < lim;) {
uint64_t left = dq[j].len - jo, take = left < lim - total ? left : lim - total;
total += take;
if (take == left) j++, jo = 0;
else jo += take;
}
VkBuffer sb;
VmaAllocation sa;
uint8_t *p = vk_staging(total, &sb, &sa);
for (uint64_t at = 0; at < total;) {
const DRange *d = &dq[i];
uint64_t left = d->len - at_i, take = left < total - at ? left : total - at;
memcpy(p + at, (const void *)(uintptr_t)(d->addr + at_i), take);
VkBufferCopy bc = {at, d->addr + at_i - d->r->import_lo, take};
vkCmdCopyBuffer(cb, sb, d->r->buf, 1, &bc);
copies++;
at += take, at_i += take;
if (at_i == d->len) i++, at_i = 0;
}
vmaFlushAllocation(vk_ctx()->vma, sa, 0, VK_WHOLE_SIZE);
StgFree *f = malloc(sizeof *f);
if (!f) rt_fatal("vk: out of memory");
f->b = sb, f->a = sa;
vk_completion_add(fence, free_shadow_stg, f);
}
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
ndq = 0;
plat_mutex_lock(&shadow_lock);
s_defer_copies += copies;
plat_mutex_unlock(&shadow_lock);
}
// Device-local scratch of the upload batch (the GPU deswizzle's linear copies, the ASTC decoder's blocks and texels):
// a pool of its own, like staging, so the replay thread neither waits on the default pool's lock while another thread
// makes a block there (the headroom thread: about 8 ms for 256 MiB) nor makes one itself for a few MiB. One block
@@ -2832,7 +2980,7 @@ void vk_mem_shutdown(void) {
plat_mutex_lock(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
pool_index.n = 0;
nm = npend = nwbl = nsnl = snl_copied = 0; // mirrors went with their pools
nm = npend = nwbl = nsnl = snl_copied = ndq = 0; // mirrors went with their pools
plat_mutex_lock(&snap_pool_lock);
for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc);
nsnap_pool = 0, snap_pool_bytes = 0;
+5 -2
View File
@@ -368,8 +368,10 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
// direct: the calling thread records, ends and submits it itself while the part being replayed stays the recorder's
// (vk_sync_part, which the caller made the recorder idle for).
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence, int direct) {
vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
// Shadow storage: the CPU's writes go in before the part is submitted (TOTK_SYNC_AHEAD: queued for the GPU or
// made after the GPU finished the readers the game may think done, vk_memory.c).
vk_mem_shadow_sync(fence);
if (!vk_mem_mirror_pending() && !vk_mem_shadow_deferred()) return VK_NULL_HANDLE;
int i = claim_slot(c, fence); // before reading slots: the claim may move it
VkCommandBuffer cb = slots[i].cb;
if (direct) {
@@ -380,6 +382,7 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence, int direct) {
rec_begin(cb, fence, 0, 0); // a command buffer of the part pool: the recorder's like the parts
}
int pair = ts_open(cb);
vk_mem_shadow_prologue(cb, fence); // before the mirror copies, which read the shadow these bring up to date
vk_mem_mirror_prologue(cb);
ts_close(cb, pair, fence);
if (direct || !rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb);