mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-21 12:55:57 +00:00
Merge pull request #917: let the game run one frame further ahead of the GPU, sync_ahead, off by default (#676 by ProSep, carried by itaskaev2)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
b2db7809b4
11 files changed
+275
-17
No files matched your search
+3
-2
@@ -60,7 +60,7 @@ static const char *const known[] = {
|
||||
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "STICK_SENS",
|
||||
"SWKBD_ANSWER",
|
||||
"SWKBD_TEST", "TEST_OOM", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_BLOCK_MIB", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_RELEASED_MIB", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UB_OVERLAY_OUT", "UPSCALER", "VB_USED", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
|
||||
"VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VMA_BLOCK_MIB", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS",
|
||||
"VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VMA_BLOCK_MIB", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS", "SHADOW_DEFER", "SYNC_AHEAD",
|
||||
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
|
||||
"ZSTD_CHECK", "ZSTD_NATIVE",
|
||||
};
|
||||
@@ -101,7 +101,7 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
|
||||
{"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_RELEASED_MIB", "performance"}, {"TEX_BLOCK_MIB", "performance"}, {"VMA_BLOCK_MIB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
|
||||
{"SYNC_AHEAD", "performance"}, {"SHADOW_DEFER", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
|
||||
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"},
|
||||
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
|
||||
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"},
|
||||
@@ -166,6 +166,7 @@ static const CfgMeta metas[] = {
|
||||
{"REPLAY_WORKERS", "0", "0..16", RESTART}, {"REPLAY_CHUNKS", "0", "0..64", RESTART},
|
||||
{"REPLAY_CHUNK_DRAWS", "128", "1..100000", RESTART}, {"REPLAY_STATE_CHECK", "0", "0|1", RESTART},
|
||||
{"REPLAY_SKEW_US", "0", "0..100000", RESTART},
|
||||
{"SYNC_AHEAD", "0", "0|1", RESTART}, {"SHADOW_DEFER", "1", "0|1", RESTART},
|
||||
{"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART},
|
||||
// shaders and pipelines
|
||||
{"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "fullscreen", "strict|any|fullscreen", RESTART},
|
||||
|
||||
@@ -43,6 +43,15 @@ _Static_assert(sizeof(nvn_mapping_t) == 0x28, "NVNmappingRequest stride");
|
||||
|
||||
const pool_t *nvn_pool_find(uint64_t addr, uint64_t len); // GPU-visible pool holding [addr, addr+len], NULL if none
|
||||
uint64_t nvn_new_serial(void); // next tex_t / prog_t serial, from 1
|
||||
// TOTK_SYNC_AHEAD (nvn_api_core.c): nvnQueuePresentTexture notes the newest fence at each present; the newest
|
||||
// previous-frame value the game was told is done although it may not be (0: none). The back end's shadow copies
|
||||
// wait for it or are queued for the GPU (vk_sync.c prologue), so the GPU never reads CPU writes the game made under
|
||||
// that assumption too early.
|
||||
void nvn_sync_ahead_present(void);
|
||||
uint64_t nvn_sync_ahead_guard(void);
|
||||
int vk_mem_shadow_on(void); // vk_memory.c: 1 with shadow storage on (valid once the Vulkan back end is up)
|
||||
void vk_mem_shadow_fence(void); // vk_memory.c: returns once every shadow sync that read the old guard is done
|
||||
void gfx_replay_wait_value(uint64_t v); // vk_backend.c: the replay thread submitted value v
|
||||
uint64_t nvn_fence_last(void); // fence value of the newest submit (0 before the first)
|
||||
int nvn_depth_minus_w(void); // 1 after DeviceSetDepthMode(NEAR_IS_MINUS_W), else 0
|
||||
void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the game wrote pool storage: back end re-reads
|
||||
|
||||
+67
-3
@@ -471,6 +471,49 @@ NVN_API(nvnQueueFenceSync) {
|
||||
s->fenced = 1;
|
||||
s->fence = nvn_fence_last();
|
||||
}
|
||||
// TOTK_SYNC_AHEAD=1: one more frame in flight. Each frame the game waits (one nvnSyncWait, frame_wait_lr) for
|
||||
// the frame it presented last; on the Switch that work is long done by then, here it is still in the replay and on the
|
||||
// GPU (about 11 ms from the submit), and the wait sets the frame time. With the switch on, that wait, and only that
|
||||
// one, waits for the frame before it instead. What the game reads back of GPU results after the wait can then be a
|
||||
// frame old. The game's CPU writes to memory the GPU still reads reach the GPU only through shadow copies, which are
|
||||
// queued for the GPU or wait (nvn_sync_ahead_guard). The wait still lasts until the replay thread submitted the asked
|
||||
// value, so no copy of that value itself picks up those writes. pa_fence: the newest fence at the latest present and
|
||||
// the one before, present thread only for writes. pa_guard: the newest value the game was told is done early.
|
||||
static uint64_t pa_fence[2], pa_guard;
|
||||
// The return address of that wait. 1.2.1: 0x7000b23dfc. 1.4.3: 0x70002db99c, the second of the two nvnSyncWait calls
|
||||
// in f_70002db8ac; like on 1.2.1 its sync comes from a two-entry ring indexed per frame, and the first call waits on
|
||||
// one object's sync behind a mode check. 0: no site known, the switch stays off.
|
||||
static uint64_t frame_wait_lr(void) {
|
||||
return rt_game_version == 121 ? 0x7000b23dfcull : rt_game_version == 143 ? 0x70002db99cull : 0;
|
||||
}
|
||||
static int sync_ahead(void) {
|
||||
static int on = -1;
|
||||
if (on < 0) {
|
||||
on = cfg_bool("TOTK_SYNC_AHEAD");
|
||||
// Without shadow storage the GPU reads the game's own memory, which the game writes ahead: nothing guards
|
||||
// that. The Vulkan back end sets the mode while it starts, before the game's first present or wait; the null
|
||||
// back end never does.
|
||||
if (on && !vk_mem_shadow_on()) {
|
||||
rt_log("nvn: TOTK_SYNC_AHEAD needs shadow storage (TOTK_SHADOW=1 on native Windows); "
|
||||
"the game waits for the GPU as before");
|
||||
on = 0;
|
||||
}
|
||||
if (on && !frame_wait_lr()) {
|
||||
rt_log("nvn: TOTK_SYNC_AHEAD: no frame wait site is known for this game version; the game waits for the "
|
||||
"GPU as before");
|
||||
on = 0;
|
||||
}
|
||||
if (on) rt_log("nvn: TOTK_SYNC_AHEAD: a wait for the frame presented last waits for the frame before it");
|
||||
}
|
||||
return on;
|
||||
}
|
||||
void nvn_sync_ahead_present(void) {
|
||||
if (!sync_ahead()) return;
|
||||
__atomic_store_n(&pa_fence[1], __atomic_load_n(&pa_fence[0], __ATOMIC_RELAXED), __ATOMIC_RELEASE);
|
||||
__atomic_store_n(&pa_fence[0], nvn_fence_last(), __ATOMIC_RELEASE);
|
||||
}
|
||||
uint64_t nvn_sync_ahead_guard(void) { return __atomic_load_n(&pa_guard, __ATOMIC_ACQUIRE); }
|
||||
|
||||
// NVNsyncWaitResult nvnSyncWait(const NVNsync *s, uint64_t timeout_ns): a sync never fenced reports a timeout at once;
|
||||
// a fenced one is released only after the back end completed that fence value
|
||||
NVN_API(nvnSyncWait) {
|
||||
@@ -478,13 +521,34 @@ NVN_API(nvnSyncWait) {
|
||||
uint64_t timeout = arg_x(c, 1);
|
||||
if (!s->fenced) { ret_x(c, NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED); return; }
|
||||
const GfxBackend *b = gfx_backend();
|
||||
if (b->fence_done() >= s->fence) { b->cpu_synced(s->fence); ret_x(c, NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED); return; }
|
||||
uint64_t want = s->fence;
|
||||
// Only the game's per-frame wait lags, and not a poll (timeout 0).
|
||||
if (timeout && c->x[30] == frame_wait_lr() && sync_ahead() && b->fence_done() < want) {
|
||||
uint64_t p0 = __atomic_load_n(&pa_fence[0], __ATOMIC_ACQUIRE);
|
||||
uint64_t p1 = __atomic_load_n(&pa_fence[1], __ATOMIC_ACQUIRE);
|
||||
if (want <= p0 && want > p1) want = p1;
|
||||
}
|
||||
if (want != s->fence) { // the replay of the asked value is done first: the guard then covers every copy of it
|
||||
{
|
||||
NVN_BLOCKING();
|
||||
gfx_replay_wait_value(s->fence);
|
||||
}
|
||||
uint64_t g = __atomic_load_n(&pa_guard, __ATOMIC_RELAXED);
|
||||
while (s->fence > g &&
|
||||
!__atomic_compare_exchange_n(&pa_guard, &g, s->fence, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {}
|
||||
vk_mem_shadow_fence(); // a shadow sync that read the old guard finishes before the game writes again
|
||||
}
|
||||
if (b->fence_done() >= want) {
|
||||
b->cpu_synced(want == s->fence ? s->fence : b->fence_done());
|
||||
ret_x(c, NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED);
|
||||
return;
|
||||
}
|
||||
int ok;
|
||||
{
|
||||
NVN_BLOCKING();
|
||||
ok = b->fence_wait(s->fence, timeout);
|
||||
ok = b->fence_wait(want, timeout);
|
||||
}
|
||||
if (ok) b->cpu_synced(s->fence);
|
||||
if (ok) b->cpu_synced(want == s->fence ? s->fence : b->fence_done());
|
||||
ret_x(c, ok ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED);
|
||||
}
|
||||
// void nvnEventBuilderSetDefaults(NVNeventBuilder *b)
|
||||
|
||||
@@ -713,6 +713,7 @@ NVN_API(nvnQueuePresentTexture) {
|
||||
const tex_t *t = (const tex_t *)o->tex[idx];
|
||||
uint64_t period = present_period(o), now = kern_now_ns();
|
||||
uint64_t slot = now + pace(&o->pace, now, period);
|
||||
nvn_sync_ahead_present();
|
||||
gfx_backend()->present(t, o->crop, slot);
|
||||
o->ticket[idx] = o->presents;
|
||||
o->avail[idx] = cfg_bool("TOTK_FRAME_QUEUE") ? slot : slot + period;
|
||||
|
||||
+1
-1
@@ -355,7 +355,7 @@ static int test_import(pool_t *pool) {
|
||||
GCHECK(n == 4 && pages[0] == a + 0x30000 && pages[3] == a + 0x33000, "CPU writes to 4 pages: %llu reported",
|
||||
(unsigned long long)n);
|
||||
vk_mem_resolve(a + 0x30000, 0x4000, &sp);
|
||||
vk_mem_shadow_sync();
|
||||
vk_mem_shadow_sync(0);
|
||||
copy_wait(sp.buffer, sp.offset, sb, 0, 0x4000);
|
||||
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
|
||||
GCHECK(!memcmp(s, m + 0x30000, 0x4000), "GPU read of CPU-written heap differs");
|
||||
|
||||
+7
-1
@@ -290,7 +290,13 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb);
|
||||
// Shadow storage (vk_memory.c): vk_sync.c calls vk_mem_shadow_sync right before each submit; the replay notes every GPU
|
||||
// write into CPU-visible memory at record time (vk_mem_gpu_pending) and reports it at the fence with vk_gpu_written,
|
||||
// which copies the bytes back into guest memory before telling the texture cache (vk_guest_written).
|
||||
void vk_mem_shadow_sync(void);
|
||||
void vk_mem_shadow_sync(uint64_t fence); // fence: the value of the part about to be submitted (0 untracked)
|
||||
// TOTK_SHADOW_DEFER: shadow updates that may not be written by the CPU while the GPU still reads the shadow are queued
|
||||
// by vk_mem_shadow_sync as ranges of guest memory. vk_mem_shadow_deferred says whether there are any;
|
||||
// vk_mem_shadow_prologue records the copies (staging -> shadow buffers) into the prologue command buffer, before
|
||||
// vk_mem_mirror_prologue's copies, which read the shadow.
|
||||
int vk_mem_shadow_deferred(void);
|
||||
void vk_mem_shadow_prologue(VkCommandBuffer cb, uint64_t fence);
|
||||
void vk_mem_gpu_pending(uint64_t addr, uint64_t size);
|
||||
void vk_gpu_written(uint64_t addr, uint64_t size);
|
||||
int vk_mem_flush_add_probe(void); // --probe-vkexec: the flush range list; 0 or the number of the failed check
|
||||
|
||||
@@ -160,6 +160,7 @@ static PlatCond rcv = PLAT_COND_INIT; // item queued, item taken, item done
|
||||
static ReplayItem rq[RQ_CAP];
|
||||
static int rq_head, rq_n, r_busy, r_started;
|
||||
static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames;
|
||||
static uint64_t r_value_done; // under rl: the newest timeline value the replay thread submitted
|
||||
// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run.
|
||||
static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N];
|
||||
// TOTK_REPLAY_CHECK: ReadTexels compared with a read after a full drain (under rl): checked, differing, differing
|
||||
@@ -192,6 +193,15 @@ uint64_t gfx_replayed_submits(void) {
|
||||
return v;
|
||||
}
|
||||
|
||||
// TOTK_SYNC_AHEAD (nvn_api_core.c nvnSyncWait): waits until the replay thread submitted value v, so every shadow copy
|
||||
// of v was made before the game writes memory v reads. The caller has given its core up.
|
||||
void gfx_replay_wait_value(uint64_t v) {
|
||||
if (!gfx_replay_async()) return;
|
||||
plat_mutex_lock(&rl);
|
||||
while (r_value_done < v) plat_cond_wait(&rcv, &rl);
|
||||
plat_mutex_unlock(&rl);
|
||||
}
|
||||
|
||||
static uint64_t submit_sync(const uint64_t *handles, int n) {
|
||||
vk_sync_start();
|
||||
vk_present_drain();
|
||||
@@ -227,6 +237,7 @@ static void replay_main(void *arg) {
|
||||
present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value);
|
||||
}
|
||||
plat_mutex_lock(&rl);
|
||||
if (it.kind != RI_RELEASE) r_value_done = it.value;
|
||||
if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no;
|
||||
else r_presents_done++;
|
||||
r_busy = 0;
|
||||
|
||||
+156
-8
@@ -71,6 +71,9 @@ typedef struct PoolRec {
|
||||
uint8_t *smap; // its mapping; offsets as in buf (from import_lo)
|
||||
uint32_t sgen; // gen_lock: every page write stamped up to this generation is in the shadow
|
||||
volatile uint8_t stouched; // on the touched list: the GPU may use it in the submission being recorded
|
||||
uint64_t suse; // shadow_lock: the newest timeline value whose submission reads the shadow
|
||||
uint64_t sdefer; // shadow_lock: the newest value whose prologue carries a deferred copy into the
|
||||
// shadow (TOTK_SHADOW_DEFER); 0 never
|
||||
} PoolRec;
|
||||
|
||||
static void mirror_forget(PoolRec *r); // pool_lock exclusive: off the mirrored list
|
||||
@@ -339,7 +342,8 @@ void vk_pool_fini(pool_t *p) {
|
||||
// recording may already name this pool: a mirrored pool's sync copies and write-backs, an import pool's buffer
|
||||
// through a texture upload or a released cache entry (vk_texture.c slot_resolve), a physical pool's memory under
|
||||
// the images vk_texture_range_gone just deferred to that submission. The game frees a pool only once its own GPU
|
||||
// work is done, so only the port's work can name it. Freed here, the submission ran with freed memory.
|
||||
// work is done, so only the port's work can name it. Freed here, the submission ran with freed memory. A shadow
|
||||
// with a deferred copy (TOTK_SHADOW_DEFER) is named by the queue (dq) until the prologue of that submission.
|
||||
pool_retiring(r);
|
||||
vk_completion_add(retire_value(), pool_retire, r);
|
||||
}
|
||||
@@ -997,6 +1001,18 @@ static int nspend, cspend;
|
||||
static int64_t s_live_n, s_live_bytes; // shadow_lock
|
||||
static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks; // shadow_lock: since the last log
|
||||
|
||||
// TOTK_SHADOW_DEFER (with TOTK_SYNC_AHEAD): a shadow update the CPU must not write yet is queued instead and executed
|
||||
// by the GPU, in queue order, at the start of the prologue of the part submitted next (vk_mem_shadow_prologue).
|
||||
// defer_on: the switch is on. sdefer_now: the pool being synced is deferred, and sdefer_fence the value its copies
|
||||
// belong to. Replay thread (sub_lock): the queue.
|
||||
typedef struct { PoolRec *r; uint64_t addr, len; } DRange;
|
||||
static int defer_on;
|
||||
static int sdefer_now;
|
||||
static uint64_t sdefer_fence;
|
||||
static DRange *dq; // guest ranges to copy into their pools' shadows
|
||||
static int ndq, cdq;
|
||||
static uint64_t s_defer_pools, s_defer_bytes, s_defer_copies, s_wait_ticks, s_waits; // shadow_lock: since the last log
|
||||
|
||||
static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
|
||||
r->import_lo = addr & ~0xfffull;
|
||||
r->import_hi = (addr + size + 0xfff) & ~0xfffull;
|
||||
@@ -1055,6 +1071,13 @@ static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b, int i) {
|
||||
if (spend[i].hi < b) shadow_copy(r, spend[i].hi, b, i + 1);
|
||||
return;
|
||||
}
|
||||
if (sdefer_now) { // the GPU copies it later: the bytes are read from guest memory when the prologue is recorded
|
||||
if (ndq == cdq) grow((void **)&dq, &cdq, sizeof *dq);
|
||||
dq[ndq++] = (DRange){r, a, b - a};
|
||||
r->sdefer = sdefer_fence;
|
||||
s_defer_bytes += b - a;
|
||||
return;
|
||||
}
|
||||
memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, b - a);
|
||||
s_copied += b - a;
|
||||
}
|
||||
@@ -1137,13 +1160,76 @@ static void shadow_copy_newer(PoolRec *r, uint32_t since) {
|
||||
if (run) shadow_copy(r, run, hi, 0);
|
||||
}
|
||||
|
||||
void vk_mem_shadow_sync(void) {
|
||||
if (!shadow_on) return;
|
||||
uint64_t t0 = qpc();
|
||||
// TOTK_SYNC_AHEAD (nvn_api_core.c): the game may have been told a value is done (at most the guard) that the GPU still
|
||||
// runs, and then write memory that value reads. The CPU copies its writes into a shadow, so while the guard is not done
|
||||
// a shadow with any reader in flight must not be written by the CPU: that reader may be the guarded value or an older
|
||||
// one (the parts of one submission share a value), and a pool records only its newest reader. A shadow with no reader
|
||||
// in flight is copied at once.
|
||||
// With TOTK_SHADOW_DEFER the update of such a shadow is queued instead (this function). Without it the replay thread
|
||||
// waits (shadow_wait_guard).
|
||||
|
||||
// shadow_lock held: whether a shadow this sync is about to update has a reader in flight.
|
||||
static int shadow_reader_busy(uint64_t done) {
|
||||
for (int i = 0; i < nstouch; i++)
|
||||
if (stouch[i]->suse > done) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// shadow_lock held (released while waiting), no TOTK_SHADOW_DEFER: waits until the guard is done when a shadow to
|
||||
// update has a reader in flight, then re-reads the guard under the lock, since the game may have raised it meanwhile. The wait
|
||||
// runs without shadow_lock: the completion thread needs it for the fence callbacks (vk_gpu_written). *g and *done are
|
||||
// current on return.
|
||||
static void shadow_wait_guard(uint64_t fence, uint64_t *g, uint64_t *done) {
|
||||
while (*g > *done && (!fence || *g < fence) && shadow_reader_busy(*done)) {
|
||||
uint64_t want = *g;
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
uint64_t t0 = qpc();
|
||||
vk_sync_fence_wait(want, UINT64_MAX);
|
||||
uint64_t dt = qpc() - t0;
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
s_wait_ticks += dt, s_waits++;
|
||||
*g = nvn_sync_ahead_guard();
|
||||
*done = vk_sync_fence_done();
|
||||
}
|
||||
}
|
||||
|
||||
// shadow_lock held, TOTK_SHADOW_DEFER: the pool's update goes through the queue when a reader of it is in flight while
|
||||
// the guard is (see above), or when an earlier deferred copy into it has not executed yet: a direct memcpy must never
|
||||
// land before it, or the queued copy would overwrite newer bytes with older ones.
|
||||
static int shadow_defers(const PoolRec *r, uint64_t g, uint64_t done) {
|
||||
if (r->sdefer > done) return 1;
|
||||
return g > done && r->suse > done;
|
||||
}
|
||||
|
||||
// nvnSyncWait raised the guard: a sync that read the old guard holds shadow_lock until its copies are made, so taking
|
||||
// the lock once here makes the game return from the wait only after such a sync finished. Later syncs see the new
|
||||
// guard.
|
||||
void vk_mem_shadow_fence(void) {
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
|
||||
int vk_mem_shadow_on(void) { return shadow_on; }
|
||||
|
||||
void vk_mem_shadow_sync(uint64_t fence) {
|
||||
if (!shadow_on) return;
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
// The guard is read under the lock: see vk_mem_shadow_fence.
|
||||
uint64_t g = nvn_sync_ahead_guard(), done = g ? vk_sync_fence_done() : 0;
|
||||
if (!defer_on && nstouch) shadow_wait_guard(fence, &g, &done);
|
||||
uint64_t t0 = qpc();
|
||||
if (nstouch) {
|
||||
plat_mutex_lock(&gen_lock);
|
||||
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i]), stouch[i]->stouched = 0;
|
||||
for (int i = 0; i < nstouch; i++) {
|
||||
PoolRec *r = stouch[i];
|
||||
sdefer_now = defer_on && fence && shadow_defers(r, g, done);
|
||||
sdefer_fence = fence;
|
||||
int before = ndq;
|
||||
shadow_sync_pool(r), r->stouched = 0;
|
||||
if (ndq > before) s_defer_pools++;
|
||||
sdefer_now = 0;
|
||||
if (fence > r->suse) r->suse = fence;
|
||||
}
|
||||
plat_mutex_unlock(&gen_lock);
|
||||
s_syncs += (uint64_t)nstouch;
|
||||
nstouch = 0;
|
||||
@@ -1152,6 +1238,8 @@ void vk_mem_shadow_sync(void) {
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
|
||||
int vk_mem_shadow_deferred(void) { return ndq; }
|
||||
|
||||
void vk_mem_gpu_pending(uint64_t addr, uint64_t size) {
|
||||
if (!shadow_on || !size) return;
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
@@ -1183,11 +1271,14 @@ static void shadow_log(uint64_t frame) {
|
||||
if (!shadow_on) return;
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
rt_log("mem: shadow at frame %llu: %lld pools %.1f MiB; since the last: %llu pool syncs, %llu queries, %.1f MiB "
|
||||
"copied in (%llu untracked), %.1f MiB written back, %.2f ms, %d GPU writes pending",
|
||||
"copied in (%llu untracked), %.1f MiB written back, %.2f ms, %d GPU writes pending; deferred (%s): %llu pools, "
|
||||
"%.1f MiB in %llu copies; replay waited for a reader %llu times, %.2f ms",
|
||||
(unsigned long long)frame, (long long)s_live_n, s_live_bytes / 1048576.0, (unsigned long long)s_syncs,
|
||||
(unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, s_wb_bytes / 1048576.0,
|
||||
qpc_ms(s_ticks), nspend);
|
||||
qpc_ms(s_ticks), nspend, defer_on ? "on" : "off", (unsigned long long)s_defer_pools,
|
||||
s_defer_bytes / 1048576.0, (unsigned long long)s_defer_copies, (unsigned long long)s_waits, qpc_ms(s_wait_ticks));
|
||||
s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = 0;
|
||||
s_defer_pools = s_defer_bytes = s_defer_copies = s_wait_ticks = s_waits = 0;
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
|
||||
@@ -1403,6 +1494,8 @@ static void mirror_mode(int discrete) {
|
||||
#else
|
||||
shadow_on = !sh || !*sh || strcmp(sh, "0");
|
||||
#endif
|
||||
defer_on = shadow_on && cfg_bool("TOTK_SYNC_AHEAD") && cfg_bool("TOTK_SHADOW_DEFER");
|
||||
if (defer_on) rt_log("mem: TOTK_SHADOW_DEFER: shadow updates that wait for a reader are queued for the GPU instead");
|
||||
if (shadow_on) rt_log("mem: shadow storage: CPU-visible pools are host-visible copies of guest memory, not imports%s",
|
||||
sh && *sh ? " (TOTK_SHADOW)" : "");
|
||||
rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why,
|
||||
@@ -2545,6 +2638,61 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
|
||||
return info.pMappedData;
|
||||
}
|
||||
|
||||
typedef struct { VkBuffer b; VmaAllocation a; } StgFree;
|
||||
static void free_shadow_stg(void *arg) {
|
||||
StgFree *f = arg;
|
||||
vmaDestroyBuffer(vk_ctx()->vma, f->b, f->a);
|
||||
free(f);
|
||||
}
|
||||
|
||||
// Records the deferred shadow copies (vk_mem_shadow_sync queued the ranges): the guest bytes go into staging buffers
|
||||
// now, the GPU copies them into the pools' shadows. The barrier before waits for every earlier command, writes and
|
||||
// reads alike (a read of the shadow must be over before the copy overwrites it); the one after makes the copies visible
|
||||
// to the rest of the queue, including the mirror copies that follow in this command buffer. The staging buffers are
|
||||
// freed when the value completes. Replay thread, sub_lock held.
|
||||
void vk_mem_shadow_prologue(VkCommandBuffer cb, uint64_t fence) {
|
||||
if (!ndq) return;
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT | VK_ACCESS_2_MEMORY_READ_BIT,
|
||||
VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
||||
const uint64_t lim = STG_BLOCK / 2;
|
||||
uint64_t copies = 0;
|
||||
int i = 0;
|
||||
uint64_t at_i = 0; // bytes of dq[i] already recorded
|
||||
while (i < ndq) {
|
||||
uint64_t total = 0, jo = at_i; // how much the next staging buffer holds
|
||||
for (int j = i; j < ndq && total < lim;) {
|
||||
uint64_t left = dq[j].len - jo, take = left < lim - total ? left : lim - total;
|
||||
total += take;
|
||||
if (take == left) j++, jo = 0;
|
||||
else jo += take;
|
||||
}
|
||||
VkBuffer sb;
|
||||
VmaAllocation sa;
|
||||
uint8_t *p = vk_staging(total, &sb, &sa);
|
||||
for (uint64_t at = 0; at < total;) {
|
||||
const DRange *d = &dq[i];
|
||||
uint64_t left = d->len - at_i, take = left < total - at ? left : total - at;
|
||||
memcpy(p + at, (const void *)(uintptr_t)(d->addr + at_i), take);
|
||||
VkBufferCopy bc = {at, d->addr + at_i - d->r->import_lo, take};
|
||||
vkCmdCopyBuffer(cb, sb, d->r->buf, 1, &bc);
|
||||
copies++;
|
||||
at += take, at_i += take;
|
||||
if (at_i == d->len) i++, at_i = 0;
|
||||
}
|
||||
vmaFlushAllocation(vk_ctx()->vma, sa, 0, VK_WHOLE_SIZE);
|
||||
StgFree *f = malloc(sizeof *f);
|
||||
if (!f) rt_fatal("vk: out of memory");
|
||||
f->b = sb, f->a = sa;
|
||||
vk_completion_add(fence, free_shadow_stg, f);
|
||||
}
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
||||
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
||||
ndq = 0;
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
s_defer_copies += copies;
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
|
||||
// Device-local scratch of the upload batch (the GPU deswizzle's linear copies, the ASTC decoder's blocks and texels):
|
||||
// a pool of its own, like staging, so the replay thread neither waits on the default pool's lock while another thread
|
||||
// makes a block there (the headroom thread: about 8 ms for 256 MiB) nor makes one itself for a few MiB. One block
|
||||
@@ -2832,7 +2980,7 @@ void vk_mem_shutdown(void) {
|
||||
plat_mutex_lock(&pool_lock);
|
||||
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
|
||||
pool_index.n = 0;
|
||||
nm = npend = nwbl = nsnl = snl_copied = 0; // mirrors went with their pools
|
||||
nm = npend = nwbl = nsnl = snl_copied = ndq = 0; // mirrors went with their pools
|
||||
plat_mutex_lock(&snap_pool_lock);
|
||||
for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc);
|
||||
nsnap_pool = 0, snap_pool_bytes = 0;
|
||||
|
||||
+5
-2
@@ -368,8 +368,10 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
|
||||
// direct: the calling thread records, ends and submits it itself while the part being replayed stays the recorder's
|
||||
// (vk_sync_part, which the caller made the recorder idle for).
|
||||
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence, int direct) {
|
||||
vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted
|
||||
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
|
||||
// Shadow storage: the CPU's writes go in before the part is submitted (TOTK_SYNC_AHEAD: queued for the GPU or
|
||||
// made after the GPU finished the readers the game may think done, vk_memory.c).
|
||||
vk_mem_shadow_sync(fence);
|
||||
if (!vk_mem_mirror_pending() && !vk_mem_shadow_deferred()) return VK_NULL_HANDLE;
|
||||
int i = claim_slot(c, fence); // before reading slots: the claim may move it
|
||||
VkCommandBuffer cb = slots[i].cb;
|
||||
if (direct) {
|
||||
@@ -380,6 +382,7 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence, int direct) {
|
||||
rec_begin(cb, fence, 0, 0); // a command buffer of the part pool: the recorder's like the parts
|
||||
}
|
||||
int pair = ts_open(cb);
|
||||
vk_mem_shadow_prologue(cb, fence); // before the mirror copies, which read the shadow these bring up to date
|
||||
vk_mem_mirror_prologue(cb);
|
||||
ts_close(cb, pair, fence);
|
||||
if (direct || !rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb);
|
||||
|
||||
Reference in new issue
Block a user