mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-21 04:55:56 +00:00
vk_backend.c present_pair: both blits are submitted at once (gfx_present.c vk_present_blit_pair, the flip queue of two), the generated picture is presented after a timeline wait on the frame's last submission and not before the pacer's display time, the game's frame after a wait on the value after both blits and half the smoothed interval between generated pictures (factor 0.1) after it, as AMD's frame interpolation swapchain and Savage Wind's present_wait_turn do: spaced by the CPU clock alone both pictures went on screen together under FIFO and G-SYNC. The swapchain gets one image more while frame_gen names a generator, for the two images held at once; the present slots stay at two (each blit waits on the fence of the blit two before, the previous frame's same picture, already shown). The generated picture's blit lays the menu and overlay out, the game's draws them again; colour and FSR 1 run per blit; the flight recorder keeps the game's frames; the generated present carries no present ID. frame_rate.c frame_step_cap_ns halves the limit, 30 drawn at least, while the last frame went out with a generated picture, so menus and fades keep the whole limit; the real-time step follows the game's presents and stays the drawn interval. With frame_gen = off the present path is unchanged. "fg present" log lines give shown and drawn fps and the present intervals (mean, sd, p99, each half of the pair) in the world. Measured on the RX 9070 and the 7900X on 4 cores, field save, camera turning, 1080p output: limit 120 immediate 117 shown and 60 drawn, intervals 8.34 ms sd 0.16 from generated to game's, 8.38 sd 1.71 back; FIFO at 144 Hz the same at 120, and 141 shown and 72 drawn without a limit; immediate without a limit 169 shown and 86 drawn against 94 drawn with frame_gen = off. Replay-thread cost 0.59-0.65 ms a drawn frame. The flip queue, the pair blit, the halving rule and the present without an ID follow PR #1089 by xellow78-lgtm. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-Authored-By: xellow78-lgtm <336264245+xellow78-lgtm@users.noreply.github.com>
702 lines
33 KiB
C
702 lines
33 KiB
C
// The Vulkan graphics back end: gfx_vulkan_backend wires the memory and
|
|
// texture layer (vk_memory.c, vk_texture.c), submission and fences (vk_sync.c) and the present. Every entry starts the
|
|
// device on first use.
|
|
//
|
|
// Present: the game texture is resolved on the calling thread and its blit is handed to a present thread, so
|
|
// nvnQueuePresentTexture returns at once (section 5). Queue order does the rest: the submissions that wrote the texture
|
|
// are already on gfx_queue, and every call that may write or free GPU state the blit reads (submit, finish, pool and
|
|
// texture teardown, texel writes) first waits until the queued blit was submitted (vk_present_drain).
|
|
//
|
|
// Back-edge pacing: the present thread submits the blit at once, then holds the host present (vkQueuePresentKHR) until
|
|
// the frame's display time from the pacer (nvn_api_obj.c), so present-to-present intervals stay even whatever the
|
|
// replay took. The blit is already on the queue, so the wait blocks no submission.
|
|
//
|
|
// Frame generation (vk_fg.cpp, frame_gen = fsr3): a frame that comes with a generated picture (the one between the
|
|
// last frame and this) is presented as a pair, both blits submitted at once (present_pair). The display shows a
|
|
// picture when the GPU has drawn it, not when the CPU hands it over, and both pictures of a pair are drawn within a
|
|
// millisecond of each other: spaced by the CPU clock alone they went on screen together and then left a gap (Savage
|
|
// Wind on FIFO and G-SYNC). So each present waits on the timeline until the GPU is done: the generated picture once
|
|
// the frame is drawn (and not before the pacer's display time), the game's half the smoothed interval between
|
|
// generated pictures after it, as AMD's frame interpolation swapchain and Savage Wind's present_wait_turn do. The game
|
|
// draws at half the frame limit meanwhile (frame_rate.c frame_step_cap_ns, gfx_frame_gen_live). The spacing is ported
|
|
// from Savage Wind (https://github.com/Revivedx/SavageWindRecomp @ bf6146c14c37fee8781a601e5a544b93dff6d75e, MIT,
|
|
// Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai),
|
|
// runtime/nvn/vulkan/device.c present_wait_turn.
|
|
//
|
|
// Asynchronous replay (TOTK_REPLAY_ASYNC, default 0; 1 turns it on): nvnQueueSubmitCommands
|
|
// and nvnQueuePresentTexture only queue. Each item gets its timeline value at enqueue (vk_sync_reserve), in queue
|
|
// order; the replay thread runs the synchronous submit and present bodies in that order, so every value is signalled
|
|
// by the submission it was handed out for. The game runs at most one frame ahead: queueing present N waits until the
|
|
// replay thread handled present N-1. Calls that read or free GPU state that queued work may use drain the queue first
|
|
// (replay_drain), each cause counted.
|
|
#include <math.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "flight.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "nvn_api.h"
|
|
#include "vk.h"
|
|
#include "vk_exec.h"
|
|
#include "platform/platform.h"
|
|
#include "vk_dlaa.h"
|
|
#include "vk_fg.h"
|
|
|
|
#define WEAK __attribute__((weak))
|
|
|
|
typedef struct { VkImage image, gen; VkFormat format; int w, h, has_crop, crop[4]; uint64_t frame, at, queued, value, pid; } PresentJob; // pid: the frame's present ID (vk_reflex_frame); gen: its generated picture (vk_fg_present_gen)
|
|
|
|
static PlatMutex pl = PLAT_MUTEX_INIT;
|
|
static PlatCond pcv = PLAT_COND_INIT; // job queued, job taken, job submitted
|
|
static PresentJob job;
|
|
static int pending, busy, present_thread;
|
|
|
|
// Sleeps on a high-resolution timer to 1 ms before at, then yields until at: the timer alone wakes up to ~0.5 ms late.
|
|
static void wait_until(uint64_t at) {
|
|
uint64_t now = kern_now_ns();
|
|
if (at > now + 1500000) plat_wait_ns(at - now - 1000000);
|
|
while (kern_now_ns() < at) plat_yield();
|
|
}
|
|
|
|
// ---- Frame generation's presents (present thread only)
|
|
static volatile int32_t fg_live; // the last frame went out with a generated picture before it (gfx_frame_gen_live)
|
|
int gfx_frame_gen_live(void) { return __atomic_load_n(&fg_live, __ATOMIC_RELAXED); }
|
|
|
|
// Present intervals in the world while frame generation is on, logged every FS_EVERY presents and over the run: all of them, from
|
|
// a generated picture to the game's frame after it, and from the game's frame to the next generated picture.
|
|
// Fixed-size histograms of 0.05 ms bins (bounded whatever the session's length).
|
|
#define FS_BINS 2000
|
|
#define FS_BIN_MS 0.05
|
|
#define FS_EVERY 2400
|
|
enum { FS_ALL, FS_GEN_REAL, FS_REAL_GEN, FS_N };
|
|
typedef struct {
|
|
uint32_t hist[FS_N][FS_BINS];
|
|
double sum[FS_N], sq[FS_N];
|
|
uint64_t n[FS_N];
|
|
uint64_t pairs, alone, t0, t1; // frames presented as a pair and alone, the window's first and last present
|
|
double gpu_wait, late; // ms: waited for the GPU before the generated picture; the game's frame past its time
|
|
} FgStats;
|
|
static FgStats fs_win, fs_run;
|
|
static uint64_t fs_last, fs_last_gen; // the last present (0 after a gap) and the last generated one
|
|
static double fs_iv; // ns between generated pictures, smoothed (factor 0.1, Savage Wind's)
|
|
|
|
static void fs_add(FgStats *s, int k, double ms) {
|
|
unsigned b = ms <= 0 ? 0 : ms / FS_BIN_MS >= FS_BINS ? FS_BINS - 1 : (unsigned)(ms / FS_BIN_MS);
|
|
s->hist[k][b]++, s->sum[k] += ms, s->sq[k] += ms * ms, s->n[k]++;
|
|
}
|
|
|
|
static double fs_p99(const FgStats *s, int k) {
|
|
uint64_t want = s->n[k] - s->n[k] / 100, got = 0;
|
|
for (unsigned b = 0; b < FS_BINS; b++)
|
|
if ((got += s->hist[k][b]) >= want) return (b + 1) * FS_BIN_MS;
|
|
return FS_BINS * FS_BIN_MS;
|
|
}
|
|
|
|
static void fs_log(const FgStats *s, const char *what) {
|
|
if (!s->n[FS_ALL] || s->t1 <= s->t0) return;
|
|
double wall = (s->t1 - s->t0) / 1e9, m[FS_N], sd[FS_N];
|
|
for (int k = 0; k < FS_N; k++) {
|
|
m[k] = s->n[k] ? s->sum[k] / s->n[k] : 0;
|
|
double v = s->n[k] ? s->sq[k] / s->n[k] - m[k] * m[k] : 0;
|
|
sd[k] = v > 0 ? sqrt(v) : 0;
|
|
}
|
|
rt_log("fg present %s: %.1f fps shown, %.1f drawn (%llu frames as pairs, %llu alone) over %.1f s; intervals mean %.3f "
|
|
"sd %.3f p99 %.2f ms; generated to game's %.3f sd %.3f, game's to generated %.3f sd %.3f; waited for the GPU "
|
|
"%.3f ms a pair, game's frame late %.3f ms a pair (means)", what, (s->n[FS_ALL] + 1) / wall,
|
|
(s->pairs + s->alone) / wall, (unsigned long long)s->pairs, (unsigned long long)s->alone, wall, m[FS_ALL],
|
|
sd[FS_ALL], fs_p99(s, FS_ALL), m[FS_GEN_REAL], sd[FS_GEN_REAL], m[FS_REAL_GEN], sd[FS_REAL_GEN],
|
|
s->pairs ? s->gpu_wait / s->pairs : 0, s->pairs ? s->late / s->pairs : 0);
|
|
}
|
|
|
|
// A present at now of kind k (FS_GEN_REAL: the game's frame after a generated picture; FS_REAL_GEN: a generated
|
|
// picture; FS_ALL: a frame alone)
|
|
static void fs_present(uint64_t now, int k) {
|
|
if (!vk_fg_active()) return;
|
|
if (frame_step_world_us() <= 0) { // (the title screen and the first load: not counted)
|
|
fs_last = 0;
|
|
return;
|
|
}
|
|
FgStats *both[2] = {&fs_win, &fs_run};
|
|
for (int i = 0; i < 2; i++) {
|
|
FgStats *s = both[i];
|
|
if (!s->t0) s->t0 = now;
|
|
if (fs_last && now - fs_last < 100000000ull) { // (a gap: a load, a menu's hitch)
|
|
double ms = (now - fs_last) / 1e6;
|
|
fs_add(s, FS_ALL, ms);
|
|
if (k != FS_ALL) fs_add(s, k, ms);
|
|
}
|
|
s->t1 = now;
|
|
s->pairs += k == FS_GEN_REAL, s->alone += k == FS_ALL;
|
|
}
|
|
fs_last = now;
|
|
if (fs_win.n[FS_ALL] >= FS_EVERY) {
|
|
fs_log(&fs_win, "last presents");
|
|
memset(&fs_win, 0, sizeof fs_win);
|
|
}
|
|
}
|
|
|
|
static void gpu_wait(uint64_t value) {
|
|
VkCtx *c = vk_ctx();
|
|
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
|
|
.pSemaphores = &c->timeline, .pValues = &value};
|
|
vkWaitSemaphores(c->device, &wi, 1000000000ull); // (drawn; a second at most)
|
|
}
|
|
|
|
// The pair's two presents (both blits submitted): the generated picture once the GPU has drawn the frame (gpu: its
|
|
// last submission's value) and the pacer's time has come, the game's frame once its blit is drawn (f) and half the
|
|
// smoothed interval between generated pictures has passed.
|
|
static void present_pair(const PresentJob *j, uint64_t gpu, uint64_t f) {
|
|
uint64_t t0 = kern_now_ns();
|
|
gpu_wait(gpu);
|
|
uint64_t t1 = kern_now_ns();
|
|
wait_until(j->at);
|
|
uint64_t now = kern_now_ns();
|
|
double dt = fs_last_gen ? (double)(now - fs_last_gen) : 0;
|
|
if (dt > 0 && dt < 1e8) fs_iv = fs_iv > 0 ? fs_iv + (dt - fs_iv) * 0.1 : dt;
|
|
fs_last_gen = now;
|
|
fs_present(now, FS_REAL_GEN);
|
|
vk_present_flip();
|
|
gpu_wait(f);
|
|
uint64_t at = now + (uint64_t)(fs_iv / 2);
|
|
if (fs_iv > 0) wait_until(at);
|
|
uint64_t t2 = kern_now_ns();
|
|
fs_present(t2, FS_GEN_REAL);
|
|
vk_present_flip();
|
|
fs_win.gpu_wait += (t1 - t0) / 1e6, fs_run.gpu_wait += (t1 - t0) / 1e6;
|
|
if (fs_iv > 0 && t2 > at) fs_win.late += (t2 - at) / 1e6, fs_run.late += (t2 - at) / 1e6;
|
|
}
|
|
|
|
static void present_main(void *arg) {
|
|
(void)arg;
|
|
perf_core_thread(PCORE_PRESENT);
|
|
uint64_t last = 0, last_queued = 0;
|
|
plat_mutex_lock(&pl);
|
|
for (;;) {
|
|
while (!pending) plat_cond_wait(&pcv, &pl);
|
|
PresentJob j = job;
|
|
pending = 0;
|
|
busy = 1;
|
|
plat_cond_broadcast(&pcv);
|
|
plat_mutex_unlock(&pl);
|
|
if (flight_on) vk_thumbs_frame(j.frame);
|
|
vk_reflex_present_frame(j.pid); // the present of this blit, and the submission after it, are the frame's
|
|
vk_reflex_submit_frame(j.pid);
|
|
// the frame's last submission, its interpolation included: no submission goes out while busy
|
|
uint64_t drawn = j.gen ? vk_sync_issued() : 0;
|
|
int pair = j.gen && vk_present_blit_pair(j.gen, j.image, j.format, j.w, j.h, j.has_crop ? j.crop : NULL);
|
|
if (!j.gen) vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.format, j.w, j.h, j.has_crop ? j.crop : NULL);
|
|
uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too
|
|
if (j.value && f != j.value)
|
|
rt_fatal("vk: the present of frame %llu signalled %llu, queued as %llu", (unsigned long long)j.frame,
|
|
(unsigned long long)f, (unsigned long long)j.value);
|
|
plat_mutex_lock(&pl);
|
|
busy = 0;
|
|
plat_cond_broadcast(&pcv);
|
|
plat_mutex_unlock(&pl);
|
|
uint64_t t0 = kern_now_ns();
|
|
if (pair) {
|
|
present_pair(&j, drawn, f);
|
|
} else {
|
|
wait_until(j.at);
|
|
fs_last_gen = 0; // (the next generated picture starts the spacing anew)
|
|
fs_present(kern_now_ns(), FS_ALL);
|
|
}
|
|
uint64_t t1 = kern_now_ns();
|
|
if (!pair) vk_present_flip();
|
|
// TOTK_TRACE=present: one line per present: the host present's interval, the interval of the game's present
|
|
// calls (when the host present went out before back-edge pacing), the wait and how far past its target it went
|
|
if (rt_trace & RT_TRACE_PRESENT)
|
|
rt_log("present: frame %llu interval %.3f ms, game %.3f ms, waited %.3f ms, %.3f ms past target",
|
|
(unsigned long long)j.frame, last ? (t1 - last) / 1e6 : 0.0,
|
|
last_queued ? (j.queued - last_queued) / 1e6 : 0.0, (t1 - t0) / 1e6,
|
|
j.at && t1 > j.at ? (t1 - j.at) / 1e6 : 0.0);
|
|
last = t1;
|
|
last_queued = j.queued;
|
|
vk_pipeline_autosave(); // starts the autosave thread once
|
|
plat_mutex_lock(&pl);
|
|
}
|
|
return;
|
|
}
|
|
|
|
void vk_present_drain(void) {
|
|
uint64_t p0 = perf_begin();
|
|
plat_mutex_lock(&pl);
|
|
while (pending || busy) plat_cond_wait(&pcv, &pl);
|
|
plat_mutex_unlock(&pl);
|
|
perf_end(PERF_DRAIN, p0);
|
|
}
|
|
|
|
// The present after the frame count: resolves the image and hands the blit to the present thread. queued: when the game
|
|
// presented; value: the timeline value queued for this present (asynchronous replay), 0 for the next one (synchronous).
|
|
static void present_image(const tex_t *t, const int crop[4], uint64_t at_ns, uint64_t queued, uint64_t frame,
|
|
uint64_t value, uint64_t pid) {
|
|
static volatile int32_t n_skipped;
|
|
vk_exec_frame_log(t, frame);
|
|
vk_dlaa_frame();
|
|
vk_mvsurvey_frame();
|
|
vk_mvprev_frame();
|
|
vk_mem_census(frame);
|
|
if (vk_dump_armed) vk_dump_present(frame);
|
|
__atomic_store_n(&gfx_counters.validation_errors, (uint64_t)vk_validation_errors_total(), __ATOMIC_RELAXED);
|
|
const VkTexImage *img = vk_texture_get(t);
|
|
if (!img || !(img->aspect & VK_IMAGE_ASPECT_COLOR_BIT)) {
|
|
if (plat_inc32(&n_skipped) <= 8)
|
|
rt_log("vk: present of texture %p (format %#x) skipped: no colour image", (const void *)t, t ? t->d.format : 0);
|
|
if (value) { // the queued value is still signalled, in order
|
|
vk_present_drain();
|
|
uint64_t f = vk_sync_submit(NULL, 0);
|
|
if (f != value) rt_fatal("vk: skipped present signalled %llu, queued as %llu", (unsigned long long)f, (unsigned long long)value);
|
|
}
|
|
return;
|
|
}
|
|
PresentJob j = {.image = img->image, .format = img->format, .w = (int)img->width, .h = (int)img->height, .frame = frame,
|
|
.at = at_ns, .queued = queued, .value = value, .pid = pid};
|
|
vk_hdr_present_source(img, &j.image, &j.format); // TOTK_HDR: the float picture the game copied into it, same size
|
|
vk_fg_present_swap(&j.image, &j.format); // TOTK_FG_TEST (frame generation checks): another picture in its place
|
|
j.gen = vk_fg_present_gen(j.image);
|
|
if (__atomic_load_n(&fg_live, __ATOMIC_RELAXED) != (j.gen != VK_NULL_HANDLE))
|
|
__atomic_store_n(&fg_live, j.gen != VK_NULL_HANDLE, __ATOMIC_RELAXED);
|
|
if (crop) {
|
|
j.has_crop = 1;
|
|
for (int i = 0; i < 4; i++) j.crop[i] = crop[i];
|
|
}
|
|
vk_upload_flush(0); // an upload vk_texture_get just recorded goes before the blit
|
|
plat_mutex_lock(&pl);
|
|
if (!present_thread) {
|
|
if (!plat_thread_spawn(present_main, NULL, "vk present", 0)) rt_fatal("vk: cannot start the present thread");
|
|
present_thread = 1;
|
|
}
|
|
while (pending) plat_cond_wait(&pcv, &pl); // one present in waiting at most
|
|
job = j;
|
|
pending = 1;
|
|
plat_cond_broadcast(&pcv);
|
|
plat_mutex_unlock(&pl);
|
|
if (rdoc_active()) { vk_present_drain(); rdoc_after_present(frame); }
|
|
}
|
|
|
|
// ---- Asynchronous replay
|
|
enum { RI_SUBMIT, RI_PRESENT, RI_RELEASE };
|
|
typedef struct {
|
|
int kind, n, has_crop, crop[4];
|
|
uint64_t value, submit_no, at, queued, pid; // pid: the game's frame when it queued the item (present ID)
|
|
uint64_t *h;
|
|
const tex_t *tex; // RI_RELEASE: a copy the item owns (the game may reuse the object's memory)
|
|
} ReplayItem;
|
|
#define RQ_CAP 1024 // a frame queues 1 submit and 1 present, and up to a few hundred releases
|
|
enum { DR_POOL_FINI, DR_MAP_VIRTUAL, DR_TEX_RELEASE, DR_PROG_RELEASE, DR_FINISH, DR_READ_TEXELS, DR_WRITE_TEXELS,
|
|
DR_GUEST_READ, DR_FRAME, DR_FULL, DR_CHECK, DR_N };
|
|
static const char *const dr_name[DR_N] = {
|
|
"pool_fini", "map_virtual", "texture_release (queued)", "program_release", "finish", "read_texels (writer fence)",
|
|
"write_texels", "guest_read (writer fence)", "frame bound", "queue full", "check drain"};
|
|
|
|
static PlatMutex rl = PLAT_MUTEX_INIT;
|
|
static PlatCond rcv = PLAT_COND_INIT; // item queued, item taken, item done
|
|
static ReplayItem rq[RQ_CAP];
|
|
static int rq_head, rq_n, r_busy, r_started;
|
|
static uint64_t r_presents_queued, r_presents_done, r_submit_done, game_frames;
|
|
static uint64_t r_value_done; // under rl: the newest timeline value the replay thread submitted
|
|
// Under rl: calls, calls that waited and the time waited per cause, since the last replay_log line and over the run.
|
|
static uint64_t dr_calls[DR_N], dr_waits[DR_N], dr_ns[DR_N], dr_tcalls[DR_N], dr_twaits[DR_N], dr_tns[DR_N];
|
|
// TOTK_REPLAY_CHECK: ReadTexels compared with a read after a full drain (under rl): checked, differing, differing
|
|
// although no submission recorded a write to the texture in between, differing with the queue idle at the first read.
|
|
static uint64_t ck_reads, ck_diff, ck_diff_unexplained, ck_diff_idle;
|
|
|
|
int vk_replay_async(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
// Off by default. Where the game waits for the GPU it overlaps the replay with the game threads (a GTX 1650 in
|
|
// the Depths went from 450-686 frames over 50 ms a minute to 10-13, issue #88), the only place a gain was
|
|
// measured. While it was the default, a GTX 1660 Ti at the 60 fps cap with the GPU at 26 % reported uneven
|
|
// motion that 0.3.4 did not have (#659). It adds up to a frame of latency.
|
|
on = cfg_bool("TOTK_REPLAY_ASYNC");
|
|
rt_log("vk: replay %s (TOTK_REPLAY_ASYNC=%d)",
|
|
on ? "on its own thread, at most one frame behind the game" : "on the submitting thread", on);
|
|
}
|
|
return on;
|
|
}
|
|
|
|
// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end.
|
|
int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); }
|
|
|
|
// The newest submit number (nvn_submit_count) whose replay is done: recording copies it used may be freed.
|
|
uint64_t gfx_replayed_submits(void) {
|
|
if (!gfx_replay_async()) return nvn_submit_count();
|
|
plat_mutex_lock_shared(&rl);
|
|
uint64_t v = r_submit_done;
|
|
plat_mutex_unlock_shared(&rl);
|
|
return v;
|
|
}
|
|
|
|
// TOTK_SYNC_AHEAD (nvn_api_core.c nvnSyncWait): waits until the replay thread submitted value v, so every shadow copy
|
|
// of v was made before the game writes memory v reads. The caller has given its core up.
|
|
void gfx_replay_wait_value(uint64_t v) {
|
|
if (!gfx_replay_async()) return;
|
|
plat_mutex_lock(&rl);
|
|
while (r_value_done < v) plat_cond_wait(&rcv, &rl);
|
|
plat_mutex_unlock(&rl);
|
|
}
|
|
|
|
static uint64_t submit_sync(const uint64_t *handles, int n) {
|
|
vk_sync_start();
|
|
vk_present_drain();
|
|
vk_texture_new_submit();
|
|
return vk_sync_submit(handles, n);
|
|
}
|
|
|
|
static void replay_main(void *arg) {
|
|
(void)arg;
|
|
perf_replay_thread();
|
|
perf_core_thread(PCORE_REPLAY);
|
|
plat_mutex_lock(&rl);
|
|
for (;;) {
|
|
while (!rq_n) plat_cond_wait(&rcv, &rl);
|
|
ReplayItem it = rq[rq_head];
|
|
rq_head = (rq_head + 1) % RQ_CAP;
|
|
rq_n--;
|
|
r_busy = 1;
|
|
plat_cond_broadcast(&rcv);
|
|
plat_mutex_unlock(&rl);
|
|
if (it.kind == RI_SUBMIT) {
|
|
vk_reflex_submit_frame(it.pid); // the replay thread submits the frame the game was on when it queued it
|
|
uint64_t f = submit_sync(it.h, it.n);
|
|
if (f != it.value)
|
|
rt_fatal("vk: replayed submission signalled %llu, queued as %llu", (unsigned long long)f,
|
|
(unsigned long long)it.value);
|
|
free(it.h);
|
|
} else if (it.kind == RI_RELEASE) {
|
|
vk_present_drain();
|
|
vk_texture_release(it.tex);
|
|
free((void *)it.tex);
|
|
} else {
|
|
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
|
|
present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value, it.pid);
|
|
}
|
|
plat_mutex_lock(&rl);
|
|
if (it.kind != RI_RELEASE) r_value_done = it.value;
|
|
if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no;
|
|
else r_presents_done++;
|
|
r_busy = 0;
|
|
plat_cond_broadcast(&rcv);
|
|
}
|
|
return;
|
|
}
|
|
|
|
static void wait_counted(int why, uint64_t t0) { // rl held
|
|
uint64_t d = kern_now_ns() - t0;
|
|
dr_waits[why]++, dr_twaits[why]++;
|
|
dr_ns[why] += d, dr_tns[why] += d;
|
|
perf_end(PERF_DRAIN, t0);
|
|
}
|
|
|
|
// A guest thread waiting for the replay gives its virtual core up, as around any blocking host wait (NVN_BLOCKING):
|
|
// guest_read (memcpy, pool invalidate), pool and texture finalize run without it, and a drain lasts up to a replay.
|
|
// Called and returning with rl held; rl is not held while the core is taken back.
|
|
static int core_away_unlocked(void) {
|
|
plat_mutex_unlock(&rl);
|
|
int away = kern_core_away();
|
|
plat_mutex_lock(&rl);
|
|
return away;
|
|
}
|
|
static void core_back_unlocked(int *away) {
|
|
if (!*away) return;
|
|
plat_mutex_unlock(&rl);
|
|
kern_core_back(away);
|
|
plat_mutex_lock(&rl);
|
|
}
|
|
|
|
// Queues an item (waits while the queue is full) and returns its value, reserved under rl so values follow queue order.
|
|
static uint64_t replay_enqueue(ReplayItem *it) {
|
|
plat_mutex_lock(&rl);
|
|
if (!r_started) {
|
|
if (!plat_thread_spawn(replay_main, NULL, "vk replay", 0)) rt_fatal("vk: cannot start the replay thread");
|
|
r_started = 1;
|
|
}
|
|
// rl is dropped while the core is taken back, and another thread may fill the queue again meanwhile: check again.
|
|
while (rq_n == RQ_CAP) {
|
|
uint64_t p0 = perf_begin();
|
|
dr_calls[DR_FULL]++, dr_tcalls[DR_FULL]++;
|
|
int away = core_away_unlocked();
|
|
while (rq_n == RQ_CAP) plat_cond_wait(&rcv, &rl);
|
|
wait_counted(DR_FULL, p0);
|
|
core_back_unlocked(&away);
|
|
}
|
|
it->value = it->kind == RI_RELEASE ? 0 : vk_sync_reserve(); // a release signals nothing
|
|
rq[(rq_head + rq_n) % RQ_CAP] = *it;
|
|
rq_n++;
|
|
plat_cond_broadcast(&rcv);
|
|
plat_mutex_unlock(&rl);
|
|
return it->value;
|
|
}
|
|
|
|
// Returns once every queued item was replayed and submitted. Back-end entries only: the replay thread never calls them.
|
|
static void replay_drain(int why) {
|
|
if (!gfx_replay_async()) return;
|
|
plat_mutex_lock(&rl);
|
|
dr_calls[why]++, dr_tcalls[why]++;
|
|
if (rq_n || r_busy) {
|
|
uint64_t p0 = perf_begin();
|
|
int away = core_away_unlocked();
|
|
while (rq_n || r_busy) plat_cond_wait(&rcv, &rl);
|
|
wait_counted(why, p0);
|
|
core_back_unlocked(&away);
|
|
}
|
|
plat_mutex_unlock(&rl);
|
|
}
|
|
|
|
// What made the game wait for the replay: since the last line (every 1800 game frames), or over the run (at exit).
|
|
static void replay_log(uint64_t frame, int total) {
|
|
char buf[1024];
|
|
int k = 0;
|
|
plat_mutex_lock(&rl);
|
|
const uint64_t *c = total ? dr_tcalls : dr_calls, *w = total ? dr_twaits : dr_waits, *t = total ? dr_tns : dr_ns;
|
|
for (int i = 0; i < DR_N && k < (int)sizeof buf - 96; i++)
|
|
if (c[i] || w[i])
|
|
k += snprintf(buf + k, sizeof buf - (size_t)k, "%s %s %llu calls, %llu waited %.1f ms", k ? ";" : "", dr_name[i],
|
|
(unsigned long long)c[i], (unsigned long long)w[i], t[i] / 1e6);
|
|
if (ck_reads && k < (int)sizeof buf - 128)
|
|
k += snprintf(buf + k, sizeof buf - (size_t)k, "; check (run): %llu reads, %llu differ, %llu without a recorded write, %llu with the queue idle",
|
|
(unsigned long long)ck_reads, (unsigned long long)ck_diff, (unsigned long long)ck_diff_unexplained,
|
|
(unsigned long long)ck_diff_idle);
|
|
memset(dr_calls, 0, sizeof dr_calls);
|
|
memset(dr_waits, 0, sizeof dr_waits);
|
|
memset(dr_ns, 0, sizeof dr_ns);
|
|
plat_mutex_unlock(&rl);
|
|
rt_log("vk: replay queue %s frame %llu:%s", total ? "over the run to" : "at", (unsigned long long)frame,
|
|
k ? buf : " no waits");
|
|
}
|
|
|
|
// The run total, once: the game exits through prof_write (boot.c _exit, the window close), probes through
|
|
// vk_backend_shutdown.
|
|
void gfx_exit_log(void) {
|
|
static int32_t done;
|
|
rec_check_log();
|
|
vk_fg_exit_log();
|
|
static int32_t fs_done;
|
|
if (!plat_xchg32(&fs_done, 1)) fs_log(&fs_run, "over the run");
|
|
vk_texture_eager_log();
|
|
vk_mem_shadow_read_log();
|
|
if (gfx_replay_async() && r_started && !plat_xchg32(&done, 1)) replay_log(game_frames, 1);
|
|
}
|
|
|
|
static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) {
|
|
static _Thread_local int pcore; // the game's render thread: the one that presents
|
|
if (!pcore) pcore = 1, perf_core_thread(PCORE_RENDER);
|
|
vk_dlaa_game_present(); // the camera jitter moves on with the game's frames (vk_dlaa.c)
|
|
vk_sync_start();
|
|
if (!gfx_replay_async()) {
|
|
uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED);
|
|
perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads
|
|
present_image(t, crop, at_ns, kern_now_ns(), frame, 0, vk_reflex_frame());
|
|
return;
|
|
}
|
|
// Frame bound: the replay thread handled the previous present before this one is queued.
|
|
plat_mutex_lock(&rl);
|
|
dr_calls[DR_FRAME]++, dr_tcalls[DR_FRAME]++;
|
|
if (r_presents_done < r_presents_queued) {
|
|
uint64_t p0 = perf_begin();
|
|
while (r_presents_done < r_presents_queued) plat_cond_wait(&rcv, &rl);
|
|
wait_counted(DR_FRAME, p0);
|
|
}
|
|
r_presents_queued++;
|
|
plat_mutex_unlock(&rl);
|
|
uint64_t frame = ++game_frames;
|
|
perf_present(frame);
|
|
if (frame % 1800 == 0) replay_log(frame, 0);
|
|
ReplayItem it = {.kind = RI_PRESENT, .tex = t, .at = at_ns, .queued = kern_now_ns(), .pid = vk_reflex_frame()};
|
|
if (crop) {
|
|
it.has_crop = 1;
|
|
for (int i = 0; i < 4; i++) it.crop[i] = crop[i];
|
|
}
|
|
replay_enqueue(&it);
|
|
}
|
|
|
|
// vk_pipeline.c drops a released program's pipelines here.
|
|
WEAK void vk_program_release(const prog_t *p) { (void)p; }
|
|
|
|
static int vk_be_pool_init(pool_t *p) { vk_sync_start(); return vk_pool_init(p); }
|
|
static void vk_be_pool_fini(pool_t *p) { vk_sync_start(); replay_drain(DR_POOL_FINI); vk_present_drain(); vk_pool_fini(p); }
|
|
static int vk_be_pool_map_virtual(pool_t *v, int n, const void *requests) {
|
|
vk_sync_start();
|
|
replay_drain(DR_MAP_VIRTUAL);
|
|
vk_present_drain();
|
|
return vk_pool_map_virtual(v, n, requests);
|
|
}
|
|
static void vk_be_texture_register(int id, const tex_t *t, const view_t *v) { vk_sync_start(); vk_texture_register(id, t, v); }
|
|
static void vk_be_sampler_register(int id, const samp_par *s) { vk_sync_start(); vk_sampler_register(id, s); }
|
|
// Asynchronous: the release is queued behind the submissions before it, so their replay still resolves the texture's
|
|
// slots to its image (a released slot follows another live texture at the storage), and the ones after it see it
|
|
// released, the order of the synchronous replay. vk_texture_release reads only the object's key fields: a copy.
|
|
static void vk_be_texture_release(const tex_t *t) {
|
|
vk_sync_start();
|
|
if (!gfx_replay_async()) {
|
|
vk_present_drain();
|
|
vk_texture_release(t);
|
|
return;
|
|
}
|
|
if (!t || !t->mem) return;
|
|
tex_t *copy = malloc(sizeof *copy);
|
|
if (!copy) rt_fatal("vk: out of memory (replay queue)");
|
|
*copy = *t;
|
|
ReplayItem it = {.kind = RI_RELEASE, .tex = copy};
|
|
plat_mutex_lock(&rl);
|
|
dr_calls[DR_TEX_RELEASE]++, dr_tcalls[DR_TEX_RELEASE]++;
|
|
plat_mutex_unlock(&rl);
|
|
replay_enqueue(&it);
|
|
}
|
|
static void vk_be_program_release(const prog_t *p) { replay_drain(DR_PROG_RELEASE); vk_program_release(p); }
|
|
// Asynchronous: the handles (nvn_submit_prepare_async ran under the front end's submit lock) are copied and queued;
|
|
// the value returned is the one the replay will signal.
|
|
static uint64_t vk_be_submit(const uint64_t *handles, int n) {
|
|
if (!gfx_replay_async()) return submit_sync(handles, n);
|
|
vk_sync_start();
|
|
ReplayItem it = {.kind = RI_SUBMIT, .n = n, .submit_no = nvn_submit_count(), .pid = vk_reflex_frame()};
|
|
if (n > 0) {
|
|
it.h = malloc(sizeof *it.h * (size_t)n);
|
|
if (!it.h) rt_fatal("vk: out of memory (replay queue)");
|
|
memcpy(it.h, handles, sizeof *it.h * (size_t)n);
|
|
}
|
|
return replay_enqueue(&it);
|
|
}
|
|
static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); }
|
|
static int replay_check(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
const char *e = cfg_get("TOTK_REPLAY_CHECK");
|
|
on = e && strcmp(e, "0");
|
|
if (on) rt_log("vk: TOTK_REPLAY_CHECK: every ReadTexels is read again after a full replay drain and compared");
|
|
}
|
|
return on;
|
|
}
|
|
|
|
// ReadTexels with the replay queued: waits for the newest submission recorded to write the texture (replayed and
|
|
// completed), not for the whole queue. The readback copy goes through the upload batch on the same queue, after every
|
|
// submission the replay has submitted. Submissions still queued or being recorded run after the copy, as on hardware
|
|
// when the GPU has not reached them yet: the game only sees their texels after waiting on a sync, and every sync
|
|
// value covers replay and GPU, so a read the game synchronised sees them. TOTK_REPLAY_CHECK compares with a full drain.
|
|
static void vk_be_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row,
|
|
uint64_t img) {
|
|
vk_sync_start();
|
|
if (!gfx_replay_async()) {
|
|
vk_read_texels(t, v, r, dst, row, img);
|
|
return;
|
|
}
|
|
uint64_t f = vk_texture_last_write(t);
|
|
plat_mutex_lock(&rl);
|
|
dr_calls[DR_READ_TEXELS]++, dr_tcalls[DR_READ_TEXELS]++;
|
|
int idle = !rq_n && !r_busy;
|
|
plat_mutex_unlock(&rl);
|
|
if (f && vk_sync_fence_done() < f) {
|
|
uint64_t t0 = kern_now_ns();
|
|
vk_sync_fence_wait(f, UINT64_MAX); // counts as a fence wait in perf
|
|
uint64_t d = kern_now_ns() - t0;
|
|
plat_mutex_lock(&rl);
|
|
dr_waits[DR_READ_TEXELS]++, dr_twaits[DR_READ_TEXELS]++;
|
|
dr_ns[DR_READ_TEXELS] += d, dr_tns[DR_READ_TEXELS] += d;
|
|
plat_mutex_unlock(&rl);
|
|
}
|
|
vk_read_texels(t, v, r, dst, row, img);
|
|
if (!replay_check()) return;
|
|
size_t n = vk_texture_texel_span(t, v, r, row, img);
|
|
uint8_t *again = n ? malloc(n) : NULL;
|
|
if (!again) return;
|
|
memcpy(again, dst, n); // bytes between the rows stay the caller's
|
|
uint64_t f0 = vk_texture_last_write(t);
|
|
replay_drain(DR_CHECK);
|
|
uint64_t f1 = vk_texture_last_write(t);
|
|
vk_read_texels(t, v, r, again, row, img);
|
|
int differ = memcmp(again, dst, n) != 0;
|
|
static volatile int32_t n_logged;
|
|
if (differ && plat_inc32(&n_logged) <= 40) {
|
|
size_t at = 0;
|
|
while (at < n && again[at] == dst[at]) at++;
|
|
rt_log("vk: replay check: ReadTexels of %p (format %#x, %dx%dx%d at %d,%d,%d) differs at byte %zu of %zu; writer "
|
|
"fence %llu at the read, %llu after the drain, done %llu; queue %s at the read",
|
|
(void *)t->mem, t->d.format, r->width, r->height, r->depth, r->xoffset, r->yoffset, r->zoffset, at, n,
|
|
(unsigned long long)f0, (unsigned long long)f1, (unsigned long long)vk_sync_fence_done(), idle ? "idle" : "busy");
|
|
}
|
|
plat_mutex_lock(&rl);
|
|
ck_reads++;
|
|
if (differ) {
|
|
ck_diff++;
|
|
if (f1 == f0) ck_diff_unexplained++;
|
|
if (idle) ck_diff_idle++;
|
|
}
|
|
plat_mutex_unlock(&rl);
|
|
free(again);
|
|
}
|
|
static void vk_be_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row,
|
|
uint64_t img) {
|
|
vk_sync_start();
|
|
replay_drain(DR_WRITE_TEXELS);
|
|
vk_present_drain();
|
|
vk_write_texels(t, v, r, src, row, img);
|
|
}
|
|
static void vk_be_guest_written(uint64_t addr, uint64_t size) { vk_sync_start(); vk_guest_written(addr, size); }
|
|
// A CPU read of GPU-written mirrored storage (memcpy, pool invalidate). With the replay queued it waits for the newest
|
|
// submission recorded to write the textures there, not for the whole queue: the write-back copy then runs after their
|
|
// renders, and clears gpu_dirty only for content that is on the queue. Submissions not replayed yet render after the
|
|
// copy, as on hardware when the GPU has not reached them; they mark the texture dirty again for a later read.
|
|
static void vk_be_guest_read(uint64_t addr, uint64_t size) {
|
|
if (!vk_texture_cpu_stale(addr, size)) return;
|
|
vk_sync_start();
|
|
if (gfx_replay_async()) {
|
|
uint64_t f = vk_texture_dirty_fence(addr, size);
|
|
plat_mutex_lock(&rl);
|
|
dr_calls[DR_GUEST_READ]++, dr_tcalls[DR_GUEST_READ]++;
|
|
plat_mutex_unlock(&rl);
|
|
if (f && vk_sync_fence_done() < f) {
|
|
uint64_t t0 = kern_now_ns();
|
|
NVN_BLOCKING();
|
|
vk_sync_fence_wait(f, UINT64_MAX);
|
|
uint64_t d = kern_now_ns() - t0;
|
|
plat_mutex_lock(&rl);
|
|
dr_waits[DR_GUEST_READ]++, dr_twaits[DR_GUEST_READ]++;
|
|
dr_ns[DR_GUEST_READ] += d, dr_tns[DR_GUEST_READ] += d;
|
|
plat_mutex_unlock(&rl);
|
|
}
|
|
}
|
|
vk_present_drain();
|
|
vk_texture_write_back(addr, size);
|
|
}
|
|
// After a fence wait: the value is done, so its replay is too; a later submission the replay records meanwhile marks
|
|
// its targets with a later fence, which write_back_small leaves alone. No drain.
|
|
static void vk_be_cpu_synced(uint64_t fence) {
|
|
if (!vk_texture_small_dirty()) return;
|
|
vk_sync_start();
|
|
vk_present_drain();
|
|
vk_texture_write_back_small(fence);
|
|
}
|
|
|
|
const GfxBackend gfx_vulkan_backend = {
|
|
.name = "vulkan",
|
|
.pool_init = vk_be_pool_init,
|
|
.pool_fini = vk_be_pool_fini,
|
|
.pool_map_virtual = vk_be_pool_map_virtual,
|
|
.texture_register = vk_be_texture_register,
|
|
.sampler_register = vk_be_sampler_register,
|
|
.texture_release = vk_be_texture_release,
|
|
.program_release = vk_be_program_release,
|
|
.submit = vk_be_submit,
|
|
.fence_wait = vk_sync_fence_wait,
|
|
.fence_done = vk_sync_fence_done,
|
|
.present = vk_be_present,
|
|
.finish = vk_be_finish,
|
|
.read_texels = vk_be_read_texels,
|
|
.write_texels = vk_be_write_texels,
|
|
.guest_written = vk_be_guest_written,
|
|
.guest_read = vk_be_guest_read,
|
|
.cpu_synced = vk_be_cpu_synced,
|
|
};
|
|
|
|
void vk_backend_shutdown(void) {
|
|
replay_drain(DR_FINISH);
|
|
gfx_exit_log();
|
|
vk_present_drain();
|
|
vk_sync_shutdown();
|
|
vk_mem_shutdown();
|
|
vk_shutdown();
|
|
}
|