Mirrors/totk-pc-recomp
1
0 Fork 0
mirror of https://github.com/valentderah/totk-pc-recomp.git synced 2026-11-11 09:51:02 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
dev
totk-pc-recomp/runtime/vk_sync.c
T
itaskaev2 2e5d59df83 Merge remote-tracking branch 'origin/dev' into carry/sync-ahead 
# Conflicts:
#	docs/architecture.md
#	runtime/cfg.c
#	runtime/vk_sync.c
2026-11-09 09:54:45 +01:00

620 lines
29 KiB
C
Raw Permalink Blame History

// Submission, fences and the completion thread of the Vulkan back end.
// One timeline semaphore (VkCtx.timeline) orders everything: each submission records its command handles into a
// primary command buffer (vk_exec_record) and signals the next timeline value, which is the NVN fence value
// the front end hands to syncs. The completion thread waits on the timeline and, in submit order, runs the CPU side
// effects queued for each value (event writes, counter reports, sync state), then frees the memory and texture objects whose users
// completed. A value counts as done only after its side effects ran, so a guest woken by fence_wait sees them.
//
// The submission shape (one timeline value per submission, signalled at ALL_COMMANDS, vkWaitSemaphores for waits, a
// failed wait treated as a lost device) follows LAS Recomp runtime/nvn/vulkan/exec.c:39-110, 355-397
// (https://github.com/peiikai/LASRecomp @ da35b83a9611d046a9a681aae668d7085954d37f, MIT, Copyright (c) 2026 Peiikai).
// Side effects written after GPU completion, as MacNativeGames does: games/totk/third_party/d2r_gfx/gfx/nvn/hle_nvn.c:7-9
// (https://github.com/ly05010419/MacNativeGames @ b75be8cfa704df3701e86130c8ced268e79d6b11, MIT, Copyright (c) 2026
// jhaizhou; origin d2r-mobile).
#include <stdlib.h>
#include <string.h>
#include "gfx.h"
#include "host.h"
#include "nvn_api.h"
#include "vk.h"
#include "platform/platform.h"
#define WEAK __attribute__((weak))
typedef struct { uint64_t fence; void (*fn)(void *); void *arg; } Pending;
typedef struct { VkCommandBuffer cb; uint64_t fence; } CmdSlot; // reusable once the timeline reached fence
static PlatMutex start_lock = PLAT_MUTEX_INIT;
static int started;
static PlatThread *completion_thread;
// Submissions: one at a time, so timeline values are signalled in the order they are handed out.
static PlatMutex sub_lock = PLAT_MUTEX_INIT;
static VkCommandPool cmd_pool;
static CmdSlot *slots;
static int nslots;
static int cur_slot; // the slot vk_sync_submit records into now (an index: take_slot may move slots)
static uint64_t cur_fence; // the value of the submission being recorded
// Shared with the completion thread (st_lock).
static PlatMutex st_lock = PLAT_MUTEX_INIT;
static PlatCond cv_work = PLAT_COND_INIT; // a submission, a callback or stop
static PlatCond cv_done = PLAT_COND_INIT; // done advanced
static uint64_t issued, done; // newest submitted value; newest value whose side effects ran
static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued)
static int stop;
static Pending *queue; // side effects in the order they were added
static int qhead, qn, qcap;
// ---- GPU time (TOTK_TRACE=perf): each submission part (vk_sync_split) writes a timestamp when it starts and when it
// ends, into a ring of query pairs. The completion thread reads a pair once the part's value is reached and adds the
// part's interval to the busy time, counting overlap with the previous part once. Off (one log line) when the queue
// family has no timestamp bits or the period is 0. Upload batches (vk_memory.c) are not timed.
#define TS_PAIRS 1024
static VkQueryPool ts_pool;
static int ts_state; // 0 not set up, 1 on, -1 off
static double ts_period; // ns per tick
static uint64_t ts_mask; // valid bits of a timestamp
static uint32_t ts_next; // next pair (atomic: parts recorded on other threads take pairs too)
static volatile uint8_t ts_busy[TS_PAIRS]; // written until the completion thread read the pair
static int ts_cur = -1; // the pair of the part being recorded, -1 untimed
static uint64_t ts_last_end; // completion thread: end of the latest part read (ticks)
static volatile int64_t ts_busy_ns, ts_parts, ts_missed;
// GPU idle (with calibrated timestamps): the gap before each timed part, split by whether the part was submitted after
// the previous one had ended (the GPU waited for the CPU: starved) or before (it waited for untimed work or a wait).
static uint64_t ts_sub_ns[TS_PAIRS]; // CPU time the part was handed to vkQueueSubmit2
static int ts_unsent[4], ts_n_unsent; // pairs closed and not yet submitted (sub_lock)
static uint64_t ts_last_end_ns; // completion thread: ts_last_end on the CPU clock
static volatile int64_t ts_idle_ns, ts_starved_ns, ts_gaps, ts_max_gap_ns, ts_queued_ns;
static int64_t cal_tick, cal_ns, cal_at; // completion thread: a calibrated (GPU tick, CPU ns) pair and when it was taken
static PFN_vkGetCalibratedTimestampsKHR cal_fn;
// On the first submission that wants timing (perf_gpu_timing: TOTK_TRACE=perf or the full performance overlay).
static void ts_setup(VkCtx *c) {
ts_state = -1;
uint32_t nq = 0;
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL);
VkQueueFamilyProperties *qp = calloc(nq ? nq : 1, sizeof *qp);
if (!qp) return;
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp);
uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0;
free(qp);
float period = c->props.limits.timestampPeriod;
if (!bits || period <= 0) {
rt_log("vk: no GPU timestamps on this queue (valid bits %u, period %.3f ns): the perf line shows no GPU time", bits, period);
return;
}
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP,
.queryCount = 2 * TS_PAIRS};
VkResult r = vkCreateQueryPool(c->device, &qi, NULL, &ts_pool);
if (r != VK_SUCCESS) {
rt_log("vk: timestamp query pool failed (%d): the perf line shows no GPU time", (int)r);
return;
}
ts_period = period;
ts_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1;
ts_state = 1;
}
typedef struct { uint32_t pair; } TsRead;
// Completion thread: refresh the GPU tick / CPU clock pair every 250 ms; 0 when there is no calibration. The host
// domain is the clock plat_now_ns reads (QPC on Windows, CLOCK_MONOTONIC elsewhere), plat_counter_hz ticks a second.
static int cal_refresh(VkCtx *c) {
if (!c->has_calibrated) return 0;
if (!cal_fn) {
cal_fn = (PFN_vkGetCalibratedTimestampsKHR)vkGetDeviceProcAddr(c->device, c->has_calibrated == 2 ? "vkGetCalibratedTimestampsKHR"
: "vkGetCalibratedTimestampsEXT");
if (!cal_fn) { c->has_calibrated = 0; return 0; }
}
int64_t now = (int64_t)plat_now_ns();
if (cal_at && now - cal_at < 250000000) return 1;
VkCalibratedTimestampInfoKHR ci[2] = {{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_DEVICE_KHR},
#ifdef _WIN32
{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_QUERY_PERFORMANCE_COUNTER_KHR}};
#else
{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, .timeDomain = VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR}};
#endif
uint64_t v[2], dev;
if (cal_fn(c->device, 2, ci, v, &dev) != VK_SUCCESS) {
if (!cal_at) { // the domain pair is not supported: no split
rt_log("vk: calibrated timestamps failed: the perf line shows no GPU idle split");
c->has_calibrated = 0;
}
return cal_at != 0;
}
cal_tick = (int64_t)(v[0] & ts_mask), cal_ns = (int64_t)((double)v[1] * 1e9 / (double)plat_counter_hz()), cal_at = now;
return 1;
}
static int64_t tick_ns(uint64_t t) { return cal_ns + (int64_t)((double)((int64_t)t - cal_tick) * ts_period); }
static void max64(volatile int64_t *m, int64_t v) {
int64_t o = __atomic_load_n(m, __ATOMIC_RELAXED);
while (v > o && !__atomic_compare_exchange_n(m, &o, v, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {}
}
// Just before vkQueueSubmit2 (sub_lock held): the parts closed since the last submit leave the CPU now.
static void ts_submitted(void) {
uint64_t now = plat_now_ns();
for (int i = 0; i < ts_n_unsent; i++) ts_sub_ns[ts_unsent[i]] = now;
ts_n_unsent = 0;
}
static void ts_read_cb(void *arg) { // completion thread, in submission order
TsRead *t = arg;
VkCtx *c = vk_ctx();
uint64_t v[2];
if (vkGetQueryPoolResults(c->device, ts_pool, 2 * t->pair, 2, sizeof v, v, sizeof v[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) {
uint64_t s = v[0] & ts_mask, e = v[1] & ts_mask;
if (e >= s) {
if (cal_refresh(c)) {
int64_t sn = tick_ns(s), en = tick_ns(e), sub = (int64_t)ts_sub_ns[t->pair];
if (ts_last_end_ns && sn > ts_last_end_ns) {
int64_t gap = sn - ts_last_end_ns, late = sub - ts_last_end_ns;
plat_add64(&ts_idle_ns, gap);
if (late > 0) plat_add64(&ts_starved_ns, late < gap ? late : gap);
if (gap > 500000) plat_inc64(&ts_gaps);
max64(&ts_max_gap_ns, gap);
}
if (sub && sn > sub) plat_add64(&ts_queued_ns, sn - sub);
if (en > ts_last_end_ns) ts_last_end_ns = en;
}
uint64_t from = s > ts_last_end ? s : ts_last_end;
if (e > from) {
int64_t ns = (int64_t)((double)(e - from) * ts_period);
plat_add64(&ts_busy_ns, ns);
perf_gpu_busy((uint64_t)ns);
}
if (e > ts_last_end) ts_last_end = e;
plat_inc64(&ts_parts);
} else plat_inc64(&ts_missed); // the counter wrapped
} else plat_inc64(&ts_missed);
__atomic_store_n(&ts_busy[t->pair], 0, __ATOMIC_RELEASE);
free(t);
}
static int ts_take(void) {
uint32_t p = __atomic_fetch_add(&ts_next, 1, __ATOMIC_RELAXED) % TS_PAIRS;
if (__atomic_exchange_n(&ts_busy[p], 1, __ATOMIC_ACQUIRE)) { plat_inc64(&ts_missed); return -1; }
return (int)p;
}
// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed.
static int ts_open(VkCommandBuffer cb) {
if (ts_state != 1 || !perf_gpu_timing()) {
if (ts_state || !perf_gpu_timing()) return -1;
ts_setup(vk_ctx()); // the overlay turned full: timestamps from now on
if (ts_state != 1) return -1;
}
int t = ts_take();
if (t < 0) return -1;
uint32_t p = (uint32_t)t;
vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2);
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p);
return (int)p;
}
// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it.
static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence) {
if (pair < 0) return;
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1);
ts_sub_ns[pair] = 0;
if (ts_n_unsent < 4) ts_unsent[ts_n_unsent++] = pair;
TsRead *t = malloc(sizeof *t);
if (!t) rt_fatal("vk: out of memory");
t->pair = (uint32_t)pair;
vk_completion_add(fence, ts_read_cb, t);
}
int vk_sync_ts_open(VkCommandBuffer cb) {
if (__atomic_load_n(&ts_state, __ATOMIC_ACQUIRE) != 1 || !perf_gpu_timing()) return -1;
int p = ts_take();
if (p < 0) return -1;
vkCmdResetQueryPool(cb, ts_pool, 2 * (uint32_t)p, 2);
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)p);
return p;
}
void vk_sync_ts_close(VkCommandBuffer cb, int pair, uint64_t fence) { ts_close(cb, pair, fence); }
// The pair of the part being recorded.
static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); }
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
ts_close(cb, ts_cur, fence);
ts_cur = -1;
}
// An empty submission records no command buffer: its pair is never written.
static void ts_drop(void) {
if (ts_cur < 0) return;
__atomic_store_n(&ts_busy[ts_cur], 0, __ATOMIC_RELEASE);
ts_cur = -1;
}
static void gpu_stats(PerfGpu *g) {
g->have_gpu = ts_state == 1;
g->gpu_ns = (uint64_t)plat_xchg64(&ts_busy_ns, 0);
g->gpu_parts = (uint64_t)plat_xchg64(&ts_parts, 0);
g->gpu_missed = (uint64_t)plat_xchg64(&ts_missed, 0);
g->have_vram = vk_mem_vram(&g->vram_used, &g->vram_budget, &g->vram_exact);
g->have_idle = vk_ctx()->has_calibrated != 0 && cal_at != 0;
g->idle_ns = (uint64_t)plat_xchg64(&ts_idle_ns, 0);
g->starved_ns = (uint64_t)plat_xchg64(&ts_starved_ns, 0);
g->queued_ns = (uint64_t)plat_xchg64(&ts_queued_ns, 0);
g->idle_gaps = (uint64_t)plat_xchg64(&ts_gaps, 0);
g->max_gap_ns = (uint64_t)plat_xchg64(&ts_max_gap_ns, 0);
}
// Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted.
WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) {
(void)cb, (void)fence;
for (int i = 0; i < n; i++) {
const op_hdr *h = handles[i] ? nvn_op_first(handles[i]) : NULL;
if (!h || h->op != OP_BEGIN || h->size < sizeof *h || nvn_op_next(h)->op != OP_END)
rt_fatal("vk: op replay is not linked");
}
return 0;
}
static void completion_main(void *arg) {
VkCtx *c = arg;
plat_mutex_lock(&st_lock);
for (;;) {
while (!stop && done == issued && !(qn && queue[qhead].fence <= done))
plat_cond_wait(&cv_work, &st_lock);
if (stop) break;
uint64_t reached = done;
if (issued > done) {
uint64_t want = done + 1;
plat_mutex_unlock(&st_lock);
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
.pSemaphores = &c->timeline, .pValues = &want};
VkResult r = vkWaitSemaphores(c->device, &wi, 100000000ull); // 100 ms: stop is checked between waits
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
vk_device_fault_report(r);
rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r);
}
vkGetSemaphoreCounterValue(c->device, c->timeline, &reached);
plat_mutex_lock(&st_lock);
if (reached > issued) reached = issued;
}
while (qn && queue[qhead].fence <= reached) {
Pending p = queue[qhead++];
qn--;
plat_mutex_unlock(&st_lock);
p.fn(p.arg);
plat_mutex_lock(&st_lock);
}
if (reached > done) {
plat_mutex_unlock(&st_lock);
vk_mem_collect();
plat_mutex_lock(&st_lock);
__atomic_store_n(&done, reached, __ATOMIC_RELEASE);
plat_cond_broadcast(&cv_done);
}
}
plat_mutex_unlock(&st_lock);
return;
}
void vk_sync_start(void) {
if (__atomic_load_n(&started, __ATOMIC_ACQUIRE)) return;
plat_mutex_lock(&start_lock);
if (!started) {
if (!vk_init()) rt_fatal("vk: no usable Vulkan device (see the vk: lines above; TOTK_GPU=null runs without a GPU)");
VkCtx *c = vk_ctx();
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
.queueFamilyIndex = c->gfx_family};
if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: submission command pool");
if (!ts_state && perf_gpu_timing()) ts_setup(c);
perf_gpu_stats = gpu_stats;
uint64_t now = 0;
vkGetSemaphoreCounterValue(c->device, c->timeline, &now);
if (issued > now) { // a new device after vk_shutdown: values keep increasing for the front end
VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline,
.value = issued};
if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) rt_fatal("vk: timeline restart at %llu", (unsigned long long)issued);
now = issued;
}
plat_mutex_lock(&st_lock);
issued = done = now;
if (reserved < now) reserved = now;
stop = 0;
plat_mutex_unlock(&st_lock);
vk_mem_note_fence(now);
completion_thread = plat_thread_new(completion_main, c, "vk completion");
if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
__atomic_store_n(&started, 1, __ATOMIC_RELEASE);
vk_astc_mode(); // decides TOTK_ASTC and builds the ASTC decoder now, not at the first ASTC upload
}
plat_mutex_unlock(&start_lock);
}
// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move
// slots[].
static int claim_slot(VkCtx *c, uint64_t fence) {
uint64_t gpu = 0;
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
for (int i = 0; i < nslots; i++)
if (slots[i].fence <= gpu) {
slots[i].fence = fence;
return i;
}
rec_drain(); // the part pool is the recorder's while it has packets (vk_rec.h): it is idle for the allocation
CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1));
if (!n) rt_fatal("vk: out of memory (command buffers)");
slots = n;
CmdSlot *s = &slots[nslots];
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
s->fence = fence;
return nslots++;
}
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
cur_slot = claim_slot(c, fence);
return &slots[cur_slot];
}
// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is
// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the
// part was recorded, sub_lock held; may move slots[].
// First brings the shadow storage up to date with the CPU's writes, which needs no command buffer.
// direct: the calling thread records, ends and submits it itself while the part being replayed stays the recorder's
// (vk_sync_part, which the caller made the recorder idle for).
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence, int direct) {
// Shadow storage: the CPU's writes go in before the part is submitted (TOTK_SYNC_AHEAD: queued for the GPU or
// made after the GPU finished the readers the game may think done, vk_memory.c).
vk_mem_shadow_sync(fence);
if (!vk_mem_mirror_pending() && !vk_mem_shadow_deferred()) return VK_NULL_HANDLE;
int i = claim_slot(c, fence); // before reading slots: the claim may move it
VkCommandBuffer cb = slots[i].cb;
if (direct) {
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
vkBeginCommandBuffer(cb, &bi);
} else {
rec_begin(cb, fence, 0, 0); // a command buffer of the part pool: the recorder's like the parts
}
int pair = ts_open(cb);
vk_mem_shadow_prologue(cb, fence); // before the mirror copies, which read the shadow these bring up to date
vk_mem_mirror_prologue(cb);
ts_close(cb, pair, fence);
if (direct || !rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb);
return cb;
}
// A chunk of a chunked replay (vk_exec.c chunks_submit), while the submission's own part is open on the recorder. The
// recorder begins, records, ends and submits the parts it owns (vk_rec.c rec_end_cb), so the prologue stays off it:
// the recorder is made idle first, as the part pool allows one thread at a time, and the prologue is recorded, ended
// and submitted here without taking the recorder's part away.
void vk_sync_part(VkCommandBuffer cb) {
VkCtx *c = vk_ctx();
vk_upload_flush(0);
rec_drain();
VkCommandBuffer pro = prologue(c, cur_fence, 1);
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
.pCommandBufferInfos = pro ? cbi : cbi + 1};
vk_queue_lock();
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r);
}
}
VkCommandBuffer vk_sync_split(void) {
VkCtx *c = vk_ctx();
VkCommandBuffer cb = slots[cur_slot].cb;
ts_end(cb, cur_fence);
if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder
vk_upload_flush(0); // the uploads this part asked for precede it in queue order
VkCommandBuffer pro = prologue(c, cur_fence, 0);
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
.pCommandBufferInfos = pro ? cbi : cbi + 1};
ts_submitted();
VkResult r = rec_submit(&si); // by the recorder when it recorded the part
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r);
}
CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached
rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence);
ts_begin(s->cb);
return s->cb;
}
// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed
// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value.
uint64_t vk_sync_reserve(void) {
vk_sync_start();
plat_mutex_lock(&st_lock);
if (reserved < issued) reserved = issued;
uint64_t v = ++reserved;
plat_mutex_unlock(&st_lock);
return v;
}
uint64_t vk_sync_submit(const uint64_t *handles, int n) {
vk_sync_start();
VkCtx *c = vk_ctx();
plat_mutex_lock(&sub_lock);
vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit
uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1;
cur_fence = fence;
// Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c
// defer), since the recording may already name it.
vk_mem_note_fence(fence);
CmdSlot *s = take_slot(c, fence);
uint64_t t0 = perf_begin();
rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence); // begun by the recorder when it records the part
ts_begin(s->cb);
int recorded = vk_exec_record(s->cb, handles, n, fence);
s = &slots[cur_slot]; // the last part (vk_sync_split)
if (recorded) ts_end(s->cb, fence);
else ts_drop();
// The recorder is done with the part: the end-of-submission crumb, then the end, by the recorder when it records.
if (!rec_end_cb(s->cb, VK_CRUMB_SUBMIT_END, fence)) vkEndCommandBuffer(s->cb);
perf_replay(t0);
// After the recording: the uploads it asked for (and every earlier one) precede it in queue order.
vk_upload_flush(0);
VkCommandBuffer pro = prologue(c, fence, 0);
s = &slots[cur_slot];
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}};
VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u),
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1,
.pSignalSemaphoreInfos = &sig};
ts_submitted();
VkResult r = rec_submit(&si); // by the recorder when it recorded the part
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r);
}
s->fence = recorded ? fence : 0;
plat_mutex_lock(&st_lock);
__atomic_store_n(&issued, fence, __ATOMIC_RELEASE);
plat_cond_broadcast(&cv_work);
plat_mutex_unlock(&st_lock);
static unsigned subs;
if (++subs % 4096 == 0) rec_stats_log();
plat_mutex_unlock(&sub_lock);
return fence;
}
static _Thread_local VkDeferred *deferred;
void vk_completion_defer(VkDeferred *d) { deferred = d; }
static void queue_put(uint64_t fence, void (*fn)(void *), void *arg) { // st_lock held
if (qhead + qn == qcap) {
if (qhead) {
memmove(queue, queue + qhead, sizeof *queue * (size_t)qn);
qhead = 0;
} else {
int cap = qcap ? qcap * 2 : 256;
Pending *q = realloc(queue, sizeof *q * (size_t)cap);
if (!q) rt_fatal("vk: out of memory (completion queue)");
queue = q, qcap = cap;
}
}
queue[qhead + qn++] = (Pending){fence, fn, arg};
// The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's,
// for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it.
if (fence <= done) plat_cond_broadcast(&cv_work);
}
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
VkDeferred *d = deferred;
if (d) {
if (d->n == d->cap) {
d->cap = d->cap ? d->cap * 2 : 64;
d->items = realloc(d->items, sizeof(Pending) * (size_t)d->cap);
if (!d->items) rt_fatal("vk: out of memory (completion queue)");
}
((Pending *)d->items)[d->n++] = (Pending){fence, fn, arg};
return;
}
plat_mutex_lock(&st_lock);
queue_put(fence, fn, arg);
plat_mutex_unlock(&st_lock);
}
void vk_completion_flush(VkDeferred *d) {
plat_mutex_lock(&st_lock);
for (int i = 0; i < d->n; i++) {
const Pending *p = &((Pending *)d->items)[i];
queue_put(p->fence, p->fn, p->arg);
}
plat_mutex_unlock(&st_lock);
d->n = 0;
}
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns);
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) {
uint64_t p0 = perf_begin();
int ok = fence_wait_impl(value, timeout_ns);
perf_end(PERF_FENCE, p0);
perf_gpu_wait(p0);
return ok;
}
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
vk_sync_start();
plat_mutex_lock(&st_lock);
int ok;
if (value > issued && value > reserved) {
ok = 0; // never submitted nor queued: never reached (as in the null back end)
} else {
uint64_t t0 = kern_now_ns();
while (done < value) {
if (timeout_ns < (1ull << 62)) {
uint64_t el = kern_now_ns() - t0;
if (el >= timeout_ns) break;
plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el);
} else {
plat_cond_wait(&cv_done, &st_lock);
}
}
ok = done >= value;
}
plat_mutex_unlock(&st_lock);
return ok;
}
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
void vk_sync_finish(void) {
vk_sync_start();
vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX);
vk_upload_flush(1);
}
void vk_sync_shutdown(void) {
plat_mutex_lock(&start_lock);
if (started) {
vk_sync_finish();
plat_mutex_lock(&st_lock);
stop = 1;
plat_cond_broadcast(&cv_work);
plat_mutex_unlock(&st_lock);
plat_thread_join(completion_thread);
completion_thread = NULL;
plat_mutex_lock(&st_lock);
qhead = qn = 0; // callbacks of values never submitted
plat_mutex_unlock(&st_lock);
plat_mutex_lock(&sub_lock);
VkCtx *c = vk_ctx();
if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
perf_gpu_stats = NULL;
if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL);
ts_pool = VK_NULL_HANDLE;
ts_state = 0;
ts_last_end = 0;
memset((void *)ts_busy, 0, sizeof ts_busy);
cmd_pool = VK_NULL_HANDLE;
free(slots);
slots = NULL;
nslots = 0;
plat_mutex_unlock(&sub_lock);
__atomic_store_n(&started, 0, __ATOMIC_RELEASE);
}
plat_mutex_unlock(&start_lock);
}
Reference in new issue View Git Blame Copy Permalink