Files
totk-pc-recomp/runtime/vk_sync.c
T
Valiantsin DzerakhandClaude Opus 5.5 97d3762ba1 Strip milestone and task labels from code comments and drop the dead gfx_stubs.c
Comments named the agent task (T1-T7, M2-M5, Task 2.x) that once owned each file or block, and pointed at local
notes the repository does not carry (backlog items, milestone plans, research notes). They now say what the code
does. gfx_stubs.c held weak stand-ins for functions that every build has defined for a long time.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 13:12:50 +03:00

466 lines
21 KiB
C

// Submission, fences and the completion thread of the Vulkan back end.
// One timeline semaphore (VkCtx.timeline) orders everything: each submission records its command handles into a
// primary command buffer (vk_exec_record) and signals the next timeline value, which is the NVN fence value
// the front end hands to syncs. The completion thread waits on the timeline and, in submit order, runs the CPU side
// effects queued for each value (event writes, counter reports, sync state), then frees the memory and texture objects whose users
// completed. A value counts as done only after its side effects ran, so a guest woken by fence_wait sees them.
//
// The submission shape (one timeline value per submission, signalled at ALL_COMMANDS, vkWaitSemaphores for waits, a
// failed wait treated as a lost device) follows LAS Recomp runtime/nvn/vulkan/exec.c:39-110, 355-397
// (https://github.com/peiikai/LASRecomp @ da35b83a9611d046a9a681aae668d7085954d37f, MIT, Copyright (c) 2026 Peiikai).
// Side effects written after GPU completion, as MacNativeGames does: games/totk/third_party/d2r_gfx/gfx/nvn/hle_nvn.c:7-9
// (https://github.com/ly05010419/MacNativeGames @ b75be8cfa704df3701e86130c8ced268e79d6b11, MIT, Copyright (c) 2026
// jhaizhou; origin d2r-mobile).
#include <stdlib.h>
#include <string.h>
#include <windows.h>
#include "gfx.h"
#include "host.h"
#include "nvn_api.h"
#include "vk.h"
#define WEAK __attribute__((weak))
typedef struct { uint64_t fence; void (*fn)(void *); void *arg; } Pending;
typedef struct { VkCommandBuffer cb; uint64_t fence; } CmdSlot; // reusable once the timeline reached fence
static SRWLOCK start_lock = SRWLOCK_INIT;
static int started;
static HANDLE completion_thread;
// Submissions: one at a time, so timeline values are signalled in the order they are handed out.
static SRWLOCK sub_lock = SRWLOCK_INIT;
static VkCommandPool cmd_pool;
static CmdSlot *slots;
static int nslots;
static int cur_slot; // the slot vk_sync_submit records into now (an index: take_slot may move slots)
static uint64_t cur_fence; // the value of the submission being recorded
// Shared with the completion thread (st_lock).
static SRWLOCK st_lock = SRWLOCK_INIT;
static CONDITION_VARIABLE cv_work = CONDITION_VARIABLE_INIT; // a submission, a callback or stop
static CONDITION_VARIABLE cv_done = CONDITION_VARIABLE_INIT; // done advanced
static uint64_t issued, done; // newest submitted value; newest value whose side effects ran
static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued)
static int stop;
static Pending *queue; // side effects in the order they were added
static int qhead, qn, qcap;
// ---- GPU time (TOTK_TRACE=perf): each submission part (vk_sync_split) writes a timestamp when it starts and when it
// ends, into a ring of query pairs. The completion thread reads a pair once the part's value is reached and adds the
// part's interval to the busy time, counting overlap with the previous part once. Off (one log line) when the queue
// family has no timestamp bits or the period is 0. Upload batches (vk_memory.c) are not timed.
#define TS_PAIRS 1024
static VkQueryPool ts_pool;
static int ts_state; // 0 not set up, 1 on, -1 off
static double ts_period; // ns per tick
static uint64_t ts_mask; // valid bits of a timestamp
static uint32_t ts_next; // submitting thread (sub_lock): next pair
static volatile uint8_t ts_busy[TS_PAIRS]; // written until the completion thread read the pair
static int ts_cur = -1; // the pair of the part being recorded, -1 untimed
static uint64_t ts_last_end; // completion thread: end of the latest part read (ticks)
static volatile LONG64 ts_busy_ns, ts_parts, ts_missed;
// On the first submission that wants timing (perf_gpu_timing: TOTK_TRACE=perf or the full performance overlay).
static void ts_setup(VkCtx *c) {
ts_state = -1;
uint32_t nq = 0;
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL);
VkQueueFamilyProperties *qp = calloc(nq ? nq : 1, sizeof *qp);
if (!qp) return;
vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp);
uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0;
free(qp);
float period = c->props.limits.timestampPeriod;
if (!bits || period <= 0) {
rt_log("vk: no GPU timestamps on this queue (valid bits %u, period %.3f ns): the perf line shows no GPU time", bits, period);
return;
}
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP,
.queryCount = 2 * TS_PAIRS};
VkResult r = vkCreateQueryPool(c->device, &qi, NULL, &ts_pool);
if (r != VK_SUCCESS) {
rt_log("vk: timestamp query pool failed (%d): the perf line shows no GPU time", (int)r);
return;
}
ts_period = period;
ts_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1;
ts_state = 1;
}
typedef struct { uint32_t pair; } TsRead;
static void ts_read_cb(void *arg) { // completion thread, in submission order
TsRead *t = arg;
VkCtx *c = vk_ctx();
uint64_t v[2];
if (vkGetQueryPoolResults(c->device, ts_pool, 2 * t->pair, 2, sizeof v, v, sizeof v[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) {
uint64_t s = v[0] & ts_mask, e = v[1] & ts_mask;
if (e >= s) {
uint64_t from = s > ts_last_end ? s : ts_last_end;
if (e > from) {
LONG64 ns = (LONG64)((double)(e - from) * ts_period);
InterlockedAdd64(&ts_busy_ns, ns);
perf_gpu_busy((uint64_t)ns);
}
if (e > ts_last_end) ts_last_end = e;
InterlockedIncrement64(&ts_parts);
} else InterlockedIncrement64(&ts_missed); // the counter wrapped
} else InterlockedIncrement64(&ts_missed);
__atomic_store_n(&ts_busy[t->pair], 0, __ATOMIC_RELEASE);
free(t);
}
// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed.
static int ts_open(VkCommandBuffer cb) {
if (ts_state != 1 || !perf_gpu_timing()) {
if (ts_state || !perf_gpu_timing()) return -1;
ts_setup(vk_ctx()); // the overlay turned full: timestamps from now on
if (ts_state != 1) return -1;
}
uint32_t p = ts_next % TS_PAIRS;
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return -1; }
ts_next++;
ts_busy[p] = 1;
vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2);
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p);
return (int)p;
}
// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it.
static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence) {
if (pair < 0) return;
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1);
TsRead *t = malloc(sizeof *t);
if (!t) rt_fatal("vk: out of memory");
t->pair = (uint32_t)pair;
vk_completion_add(fence, ts_read_cb, t);
}
// The pair of the part being recorded.
static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); }
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
ts_close(cb, ts_cur, fence);
ts_cur = -1;
}
// An empty submission records no command buffer: its pair is never written.
static void ts_drop(void) {
if (ts_cur < 0) return;
__atomic_store_n(&ts_busy[ts_cur], 0, __ATOMIC_RELEASE);
ts_cur = -1;
}
static void gpu_stats(PerfGpu *g) {
g->have_gpu = ts_state == 1;
g->gpu_ns = (uint64_t)InterlockedExchange64(&ts_busy_ns, 0);
g->gpu_parts = (uint64_t)InterlockedExchange64(&ts_parts, 0);
g->gpu_missed = (uint64_t)InterlockedExchange64(&ts_missed, 0);
g->have_vram = vk_mem_vram(&g->vram_used, &g->vram_budget, &g->vram_exact);
}
// Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted.
WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) {
(void)cb, (void)fence;
for (int i = 0; i < n; i++) {
const op_hdr *h = handles[i] ? nvn_op_first(handles[i]) : NULL;
if (!h || h->op != OP_BEGIN || h->size < sizeof *h || nvn_op_next(h)->op != OP_END)
rt_fatal("vk: op replay is not linked");
}
return 0;
}
static DWORD WINAPI completion_main(void *arg) {
VkCtx *c = arg;
AcquireSRWLockExclusive(&st_lock);
for (;;) {
while (!stop && done == issued && !(qn && queue[qhead].fence <= done))
SleepConditionVariableSRW(&cv_work, &st_lock, INFINITE, 0);
if (stop) break;
uint64_t reached = done;
if (issued > done) {
uint64_t want = done + 1;
ReleaseSRWLockExclusive(&st_lock);
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
.pSemaphores = &c->timeline, .pValues = &want};
VkResult r = vkWaitSemaphores(c->device, &wi, 100000000ull); // 100 ms: stop is checked between waits
if (r != VK_SUCCESS && r != VK_TIMEOUT) rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r);
vkGetSemaphoreCounterValue(c->device, c->timeline, &reached);
AcquireSRWLockExclusive(&st_lock);
if (reached > issued) reached = issued;
}
while (qn && queue[qhead].fence <= reached) {
Pending p = queue[qhead++];
qn--;
ReleaseSRWLockExclusive(&st_lock);
p.fn(p.arg);
AcquireSRWLockExclusive(&st_lock);
}
if (reached > done) {
ReleaseSRWLockExclusive(&st_lock);
vk_mem_collect();
AcquireSRWLockExclusive(&st_lock);
__atomic_store_n(&done, reached, __ATOMIC_RELEASE);
WakeAllConditionVariable(&cv_done);
}
}
ReleaseSRWLockExclusive(&st_lock);
return 0;
}
void vk_sync_start(void) {
if (__atomic_load_n(&started, __ATOMIC_ACQUIRE)) return;
AcquireSRWLockExclusive(&start_lock);
if (!started) {
if (!vk_init()) rt_fatal("vk: no usable Vulkan device (see the vk: lines above; TOTK_GPU=null runs without a GPU)");
VkCtx *c = vk_ctx();
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
.queueFamilyIndex = c->gfx_family};
if (vkCreateCommandPool(c->device, &pi, NULL, &cmd_pool) != VK_SUCCESS) rt_fatal("vk: submission command pool");
if (!ts_state && perf_gpu_timing()) ts_setup(c);
perf_gpu_stats = gpu_stats;
uint64_t now = 0;
vkGetSemaphoreCounterValue(c->device, c->timeline, &now);
if (issued > now) { // a new device after vk_shutdown: values keep increasing for the front end
VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline,
.value = issued};
if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) rt_fatal("vk: timeline restart at %llu", (unsigned long long)issued);
now = issued;
}
AcquireSRWLockExclusive(&st_lock);
issued = done = now;
if (reserved < now) reserved = now;
stop = 0;
ReleaseSRWLockExclusive(&st_lock);
vk_mem_note_submit(now);
completion_thread = CreateThread(NULL, 0, completion_main, c, 0, NULL);
if (!completion_thread) rt_fatal("vk: cannot start the completion thread (error %lu)", GetLastError());
__atomic_store_n(&started, 1, __ATOMIC_RELEASE);
vk_astc_mode(); // decides TOTK_ASTC and builds the ASTC decoder now, not at the first ASTC upload
}
ReleaseSRWLockExclusive(&start_lock);
}
// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move
// slots[].
static int claim_slot(VkCtx *c, uint64_t fence) {
uint64_t gpu = 0;
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
for (int i = 0; i < nslots; i++)
if (slots[i].fence <= gpu) {
slots[i].fence = fence;
return i;
}
CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1));
if (!n) rt_fatal("vk: out of memory (command buffers)");
slots = n;
CmdSlot *s = &slots[nslots];
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
s->fence = fence;
return nslots++;
}
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
cur_slot = claim_slot(c, fence);
return &slots[cur_slot];
}
// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is
// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the
// part was recorded, sub_lock held; may move slots[].
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
int i = claim_slot(c, fence); // before reading slots: the claim may move it
VkCommandBuffer cb = slots[i].cb;
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
vkBeginCommandBuffer(cb, &bi);
int pair = ts_open(cb);
vk_mem_mirror_prologue(cb);
ts_close(cb, pair, fence);
vkEndCommandBuffer(cb);
return cb;
}
VkCommandBuffer vk_sync_split(void) {
VkCtx *c = vk_ctx();
VkCommandBuffer cb = slots[cur_slot].cb;
ts_end(cb, cur_fence);
vkEndCommandBuffer(cb);
vk_upload_flush(0); // the uploads this part asked for precede it in queue order
VkCommandBuffer pro = prologue(c, cur_fence);
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
.pCommandBufferInfos = pro ? cbi : cbi + 1};
vk_queue_lock();
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
if (r != VK_SUCCESS) rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r);
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
vkBeginCommandBuffer(s->cb, &bi);
ts_begin(s->cb);
return s->cb;
}
// Asynchronous replay (vk_backend.c): the value the next queued submission or present will signal. Values are handed
// out in queue order and the replay thread submits them in that order, so each vk_sync_submit gets exactly its value.
uint64_t vk_sync_reserve(void) {
vk_sync_start();
AcquireSRWLockExclusive(&st_lock);
if (reserved < issued) reserved = issued;
uint64_t v = ++reserved;
ReleaseSRWLockExclusive(&st_lock);
return v;
}
uint64_t vk_sync_submit(const uint64_t *handles, int n) {
vk_sync_start();
VkCtx *c = vk_ctx();
AcquireSRWLockExclusive(&sub_lock);
vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit
uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1;
cur_fence = fence;
CmdSlot *s = take_slot(c, fence);
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
uint64_t t0 = perf_begin();
vkBeginCommandBuffer(s->cb, &bi);
ts_begin(s->cb);
int recorded = vk_exec_record(s->cb, handles, n, fence);
s = &slots[cur_slot]; // the last part (vk_sync_split)
if (recorded) ts_end(s->cb, fence);
else ts_drop();
vkEndCommandBuffer(s->cb);
perf_replay(t0);
// After the recording: the uploads it asked for (and every earlier one) precede it in queue order.
vk_upload_flush(0);
VkCommandBuffer pro = prologue(c, fence);
s = &slots[cur_slot];
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}};
VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u),
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1,
.pSignalSemaphoreInfos = &sig};
vk_queue_lock();
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
if (r != VK_SUCCESS) rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r);
s->fence = recorded ? fence : 0;
vk_mem_note_submit(fence);
AcquireSRWLockExclusive(&st_lock);
__atomic_store_n(&issued, fence, __ATOMIC_RELEASE);
WakeAllConditionVariable(&cv_work);
ReleaseSRWLockExclusive(&st_lock);
ReleaseSRWLockExclusive(&sub_lock);
return fence;
}
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
AcquireSRWLockExclusive(&st_lock);
if (qhead + qn == qcap) {
if (qhead) {
memmove(queue, queue + qhead, sizeof *queue * (size_t)qn);
qhead = 0;
} else {
int cap = qcap ? qcap * 2 : 256;
Pending *q = realloc(queue, sizeof *q * (size_t)cap);
if (!q) rt_fatal("vk: out of memory (completion queue)");
queue = q, qcap = cap;
}
}
queue[qhead + qn++] = (Pending){fence, fn, arg};
// The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's,
// for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it.
if (fence <= done) WakeAllConditionVariable(&cv_work);
ReleaseSRWLockExclusive(&st_lock);
}
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns);
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) {
uint64_t p0 = perf_begin();
int ok = fence_wait_impl(value, timeout_ns);
perf_end(PERF_FENCE, p0);
perf_gpu_wait(p0);
return ok;
}
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
vk_sync_start();
AcquireSRWLockExclusive(&st_lock);
int ok;
if (value > issued && value > reserved) {
ok = 0; // never submitted nor queued: never reached (as in the null back end)
} else {
uint64_t t0 = kern_now_ns();
while (done < value) {
DWORD ms = INFINITE;
if (timeout_ns < (1ull << 62)) {
uint64_t el = kern_now_ns() - t0;
if (el >= timeout_ns) break;
uint64_t left = (timeout_ns - el + 999999) / 1000000;
ms = left >= INFINITE ? INFINITE - 1 : (DWORD)left;
}
SleepConditionVariableSRW(&cv_done, &st_lock, ms, 0);
}
ok = done >= value;
}
ReleaseSRWLockExclusive(&st_lock);
return ok;
}
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
void vk_sync_finish(void) {
vk_sync_start();
vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX);
vk_upload_flush(1);
}
void vk_sync_shutdown(void) {
AcquireSRWLockExclusive(&start_lock);
if (started) {
vk_sync_finish();
AcquireSRWLockExclusive(&st_lock);
stop = 1;
WakeAllConditionVariable(&cv_work);
ReleaseSRWLockExclusive(&st_lock);
WaitForSingleObject(completion_thread, INFINITE);
CloseHandle(completion_thread);
completion_thread = NULL;
AcquireSRWLockExclusive(&st_lock);
qhead = qn = 0; // callbacks of values never submitted
ReleaseSRWLockExclusive(&st_lock);
AcquireSRWLockExclusive(&sub_lock);
VkCtx *c = vk_ctx();
if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
perf_gpu_stats = NULL;
if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL);
ts_pool = VK_NULL_HANDLE;
ts_state = 0;
ts_last_end = 0;
memset((void *)ts_busy, 0, sizeof ts_busy);
cmd_pool = VK_NULL_HANDLE;
free(slots);
slots = NULL;
nslots = 0;
ReleaseSRWLockExclusive(&sub_lock);
__atomic_store_n(&started, 0, __ATOMIC_RELEASE);
}
ReleaseSRWLockExclusive(&start_lock);
}