Merge the device-local mirror of buffer-bound pools on discrete GPUs and the frozen idle regions that cut write-watch queries

On the GTX 1050 Ti the GPU time per frame in the world drops from 110-120 ms to 10-11 ms. TOTK_MEMORY=verify on the
same machine compared 4.45 million mirror pages over a 74-second run with 0 mismatches.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-25 09:43:34 +03:00
commit 6b1d15534e
18 files changed
+1066 -98

No files matched your search

+2 -2
View File
@@ -27,10 +27,10 @@ static const char *const known[] = {
"BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL",
"BIND_ZR",
"CLOTH_REALTIME", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT",
"FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "GPL",
"FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "FREEZE_IDLE", "GPL",
"GPL_DRAW_BUILD",
"GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MINIMAL_LAYOUT", "MOTION", "MOUSE_SENS", "MOVIE_HOST",
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST",
"MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SYNC", "PREWARM_XLATE",
"PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_STAGE", "RDOC",
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
+7 -1
View File
@@ -691,7 +691,13 @@ static File *file_of(uint64_t h, HANDLE *fh, int *mode) {
static int64_t host_read_at(HANDLE f, void *buf, uint32_t n, uint64_t off) {
OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32)};
DWORD got = 0;
if (!ReadFile(f, buf, n, &got, &ov)) return GetLastError() == ERROR_HANDLE_EOF ? 0 : -1;
mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting
if (!ReadFile(f, buf, n, &got, &ov)) {
DWORD e = GetLastError();
if (e == ERROR_NOACCESS) // a read-only page in the buffer: a frozen region missed its thaw
rt_log("fs: read of %u bytes into %p refused: the buffer is not writable", n, buf);
return e == ERROR_HANDLE_EOF ? 0 : -1;
}
return got;
}
+17
View File
@@ -5,6 +5,7 @@
// Needs no game files.
#include <stddef.h>
#include <stdio.h>
#include <windows.h>
#include "gfx.h"
@@ -60,6 +61,22 @@ static void check_write_watch(void) {
printf("write-watch: three pages written, max 2: %llu then %llu\n", (unsigned long long)n, (unsigned long long)m);
CHECK(n == 2 && m == 1, "max 2 reported %llu, then %llu (want 2, 1)", (unsigned long long)n, (unsigned long long)m);
// Frozen region: read-only until the first write thaws it (the write lands and is reported); mem_thaw (a file
// read) thaws it and holds it thawed; SetHeapSize below decommits it.
mem_written_pages(heap, 0x200000, pages, 8);
CHECK(mem_freeze(heap) == 1 && mem_frozen(heap), "freeze of an idle committed 2 MiB region");
MEMORY_BASIC_INFORMATION mbi;
CHECK(VirtualQuery((void *)(uintptr_t)heap, &mbi, sizeof mbi) && mbi.Protect == PAGE_READONLY, "frozen region is not read-only");
p[0x3000] = 7;
CHECK(!mem_frozen(heap) && p[0x3000] == 7, "a write into a frozen region did not thaw it");
n = mem_written_pages(heap, 0x200000, pages, 8);
CHECK(n == 1 && pages[0] == heap + 0x3000, "write after a thaw: %llu pages reported (want 1)", (unsigned long long)n);
CHECK(mem_freeze(heap) == 1, "refreeze after a write thaw");
mem_thaw(heap + 0x6000, 0x1000);
CHECK(!mem_frozen(heap) && mem_freeze(heap) == 0, "mem_thaw did not thaw and hold the region");
p[0x6000] = 8;
printf("write-watch: frozen region thawed by a write (reported once) and by mem_thaw (held)\n");
uint64_t a = g->alias.base + 0x10000;
CHECK(mem_map_physical(a, 0x4000) == 0, "MapPhysicalMemory");
mem_written_pages(a, 0x4000, pages, 8);
+12
View File
@@ -128,6 +128,18 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64
// past max stays reported next time). The range must lie
// inside the stack/heap block or the alias region
// (write watch); otherwise returns UINT64_MAX
uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max);
// the same without the reset: the bits stay for the next
// mem_written_pages (verify checks only)
int mem_freeze(uint64_t a); // makes the 2 MiB region at a (2 MiB aligned, watched,
// committed PAGE_READWRITE) read-only until a write or
// mem_thaw thaws it; 1 frozen (or already), 0 refused
int mem_frozen(uint64_t a); // the 2 MiB region of a is frozen: no page in it changed
// since its freeze
void mem_thaw(uint64_t a, uint64_t n); // makes the frozen regions over [a, a+n) writable again;
// before a kernel write (file read) into guest memory
void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws);
// regions frozen now; freezes and thaws since the last call
void mem_kernel_free(void *p, uint64_t size); // TLR/TLS reuse after thread exit
// ---- modules (loader.c, T1)
+136 -10
View File
@@ -67,21 +67,142 @@ static void *reserve_at(uint64_t at, uint64_t size, DWORD extra) {
return p;
}
// ---- frozen regions
// A 2 MiB region of the watched reservations that nothing wrote for a while can be made read-only (mem_freeze), so the
// back end's write-watch queries skip it: no page of a frozen region changes before a thaw. The first write faults and
// thaw_handler makes the region writable again before the write retries. A kernel write into guest memory (a file
// read) fails on a read-only page instead of faulting, so file reads thaw their buffer first (mem_thaw), and so does
// every commit, decommit and protection change here. A region stays thawed for FRZ_HOLD_MS after a mem_thaw: a file
// read still in flight writes its buffer later. frz_lock is a leaf lock: nothing writes guest memory under it.
#define FRZ_SHIFT 21
#define FRZ_SIZE (1ull << FRZ_SHIFT)
#define FRZ_HOLD_MS 5000u
static volatile uint8_t frz[RT_ADDRESS_LIMIT >> FRZ_SHIFT];
static uint32_t thaw_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: GetTickCount of the last thaw request, 0 never
static volatile LONG nfrz;
static SRWLOCK frz_lock = SRWLOCK_INIT;
static volatile LONG64 frz_count[4]; // freezes; thaws by a write fault, by mem_thaw, by a mapping change
static int watched(uint64_t addr, uint64_t size) {
const RtGuestLayout *g = &L;
return (addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size) ||
(addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size);
}
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
static void thaw_locked(uint64_t r, int why) {
if (why == 2) thaw_ms[r] = (uint32_t)GetTickCount64() | 1;
if (!frz[r]) return;
DWORD old;
uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE;
if (!VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READWRITE, &old)) {
// Not all committed (cannot happen while mapping changes thaw under frz_lock): the read-only runs only.
MEMORY_BASIC_INFORMATION mbi;
while (a < end && VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi)) {
uint64_t b = (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize;
if (b > end) b = end;
if (mbi.State == MEM_COMMIT && mbi.Protect == PAGE_READONLY &&
!VirtualProtect((void *)(uintptr_t)a, b - a, PAGE_READWRITE, &old))
rt_fatal("memory: thawing %#llx+%#llx failed (error %lu)", (unsigned long long)a,
(unsigned long long)(b - a), GetLastError());
a = b;
}
}
__atomic_store_n(&frz[r], 0, __ATOMIC_RELEASE);
InterlockedDecrement(&nfrz);
InterlockedIncrement64(&frz_count[why]);
}
// frz_lock held exclusively.
static void thaw_range_locked(uint64_t a, uint64_t n, int why) {
if (!n || a >= RT_ADDRESS_LIMIT) return;
uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT;
for (uint64_t r = r0; r <= r1; r++) thaw_locked(r, why);
}
void mem_thaw(uint64_t a, uint64_t n) {
AcquireSRWLockExclusive(&frz_lock); // also when nothing is frozen: the hold keeps a read's buffer writable
thaw_range_locked(a, n, 2);
ReleaseSRWLockExclusive(&frz_lock);
}
int mem_freeze(uint64_t a) {
if ((a & (FRZ_SIZE - 1)) || !watched(a, FRZ_SIZE)) return 0;
uint64_t r = a >> FRZ_SHIFT;
MEMORY_BASIC_INFORMATION mbi;
DWORD old;
int ok = 0;
AcquireSRWLockExclusive(&frz_lock);
if (frz[r]) ok = 1;
else if (thaw_ms[r] && (int32_t)((uint32_t)GetTickCount64() - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) ok = 0; // |1 may be ahead
else if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT &&
mbi.Protect == PAGE_READWRITE && (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize >= a + FRZ_SIZE &&
VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READONLY, &old)) {
__atomic_store_n(&frz[r], 1, __ATOMIC_RELEASE);
InterlockedIncrement(&nfrz);
InterlockedIncrement64(&frz_count[0]);
ok = 1;
}
ReleaseSRWLockExclusive(&frz_lock);
return ok;
}
int mem_frozen(uint64_t a) { return a < RT_ADDRESS_LIMIT && __atomic_load_n(&frz[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE); }
void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws) {
*frozen = (uint64_t)__atomic_load_n(&nfrz, __ATOMIC_RELAXED);
*freezes = (uint64_t)InterlockedExchange64(&frz_count[0], 0);
*fault_thaws = (uint64_t)InterlockedExchange64(&frz_count[1], 0);
*other_thaws = (uint64_t)InterlockedExchange64(&frz_count[2], 0) + (uint64_t)InterlockedExchange64(&frz_count[3], 0);
}
// The first write into a frozen region thaws it and retries. A fault in a region another thread thawed meanwhile
// retries too; any other fault goes on to the crash handler.
static LONG CALLBACK thaw_handler(EXCEPTION_POINTERS *ep) {
const EXCEPTION_RECORD *er = ep->ExceptionRecord;
if (er->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || er->NumberParameters < 2 || er->ExceptionInformation[0] != 1)
return EXCEPTION_CONTINUE_SEARCH;
uint64_t a = er->ExceptionInformation[1];
if (a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return EXCEPTION_CONTINUE_SEARCH;
AcquireSRWLockExclusive(&frz_lock);
int was = frz[a >> FRZ_SHIFT];
thaw_locked(a >> FRZ_SHIFT, 1);
ReleaseSRWLockExclusive(&frz_lock);
if (was) return EXCEPTION_CONTINUE_EXECUTION;
MEMORY_BASIC_INFORMATION mbi;
if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT && mbi.Protect == PAGE_READWRITE)
return EXCEPTION_CONTINUE_EXECUTION;
return EXCEPTION_CONTINUE_SEARCH;
}
// Mapping changes thaw the regions they touch and run under frz_lock, so no freeze sees them half done.
static void commit(uint64_t a, uint64_t n, const char *what) {
if (n && !VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE))
AcquireSRWLockExclusive(&frz_lock);
thaw_range_locked(a, n, 3);
int ok = !n || VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE);
ReleaseSRWLockExclusive(&frz_lock);
if (!ok)
rt_fatal("memory: commit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
(unsigned long long)n, GetLastError());
}
static void decommit(uint64_t a, uint64_t n, const char *what) {
if (n && !VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT))
AcquireSRWLockExclusive(&frz_lock);
thaw_range_locked(a, n, 3);
int ok = !n || VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT);
ReleaseSRWLockExclusive(&frz_lock);
if (!ok)
rt_fatal("memory: decommit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
(unsigned long long)n, GetLastError());
}
static void protect(uint64_t a, uint64_t n, DWORD prot, const char *what) {
DWORD old;
if (n && !VirtualProtect((void *)(uintptr_t)a, n, prot, &old))
AcquireSRWLockExclusive(&frz_lock);
thaw_range_locked(a, n, 3);
int ok = !n || VirtualProtect((void *)(uintptr_t)a, n, prot, &old);
ReleaseSRWLockExclusive(&frz_lock);
if (!ok)
rt_fatal("memory: protect %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
(unsigned long long)n, GetLastError());
}
@@ -129,6 +250,7 @@ void mem_init(void) {
if (reserve_at(at, ALIAS_SIZE, MEM_WRITE_WATCH)) alias = at;
if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39");
AddVectoredExceptionHandler(1, thaw_handler); // before the crash handler
alias_pages = VirtualAlloc(NULL, ALIAS_PAGES, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
if (!alias_pages) rt_fatal("memory: cannot allocate the alias page map");
const char *pool = cfg_get("TOTK_APP_POOL");
@@ -148,20 +270,24 @@ const RtGuestLayout *mem_layout(void) { return &L; }
// Adapted from LAS Recomp runtime/platform/windows.c:40-63 @ da35b83 (MIT, Copyright (c) 2026 Peiikai): GetWriteWatch
// with the reset flag. A range inside one watched reservation (stack and heap block, or alias region) only.
uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, DWORD flags) {
if (!max || !size) return 0;
const RtGuestLayout *g = &L;
int low_block = addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size;
int alias = addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size;
if (!low_block && !alias) return UINT64_MAX;
if (!watched(addr, size)) return UINT64_MAX;
ULONG_PTR n = (ULONG_PTR)max;
DWORD gran;
_Static_assert(sizeof(PVOID) == sizeof(uint64_t), "page addresses are written straight into a uint64_t array");
if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran))
return UINT64_MAX;
if (GetWriteWatch(flags, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran)) return UINT64_MAX;
return n;
}
uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
return written_pages(addr, size, pages, max, WRITE_WATCH_FLAG_RESET);
}
uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
return written_pages(addr, size, pages, max, 0);
}
static uint64_t image_bytes(void) {
uint64_t n = 0;
for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size;
+3
View File
@@ -99,3 +99,6 @@ const char *nvn_op_name(uint32_t op); // "DRAW_IDX", ...; "
// MemoryPoolInvalidateMappedRange: the CPU is about to read pool storage the GPU wrote. Weak no-op in nvn_ext.c; a back
// end that renders into host-side copies defines it and writes them back.
void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size);
// MemoryPoolFlushMappedRange: the CPU wrote [mem + off, mem + off + size). Weak no-op in nvn_ext.c; a back end that keeps
// device-side copies of pool memory (vk_memory.c's VRAM mirror) defines it and copies the range again.
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size);
+7 -2
View File
@@ -303,9 +303,14 @@ void nvn_flush_stats(uint64_t out[2][3][2]) {
for (int j = 0; j < 2; j++) out[i][k][j] = __atomic_exchange_n(&flush_stat[i][k][j], 0, __ATOMIC_RELAXED);
}
// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent
// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent; a
// back end with device-side copies hears of it (nvn_pool_flushed)
NVN_API(nvnMemoryPoolFlushMappedRange) {
if (rt_trace & RT_TRACE_PERF) flush_note(0, arg_p(c, 0), arg_x(c, 2));
const pool_t *p = arg_p(c, 0);
int64_t off = (int64_t)arg_x(c, 1);
uint64_t size = arg_x(c, 2);
if (rt_trace & RT_TRACE_PERF) flush_note(0, p, size);
if (p && off >= 0 && (uint64_t)off < p->size) nvn_pool_flushed(p, (uint64_t)off, size < p->size - (uint64_t)off ? size : p->size - (uint64_t)off);
}
// NVNmemoryPoolFlags nvnMemoryPoolGetFlags(const NVNmemoryPool *p)
NVN_API(nvnMemoryPoolGetFlags) { ret_x(c, ((const pool_t *)arg_p(c, 0))->flags); }
+2
View File
@@ -20,6 +20,8 @@ void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size) {
gfx_backend()->guest_read((uintptr_t)p->mem + off, size);
}
__attribute__((weak)) void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { (void)p, (void)off, (void)size; }
// ---------- debug labels, markers, groups, checkpoints: meaningful only to NVN debugging tools
// void nvn<Object>SetDebugLabel(NVN<object> *o, const char *label)
NVN_API(nvnDeviceSetDebugLabel) { (void)c; }
+7 -2
View File
@@ -84,11 +84,16 @@ static int64_t read_at(HANDLE f, void *buf, uint32_t n, uint64_t off) {
rt_fatal("RomFS: CreateEvent failed (%lu)", GetLastError());
OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32), .hEvent = ev};
DWORD got = 0;
mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting
if (!ReadFile(f, buf, n, &got, &ov)) {
DWORD e = GetLastError();
if (e == ERROR_HANDLE_EOF) return 0;
if (e != ERROR_IO_PENDING) return -1;
if (!GetOverlappedResult(f, &ov, &got, TRUE)) return GetLastError() == ERROR_HANDLE_EOF ? 0 : -1;
if (e == ERROR_IO_PENDING && !GetOverlappedResult(f, &ov, &got, TRUE)) e = GetLastError();
else if (e == ERROR_IO_PENDING) return got;
if (e == ERROR_HANDLE_EOF) return 0;
if (e == ERROR_NOACCESS) // a read-only page in the buffer: a frozen region missed its thaw
rt_log("RomFS: read of %u bytes into %p refused: the buffer is not writable", n, buf);
return -1;
}
return got;
}
+26
View File
@@ -138,6 +138,30 @@ void vk_mem_count(int cls, int use, uint64_t bytes);
void vk_mem_bound_note(void *pool);
#define VK_MEM_BOUND(sp) do { if ((rt_trace & RT_TRACE_PERF) && (sp).pool) vk_mem_bound_note((sp).pool); } while (0)
void vk_mem_dirty_submit(void); // vk_sync_submit, under its lock, before the replay
// VRAM mirror of buffer-bound pools (vk_memory.c, build/mem-plan.md). vk_mirror_on: TOTK_MEMORY=mirror|verify, or the
// device class is discrete and TOTK_MEMORY is unset or auto; set at device selection. The replay (submitting thread
// under vk_sync_submit's lock) calls:
// - VK_MEM_USE at a vertex, index, uniform, storage or indirect binding of a span vk_mem_resolve returned: the pool's
// mirror is created on first use and sp.buffer becomes it (same offsets); a pool that stays imported keeps its buffer.
// - VK_MEM_PICK where a copy, fill or update reads or writes a span: the mirror if the pool has one, never a new one.
// - vk_mem_gpu_wrote after recording a GPU write of [sp.offset, sp.offset + size) into sp.buffer: a mirror's bytes go
// back to guest memory at the end of the submission. No-op for imported buffers.
// - vk_mem_mirror_begin at the start of a submission's recording, vk_mem_mirror_end (records the write-backs; returns
// the commands recorded) after its last barrier.
// vk_sync.c records the copies the mirrors' first accesses queued (vk_mem_mirror_pending) into a prologue command
// buffer with vk_mem_mirror_prologue and submits it right before each part.
extern int vk_mirror_on;
void vk_mem_use(VkMemSpan *sp);
void vk_mem_pick(VkMemSpan *sp);
#define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0)
#define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0)
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size);
void vk_mem_mirror_begin(void);
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence);
int vk_mem_mirror_pending(void);
void vk_mem_mirror_prologue(VkCommandBuffer cb);
int vk_mem_mirror_probe(uint64_t addr); // --probe-vkexec: the pool at addr gets its mirror as on a first buffer use;
// 1 when it has one
void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m); // once
int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked)
void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and,
@@ -223,6 +247,7 @@ void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); /
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads);
void vk_texture_shutdown(void); // vk_mem_shutdown calls it
void vk_texture_new_submit(void); // a queue submit starts: draw bindings check storage again
uint32_t vk_texture_epoch(void); // the write-watch snapshot epoch vk_texture_new_submit started
VkSampler vk_sampler_by_id(int id); // VK_NULL_HANDLE for an empty slot
VkSampler vk_sampler_for(const samp_par *s); // cached by content
int vk_sampler_params(int id, samp_par *out); // parameters of a sampler slot; 0 for an empty slot
@@ -237,6 +262,7 @@ void vk_sync_start(void); // vk_init (fatal without a device) a
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
uint64_t vk_sync_fence_done(void); // highest done value
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
void vk_sync_finish(void); // every submitted value done
void vk_sync_shutdown(void); // finish, stop the completion thread, free the command buffers
// The op-stream replay (vk_exec.c: T6c): records the command handles of one submission into cb, which the caller has
+1
View File
@@ -399,6 +399,7 @@ static void buf_snap(VkExec *ex, int pass) {
if (!bd_size) return;
VkMemSpan s;
if (!vk_mem_resolve(bd_addr, bd_size, &s) || s.avail < bd_size) return;
VK_MEM_PICK(s); // the GPU's view of the buffer
BufJob *j = calloc(1, sizeof *j);
if (!j) rt_fatal("vk_dump: out of memory");
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bd_size,
+2
View File
@@ -688,6 +688,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
vk_exec_frame.submits++;
split_draws = part_start = 0;
x_here = 1;
if (n) vk_mem_mirror_begin();
for (int i = 0; i < n; i++) {
if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i);
walk(ex, nvn_op_first(handles[i]), 0);
@@ -695,6 +696,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
vk_exec_pass_end(ex);
x_here = 0;
if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes
if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence); // VRAM mirror bytes the GPU wrote go back to guest memory
if (ex->nviews) {
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
if (!l) rt_fatal("vk: out of memory");
+19 -6
View File
@@ -569,16 +569,20 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) {
return (VkDescriptorBufferInfo){Z.zbuf, 0, max < ZERO_BYTES ? max : ZERO_BYTES};
}
// A buffer descriptor for [addr, addr+size) of guest memory: its backing when aligned, a copy in transient memory when
// the backing is host memory with an unaligned offset. 0 when neither works.
static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, VkDescriptorBufferInfo *out) {
// A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when
// aligned, a copy in transient memory when the backing is host memory with an unaligned offset. 0 when neither works.
// written: the shader stores into it (a mirror's range goes back to guest memory at the end of the submission).
static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, int written,
VkDescriptorBufferInfo *out) {
VkMemSpan sp;
if (!addr || !vk_mem_resolve(addr, 1, &sp)) return 0;
uint64_t n = size && size < sp.avail ? size : sp.avail;
if (n > max) n = max;
if (!n) return 0;
if (sp.offset % align == 0) {
VK_MEM_USE(sp);
*out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n};
if (written) vk_mem_gpu_wrote(&sp, n);
vk_mem_count(sp.cls, use, n);
VK_MEM_BOUND(sp);
return 1;
@@ -665,11 +669,12 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) {
uint32_t len, c1;
if (find_stage(ex->prog, st, &ctl, &code, &len)) {
memcpy(&c1, ctl + 0x700, 4);
if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, &bi)) return bi;
if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, 0, &bi))
return bi;
}
} else if (slot >= 3 && slot - 3 < 18) {
const p_bind_buf *b = &ex->ubo[st][slot - 3];
if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, &bi)) return bi;
if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, 0, &bi)) return bi;
}
return zero_info(max);
}
@@ -699,7 +704,8 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *
const p_bind_buf *b = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot];
// vk_caps.c: shaders translated for sbuf_align address the buffer from its address rounded down to it.
uint64_t lead = vk_caps.sbuf_align ? b->addr & (vk_caps.sbuf_align - 1) : 0;
if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO, &bi)) {
if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO,
written, &bi)) {
if (written && bi.buffer != VK_NULL_HANDLE) note_written(ex, b->addr - lead, bi.range);
return bi;
}
@@ -1095,6 +1101,7 @@ static void bind_vertex_buffers(VkExec *ex) {
const NVNbufferRange *r = &ex->vbuf[b];
VkMemSpan sp;
if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue;
VK_MEM_USE(sp);
uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail;
vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool;
}
@@ -1271,6 +1278,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) {
} else if (sp.offset % isize[d->itype]) {
return warn_once(W_INDEX, "index buffer %#llx is not aligned to its index size: draw skipped", (unsigned long long)d->ibuf);
} else {
VK_MEM_USE(sp);
vkCmdBindIndexBuffer(ex->cb, sp.buffer, sp.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
ex->recorded++;
vk_mem_count(sp.cls, VK_MUSE_IB, bytes);
@@ -1295,6 +1303,8 @@ static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) {
return warn_once(W_INDEX, "index buffer %#llx is not backed or not aligned: indirect draws skipped", (unsigned long long)d->ibuf);
if (!prepare_draw(ex, d->prim, 0x7fffffff, restart_for(ex, d->prim, d->itype))) return;
vk_exec_frame.draws++;
VK_MEM_USE(ip);
VK_MEM_USE(ib);
vkCmdBindIndexBuffer(ex->cb, ib.buffer, ib.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
vkCmdDrawIndexedIndirect(ex->cb, ip.buffer, ip.offset, 1, sizeof(VkDrawIndexedIndirectCommand));
ex->recorded += 2;
@@ -1341,6 +1351,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) {
VkPipeline p = vk_pipeline_compute(cs, &info);
if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped");
vk_exec_barrier(ex);
VK_MEM_USE(ip);
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
c0_dim[0] = c0_dim[1] = c0_dim[2] = 0;
B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads
@@ -1368,8 +1379,10 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) {
VkBuffer b;
VkDeviceSize o;
memcpy(scratch(ex, u->size, &b, &o), u->data, u->size);
VK_MEM_PICK(sp);
VkBufferCopy cp = {o, sp.offset, u->size};
vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp);
vk_mem_gpu_wrote(&sp, u->size);
ex->recorded++;
vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size);
note_written(ex, u->dst, u->size);
+55 -1
View File
@@ -802,6 +802,54 @@ static int test_nvnx(const GfxBackend *b, uint8_t *guest, int *bytes) {
return 0;
}
// VRAM mirror (vk_memory.c; skipped in import mode): a pool given its mirror as on a first buffer use, after which
// buffer copies and fills read and write the mirror. The first use copies the whole pool; CPU writes between
// submissions reach the mirror through write watch; GPU writes reach guest memory by the fence, and only the bytes
// written: CPU bytes beside a GPU fill keep their values.
static int test_mirror(const GfxBackend *b, uint8_t *guest, int *bytes) {
*bytes = 0;
if (!vk_mirror_on) return 0;
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x22, .mem = guest + 0xC00000, .size = 0x100000,
.game_mem = guest + 0xC00000};
CHECK(b->pool_init(&pool), "pool_init of a CPU-visible pool for the mirror part");
uint8_t *m = pool.mem;
for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 13 + 1);
memset(m + 0x1000, 0xEE, 0x5000);
CHECK(vk_mem_mirror_probe((uint64_t)(uintptr_t)m), "the pool got no VRAM mirror");
static _Alignas(8) uint8_t stream[1024];
uint64_t a = (uint64_t)(uintptr_t)m;
// 1: B2B in the mirror after its first-use copy, and a fill; both written back.
uint8_t *p = put(stream, OP_BEGIN, NULL, 0);
const p_copy_b2b c1 = {a, a + 0x1000, 4096};
const p_clear_buf f1 = {a + 0x2000, 0x800, 0x5a5a5a5au, 0};
p = put(p, OP_COPY_B2B, &c1, sizeof c1);
p = put(p, OP_CLEAR_BUF, &f1, sizeof f1);
CHECK(xf_submit(b, stream, p), "fence_wait of the first mirror submission timed out");
for (int i = 0; i < 4096; i++) CHECK(m[0x1000 + i] == m[i], "mirror B2B byte %d is %02x, expected %02x", i, m[0x1000 + i], m[i]);
for (int i = 0; i < 0x800; i++) CHECK(m[0x2000 + i] == 0x5a, "mirror fill byte %d is %02x", i, m[0x2000 + i]);
for (int i = 0x800; i < 0x1000; i++) CHECK(m[0x2000 + i] == 0xEE, "byte %#x past the mirror fill is %02x", i, m[0x2000 + i]);
// 2: the CPU rewrites the source; the next submission copies the new bytes.
for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 7 + 5);
p = put(stream, OP_BEGIN, NULL, 0);
const p_copy_b2b c2 = {a, a + 0x3000, 4096};
p = put(p, OP_COPY_B2B, &c2, sizeof c2);
CHECK(xf_submit(b, stream, p), "fence_wait of the second mirror submission timed out");
for (int i = 0; i < 4096; i++)
CHECK(m[0x3000 + i] == m[i], "B2B byte %d after a CPU write is %02x, expected %02x (write not mirrored)", i,
m[0x3000 + i], m[i]);
// 3: CPU bytes in the page of a GPU fill, written before the submission, survive the write-back.
memset(m + 0x4000, 0x11, 0x800);
p = put(stream, OP_BEGIN, NULL, 0);
const p_clear_buf f3 = {a + 0x4800, 0x800, 0x77777777u, 0};
p = put(p, OP_CLEAR_BUF, &f3, sizeof f3);
CHECK(xf_submit(b, stream, p), "fence_wait of the third mirror submission timed out");
for (int i = 0; i < 0x800; i++) CHECK(m[0x4000 + i] == 0x11, "CPU byte %#x beside a fill is %02x", i, m[0x4000 + i]);
for (int i = 0; i < 0x800; i++) CHECK(m[0x4800 + i] == 0x77, "fill byte %#x is %02x", i, m[0x4800 + i]);
b->pool_fini(&pool);
*bytes = 4096 + 0x1000 + 4096 + 0x1000;
return 0;
}
int probe_vkexec(void) {
layouts_equal = 0;
main_tid = GetCurrentThreadId();
@@ -819,8 +867,9 @@ int probe_vkexec(void) {
if (!bad) bad = test_xfer(b, guest, &xfer_bytes);
int zero_bytes = 0;
if (!bad) bad = test_c0() || test_zero(b, guest, &zero_bytes);
int nvnx_bytes = 0;
int nvnx_bytes = 0, mirror_bytes = 0;
if (!bad) bad = test_nvnx(b, guest, &nvnx_bytes);
if (!bad) bad = test_mirror(b, guest, &mirror_bytes);
b->finish();
vk_pipeline_shutdown(); // the pipelines and the draw part's stand-ins go before the device
vk_exec_draw_shutdown();
@@ -860,5 +909,10 @@ int probe_vkexec(void) {
printf("probe-vkexec: c0 and zero parts passed (c0 tables, dispatch size, point size and base instance at their offsets, "
"unbound slots zero; %d bytes of an unwritten physical-pool texture read zero, TOTK_FIX_ZERO=%s)\n",
zero_bytes, cfg_get("TOTK_FIX_ZERO") ? cfg_get("TOTK_FIX_ZERO") : "1");
if (vk_mirror_on)
printf("probe-vkexec: mirror part passed (B2B and fill in a VRAM mirror written back by the fence, a CPU write "
"between submissions mirrored, CPU bytes beside a fill kept, %d bytes compared)\n", mirror_bytes);
else
printf("probe-vkexec: mirror part skipped (memory mode import)\n");
return 0;
}
+16 -11
View File
@@ -379,9 +379,10 @@ static VkExtent3D sub_extent(const Sub *s) {
return (VkExtent3D){s->w, s->h, s->img->type == VK_IMAGE_TYPE_3D ? s->nz : 1};
}
// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span.
// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span (a VRAM mirror when the pool
// has one).
static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *size, VkBufferImageCopy *c,
VkBuffer *buf, int *cls) {
VkMemSpan *span) {
uint32_t wb = (s->w + (uint32_t)s->bw - 1) / (uint32_t)s->bw, hb = (s->h + (uint32_t)s->bh - 1) / (uint32_t)s->bh;
uint64_t row = ex->copy_stride > 0 ? (uint64_t)ex->copy_stride : (uint64_t)wb * (uint64_t)s->bpb;
if (row % (uint64_t)s->bpb || row < (uint64_t)wb * (uint64_t)s->bpb) {
@@ -397,8 +398,8 @@ static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *
(unsigned long long)addr, (unsigned long long)*size);
return 0;
}
*buf = m.buffer;
*cls = m.cls;
VK_MEM_PICK(m);
*span = m;
*c = (VkBufferImageCopy){.bufferOffset = m.offset, .bufferRowLength = (uint32_t)(row / (uint64_t)s->bpb) * (uint32_t)s->bw,
.bufferImageHeight = hb * (uint32_t)s->bh, .imageSubresource = sub_layers(s),
.imageOffset = sub_offset(s), .imageExtent = sub_extent(s)};
@@ -410,17 +411,16 @@ static void copy_buffer_image(VkExec *ex, const p_copy_b2t *p, int to_image) {
if (!sub_resolve(p->tex, &p->view, p->has_view, &p->r, 1, &s)) return;
uint64_t size;
VkBufferImageCopy c;
VkBuffer buf;
int cls;
if (!buffer_side(ex, &s, p->src, &size, &c, &buf, &cls)) return;
vk_mem_count(cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size);
VkMemSpan m;
if (!buffer_side(ex, &s, p->src, &size, &c, &m)) return;
vk_mem_count(m.cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size);
vk_exec_pass_end(ex);
vk_exec_barrier(ex);
if (to_image) vkCmdCopyBufferToImage(ex->cb, buf, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c);
else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, buf, 1, &c);
if (to_image) vkCmdCopyBufferToImage(ex->cb, m.buffer, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c);
else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, m.buffer, 1, &c);
ex->recorded++;
if (to_image) vk_texture_gpu_written(p->tex);
else note_gpu_write(ex, p->src, size);
else vk_mem_gpu_wrote(&m, size), note_gpu_write(ex, p->src, size);
}
static void copy_t2t(VkExec *ex, const p_copy_t2t *p) {
@@ -476,6 +476,8 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) {
}
uint64_t n = left < s.avail ? left : s.avail;
if (n > d.avail) n = d.avail;
VK_MEM_PICK(s);
VK_MEM_PICK(d);
if (s.buffer == d.buffer && s.offset < d.offset + n && d.offset < s.offset + n) {
warn_once(W_OVERLAP, "buffer copy %#llx -> %#llx overlaps within one buffer: skipped",
(unsigned long long)p->src, (unsigned long long)p->dst);
@@ -484,6 +486,7 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) {
if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1;
VkBufferCopy bc = {s.offset, d.offset, n};
vkCmdCopyBuffer(ex->cb, s.buffer, d.buffer, 1, &bc);
vk_mem_gpu_wrote(&d, n);
ex->recorded++;
vk_mem_count(s.cls, VK_MUSE_COPY_SRC, n);
vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n);
@@ -514,7 +517,9 @@ static void clear_buf(VkExec *ex, const p_clear_buf *p) {
uint64_t n = left < d.avail ? left : d.avail & ~3ull;
if (!n) break;
if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1;
VK_MEM_PICK(d);
vkCmdFillBuffer(ex->cb, d.buffer, d.offset, n, p->value);
vk_mem_gpu_wrote(&d, n);
ex->recorded++;
vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n);
dst += n, left -= n;
+693 -40
View File
@@ -22,8 +22,26 @@
#include "nvn_pool_index.h"
#include "vk.h"
// Raw performance counter ticks for the tracking cost (no division on the replay's path); qpc_ms converts.
static uint64_t qpc(void) {
LARGE_INTEGER t;
QueryPerformanceCounter(&t);
return (uint64_t)t.QuadPart;
}
static double qpc_ms(uint64_t ticks) {
static double per_ms;
if (!per_ms) {
LARGE_INTEGER f;
QueryPerformanceFrequency(&f);
per_ms = (double)f.QuadPart / 1000.0;
}
return (double)ticks / per_ms;
}
// ---------------------------------------------------------------- pools
enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT };
enum { MS_NONE, MS_ON, MS_REFUSED };
typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page -> physical pool and offset
@@ -44,8 +62,22 @@ typedef struct PoolRec {
uint32_t synced_gen; // page generation its lazy-mirror count covers; DIRTY_UNSYNCED until scanned
uint8_t bufbound; // ever bound as a vertex, index, uniform, storage or indirect buffer
uint8_t untracked; // the last dirty scan found no write watch over it
// VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only.
uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED
VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf
VmaAllocation malloc_;
uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync)
uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory
uint32_t mgen; // gen_lock: every page write stamped up to this generation is in the mirror
// (or in a copy queued before its next use)
uint64_t fl_lo, fl_hi; // flush_lock: nvnMemoryPoolFlushMappedRange bytes not copied yet (fl_hi 0: none)
} PoolRec;
static void mirror_forget(PoolRec *r); // pool_lock exclusive: off the mirrored list
static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown)
static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record
static void grow(void **arr, int *cap, size_t elem);
static SRWLOCK pool_lock = SRWLOCK_INIT;
// Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits
// stay valid while it is unchanged.
@@ -197,6 +229,7 @@ static PoolRec *phys_find(const pool_t *key) {
}
static void destroy_rec(VkCtx *c, PoolRec *r) {
if (r->mbuf) vmaDestroyBuffer(c->vma, r->mbuf, r->malloc_);
if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL);
if (r->mem) vkFreeMemory(c->device, r->mem, NULL);
if (r->alloc) vmaFreeMemory(c->vma, r->alloc);
@@ -226,6 +259,7 @@ int vk_pool_init(pool_t *p) {
ok = alloc_physical(c, r);
} else {
r->kind = POOL_IMPORT;
mem_thaw((uintptr_t)p->mem, p->size); // the import may need write access to the pages
ok = import_pool(c, r, (uintptr_t)p->mem, p->size);
if (!ok) {
// Not fatal for the game: the pool works for the CPU, and vk_mem_resolve reports its addresses as unbacked.
@@ -259,6 +293,7 @@ void vk_pool_fini(pool_t *p) {
int had = pidx_remove(&pool_index, p, &e);
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
PoolRec *r = had ? e.buf : NULL;
if (r && r->mstate == MS_ON) mirror_forget(r);
if (r && r->kind == POOL_PHYS) {
for (int i = 0; i < nphys; i++)
if (phys_list[i] == r) { phys_list[i] = phys_list[--nphys]; break; }
@@ -271,6 +306,7 @@ void vk_pool_fini(pool_t *p) {
}
ReleaseSRWLockExclusive(&pool_lock);
if (!r) return;
if (r->kind == POOL_IMPORT) mem_thaw(r->lo, r->size); // the storage goes back to the guest writable
// Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the
// guest gets the storage back when this returns.
vk_texture_range_gone(r->lo, r->size, r->alloc);
@@ -281,7 +317,10 @@ void vk_pool_fini(pool_t *p) {
vk_queue_unlock();
}
vk_mem_collect();
destroy_rec(c, r);
// A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may
// name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued).
if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r);
else destroy_rec(c, r);
}
// Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c:54-67); apply them in order.
@@ -414,9 +453,13 @@ static void flag_names(uint32_t f, char *buf, size_t n) {
// run-game saves), at frame 600, and with TOTK_TRACE=perf every 1800 frames.
static void dirty_frame(uint64_t frame);
static void mirror_log(uint64_t frame);
void vk_mem_census(uint64_t frame) {
static uint64_t window_start;
if (rt_trace & RT_TRACE_PERF) dirty_frame(frame);
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame); // and the
// write-watch line
if (frame == 1200) { // loading is over: the first window starts here
for (int c = 0; c < VK_MCLS_N; c++)
for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0);
@@ -497,6 +540,36 @@ static uint32_t *gen_chunk[GEN_CHUNKS];
static uint32_t gen_now; // last generation handed out
static SRWLOCK gen_lock = SRWLOCK_INIT;
static uint64_t gen_pages[1024];
// The newest generation stamped on any page of each 2 MiB region (the write-watch snapshot regions below): the mirror
// sync skips regions with nothing newer than its pool's mirror.
#define REG_SHIFT 21
static uint32_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT];
// Every write-watch query of the generation table (gen_lock held exclusively): calls, bytes and time since the last
// mirror summary, all consumers together; the queries that found written pages (they also flush TLBs) apart.
static uint64_t ww_calls, ww_bytes, ww_ticks, ww_dirty_calls, ww_dirty_ticks, ww_pages;
static uint64_t ww_query(uint64_t addr, uint64_t size) {
uint64_t t0 = qpc();
uint64_t n = mem_written_pages(addr, size, gen_pages, sizeof gen_pages / sizeof *gen_pages);
uint64_t dt = qpc() - t0;
ww_ticks += dt, ww_calls++, ww_bytes += size;
if (n && n != UINT64_MAX) ww_dirty_calls++, ww_dirty_ticks += dt, ww_pages += n;
return n;
}
// Frozen regions (mirror mode): a 2 MiB region whose pages nothing wrote for freeze_idle submissions, and which lies
// wholly in imported pools, is made read-only (memory.c mem_freeze), and write-watch queries skip it until a write or a
// file read thaws it. Most of the pool memory the mirror and the texture checks cover is static (meshes, textures), and
// a query costs about 1 us per MiB, so this keeps the queries to the regions the CPU writes.
#define FREEZE_PER_SUB 32
static int freeze_idle; // 0: off
static uint32_t ww_ep = 1; // gen_lock: the submission being recorded (vk_mem_mirror_begin)
static uint32_t reg_last[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of the region's last stamp
static uint32_t reg_tried[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of its last freeze candidacy
static uint32_t *fcand; // gen_lock: regions to try at the next freeze pass
static int nfcand, cfcand;
static uint64_t n_frz_nopool, n_frz_refused; // gen_lock: candidates not wholly in pools, refused by mem_freeze
static uint32_t *gen_slot(uint64_t page_addr, int create) {
uint64_t ci = page_addr >> GEN_CHUNK_SHIFT;
@@ -509,11 +582,17 @@ static uint32_t *gen_slot(uint64_t page_addr, int create) {
return &gen_chunk[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)];
}
// gen_lock held exclusively; g is the newest generation handed out.
static void stamp_page(uint64_t a, uint32_t g) {
uint32_t *s = gen_slot(a, 1);
if (!s) return;
*s = g;
reg_max[a >> REG_SHIFT] = g;
reg_last[a >> REG_SHIFT] = ww_ep;
}
static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) {
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) {
uint32_t *s = gen_slot(a, 1);
if (s) *s = g;
}
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page(a, g);
}
void vk_guest_written(uint64_t addr, uint64_t size) {
@@ -529,14 +608,12 @@ int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now)
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
AcquireSRWLockExclusive(&gen_lock);
uint32_t g = ++gen_now;
int tracked = 1;
for (;;) {
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
int tracked = 1, frozen = 1;
for (uint64_t a = lo & ~((1ull << REG_SHIFT) - 1); a < hi && frozen; a += 1ull << REG_SHIFT) frozen = mem_frozen(a);
while (!frozen) {
uint64_t n = ww_query(lo, hi - lo);
if (n == UINT64_MAX) { tracked = 0; break; }
for (uint64_t k = 0; k < n; k++) {
uint32_t *s = gen_slot(gen_pages[k], 1);
if (s) *s = g;
}
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call
}
int changed = 0;
@@ -560,6 +637,44 @@ static uint32_t snap_at[SNAP_REGIONS]; // epoch of the region's last snapsh
static uint8_t snap_untracked[SNAP_REGIONS];
static uint64_t n_snap_checks, n_snap_calls, n_snap_untracked;
// gen_lock held exclusively. Queries the regions r0..r1 that were not snapped in epoch, one query per run of them;
// returns the number of queries.
static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) {
uint32_t g = 0;
uint64_t calls = 0;
for (uint64_t r = r0; r <= r1;) {
if (snap_at[r] == epoch) { r++; continue; }
if (mem_frozen(r << SNAP_SHIFT)) { // nothing in it changed since its freeze
snap_at[r] = epoch, snap_untracked[r] = 0;
r++;
continue;
}
uint64_t k = r; // one query over a run of regions not seen in this epoch
while (k + 1 <= r1 && snap_at[k + 1] != epoch && !mem_frozen((k + 1) << SNAP_SHIFT)) k++;
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
int tracked = 1;
if (!g) g = ++gen_now;
for (;;) {
uint64_t n = ww_query(a, b - a);
if (n == UINT64_MAX) { tracked = 0; break; }
for (uint64_t j = 0; j < n; j++) stamp_page(gen_pages[j], g);
if (n < sizeof gen_pages / sizeof *gen_pages) break;
}
calls++;
for (uint64_t j = r; j <= k; j++) {
snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
if (tracked && freeze_idle && ww_ep - reg_last[j] >= (uint32_t)freeze_idle &&
ww_ep - reg_tried[j] >= (uint32_t)freeze_idle) {
reg_tried[j] = ww_ep;
if (nfcand == cfcand) grow((void **)&fcand, &cfcand, sizeof *fcand);
fcand[nfcand++] = (uint32_t)j;
}
}
r = k + 1;
}
return calls;
}
int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) {
if (!size) { *now = since; return 0; }
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
@@ -572,26 +687,7 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e
if (excl) {
ReleaseSRWLockShared(&gen_lock);
AcquireSRWLockExclusive(&gen_lock);
uint32_t g = ++gen_now;
for (uint64_t r = r0; r <= r1;) {
if (snap_at[r] == epoch) { r++; continue; }
uint64_t k = r; // one query over a run of regions not seen in this epoch
while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++;
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
int tracked = 1;
for (;;) {
uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages);
if (n == UINT64_MAX) { tracked = 0; break; }
for (uint64_t j = 0; j < n; j++) {
uint32_t *s = gen_slot(gen_pages[j], 1);
if (s) *s = g;
}
if (n < sizeof gen_pages / sizeof *gen_pages) break;
}
n_snap_calls++;
for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
r = k + 1;
}
n_snap_calls += snap_regions(r0, r1, epoch);
}
int untracked = 0, changed = 0;
for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r];
@@ -661,18 +757,15 @@ static uint64_t page_hi(const PoolRec *r) { return (r->lo + r->size + 0xfff) & ~
// gen_lock held exclusively. 0: the range is not write-watched.
static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
for (;;) {
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
uint64_t n = ww_query(lo, hi - lo);
if (n == UINT64_MAX) return 0;
for (uint64_t k = 0; k < n; k++) {
uint32_t *s = gen_slot(gen_pages[k], 1);
if (s) *s = g;
}
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
if (n < sizeof gen_pages / sizeof *gen_pages) return 1;
}
}
void vk_mem_dirty_submit(void) {
if (!(rt_trace & RT_TRACE_PERF)) return;
if (!(rt_trace & RT_TRACE_PERF) || vk_mirror_on) return; // the mirror's own line replaces the census
uint64_t t0 = kern_now_ns();
uint32_t prev = dirty_sub; // the submit whose replay noted the bindings
dirty_sub = prev + 1 ? prev + 1 : 1;
@@ -793,8 +886,563 @@ static void dirty_frame(uint64_t frame) {
rt_log("%s", line);
}
// ---------------------------------------------------------------- VRAM mirror of buffer-bound pools
// build/mem-plan.md, "Design: VRAM mirror of buffer-bound pools". An imported pool keeps vertices, indices, constants
// and storage buffers in system memory, so a discrete GPU reads them over PCIe on every draw (GTX 1050 Ti and RTX
// 3090 Ti: about 110 ms of GPU time per frame in the field). In mirror mode (device class discrete, TOTK_MEMORY) an
// imported pool gets a device-local buffer over its imported range the first time the replay uses it as a vertex,
// index, uniform, storage or indirect buffer (vk_mem_use). From then on those bindings, and the buffer copies, fills
// and uniform updates that touch the pool (vk_mem_pick), use the mirror at the same offsets. Guest memory stays what
// the CPU reads and writes; textures and every host path that reads or writes pool bytes keep using it.
// - CPU -> mirror: the first access of a submission to a mirror syncs it (mirror_sync): the 2 MiB write-watch
// snapshots of its regions for this epoch (snap_regions, shared with the texture checks: the OS bits keep one
// consumer, the shared page generations) and every page stamped after the mirror's mgen goes into a copy; so do the
// nvnMemoryPoolFlushMappedRange ranges. The first use copies the whole range. The copies run in a prologue command
// buffer that vk_sync.c submits right before the part that accessed the mirror first, between full barriers.
// vk_mem_mirror_begin snaps the regions of the pools the previous submission accessed in a few merged queries.
// - GPU -> guest: what the replay writes into a mirror (storage buffers its shaders write, buffer copies, fills,
// uniform updates, texture-to-buffer copies: vk_mem_gpu_wrote) goes back into the imported buffer at the end of the
// submission, exact byte ranges, before the fence signals (vk_mem_mirror_end).
// - TOTK_MEMORY=verify: every submission also copies up to VERIFY_BYTES of mirror pages, round robin, into host memory;
// at its fence the pages the CPU has not written since their sync are compared with guest memory.
// A pool stays imported when its mirror would not fit the VRAM budget, its range is not write-watched, it overlaps
// another pool or the allocation fails.
#define VERIFY_BYTES (4u << 20)
#define VERIFY_SAMPLES 64
#define VERIFY_LOG 16
enum { MR_BUDGET, MR_UNTRACKED, MR_OVERLAP, MR_ALLOC, MR_N };
int vk_mirror_on;
static int mirror_verify;
static int mirror_gap = 2; // pre-snap: hot pools at most this many 2 MiB regions apart share one query
static PoolRec **mlist; // pool_lock: mirrored pools by import_lo
static int nm, mcap;
static uint32_t msub; // replay: number of the submission being recorded
static uint32_t mepoch; // replay: its write-watch snapshot epoch (vk_texture_epoch)
typedef struct { PoolRec *r; uint64_t off, size; } MRange;
static MRange *pend, *wbl; // replay: copies for the next prologue; GPU writes into mirrors this submission
static int npend, cpend, nwbl, cwbl;
static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions
static int cregions;
static SRWLOCK flush_lock = SRWLOCK_INIT;
typedef struct {
uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges;
uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes;
} MirrorAcc;
static MirrorAcc mrep; // replay: this submission
static MirrorAcc macc; // mirror_stat_lock: since the last summary
static SRWLOCK mirror_stat_lock = SRWLOCK_INIT;
static volatile LONG64 m_live_n, m_live_bytes, m_refused_n[MR_N], m_refused_bytes[MR_N];
static volatile LONG64 m_flush_calls, m_flush_bytes, v_pages, v_skipped, v_mismatch;
static void mirror_mode(int discrete) {
const char *e = cfg_get("TOTK_MEMORY");
int mode = discrete ? 1 : 0; // 0 import, 1 mirror, 2 verify
const char *why = discrete ? "auto, discrete gpu" : "auto, uma";
if (e && *e && strcmp(e, "auto")) {
if (!strcmp(e, "import")) mode = 0, why = "TOTK_MEMORY=import";
else if (!strcmp(e, "mirror")) mode = 1, why = "TOTK_MEMORY=mirror";
else if (!strcmp(e, "verify")) mode = 2, why = "TOTK_MEMORY=verify";
else rt_log("mem: TOTK_MEMORY=%s is not import, mirror, verify or auto: auto", e);
}
vk_mirror_on = mode > 0;
mirror_verify = mode == 2;
const char *g = cfg_get("TOTK_MIRROR_GAP");
mirror_gap = g && *g ? atoi(g) : 2;
if (mirror_gap < 0) mirror_gap = 0;
const char *fi = cfg_get("TOTK_FREEZE_IDLE"); // submissions a region stays unwritten before it is frozen
freeze_idle = !vk_mirror_on ? 0 : fi && *fi ? atoi(fi) : 300;
if (freeze_idle < 0) freeze_idle = 0;
rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why,
mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory");
}
static void range_add(MRange **a, int *n, int *cap, PoolRec *r, uint64_t off, uint64_t size) {
if (*n && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) {
(*a)[*n - 1].size += size;
return;
}
if (*n == *cap) grow((void **)a, cap, sizeof **a);
(*a)[(*n)++] = (MRange){r, off, size};
}
static int range_cmp(const void *x, const void *y) {
const MRange *a = x, *b = y;
if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1;
return a->off < b->off ? -1 : a->off > b->off;
}
// Records the ranges as one vkCmdCopyBuffer per pool, overlapping and adjacent ranges merged; to_mirror: buf -> mbuf,
// else mbuf -> buf. Returns the commands recorded; adds what they move to *bytes and *copies.
static int copy_ranges(VkCommandBuffer cb, MRange *a, int n, int to_mirror, uint64_t *bytes, uint64_t *copies) {
qsort(a, (size_t)n, sizeof *a, range_cmp);
int rec = 0;
for (int i = 0; i < n;) {
PoolRec *r = a[i].r;
uint64_t lim = r->import_hi - r->import_lo;
int k = 0;
for (; i < n && a[i].r == r; i++) {
uint64_t lo = a[i].off, hi = a[i].off + a[i].size;
if (hi > lim) hi = lim;
if (lo >= hi) continue;
if (k && lo <= mregions[k - 1].srcOffset + mregions[k - 1].size) {
uint64_t end = mregions[k - 1].srcOffset + mregions[k - 1].size;
if (hi > end) mregions[k - 1].size += hi - end;
continue;
}
if (k == cregions) grow((void **)&mregions, &cregions, sizeof *mregions);
mregions[k++] = (VkBufferCopy){lo, lo, hi - lo};
}
if (!k) continue;
if (to_mirror) vkCmdCopyBuffer(cb, r->buf, r->mbuf, (uint32_t)k, mregions);
else vkCmdCopyBuffer(cb, r->mbuf, r->buf, (uint32_t)k, mregions);
rec++;
for (int j = 0; j < k; j++) *bytes += mregions[j].size;
*copies += (uint64_t)k;
}
return rec;
}
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
VkAccessFlags2 da) {
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = ss, .srcAccessMask = sa,
.dstStageMask = ds, .dstAccessMask = da};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(cb, &di);
}
static void refuse(PoolRec *r, int why, const char *what) {
r->mstate = MS_REFUSED;
uint64_t n = r->import_hi - r->import_lo;
if (InterlockedIncrement64(&m_refused_n[why]) <= 4) {
char f[160];
flag_names(r->flags, f, sizeof f);
rt_log("mem: pool %#llx+%#llx (%s) stays imported: %s", (unsigned long long)r->lo, (unsigned long long)r->size,
f, what);
}
InterlockedAdd64(&m_refused_bytes[why], (LONG64)n);
}
// Replay thread: the pool's mirror on its first buffer use.
static void mirror_create(PoolRec *r) {
VkCtx *c = vk_ctx();
uint64_t n = r->import_hi - r->import_lo;
r->mstate = MS_REFUSED;
if (r->kind != POOL_IMPORT || !r->buf || !n) return;
AcquireSRWLockShared(&pool_lock);
int i = pidx_upper(&pool_index, r->lo) - 1;
while (i >= 0 && pool_index.e[i].buf != r) i--;
int alone = i >= 0 && (i == 0 || pool_index.maxhi[i - 1] <= pool_index.e[i].lo) &&
(i + 1 == pool_index.n || pool_index.e[i + 1].lo >= pool_index.e[i].hi);
ReleaseSRWLockShared(&pool_lock);
if (i < 0) return; // finalized meanwhile
if (!alone) return refuse(r, MR_OVERLAP, "it overlaps another pool");
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
if (r1 >= SNAP_REGIONS) return refuse(r, MR_UNTRACKED, "outside the write-watched address space");
uint64_t used, budget;
int exact;
if (vk_mem_vram(&used, &budget, &exact) && used + n + budget / 8 > budget) {
char why[128];
snprintf(why, sizeof why, "its %.1f MiB mirror does not fit the VRAM budget (%.0f of %.0f MiB in use)",
n / 1048576.0, used / 1048576.0, budget / 1048576.0);
return refuse(r, MR_BUDGET, why);
}
AcquireSRWLockExclusive(&gen_lock);
mrep.sync_queries += snap_regions(r0, r1, mepoch);
int untracked = 0;
for (uint64_t k = r0; k <= r1; k++) untracked |= snap_untracked[k];
r->mgen = gen_now; // the whole-range copy below holds every write observed so far
ReleaseSRWLockExclusive(&gen_lock);
if (untracked) return refuse(r, MR_UNTRACKED, "its range is not write-watched");
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = n, .usage = pool_usage(c),
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
VkResult res = vmaCreateBuffer(c->vma, &bi, &ai, &r->mbuf, &r->malloc_, NULL);
if (res != VK_SUCCESS) {
r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL;
char why[96];
snprintf(why, sizeof why, "device-local allocation of %.1f MiB failed (%d)", n / 1048576.0, (int)res);
return refuse(r, MR_ALLOC, why);
}
AcquireSRWLockExclusive(&pool_lock);
if (nm == mcap) grow((void **)&mlist, &mcap, sizeof *mlist);
int at = nm;
while (at > 0 && mlist[at - 1]->import_lo > r->import_lo) at--;
memmove(mlist + at + 1, mlist + at, (size_t)(nm - at) * sizeof *mlist);
mlist[at] = r;
nm++;
ReleaseSRWLockExclusive(&pool_lock);
r->acc_sub = msub;
r->mstate = MS_ON;
range_add(&pend, &npend, &cpend, r, 0, n);
mrep.first_n++, mrep.first_bytes += n;
InterlockedIncrement64(&m_live_n);
InterlockedAdd64(&m_live_bytes, (LONG64)n);
}
static void mirror_forget(PoolRec *r) {
for (int i = 0; i < nm; i++)
if (mlist[i] == r) {
memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist);
nm--;
InterlockedDecrement64(&m_live_n);
InterlockedAdd64(&m_live_bytes, -(LONG64)(r->import_hi - r->import_lo));
break;
}
}
// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran.
static PoolRec **retiring;
static int nretiring, cretiring;
static SRWLOCK retire_lock = SRWLOCK_INIT;
static void mirror_retiring(PoolRec *r) {
AcquireSRWLockExclusive(&retire_lock);
if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring);
retiring[nretiring++] = r;
ReleaseSRWLockExclusive(&retire_lock);
}
static void mirror_retire(void *arg) {
AcquireSRWLockExclusive(&retire_lock);
for (int i = 0; i < nretiring; i++)
if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; }
ReleaseSRWLockExclusive(&retire_lock);
destroy_rec(vk_ctx(), arg);
}
// Replay thread, the first access of this submission: queues the copies of the pages written since the last sync.
static void mirror_sync(PoolRec *r) {
uint64_t t0 = qpc();
r->acc_sub = msub;
uint64_t lo = r->import_lo, hi = r->import_hi;
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
int excl = 0;
AcquireSRWLockShared(&gen_lock);
for (uint64_t k = r0; k <= r1; k++)
if (snap_at[k] != mepoch) { excl = 1; break; }
if (excl) {
ReleaseSRWLockShared(&gen_lock);
AcquireSRWLockExclusive(&gen_lock);
mrep.sync_queries += snap_regions(r0, r1, mepoch);
}
uint32_t since = r->mgen;
for (uint64_t k = r0; k <= r1; k++) {
if (reg_max[k] <= since) continue;
uint64_t a = k << SNAP_SHIFT, b = a + (1ull << SNAP_SHIFT);
if (a < lo) a = lo;
if (b > hi) b = hi;
const uint32_t *ch = gen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk
if (!ch) continue;
for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT)
if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since)
range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT);
}
r->mgen = gen_now;
if (excl) ReleaseSRWLockExclusive(&gen_lock);
else ReleaseSRWLockShared(&gen_lock);
if (__atomic_load_n(&r->fl_hi, __ATOMIC_ACQUIRE)) {
AcquireSRWLockExclusive(&flush_lock);
uint64_t a = r->fl_lo, b = r->fl_hi;
r->fl_hi = 0;
ReleaseSRWLockExclusive(&flush_lock);
a = a > lo ? a & ~0xfffull : lo;
b = b < hi ? (b + 0xfff) & ~0xfffull : hi;
if (a < b) range_add(&pend, &npend, &cpend, r, a - lo, b - a);
}
mrep.syncs++;
mrep.sync_t += qpc() - t0;
}
void vk_mem_use(VkMemSpan *sp) {
PoolRec *r = sp->pool;
if (r->mstate == MS_NONE) mirror_create(r);
if (r->mstate != MS_ON) return;
if (r->acc_sub != msub) mirror_sync(r);
sp->buffer = r->mbuf;
}
void vk_mem_pick(VkMemSpan *sp) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON) return;
if (r->acc_sub != msub) mirror_sync(r);
sp->buffer = r->mbuf;
}
int vk_mem_mirror_probe(uint64_t addr) {
VkMemSpan sp;
if (!vk_mirror_on || !vk_mem_resolve(addr, 1, &sp) || !sp.pool) return 0;
mepoch = vk_texture_epoch();
vk_mem_use(&sp);
return ((PoolRec *)sp.pool)->mstate == MS_ON;
}
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return;
range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size);
}
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) {
if (!vk_mirror_on || !size || !p->mem) return;
InterlockedIncrement64(&m_flush_calls);
InterlockedAdd64(&m_flush_bytes, (LONG64)size);
AcquireSRWLockShared(&pool_lock);
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)p->mem, 0);
PoolRec *r = e && e->key == p ? e->buf : NULL;
if (r && r->mstate == MS_ON) {
uint64_t a = (uintptr_t)p->mem + off, b = a + size;
AcquireSRWLockExclusive(&flush_lock);
if (!r->fl_hi) r->fl_lo = a, r->fl_hi = b;
else {
if (a < r->fl_lo) r->fl_lo = a;
if (b > r->fl_hi) r->fl_hi = b;
}
ReleaseSRWLockExclusive(&flush_lock);
}
ReleaseSRWLockShared(&pool_lock);
}
// pool_lock shared: [a, b) lies wholly in the host ranges of imported pools with a GPU buffer.
static int pools_cover(uint64_t a, uint64_t b) {
uint64_t lo[64], hi[64];
int n = 0;
for (int i = pidx_upper(&pool_index, b - 1) - 1; i >= 0 && pool_index.maxhi[i] > a; i--) {
const PoolRec *r = pool_index.e[i].buf;
if (r->kind != POOL_IMPORT || !r->buf || r->import_hi <= a || r->import_lo >= b) continue;
if (n == 64) return 0;
int at = n++;
for (; at > 0 && lo[at - 1] > r->import_lo; at--) lo[at] = lo[at - 1], hi[at] = hi[at - 1];
lo[at] = r->import_lo, hi[at] = r->import_hi;
}
uint64_t end = a;
for (int i = 0; i < n && end < b; i++) {
if (lo[i] > end) return 0;
if (hi[i] > end) end = hi[i];
}
return end >= b;
}
// pool_lock shared, gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the
// protection collects the writes made before it took effect.
static void freeze_pass(void) {
for (int done = 0; nfcand && done < FREEZE_PER_SUB;) {
uint64_t j = fcand[--nfcand], a = j << REG_SHIFT, b = a + (1ull << REG_SHIFT);
if (ww_ep - reg_last[j] < (uint32_t)freeze_idle || mem_frozen(a)) continue;
if (!pools_cover(a, b)) { n_frz_nopool++; continue; }
done++;
if (!mem_freeze(a)) { n_frz_refused++; continue; }
query_stamp(a, b, ++gen_now);
}
}
void vk_mem_mirror_begin(void) {
if (!vk_mirror_on) return;
uint64_t t0 = qpc();
uint32_t prev = msub;
msub = msub + 1 ? msub + 1 : 1;
mepoch = vk_texture_epoch();
uint64_t q = 0, run0 = 0, run1 = 0;
int have = 0;
AcquireSRWLockShared(&pool_lock);
AcquireSRWLockExclusive(&gen_lock);
ww_ep++;
uint64_t b0 = ww_bytes;
for (int i = 0; i < nm; i++) {
const PoolRec *r = mlist[i];
if (r->acc_sub != prev) continue;
uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT;
if (have && a <= run1 + 1 + (uint64_t)mirror_gap) {
if (b > run1) run1 = b;
continue;
}
if (have) q += snap_regions(run0, run1, mepoch);
run0 = a, run1 = b, have = 1;
}
if (have) q += snap_regions(run0, run1, mepoch);
mrep.presnap_bytes += ww_bytes - b0;
if (freeze_idle) freeze_pass();
ReleaseSRWLockExclusive(&gen_lock);
ReleaseSRWLockShared(&pool_lock);
mrep.subs++;
mrep.presnap_queries += q;
mrep.presnap_t += qpc() - t0;
}
int vk_mem_mirror_pending(void) { return npend; }
void vk_mem_mirror_prologue(VkCommandBuffer cb) {
if (!npend) return;
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies);
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
npend = 0;
mrep.prologues++;
}
// ---- verify mode
typedef struct { PoolRec *r; uint64_t off, len, dst; uint32_t mgen; } VSample;
typedef struct { uint32_t sub; int n; VkBuffer buf; VmaAllocation alloc; const uint8_t *map; VSample s[VERIFY_SAMPLES]; } VJob;
static int ver_i;
static uint64_t ver_off;
// The CPU wrote the page since the mirror's sync that mgen records: stamped later, or its write-watch bit is set.
static int page_dirty(uint64_t a, uint32_t mgen) {
uint32_t *s = gen_slot(a, 0);
if (s && __atomic_load_n(s, __ATOMIC_RELAXED) > mgen) return 1;
uint64_t p[1];
return mem_written_peek(a, 1ull << GEN_PAGE_SHIFT, p, 1) != 0; // UINT64_MAX (not watched) counts as written
}
static void verify_cb(void *arg) { // completion thread
VJob *j = arg;
static volatile LONG logged;
uint64_t pages = 0, skipped = 0, bad = 0;
for (int i = 0; i < j->n; i++) {
const VSample *s = &j->s[i];
PoolRec *r = s->r;
uint32_t wb = __atomic_load_n(&r->wb_sub, __ATOMIC_RELAXED);
if (wb != j->sub && wb - j->sub < 0x80000000u) { // a later submission wrote back into it
skipped += s->len >> GEN_PAGE_SHIFT;
continue;
}
for (uint64_t o = 0; o < s->len; o += 1ull << GEN_PAGE_SHIFT) {
uint64_t a = r->import_lo + s->off + o;
const uint8_t *m = j->map + s->dst + o, *g = (const uint8_t *)(uintptr_t)a;
pages++;
if (!memcmp(m, g, 1ull << GEN_PAGE_SHIFT)) continue;
if (page_dirty(a, s->mgen)) { skipped++; continue; }
bad++;
if (InterlockedIncrement(&logged) <= VERIFY_LOG) {
int k = 0, e = (1 << GEN_PAGE_SHIFT) - 1;
while (k < e && m[k] == g[k]) k++;
while (e > k && m[e] == g[e]) e--;
char f[160];
flag_names(r->flags, f, sizeof f);
rt_log("mem: verify mismatch: pool %#llx+%#llx (%s) offset %#llx: bytes %#x..%#x of the page differ, "
"mirror %02x guest %02x at the first (submission %u)", (unsigned long long)r->lo,
(unsigned long long)r->size, f, (unsigned long long)(a - r->lo), k, e, m[k], g[k], j->sub);
}
}
}
InterlockedAdd64(&v_pages, (LONG64)pages);
InterlockedAdd64(&v_skipped, (LONG64)skipped);
InterlockedAdd64(&v_mismatch, (LONG64)bad);
vmaDestroyBuffer(vk_ctx()->vma, j->buf, j->alloc);
free(j);
}
static int verify_record(VkCommandBuffer cb, uint64_t fence) {
VJob *j = calloc(1, sizeof *j);
if (!j) rt_fatal("vk: out of memory");
uint64_t left = VERIFY_BYTES, dst = 0;
AcquireSRWLockShared(&pool_lock);
for (int tries = 0; nm && left && j->n < VERIFY_SAMPLES && tries <= nm; tries++) {
if (ver_i >= nm) ver_i = 0, ver_off = 0;
PoolRec *r = mlist[ver_i];
uint64_t size = r->import_hi - r->import_lo;
if (ver_off >= size) { ver_i++, ver_off = 0; continue; }
uint64_t len = size - ver_off < left ? size - ver_off : left;
j->s[j->n++] = (VSample){r, ver_off, len, dst, r->mgen};
dst += len, left -= len, ver_off += len;
if (ver_off >= size) ver_i++, ver_off = 0;
}
ReleaseSRWLockShared(&pool_lock);
if (!j->n) { free(j); return 0; }
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = dst,
.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
VmaAllocationInfo info;
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &j->buf, &j->alloc, &info) != VK_SUCCESS) { free(j); return 0; }
j->map = info.pMappedData;
j->sub = msub;
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT);
for (int i = 0; i < j->n; i++) {
VkBufferCopy c = {j->s[i].off, j->s[i].dst, j->s[i].len};
vkCmdCopyBuffer(cb, j->s[i].r->mbuf, j->buf, 1, &c);
}
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
VK_ACCESS_2_HOST_READ_BIT);
vk_completion_add(fence, verify_cb, j);
return j->n + 2;
}
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
if (!vk_mirror_on) return 0;
int rec = 0;
if (nwbl) {
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT);
for (int i = 0; i < nwbl; i++) wbl[i].r->wb_sub = msub;
mrep.wb_ranges += (uint64_t)nwbl;
rec += copy_ranges(cb, wbl, nwbl, 0, &mrep.wb_bytes, &mrep.wb_copies) + 2;
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
nwbl = 0;
}
if (mirror_verify) rec += verify_record(cb, fence);
AcquireSRWLockExclusive(&mirror_stat_lock);
uint64_t *d = (uint64_t *)&macc, *s = (uint64_t *)&mrep;
for (size_t i = 0; i < sizeof macc / sizeof(uint64_t); i++) d[i] += s[i];
ReleaseSRWLockExclusive(&mirror_stat_lock);
memset(&mrep, 0, sizeof mrep);
return rec;
}
// One summary line: with TOTK_TRACE=perf every 60 frames, else at frames 600 and 1800.
static void mirror_log(uint64_t frame) {
static uint64_t last, ww_last;
AcquireSRWLockExclusive(&gen_lock);
uint64_t wc = ww_calls, wb = ww_bytes, wt = ww_ticks, dc = ww_dirty_calls, dt = ww_dirty_ticks, wp = ww_pages;
ww_pages = 0;
uint64_t np = n_frz_nopool, nr = n_frz_refused;
ww_calls = ww_bytes = ww_ticks = ww_dirty_calls = ww_dirty_ticks = n_frz_nopool = n_frz_refused = 0;
ReleaseSRWLockExclusive(&gen_lock);
double wf = frame > ww_last ? (double)(frame - ww_last) : 1.0;
ww_last = frame;
uint64_t fz, ff, ft, fo;
mem_freeze_stats(&fz, &ff, &ft, &fo);
if (rt_trace & RT_TRACE_PERF)
rt_log("mem: write-watch frame %llu: per frame, every consumer: %.1f queries over %.1f MiB, %.3f ms (%.1f found "
"%.0f written pages, %.3f ms); frozen %.0f MiB; since the last line: freezes %llu, thaws by a write %llu, by a file read "
"or mapping change %llu, candidates not wholly in pools %llu, refused %llu",
(unsigned long long)frame, wc / wf, wb / wf / 1048576.0, qpc_ms(wt) / wf, dc / wf, wp / wf, qpc_ms(dt) / wf,
fz * 2.0, (unsigned long long)ff, (unsigned long long)ft, (unsigned long long)fo, (unsigned long long)np,
(unsigned long long)nr);
if (!vk_mirror_on) return;
AcquireSRWLockExclusive(&mirror_stat_lock);
MirrorAcc a = macc;
memset(&macc, 0, sizeof macc);
ReleaseSRWLockExclusive(&mirror_stat_lock);
double f = frame > last ? (double)(frame - last) : 1.0, M = 1048576.0;
last = frame;
uint64_t fc = (uint64_t)InterlockedExchange64(&m_flush_calls, 0), fb = (uint64_t)InterlockedExchange64(&m_flush_bytes, 0);
char ver[160] = "";
if (mirror_verify)
snprintf(ver, sizeof ver, "; verify: pages compared %lld, skipped %lld (written since), mismatches %lld",
(long long)InterlockedExchange64(&v_pages, 0), (long long)InterlockedExchange64(&v_skipped, 0),
(long long)InterlockedExchange64(&v_mismatch, 0));
rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked "
"%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, "
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries "
"%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f)%s",
(unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET],
m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP],
(long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f,
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M,
a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f,
qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver);
}
// Device class (build/mem-plan.md): INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a
// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Only logged for now.
// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Sets the mirror mode.
void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m) {
int big_local = -1, host_heap = -1;
for (uint32_t h = 0; h < m->memoryHeapCount; h++) {
@@ -811,6 +1459,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic
else if (big_local < 0) cls = "uma", why = "no device-local heap of 1 GiB or more";
else if (host_heap < 0) cls = "uma", why = "every heap is device-local";
else cls = "discrete", why = "device-local heap separate from host memory";
int discrete = cls[0] == 'd';
char line[2048];
size_t k = (size_t)snprintf(line, sizeof line, "mem: class %s (%s; device type %d %s, %s)", cls, why, t, tn, p->deviceName);
for (uint32_t h = 0; h < m->memoryHeapCount; h++)
@@ -830,6 +1479,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic
f & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT ? " LAZY" : "");
}
rt_log("%s", line);
mirror_mode(discrete);
}
// ---------------------------------------------------------------- upload batches and deferred destruction
@@ -1069,6 +1719,9 @@ void vk_mem_shutdown(void) {
AcquireSRWLockExclusive(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
pool_index.n = 0;
nm = npend = nwbl = 0; // mirrors went with their pools
for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran
nretiring = 0;
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
nphys = 0;
ReleaseSRWLockExclusive(&pool_lock);
+59 -23
View File
@@ -107,27 +107,33 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order
free(t);
}
// At the start of a part's command buffer (sub_lock held).
static void ts_begin(VkCommandBuffer cb) {
ts_cur = -1;
if (ts_state != 1) return;
// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed.
static int ts_open(VkCommandBuffer cb) {
if (ts_state != 1) return -1;
uint32_t p = ts_next % TS_PAIRS;
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return; }
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return -1; }
ts_next++;
ts_busy[p] = 1;
vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2);
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p);
ts_cur = (int)p;
return (int)p;
}
// At the end of the part, before vkEndCommandBuffer; fence: the value whose completion covers the part.
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
if (ts_cur < 0) return;
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)ts_cur + 1);
// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it.
static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence) {
if (pair < 0) return;
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1);
TsRead *t = malloc(sizeof *t);
if (!t) rt_fatal("vk: out of memory");
t->pair = (uint32_t)ts_cur;
t->pair = (uint32_t)pair;
vk_completion_add(fence, ts_read_cb, t);
}
// The pair of the part being recorded.
static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); }
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
ts_close(cb, ts_cur, fence);
ts_cur = -1;
}
@@ -227,27 +233,47 @@ void vk_sync_start(void) {
ReleaseSRWLockExclusive(&start_lock);
}
// A command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held.
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move
// slots[].
static int claim_slot(VkCtx *c, uint64_t fence) {
uint64_t gpu = 0;
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
for (int i = 0; i < nslots; i++)
if (slots[i].fence <= gpu) {
slots[i].fence = fence;
cur_slot = i;
return &slots[i];
return i;
}
CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1));
if (!n) rt_fatal("vk: out of memory (command buffers)");
slots = n;
CmdSlot *s = &slots[nslots];
cur_slot = nslots;
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
s->fence = fence;
nslots++;
return s;
return nslots++;
}
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
cur_slot = claim_slot(c, fence);
return &slots[cur_slot];
}
// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is
// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the
// part was recorded, sub_lock held; may move slots[].
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
int i = claim_slot(c, fence); // before reading slots: the claim may move it
VkCommandBuffer cb = slots[i].cb;
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
vkBeginCommandBuffer(cb, &bi);
int pair = ts_open(cb);
vk_mem_mirror_prologue(cb);
ts_close(cb, pair, fence);
vkEndCommandBuffer(cb);
return cb;
}
VkCommandBuffer vk_sync_split(void) {
@@ -256,8 +282,11 @@ VkCommandBuffer vk_sync_split(void) {
ts_end(cb, cur_fence);
vkEndCommandBuffer(cb);
vk_upload_flush(0); // the uploads this part asked for precede it in queue order
VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = 1, .pCommandBufferInfos = &cbi};
VkCommandBuffer pro = prologue(c, cur_fence);
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
.pCommandBufferInfos = pro ? cbi : cbi + 1};
vk_queue_lock();
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
@@ -292,11 +321,16 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
perf_replay(t0);
// After the recording: the uploads it asked for (and every earlier one) precede it in queue order.
vk_upload_flush(0);
VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb};
VkCommandBuffer pro = prologue(c, fence);
s = &slots[cur_slot];
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}};
VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = recorded ? 1 : 0,
.pCommandBufferInfos = &cbi, .signalSemaphoreInfoCount = 1, .pSignalSemaphoreInfos = &sig};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u),
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1,
.pSignalSemaphoreInfos = &sig};
vk_queue_lock();
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
@@ -365,6 +399,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
void vk_sync_finish(void) {
vk_sync_start();
vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX);
+2
View File
@@ -888,6 +888,8 @@ int vk_texture_slot_swz(int id, uint8_t swz[4]) {
return ok;
}
uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); }
void vk_texture_new_submit(void) {
lock();
if (!++submit_epoch) submit_epoch = 1;