mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 20:25:57 +00:00
Merge the device-local mirror of buffer-bound pools on discrete GPUs and the frozen idle regions that cut write-watch queries
On the GTX 1050 Ti the GPU time per frame in the world drops from 110-120 ms to 10-11 ms. TOTK_MEMORY=verify on the same machine compared 4.45 million mirror pages over a 74-second run with 0 mismatches. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
6b1d15534e
18 files changed
+1066
-98
No files matched your search
+2
-2
@@ -27,10 +27,10 @@ static const char *const known[] = {
|
||||
"BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL",
|
||||
"BIND_ZR",
|
||||
"CLOTH_REALTIME", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT",
|
||||
"FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "GPL",
|
||||
"FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "FREEZE_IDLE", "GPL",
|
||||
"GPL_DRAW_BUILD",
|
||||
"GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
|
||||
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MINIMAL_LAYOUT", "MOTION", "MOUSE_SENS", "MOVIE_HOST",
|
||||
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST",
|
||||
"MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SYNC", "PREWARM_XLATE",
|
||||
"PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_STAGE", "RDOC",
|
||||
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
|
||||
|
||||
+7
-1
@@ -691,7 +691,13 @@ static File *file_of(uint64_t h, HANDLE *fh, int *mode) {
|
||||
static int64_t host_read_at(HANDLE f, void *buf, uint32_t n, uint64_t off) {
|
||||
OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32)};
|
||||
DWORD got = 0;
|
||||
if (!ReadFile(f, buf, n, &got, &ov)) return GetLastError() == ERROR_HANDLE_EOF ? 0 : -1;
|
||||
mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting
|
||||
if (!ReadFile(f, buf, n, &got, &ov)) {
|
||||
DWORD e = GetLastError();
|
||||
if (e == ERROR_NOACCESS) // a read-only page in the buffer: a frozen region missed its thaw
|
||||
rt_log("fs: read of %u bytes into %p refused: the buffer is not writable", n, buf);
|
||||
return e == ERROR_HANDLE_EOF ? 0 : -1;
|
||||
}
|
||||
return got;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
// Needs no game files.
|
||||
#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
#include <windows.h>
|
||||
|
||||
#include "gfx.h"
|
||||
|
||||
@@ -60,6 +61,22 @@ static void check_write_watch(void) {
|
||||
printf("write-watch: three pages written, max 2: %llu then %llu\n", (unsigned long long)n, (unsigned long long)m);
|
||||
CHECK(n == 2 && m == 1, "max 2 reported %llu, then %llu (want 2, 1)", (unsigned long long)n, (unsigned long long)m);
|
||||
|
||||
// Frozen region: read-only until the first write thaws it (the write lands and is reported); mem_thaw (a file
|
||||
// read) thaws it and holds it thawed; SetHeapSize below decommits it.
|
||||
mem_written_pages(heap, 0x200000, pages, 8);
|
||||
CHECK(mem_freeze(heap) == 1 && mem_frozen(heap), "freeze of an idle committed 2 MiB region");
|
||||
MEMORY_BASIC_INFORMATION mbi;
|
||||
CHECK(VirtualQuery((void *)(uintptr_t)heap, &mbi, sizeof mbi) && mbi.Protect == PAGE_READONLY, "frozen region is not read-only");
|
||||
p[0x3000] = 7;
|
||||
CHECK(!mem_frozen(heap) && p[0x3000] == 7, "a write into a frozen region did not thaw it");
|
||||
n = mem_written_pages(heap, 0x200000, pages, 8);
|
||||
CHECK(n == 1 && pages[0] == heap + 0x3000, "write after a thaw: %llu pages reported (want 1)", (unsigned long long)n);
|
||||
CHECK(mem_freeze(heap) == 1, "refreeze after a write thaw");
|
||||
mem_thaw(heap + 0x6000, 0x1000);
|
||||
CHECK(!mem_frozen(heap) && mem_freeze(heap) == 0, "mem_thaw did not thaw and hold the region");
|
||||
p[0x6000] = 8;
|
||||
printf("write-watch: frozen region thawed by a write (reported once) and by mem_thaw (held)\n");
|
||||
|
||||
uint64_t a = g->alias.base + 0x10000;
|
||||
CHECK(mem_map_physical(a, 0x4000) == 0, "MapPhysicalMemory");
|
||||
mem_written_pages(a, 0x4000, pages, 8);
|
||||
|
||||
@@ -128,6 +128,18 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64
|
||||
// past max stays reported next time). The range must lie
|
||||
// inside the stack/heap block or the alias region
|
||||
// (write watch); otherwise returns UINT64_MAX
|
||||
uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max);
|
||||
// the same without the reset: the bits stay for the next
|
||||
// mem_written_pages (verify checks only)
|
||||
int mem_freeze(uint64_t a); // makes the 2 MiB region at a (2 MiB aligned, watched,
|
||||
// committed PAGE_READWRITE) read-only until a write or
|
||||
// mem_thaw thaws it; 1 frozen (or already), 0 refused
|
||||
int mem_frozen(uint64_t a); // the 2 MiB region of a is frozen: no page in it changed
|
||||
// since its freeze
|
||||
void mem_thaw(uint64_t a, uint64_t n); // makes the frozen regions over [a, a+n) writable again;
|
||||
// before a kernel write (file read) into guest memory
|
||||
void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws);
|
||||
// regions frozen now; freezes and thaws since the last call
|
||||
void mem_kernel_free(void *p, uint64_t size); // TLR/TLS reuse after thread exit
|
||||
|
||||
// ---- modules (loader.c, T1)
|
||||
|
||||
+136
-10
@@ -67,21 +67,142 @@ static void *reserve_at(uint64_t at, uint64_t size, DWORD extra) {
|
||||
return p;
|
||||
}
|
||||
|
||||
// ---- frozen regions
|
||||
// A 2 MiB region of the watched reservations that nothing wrote for a while can be made read-only (mem_freeze), so the
|
||||
// back end's write-watch queries skip it: no page of a frozen region changes before a thaw. The first write faults and
|
||||
// thaw_handler makes the region writable again before the write retries. A kernel write into guest memory (a file
|
||||
// read) fails on a read-only page instead of faulting, so file reads thaw their buffer first (mem_thaw), and so does
|
||||
// every commit, decommit and protection change here. A region stays thawed for FRZ_HOLD_MS after a mem_thaw: a file
|
||||
// read still in flight writes its buffer later. frz_lock is a leaf lock: nothing writes guest memory under it.
|
||||
#define FRZ_SHIFT 21
|
||||
#define FRZ_SIZE (1ull << FRZ_SHIFT)
|
||||
#define FRZ_HOLD_MS 5000u
|
||||
static volatile uint8_t frz[RT_ADDRESS_LIMIT >> FRZ_SHIFT];
|
||||
static uint32_t thaw_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: GetTickCount of the last thaw request, 0 never
|
||||
static volatile LONG nfrz;
|
||||
static SRWLOCK frz_lock = SRWLOCK_INIT;
|
||||
static volatile LONG64 frz_count[4]; // freezes; thaws by a write fault, by mem_thaw, by a mapping change
|
||||
|
||||
static int watched(uint64_t addr, uint64_t size) {
|
||||
const RtGuestLayout *g = &L;
|
||||
return (addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size) ||
|
||||
(addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size);
|
||||
}
|
||||
|
||||
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
|
||||
static void thaw_locked(uint64_t r, int why) {
|
||||
if (why == 2) thaw_ms[r] = (uint32_t)GetTickCount64() | 1;
|
||||
if (!frz[r]) return;
|
||||
DWORD old;
|
||||
uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE;
|
||||
if (!VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READWRITE, &old)) {
|
||||
// Not all committed (cannot happen while mapping changes thaw under frz_lock): the read-only runs only.
|
||||
MEMORY_BASIC_INFORMATION mbi;
|
||||
while (a < end && VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi)) {
|
||||
uint64_t b = (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize;
|
||||
if (b > end) b = end;
|
||||
if (mbi.State == MEM_COMMIT && mbi.Protect == PAGE_READONLY &&
|
||||
!VirtualProtect((void *)(uintptr_t)a, b - a, PAGE_READWRITE, &old))
|
||||
rt_fatal("memory: thawing %#llx+%#llx failed (error %lu)", (unsigned long long)a,
|
||||
(unsigned long long)(b - a), GetLastError());
|
||||
a = b;
|
||||
}
|
||||
}
|
||||
__atomic_store_n(&frz[r], 0, __ATOMIC_RELEASE);
|
||||
InterlockedDecrement(&nfrz);
|
||||
InterlockedIncrement64(&frz_count[why]);
|
||||
}
|
||||
|
||||
// frz_lock held exclusively.
|
||||
static void thaw_range_locked(uint64_t a, uint64_t n, int why) {
|
||||
if (!n || a >= RT_ADDRESS_LIMIT) return;
|
||||
uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT;
|
||||
for (uint64_t r = r0; r <= r1; r++) thaw_locked(r, why);
|
||||
}
|
||||
|
||||
void mem_thaw(uint64_t a, uint64_t n) {
|
||||
AcquireSRWLockExclusive(&frz_lock); // also when nothing is frozen: the hold keeps a read's buffer writable
|
||||
thaw_range_locked(a, n, 2);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
}
|
||||
|
||||
int mem_freeze(uint64_t a) {
|
||||
if ((a & (FRZ_SIZE - 1)) || !watched(a, FRZ_SIZE)) return 0;
|
||||
uint64_t r = a >> FRZ_SHIFT;
|
||||
MEMORY_BASIC_INFORMATION mbi;
|
||||
DWORD old;
|
||||
int ok = 0;
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
if (frz[r]) ok = 1;
|
||||
else if (thaw_ms[r] && (int32_t)((uint32_t)GetTickCount64() - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) ok = 0; // |1 may be ahead
|
||||
else if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT &&
|
||||
mbi.Protect == PAGE_READWRITE && (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize >= a + FRZ_SIZE &&
|
||||
VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READONLY, &old)) {
|
||||
__atomic_store_n(&frz[r], 1, __ATOMIC_RELEASE);
|
||||
InterlockedIncrement(&nfrz);
|
||||
InterlockedIncrement64(&frz_count[0]);
|
||||
ok = 1;
|
||||
}
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
return ok;
|
||||
}
|
||||
|
||||
int mem_frozen(uint64_t a) { return a < RT_ADDRESS_LIMIT && __atomic_load_n(&frz[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE); }
|
||||
|
||||
void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws) {
|
||||
*frozen = (uint64_t)__atomic_load_n(&nfrz, __ATOMIC_RELAXED);
|
||||
*freezes = (uint64_t)InterlockedExchange64(&frz_count[0], 0);
|
||||
*fault_thaws = (uint64_t)InterlockedExchange64(&frz_count[1], 0);
|
||||
*other_thaws = (uint64_t)InterlockedExchange64(&frz_count[2], 0) + (uint64_t)InterlockedExchange64(&frz_count[3], 0);
|
||||
}
|
||||
|
||||
// The first write into a frozen region thaws it and retries. A fault in a region another thread thawed meanwhile
|
||||
// retries too; any other fault goes on to the crash handler.
|
||||
static LONG CALLBACK thaw_handler(EXCEPTION_POINTERS *ep) {
|
||||
const EXCEPTION_RECORD *er = ep->ExceptionRecord;
|
||||
if (er->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || er->NumberParameters < 2 || er->ExceptionInformation[0] != 1)
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
uint64_t a = er->ExceptionInformation[1];
|
||||
if (a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return EXCEPTION_CONTINUE_SEARCH;
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
int was = frz[a >> FRZ_SHIFT];
|
||||
thaw_locked(a >> FRZ_SHIFT, 1);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
if (was) return EXCEPTION_CONTINUE_EXECUTION;
|
||||
MEMORY_BASIC_INFORMATION mbi;
|
||||
if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT && mbi.Protect == PAGE_READWRITE)
|
||||
return EXCEPTION_CONTINUE_EXECUTION;
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
}
|
||||
|
||||
// Mapping changes thaw the regions they touch and run under frz_lock, so no freeze sees them half done.
|
||||
static void commit(uint64_t a, uint64_t n, const char *what) {
|
||||
if (n && !VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE))
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
thaw_range_locked(a, n, 3);
|
||||
int ok = !n || VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
if (!ok)
|
||||
rt_fatal("memory: commit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
|
||||
(unsigned long long)n, GetLastError());
|
||||
}
|
||||
|
||||
static void decommit(uint64_t a, uint64_t n, const char *what) {
|
||||
if (n && !VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT))
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
thaw_range_locked(a, n, 3);
|
||||
int ok = !n || VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
if (!ok)
|
||||
rt_fatal("memory: decommit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
|
||||
(unsigned long long)n, GetLastError());
|
||||
}
|
||||
|
||||
static void protect(uint64_t a, uint64_t n, DWORD prot, const char *what) {
|
||||
DWORD old;
|
||||
if (n && !VirtualProtect((void *)(uintptr_t)a, n, prot, &old))
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
thaw_range_locked(a, n, 3);
|
||||
int ok = !n || VirtualProtect((void *)(uintptr_t)a, n, prot, &old);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
if (!ok)
|
||||
rt_fatal("memory: protect %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
|
||||
(unsigned long long)n, GetLastError());
|
||||
}
|
||||
@@ -129,6 +250,7 @@ void mem_init(void) {
|
||||
if (reserve_at(at, ALIAS_SIZE, MEM_WRITE_WATCH)) alias = at;
|
||||
if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39");
|
||||
|
||||
AddVectoredExceptionHandler(1, thaw_handler); // before the crash handler
|
||||
alias_pages = VirtualAlloc(NULL, ALIAS_PAGES, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
if (!alias_pages) rt_fatal("memory: cannot allocate the alias page map");
|
||||
const char *pool = cfg_get("TOTK_APP_POOL");
|
||||
@@ -148,20 +270,24 @@ const RtGuestLayout *mem_layout(void) { return &L; }
|
||||
|
||||
// Adapted from LAS Recomp runtime/platform/windows.c:40-63 @ da35b83 (MIT, Copyright (c) 2026 Peiikai): GetWriteWatch
|
||||
// with the reset flag. A range inside one watched reservation (stack and heap block, or alias region) only.
|
||||
uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
|
||||
static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, DWORD flags) {
|
||||
if (!max || !size) return 0;
|
||||
const RtGuestLayout *g = &L;
|
||||
int low_block = addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size;
|
||||
int alias = addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size;
|
||||
if (!low_block && !alias) return UINT64_MAX;
|
||||
if (!watched(addr, size)) return UINT64_MAX;
|
||||
ULONG_PTR n = (ULONG_PTR)max;
|
||||
DWORD gran;
|
||||
_Static_assert(sizeof(PVOID) == sizeof(uint64_t), "page addresses are written straight into a uint64_t array");
|
||||
if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran))
|
||||
return UINT64_MAX;
|
||||
if (GetWriteWatch(flags, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran)) return UINT64_MAX;
|
||||
return n;
|
||||
}
|
||||
|
||||
uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
|
||||
return written_pages(addr, size, pages, max, WRITE_WATCH_FLAG_RESET);
|
||||
}
|
||||
|
||||
uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
|
||||
return written_pages(addr, size, pages, max, 0);
|
||||
}
|
||||
|
||||
static uint64_t image_bytes(void) {
|
||||
uint64_t n = 0;
|
||||
for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size;
|
||||
|
||||
@@ -99,3 +99,6 @@ const char *nvn_op_name(uint32_t op); // "DRAW_IDX", ...; "
|
||||
// MemoryPoolInvalidateMappedRange: the CPU is about to read pool storage the GPU wrote. Weak no-op in nvn_ext.c; a back
|
||||
// end that renders into host-side copies defines it and writes them back.
|
||||
void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size);
|
||||
// MemoryPoolFlushMappedRange: the CPU wrote [mem + off, mem + off + size). Weak no-op in nvn_ext.c; a back end that keeps
|
||||
// device-side copies of pool memory (vk_memory.c's VRAM mirror) defines it and copies the range again.
|
||||
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size);
|
||||
@@ -303,9 +303,14 @@ void nvn_flush_stats(uint64_t out[2][3][2]) {
|
||||
for (int j = 0; j < 2; j++) out[i][k][j] = __atomic_exchange_n(&flush_stat[i][k][j], 0, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent
|
||||
// void nvnMemoryPoolFlushMappedRange(const NVNmemoryPool *p, ptrdiff_t offset, size_t size): storage is coherent; a
|
||||
// back end with device-side copies hears of it (nvn_pool_flushed)
|
||||
NVN_API(nvnMemoryPoolFlushMappedRange) {
|
||||
if (rt_trace & RT_TRACE_PERF) flush_note(0, arg_p(c, 0), arg_x(c, 2));
|
||||
const pool_t *p = arg_p(c, 0);
|
||||
int64_t off = (int64_t)arg_x(c, 1);
|
||||
uint64_t size = arg_x(c, 2);
|
||||
if (rt_trace & RT_TRACE_PERF) flush_note(0, p, size);
|
||||
if (p && off >= 0 && (uint64_t)off < p->size) nvn_pool_flushed(p, (uint64_t)off, size < p->size - (uint64_t)off ? size : p->size - (uint64_t)off);
|
||||
}
|
||||
// NVNmemoryPoolFlags nvnMemoryPoolGetFlags(const NVNmemoryPool *p)
|
||||
NVN_API(nvnMemoryPoolGetFlags) { ret_x(c, ((const pool_t *)arg_p(c, 0))->flags); }
|
||||
|
||||
@@ -20,6 +20,8 @@ void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size) {
|
||||
gfx_backend()->guest_read((uintptr_t)p->mem + off, size);
|
||||
}
|
||||
|
||||
__attribute__((weak)) void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { (void)p, (void)off, (void)size; }
|
||||
|
||||
// ---------- debug labels, markers, groups, checkpoints: meaningful only to NVN debugging tools
|
||||
// void nvn<Object>SetDebugLabel(NVN<object> *o, const char *label)
|
||||
NVN_API(nvnDeviceSetDebugLabel) { (void)c; }
|
||||
|
||||
+7
-2
@@ -84,11 +84,16 @@ static int64_t read_at(HANDLE f, void *buf, uint32_t n, uint64_t off) {
|
||||
rt_fatal("RomFS: CreateEvent failed (%lu)", GetLastError());
|
||||
OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32), .hEvent = ev};
|
||||
DWORD got = 0;
|
||||
mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting
|
||||
if (!ReadFile(f, buf, n, &got, &ov)) {
|
||||
DWORD e = GetLastError();
|
||||
if (e == ERROR_HANDLE_EOF) return 0;
|
||||
if (e != ERROR_IO_PENDING) return -1;
|
||||
if (!GetOverlappedResult(f, &ov, &got, TRUE)) return GetLastError() == ERROR_HANDLE_EOF ? 0 : -1;
|
||||
if (e == ERROR_IO_PENDING && !GetOverlappedResult(f, &ov, &got, TRUE)) e = GetLastError();
|
||||
else if (e == ERROR_IO_PENDING) return got;
|
||||
if (e == ERROR_HANDLE_EOF) return 0;
|
||||
if (e == ERROR_NOACCESS) // a read-only page in the buffer: a frozen region missed its thaw
|
||||
rt_log("RomFS: read of %u bytes into %p refused: the buffer is not writable", n, buf);
|
||||
return -1;
|
||||
}
|
||||
return got;
|
||||
}
|
||||
|
||||
@@ -138,6 +138,30 @@ void vk_mem_count(int cls, int use, uint64_t bytes);
|
||||
void vk_mem_bound_note(void *pool);
|
||||
#define VK_MEM_BOUND(sp) do { if ((rt_trace & RT_TRACE_PERF) && (sp).pool) vk_mem_bound_note((sp).pool); } while (0)
|
||||
void vk_mem_dirty_submit(void); // vk_sync_submit, under its lock, before the replay
|
||||
// VRAM mirror of buffer-bound pools (vk_memory.c, build/mem-plan.md). vk_mirror_on: TOTK_MEMORY=mirror|verify, or the
|
||||
// device class is discrete and TOTK_MEMORY is unset or auto; set at device selection. The replay (submitting thread
|
||||
// under vk_sync_submit's lock) calls:
|
||||
// - VK_MEM_USE at a vertex, index, uniform, storage or indirect binding of a span vk_mem_resolve returned: the pool's
|
||||
// mirror is created on first use and sp.buffer becomes it (same offsets); a pool that stays imported keeps its buffer.
|
||||
// - VK_MEM_PICK where a copy, fill or update reads or writes a span: the mirror if the pool has one, never a new one.
|
||||
// - vk_mem_gpu_wrote after recording a GPU write of [sp.offset, sp.offset + size) into sp.buffer: a mirror's bytes go
|
||||
// back to guest memory at the end of the submission. No-op for imported buffers.
|
||||
// - vk_mem_mirror_begin at the start of a submission's recording, vk_mem_mirror_end (records the write-backs; returns
|
||||
// the commands recorded) after its last barrier.
|
||||
// vk_sync.c records the copies the mirrors' first accesses queued (vk_mem_mirror_pending) into a prologue command
|
||||
// buffer with vk_mem_mirror_prologue and submits it right before each part.
|
||||
extern int vk_mirror_on;
|
||||
void vk_mem_use(VkMemSpan *sp);
|
||||
void vk_mem_pick(VkMemSpan *sp);
|
||||
#define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0)
|
||||
#define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0)
|
||||
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size);
|
||||
void vk_mem_mirror_begin(void);
|
||||
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence);
|
||||
int vk_mem_mirror_pending(void);
|
||||
void vk_mem_mirror_prologue(VkCommandBuffer cb);
|
||||
int vk_mem_mirror_probe(uint64_t addr); // --probe-vkexec: the pool at addr gets its mirror as on a first buffer use;
|
||||
// 1 when it has one
|
||||
void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m); // once
|
||||
int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked)
|
||||
void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and,
|
||||
@@ -223,6 +247,7 @@ void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); /
|
||||
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads);
|
||||
void vk_texture_shutdown(void); // vk_mem_shutdown calls it
|
||||
void vk_texture_new_submit(void); // a queue submit starts: draw bindings check storage again
|
||||
uint32_t vk_texture_epoch(void); // the write-watch snapshot epoch vk_texture_new_submit started
|
||||
VkSampler vk_sampler_by_id(int id); // VK_NULL_HANDLE for an empty slot
|
||||
VkSampler vk_sampler_for(const samp_par *s); // cached by content
|
||||
int vk_sampler_params(int id, samp_par *out); // parameters of a sampler slot; 0 for an empty slot
|
||||
@@ -237,6 +262,7 @@ void vk_sync_start(void); // vk_init (fatal without a device) a
|
||||
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
|
||||
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
|
||||
uint64_t vk_sync_fence_done(void); // highest done value
|
||||
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
|
||||
void vk_sync_finish(void); // every submitted value done
|
||||
void vk_sync_shutdown(void); // finish, stop the completion thread, free the command buffers
|
||||
// The op-stream replay (vk_exec.c: T6c): records the command handles of one submission into cb, which the caller has
|
||||
|
||||
@@ -399,6 +399,7 @@ static void buf_snap(VkExec *ex, int pass) {
|
||||
if (!bd_size) return;
|
||||
VkMemSpan s;
|
||||
if (!vk_mem_resolve(bd_addr, bd_size, &s) || s.avail < bd_size) return;
|
||||
VK_MEM_PICK(s); // the GPU's view of the buffer
|
||||
BufJob *j = calloc(1, sizeof *j);
|
||||
if (!j) rt_fatal("vk_dump: out of memory");
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bd_size,
|
||||
|
||||
@@ -688,6 +688,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
|
||||
vk_exec_frame.submits++;
|
||||
split_draws = part_start = 0;
|
||||
x_here = 1;
|
||||
if (n) vk_mem_mirror_begin();
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (!handles[i]) rt_fatal("vk: command handle %d of a submission is null", i);
|
||||
walk(ex, nvn_op_first(handles[i]), 0);
|
||||
@@ -695,6 +696,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
|
||||
vk_exec_pass_end(ex);
|
||||
x_here = 0;
|
||||
if (ex->recorded) vk_exec_barrier(ex); // later submissions see this one's writes
|
||||
if (n) ex->recorded += vk_mem_mirror_end(ex->cb, fence); // VRAM mirror bytes the GPU wrote go back to guest memory
|
||||
if (ex->nviews) {
|
||||
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
|
||||
if (!l) rt_fatal("vk: out of memory");
|
||||
|
||||
+19
-6
@@ -569,16 +569,20 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) {
|
||||
return (VkDescriptorBufferInfo){Z.zbuf, 0, max < ZERO_BYTES ? max : ZERO_BYTES};
|
||||
}
|
||||
|
||||
// A buffer descriptor for [addr, addr+size) of guest memory: its backing when aligned, a copy in transient memory when
|
||||
// the backing is host memory with an unaligned offset. 0 when neither works.
|
||||
static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, VkDescriptorBufferInfo *out) {
|
||||
// A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when
|
||||
// aligned, a copy in transient memory when the backing is host memory with an unaligned offset. 0 when neither works.
|
||||
// written: the shader stores into it (a mirror's range goes back to guest memory at the end of the submission).
|
||||
static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, uint64_t max, int use, int written,
|
||||
VkDescriptorBufferInfo *out) {
|
||||
VkMemSpan sp;
|
||||
if (!addr || !vk_mem_resolve(addr, 1, &sp)) return 0;
|
||||
uint64_t n = size && size < sp.avail ? size : sp.avail;
|
||||
if (n > max) n = max;
|
||||
if (!n) return 0;
|
||||
if (sp.offset % align == 0) {
|
||||
VK_MEM_USE(sp);
|
||||
*out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n};
|
||||
if (written) vk_mem_gpu_wrote(&sp, n);
|
||||
vk_mem_count(sp.cls, use, n);
|
||||
VK_MEM_BOUND(sp);
|
||||
return 1;
|
||||
@@ -665,11 +669,12 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) {
|
||||
uint32_t len, c1;
|
||||
if (find_stage(ex->prog, st, &ctl, &code, &len)) {
|
||||
memcpy(&c1, ctl + 0x700, 4);
|
||||
if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, &bi)) return bi;
|
||||
if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, 0, &bi))
|
||||
return bi;
|
||||
}
|
||||
} else if (slot >= 3 && slot - 3 < 18) {
|
||||
const p_bind_buf *b = &ex->ubo[st][slot - 3];
|
||||
if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, &bi)) return bi;
|
||||
if (bind_range(ex, b->addr, b->size, align, max, VK_MUSE_UBO, 0, &bi)) return bi;
|
||||
}
|
||||
return zero_info(max);
|
||||
}
|
||||
@@ -699,7 +704,8 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *
|
||||
const p_bind_buf *b = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot];
|
||||
// vk_caps.c: shaders translated for sbuf_align address the buffer from its address rounded down to it.
|
||||
uint64_t lead = vk_caps.sbuf_align ? b->addr & (vk_caps.sbuf_align - 1) : 0;
|
||||
if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO, &bi)) {
|
||||
if (bind_range(ex, b->addr - lead, b->size ? b->size + lead : 0, align, max, ubo ? VK_MUSE_UBO : VK_MUSE_SSBO,
|
||||
written, &bi)) {
|
||||
if (written && bi.buffer != VK_NULL_HANDLE) note_written(ex, b->addr - lead, bi.range);
|
||||
return bi;
|
||||
}
|
||||
@@ -1095,6 +1101,7 @@ static void bind_vertex_buffers(VkExec *ex) {
|
||||
const NVNbufferRange *r = &ex->vbuf[b];
|
||||
VkMemSpan sp;
|
||||
if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue;
|
||||
VK_MEM_USE(sp);
|
||||
uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail;
|
||||
vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool;
|
||||
}
|
||||
@@ -1271,6 +1278,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) {
|
||||
} else if (sp.offset % isize[d->itype]) {
|
||||
return warn_once(W_INDEX, "index buffer %#llx is not aligned to its index size: draw skipped", (unsigned long long)d->ibuf);
|
||||
} else {
|
||||
VK_MEM_USE(sp);
|
||||
vkCmdBindIndexBuffer(ex->cb, sp.buffer, sp.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
|
||||
ex->recorded++;
|
||||
vk_mem_count(sp.cls, VK_MUSE_IB, bytes);
|
||||
@@ -1295,6 +1303,8 @@ static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) {
|
||||
return warn_once(W_INDEX, "index buffer %#llx is not backed or not aligned: indirect draws skipped", (unsigned long long)d->ibuf);
|
||||
if (!prepare_draw(ex, d->prim, 0x7fffffff, restart_for(ex, d->prim, d->itype))) return;
|
||||
vk_exec_frame.draws++;
|
||||
VK_MEM_USE(ip);
|
||||
VK_MEM_USE(ib);
|
||||
vkCmdBindIndexBuffer(ex->cb, ib.buffer, ib.offset, d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
|
||||
vkCmdDrawIndexedIndirect(ex->cb, ip.buffer, ip.offset, 1, sizeof(VkDrawIndexedIndirectCommand));
|
||||
ex->recorded += 2;
|
||||
@@ -1341,6 +1351,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) {
|
||||
VkPipeline p = vk_pipeline_compute(cs, &info);
|
||||
if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped");
|
||||
vk_exec_barrier(ex);
|
||||
VK_MEM_USE(ip);
|
||||
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
|
||||
c0_dim[0] = c0_dim[1] = c0_dim[2] = 0;
|
||||
B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads
|
||||
@@ -1368,8 +1379,10 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) {
|
||||
VkBuffer b;
|
||||
VkDeviceSize o;
|
||||
memcpy(scratch(ex, u->size, &b, &o), u->data, u->size);
|
||||
VK_MEM_PICK(sp);
|
||||
VkBufferCopy cp = {o, sp.offset, u->size};
|
||||
vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp);
|
||||
vk_mem_gpu_wrote(&sp, u->size);
|
||||
ex->recorded++;
|
||||
vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size);
|
||||
note_written(ex, u->dst, u->size);
|
||||
|
||||
+55
-1
@@ -802,6 +802,54 @@ static int test_nvnx(const GfxBackend *b, uint8_t *guest, int *bytes) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// VRAM mirror (vk_memory.c; skipped in import mode): a pool given its mirror as on a first buffer use, after which
|
||||
// buffer copies and fills read and write the mirror. The first use copies the whole pool; CPU writes between
|
||||
// submissions reach the mirror through write watch; GPU writes reach guest memory by the fence, and only the bytes
|
||||
// written: CPU bytes beside a GPU fill keep their values.
|
||||
static int test_mirror(const GfxBackend *b, uint8_t *guest, int *bytes) {
|
||||
*bytes = 0;
|
||||
if (!vk_mirror_on) return 0;
|
||||
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x22, .mem = guest + 0xC00000, .size = 0x100000,
|
||||
.game_mem = guest + 0xC00000};
|
||||
CHECK(b->pool_init(&pool), "pool_init of a CPU-visible pool for the mirror part");
|
||||
uint8_t *m = pool.mem;
|
||||
for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 13 + 1);
|
||||
memset(m + 0x1000, 0xEE, 0x5000);
|
||||
CHECK(vk_mem_mirror_probe((uint64_t)(uintptr_t)m), "the pool got no VRAM mirror");
|
||||
static _Alignas(8) uint8_t stream[1024];
|
||||
uint64_t a = (uint64_t)(uintptr_t)m;
|
||||
// 1: B2B in the mirror after its first-use copy, and a fill; both written back.
|
||||
uint8_t *p = put(stream, OP_BEGIN, NULL, 0);
|
||||
const p_copy_b2b c1 = {a, a + 0x1000, 4096};
|
||||
const p_clear_buf f1 = {a + 0x2000, 0x800, 0x5a5a5a5au, 0};
|
||||
p = put(p, OP_COPY_B2B, &c1, sizeof c1);
|
||||
p = put(p, OP_CLEAR_BUF, &f1, sizeof f1);
|
||||
CHECK(xf_submit(b, stream, p), "fence_wait of the first mirror submission timed out");
|
||||
for (int i = 0; i < 4096; i++) CHECK(m[0x1000 + i] == m[i], "mirror B2B byte %d is %02x, expected %02x", i, m[0x1000 + i], m[i]);
|
||||
for (int i = 0; i < 0x800; i++) CHECK(m[0x2000 + i] == 0x5a, "mirror fill byte %d is %02x", i, m[0x2000 + i]);
|
||||
for (int i = 0x800; i < 0x1000; i++) CHECK(m[0x2000 + i] == 0xEE, "byte %#x past the mirror fill is %02x", i, m[0x2000 + i]);
|
||||
// 2: the CPU rewrites the source; the next submission copies the new bytes.
|
||||
for (int i = 0; i < 4096; i++) m[i] = (uint8_t)(i * 7 + 5);
|
||||
p = put(stream, OP_BEGIN, NULL, 0);
|
||||
const p_copy_b2b c2 = {a, a + 0x3000, 4096};
|
||||
p = put(p, OP_COPY_B2B, &c2, sizeof c2);
|
||||
CHECK(xf_submit(b, stream, p), "fence_wait of the second mirror submission timed out");
|
||||
for (int i = 0; i < 4096; i++)
|
||||
CHECK(m[0x3000 + i] == m[i], "B2B byte %d after a CPU write is %02x, expected %02x (write not mirrored)", i,
|
||||
m[0x3000 + i], m[i]);
|
||||
// 3: CPU bytes in the page of a GPU fill, written before the submission, survive the write-back.
|
||||
memset(m + 0x4000, 0x11, 0x800);
|
||||
p = put(stream, OP_BEGIN, NULL, 0);
|
||||
const p_clear_buf f3 = {a + 0x4800, 0x800, 0x77777777u, 0};
|
||||
p = put(p, OP_CLEAR_BUF, &f3, sizeof f3);
|
||||
CHECK(xf_submit(b, stream, p), "fence_wait of the third mirror submission timed out");
|
||||
for (int i = 0; i < 0x800; i++) CHECK(m[0x4000 + i] == 0x11, "CPU byte %#x beside a fill is %02x", i, m[0x4000 + i]);
|
||||
for (int i = 0; i < 0x800; i++) CHECK(m[0x4800 + i] == 0x77, "fill byte %#x is %02x", i, m[0x4800 + i]);
|
||||
b->pool_fini(&pool);
|
||||
*bytes = 4096 + 0x1000 + 4096 + 0x1000;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int probe_vkexec(void) {
|
||||
layouts_equal = 0;
|
||||
main_tid = GetCurrentThreadId();
|
||||
@@ -819,8 +867,9 @@ int probe_vkexec(void) {
|
||||
if (!bad) bad = test_xfer(b, guest, &xfer_bytes);
|
||||
int zero_bytes = 0;
|
||||
if (!bad) bad = test_c0() || test_zero(b, guest, &zero_bytes);
|
||||
int nvnx_bytes = 0;
|
||||
int nvnx_bytes = 0, mirror_bytes = 0;
|
||||
if (!bad) bad = test_nvnx(b, guest, &nvnx_bytes);
|
||||
if (!bad) bad = test_mirror(b, guest, &mirror_bytes);
|
||||
b->finish();
|
||||
vk_pipeline_shutdown(); // the pipelines and the draw part's stand-ins go before the device
|
||||
vk_exec_draw_shutdown();
|
||||
@@ -860,5 +909,10 @@ int probe_vkexec(void) {
|
||||
printf("probe-vkexec: c0 and zero parts passed (c0 tables, dispatch size, point size and base instance at their offsets, "
|
||||
"unbound slots zero; %d bytes of an unwritten physical-pool texture read zero, TOTK_FIX_ZERO=%s)\n",
|
||||
zero_bytes, cfg_get("TOTK_FIX_ZERO") ? cfg_get("TOTK_FIX_ZERO") : "1");
|
||||
if (vk_mirror_on)
|
||||
printf("probe-vkexec: mirror part passed (B2B and fill in a VRAM mirror written back by the fence, a CPU write "
|
||||
"between submissions mirrored, CPU bytes beside a fill kept, %d bytes compared)\n", mirror_bytes);
|
||||
else
|
||||
printf("probe-vkexec: mirror part skipped (memory mode import)\n");
|
||||
return 0;
|
||||
}
|
||||
+16
-11
@@ -379,9 +379,10 @@ static VkExtent3D sub_extent(const Sub *s) {
|
||||
return (VkExtent3D){s->w, s->h, s->img->type == VK_IMAGE_TYPE_3D ? s->nz : 1};
|
||||
}
|
||||
|
||||
// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span.
|
||||
// Buffer side of a texture <-> buffer copy: bytes covered, pitches, Vulkan buffer span (a VRAM mirror when the pool
|
||||
// has one).
|
||||
static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *size, VkBufferImageCopy *c,
|
||||
VkBuffer *buf, int *cls) {
|
||||
VkMemSpan *span) {
|
||||
uint32_t wb = (s->w + (uint32_t)s->bw - 1) / (uint32_t)s->bw, hb = (s->h + (uint32_t)s->bh - 1) / (uint32_t)s->bh;
|
||||
uint64_t row = ex->copy_stride > 0 ? (uint64_t)ex->copy_stride : (uint64_t)wb * (uint64_t)s->bpb;
|
||||
if (row % (uint64_t)s->bpb || row < (uint64_t)wb * (uint64_t)s->bpb) {
|
||||
@@ -397,8 +398,8 @@ static int buffer_side(const VkExec *ex, const Sub *s, uint64_t addr, uint64_t *
|
||||
(unsigned long long)addr, (unsigned long long)*size);
|
||||
return 0;
|
||||
}
|
||||
*buf = m.buffer;
|
||||
*cls = m.cls;
|
||||
VK_MEM_PICK(m);
|
||||
*span = m;
|
||||
*c = (VkBufferImageCopy){.bufferOffset = m.offset, .bufferRowLength = (uint32_t)(row / (uint64_t)s->bpb) * (uint32_t)s->bw,
|
||||
.bufferImageHeight = hb * (uint32_t)s->bh, .imageSubresource = sub_layers(s),
|
||||
.imageOffset = sub_offset(s), .imageExtent = sub_extent(s)};
|
||||
@@ -410,17 +411,16 @@ static void copy_buffer_image(VkExec *ex, const p_copy_b2t *p, int to_image) {
|
||||
if (!sub_resolve(p->tex, &p->view, p->has_view, &p->r, 1, &s)) return;
|
||||
uint64_t size;
|
||||
VkBufferImageCopy c;
|
||||
VkBuffer buf;
|
||||
int cls;
|
||||
if (!buffer_side(ex, &s, p->src, &size, &c, &buf, &cls)) return;
|
||||
vk_mem_count(cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size);
|
||||
VkMemSpan m;
|
||||
if (!buffer_side(ex, &s, p->src, &size, &c, &m)) return;
|
||||
vk_mem_count(m.cls, to_image ? VK_MUSE_COPY_SRC : VK_MUSE_GPU_WRITE, size);
|
||||
vk_exec_pass_end(ex);
|
||||
vk_exec_barrier(ex);
|
||||
if (to_image) vkCmdCopyBufferToImage(ex->cb, buf, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c);
|
||||
else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, buf, 1, &c);
|
||||
if (to_image) vkCmdCopyBufferToImage(ex->cb, m.buffer, s.img->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c);
|
||||
else vkCmdCopyImageToBuffer(ex->cb, s.img->image, VK_IMAGE_LAYOUT_GENERAL, m.buffer, 1, &c);
|
||||
ex->recorded++;
|
||||
if (to_image) vk_texture_gpu_written(p->tex);
|
||||
else note_gpu_write(ex, p->src, size);
|
||||
else vk_mem_gpu_wrote(&m, size), note_gpu_write(ex, p->src, size);
|
||||
}
|
||||
|
||||
static void copy_t2t(VkExec *ex, const p_copy_t2t *p) {
|
||||
@@ -476,6 +476,8 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) {
|
||||
}
|
||||
uint64_t n = left < s.avail ? left : s.avail;
|
||||
if (n > d.avail) n = d.avail;
|
||||
VK_MEM_PICK(s);
|
||||
VK_MEM_PICK(d);
|
||||
if (s.buffer == d.buffer && s.offset < d.offset + n && d.offset < s.offset + n) {
|
||||
warn_once(W_OVERLAP, "buffer copy %#llx -> %#llx overlaps within one buffer: skipped",
|
||||
(unsigned long long)p->src, (unsigned long long)p->dst);
|
||||
@@ -484,6 +486,7 @@ static void copy_b2b(VkExec *ex, const p_copy_b2b *p) {
|
||||
if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1;
|
||||
VkBufferCopy bc = {s.offset, d.offset, n};
|
||||
vkCmdCopyBuffer(ex->cb, s.buffer, d.buffer, 1, &bc);
|
||||
vk_mem_gpu_wrote(&d, n);
|
||||
ex->recorded++;
|
||||
vk_mem_count(s.cls, VK_MUSE_COPY_SRC, n);
|
||||
vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n);
|
||||
@@ -514,7 +517,9 @@ static void clear_buf(VkExec *ex, const p_clear_buf *p) {
|
||||
uint64_t n = left < d.avail ? left : d.avail & ~3ull;
|
||||
if (!n) break;
|
||||
if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1;
|
||||
VK_MEM_PICK(d);
|
||||
vkCmdFillBuffer(ex->cb, d.buffer, d.offset, n, p->value);
|
||||
vk_mem_gpu_wrote(&d, n);
|
||||
ex->recorded++;
|
||||
vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n);
|
||||
dst += n, left -= n;
|
||||
|
||||
+693
-40
@@ -22,8 +22,26 @@
|
||||
#include "nvn_pool_index.h"
|
||||
#include "vk.h"
|
||||
|
||||
// Raw performance counter ticks for the tracking cost (no division on the replay's path); qpc_ms converts.
|
||||
static uint64_t qpc(void) {
|
||||
LARGE_INTEGER t;
|
||||
QueryPerformanceCounter(&t);
|
||||
return (uint64_t)t.QuadPart;
|
||||
}
|
||||
|
||||
static double qpc_ms(uint64_t ticks) {
|
||||
static double per_ms;
|
||||
if (!per_ms) {
|
||||
LARGE_INTEGER f;
|
||||
QueryPerformanceFrequency(&f);
|
||||
per_ms = (double)f.QuadPart / 1000.0;
|
||||
}
|
||||
return (double)ticks / per_ms;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- pools
|
||||
enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT };
|
||||
enum { MS_NONE, MS_ON, MS_REFUSED };
|
||||
|
||||
typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page -> physical pool and offset
|
||||
|
||||
@@ -44,8 +62,22 @@ typedef struct PoolRec {
|
||||
uint32_t synced_gen; // page generation its lazy-mirror count covers; DIRTY_UNSYNCED until scanned
|
||||
uint8_t bufbound; // ever bound as a vertex, index, uniform, storage or indirect buffer
|
||||
uint8_t untracked; // the last dirty scan found no write watch over it
|
||||
// VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only.
|
||||
uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED
|
||||
VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf
|
||||
VmaAllocation malloc_;
|
||||
uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync)
|
||||
uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory
|
||||
uint32_t mgen; // gen_lock: every page write stamped up to this generation is in the mirror
|
||||
// (or in a copy queued before its next use)
|
||||
uint64_t fl_lo, fl_hi; // flush_lock: nvnMemoryPoolFlushMappedRange bytes not copied yet (fl_hi 0: none)
|
||||
} PoolRec;
|
||||
|
||||
static void mirror_forget(PoolRec *r); // pool_lock exclusive: off the mirrored list
|
||||
static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown)
|
||||
static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record
|
||||
static void grow(void **arr, int *cap, size_t elem);
|
||||
|
||||
static SRWLOCK pool_lock = SRWLOCK_INIT;
|
||||
// Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits
|
||||
// stay valid while it is unchanged.
|
||||
@@ -197,6 +229,7 @@ static PoolRec *phys_find(const pool_t *key) {
|
||||
}
|
||||
|
||||
static void destroy_rec(VkCtx *c, PoolRec *r) {
|
||||
if (r->mbuf) vmaDestroyBuffer(c->vma, r->mbuf, r->malloc_);
|
||||
if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL);
|
||||
if (r->mem) vkFreeMemory(c->device, r->mem, NULL);
|
||||
if (r->alloc) vmaFreeMemory(c->vma, r->alloc);
|
||||
@@ -226,6 +259,7 @@ int vk_pool_init(pool_t *p) {
|
||||
ok = alloc_physical(c, r);
|
||||
} else {
|
||||
r->kind = POOL_IMPORT;
|
||||
mem_thaw((uintptr_t)p->mem, p->size); // the import may need write access to the pages
|
||||
ok = import_pool(c, r, (uintptr_t)p->mem, p->size);
|
||||
if (!ok) {
|
||||
// Not fatal for the game: the pool works for the CPU, and vk_mem_resolve reports its addresses as unbacked.
|
||||
@@ -259,6 +293,7 @@ void vk_pool_fini(pool_t *p) {
|
||||
int had = pidx_remove(&pool_index, p, &e);
|
||||
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
||||
PoolRec *r = had ? e.buf : NULL;
|
||||
if (r && r->mstate == MS_ON) mirror_forget(r);
|
||||
if (r && r->kind == POOL_PHYS) {
|
||||
for (int i = 0; i < nphys; i++)
|
||||
if (phys_list[i] == r) { phys_list[i] = phys_list[--nphys]; break; }
|
||||
@@ -271,6 +306,7 @@ void vk_pool_fini(pool_t *p) {
|
||||
}
|
||||
ReleaseSRWLockExclusive(&pool_lock);
|
||||
if (!r) return;
|
||||
if (r->kind == POOL_IMPORT) mem_thaw(r->lo, r->size); // the storage goes back to the guest writable
|
||||
// Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the
|
||||
// guest gets the storage back when this returns.
|
||||
vk_texture_range_gone(r->lo, r->size, r->alloc);
|
||||
@@ -281,7 +317,10 @@ void vk_pool_fini(pool_t *p) {
|
||||
vk_queue_unlock();
|
||||
}
|
||||
vk_mem_collect();
|
||||
destroy_rec(c, r);
|
||||
// A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may
|
||||
// name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued).
|
||||
if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r);
|
||||
else destroy_rec(c, r);
|
||||
}
|
||||
|
||||
// Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c:54-67); apply them in order.
|
||||
@@ -414,9 +453,13 @@ static void flag_names(uint32_t f, char *buf, size_t n) {
|
||||
// run-game saves), at frame 600, and with TOTK_TRACE=perf every 1800 frames.
|
||||
static void dirty_frame(uint64_t frame);
|
||||
|
||||
static void mirror_log(uint64_t frame);
|
||||
|
||||
void vk_mem_census(uint64_t frame) {
|
||||
static uint64_t window_start;
|
||||
if (rt_trace & RT_TRACE_PERF) dirty_frame(frame);
|
||||
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
|
||||
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame); // and the
|
||||
// write-watch line
|
||||
if (frame == 1200) { // loading is over: the first window starts here
|
||||
for (int c = 0; c < VK_MCLS_N; c++)
|
||||
for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0);
|
||||
@@ -497,6 +540,36 @@ static uint32_t *gen_chunk[GEN_CHUNKS];
|
||||
static uint32_t gen_now; // last generation handed out
|
||||
static SRWLOCK gen_lock = SRWLOCK_INIT;
|
||||
static uint64_t gen_pages[1024];
|
||||
// The newest generation stamped on any page of each 2 MiB region (the write-watch snapshot regions below): the mirror
|
||||
// sync skips regions with nothing newer than its pool's mirror.
|
||||
#define REG_SHIFT 21
|
||||
static uint32_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT];
|
||||
|
||||
// Every write-watch query of the generation table (gen_lock held exclusively): calls, bytes and time since the last
|
||||
// mirror summary, all consumers together; the queries that found written pages (they also flush TLBs) apart.
|
||||
static uint64_t ww_calls, ww_bytes, ww_ticks, ww_dirty_calls, ww_dirty_ticks, ww_pages;
|
||||
|
||||
static uint64_t ww_query(uint64_t addr, uint64_t size) {
|
||||
uint64_t t0 = qpc();
|
||||
uint64_t n = mem_written_pages(addr, size, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
||||
uint64_t dt = qpc() - t0;
|
||||
ww_ticks += dt, ww_calls++, ww_bytes += size;
|
||||
if (n && n != UINT64_MAX) ww_dirty_calls++, ww_dirty_ticks += dt, ww_pages += n;
|
||||
return n;
|
||||
}
|
||||
|
||||
// Frozen regions (mirror mode): a 2 MiB region whose pages nothing wrote for freeze_idle submissions, and which lies
|
||||
// wholly in imported pools, is made read-only (memory.c mem_freeze), and write-watch queries skip it until a write or a
|
||||
// file read thaws it. Most of the pool memory the mirror and the texture checks cover is static (meshes, textures), and
|
||||
// a query costs about 1 us per MiB, so this keeps the queries to the regions the CPU writes.
|
||||
#define FREEZE_PER_SUB 32
|
||||
static int freeze_idle; // 0: off
|
||||
static uint32_t ww_ep = 1; // gen_lock: the submission being recorded (vk_mem_mirror_begin)
|
||||
static uint32_t reg_last[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of the region's last stamp
|
||||
static uint32_t reg_tried[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of its last freeze candidacy
|
||||
static uint32_t *fcand; // gen_lock: regions to try at the next freeze pass
|
||||
static int nfcand, cfcand;
|
||||
static uint64_t n_frz_nopool, n_frz_refused; // gen_lock: candidates not wholly in pools, refused by mem_freeze
|
||||
|
||||
static uint32_t *gen_slot(uint64_t page_addr, int create) {
|
||||
uint64_t ci = page_addr >> GEN_CHUNK_SHIFT;
|
||||
@@ -509,11 +582,17 @@ static uint32_t *gen_slot(uint64_t page_addr, int create) {
|
||||
return &gen_chunk[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)];
|
||||
}
|
||||
|
||||
// gen_lock held exclusively; g is the newest generation handed out.
|
||||
static void stamp_page(uint64_t a, uint32_t g) {
|
||||
uint32_t *s = gen_slot(a, 1);
|
||||
if (!s) return;
|
||||
*s = g;
|
||||
reg_max[a >> REG_SHIFT] = g;
|
||||
reg_last[a >> REG_SHIFT] = ww_ep;
|
||||
}
|
||||
|
||||
static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) {
|
||||
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) {
|
||||
uint32_t *s = gen_slot(a, 1);
|
||||
if (s) *s = g;
|
||||
}
|
||||
for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page(a, g);
|
||||
}
|
||||
|
||||
void vk_guest_written(uint64_t addr, uint64_t size) {
|
||||
@@ -529,14 +608,12 @@ int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now)
|
||||
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
uint32_t g = ++gen_now;
|
||||
int tracked = 1;
|
||||
for (;;) {
|
||||
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
||||
int tracked = 1, frozen = 1;
|
||||
for (uint64_t a = lo & ~((1ull << REG_SHIFT) - 1); a < hi && frozen; a += 1ull << REG_SHIFT) frozen = mem_frozen(a);
|
||||
while (!frozen) {
|
||||
uint64_t n = ww_query(lo, hi - lo);
|
||||
if (n == UINT64_MAX) { tracked = 0; break; }
|
||||
for (uint64_t k = 0; k < n; k++) {
|
||||
uint32_t *s = gen_slot(gen_pages[k], 1);
|
||||
if (s) *s = g;
|
||||
}
|
||||
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
|
||||
if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call
|
||||
}
|
||||
int changed = 0;
|
||||
@@ -560,6 +637,44 @@ static uint32_t snap_at[SNAP_REGIONS]; // epoch of the region's last snapsh
|
||||
static uint8_t snap_untracked[SNAP_REGIONS];
|
||||
static uint64_t n_snap_checks, n_snap_calls, n_snap_untracked;
|
||||
|
||||
// gen_lock held exclusively. Queries the regions r0..r1 that were not snapped in epoch, one query per run of them;
|
||||
// returns the number of queries.
|
||||
static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) {
|
||||
uint32_t g = 0;
|
||||
uint64_t calls = 0;
|
||||
for (uint64_t r = r0; r <= r1;) {
|
||||
if (snap_at[r] == epoch) { r++; continue; }
|
||||
if (mem_frozen(r << SNAP_SHIFT)) { // nothing in it changed since its freeze
|
||||
snap_at[r] = epoch, snap_untracked[r] = 0;
|
||||
r++;
|
||||
continue;
|
||||
}
|
||||
uint64_t k = r; // one query over a run of regions not seen in this epoch
|
||||
while (k + 1 <= r1 && snap_at[k + 1] != epoch && !mem_frozen((k + 1) << SNAP_SHIFT)) k++;
|
||||
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
|
||||
int tracked = 1;
|
||||
if (!g) g = ++gen_now;
|
||||
for (;;) {
|
||||
uint64_t n = ww_query(a, b - a);
|
||||
if (n == UINT64_MAX) { tracked = 0; break; }
|
||||
for (uint64_t j = 0; j < n; j++) stamp_page(gen_pages[j], g);
|
||||
if (n < sizeof gen_pages / sizeof *gen_pages) break;
|
||||
}
|
||||
calls++;
|
||||
for (uint64_t j = r; j <= k; j++) {
|
||||
snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
|
||||
if (tracked && freeze_idle && ww_ep - reg_last[j] >= (uint32_t)freeze_idle &&
|
||||
ww_ep - reg_tried[j] >= (uint32_t)freeze_idle) {
|
||||
reg_tried[j] = ww_ep;
|
||||
if (nfcand == cfcand) grow((void **)&fcand, &cfcand, sizeof *fcand);
|
||||
fcand[nfcand++] = (uint32_t)j;
|
||||
}
|
||||
}
|
||||
r = k + 1;
|
||||
}
|
||||
return calls;
|
||||
}
|
||||
|
||||
int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) {
|
||||
if (!size) { *now = since; return 0; }
|
||||
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
||||
@@ -572,26 +687,7 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e
|
||||
if (excl) {
|
||||
ReleaseSRWLockShared(&gen_lock);
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
uint32_t g = ++gen_now;
|
||||
for (uint64_t r = r0; r <= r1;) {
|
||||
if (snap_at[r] == epoch) { r++; continue; }
|
||||
uint64_t k = r; // one query over a run of regions not seen in this epoch
|
||||
while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++;
|
||||
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
|
||||
int tracked = 1;
|
||||
for (;;) {
|
||||
uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
||||
if (n == UINT64_MAX) { tracked = 0; break; }
|
||||
for (uint64_t j = 0; j < n; j++) {
|
||||
uint32_t *s = gen_slot(gen_pages[j], 1);
|
||||
if (s) *s = g;
|
||||
}
|
||||
if (n < sizeof gen_pages / sizeof *gen_pages) break;
|
||||
}
|
||||
n_snap_calls++;
|
||||
for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
|
||||
r = k + 1;
|
||||
}
|
||||
n_snap_calls += snap_regions(r0, r1, epoch);
|
||||
}
|
||||
int untracked = 0, changed = 0;
|
||||
for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r];
|
||||
@@ -661,18 +757,15 @@ static uint64_t page_hi(const PoolRec *r) { return (r->lo + r->size + 0xfff) & ~
|
||||
// gen_lock held exclusively. 0: the range is not write-watched.
|
||||
static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
|
||||
for (;;) {
|
||||
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
||||
uint64_t n = ww_query(lo, hi - lo);
|
||||
if (n == UINT64_MAX) return 0;
|
||||
for (uint64_t k = 0; k < n; k++) {
|
||||
uint32_t *s = gen_slot(gen_pages[k], 1);
|
||||
if (s) *s = g;
|
||||
}
|
||||
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
|
||||
if (n < sizeof gen_pages / sizeof *gen_pages) return 1;
|
||||
}
|
||||
}
|
||||
|
||||
void vk_mem_dirty_submit(void) {
|
||||
if (!(rt_trace & RT_TRACE_PERF)) return;
|
||||
if (!(rt_trace & RT_TRACE_PERF) || vk_mirror_on) return; // the mirror's own line replaces the census
|
||||
uint64_t t0 = kern_now_ns();
|
||||
uint32_t prev = dirty_sub; // the submit whose replay noted the bindings
|
||||
dirty_sub = prev + 1 ? prev + 1 : 1;
|
||||
@@ -793,8 +886,563 @@ static void dirty_frame(uint64_t frame) {
|
||||
rt_log("%s", line);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- VRAM mirror of buffer-bound pools
|
||||
// build/mem-plan.md, "Design: VRAM mirror of buffer-bound pools". An imported pool keeps vertices, indices, constants
|
||||
// and storage buffers in system memory, so a discrete GPU reads them over PCIe on every draw (GTX 1050 Ti and RTX
|
||||
// 3090 Ti: about 110 ms of GPU time per frame in the field). In mirror mode (device class discrete, TOTK_MEMORY) an
|
||||
// imported pool gets a device-local buffer over its imported range the first time the replay uses it as a vertex,
|
||||
// index, uniform, storage or indirect buffer (vk_mem_use). From then on those bindings, and the buffer copies, fills
|
||||
// and uniform updates that touch the pool (vk_mem_pick), use the mirror at the same offsets. Guest memory stays what
|
||||
// the CPU reads and writes; textures and every host path that reads or writes pool bytes keep using it.
|
||||
// - CPU -> mirror: the first access of a submission to a mirror syncs it (mirror_sync): the 2 MiB write-watch
|
||||
// snapshots of its regions for this epoch (snap_regions, shared with the texture checks: the OS bits keep one
|
||||
// consumer, the shared page generations) and every page stamped after the mirror's mgen goes into a copy; so do the
|
||||
// nvnMemoryPoolFlushMappedRange ranges. The first use copies the whole range. The copies run in a prologue command
|
||||
// buffer that vk_sync.c submits right before the part that accessed the mirror first, between full barriers.
|
||||
// vk_mem_mirror_begin snaps the regions of the pools the previous submission accessed in a few merged queries.
|
||||
// - GPU -> guest: what the replay writes into a mirror (storage buffers its shaders write, buffer copies, fills,
|
||||
// uniform updates, texture-to-buffer copies: vk_mem_gpu_wrote) goes back into the imported buffer at the end of the
|
||||
// submission, exact byte ranges, before the fence signals (vk_mem_mirror_end).
|
||||
// - TOTK_MEMORY=verify: every submission also copies up to VERIFY_BYTES of mirror pages, round robin, into host memory;
|
||||
// at its fence the pages the CPU has not written since their sync are compared with guest memory.
|
||||
// A pool stays imported when its mirror would not fit the VRAM budget, its range is not write-watched, it overlaps
|
||||
// another pool or the allocation fails.
|
||||
#define VERIFY_BYTES (4u << 20)
|
||||
#define VERIFY_SAMPLES 64
|
||||
#define VERIFY_LOG 16
|
||||
enum { MR_BUDGET, MR_UNTRACKED, MR_OVERLAP, MR_ALLOC, MR_N };
|
||||
|
||||
int vk_mirror_on;
|
||||
static int mirror_verify;
|
||||
static int mirror_gap = 2; // pre-snap: hot pools at most this many 2 MiB regions apart share one query
|
||||
static PoolRec **mlist; // pool_lock: mirrored pools by import_lo
|
||||
static int nm, mcap;
|
||||
static uint32_t msub; // replay: number of the submission being recorded
|
||||
static uint32_t mepoch; // replay: its write-watch snapshot epoch (vk_texture_epoch)
|
||||
typedef struct { PoolRec *r; uint64_t off, size; } MRange;
|
||||
static MRange *pend, *wbl; // replay: copies for the next prologue; GPU writes into mirrors this submission
|
||||
static int npend, cpend, nwbl, cwbl;
|
||||
static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions
|
||||
static int cregions;
|
||||
static SRWLOCK flush_lock = SRWLOCK_INIT;
|
||||
|
||||
typedef struct {
|
||||
uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges;
|
||||
uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes;
|
||||
} MirrorAcc;
|
||||
static MirrorAcc mrep; // replay: this submission
|
||||
static MirrorAcc macc; // mirror_stat_lock: since the last summary
|
||||
static SRWLOCK mirror_stat_lock = SRWLOCK_INIT;
|
||||
static volatile LONG64 m_live_n, m_live_bytes, m_refused_n[MR_N], m_refused_bytes[MR_N];
|
||||
static volatile LONG64 m_flush_calls, m_flush_bytes, v_pages, v_skipped, v_mismatch;
|
||||
|
||||
static void mirror_mode(int discrete) {
|
||||
const char *e = cfg_get("TOTK_MEMORY");
|
||||
int mode = discrete ? 1 : 0; // 0 import, 1 mirror, 2 verify
|
||||
const char *why = discrete ? "auto, discrete gpu" : "auto, uma";
|
||||
if (e && *e && strcmp(e, "auto")) {
|
||||
if (!strcmp(e, "import")) mode = 0, why = "TOTK_MEMORY=import";
|
||||
else if (!strcmp(e, "mirror")) mode = 1, why = "TOTK_MEMORY=mirror";
|
||||
else if (!strcmp(e, "verify")) mode = 2, why = "TOTK_MEMORY=verify";
|
||||
else rt_log("mem: TOTK_MEMORY=%s is not import, mirror, verify or auto: auto", e);
|
||||
}
|
||||
vk_mirror_on = mode > 0;
|
||||
mirror_verify = mode == 2;
|
||||
const char *g = cfg_get("TOTK_MIRROR_GAP");
|
||||
mirror_gap = g && *g ? atoi(g) : 2;
|
||||
if (mirror_gap < 0) mirror_gap = 0;
|
||||
const char *fi = cfg_get("TOTK_FREEZE_IDLE"); // submissions a region stays unwritten before it is frozen
|
||||
freeze_idle = !vk_mirror_on ? 0 : fi && *fi ? atoi(fi) : 300;
|
||||
if (freeze_idle < 0) freeze_idle = 0;
|
||||
rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why,
|
||||
mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory");
|
||||
}
|
||||
|
||||
static void range_add(MRange **a, int *n, int *cap, PoolRec *r, uint64_t off, uint64_t size) {
|
||||
if (*n && (*a)[*n - 1].r == r && (*a)[*n - 1].off + (*a)[*n - 1].size == off) {
|
||||
(*a)[*n - 1].size += size;
|
||||
return;
|
||||
}
|
||||
if (*n == *cap) grow((void **)a, cap, sizeof **a);
|
||||
(*a)[(*n)++] = (MRange){r, off, size};
|
||||
}
|
||||
|
||||
static int range_cmp(const void *x, const void *y) {
|
||||
const MRange *a = x, *b = y;
|
||||
if (a->r != b->r) return (uintptr_t)a->r < (uintptr_t)b->r ? -1 : 1;
|
||||
return a->off < b->off ? -1 : a->off > b->off;
|
||||
}
|
||||
|
||||
// Records the ranges as one vkCmdCopyBuffer per pool, overlapping and adjacent ranges merged; to_mirror: buf -> mbuf,
|
||||
// else mbuf -> buf. Returns the commands recorded; adds what they move to *bytes and *copies.
|
||||
static int copy_ranges(VkCommandBuffer cb, MRange *a, int n, int to_mirror, uint64_t *bytes, uint64_t *copies) {
|
||||
qsort(a, (size_t)n, sizeof *a, range_cmp);
|
||||
int rec = 0;
|
||||
for (int i = 0; i < n;) {
|
||||
PoolRec *r = a[i].r;
|
||||
uint64_t lim = r->import_hi - r->import_lo;
|
||||
int k = 0;
|
||||
for (; i < n && a[i].r == r; i++) {
|
||||
uint64_t lo = a[i].off, hi = a[i].off + a[i].size;
|
||||
if (hi > lim) hi = lim;
|
||||
if (lo >= hi) continue;
|
||||
if (k && lo <= mregions[k - 1].srcOffset + mregions[k - 1].size) {
|
||||
uint64_t end = mregions[k - 1].srcOffset + mregions[k - 1].size;
|
||||
if (hi > end) mregions[k - 1].size += hi - end;
|
||||
continue;
|
||||
}
|
||||
if (k == cregions) grow((void **)&mregions, &cregions, sizeof *mregions);
|
||||
mregions[k++] = (VkBufferCopy){lo, lo, hi - lo};
|
||||
}
|
||||
if (!k) continue;
|
||||
if (to_mirror) vkCmdCopyBuffer(cb, r->buf, r->mbuf, (uint32_t)k, mregions);
|
||||
else vkCmdCopyBuffer(cb, r->mbuf, r->buf, (uint32_t)k, mregions);
|
||||
rec++;
|
||||
for (int j = 0; j < k; j++) *bytes += mregions[j].size;
|
||||
*copies += (uint64_t)k;
|
||||
}
|
||||
return rec;
|
||||
}
|
||||
|
||||
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
|
||||
VkAccessFlags2 da) {
|
||||
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = ss, .srcAccessMask = sa,
|
||||
.dstStageMask = ds, .dstAccessMask = da};
|
||||
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
||||
vkCmdPipelineBarrier2(cb, &di);
|
||||
}
|
||||
|
||||
static void refuse(PoolRec *r, int why, const char *what) {
|
||||
r->mstate = MS_REFUSED;
|
||||
uint64_t n = r->import_hi - r->import_lo;
|
||||
if (InterlockedIncrement64(&m_refused_n[why]) <= 4) {
|
||||
char f[160];
|
||||
flag_names(r->flags, f, sizeof f);
|
||||
rt_log("mem: pool %#llx+%#llx (%s) stays imported: %s", (unsigned long long)r->lo, (unsigned long long)r->size,
|
||||
f, what);
|
||||
}
|
||||
InterlockedAdd64(&m_refused_bytes[why], (LONG64)n);
|
||||
}
|
||||
|
||||
// Replay thread: the pool's mirror on its first buffer use.
|
||||
static void mirror_create(PoolRec *r) {
|
||||
VkCtx *c = vk_ctx();
|
||||
uint64_t n = r->import_hi - r->import_lo;
|
||||
r->mstate = MS_REFUSED;
|
||||
if (r->kind != POOL_IMPORT || !r->buf || !n) return;
|
||||
AcquireSRWLockShared(&pool_lock);
|
||||
int i = pidx_upper(&pool_index, r->lo) - 1;
|
||||
while (i >= 0 && pool_index.e[i].buf != r) i--;
|
||||
int alone = i >= 0 && (i == 0 || pool_index.maxhi[i - 1] <= pool_index.e[i].lo) &&
|
||||
(i + 1 == pool_index.n || pool_index.e[i + 1].lo >= pool_index.e[i].hi);
|
||||
ReleaseSRWLockShared(&pool_lock);
|
||||
if (i < 0) return; // finalized meanwhile
|
||||
if (!alone) return refuse(r, MR_OVERLAP, "it overlaps another pool");
|
||||
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
|
||||
if (r1 >= SNAP_REGIONS) return refuse(r, MR_UNTRACKED, "outside the write-watched address space");
|
||||
uint64_t used, budget;
|
||||
int exact;
|
||||
if (vk_mem_vram(&used, &budget, &exact) && used + n + budget / 8 > budget) {
|
||||
char why[128];
|
||||
snprintf(why, sizeof why, "its %.1f MiB mirror does not fit the VRAM budget (%.0f of %.0f MiB in use)",
|
||||
n / 1048576.0, used / 1048576.0, budget / 1048576.0);
|
||||
return refuse(r, MR_BUDGET, why);
|
||||
}
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
mrep.sync_queries += snap_regions(r0, r1, mepoch);
|
||||
int untracked = 0;
|
||||
for (uint64_t k = r0; k <= r1; k++) untracked |= snap_untracked[k];
|
||||
r->mgen = gen_now; // the whole-range copy below holds every write observed so far
|
||||
ReleaseSRWLockExclusive(&gen_lock);
|
||||
if (untracked) return refuse(r, MR_UNTRACKED, "its range is not write-watched");
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = n, .usage = pool_usage(c),
|
||||
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
|
||||
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
||||
.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
|
||||
VkResult res = vmaCreateBuffer(c->vma, &bi, &ai, &r->mbuf, &r->malloc_, NULL);
|
||||
if (res != VK_SUCCESS) {
|
||||
r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL;
|
||||
char why[96];
|
||||
snprintf(why, sizeof why, "device-local allocation of %.1f MiB failed (%d)", n / 1048576.0, (int)res);
|
||||
return refuse(r, MR_ALLOC, why);
|
||||
}
|
||||
AcquireSRWLockExclusive(&pool_lock);
|
||||
if (nm == mcap) grow((void **)&mlist, &mcap, sizeof *mlist);
|
||||
int at = nm;
|
||||
while (at > 0 && mlist[at - 1]->import_lo > r->import_lo) at--;
|
||||
memmove(mlist + at + 1, mlist + at, (size_t)(nm - at) * sizeof *mlist);
|
||||
mlist[at] = r;
|
||||
nm++;
|
||||
ReleaseSRWLockExclusive(&pool_lock);
|
||||
r->acc_sub = msub;
|
||||
r->mstate = MS_ON;
|
||||
range_add(&pend, &npend, &cpend, r, 0, n);
|
||||
mrep.first_n++, mrep.first_bytes += n;
|
||||
InterlockedIncrement64(&m_live_n);
|
||||
InterlockedAdd64(&m_live_bytes, (LONG64)n);
|
||||
}
|
||||
|
||||
static void mirror_forget(PoolRec *r) {
|
||||
for (int i = 0; i < nm; i++)
|
||||
if (mlist[i] == r) {
|
||||
memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist);
|
||||
nm--;
|
||||
InterlockedDecrement64(&m_live_n);
|
||||
InterlockedAdd64(&m_live_bytes, -(LONG64)(r->import_hi - r->import_lo));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran.
|
||||
static PoolRec **retiring;
|
||||
static int nretiring, cretiring;
|
||||
static SRWLOCK retire_lock = SRWLOCK_INIT;
|
||||
|
||||
static void mirror_retiring(PoolRec *r) {
|
||||
AcquireSRWLockExclusive(&retire_lock);
|
||||
if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring);
|
||||
retiring[nretiring++] = r;
|
||||
ReleaseSRWLockExclusive(&retire_lock);
|
||||
}
|
||||
|
||||
static void mirror_retire(void *arg) {
|
||||
AcquireSRWLockExclusive(&retire_lock);
|
||||
for (int i = 0; i < nretiring; i++)
|
||||
if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; }
|
||||
ReleaseSRWLockExclusive(&retire_lock);
|
||||
destroy_rec(vk_ctx(), arg);
|
||||
}
|
||||
|
||||
// Replay thread, the first access of this submission: queues the copies of the pages written since the last sync.
|
||||
static void mirror_sync(PoolRec *r) {
|
||||
uint64_t t0 = qpc();
|
||||
r->acc_sub = msub;
|
||||
uint64_t lo = r->import_lo, hi = r->import_hi;
|
||||
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
||||
int excl = 0;
|
||||
AcquireSRWLockShared(&gen_lock);
|
||||
for (uint64_t k = r0; k <= r1; k++)
|
||||
if (snap_at[k] != mepoch) { excl = 1; break; }
|
||||
if (excl) {
|
||||
ReleaseSRWLockShared(&gen_lock);
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
mrep.sync_queries += snap_regions(r0, r1, mepoch);
|
||||
}
|
||||
uint32_t since = r->mgen;
|
||||
for (uint64_t k = r0; k <= r1; k++) {
|
||||
if (reg_max[k] <= since) continue;
|
||||
uint64_t a = k << SNAP_SHIFT, b = a + (1ull << SNAP_SHIFT);
|
||||
if (a < lo) a = lo;
|
||||
if (b > hi) b = hi;
|
||||
const uint32_t *ch = gen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk
|
||||
if (!ch) continue;
|
||||
for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT)
|
||||
if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since)
|
||||
range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT);
|
||||
}
|
||||
r->mgen = gen_now;
|
||||
if (excl) ReleaseSRWLockExclusive(&gen_lock);
|
||||
else ReleaseSRWLockShared(&gen_lock);
|
||||
if (__atomic_load_n(&r->fl_hi, __ATOMIC_ACQUIRE)) {
|
||||
AcquireSRWLockExclusive(&flush_lock);
|
||||
uint64_t a = r->fl_lo, b = r->fl_hi;
|
||||
r->fl_hi = 0;
|
||||
ReleaseSRWLockExclusive(&flush_lock);
|
||||
a = a > lo ? a & ~0xfffull : lo;
|
||||
b = b < hi ? (b + 0xfff) & ~0xfffull : hi;
|
||||
if (a < b) range_add(&pend, &npend, &cpend, r, a - lo, b - a);
|
||||
}
|
||||
mrep.syncs++;
|
||||
mrep.sync_t += qpc() - t0;
|
||||
}
|
||||
|
||||
void vk_mem_use(VkMemSpan *sp) {
|
||||
PoolRec *r = sp->pool;
|
||||
if (r->mstate == MS_NONE) mirror_create(r);
|
||||
if (r->mstate != MS_ON) return;
|
||||
if (r->acc_sub != msub) mirror_sync(r);
|
||||
sp->buffer = r->mbuf;
|
||||
}
|
||||
|
||||
void vk_mem_pick(VkMemSpan *sp) {
|
||||
PoolRec *r = sp->pool;
|
||||
if (!r || r->mstate != MS_ON) return;
|
||||
if (r->acc_sub != msub) mirror_sync(r);
|
||||
sp->buffer = r->mbuf;
|
||||
}
|
||||
|
||||
int vk_mem_mirror_probe(uint64_t addr) {
|
||||
VkMemSpan sp;
|
||||
if (!vk_mirror_on || !vk_mem_resolve(addr, 1, &sp) || !sp.pool) return 0;
|
||||
mepoch = vk_texture_epoch();
|
||||
vk_mem_use(&sp);
|
||||
return ((PoolRec *)sp.pool)->mstate == MS_ON;
|
||||
}
|
||||
|
||||
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) {
|
||||
PoolRec *r = sp->pool;
|
||||
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return;
|
||||
range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size);
|
||||
}
|
||||
|
||||
void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) {
|
||||
if (!vk_mirror_on || !size || !p->mem) return;
|
||||
InterlockedIncrement64(&m_flush_calls);
|
||||
InterlockedAdd64(&m_flush_bytes, (LONG64)size);
|
||||
AcquireSRWLockShared(&pool_lock);
|
||||
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)p->mem, 0);
|
||||
PoolRec *r = e && e->key == p ? e->buf : NULL;
|
||||
if (r && r->mstate == MS_ON) {
|
||||
uint64_t a = (uintptr_t)p->mem + off, b = a + size;
|
||||
AcquireSRWLockExclusive(&flush_lock);
|
||||
if (!r->fl_hi) r->fl_lo = a, r->fl_hi = b;
|
||||
else {
|
||||
if (a < r->fl_lo) r->fl_lo = a;
|
||||
if (b > r->fl_hi) r->fl_hi = b;
|
||||
}
|
||||
ReleaseSRWLockExclusive(&flush_lock);
|
||||
}
|
||||
ReleaseSRWLockShared(&pool_lock);
|
||||
}
|
||||
|
||||
// pool_lock shared: [a, b) lies wholly in the host ranges of imported pools with a GPU buffer.
|
||||
static int pools_cover(uint64_t a, uint64_t b) {
|
||||
uint64_t lo[64], hi[64];
|
||||
int n = 0;
|
||||
for (int i = pidx_upper(&pool_index, b - 1) - 1; i >= 0 && pool_index.maxhi[i] > a; i--) {
|
||||
const PoolRec *r = pool_index.e[i].buf;
|
||||
if (r->kind != POOL_IMPORT || !r->buf || r->import_hi <= a || r->import_lo >= b) continue;
|
||||
if (n == 64) return 0;
|
||||
int at = n++;
|
||||
for (; at > 0 && lo[at - 1] > r->import_lo; at--) lo[at] = lo[at - 1], hi[at] = hi[at - 1];
|
||||
lo[at] = r->import_lo, hi[at] = r->import_hi;
|
||||
}
|
||||
uint64_t end = a;
|
||||
for (int i = 0; i < n && end < b; i++) {
|
||||
if (lo[i] > end) return 0;
|
||||
if (hi[i] > end) end = hi[i];
|
||||
}
|
||||
return end >= b;
|
||||
}
|
||||
|
||||
// pool_lock shared, gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the
|
||||
// protection collects the writes made before it took effect.
|
||||
static void freeze_pass(void) {
|
||||
for (int done = 0; nfcand && done < FREEZE_PER_SUB;) {
|
||||
uint64_t j = fcand[--nfcand], a = j << REG_SHIFT, b = a + (1ull << REG_SHIFT);
|
||||
if (ww_ep - reg_last[j] < (uint32_t)freeze_idle || mem_frozen(a)) continue;
|
||||
if (!pools_cover(a, b)) { n_frz_nopool++; continue; }
|
||||
done++;
|
||||
if (!mem_freeze(a)) { n_frz_refused++; continue; }
|
||||
query_stamp(a, b, ++gen_now);
|
||||
}
|
||||
}
|
||||
|
||||
void vk_mem_mirror_begin(void) {
|
||||
if (!vk_mirror_on) return;
|
||||
uint64_t t0 = qpc();
|
||||
uint32_t prev = msub;
|
||||
msub = msub + 1 ? msub + 1 : 1;
|
||||
mepoch = vk_texture_epoch();
|
||||
uint64_t q = 0, run0 = 0, run1 = 0;
|
||||
int have = 0;
|
||||
AcquireSRWLockShared(&pool_lock);
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
ww_ep++;
|
||||
uint64_t b0 = ww_bytes;
|
||||
for (int i = 0; i < nm; i++) {
|
||||
const PoolRec *r = mlist[i];
|
||||
if (r->acc_sub != prev) continue;
|
||||
uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT;
|
||||
if (have && a <= run1 + 1 + (uint64_t)mirror_gap) {
|
||||
if (b > run1) run1 = b;
|
||||
continue;
|
||||
}
|
||||
if (have) q += snap_regions(run0, run1, mepoch);
|
||||
run0 = a, run1 = b, have = 1;
|
||||
}
|
||||
if (have) q += snap_regions(run0, run1, mepoch);
|
||||
mrep.presnap_bytes += ww_bytes - b0;
|
||||
if (freeze_idle) freeze_pass();
|
||||
ReleaseSRWLockExclusive(&gen_lock);
|
||||
ReleaseSRWLockShared(&pool_lock);
|
||||
mrep.subs++;
|
||||
mrep.presnap_queries += q;
|
||||
mrep.presnap_t += qpc() - t0;
|
||||
}
|
||||
|
||||
int vk_mem_mirror_pending(void) { return npend; }
|
||||
|
||||
void vk_mem_mirror_prologue(VkCommandBuffer cb) {
|
||||
if (!npend) return;
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
||||
copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies);
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
||||
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
||||
npend = 0;
|
||||
mrep.prologues++;
|
||||
}
|
||||
|
||||
// ---- verify mode
|
||||
typedef struct { PoolRec *r; uint64_t off, len, dst; uint32_t mgen; } VSample;
|
||||
typedef struct { uint32_t sub; int n; VkBuffer buf; VmaAllocation alloc; const uint8_t *map; VSample s[VERIFY_SAMPLES]; } VJob;
|
||||
static int ver_i;
|
||||
static uint64_t ver_off;
|
||||
|
||||
// The CPU wrote the page since the mirror's sync that mgen records: stamped later, or its write-watch bit is set.
|
||||
static int page_dirty(uint64_t a, uint32_t mgen) {
|
||||
uint32_t *s = gen_slot(a, 0);
|
||||
if (s && __atomic_load_n(s, __ATOMIC_RELAXED) > mgen) return 1;
|
||||
uint64_t p[1];
|
||||
return mem_written_peek(a, 1ull << GEN_PAGE_SHIFT, p, 1) != 0; // UINT64_MAX (not watched) counts as written
|
||||
}
|
||||
|
||||
static void verify_cb(void *arg) { // completion thread
|
||||
VJob *j = arg;
|
||||
static volatile LONG logged;
|
||||
uint64_t pages = 0, skipped = 0, bad = 0;
|
||||
for (int i = 0; i < j->n; i++) {
|
||||
const VSample *s = &j->s[i];
|
||||
PoolRec *r = s->r;
|
||||
uint32_t wb = __atomic_load_n(&r->wb_sub, __ATOMIC_RELAXED);
|
||||
if (wb != j->sub && wb - j->sub < 0x80000000u) { // a later submission wrote back into it
|
||||
skipped += s->len >> GEN_PAGE_SHIFT;
|
||||
continue;
|
||||
}
|
||||
for (uint64_t o = 0; o < s->len; o += 1ull << GEN_PAGE_SHIFT) {
|
||||
uint64_t a = r->import_lo + s->off + o;
|
||||
const uint8_t *m = j->map + s->dst + o, *g = (const uint8_t *)(uintptr_t)a;
|
||||
pages++;
|
||||
if (!memcmp(m, g, 1ull << GEN_PAGE_SHIFT)) continue;
|
||||
if (page_dirty(a, s->mgen)) { skipped++; continue; }
|
||||
bad++;
|
||||
if (InterlockedIncrement(&logged) <= VERIFY_LOG) {
|
||||
int k = 0, e = (1 << GEN_PAGE_SHIFT) - 1;
|
||||
while (k < e && m[k] == g[k]) k++;
|
||||
while (e > k && m[e] == g[e]) e--;
|
||||
char f[160];
|
||||
flag_names(r->flags, f, sizeof f);
|
||||
rt_log("mem: verify mismatch: pool %#llx+%#llx (%s) offset %#llx: bytes %#x..%#x of the page differ, "
|
||||
"mirror %02x guest %02x at the first (submission %u)", (unsigned long long)r->lo,
|
||||
(unsigned long long)r->size, f, (unsigned long long)(a - r->lo), k, e, m[k], g[k], j->sub);
|
||||
}
|
||||
}
|
||||
}
|
||||
InterlockedAdd64(&v_pages, (LONG64)pages);
|
||||
InterlockedAdd64(&v_skipped, (LONG64)skipped);
|
||||
InterlockedAdd64(&v_mismatch, (LONG64)bad);
|
||||
vmaDestroyBuffer(vk_ctx()->vma, j->buf, j->alloc);
|
||||
free(j);
|
||||
}
|
||||
|
||||
static int verify_record(VkCommandBuffer cb, uint64_t fence) {
|
||||
VJob *j = calloc(1, sizeof *j);
|
||||
if (!j) rt_fatal("vk: out of memory");
|
||||
uint64_t left = VERIFY_BYTES, dst = 0;
|
||||
AcquireSRWLockShared(&pool_lock);
|
||||
for (int tries = 0; nm && left && j->n < VERIFY_SAMPLES && tries <= nm; tries++) {
|
||||
if (ver_i >= nm) ver_i = 0, ver_off = 0;
|
||||
PoolRec *r = mlist[ver_i];
|
||||
uint64_t size = r->import_hi - r->import_lo;
|
||||
if (ver_off >= size) { ver_i++, ver_off = 0; continue; }
|
||||
uint64_t len = size - ver_off < left ? size - ver_off : left;
|
||||
j->s[j->n++] = (VSample){r, ver_off, len, dst, r->mgen};
|
||||
dst += len, left -= len, ver_off += len;
|
||||
if (ver_off >= size) ver_i++, ver_off = 0;
|
||||
}
|
||||
ReleaseSRWLockShared(&pool_lock);
|
||||
if (!j->n) { free(j); return 0; }
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = dst,
|
||||
.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
||||
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
||||
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
||||
VmaAllocationInfo info;
|
||||
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &j->buf, &j->alloc, &info) != VK_SUCCESS) { free(j); return 0; }
|
||||
j->map = info.pMappedData;
|
||||
j->sub = msub;
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_TRANSFER_READ_BIT);
|
||||
for (int i = 0; i < j->n; i++) {
|
||||
VkBufferCopy c = {j->s[i].off, j->s[i].dst, j->s[i].len};
|
||||
vkCmdCopyBuffer(cb, j->s[i].r->mbuf, j->buf, 1, &c);
|
||||
}
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
|
||||
VK_ACCESS_2_HOST_READ_BIT);
|
||||
vk_completion_add(fence, verify_cb, j);
|
||||
return j->n + 2;
|
||||
}
|
||||
|
||||
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
|
||||
if (!vk_mirror_on) return 0;
|
||||
int rec = 0;
|
||||
if (nwbl) {
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_TRANSFER_READ_BIT);
|
||||
for (int i = 0; i < nwbl; i++) wbl[i].r->wb_sub = msub;
|
||||
mrep.wb_ranges += (uint64_t)nwbl;
|
||||
rec += copy_ranges(cb, wbl, nwbl, 0, &mrep.wb_bytes, &mrep.wb_copies) + 2;
|
||||
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
|
||||
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
||||
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
|
||||
nwbl = 0;
|
||||
}
|
||||
if (mirror_verify) rec += verify_record(cb, fence);
|
||||
AcquireSRWLockExclusive(&mirror_stat_lock);
|
||||
uint64_t *d = (uint64_t *)&macc, *s = (uint64_t *)&mrep;
|
||||
for (size_t i = 0; i < sizeof macc / sizeof(uint64_t); i++) d[i] += s[i];
|
||||
ReleaseSRWLockExclusive(&mirror_stat_lock);
|
||||
memset(&mrep, 0, sizeof mrep);
|
||||
return rec;
|
||||
}
|
||||
|
||||
// One summary line: with TOTK_TRACE=perf every 60 frames, else at frames 600 and 1800.
|
||||
static void mirror_log(uint64_t frame) {
|
||||
static uint64_t last, ww_last;
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
uint64_t wc = ww_calls, wb = ww_bytes, wt = ww_ticks, dc = ww_dirty_calls, dt = ww_dirty_ticks, wp = ww_pages;
|
||||
ww_pages = 0;
|
||||
uint64_t np = n_frz_nopool, nr = n_frz_refused;
|
||||
ww_calls = ww_bytes = ww_ticks = ww_dirty_calls = ww_dirty_ticks = n_frz_nopool = n_frz_refused = 0;
|
||||
ReleaseSRWLockExclusive(&gen_lock);
|
||||
double wf = frame > ww_last ? (double)(frame - ww_last) : 1.0;
|
||||
ww_last = frame;
|
||||
uint64_t fz, ff, ft, fo;
|
||||
mem_freeze_stats(&fz, &ff, &ft, &fo);
|
||||
if (rt_trace & RT_TRACE_PERF)
|
||||
rt_log("mem: write-watch frame %llu: per frame, every consumer: %.1f queries over %.1f MiB, %.3f ms (%.1f found "
|
||||
"%.0f written pages, %.3f ms); frozen %.0f MiB; since the last line: freezes %llu, thaws by a write %llu, by a file read "
|
||||
"or mapping change %llu, candidates not wholly in pools %llu, refused %llu",
|
||||
(unsigned long long)frame, wc / wf, wb / wf / 1048576.0, qpc_ms(wt) / wf, dc / wf, wp / wf, qpc_ms(dt) / wf,
|
||||
fz * 2.0, (unsigned long long)ff, (unsigned long long)ft, (unsigned long long)fo, (unsigned long long)np,
|
||||
(unsigned long long)nr);
|
||||
if (!vk_mirror_on) return;
|
||||
AcquireSRWLockExclusive(&mirror_stat_lock);
|
||||
MirrorAcc a = macc;
|
||||
memset(&macc, 0, sizeof macc);
|
||||
ReleaseSRWLockExclusive(&mirror_stat_lock);
|
||||
double f = frame > last ? (double)(frame - last) : 1.0, M = 1048576.0;
|
||||
last = frame;
|
||||
uint64_t fc = (uint64_t)InterlockedExchange64(&m_flush_calls, 0), fb = (uint64_t)InterlockedExchange64(&m_flush_bytes, 0);
|
||||
char ver[160] = "";
|
||||
if (mirror_verify)
|
||||
snprintf(ver, sizeof ver, "; verify: pages compared %lld, skipped %lld (written since), mismatches %lld",
|
||||
(long long)InterlockedExchange64(&v_pages, 0), (long long)InterlockedExchange64(&v_skipped, 0),
|
||||
(long long)InterlockedExchange64(&v_mismatch, 0));
|
||||
rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked "
|
||||
"%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, "
|
||||
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries "
|
||||
"%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f)%s",
|
||||
(unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET],
|
||||
m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP],
|
||||
(long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f,
|
||||
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M,
|
||||
a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f,
|
||||
qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver);
|
||||
}
|
||||
|
||||
// Device class (build/mem-plan.md): INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a
|
||||
// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Only logged for now.
|
||||
// heap that is not device-local (host memory) -> discrete; otherwise uma (import everywhere). Sets the mirror mode.
|
||||
void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m) {
|
||||
int big_local = -1, host_heap = -1;
|
||||
for (uint32_t h = 0; h < m->memoryHeapCount; h++) {
|
||||
@@ -811,6 +1459,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic
|
||||
else if (big_local < 0) cls = "uma", why = "no device-local heap of 1 GiB or more";
|
||||
else if (host_heap < 0) cls = "uma", why = "every heap is device-local";
|
||||
else cls = "discrete", why = "device-local heap separate from host memory";
|
||||
int discrete = cls[0] == 'd';
|
||||
char line[2048];
|
||||
size_t k = (size_t)snprintf(line, sizeof line, "mem: class %s (%s; device type %d %s, %s)", cls, why, t, tn, p->deviceName);
|
||||
for (uint32_t h = 0; h < m->memoryHeapCount; h++)
|
||||
@@ -830,6 +1479,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic
|
||||
f & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT ? " LAZY" : "");
|
||||
}
|
||||
rt_log("%s", line);
|
||||
mirror_mode(discrete);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- upload batches and deferred destruction
|
||||
@@ -1069,6 +1719,9 @@ void vk_mem_shutdown(void) {
|
||||
AcquireSRWLockExclusive(&pool_lock);
|
||||
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
|
||||
pool_index.n = 0;
|
||||
nm = npend = nwbl = 0; // mirrors went with their pools
|
||||
for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran
|
||||
nretiring = 0;
|
||||
__atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE);
|
||||
nphys = 0;
|
||||
ReleaseSRWLockExclusive(&pool_lock);
|
||||
|
||||
+59
-23
@@ -107,27 +107,33 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order
|
||||
free(t);
|
||||
}
|
||||
|
||||
// At the start of a part's command buffer (sub_lock held).
|
||||
static void ts_begin(VkCommandBuffer cb) {
|
||||
ts_cur = -1;
|
||||
if (ts_state != 1) return;
|
||||
// At the start of a command buffer (sub_lock held): the pair it times, -1 untimed.
|
||||
static int ts_open(VkCommandBuffer cb) {
|
||||
if (ts_state != 1) return -1;
|
||||
uint32_t p = ts_next % TS_PAIRS;
|
||||
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return; }
|
||||
if (__atomic_load_n(&ts_busy[p], __ATOMIC_ACQUIRE)) { InterlockedIncrement64(&ts_missed); return -1; }
|
||||
ts_next++;
|
||||
ts_busy[p] = 1;
|
||||
vkCmdResetQueryPool(cb, ts_pool, 2 * p, 2);
|
||||
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, ts_pool, 2 * p);
|
||||
ts_cur = (int)p;
|
||||
return (int)p;
|
||||
}
|
||||
|
||||
// At the end of the part, before vkEndCommandBuffer; fence: the value whose completion covers the part.
|
||||
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
|
||||
if (ts_cur < 0) return;
|
||||
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)ts_cur + 1);
|
||||
// At its end, before vkEndCommandBuffer; fence: the value whose completion covers it.
|
||||
static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence) {
|
||||
if (pair < 0) return;
|
||||
vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * (uint32_t)pair + 1);
|
||||
TsRead *t = malloc(sizeof *t);
|
||||
if (!t) rt_fatal("vk: out of memory");
|
||||
t->pair = (uint32_t)ts_cur;
|
||||
t->pair = (uint32_t)pair;
|
||||
vk_completion_add(fence, ts_read_cb, t);
|
||||
}
|
||||
|
||||
// The pair of the part being recorded.
|
||||
static void ts_begin(VkCommandBuffer cb) { ts_cur = ts_open(cb); }
|
||||
|
||||
static void ts_end(VkCommandBuffer cb, uint64_t fence) {
|
||||
ts_close(cb, ts_cur, fence);
|
||||
ts_cur = -1;
|
||||
}
|
||||
|
||||
@@ -227,27 +233,47 @@ void vk_sync_start(void) {
|
||||
ReleaseSRWLockExclusive(&start_lock);
|
||||
}
|
||||
|
||||
// A command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held.
|
||||
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
|
||||
// The index of a command buffer the GPU no longer uses, claimed for value `fence`. Called with sub_lock held; may move
|
||||
// slots[].
|
||||
static int claim_slot(VkCtx *c, uint64_t fence) {
|
||||
uint64_t gpu = 0;
|
||||
vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu);
|
||||
for (int i = 0; i < nslots; i++)
|
||||
if (slots[i].fence <= gpu) {
|
||||
slots[i].fence = fence;
|
||||
cur_slot = i;
|
||||
return &slots[i];
|
||||
return i;
|
||||
}
|
||||
CmdSlot *n = realloc(slots, sizeof *n * (size_t)(nslots + 1));
|
||||
if (!n) rt_fatal("vk: out of memory (command buffers)");
|
||||
slots = n;
|
||||
CmdSlot *s = &slots[nslots];
|
||||
cur_slot = nslots;
|
||||
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = cmd_pool,
|
||||
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
||||
if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation");
|
||||
s->fence = fence;
|
||||
nslots++;
|
||||
return s;
|
||||
return nslots++;
|
||||
}
|
||||
|
||||
static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
|
||||
cur_slot = claim_slot(c, fence);
|
||||
return &slots[cur_slot];
|
||||
}
|
||||
|
||||
// The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is
|
||||
// submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the
|
||||
// part was recorded, sub_lock held; may move slots[].
|
||||
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
|
||||
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
|
||||
int i = claim_slot(c, fence); // before reading slots: the claim may move it
|
||||
VkCommandBuffer cb = slots[i].cb;
|
||||
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
||||
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
||||
vkBeginCommandBuffer(cb, &bi);
|
||||
int pair = ts_open(cb);
|
||||
vk_mem_mirror_prologue(cb);
|
||||
ts_close(cb, pair, fence);
|
||||
vkEndCommandBuffer(cb);
|
||||
return cb;
|
||||
}
|
||||
|
||||
VkCommandBuffer vk_sync_split(void) {
|
||||
@@ -256,8 +282,11 @@ VkCommandBuffer vk_sync_split(void) {
|
||||
ts_end(cb, cur_fence);
|
||||
vkEndCommandBuffer(cb);
|
||||
vk_upload_flush(0); // the uploads this part asked for precede it in queue order
|
||||
VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb};
|
||||
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = 1, .pCommandBufferInfos = &cbi};
|
||||
VkCommandBuffer pro = prologue(c, cur_fence);
|
||||
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
|
||||
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
|
||||
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
|
||||
.pCommandBufferInfos = pro ? cbi : cbi + 1};
|
||||
vk_queue_lock();
|
||||
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
|
||||
vk_queue_unlock();
|
||||
@@ -292,11 +321,16 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
|
||||
perf_replay(t0);
|
||||
// After the recording: the uploads it asked for (and every earlier one) precede it in queue order.
|
||||
vk_upload_flush(0);
|
||||
VkCommandBufferSubmitInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb};
|
||||
VkCommandBuffer pro = prologue(c, fence);
|
||||
s = &slots[cur_slot];
|
||||
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
|
||||
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}};
|
||||
VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
|
||||
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
|
||||
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = recorded ? 1 : 0,
|
||||
.pCommandBufferInfos = &cbi, .signalSemaphoreInfoCount = 1, .pSignalSemaphoreInfos = &sig};
|
||||
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
|
||||
.commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u),
|
||||
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1,
|
||||
.pSignalSemaphoreInfos = &sig};
|
||||
vk_queue_lock();
|
||||
VkResult r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
|
||||
vk_queue_unlock();
|
||||
@@ -365,6 +399,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
|
||||
|
||||
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
|
||||
|
||||
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
|
||||
|
||||
void vk_sync_finish(void) {
|
||||
vk_sync_start();
|
||||
vk_sync_fence_wait(__atomic_load_n(&issued, __ATOMIC_ACQUIRE), UINT64_MAX);
|
||||
|
||||
@@ -888,6 +888,8 @@ int vk_texture_slot_swz(int id, uint8_t swz[4]) {
|
||||
return ok;
|
||||
}
|
||||
|
||||
uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); }
|
||||
|
||||
void vk_texture_new_submit(void) {
|
||||
lock();
|
||||
if (!++submit_epoch) submit_epoch = 1;
|
||||
|
||||
Reference in new issue
Block a user