mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-18 20:15:56 +00:00
Freeze idle pool regions read-only so write-watch queries skip them
The mirror and the texture checks queried about 930 MiB of write-watched memory per frame (0.96 ms on the RX 9070 machine, about 3 ms on an 8th-gen i7), at about 1 us per MiB, and 770 MiB of it lay in 2 MiB regions nothing had written for two seconds. In mirror mode a region left unwritten for TOTK_FREEZE_IDLE submissions (300; 0 turns it off) and lying wholly in imported pools is now made read-only (memory.c mem_freeze), and the snapshot and change queries skip it. The first write faults and a vectored handler thaws the region before the write retries; file reads into guest memory, which the kernel would refuse on a read-only page, thaw their buffer first and log a refusal, as do pool init and finalize, and commits, decommits and protection changes thaw under the same lock. A region stays thawed for 5 s after a file read. Field: 66 queries over 380 MiB, 0.56 ms per frame; 0.38 ms of it are the queries that find writes. The perf line `mem: write-watch` gives queries, bytes, time, frozen MiB, freezes and thaws. The gfx-abi probe covers freeze, thaw by a write and the hold after mem_thaw. TOTK_MEMORY, TOTK_MIRROR_GAP and TOTK_FREEZE_IDLE are known settings now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1fe552ae0a
commit
51567ac32a
7 files changed
+303
-28
No files matched your search
+2
-2
@@ -27,10 +27,10 @@ static const char *const known[] = {
|
||||
"BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL",
|
||||
"BIND_ZR",
|
||||
"CLOTH_REALTIME", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT",
|
||||
"FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "GPL",
|
||||
"FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "FREEZE_IDLE", "GPL",
|
||||
"GPL_DRAW_BUILD",
|
||||
"GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
|
||||
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MINIMAL_LAYOUT", "MOTION", "MOUSE_SENS", "MOVIE_HOST",
|
||||
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST",
|
||||
"MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SYNC", "PREWARM_XLATE",
|
||||
"PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_STAGE", "RDOC",
|
||||
"RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
|
||||
|
||||
+7
-1
@@ -691,7 +691,13 @@ static File *file_of(uint64_t h, HANDLE *fh, int *mode) {
|
||||
static int64_t host_read_at(HANDLE f, void *buf, uint32_t n, uint64_t off) {
|
||||
OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32)};
|
||||
DWORD got = 0;
|
||||
if (!ReadFile(f, buf, n, &got, &ov)) return GetLastError() == ERROR_HANDLE_EOF ? 0 : -1;
|
||||
mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting
|
||||
if (!ReadFile(f, buf, n, &got, &ov)) {
|
||||
DWORD e = GetLastError();
|
||||
if (e == ERROR_NOACCESS) // a read-only page in the buffer: a frozen region missed its thaw
|
||||
rt_log("fs: read of %u bytes into %p refused: the buffer is not writable", n, buf);
|
||||
return e == ERROR_HANDLE_EOF ? 0 : -1;
|
||||
}
|
||||
return got;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
// Needs no game files.
|
||||
#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
#include <windows.h>
|
||||
|
||||
#include "gfx.h"
|
||||
|
||||
@@ -60,6 +61,22 @@ static void check_write_watch(void) {
|
||||
printf("write-watch: three pages written, max 2: %llu then %llu\n", (unsigned long long)n, (unsigned long long)m);
|
||||
CHECK(n == 2 && m == 1, "max 2 reported %llu, then %llu (want 2, 1)", (unsigned long long)n, (unsigned long long)m);
|
||||
|
||||
// Frozen region: read-only until the first write thaws it (the write lands and is reported); mem_thaw (a file
|
||||
// read) thaws it and holds it thawed; SetHeapSize below decommits it.
|
||||
mem_written_pages(heap, 0x200000, pages, 8);
|
||||
CHECK(mem_freeze(heap) == 1 && mem_frozen(heap), "freeze of an idle committed 2 MiB region");
|
||||
MEMORY_BASIC_INFORMATION mbi;
|
||||
CHECK(VirtualQuery((void *)(uintptr_t)heap, &mbi, sizeof mbi) && mbi.Protect == PAGE_READONLY, "frozen region is not read-only");
|
||||
p[0x3000] = 7;
|
||||
CHECK(!mem_frozen(heap) && p[0x3000] == 7, "a write into a frozen region did not thaw it");
|
||||
n = mem_written_pages(heap, 0x200000, pages, 8);
|
||||
CHECK(n == 1 && pages[0] == heap + 0x3000, "write after a thaw: %llu pages reported (want 1)", (unsigned long long)n);
|
||||
CHECK(mem_freeze(heap) == 1, "refreeze after a write thaw");
|
||||
mem_thaw(heap + 0x6000, 0x1000);
|
||||
CHECK(!mem_frozen(heap) && mem_freeze(heap) == 0, "mem_thaw did not thaw and hold the region");
|
||||
p[0x6000] = 8;
|
||||
printf("write-watch: frozen region thawed by a write (reported once) and by mem_thaw (held)\n");
|
||||
|
||||
uint64_t a = g->alias.base + 0x10000;
|
||||
CHECK(mem_map_physical(a, 0x4000) == 0, "MapPhysicalMemory");
|
||||
mem_written_pages(a, 0x4000, pages, 8);
|
||||
|
||||
@@ -131,6 +131,15 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64
|
||||
uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max);
|
||||
// the same without the reset: the bits stay for the next
|
||||
// mem_written_pages (verify checks only)
|
||||
int mem_freeze(uint64_t a); // makes the 2 MiB region at a (2 MiB aligned, watched,
|
||||
// committed PAGE_READWRITE) read-only until a write or
|
||||
// mem_thaw thaws it; 1 frozen (or already), 0 refused
|
||||
int mem_frozen(uint64_t a); // the 2 MiB region of a is frozen: no page in it changed
|
||||
// since its freeze
|
||||
void mem_thaw(uint64_t a, uint64_t n); // makes the frozen regions over [a, a+n) writable again;
|
||||
// before a kernel write (file read) into guest memory
|
||||
void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws);
|
||||
// regions frozen now; freezes and thaws since the last call
|
||||
void mem_kernel_free(void *p, uint64_t size); // TLR/TLS reuse after thread exit
|
||||
|
||||
// ---- modules (loader.c, T1)
|
||||
|
||||
+126
-7
@@ -67,21 +67,142 @@ static void *reserve_at(uint64_t at, uint64_t size, DWORD extra) {
|
||||
return p;
|
||||
}
|
||||
|
||||
// ---- frozen regions
|
||||
// A 2 MiB region of the watched reservations that nothing wrote for a while can be made read-only (mem_freeze), so the
|
||||
// back end's write-watch queries skip it: no page of a frozen region changes before a thaw. The first write faults and
|
||||
// thaw_handler makes the region writable again before the write retries. A kernel write into guest memory (a file
|
||||
// read) fails on a read-only page instead of faulting, so file reads thaw their buffer first (mem_thaw), and so does
|
||||
// every commit, decommit and protection change here. A region stays thawed for FRZ_HOLD_MS after a mem_thaw: a file
|
||||
// read still in flight writes its buffer later. frz_lock is a leaf lock: nothing writes guest memory under it.
|
||||
#define FRZ_SHIFT 21
|
||||
#define FRZ_SIZE (1ull << FRZ_SHIFT)
|
||||
#define FRZ_HOLD_MS 5000u
|
||||
static volatile uint8_t frz[RT_ADDRESS_LIMIT >> FRZ_SHIFT];
|
||||
static uint32_t thaw_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: GetTickCount of the last thaw request, 0 never
|
||||
static volatile LONG nfrz;
|
||||
static SRWLOCK frz_lock = SRWLOCK_INIT;
|
||||
static volatile LONG64 frz_count[4]; // freezes; thaws by a write fault, by mem_thaw, by a mapping change
|
||||
|
||||
static int watched(uint64_t addr, uint64_t size) {
|
||||
const RtGuestLayout *g = &L;
|
||||
return (addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size) ||
|
||||
(addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size);
|
||||
}
|
||||
|
||||
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
|
||||
static void thaw_locked(uint64_t r, int why) {
|
||||
if (why == 2) thaw_ms[r] = (uint32_t)GetTickCount64() | 1;
|
||||
if (!frz[r]) return;
|
||||
DWORD old;
|
||||
uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE;
|
||||
if (!VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READWRITE, &old)) {
|
||||
// Not all committed (cannot happen while mapping changes thaw under frz_lock): the read-only runs only.
|
||||
MEMORY_BASIC_INFORMATION mbi;
|
||||
while (a < end && VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi)) {
|
||||
uint64_t b = (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize;
|
||||
if (b > end) b = end;
|
||||
if (mbi.State == MEM_COMMIT && mbi.Protect == PAGE_READONLY &&
|
||||
!VirtualProtect((void *)(uintptr_t)a, b - a, PAGE_READWRITE, &old))
|
||||
rt_fatal("memory: thawing %#llx+%#llx failed (error %lu)", (unsigned long long)a,
|
||||
(unsigned long long)(b - a), GetLastError());
|
||||
a = b;
|
||||
}
|
||||
}
|
||||
__atomic_store_n(&frz[r], 0, __ATOMIC_RELEASE);
|
||||
InterlockedDecrement(&nfrz);
|
||||
InterlockedIncrement64(&frz_count[why]);
|
||||
}
|
||||
|
||||
// frz_lock held exclusively.
|
||||
static void thaw_range_locked(uint64_t a, uint64_t n, int why) {
|
||||
if (!n || a >= RT_ADDRESS_LIMIT) return;
|
||||
uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT;
|
||||
for (uint64_t r = r0; r <= r1; r++) thaw_locked(r, why);
|
||||
}
|
||||
|
||||
void mem_thaw(uint64_t a, uint64_t n) {
|
||||
AcquireSRWLockExclusive(&frz_lock); // also when nothing is frozen: the hold keeps a read's buffer writable
|
||||
thaw_range_locked(a, n, 2);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
}
|
||||
|
||||
int mem_freeze(uint64_t a) {
|
||||
if ((a & (FRZ_SIZE - 1)) || !watched(a, FRZ_SIZE)) return 0;
|
||||
uint64_t r = a >> FRZ_SHIFT;
|
||||
MEMORY_BASIC_INFORMATION mbi;
|
||||
DWORD old;
|
||||
int ok = 0;
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
if (frz[r]) ok = 1;
|
||||
else if (thaw_ms[r] && (int32_t)((uint32_t)GetTickCount64() - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) ok = 0; // |1 may be ahead
|
||||
else if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT &&
|
||||
mbi.Protect == PAGE_READWRITE && (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize >= a + FRZ_SIZE &&
|
||||
VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READONLY, &old)) {
|
||||
__atomic_store_n(&frz[r], 1, __ATOMIC_RELEASE);
|
||||
InterlockedIncrement(&nfrz);
|
||||
InterlockedIncrement64(&frz_count[0]);
|
||||
ok = 1;
|
||||
}
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
return ok;
|
||||
}
|
||||
|
||||
int mem_frozen(uint64_t a) { return a < RT_ADDRESS_LIMIT && __atomic_load_n(&frz[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE); }
|
||||
|
||||
void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws) {
|
||||
*frozen = (uint64_t)__atomic_load_n(&nfrz, __ATOMIC_RELAXED);
|
||||
*freezes = (uint64_t)InterlockedExchange64(&frz_count[0], 0);
|
||||
*fault_thaws = (uint64_t)InterlockedExchange64(&frz_count[1], 0);
|
||||
*other_thaws = (uint64_t)InterlockedExchange64(&frz_count[2], 0) + (uint64_t)InterlockedExchange64(&frz_count[3], 0);
|
||||
}
|
||||
|
||||
// The first write into a frozen region thaws it and retries. A fault in a region another thread thawed meanwhile
|
||||
// retries too; any other fault goes on to the crash handler.
|
||||
static LONG CALLBACK thaw_handler(EXCEPTION_POINTERS *ep) {
|
||||
const EXCEPTION_RECORD *er = ep->ExceptionRecord;
|
||||
if (er->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || er->NumberParameters < 2 || er->ExceptionInformation[0] != 1)
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
uint64_t a = er->ExceptionInformation[1];
|
||||
if (a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return EXCEPTION_CONTINUE_SEARCH;
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
int was = frz[a >> FRZ_SHIFT];
|
||||
thaw_locked(a >> FRZ_SHIFT, 1);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
if (was) return EXCEPTION_CONTINUE_EXECUTION;
|
||||
MEMORY_BASIC_INFORMATION mbi;
|
||||
if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT && mbi.Protect == PAGE_READWRITE)
|
||||
return EXCEPTION_CONTINUE_EXECUTION;
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
}
|
||||
|
||||
// Mapping changes thaw the regions they touch and run under frz_lock, so no freeze sees them half done.
|
||||
static void commit(uint64_t a, uint64_t n, const char *what) {
|
||||
if (n && !VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE))
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
thaw_range_locked(a, n, 3);
|
||||
int ok = !n || VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
if (!ok)
|
||||
rt_fatal("memory: commit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
|
||||
(unsigned long long)n, GetLastError());
|
||||
}
|
||||
|
||||
static void decommit(uint64_t a, uint64_t n, const char *what) {
|
||||
if (n && !VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT))
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
thaw_range_locked(a, n, 3);
|
||||
int ok = !n || VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
if (!ok)
|
||||
rt_fatal("memory: decommit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
|
||||
(unsigned long long)n, GetLastError());
|
||||
}
|
||||
|
||||
static void protect(uint64_t a, uint64_t n, DWORD prot, const char *what) {
|
||||
DWORD old;
|
||||
if (n && !VirtualProtect((void *)(uintptr_t)a, n, prot, &old))
|
||||
AcquireSRWLockExclusive(&frz_lock);
|
||||
thaw_range_locked(a, n, 3);
|
||||
int ok = !n || VirtualProtect((void *)(uintptr_t)a, n, prot, &old);
|
||||
ReleaseSRWLockExclusive(&frz_lock);
|
||||
if (!ok)
|
||||
rt_fatal("memory: protect %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
|
||||
(unsigned long long)n, GetLastError());
|
||||
}
|
||||
@@ -129,6 +250,7 @@ void mem_init(void) {
|
||||
if (reserve_at(at, ALIAS_SIZE, MEM_WRITE_WATCH)) alias = at;
|
||||
if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39");
|
||||
|
||||
AddVectoredExceptionHandler(1, thaw_handler); // before the crash handler
|
||||
alias_pages = VirtualAlloc(NULL, ALIAS_PAGES, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
if (!alias_pages) rt_fatal("memory: cannot allocate the alias page map");
|
||||
const char *pool = cfg_get("TOTK_APP_POOL");
|
||||
@@ -150,10 +272,7 @@ const RtGuestLayout *mem_layout(void) { return &L; }
|
||||
// with the reset flag. A range inside one watched reservation (stack and heap block, or alias region) only.
|
||||
static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, DWORD flags) {
|
||||
if (!max || !size) return 0;
|
||||
const RtGuestLayout *g = &L;
|
||||
int low_block = addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size;
|
||||
int alias = addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size;
|
||||
if (!low_block && !alias) return UINT64_MAX;
|
||||
if (!watched(addr, size)) return UINT64_MAX;
|
||||
ULONG_PTR n = (ULONG_PTR)max;
|
||||
DWORD gran;
|
||||
_Static_assert(sizeof(PVOID) == sizeof(uint64_t), "page addresses are written straight into a uint64_t array");
|
||||
|
||||
+7
-2
@@ -84,11 +84,16 @@ static int64_t read_at(HANDLE f, void *buf, uint32_t n, uint64_t off) {
|
||||
rt_fatal("RomFS: CreateEvent failed (%lu)", GetLastError());
|
||||
OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32), .hEvent = ev};
|
||||
DWORD got = 0;
|
||||
mem_thaw((uintptr_t)buf, n); // the kernel's write would fail on a frozen page instead of faulting
|
||||
if (!ReadFile(f, buf, n, &got, &ov)) {
|
||||
DWORD e = GetLastError();
|
||||
if (e == ERROR_HANDLE_EOF) return 0;
|
||||
if (e != ERROR_IO_PENDING) return -1;
|
||||
if (!GetOverlappedResult(f, &ov, &got, TRUE)) return GetLastError() == ERROR_HANDLE_EOF ? 0 : -1;
|
||||
if (e == ERROR_IO_PENDING && !GetOverlappedResult(f, &ov, &got, TRUE)) e = GetLastError();
|
||||
else if (e == ERROR_IO_PENDING) return got;
|
||||
if (e == ERROR_HANDLE_EOF) return 0;
|
||||
if (e == ERROR_NOACCESS) // a read-only page in the buffer: a frozen region missed its thaw
|
||||
rt_log("RomFS: read of %u bytes into %p refused: the buffer is not writable", n, buf);
|
||||
return -1;
|
||||
}
|
||||
return got;
|
||||
}
|
||||
|
||||
+135
-16
@@ -22,6 +22,23 @@
|
||||
#include "nvn_pool_index.h"
|
||||
#include "vk.h"
|
||||
|
||||
// Raw performance counter ticks for the tracking cost (no division on the replay's path); qpc_ms converts.
|
||||
static uint64_t qpc(void) {
|
||||
LARGE_INTEGER t;
|
||||
QueryPerformanceCounter(&t);
|
||||
return (uint64_t)t.QuadPart;
|
||||
}
|
||||
|
||||
static double qpc_ms(uint64_t ticks) {
|
||||
static double per_ms;
|
||||
if (!per_ms) {
|
||||
LARGE_INTEGER f;
|
||||
QueryPerformanceFrequency(&f);
|
||||
per_ms = (double)f.QuadPart / 1000.0;
|
||||
}
|
||||
return (double)ticks / per_ms;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- pools
|
||||
enum { POOL_IMPORT = 1, POOL_PHYS, POOL_VIRT };
|
||||
enum { MS_NONE, MS_ON, MS_REFUSED };
|
||||
@@ -242,6 +259,7 @@ int vk_pool_init(pool_t *p) {
|
||||
ok = alloc_physical(c, r);
|
||||
} else {
|
||||
r->kind = POOL_IMPORT;
|
||||
mem_thaw((uintptr_t)p->mem, p->size); // the import may need write access to the pages
|
||||
ok = import_pool(c, r, (uintptr_t)p->mem, p->size);
|
||||
if (!ok) {
|
||||
// Not fatal for the game: the pool works for the CPU, and vk_mem_resolve reports its addresses as unbacked.
|
||||
@@ -288,6 +306,7 @@ void vk_pool_fini(pool_t *p) {
|
||||
}
|
||||
ReleaseSRWLockExclusive(&pool_lock);
|
||||
if (!r) return;
|
||||
if (r->kind == POOL_IMPORT) mem_thaw(r->lo, r->size); // the storage goes back to the guest writable
|
||||
// Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the
|
||||
// guest gets the storage back when this returns.
|
||||
vk_texture_range_gone(r->lo, r->size, r->alloc);
|
||||
@@ -439,7 +458,8 @@ static void mirror_log(uint64_t frame);
|
||||
void vk_mem_census(uint64_t frame) {
|
||||
static uint64_t window_start;
|
||||
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
|
||||
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame);
|
||||
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame); // and the
|
||||
// write-watch line
|
||||
if (frame == 1200) { // loading is over: the first window starts here
|
||||
for (int c = 0; c < VK_MCLS_N; c++)
|
||||
for (int u = 0; u < VK_MUSE_N; u++) InterlockedExchange64(&traffic[c][u], 0);
|
||||
@@ -525,6 +545,32 @@ static uint64_t gen_pages[1024];
|
||||
#define REG_SHIFT 21
|
||||
static uint32_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT];
|
||||
|
||||
// Every write-watch query of the generation table (gen_lock held exclusively): calls, bytes and time since the last
|
||||
// mirror summary, all consumers together; the queries that found written pages (they also flush TLBs) apart.
|
||||
static uint64_t ww_calls, ww_bytes, ww_ticks, ww_dirty_calls, ww_dirty_ticks, ww_pages;
|
||||
|
||||
static uint64_t ww_query(uint64_t addr, uint64_t size) {
|
||||
uint64_t t0 = qpc();
|
||||
uint64_t n = mem_written_pages(addr, size, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
||||
uint64_t dt = qpc() - t0;
|
||||
ww_ticks += dt, ww_calls++, ww_bytes += size;
|
||||
if (n && n != UINT64_MAX) ww_dirty_calls++, ww_dirty_ticks += dt, ww_pages += n;
|
||||
return n;
|
||||
}
|
||||
|
||||
// Frozen regions (mirror mode): a 2 MiB region whose pages nothing wrote for freeze_idle submissions, and which lies
|
||||
// wholly in imported pools, is made read-only (memory.c mem_freeze), and write-watch queries skip it until a write or a
|
||||
// file read thaws it. Most of the pool memory the mirror and the texture checks cover is static (meshes, textures), and
|
||||
// a query costs about 1 us per MiB, so this keeps the queries to the regions the CPU writes.
|
||||
#define FREEZE_PER_SUB 32
|
||||
static int freeze_idle; // 0: off
|
||||
static uint32_t ww_ep = 1; // gen_lock: the submission being recorded (vk_mem_mirror_begin)
|
||||
static uint32_t reg_last[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of the region's last stamp
|
||||
static uint32_t reg_tried[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of its last freeze candidacy
|
||||
static uint32_t *fcand; // gen_lock: regions to try at the next freeze pass
|
||||
static int nfcand, cfcand;
|
||||
static uint64_t n_frz_nopool, n_frz_refused; // gen_lock: candidates not wholly in pools, refused by mem_freeze
|
||||
|
||||
static uint32_t *gen_slot(uint64_t page_addr, int create) {
|
||||
uint64_t ci = page_addr >> GEN_CHUNK_SHIFT;
|
||||
if (ci >= GEN_CHUNKS) return NULL;
|
||||
@@ -542,6 +588,7 @@ static void stamp_page(uint64_t a, uint32_t g) {
|
||||
if (!s) return;
|
||||
*s = g;
|
||||
reg_max[a >> REG_SHIFT] = g;
|
||||
reg_last[a >> REG_SHIFT] = ww_ep;
|
||||
}
|
||||
|
||||
static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) {
|
||||
@@ -561,9 +608,10 @@ int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now)
|
||||
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
uint32_t g = ++gen_now;
|
||||
int tracked = 1;
|
||||
for (;;) {
|
||||
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
||||
int tracked = 1, frozen = 1;
|
||||
for (uint64_t a = lo & ~((1ull << REG_SHIFT) - 1); a < hi && frozen; a += 1ull << REG_SHIFT) frozen = mem_frozen(a);
|
||||
while (!frozen) {
|
||||
uint64_t n = ww_query(lo, hi - lo);
|
||||
if (n == UINT64_MAX) { tracked = 0; break; }
|
||||
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
|
||||
if (n < sizeof gen_pages / sizeof *gen_pages) break; // a full answer may have left pages for the next call
|
||||
@@ -596,19 +644,32 @@ static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) {
|
||||
uint64_t calls = 0;
|
||||
for (uint64_t r = r0; r <= r1;) {
|
||||
if (snap_at[r] == epoch) { r++; continue; }
|
||||
if (mem_frozen(r << SNAP_SHIFT)) { // nothing in it changed since its freeze
|
||||
snap_at[r] = epoch, snap_untracked[r] = 0;
|
||||
r++;
|
||||
continue;
|
||||
}
|
||||
uint64_t k = r; // one query over a run of regions not seen in this epoch
|
||||
while (k + 1 <= r1 && snap_at[k + 1] != epoch) k++;
|
||||
while (k + 1 <= r1 && snap_at[k + 1] != epoch && !mem_frozen((k + 1) << SNAP_SHIFT)) k++;
|
||||
uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT;
|
||||
int tracked = 1;
|
||||
if (!g) g = ++gen_now;
|
||||
for (;;) {
|
||||
uint64_t n = mem_written_pages(a, b - a, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
||||
uint64_t n = ww_query(a, b - a);
|
||||
if (n == UINT64_MAX) { tracked = 0; break; }
|
||||
for (uint64_t j = 0; j < n; j++) stamp_page(gen_pages[j], g);
|
||||
if (n < sizeof gen_pages / sizeof *gen_pages) break;
|
||||
}
|
||||
calls++;
|
||||
for (uint64_t j = r; j <= k; j++) snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
|
||||
for (uint64_t j = r; j <= k; j++) {
|
||||
snap_at[j] = epoch, snap_untracked[j] = (uint8_t)!tracked;
|
||||
if (tracked && freeze_idle && ww_ep - reg_last[j] >= (uint32_t)freeze_idle &&
|
||||
ww_ep - reg_tried[j] >= (uint32_t)freeze_idle) {
|
||||
reg_tried[j] = ww_ep;
|
||||
if (nfcand == cfcand) grow((void **)&fcand, &cfcand, sizeof *fcand);
|
||||
fcand[nfcand++] = (uint32_t)j;
|
||||
}
|
||||
}
|
||||
r = k + 1;
|
||||
}
|
||||
return calls;
|
||||
@@ -696,7 +757,7 @@ static uint64_t page_hi(const PoolRec *r) { return (r->lo + r->size + 0xfff) & ~
|
||||
// gen_lock held exclusively. 0: the range is not write-watched.
|
||||
static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
|
||||
for (;;) {
|
||||
uint64_t n = mem_written_pages(lo, hi - lo, gen_pages, sizeof gen_pages / sizeof *gen_pages);
|
||||
uint64_t n = ww_query(lo, hi - lo);
|
||||
if (n == UINT64_MAX) return 0;
|
||||
for (uint64_t k = 0; k < n; k++) stamp_page(gen_pages[k], g);
|
||||
if (n < sizeof gen_pages / sizeof *gen_pages) return 1;
|
||||
@@ -867,7 +928,7 @@ static SRWLOCK flush_lock = SRWLOCK_INIT;
|
||||
|
||||
typedef struct {
|
||||
uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges;
|
||||
uint64_t presnap_queries, sync_queries, cpu_ns, prologues;
|
||||
uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes;
|
||||
} MirrorAcc;
|
||||
static MirrorAcc mrep; // replay: this submission
|
||||
static MirrorAcc macc; // mirror_stat_lock: since the last summary
|
||||
@@ -890,6 +951,9 @@ static void mirror_mode(int discrete) {
|
||||
const char *g = cfg_get("TOTK_MIRROR_GAP");
|
||||
mirror_gap = g && *g ? atoi(g) : 2;
|
||||
if (mirror_gap < 0) mirror_gap = 0;
|
||||
const char *fi = cfg_get("TOTK_FREEZE_IDLE"); // submissions a region stays unwritten before it is frozen
|
||||
freeze_idle = !vk_mirror_on ? 0 : fi && *fi ? atoi(fi) : 300;
|
||||
if (freeze_idle < 0) freeze_idle = 0;
|
||||
rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why,
|
||||
mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory");
|
||||
}
|
||||
@@ -1051,7 +1115,7 @@ static void mirror_retire(void *arg) {
|
||||
|
||||
// Replay thread, the first access of this submission: queues the copies of the pages written since the last sync.
|
||||
static void mirror_sync(PoolRec *r) {
|
||||
uint64_t t0 = kern_now_ns();
|
||||
uint64_t t0 = qpc();
|
||||
r->acc_sub = msub;
|
||||
uint64_t lo = r->import_lo, hi = r->import_hi;
|
||||
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
|
||||
@@ -1089,7 +1153,7 @@ static void mirror_sync(PoolRec *r) {
|
||||
if (a < b) range_add(&pend, &npend, &cpend, r, a - lo, b - a);
|
||||
}
|
||||
mrep.syncs++;
|
||||
mrep.cpu_ns += kern_now_ns() - t0;
|
||||
mrep.sync_t += qpc() - t0;
|
||||
}
|
||||
|
||||
void vk_mem_use(VkMemSpan *sp) {
|
||||
@@ -1141,9 +1205,42 @@ void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) {
|
||||
ReleaseSRWLockShared(&pool_lock);
|
||||
}
|
||||
|
||||
// pool_lock shared: [a, b) lies wholly in the host ranges of imported pools with a GPU buffer.
|
||||
static int pools_cover(uint64_t a, uint64_t b) {
|
||||
uint64_t lo[64], hi[64];
|
||||
int n = 0;
|
||||
for (int i = pidx_upper(&pool_index, b - 1) - 1; i >= 0 && pool_index.maxhi[i] > a; i--) {
|
||||
const PoolRec *r = pool_index.e[i].buf;
|
||||
if (r->kind != POOL_IMPORT || !r->buf || r->import_hi <= a || r->import_lo >= b) continue;
|
||||
if (n == 64) return 0;
|
||||
int at = n++;
|
||||
for (; at > 0 && lo[at - 1] > r->import_lo; at--) lo[at] = lo[at - 1], hi[at] = hi[at - 1];
|
||||
lo[at] = r->import_lo, hi[at] = r->import_hi;
|
||||
}
|
||||
uint64_t end = a;
|
||||
for (int i = 0; i < n && end < b; i++) {
|
||||
if (lo[i] > end) return 0;
|
||||
if (hi[i] > end) end = hi[i];
|
||||
}
|
||||
return end >= b;
|
||||
}
|
||||
|
||||
// pool_lock shared, gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the
|
||||
// protection collects the writes made before it took effect.
|
||||
static void freeze_pass(void) {
|
||||
for (int done = 0; nfcand && done < FREEZE_PER_SUB;) {
|
||||
uint64_t j = fcand[--nfcand], a = j << REG_SHIFT, b = a + (1ull << REG_SHIFT);
|
||||
if (ww_ep - reg_last[j] < (uint32_t)freeze_idle || mem_frozen(a)) continue;
|
||||
if (!pools_cover(a, b)) { n_frz_nopool++; continue; }
|
||||
done++;
|
||||
if (!mem_freeze(a)) { n_frz_refused++; continue; }
|
||||
query_stamp(a, b, ++gen_now);
|
||||
}
|
||||
}
|
||||
|
||||
void vk_mem_mirror_begin(void) {
|
||||
if (!vk_mirror_on) return;
|
||||
uint64_t t0 = kern_now_ns();
|
||||
uint64_t t0 = qpc();
|
||||
uint32_t prev = msub;
|
||||
msub = msub + 1 ? msub + 1 : 1;
|
||||
mepoch = vk_texture_epoch();
|
||||
@@ -1151,6 +1248,8 @@ void vk_mem_mirror_begin(void) {
|
||||
int have = 0;
|
||||
AcquireSRWLockShared(&pool_lock);
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
ww_ep++;
|
||||
uint64_t b0 = ww_bytes;
|
||||
for (int i = 0; i < nm; i++) {
|
||||
const PoolRec *r = mlist[i];
|
||||
if (r->acc_sub != prev) continue;
|
||||
@@ -1163,11 +1262,13 @@ void vk_mem_mirror_begin(void) {
|
||||
run0 = a, run1 = b, have = 1;
|
||||
}
|
||||
if (have) q += snap_regions(run0, run1, mepoch);
|
||||
mrep.presnap_bytes += ww_bytes - b0;
|
||||
if (freeze_idle) freeze_pass();
|
||||
ReleaseSRWLockExclusive(&gen_lock);
|
||||
ReleaseSRWLockShared(&pool_lock);
|
||||
mrep.subs++;
|
||||
mrep.presnap_queries += q;
|
||||
mrep.cpu_ns += kern_now_ns() - t0;
|
||||
mrep.presnap_t += qpc() - t0;
|
||||
}
|
||||
|
||||
int vk_mem_mirror_pending(void) { return npend; }
|
||||
@@ -1297,7 +1398,24 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
|
||||
|
||||
// One summary line: with TOTK_TRACE=perf every 60 frames, else at frames 600 and 1800.
|
||||
static void mirror_log(uint64_t frame) {
|
||||
static uint64_t last;
|
||||
static uint64_t last, ww_last;
|
||||
AcquireSRWLockExclusive(&gen_lock);
|
||||
uint64_t wc = ww_calls, wb = ww_bytes, wt = ww_ticks, dc = ww_dirty_calls, dt = ww_dirty_ticks, wp = ww_pages;
|
||||
ww_pages = 0;
|
||||
uint64_t np = n_frz_nopool, nr = n_frz_refused;
|
||||
ww_calls = ww_bytes = ww_ticks = ww_dirty_calls = ww_dirty_ticks = n_frz_nopool = n_frz_refused = 0;
|
||||
ReleaseSRWLockExclusive(&gen_lock);
|
||||
double wf = frame > ww_last ? (double)(frame - ww_last) : 1.0;
|
||||
ww_last = frame;
|
||||
uint64_t fz, ff, ft, fo;
|
||||
mem_freeze_stats(&fz, &ff, &ft, &fo);
|
||||
if (rt_trace & RT_TRACE_PERF)
|
||||
rt_log("mem: write-watch frame %llu: per frame, every consumer: %.1f queries over %.1f MiB, %.3f ms (%.1f found "
|
||||
"%.0f written pages, %.3f ms); frozen %.0f MiB; since the last line: freezes %llu, thaws by a write %llu, by a file read "
|
||||
"or mapping change %llu, candidates not wholly in pools %llu, refused %llu",
|
||||
(unsigned long long)frame, wc / wf, wb / wf / 1048576.0, qpc_ms(wt) / wf, dc / wf, wp / wf, qpc_ms(dt) / wf,
|
||||
fz * 2.0, (unsigned long long)ff, (unsigned long long)ft, (unsigned long long)fo, (unsigned long long)np,
|
||||
(unsigned long long)nr);
|
||||
if (!vk_mirror_on) return;
|
||||
AcquireSRWLockExclusive(&mirror_stat_lock);
|
||||
MirrorAcc a = macc;
|
||||
@@ -1314,12 +1432,13 @@ static void mirror_log(uint64_t frame) {
|
||||
rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked "
|
||||
"%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, "
|
||||
"first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, flush calls %.1f (%.3f MiB), queries "
|
||||
"%.1f pre-snap + %.1f sync, prologues %.2f, tracking cpu %.3f ms%s",
|
||||
"%.1f pre-snap (%.1f MiB) + %.1f sync, prologues %.2f, tracking cpu %.3f ms (pre-snap %.3f, syncs %.3f)%s",
|
||||
(unsigned long long)frame, (long long)m_live_n, m_live_bytes / M, (long long)m_refused_n[MR_BUDGET],
|
||||
m_refused_bytes[MR_BUDGET] / M, (long long)m_refused_n[MR_UNTRACKED], (long long)m_refused_n[MR_OVERLAP],
|
||||
(long long)m_refused_n[MR_ALLOC], a.subs / f, a.syncs / f, a.upload_bytes / f / M, a.upload_copies / f,
|
||||
a.first_n / f, a.first_bytes / f / M, a.wb_bytes / f / M, a.wb_copies / f, fc / f, fb / f / M,
|
||||
a.presnap_queries / f, a.sync_queries / f, a.prologues / f, a.cpu_ns / f / 1e6, ver);
|
||||
a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f,
|
||||
qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver);
|
||||
}
|
||||
|
||||
// Device class (build/mem-plan.md): INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a
|
||||
|
||||
Reference in new issue
Block a user