From 1f7d80306e7871221fe32a1e449f8babee32563d Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Wed, 28 Oct 2026 21:22:17 +0300 Subject: [PATCH] Freeze idle regions that also hold memory outside NVN pools, and freeze after 60 idle submissions instead of 300 Write-watch queries skip frozen 2 MiB regions, but a region was frozen only when it lay wholly in imported pools and wholly committed. NVN pools are small and not 2 MiB aligned, so most regions also hold other guest memory or reserved holes: in town 340 of the 436 MiB queried every frame lay in regions nothing had written for 300 submissions. On the GTX 1050 Ti laptop the queries took up to 2.2 ms a frame, 1.4 ms of it in queries that found no write. mem_freeze now makes the committed pages of a region read-only when all of them are read-write and leaves reserved holes alone; thaw_locked already restored such regions run by run. The pool rule is gone: the only kernel writes into guest memory are file reads, which thaw their buffer first, and every user-mode write faults and thaws as before. TOTK_FREEZE_IDLE defaults to 60 (in a fight 100 of 257 queried MiB lay in regions idle for 30-299 submissions; a thaw by a write costs about 60 us on the RX 9070 machine), and import mode freezes too, for the texture checks of APUs. RX 9070, per frame after load: town 436 -> 96 MiB queried, 0.68 -> 0.38 ms; fight 168 MiB, 0.45 ms (257 MiB, 0.57 ms with the new rule at 300 submissions), about 20 thaws a second. TOTK_MEMORY=verify in the fight: 3.65 million pages compared, 0 mismatches. The gfx-abi probe covers a region with reserved holes. Co-Authored-By: Claude Opus 5.5 --- runtime/gfx_abi_probe.c | 11 +++++++ runtime/memory.c | 34 +++++++++++++++++---- runtime/vk_memory.c | 65 ++++++++++++++++++----------------------- 3 files changed, 67 insertions(+), 43 deletions(-) diff --git a/runtime/gfx_abi_probe.c b/runtime/gfx_abi_probe.c index e84bc62c..69eb4709 100644 --- a/runtime/gfx_abi_probe.c +++ b/runtime/gfx_abi_probe.c @@ -84,6 +84,17 @@ static void check_write_watch(void) { n = mem_written_pages(a, 0x4000, pages, 8); CHECK(n == 1 && pages[0] == a + 0x1000, "alias write: %llu pages reported (want 1)", (unsigned long long)n); CHECK(mem_written_pages(a, 0x4000, pages, 8) == 0, "alias: second call reported pages"); + + // A region with reserved holes freezes its committed pages only, and a write thaws them without committing a hole. + uint64_t ra = a & ~0x1fffffull; + CHECK(mem_freeze(ra) == 1 && mem_frozen(ra), "freeze of a 2 MiB region with reserved holes"); + CHECK(VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.Protect == PAGE_READONLY, "its committed pages are not read-only"); + *(volatile uint8_t *)(uintptr_t)(a + 0x2000) = 9; + CHECK(!mem_frozen(ra) && *(volatile uint8_t *)(uintptr_t)(a + 0x2000) == 9, "a write did not thaw the region with holes"); + CHECK(VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.Protect == PAGE_READWRITE && mbi.RegionSize == 0x4000, + "the thawed pages are not read-write"); + CHECK(VirtualQuery((void *)(uintptr_t)(a + 0x4000), &mbi, sizeof mbi) && mbi.State == MEM_RESERVE, "a hole got committed"); + printf("write-watch: region with reserved holes frozen and thawed by a write\n"); CHECK(mem_unmap_physical(a, 0x4000) == 0, "UnmapPhysicalMemory"); CHECK(mem_written_pages(RT_KERNEL_BASE, 0x1000, pages, 8) == UINT64_MAX, "kernel region is not write-watched"); CHECK(mem_set_heap_size(0, &heap) == 0, "SetHeapSize 0"); diff --git a/runtime/memory.c b/runtime/memory.c index 8e97143d..9c718dea 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -96,7 +96,8 @@ static void thaw_locked(uint64_t r, int why) { DWORD old; uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE; if (!VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READWRITE, &old)) { - // Not all committed (cannot happen while mapping changes thaw under frz_lock): the read-only runs only. + // Not all committed (freeze_locked left reserved holes): the read-only runs only. Every committed page of a + // frozen region was read-write at the freeze, and mapping changes thaw before they run, so these are all ours. MEMORY_BASIC_INFORMATION mbi; while (a < end && VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi)) { uint64_t b = (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize; @@ -126,18 +127,39 @@ void mem_thaw(uint64_t a, uint64_t n) { ReleaseSRWLockExclusive(&frz_lock); } +// frz_lock held exclusively. Makes the committed pages of region r read-only when every one of them is read-write; +// reserved holes (heap pages given back) stay as they are, and thaw_locked finds the read-only runs again. +static int freeze_locked(uint64_t r) { + uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, run[16][2]; + int n = 0; + MEMORY_BASIC_INFORMATION mbi; + for (uint64_t x = a; x < end; ) { + if (!VirtualQuery((void *)(uintptr_t)x, &mbi, sizeof mbi)) return 0; + uint64_t y = (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize; + if (y > end) y = end; + if (mbi.State == MEM_COMMIT) { + if (mbi.Protect != PAGE_READWRITE || n == 16) return 0; + run[n][0] = x, run[n][1] = y, n++; + } else if (mbi.State != MEM_RESERVE) return 0; + x = y; + } + DWORD old; + for (int i = 0; i < n; i++) + if (!VirtualProtect((void *)(uintptr_t)run[i][0], run[i][1] - run[i][0], PAGE_READONLY, &old)) { + while (i--) VirtualProtect((void *)(uintptr_t)run[i][0], run[i][1] - run[i][0], PAGE_READWRITE, &old); + return 0; + } + return n > 0; +} + int mem_freeze(uint64_t a) { if ((a & (FRZ_SIZE - 1)) || !watched(a, FRZ_SIZE)) return 0; uint64_t r = a >> FRZ_SHIFT; - MEMORY_BASIC_INFORMATION mbi; - DWORD old; int ok = 0; AcquireSRWLockExclusive(&frz_lock); if (frz[r]) ok = 1; else if (thaw_ms[r] && (int32_t)((uint32_t)GetTickCount64() - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) ok = 0; // |1 may be ahead - else if (VirtualQuery((void *)(uintptr_t)a, &mbi, sizeof mbi) && mbi.State == MEM_COMMIT && - mbi.Protect == PAGE_READWRITE && (uint64_t)(uintptr_t)mbi.BaseAddress + mbi.RegionSize >= a + FRZ_SIZE && - VirtualProtect((void *)(uintptr_t)a, FRZ_SIZE, PAGE_READONLY, &old)) { + else if (freeze_locked(r)) { __atomic_store_n(&frz[r], 1, __ATOMIC_RELEASE); InterlockedIncrement(&nfrz); InterlockedIncrement64(&frz_count[0]); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 3601b714..23c967da 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -599,10 +599,14 @@ static uint64_t ww_query(uint64_t addr, uint64_t size) { return n; } -// Frozen regions (mirror mode): a 2 MiB region whose pages nothing wrote for freeze_idle submissions, and which lies -// wholly in imported pools, is made read-only (memory.c mem_freeze), and write-watch queries skip it until a write or a -// file read thaws it. Most of the pool memory the mirror and the texture checks cover is static (meshes, textures), and -// a query costs about 1 us per MiB, so this keeps the queries to the regions the CPU writes. +// Frozen regions: a 2 MiB region whose pages nothing wrote for freeze_idle submissions is made read-only +// (memory.c mem_freeze), and write-watch queries skip it until a write or a file read thaws it. Most of the pool memory +// the mirror and the texture checks cover is static (meshes, textures), and a query costs about 1 us per MiB (4-5 on an +// 8th-gen i7), so this keeps the queries to the regions the CPU writes. The region need not lie wholly in pools: NVN +// pools are small and not 2 MiB aligned, so most regions also hold other guest memory, and a rule that refused those +// left 340 of 436 queried MiB a frame unfrozen in town although nothing had written them for 300 submissions. The +// other memory is safe to freeze for the same reason pools are: the only kernel writes into guest memory are file +// reads, which thaw their buffer first (fs_host.c, romfs.c), and every user-mode write faults and thaws. #define FREEZE_PER_SUB 32 static int freeze_idle; // 0: off static uint32_t ww_ep = 1; // gen_lock: the submission being recorded (vk_mem_mirror_begin) @@ -610,7 +614,7 @@ static uint32_t reg_last[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep o static uint32_t reg_tried[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep of its last freeze candidacy static uint32_t *fcand; // gen_lock: regions to try at the next freeze pass static int nfcand, cfcand; -static uint64_t n_frz_nopool, n_frz_refused; // gen_lock: candidates not wholly in pools, refused by mem_freeze +static uint64_t n_frz_refused; // gen_lock: candidates mem_freeze refused static uint32_t *gen_slot(uint64_t page_addr, int create) { uint64_t ci = page_addr >> GEN_CHUNK_SHIFT; @@ -998,8 +1002,10 @@ static void mirror_mode(int discrete) { const char *g = cfg_get("TOTK_MIRROR_GAP"); mirror_gap = g && *g ? atoi(g) : 2; if (mirror_gap < 0) mirror_gap = 0; - const char *fi = cfg_get("TOTK_FREEZE_IDLE"); // submissions a region stays unwritten before it is frozen - freeze_idle = !vk_mirror_on ? 0 : fi && *fi ? atoi(fi) : 300; + // Submissions a region stays unwritten before it is frozen. 60 rather than 300: in a fight 100 of the 257 MiB + // queried a frame lay in regions idle for 30-299 submissions, and a thaw by a write costs about 60 us. + const char *fi = cfg_get("TOTK_FREEZE_IDLE"); + freeze_idle = fi && *fi ? atoi(fi) : 60; if (freeze_idle < 0) freeze_idle = 0; rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why, mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory"); @@ -1254,41 +1260,27 @@ void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { ReleaseSRWLockShared(&pool_lock); } -// pool_lock shared: [a, b) lies wholly in the host ranges of imported pools with a GPU buffer. -static int pools_cover(uint64_t a, uint64_t b) { - uint64_t lo[64], hi[64]; - int n = 0; - for (int i = pidx_upper(&pool_index, b - 1) - 1; i >= 0 && pool_index.maxhi[i] > a; i--) { - const PoolRec *r = pool_index.e[i].buf; - if (r->kind != POOL_IMPORT || !r->buf || r->import_hi <= a || r->import_lo >= b) continue; - if (n == 64) return 0; - int at = n++; - for (; at > 0 && lo[at - 1] > r->import_lo; at--) lo[at] = lo[at - 1], hi[at] = hi[at - 1]; - lo[at] = r->import_lo, hi[at] = r->import_hi; - } - uint64_t end = a; - for (int i = 0; i < n && end < b; i++) { - if (lo[i] > end) return 0; - if (hi[i] > end) end = hi[i]; - } - return end >= b; -} - -// pool_lock shared, gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the +// gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the // protection collects the writes made before it took effect. static void freeze_pass(void) { for (int done = 0; nfcand && done < FREEZE_PER_SUB;) { - uint64_t j = fcand[--nfcand], a = j << REG_SHIFT, b = a + (1ull << REG_SHIFT); + uint64_t j = fcand[--nfcand], a = j << REG_SHIFT; if (ww_ep - reg_last[j] < (uint32_t)freeze_idle || mem_frozen(a)) continue; - if (!pools_cover(a, b)) { n_frz_nopool++; continue; } done++; if (!mem_freeze(a)) { n_frz_refused++; continue; } - query_stamp(a, b, ++gen_now); + query_stamp(a, a + (1ull << REG_SHIFT), ++gen_now); } } void vk_mem_mirror_begin(void) { - if (!vk_mirror_on) return; + if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions + if (!freeze_idle) return; + AcquireSRWLockExclusive(&gen_lock); + ww_ep++; + freeze_pass(); + ReleaseSRWLockExclusive(&gen_lock); + return; + } uint64_t t0 = qpc(); uint32_t prev = msub; msub = msub + 1 ? msub + 1 : 1; @@ -1452,8 +1444,8 @@ static void mirror_log(uint64_t frame) { AcquireSRWLockExclusive(&gen_lock); uint64_t wc = ww_calls, wb = ww_bytes, wt = ww_ticks, dc = ww_dirty_calls, dt = ww_dirty_ticks, wp = ww_pages; ww_pages = 0; - uint64_t np = n_frz_nopool, nr = n_frz_refused; - ww_calls = ww_bytes = ww_ticks = ww_dirty_calls = ww_dirty_ticks = n_frz_nopool = n_frz_refused = 0; + uint64_t nr = n_frz_refused; + ww_calls = ww_bytes = ww_ticks = ww_dirty_calls = ww_dirty_ticks = n_frz_refused = 0; ReleaseSRWLockExclusive(&gen_lock); double wf = frame > ww_last ? (double)(frame - ww_last) : 1.0; ww_last = frame; @@ -1462,10 +1454,9 @@ static void mirror_log(uint64_t frame) { if (rt_trace & RT_TRACE_PERF) rt_log("mem: write-watch frame %llu: per frame, every consumer: %.1f queries over %.1f MiB, %.3f ms (%.1f found " "%.0f written pages, %.3f ms); frozen %.0f MiB; since the last line: freezes %llu, thaws by a write %llu, by a file read " - "or mapping change %llu, candidates not wholly in pools %llu, refused %llu", + "or mapping change %llu, refused %llu", (unsigned long long)frame, wc / wf, wb / wf / 1048576.0, qpc_ms(wt) / wf, dc / wf, wp / wf, qpc_ms(dt) / wf, - fz * 2.0, (unsigned long long)ff, (unsigned long long)ft, (unsigned long long)fo, (unsigned long long)np, - (unsigned long long)nr); + fz * 2.0, (unsigned long long)ff, (unsigned long long)ft, (unsigned long long)fo, (unsigned long long)nr); if (!vk_mirror_on) return; AcquireSRWLockExclusive(&mirror_stat_lock); MirrorAcc a = macc;