From 9a81e3e2f15bcb7b897034bfda012da27adccd8c Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 6 Nov 2026 10:07:38 +0300 Subject: [PATCH] Keep regions the GPU writes from being frozen read-only, and thaw a frozen one before a GPU write. Storage-buffer stores, copies and fills land in imported guest memory without a fault or a write-watch bit, so a region only the GPU wrote looked idle and was frozen while the GPU kept writing it. LAS Recomp never changes the protection of imported memory because of NVIDIA. A possible cause of the NVIDIA "write invalid" device faults (#788, #802), not proven. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_memory.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 7ee7acb8..f4ac80ce 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -633,6 +633,17 @@ static uint32_t reg_tried[RT_ADDRESS_LIMIT >> REG_SHIFT]; // gen_lock: ww_ep o static uint32_t *fcand; // gen_lock: regions to try at the next freeze pass static int nfcand, cfcand; static uint64_t n_frz_refused; // gen_lock: candidates mem_freeze refused +// Regions the GPU writes. Storage-buffer stores, buffer copies and fills land in imported guest memory, directly (import +// mode) or through a pool's mirror that vk_mem_mirror_end copies or merges into the imported buffer. They are not CPU +// writes: no fault thaws the region and no write-watch bit is set, so a region only the GPU writes looked idle, was +// frozen, and the GPU went on writing pages the CPU side had made read-only. LAS Recomp never changes the protection of +// imported memory: on NVIDIA, VirtualProtect on pages imported with VK_EXT_external_memory_host left the GPU reading +// wrong data (runtime/nvn/vulkan/exec.c, the comment on its write watch); a GPU write there is the likely cause of the +// device faults "write invalid" at a page of guest memory on NVIDIA (#788, #802). reg_gpu holds the timeline value of +// the newest submission that writes into the region (gpu_writes): freeze_pass leaves the region alone until the GPU is +// past that value, and a frozen region is thawed when a write into it is recorded. +static uint64_t reg_gpu[RT_ADDRESS_LIMIT >> REG_SHIFT]; // replay thread +static volatile int64_t n_frz_gpu, n_thaw_gpu; // freeze candidates kept for a GPU write; thaws for one (mirror_log) static uint32_t *gen_slot(uint64_t page_addr, int create) { uint64_t ci = page_addr >> GEN_CHUNK_SHIFT; @@ -1693,8 +1704,26 @@ static void snap_free(void *arg) { // completion thread: the submission is don free(f); } +// Record time: the submission being recorded writes [off, off+size) of r's guest range, directly into the imported +// buffer or into the mirror whose bytes go back at its end. Keeps the regions unfrozen until the GPU is past it +// (reg_gpu). Shadow storage needs nothing: the GPU writes the shadow and the CPU copies it back, thawing as it goes. +static void gpu_writes(const PoolRec *r, uint64_t off, uint64_t size) { + if (!freeze_idle || r->smap) return; + uint64_t a = r->import_lo + off, b = a + size; + if (b <= a || b > RT_ADDRESS_LIMIT) return; + uint64_t f = vk_exec_recording_fence(); + if (!f) f = vk_sync_issued() + 1; + int frozen = 0; + for (uint64_t j = a >> REG_SHIFT; j <= (b - 1) >> REG_SHIFT; j++) { + if (__atomic_load_n(®_gpu[j], __ATOMIC_RELAXED) < f) __atomic_store_n(®_gpu[j], f, __ATOMIC_RELAXED); + frozen |= mem_frozen(j << REG_SHIFT); + } + if (frozen) mem_thaw(a, b - a), plat_inc64(&n_thaw_gpu); +} + static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { PoolRec *r = sp->pool; + if (r && size) gpu_writes(r, sp->offset, size); if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; // A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies. if (shader && !(sp->offset & 3) && !(size & 3) && size <= MERGE_MAX && merging() && snap_add(r, sp->offset, size)) @@ -1778,9 +1807,11 @@ void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { // gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the // protection collects the writes made before it took effect. static void freeze_pass(void) { + uint64_t gpu_done = vk_sync_fence_done(); for (int done = 0; nfcand && done < FREEZE_PER_SUB;) { uint64_t j = fcand[--nfcand], a = j << REG_SHIFT; if (ww_ep - reg_last[j] < (uint32_t)freeze_idle || mem_frozen(a)) continue; + if (__atomic_load_n(®_gpu[j], __ATOMIC_RELAXED) > gpu_done) { plat_inc64(&n_frz_gpu); continue; } // gpu_writes done++; if (!mem_freeze(a)) { n_frz_refused++; continue; } query_stamp(a, a + (1ull << REG_SHIFT), ++gen_now); @@ -1998,6 +2029,10 @@ static void mirror_log(uint64_t frame) { "or mapping change %llu, refused %llu", (unsigned long long)frame, wc / wf, wb / wf / 1048576.0, qpc_ms(wt) / wf, dc / wf, wp / wf, qpc_ms(dt) / wf, fz * 2.0, (unsigned long long)ff, (unsigned long long)ft, (unsigned long long)fo, (unsigned long long)nr); + int64_t kg = plat_xchg64(&n_frz_gpu, 0), tg = plat_xchg64(&n_thaw_gpu, 0); + if (kg || tg) // gpu_writes: before this, every count here was a region frozen under a GPU write + rt_log("mem: frame %llu: regions the GPU writes, since the last line: %lld kept from freezing, %lld frozen ones " + "thawed", (unsigned long long)frame, (long long)kg, (long long)tg); if (!vk_mirror_on) return; plat_mutex_lock(&mirror_stat_lock); MirrorAcc a = macc;