diff --git a/runtime/gfx_null.c b/runtime/gfx_null.c index 073c3182..0f89abc1 100644 --- a/runtime/gfx_null.c +++ b/runtime/gfx_null.c @@ -100,6 +100,7 @@ static void null_replay(uint64_t handle) { } static uint64_t null_submit(const uint64_t *handles, int n) { + nvn_pins_sweep(nvn_submit_count(), NULL, 0); // no replay state outlives a submission here for (int i = 0; i < n; i++) null_replay(handles[i]); return __atomic_add_fetch(&fence, 1, __ATOMIC_ACQ_REL); } diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index 879174fa..b74253ab 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -75,7 +75,8 @@ uint64_t nvn_tex_view_offset(const tex_t *t, const view_t *v); // A command handle (what QueueSubmitCommands receives) points at a host copy of one recording: a flat sequence of // 8-byte-aligned records {op_hdr, payload}, OP_BEGIN first and OP_END last. Called recordings are inlined, so a copy // holds no OP_CALL or OP_JUMP. Texture pointers in payloads (p_set_rt, p_copy_b2t, p_copy_t2t, p_clear_tex) are -// immutable host copies that live for the run, one per tex_t.serial. Deferred binds (p_bind_handles.src != 0) hold the +// immutable host copies ("pins"), one per tex_t.serial, that live while a recording copy names them and a while after +// (nvn_pins_sweep). Deferred binds (p_bind_handles.src != 0) hold the // handles read at submit (nvn_submit_prepare). Payloads of the ops nvn_obj.h does not define: typedef struct { volatile uint32_t *val; uint32_t mode, value; } p_signal_event; // OP_SIGNAL_EVENT; mode: NVNeventSignalMode typedef struct { uint64_t type, addr; } p_report_counter; // OP_REPORT_COUNTER: NVNcounterType, NVNcounterData address @@ -99,6 +100,14 @@ int nvn_handle_live(uint64_t handle); // 1 while the handle // not added again) uint64_t nvn_handle_bytes(uint64_t handle); // bytes of the copy, OP_BEGIN to OP_END; 0 if not live uint64_t nvn_submit_count(void); // QueueSubmitCommands calls so far +// Submits a texture pin outlives its last tex_pin and the release of the last recording copy naming it: about 20 s at +// one submit a frame (MacNativeGames frees after two 600-submit windows). +#define NVN_PIN_AGE 1200 +// Frees the texture pins no recording copy has named for NVN_PIN_AGE submits (nvn_api_cmd.c, "texture pins"), except keep[0..nkeep): +// the replay's current render targets, which carry over to the next submission. Called between submissions by the +// thread that replays them; now = nvn_submit_count(). +void nvn_pins_sweep(uint64_t now, const tex_t *const *keep, int nkeep); +void nvn_pins_stats(uint64_t *live, uint64_t *freed); // pins that exist, pins freed so far const char *nvn_op_name(uint32_t op); // "DRAW_IDX", ...; "?" past OP_COUNT_ // MemoryPoolInvalidateMappedRange: the CPU is about to read pool storage the GPU wrote. Weak no-op in nvn_ext.c; a back // end that renders into host-side copies defines it and writes them back. diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index 1dbff42c..21644b76 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -4,7 +4,7 @@ // games/totk/third_party/d2r_gfx/gfx/nvn/hle_nvn.c @ b75be8cf (MIT, Copyright (c) 2026 jhaizhou; origin d2r-mobile), // lines 647-1062, 1261-1286, 1486-1760 and 1975-1990, and the *Deferred and memory-callback entry points of // totk_nvn_ext.c in the same directory. Metal, mach, the translation thread, Apple diagnostics, the per-frame -// statistics and the long-session sweeps (copy marks in control memory, texture pin sweep) dropped. +// statistics and the copy marks in control memory dropped; the texture pin sweep replaced by counting (texture pins). // // Records are written into a growable host buffer per command buffer, never into the game's command memory: games hand // the same command memory to several command buffers recording at once (budgeted for real NVN encodings), and our @@ -17,6 +17,7 @@ #include #include +#include "cfg.h" #include "gfx.h" #include "host.h" #include "nvn_api.h" @@ -235,30 +236,169 @@ static void copy_view(view_t *dst, uint8_t *has, const NVNtextureView *v) { // ---------- texture pins // Recordings refer to textures by pointer, but the game may finalize or reuse an NVNtexture after recording and before // the submit (NVN itself captured only GPU state). Records therefore point at an immutable host copy per texture -// initialization (tex_t.serial). The copies are kept for the whole run (MacNativeGames sweeps them for long sessions). -static PlatMutex pin_lock = PLAT_MUTEX_INIT; -static nvn_hmap pins = {.klen = 8}; // serial -> tex_t copy +// initialization (tex_t.serial). The game initializes its transient targets again every frame, about 29 new serials a +// frame on 1.4.3, so copies kept for the run grew the heap by about 44 MiB per minute of play, standing still too, until +// the commit limit ended the game after a few hours. A copy is freed once nothing can read it any more: +// - refs counts the recording copies whose ops name it: live ones, released ones a queued submission still reads, and +// per-submission clones (pin_refs, at flatten_recording and wherever such a copy is freed or cloned); +// - a recording in progress holds pins no copy counts yet, through its own tex_pin calls and through ops inlined from a +// copy that was released since: a pin goes only NVN_PIN_AGE submits after its last tex_pin and after refs fell to 0; +// - the replay's current render targets carry over to later submissions and are compared by pointer (vk_exec.c set_rt), +// so a freed address reused by a new pin would read as the same target: nvn_pins_sweep runs between submissions on +// the thread that replays them and keeps the pins it is given. +// MacNativeGames hit the same growth (3.16 million copies, 405 MB after 27 minutes) and marks pins by walking every live +// recording each window (hle_nvn.c pin_scan); counting at copy and release walks each copy's ops once more, when it is +// freed. TOTK_PIN_SWEEP=0 keeps every copy. +typedef struct { + tex_t t; // first: ops hold pins as tex_t pointers + uint64_t last_use; // nsubmits at the last tex_pin or the last drop of refs + uint32_t refs; + uint8_t queued; // in pin_q +} pin_t; +_Static_assert(offsetof(pin_t, t) == 0, "ops hold pins as tex_t pointers"); +typedef struct { pin_t *p; uint64_t at; } pin_due; +static PlatMutex pin_lock = PLAT_MUTEX_INIT; // lock order: copy_lock, then pin_lock +static nvn_hmap pins = {.klen = 8}; // serial -> pin_t +static pin_due *pin_q; // pins with no counted copy, oldest first: [pin_qh, pin_qn) +static size_t pin_qh, pin_qn, pin_qcap; +static uint64_t pins_live, pins_freed; +static uint64_t nsubmits; // QueueSubmitCommands calls so far + +static void pin_queue(pin_t *p, uint64_t at) { // pin_lock held + if (p->queued) return; + if (pin_qh && pin_qh * 2 >= pin_qn) { + memmove(pin_q, pin_q + pin_qh, sizeof *pin_q * (pin_qn - pin_qh)); + pin_qn -= pin_qh, pin_qh = 0; + } + if (pin_qn == pin_qcap) { + pin_qcap = pin_qcap ? pin_qcap * 2 : 4096; + if (!(pin_q = realloc(pin_q, sizeof *pin_q * pin_qcap))) rt_fatal("nvn: out of memory (texture pin queue)"); + } + pin_q[pin_qn++] = (pin_due){p, at}; + p->queued = 1; +} static const tex_t *tex_pin(const NVNtexture *nt) { const tex_t *t = (const tex_t *)nt; if (!t) return NULL; uint64_t serial = t->serial; if (!serial) rt_fatal("nvn: a recording references the uninitialized texture object %p", (const void *)t); - plat_mutex_lock_shared(&pin_lock); - const tex_t *r = nvn_hm_get(&pins, &serial); + uint64_t now = __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); + plat_mutex_lock_shared(&pin_lock); // nvn_pins_sweep frees under the exclusive lock + pin_t *r = nvn_hm_get(&pins, &serial); + if (r) __atomic_store_n(&r->last_use, now, __ATOMIC_RELAXED); plat_mutex_unlock_shared(&pin_lock); - if (r) return r; + if (r) return &r->t; plat_mutex_lock(&pin_lock); r = nvn_hm_get(&pins, &serial); if (!r) { - tex_t *copy = malloc(sizeof *copy); - if (!copy) rt_fatal("nvn: texture pin copy"); - *copy = *t; - nvn_hm_put(&pins, &serial, copy); - r = copy; + if (!(r = malloc(sizeof *r))) rt_fatal("nvn: texture pin copy"); + *r = (pin_t){.t = *t}; + nvn_hm_put(&pins, &serial, r); + pins_live++; + pin_queue(r, now); // freed like any other if no copy ever names it + } + __atomic_store_n(&r->last_use, now, __ATOMIC_RELAXED); + plat_mutex_unlock(&pin_lock); + return &r->t; +} + +// The pins an op's payload names, into out (at most 9); returns how many. +static int op_pins(uint32_t op, const void *a, const tex_t *out[9]) { + int k = 0; + switch (op) { + case OP_SET_RT: { + const p_set_rt *r = a; + int n = r->n < 0 ? 0 : r->n > 8 ? 8 : r->n; + for (int i = 0; i < n; i++) + if (r->color[i]) out[k++] = r->color[i]; + if (r->depth) out[k++] = r->depth; + break; + } + case OP_COPY_B2T: case OP_COPY_T2B: + if (((const p_copy_b2t *)a)->tex) out[k++] = ((const p_copy_b2t *)a)->tex; + break; + case OP_CLEAR_TEX: + if (((const p_clear_tex *)a)->tex) out[k++] = ((const p_clear_tex *)a)->tex; + break; + case OP_COPY_T2T: { + const p_copy_t2t *c = a; + if (c->src) out[k++] = c->src; + if (c->dst) out[k++] = c->dst; + break; + } + default: break; + } + return k; +} + +static void pin_ref(const tex_t *t, int delta, uint64_t now) { + pin_t *p = (pin_t *)(uintptr_t)t; + if (delta > 0) { + __atomic_add_fetch(&p->refs, 1, __ATOMIC_RELAXED); + return; + } + __atomic_store_n(&p->last_use, now, __ATOMIC_RELAXED); + if (__atomic_sub_fetch(&p->refs, 1, __ATOMIC_ACQ_REL)) return; + plat_mutex_lock(&pin_lock); + pin_queue(p, now); + plat_mutex_unlock(&pin_lock); +} + +// Counts (delta 1) or uncounts (-1) the pins named by the copy at handle, OP_BEGIN to OP_END; returns how many. +static uint32_t pin_refs(const uint8_t *copy, int delta) { + uint64_t now = __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); + uint32_t n = 0; + for (const op_hdr *h = nvn_op_first((uintptr_t)copy); h->op != OP_END; h = nvn_op_next(h)) { + const tex_t *p[9]; + int k = op_pins(h->op, nvn_op_payload(h), p); + for (int i = 0; i < k; i++) pin_ref(p[i], delta, now); + n += (uint32_t)k; + } + return n; +} + +void nvn_pins_sweep(uint64_t now, const tex_t *const *keep, int nkeep) { + if (now < NVN_PIN_AGE) return; + static int on = -1; + if (on < 0) { + const char *s = cfg_get("TOTK_PIN_SWEEP"); + on = !(s && !strcmp(s, "0")); + if (!on) rt_log("nvn: TOTK_PIN_SWEEP=0: texture pin copies are kept for the run (the heap grows in long sessions)"); + } + if (!on) return; + static uint64_t logged; + plat_mutex_lock(&pin_lock); + for (int budget = 1024; pin_qh < pin_qn && budget; budget--) { + pin_due d = pin_q[pin_qh]; + if (d.at > now - NVN_PIN_AGE) break; + pin_qh++; + pin_t *p = d.p; + p->queued = 0; + if (__atomic_load_n(&p->refs, __ATOMIC_ACQUIRE)) continue; // queued again when the count falls to 0 + uint64_t last = __atomic_load_n(&p->last_use, __ATOMIC_RELAXED); + int kept = 0; + for (int i = 0; i < nkeep; i++) kept |= keep[i] == &p->t; + if (kept || last > now - NVN_PIN_AGE) { + pin_queue(p, kept ? now : last); + continue; + } + nvn_hm_del(&pins, &p->t.serial); + free(p); + pins_live--, pins_freed++; + } + if (now / 7200 != logged) { + logged = now / 7200; + rt_log("nvn: texture pins at submit %llu: %llu live, %llu waiting, %llu freed so far", (unsigned long long)now, + (unsigned long long)pins_live, (unsigned long long)(pin_qn - pin_qh), (unsigned long long)pins_freed); } plat_mutex_unlock(&pin_lock); - return r; +} + +void nvn_pins_stats(uint64_t *live, uint64_t *freed) { + plat_mutex_lock_shared(&pin_lock); + *live = pins_live, *freed = pins_freed; + plat_mutex_unlock_shared(&pin_lock); } // ---------- recording copies @@ -266,14 +406,15 @@ static const tex_t *tex_pin(const NVNtexture *nt) { // adding control memory again releases the copies recorded there without walking every live copy. 4 KiB pages cost // about 500 lookups per AddControlMemory under the exclusive lock (1.4.3 field: 53 calls a frame, 2 MiB ranges, 3.7 // copies per page), and the recording threads waited on it. -typedef struct { uint64_t magic, bytes, made_at, used_at, deferred; } copy_prefix; // deferred: 0 unknown, 1 none, 2 some +// deferred: 0 unknown, 1 none, 2 some; pins: texture pins its ops name, counted in their refs (pin_refs) +typedef struct { uint64_t magic, bytes, made_at, used_at, deferred, pins; } copy_prefix; _Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned"); typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy; typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket; static PlatMutex copy_lock = PLAT_MUTEX_INIT; // guards the two maps, the counters and every prefix static nvn_hmap copy_pages = {.klen = 8}; // page -> copy_bucket static nvn_hmap copy_live = {.klen = 8}; // handle -> copy_prefix of a live copy -static uint64_t ncopies, copy_bytes, copies_made, nsubmits; +static uint64_t ncopies, copy_bytes, copies_made; // Asynchronous replay: copies released by the game, and per-submission clones, that a queued submission still reads. // Freed once gfx_replayed_submits() reaches their used_at. Empty with synchronous replay. static copy_prefix **zombies; @@ -288,10 +429,15 @@ static void zombie_add(copy_prefix *pf) { // copy_lock held zombies[nzombies++] = pf; } +static void copy_free(copy_prefix *pf) { // copy_lock held + if (pf->pins) pin_refs((const uint8_t *)(pf + 1), -1); + free(pf); +} + static void zombie_sweep(uint64_t replayed) { // copy_lock held for (uint32_t i = 0; i < nzombies;) { if (zombies[i]->used_at <= replayed) { - free(zombies[i]); + copy_free(zombies[i]); zombies[i] = zombies[--nzombies]; } else { i++; @@ -337,19 +483,24 @@ static uint8_t *flatten_recording(const uint8_t *p, size_t n) { uint8_t *out = realloc_wait(NULL, sizeof(copy_prefix) + n, "command copy"); if (!out) rt_fatal("nvn: command copy allocation"); memcpy(out + sizeof(copy_prefix), p, n); + uint64_t pins_named = 0; for (size_t at = 0;;) { // the records stay checked: a corrupt one stops here, not in the replay op_hdr h; memcpy(&h, p + at, sizeof h); if (h.size < 8 || (h.size & 7) || h.op >= OP_COUNT_ || h.op == OP_JUMP || h.op == OP_CALL || at + h.size > n || (!at && h.op != OP_BEGIN)) rt_fatal("nvn: corrupt recording at %p (op %u size %u)", (const void *)(p + at), h.op, h.size); + const tex_t *pinned[9]; + int k = op_pins(h.op, out + sizeof(copy_prefix) + at + 8, pinned); + for (int i = 0; i < k; i++) pin_ref(pinned[i], 1, 0); + pins_named += (uint64_t)k; at += h.size; if (h.op == OP_END) { if (at != n) rt_fatal("nvn: recording at %p ends at %zu of %zu bytes", (const void *)p, at, n); break; } } - *(copy_prefix *)out = (copy_prefix){COPY_MAGIC, n, 0, 0}; + *(copy_prefix *)out = (copy_prefix){.magic = COPY_MAGIC, .bytes = n, .pins = pins_named}; return out + sizeof(copy_prefix); } @@ -399,7 +550,7 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { copy_bytes -= pf->bytes; pf->magic = 0; if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it - else free(pf); + else copy_free(pf); b->v[i] = b->v[--b->n]; } if (!b->n) { @@ -1035,6 +1186,7 @@ void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { copy_prefix *cl = malloc(sz); if (!cl) rt_fatal("nvn: out of memory (recording clone, %zu bytes)", sz); memcpy(cl, pf, sz); + if (cl->pins) pin_refs((const uint8_t *)(cl + 1), 1); cl->magic = 0; // not a live handle: the game never sees it cl->used_at = now; zombie_add(cl); diff --git a/runtime/nvn_probe.c b/runtime/nvn_probe.c index 9d1a79f8..8431d61a 100644 --- a/runtime/nvn_probe.c +++ b/runtime/nvn_probe.c @@ -694,6 +694,55 @@ static void parse_all(uint64_t h) { n_ops = OP_COUNT_ - OP_SIGNAL_EVENT; } +// A recording naming tex3 as its render target; *pin: the texture pin in its SET_RT payload. +static uint64_t record_target(uint8_t *ctl, const tex_t **pin) { + const NVNtexture *colors[1] = {&tex3}; + begin(&cmd2, ctl, 0x3a000); + CALL(nvnCommandBufferSetRenderTargets, U(&cmd2), 1, U(colors), 0, 0, 0); + uint64_t h = CALL(nvnCommandBufferEndRecording, U(&cmd2)); + *pin = NULL; + for (const op_hdr *o = nvn_op_first(h); o->op != OP_END; o = nvn_op_next(o)) + if (o->op == OP_SET_RT) *pin = ((const p_set_rt *)nvn_op_payload(o))->color[0]; + return h; +} + +// Texture pins are freed NVN_PIN_AGE submits after the last recording copy naming them is released, unless the replay +// keeps them as its current render targets; a texture named again gets a new pin. +static void probe_pins(void) { + uint8_t *ctl = mem_kernel_alloc(0x1000, 16); + set_tex(NVN_FORMAT_RGBA8, 16, 16, 1, 0x20000); + CHECK((uint8_t)CALL(nvnTextureInitialize, U(&tex3), U(&texb)) == 1, "pin probe texture"); + uint64_t serial = ((const tex_t *)(const void *)&tex3)->serial; + uint64_t live0, freed0, live, freed; + nvn_pins_stats(&live0, &freed0); + const tex_t *pin; + uint64_t h = record_target(ctl, &pin); + nvn_pins_stats(&live, &freed); + CHECK(pin && pin->serial == serial && live == live0 + 1, "a recording naming a new texture makes one pin"); + uint64_t s = nvn_submit_count(); + nvn_pins_sweep(s + 10 * NVN_PIN_AGE, NULL, 0); + nvn_pins_stats(&live, &freed); + CHECK(live == live0 + 1 && freed == freed0 && nvn_handle_live(h), "a pin a live recording names stays"); + CALL(nvnCommandBufferAddControlMemory, U(&cmd2), U(ctl), 0x1000); // retires h: the pin has no copy left + CHECK(!nvn_handle_live(h), "the pin's recording is retired"); + nvn_pins_sweep(s + NVN_PIN_AGE - 1, NULL, 0); + nvn_pins_stats(&live, &freed); + CHECK(live == live0 + 1 && freed == freed0, "a pin stays NVN_PIN_AGE submits after its last copy went"); + nvn_pins_sweep(s + NVN_PIN_AGE, &pin, 1); + nvn_pins_stats(&live, &freed); + CHECK(live == live0 + 1 && freed == freed0, "the replay's current render target is kept"); + nvn_pins_sweep(s + 2 * NVN_PIN_AGE, NULL, 0); + nvn_pins_stats(&live, &freed); + CHECK(live == live0 && freed == freed0 + 1, "an unnamed, unkept pin is freed (live %llu of %llu, freed %llu of %llu)", + (unsigned long long)live, (unsigned long long)live0, (unsigned long long)freed, (unsigned long long)freed0 + 1); + h = record_target(ctl, &pin); + nvn_pins_stats(&live, &freed); + CHECK(pin && pin->serial == serial && live == live0 + 1, "naming the texture again makes a new pin"); + CALL(nvnCommandBufferAddControlMemory, U(&cmd2), U(ctl), 0x1000); + CALL(nvnCommandBufferFinalize, U(&cmd2)); + CALL(nvnTextureFinalize, U(&tex3)); +} + static void probe_commands(void) { ctlmem1 = mem_kernel_alloc(0x1000, 16); ctlmem2 = mem_kernel_alloc(0x1000, 16); @@ -722,6 +771,7 @@ static void probe_commands(void) { CALL(nvnCommandBufferAddControlMemory, U(&cmd2), U(ctlmem2), 0x1000); CHECK(!nvn_handle_live(h2) && nvn_handle_live(h), "reusing control memory retires only the handles recorded in it"); CALL(nvnCommandBufferFinalize, U(&cmd2)); + probe_pins(); n_objects += 5; } diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 2709471d..5b863a16 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -919,6 +919,10 @@ static void walk(VkExec *ex, const op_hdr *h, int depth) { int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { VkExec *ex = &X; if (!x_ready) state_reset(ex), vk_dump_init(), x_ready = 1; + const tex_t *keep[9]; // the render targets carry over to this submission (set_rt compares them by pointer) + memcpy(keep, ex->rt.color, sizeof ex->rt.color); + keep[8] = ex->rt.depth; + nvn_pins_sweep(nvn_submit_count(), keep, 9); ex->cb = cb; ex->fence = fence; ex->recorded = 0; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 9e61e34a..897eadf5 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -24,7 +24,7 @@ enum { // One render target of OP_SET_RT, resolved to its vk_texture.c image for the current command buffer. typedef struct { - const tex_t *tex; // payload copy (lives for the run); NULL: no target in this slot + const tex_t *tex; // texture pin of ex->rt (nvn_pins_sweep keeps it); NULL: no target in this slot VkImage image; // VK_NULL_HANDLE: no target or not renderable (logged once) VkFormat format; // view format: the view's when it reinterprets a same-size format VkImageViewType view_type; // 2D, or 2D_ARRAY for more than one layer