mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-11 09:51:02 +00:00
Valiantsin DzerakhandClaude Opus 5.5
5c8f3ee5c6
Three game threads flatten recordings, inline them and add control memory at once, and every large copy came from a fresh VirtualAlloc, so each one paid zero-page faults, and release_copies freed them under the exclusive copy lock. Copies are now freed after the lock through a per-thread dead list, blocks of 256 KiB to 16 MiB come from a small best-fit cache, and each copy keeps its pin list so a free or clone no longer walks the records again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1349 lines
69 KiB
C
1349 lines
69 KiB
C
// NVN front end, command-buffer slice: command buffers, their memory, recording
|
|
// as op records (nvn_obj.h), recording copies behind command handles, and the submit-time resolution of deferred binds
|
|
// (nvn_submit_prepare, called by nvnQueueSubmitCommands in nvn_api_core.c). Ported from MacNativeGames
|
|
// games/totk/third_party/d2r_gfx/gfx/nvn/hle_nvn.c @ b75be8cf (MIT, Copyright (c) 2026 jhaizhou; origin d2r-mobile),
|
|
// lines 647-1062, 1261-1286, 1486-1760 and 1975-1990, and the *Deferred and memory-callback entry points of
|
|
// totk_nvn_ext.c in the same directory. Metal, mach, the translation thread, Apple diagnostics, the per-frame
|
|
// statistics and the copy marks in control memory dropped; the texture pin sweep replaced by counting (texture pins).
|
|
//
|
|
// Records are written into a growable host buffer per command buffer, never into the game's command memory: games hand
|
|
// the same command memory to several command buffers recording at once (budgeted for real NVN encodings), and our
|
|
// records are larger. The game's command and control memory is accounted only. EndRecording copies the finished
|
|
// recording into host memory; the handle is that copy (a flat OP_BEGIN ... OP_END sequence) and lives until the game
|
|
// adds the same control memory again. CallCommands and CopyCommands inline the called recording, so copies hold no
|
|
// OP_CALL or OP_JUMP.
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "nvn_api.h"
|
|
#include "nvn_hmap.h"
|
|
#include "platform/platform.h"
|
|
|
|
#define CMD_MAGIC MAGIC('C', 'M', 'D', 'B')
|
|
#define COPY_MAGIC 0x59504f43444d43ull // "CMDCOPY": prefix of a live recording copy
|
|
#define OP_RESERVE 24 // room always kept at the end of the recording buffer
|
|
#define MIN_COMMAND_SIZE 512 // NVN_DEVICE_INFO_COMMAND_BUFFER_MIN_COMMAND_SIZE (nvn_api_core.c devinfo)
|
|
#define MIN_CONTROL_SIZE 256 // NVN_DEVICE_INFO_COMMAND_BUFFER_MIN_CONTROL_SIZE
|
|
#define CTL_PER_RECORDING 64 // control memory one recording is reported to use
|
|
#define COPY_PAGE_SHIFT 16 // recording copy index granularity (release_copies)
|
|
|
|
static cmd_t *cmd_of(const Cpu *c) { return arg_p(c, 0); }
|
|
|
|
// ---------- memory callbacks
|
|
// NVN calls the game's memory callback when a recording needs command or control memory and none is left. Records
|
|
// live in host memory here, so the callback is called only where the hardware certainly would: a recording starts
|
|
// without any command memory, or ends without any control memory. It runs on the calling guest thread through
|
|
// rt_call; whatever memory it adds is accounted. Set{Command,Control}MemoryCallbackEnabled(false) suppress it.
|
|
enum { CB_NO_COMMAND = 1, CB_NO_CONTROL = 2 };
|
|
static PlatMutex cb_lock = PLAT_MUTEX_INIT;
|
|
static nvn_hmap cb_off = {.klen = 8}; // command buffer -> disabled callback kinds | 0x100
|
|
|
|
static unsigned cb_disabled(const cmd_t *o) {
|
|
plat_mutex_lock_shared(&cb_lock);
|
|
uintptr_t v = (uintptr_t)nvn_hm_get(&cb_off, &o);
|
|
plat_mutex_unlock_shared(&cb_lock);
|
|
return (unsigned)v & 0xff;
|
|
}
|
|
|
|
static void cb_set_disabled(const cmd_t *o, unsigned kind, int disabled) {
|
|
plat_mutex_lock(&cb_lock);
|
|
unsigned v = (unsigned)(uintptr_t)nvn_hm_get(&cb_off, &o) & 0xff;
|
|
v = disabled ? v | kind : v & ~kind;
|
|
if (v) nvn_hm_put(&cb_off, &o, (void *)(uintptr_t)(v | 0x100));
|
|
else nvn_hm_del(&cb_off, &o);
|
|
plat_mutex_unlock(&cb_lock);
|
|
}
|
|
|
|
static void memory_event(Cpu *c, cmd_t *o, int event, uint64_t min) {
|
|
if (!o->cb || (cb_disabled(o) & (event == NVN_COMMAND_BUFFER_MEMORY_EVENT_OUT_OF_COMMAND_MEMORY ? CB_NO_COMMAND
|
|
: CB_NO_CONTROL)))
|
|
return;
|
|
NVN_LOG("command buffer %p: memory callback %p (%s memory, %llu bytes)", (void *)o, (void *)o->cb,
|
|
event == NVN_COMMAND_BUFFER_MEMORY_EVENT_OUT_OF_COMMAND_MEMORY ? "command" : "control",
|
|
(unsigned long long)min);
|
|
rt_call(c, (uint64_t)(uintptr_t)o->cb, 4, (uint64_t)(uintptr_t)o, (uint64_t)event, min,
|
|
(uint64_t)(uintptr_t)o->cb_data);
|
|
}
|
|
|
|
// ---------- recording buffer
|
|
static void put_hdr(uint8_t *p, uint32_t op, uint32_t size) {
|
|
op_hdr h = {op, size};
|
|
memcpy(p, &h, sizeof h);
|
|
}
|
|
|
|
// 1.4.3 initializes and finalizes command buffers about 2000 times a second while a scene loads and several times a
|
|
// frame in play, and its two largest per-frame recordings take 1-2 MiB. Freeing the recording buffer at Finalize made
|
|
// every new recording grow again from 64 KiB through up to five reallocations and copies, the last ones fresh 1-2 MiB
|
|
// blocks the heap takes from the system: new commit and zero-filled pages twice a frame. Those growths were most of the
|
|
// process's new commit requests, so when the system ran short for a moment (a Depths load at render scale 200 on 16 GB)
|
|
// a 1 MiB growth was the allocation that failed. Finalized buffers therefore wait in a small pool for the next
|
|
// recording, and a failed allocation waits for memory instead of ending the game (realloc_wait).
|
|
#define POOL_SLOTS 16
|
|
#define POOL_MAX_BYTES ((size_t)4 << 20) // a larger buffer (a recording that inlined a big one) goes back to the heap
|
|
static PlatMutex pool_lock = PLAT_MUTEX_INIT;
|
|
static struct { uint8_t *p; size_t size; } pool[POOL_SLOTS];
|
|
static uint32_t npool;
|
|
|
|
// The largest pooled buffer and its size, or NULL: the largest, so the big per-frame recordings stop growing once the
|
|
// pool holds enough buffers of their size.
|
|
static uint8_t *pool_take(size_t *size) {
|
|
uint8_t *p = NULL;
|
|
plat_mutex_lock(&pool_lock);
|
|
if (npool) {
|
|
uint32_t best = 0;
|
|
for (uint32_t i = 1; i < npool; i++)
|
|
if (pool[i].size > pool[best].size) best = i;
|
|
p = pool[best].p;
|
|
*size = pool[best].size;
|
|
pool[best] = pool[--npool];
|
|
}
|
|
plat_mutex_unlock(&pool_lock);
|
|
return p;
|
|
}
|
|
|
|
// Keeps a finalized buffer when the pool has room or holds a smaller one (which is freed instead).
|
|
static void pool_put(uint8_t *p, size_t size) {
|
|
if (!p) return;
|
|
if (size <= POOL_MAX_BYTES) {
|
|
plat_mutex_lock(&pool_lock);
|
|
if (npool < POOL_SLOTS) {
|
|
pool[npool].p = p;
|
|
pool[npool++].size = size;
|
|
p = NULL;
|
|
} else {
|
|
uint32_t small = 0;
|
|
for (uint32_t i = 1; i < npool; i++)
|
|
if (pool[i].size < pool[small].size) small = i;
|
|
if (pool[small].size < size) {
|
|
uint8_t *drop = pool[small].p;
|
|
pool[small].p = p;
|
|
pool[small].size = size;
|
|
p = drop;
|
|
}
|
|
}
|
|
plat_mutex_unlock(&pool_lock);
|
|
}
|
|
free(p);
|
|
}
|
|
|
|
static void pool_drain(void) {
|
|
plat_mutex_lock(&pool_lock);
|
|
while (npool) free(pool[--npool].p);
|
|
plat_mutex_unlock(&pool_lock);
|
|
}
|
|
|
|
// realloc that waits out a short commit shortage: Windows refuses commit beyond the current limit while it grows a
|
|
// system-managed page file, so the same request succeeds a moment later, and a stall of some frames is better than
|
|
// ending the game. Frees the pooled buffers first. NULL only after about 30 s without memory; the caller ends the game.
|
|
// Never called with a lock held.
|
|
static void big_drain(void);
|
|
|
|
static void *realloc_wait(void *old, size_t n, const char *what) {
|
|
void *p = realloc(old, n);
|
|
if (p) return p;
|
|
pool_drain();
|
|
big_drain();
|
|
static uint32_t reports;
|
|
int report = __atomic_fetch_add(&reports, 1, __ATOMIC_RELAXED) < 8;
|
|
if (report) {
|
|
double proc, used, limit;
|
|
plat_commit_info(&proc, &used, &limit);
|
|
rt_log("nvn: %s of %zu bytes failed (memory commit: this process %.0f MB, the system %.0f of %.0f MB); waiting "
|
|
"for memory", what, n, proc, used, limit);
|
|
}
|
|
uint64_t t0 = plat_now_ms();
|
|
while (plat_now_ms() - t0 < 30000) {
|
|
plat_sleep_ms(10);
|
|
if ((p = realloc(old, n))) {
|
|
if (report)
|
|
rt_log("nvn: %s of %zu bytes succeeded after %llu ms", what, n,
|
|
(unsigned long long)(plat_now_ms() - t0));
|
|
return p;
|
|
}
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static void cmd_reserve(cmd_t *o, size_t need) {
|
|
if (o->seg && o->cur + need + OP_RESERVE <= o->end) return;
|
|
if (!o->seg) { // the first record since Initialize: a pooled buffer when one waits
|
|
size_t size = 0;
|
|
uint8_t *p = pool_take(&size);
|
|
if (p) {
|
|
o->seg = o->cur = p;
|
|
o->end = p + size;
|
|
if (need + OP_RESERVE <= size) return;
|
|
}
|
|
}
|
|
size_t used = o->seg ? (size_t)(o->cur - o->seg) : 0;
|
|
size_t cap = o->seg ? (size_t)(o->end - o->seg) : 0;
|
|
if (!cap) cap = 64 << 10;
|
|
while (used + need + OP_RESERVE > cap) cap *= 2;
|
|
uint8_t *b = realloc_wait(o->seg, cap, "command recording buffer");
|
|
// Still failing after the wait: the system has no commit left (the fatal report states it), or seg is no block of
|
|
// ours: a command buffer object the game overwrote or copied shows a wrong magic or more used than its old size.
|
|
if (!b)
|
|
rt_fatal("nvn: command recording buffer of %zu bytes (command buffer %p, magic %#x, buffer %p of %zu bytes, %zu "
|
|
"used, %zu asked)", cap, (void *)o, o->magic, (void *)o->seg, o->seg ? (size_t)(o->end - o->seg) : 0,
|
|
used, need);
|
|
o->seg = b;
|
|
o->cur = b + used;
|
|
o->end = b + cap;
|
|
}
|
|
|
|
// Space for one record (header included); the header's op is filled by the caller.
|
|
static uint8_t *cmd_alloc(cmd_t *o, uint32_t size) {
|
|
size = (size + 7) & ~7u;
|
|
cmd_reserve(o, size);
|
|
uint8_t *p = o->cur;
|
|
o->cur += size;
|
|
o->cmd_used += size;
|
|
o->hw_used += size;
|
|
o->last_rec = size;
|
|
put_hdr(p, 0, size);
|
|
return p;
|
|
}
|
|
|
|
// The game sums nvnCommandBufferGetCommandMemoryUsed over recordings to size resource heaps, so usage must be what the
|
|
// real NVN push-buffer encoding takes, not our record size. Known encodings (Maxwell methods, from a real recording):
|
|
// DrawElements 12 words, DrawElementsBaseVertex 11, BindVertexAttribState 1+16, BindVertexStreamState 16+n,
|
|
// BindVertexBuffer 6 words each.
|
|
static void hw_fix(cmd_t *o, size_t bytes) {
|
|
o->hw_used += bytes;
|
|
o->hw_used -= o->last_rec;
|
|
o->last_rec = bytes;
|
|
}
|
|
|
|
// A zeroed payload of one record.
|
|
static void *rec(cmd_t *o, uint32_t op, size_t payload) {
|
|
uint32_t sz = (uint32_t)((8 + payload + 7) & ~(size_t)7);
|
|
uint8_t *p = cmd_alloc(o, sz);
|
|
put_hdr(p, op, sz);
|
|
memset(p + 8, 0, sz - 8);
|
|
return p + 8;
|
|
}
|
|
#define REC(o, OP, T) ((T *)rec((o), (OP), sizeof(T)))
|
|
// Array-carrying binds take the prefix fields plus n elements; the replay reads only the first n.
|
|
#define RECN(o, OP, T, ARR, N) ((T *)rec((o), (OP), offsetof(T, ARR) + sizeof(((T *)0)->ARR[0]) * (size_t)(N)))
|
|
|
|
static void copy_view(view_t *dst, uint8_t *has, const NVNtextureView *v) {
|
|
*has = v != NULL;
|
|
if (v) *dst = *(const view_t *)v;
|
|
}
|
|
|
|
// ---------- texture pins
|
|
// Recordings refer to textures by pointer, but the game may finalize or reuse an NVNtexture after recording and before
|
|
// the submit (NVN itself captured only GPU state). Records therefore point at an immutable host copy per texture
|
|
// initialization (tex_t.serial). The game initializes its transient targets again every frame, about 29 new serials a
|
|
// frame on 1.4.3, so copies kept for the run grew the heap by about 44 MiB per minute of play, standing still too, until
|
|
// the commit limit ended the game after a few hours. A copy is freed once nothing can read it any more:
|
|
// - refs counts the recording copies whose ops name it: live ones, released ones a queued submission still reads, and
|
|
// per-submission clones (pin_refs, at flatten_recording and wherever such a copy is freed or cloned);
|
|
// - a recording in progress holds pins no copy counts yet, through its own tex_pin calls and through ops inlined from a
|
|
// copy that was released since: a pin goes only NVN_PIN_AGE submits after its last tex_pin and after refs fell to 0;
|
|
// - the replay's current render targets carry over to later submissions and are compared by pointer (vk_exec.c set_rt),
|
|
// so a freed address reused by a new pin would read as the same target: nvn_pins_sweep runs between submissions on
|
|
// the thread that replays them and keeps the pins it is given.
|
|
// MacNativeGames hit the same growth (3.16 million copies, 405 MB after 27 minutes) and marks pins by walking every live
|
|
// recording each window (hle_nvn.c pin_scan); counting at copy and release walks each copy's ops once more, when it is
|
|
// freed. TOTK_PIN_SWEEP=0 keeps every copy.
|
|
typedef struct {
|
|
tex_t t; // first: ops hold pins as tex_t pointers
|
|
uint64_t last_use; // nsubmits at the last tex_pin or the last drop of refs
|
|
uint32_t refs;
|
|
uint8_t queued; // in pin_q
|
|
} pin_t;
|
|
_Static_assert(offsetof(pin_t, t) == 0, "ops hold pins as tex_t pointers");
|
|
typedef struct { pin_t *p; uint64_t at; } pin_due;
|
|
static PlatMutex pin_lock = PLAT_MUTEX_INIT; // lock order: copy_lock, then pin_lock
|
|
static nvn_hmap pins = {.klen = 8}; // serial -> pin_t
|
|
static pin_due *pin_q; // pins with no counted copy, oldest first: [pin_qh, pin_qn)
|
|
static size_t pin_qh, pin_qn, pin_qcap;
|
|
static uint64_t pins_live, pins_freed;
|
|
static uint64_t nsubmits; // QueueSubmitCommands calls so far
|
|
|
|
static void pin_queue(pin_t *p, uint64_t at) { // pin_lock held
|
|
if (p->queued) return;
|
|
if (pin_qh && pin_qh * 2 >= pin_qn) {
|
|
memmove(pin_q, pin_q + pin_qh, sizeof *pin_q * (pin_qn - pin_qh));
|
|
pin_qn -= pin_qh, pin_qh = 0;
|
|
}
|
|
if (pin_qn == pin_qcap) {
|
|
pin_qcap = pin_qcap ? pin_qcap * 2 : 4096;
|
|
if (!(pin_q = realloc(pin_q, sizeof *pin_q * pin_qcap))) rt_fatal("nvn: out of memory (texture pin queue)");
|
|
}
|
|
pin_q[pin_qn++] = (pin_due){p, at};
|
|
p->queued = 1;
|
|
}
|
|
|
|
static const tex_t *tex_pin(const NVNtexture *nt) {
|
|
const tex_t *t = (const tex_t *)nt;
|
|
if (!t) return NULL;
|
|
uint64_t serial = t->serial;
|
|
if (!serial) rt_fatal("nvn: a recording references the uninitialized texture object %p", (const void *)t);
|
|
uint64_t now = __atomic_load_n(&nsubmits, __ATOMIC_RELAXED);
|
|
plat_mutex_lock_shared(&pin_lock); // nvn_pins_sweep frees under the exclusive lock
|
|
pin_t *r = nvn_hm_get(&pins, &serial);
|
|
if (r) __atomic_store_n(&r->last_use, now, __ATOMIC_RELAXED);
|
|
plat_mutex_unlock_shared(&pin_lock);
|
|
if (r) return &r->t;
|
|
plat_mutex_lock(&pin_lock);
|
|
r = nvn_hm_get(&pins, &serial);
|
|
if (!r) {
|
|
if (!(r = malloc(sizeof *r))) rt_fatal("nvn: texture pin copy");
|
|
*r = (pin_t){.t = *t};
|
|
nvn_hm_put(&pins, &serial, r);
|
|
pins_live++;
|
|
pin_queue(r, now); // freed like any other if no copy ever names it
|
|
}
|
|
__atomic_store_n(&r->last_use, now, __ATOMIC_RELAXED);
|
|
plat_mutex_unlock(&pin_lock);
|
|
return &r->t;
|
|
}
|
|
|
|
// The pins an op's payload names, into out (at most 9); returns how many.
|
|
static int op_pins(uint32_t op, const void *a, const tex_t *out[9]) {
|
|
int k = 0;
|
|
switch (op) {
|
|
case OP_SET_RT: {
|
|
const p_set_rt *r = a;
|
|
int n = r->n < 0 ? 0 : r->n > 8 ? 8 : r->n;
|
|
for (int i = 0; i < n; i++)
|
|
if (r->color[i]) out[k++] = r->color[i];
|
|
if (r->depth) out[k++] = r->depth;
|
|
break;
|
|
}
|
|
case OP_COPY_B2T: case OP_COPY_T2B:
|
|
if (((const p_copy_b2t *)a)->tex) out[k++] = ((const p_copy_b2t *)a)->tex;
|
|
break;
|
|
case OP_CLEAR_TEX:
|
|
if (((const p_clear_tex *)a)->tex) out[k++] = ((const p_clear_tex *)a)->tex;
|
|
break;
|
|
case OP_COPY_T2T: {
|
|
const p_copy_t2t *c = a;
|
|
if (c->src) out[k++] = c->src;
|
|
if (c->dst) out[k++] = c->dst;
|
|
break;
|
|
}
|
|
default: break;
|
|
}
|
|
return k;
|
|
}
|
|
|
|
static void pin_ref(const tex_t *t, int delta, uint64_t now) {
|
|
pin_t *p = (pin_t *)(uintptr_t)t;
|
|
if (delta > 0) {
|
|
__atomic_add_fetch(&p->refs, 1, __ATOMIC_RELAXED);
|
|
return;
|
|
}
|
|
__atomic_store_n(&p->last_use, now, __ATOMIC_RELAXED);
|
|
if (__atomic_sub_fetch(&p->refs, 1, __ATOMIC_ACQ_REL)) return;
|
|
plat_mutex_lock(&pin_lock);
|
|
pin_queue(p, now);
|
|
plat_mutex_unlock(&pin_lock);
|
|
}
|
|
|
|
// Counts (delta 1) or uncounts (-1) the pins named by the copy at handle, OP_BEGIN to OP_END; returns how many.
|
|
static uint32_t pin_refs(const uint8_t *copy, int delta) {
|
|
uint64_t now = __atomic_load_n(&nsubmits, __ATOMIC_RELAXED);
|
|
uint32_t n = 0;
|
|
for (const op_hdr *h = nvn_op_first((uintptr_t)copy); h->op != OP_END; h = nvn_op_next(h)) {
|
|
const tex_t *p[9];
|
|
int k = op_pins(h->op, nvn_op_payload(h), p);
|
|
for (int i = 0; i < k; i++) pin_ref(p[i], delta, now);
|
|
n += (uint32_t)k;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
void nvn_pins_sweep(uint64_t now, const tex_t *const *keep, int nkeep) {
|
|
if (now < NVN_PIN_AGE) return;
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
const char *s = cfg_get("TOTK_PIN_SWEEP");
|
|
on = !(s && !strcmp(s, "0"));
|
|
if (!on) rt_log("nvn: TOTK_PIN_SWEEP=0: texture pin copies are kept for the run (the heap grows in long sessions)");
|
|
}
|
|
if (!on) return;
|
|
static uint64_t logged;
|
|
plat_mutex_lock(&pin_lock);
|
|
for (int budget = 1024; pin_qh < pin_qn && budget; budget--) {
|
|
pin_due d = pin_q[pin_qh];
|
|
if (d.at > now - NVN_PIN_AGE) break;
|
|
pin_qh++;
|
|
pin_t *p = d.p;
|
|
p->queued = 0;
|
|
if (__atomic_load_n(&p->refs, __ATOMIC_ACQUIRE)) continue; // queued again when the count falls to 0
|
|
uint64_t last = __atomic_load_n(&p->last_use, __ATOMIC_RELAXED);
|
|
int kept = 0;
|
|
for (int i = 0; i < nkeep; i++) kept |= keep[i] == &p->t;
|
|
if (kept || last > now - NVN_PIN_AGE) {
|
|
pin_queue(p, kept ? now : last);
|
|
continue;
|
|
}
|
|
nvn_hm_del(&pins, &p->t.serial);
|
|
free(p);
|
|
pins_live--, pins_freed++;
|
|
}
|
|
if (now / 7200 != logged) {
|
|
logged = now / 7200;
|
|
rt_log("nvn: texture pins at submit %llu: %llu live, %llu waiting, %llu freed so far", (unsigned long long)now,
|
|
(unsigned long long)pins_live, (unsigned long long)(pin_qn - pin_qh), (unsigned long long)pins_freed);
|
|
}
|
|
plat_mutex_unlock(&pin_lock);
|
|
}
|
|
|
|
void nvn_pins_stats(uint64_t *live, uint64_t *freed) {
|
|
plat_mutex_lock_shared(&pin_lock);
|
|
*live = pins_live, *freed = pins_freed;
|
|
plat_mutex_unlock_shared(&pin_lock);
|
|
}
|
|
|
|
// ---------- recording copies
|
|
// A handle points just past a copy_prefix. Copies are indexed by the 64 KiB block of their control-memory position, so
|
|
// adding control memory again releases the copies recorded there without walking every live copy. 4 KiB pages cost
|
|
// about 500 lookups per AddControlMemory under the exclusive lock (1.4.3 field: 53 calls a frame, 2 MiB ranges, 3.7
|
|
// copies per page), and the recording threads waited on it.
|
|
// deferred: 0 unknown, 1 none, 2 some; pins: texture pins its ops name, counted in their refs; pin_v: those pins, in
|
|
// record order, or NULL (then pin_refs walks the records); cap: bytes of the block from the large-block cache
|
|
// (copy_block), 0 for a plain heap block
|
|
typedef struct { uint64_t magic, bytes, made_at, used_at, deferred, pins, cap; const tex_t **pin_v; } copy_prefix;
|
|
_Static_assert(sizeof(copy_prefix) % 8 == 0, "records stay 8-byte aligned");
|
|
typedef struct { uintptr_t lo, hi; uint8_t *copy; } cmd_copy;
|
|
typedef struct { cmd_copy *v; uint32_t n, cap; } copy_bucket;
|
|
static PlatMutex copy_lock = PLAT_MUTEX_INIT; // guards the two maps, the counters and every prefix
|
|
static nvn_hmap copy_pages = {.klen = 8}; // page -> copy_bucket
|
|
static nvn_hmap copy_live = {.klen = 8}; // handle -> copy_prefix of a live copy
|
|
static uint64_t ncopies, copy_bytes, copies_made;
|
|
// Asynchronous replay: copies released by the game, and per-submission clones, that a queued submission still reads.
|
|
// Freed once gfx_replayed_submits() reaches their used_at. Empty with synchronous replay.
|
|
static copy_prefix **zombies;
|
|
static uint32_t nzombies, cap_zombies;
|
|
|
|
static void zombie_add(copy_prefix *pf) { // copy_lock held
|
|
if (nzombies == cap_zombies) {
|
|
cap_zombies = cap_zombies ? cap_zombies * 2 : 64;
|
|
zombies = realloc(zombies, sizeof *zombies * cap_zombies);
|
|
if (!zombies) rt_fatal("nvn: out of memory (released recording copies)");
|
|
}
|
|
zombies[nzombies++] = pf;
|
|
}
|
|
|
|
// Blocks of large copies are kept for the next ones. 1.4.3 makes two copies of 0.5-2 MiB a frame (the per-frame
|
|
// recordings that inline many others); above the heap's direct-allocation threshold (about 512 KiB) every malloc mapped
|
|
// fresh zero-filled pages and every free unmapped them, so the copy ran at half the speed of a small one and the free
|
|
// held copy_lock for tens of microseconds. A block is reused for a copy that fills at least half of it.
|
|
#define BIG_MIN ((size_t)256 << 10) // smaller blocks come from the heap and go back to it
|
|
#define BIG_MAX ((size_t)16 << 20) // a larger block is never kept
|
|
#define BIG_SLOTS 8
|
|
static PlatMutex big_lock = PLAT_MUTEX_INIT; // lock order: copy_lock, then big_lock
|
|
static struct { copy_prefix *p; size_t cap; } big[BIG_SLOTS];
|
|
static uint32_t nbig;
|
|
|
|
static void big_drain(void) {
|
|
plat_mutex_lock(&big_lock);
|
|
while (nbig) free(big[--nbig].p);
|
|
plat_mutex_unlock(&big_lock);
|
|
}
|
|
|
|
// A block for a copy of n bytes after its prefix; prefix.cap says where it goes back (copy_block_free). wait: a failed
|
|
// allocation waits for memory (realloc_wait), which is never done with a lock held; else it ends the game at once.
|
|
static copy_prefix *copy_block(size_t n, int wait, const char *what) {
|
|
size_t total = sizeof(copy_prefix) + n, cap = 0;
|
|
copy_prefix *pf = NULL;
|
|
if (total >= BIG_MIN && total <= BIG_MAX) {
|
|
plat_mutex_lock(&big_lock);
|
|
uint32_t best = BIG_SLOTS;
|
|
for (uint32_t i = 0; i < nbig; i++)
|
|
if (big[i].cap >= total && big[i].cap / 2 <= total && (best == BIG_SLOTS || big[i].cap < big[best].cap))
|
|
best = i;
|
|
if (best < BIG_SLOTS) {
|
|
pf = big[best].p;
|
|
cap = big[best].cap;
|
|
big[best] = big[--nbig];
|
|
}
|
|
plat_mutex_unlock(&big_lock);
|
|
if (!pf) cap = (total + total / 8 + 0xffff) & ~(size_t)0xffff; // room for the next frame's slightly larger one
|
|
}
|
|
if (!pf) pf = wait ? realloc_wait(NULL, cap ? cap : total, what) : malloc(cap ? cap : total);
|
|
if (!pf) rt_fatal("nvn: %s allocation (%zu bytes)", what, cap ? cap : total);
|
|
pf->cap = cap;
|
|
return pf;
|
|
}
|
|
|
|
static void copy_block_free(copy_prefix *pf) {
|
|
size_t cap = pf->cap;
|
|
if (cap) {
|
|
plat_mutex_lock(&big_lock);
|
|
if (nbig < BIG_SLOTS) {
|
|
big[nbig].p = pf;
|
|
big[nbig++].cap = cap;
|
|
pf = NULL;
|
|
} else {
|
|
uint32_t small = 0;
|
|
for (uint32_t i = 1; i < nbig; i++)
|
|
if (big[i].cap < big[small].cap) small = i;
|
|
if (big[small].cap < cap) {
|
|
copy_prefix *drop = big[small].p;
|
|
big[small].p = pf;
|
|
big[small].cap = cap;
|
|
pf = drop;
|
|
}
|
|
}
|
|
plat_mutex_unlock(&big_lock);
|
|
}
|
|
free(pf);
|
|
}
|
|
|
|
// Counts (delta 1) or uncounts (-1) the pins a copy names: from its list, without walking its records. A copy of a
|
|
// per-frame recording is 0.5-2 MiB of records naming a few pins; walking them again at every free cost about 40 us.
|
|
static void copy_pins_ref(const copy_prefix *pf, int delta) {
|
|
if (!pf->pins) return;
|
|
if (!pf->pin_v) {
|
|
pin_refs((const uint8_t *)(pf + 1), delta);
|
|
return;
|
|
}
|
|
uint64_t now = __atomic_load_n(&nsubmits, __ATOMIC_RELAXED);
|
|
for (uint64_t i = 0; i < pf->pins; i++) pin_ref(pf->pin_v[i], delta, now);
|
|
}
|
|
|
|
// A copy in no map and no zombie list: nothing else can reach it, so copy_lock need not be held.
|
|
static void copy_free(copy_prefix *pf) {
|
|
copy_pins_ref(pf, -1);
|
|
free((void *)pf->pin_v);
|
|
copy_block_free(pf);
|
|
}
|
|
|
|
// Copies retired under copy_lock and freed after it is released: the frees, the unpinning walk over their records and
|
|
// the large blocks kept the lock exclusive for most of release_copies, while the game's other recording threads waited
|
|
// for it in copy_touch (1.4.3 town: three threads, 0.6 ms a frame of waiting).
|
|
typedef struct { copy_prefix **v; uint32_t n, cap; } dead_list;
|
|
static __thread dead_list dead;
|
|
|
|
static void dead_add(copy_prefix *pf) {
|
|
if (dead.n == dead.cap) {
|
|
uint32_t cap = dead.cap ? dead.cap * 2 : 256;
|
|
copy_prefix **v = realloc(dead.v, sizeof *v * cap);
|
|
if (!v) { copy_free(pf); return; } // out of memory: free it here, under the lock, as before
|
|
dead.v = v;
|
|
dead.cap = cap;
|
|
}
|
|
dead.v[dead.n++] = pf;
|
|
}
|
|
|
|
static void dead_free(void) { // copy_lock not held
|
|
for (uint32_t i = 0; i < dead.n; i++) copy_free(dead.v[i]);
|
|
dead.n = 0;
|
|
}
|
|
|
|
static void zombie_sweep(uint64_t replayed) { // copy_lock held; the swept copies go to the dead list
|
|
for (uint32_t i = 0; i < nzombies;) {
|
|
if (zombies[i]->used_at <= replayed) {
|
|
dead_add(zombies[i]);
|
|
zombies[i] = zombies[--nzombies];
|
|
} else {
|
|
i++;
|
|
}
|
|
}
|
|
}
|
|
|
|
static copy_prefix *copy_find(uint64_t handle) { // copy_lock held
|
|
return nvn_hm_get(©_live, &handle);
|
|
}
|
|
|
|
int nvn_handle_live(uint64_t handle) {
|
|
plat_mutex_lock_shared(©_lock);
|
|
int live = copy_find(handle) != NULL;
|
|
plat_mutex_unlock_shared(©_lock);
|
|
return live;
|
|
}
|
|
|
|
uint64_t nvn_handle_bytes(uint64_t handle) {
|
|
plat_mutex_lock_shared(©_lock);
|
|
const copy_prefix *pf = copy_find(handle);
|
|
uint64_t n = pf ? pf->bytes : 0;
|
|
plat_mutex_unlock_shared(©_lock);
|
|
return n;
|
|
}
|
|
|
|
// Marks a live copy used by submission now; its size from OP_BEGIN to OP_END inclusive.
|
|
static uint64_t copy_touch(uint64_t handle, uint64_t now, const char *what) {
|
|
plat_mutex_lock_shared(©_lock);
|
|
copy_prefix *pf = copy_find(handle);
|
|
uint64_t bytes = 0;
|
|
if (pf) __atomic_store_n(&pf->used_at, now, __ATOMIC_RELAXED), bytes = pf->bytes;
|
|
plat_mutex_unlock_shared(©_lock);
|
|
if (!pf) rt_fatal("nvn: %s: command handle %#llx is not a live recording (released or never recorded)", what,
|
|
(unsigned long long)handle);
|
|
return bytes;
|
|
}
|
|
|
|
// The recording [p, p + n), OP_BEGIN to OP_END (the recording buffer is one block: recording emits no OP_JUMP), copied
|
|
// into one host block (copy_block: its exact size, or a kept large block). One allocation and one copy: growing the
|
|
// copy record by record cost the game's main thread about 1 ms a frame in reallocations and small copies (1.4.3,
|
|
// field). The pins the records name are counted and listed (copy_prefix.pin_v) on the way.
|
|
static uint8_t *flatten_recording(const uint8_t *p, size_t n) {
|
|
copy_prefix *block = copy_block(n, 1, "command copy");
|
|
uint8_t *out = (uint8_t *)block;
|
|
memcpy(out + sizeof(copy_prefix), p, n);
|
|
static __thread struct { const tex_t **v; uint64_t cap; } named; // the pins of this recording, reused
|
|
uint64_t pins_named = 0;
|
|
int listed = 1; // 0 after a failed growth: the copy goes without a list
|
|
for (size_t at = 0;;) { // the records stay checked: a corrupt one stops here, not in the replay
|
|
op_hdr h;
|
|
memcpy(&h, p + at, sizeof h);
|
|
if (h.size < 8 || (h.size & 7) || h.op >= OP_COUNT_ || h.op == OP_JUMP || h.op == OP_CALL || at + h.size > n ||
|
|
(!at && h.op != OP_BEGIN))
|
|
rt_fatal("nvn: corrupt recording at %p (op %u size %u)", (const void *)(p + at), h.op, h.size);
|
|
const tex_t *pinned[9];
|
|
int k = op_pins(h.op, out + sizeof(copy_prefix) + at + 8, pinned);
|
|
for (int i = 0; i < k; i++) pin_ref(pinned[i], 1, 0);
|
|
if (k && listed && pins_named + (uint64_t)k > named.cap) {
|
|
uint64_t cap = named.cap ? named.cap * 2 : 64;
|
|
while (cap < pins_named + (uint64_t)k) cap *= 2;
|
|
const tex_t **v = realloc((void *)named.v, sizeof *v * cap);
|
|
if (v) named.v = v, named.cap = cap;
|
|
else listed = 0;
|
|
}
|
|
if (k && listed) memcpy((void *)(named.v + pins_named), pinned, sizeof *pinned * (size_t)k);
|
|
pins_named += (uint64_t)k;
|
|
at += h.size;
|
|
if (h.op == OP_END) {
|
|
if (at != n) rt_fatal("nvn: recording at %p ends at %zu of %zu bytes", (const void *)p, at, n);
|
|
break;
|
|
}
|
|
}
|
|
const tex_t **pin_v = NULL;
|
|
if (pins_named && listed && (pin_v = malloc(sizeof *pin_v * pins_named)))
|
|
memcpy((void *)pin_v, (const void *)named.v, sizeof *pin_v * pins_named);
|
|
*block = (copy_prefix){.magic = COPY_MAGIC, .bytes = n, .pins = pins_named, .cap = block->cap, .pin_v = pin_v};
|
|
return out + sizeof(copy_prefix);
|
|
}
|
|
|
|
static void copy_add(uintptr_t lo, uintptr_t hi, uint8_t *copy) {
|
|
copy_prefix *pf = (copy_prefix *)copy - 1;
|
|
uint64_t handle = (uintptr_t)copy;
|
|
uintptr_t page = lo >> COPY_PAGE_SHIFT;
|
|
plat_mutex_lock(©_lock);
|
|
pf->made_at = nsubmits;
|
|
copy_bucket *b = nvn_hm_get(©_pages, &page);
|
|
if (!b) {
|
|
b = calloc(1, sizeof *b);
|
|
if (!b) rt_fatal("nvn: command copy index");
|
|
nvn_hm_put(©_pages, &page, b);
|
|
}
|
|
if (b->n == b->cap) {
|
|
b->cap = b->cap ? b->cap * 2 : 8;
|
|
b->v = realloc(b->v, b->cap * sizeof *b->v);
|
|
if (!b->v) rt_fatal("nvn: command copy index");
|
|
}
|
|
b->v[b->n++] = (cmd_copy){lo, hi, copy};
|
|
nvn_hm_put(©_live, &handle, pf);
|
|
ncopies++;
|
|
copy_bytes += pf->bytes;
|
|
if (++copies_made % 20000 == 0)
|
|
NVN_LOG("recording copies: %llu made, %llu live, %.1f MiB, submit %llu", (unsigned long long)copies_made,
|
|
(unsigned long long)ncopies, copy_bytes / 1048576.0, (unsigned long long)nsubmits);
|
|
plat_mutex_unlock(©_lock);
|
|
}
|
|
|
|
// Adding control memory again retires every recording copy made in it (the driver's handle lived there).
|
|
static void release_copies(uintptr_t lo, uintptr_t hi) {
|
|
if (hi <= lo) return;
|
|
uint64_t replayed = gfx_replayed_submits();
|
|
plat_mutex_lock(©_lock);
|
|
if (nzombies) zombie_sweep(replayed);
|
|
for (uintptr_t pg = lo >> COPY_PAGE_SHIFT; pg <= (hi - 1) >> COPY_PAGE_SHIFT; pg++) {
|
|
copy_bucket *b = nvn_hm_get(©_pages, &pg);
|
|
if (!b) continue;
|
|
for (uint32_t i = 0; i < b->n;) {
|
|
cmd_copy *k = &b->v[i];
|
|
if (!(k->lo < hi && lo < k->hi)) { i++; continue; }
|
|
uint64_t handle = (uintptr_t)k->copy;
|
|
copy_prefix *pf = (copy_prefix *)k->copy - 1;
|
|
nvn_hm_del(©_live, &handle);
|
|
ncopies--;
|
|
copy_bytes -= pf->bytes;
|
|
pf->magic = 0;
|
|
if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it
|
|
else dead_add(pf);
|
|
b->v[i] = b->v[--b->n];
|
|
}
|
|
if (!b->n) {
|
|
nvn_hm_del(©_pages, &pg);
|
|
free(b->v);
|
|
free(b);
|
|
}
|
|
}
|
|
plat_mutex_unlock(©_lock);
|
|
dead_free();
|
|
}
|
|
|
|
// ---------- CommandBuffer: object and memory
|
|
// NVNboolean nvnCommandBufferInitialize(NVNcommandBuffer *cmd, NVNdevice *device): the inline fast-path fields are kept
|
|
// at ~0 / 0 so NVN 55 inline code never finds room and calls the API (nvn_obj.h cmd_t)
|
|
NVN_API(nvnCommandBufferInitialize) {
|
|
cmd_t *o = cmd_of(c);
|
|
// Initialize over an object that was never finalized drops its recording buffer (seg lives in host memory). Logged
|
|
// only: the old bytes may be a copy whose buffer another object owns, so freeing it here is not safe.
|
|
static uint32_t reinits;
|
|
if (o->magic == CMD_MAGIC && o->seg && __atomic_fetch_add(&reinits, 1, __ATOMIC_RELAXED) < 8)
|
|
rt_log("nvn: Initialize over the command buffer %p, which was not finalized; its %zu-byte recording buffer %p "
|
|
"is dropped", (void *)o, (size_t)(o->end - o->seg), (void *)o->seg);
|
|
memset(o, 0, sizeof *o);
|
|
o->magic = CMD_MAGIC;
|
|
o->dev = arg_p(c, 1);
|
|
o->inline_cur = ~0ull;
|
|
o->inline_end = 0;
|
|
cb_set_disabled(o, CB_NO_COMMAND | CB_NO_CONTROL, 0);
|
|
ret_x(c, 1);
|
|
}
|
|
// void nvnCommandBufferFinalize(NVNcommandBuffer *cmd): handles outlive their recorder
|
|
NVN_API(nvnCommandBufferFinalize) {
|
|
cmd_t *o = cmd_of(c);
|
|
pool_put(o->seg, (size_t)(o->end - o->seg));
|
|
o->seg = o->cur = o->end = NULL;
|
|
o->magic = 0;
|
|
cb_set_disabled(o, CB_NO_COMMAND | CB_NO_CONTROL, 0);
|
|
}
|
|
// void nvnCommandBufferSetMemoryCallback(NVNcommandBuffer *cmd, PFNNVNCOMMANDBUFFERMEMORYCALLBACKPROC callback)
|
|
NVN_API(nvnCommandBufferSetMemoryCallback) { cmd_of(c)->cb = arg_p(c, 1); }
|
|
// void nvnCommandBufferSetMemoryCallbackData(NVNcommandBuffer *cmd, void *data)
|
|
NVN_API(nvnCommandBufferSetMemoryCallbackData) { cmd_of(c)->cb_data = arg_p(c, 1); }
|
|
// PFNNVNCOMMANDBUFFERMEMORYCALLBACKPROC nvnCommandBufferGetMemoryCallback(const NVNcommandBuffer *cmd)
|
|
NVN_API(nvnCommandBufferGetMemoryCallback) { ret_x(c, (uintptr_t)cmd_of(c)->cb); }
|
|
// void *nvnCommandBufferGetMemoryCallbackData(const NVNcommandBuffer *cmd) (nvn.h declares it void)
|
|
NVN_API(nvnCommandBufferGetMemoryCallbackData) { ret_x(c, (uintptr_t)cmd_of(c)->cb_data); }
|
|
// void nvnCommandBufferSetCommandMemoryCallbackEnabled / SetControlMemoryCallbackEnabled(NVNcommandBuffer *cmd,
|
|
// NVNboolean enabled) (NVN 55, no prototype in nvn.h)
|
|
NVN_API(nvnCommandBufferSetCommandMemoryCallbackEnabled) { cb_set_disabled(cmd_of(c), CB_NO_COMMAND, !nvn_arg_bool(c, 1)); }
|
|
NVN_API(nvnCommandBufferSetControlMemoryCallbackEnabled) { cb_set_disabled(cmd_of(c), CB_NO_CONTROL, !nvn_arg_bool(c, 1)); }
|
|
// void nvnCommandBufferAddCommandMemory(NVNcommandBuffer *cmd, const NVNmemoryPool *pool, ptrdiff_t offset, size_t size):
|
|
// accounting only; command memory can be shared by independent recordings with distinct control memory
|
|
NVN_API(nvnCommandBufferAddCommandMemory) {
|
|
cmd_t *o = cmd_of(c);
|
|
const pool_t *p = arg_p(c, 1);
|
|
o->cmd_base = (uintptr_t)(p ? p->mem : NULL) + (uintptr_t)arg_x(c, 2);
|
|
o->game_size = arg_x(c, 3);
|
|
o->cmd_used = 0;
|
|
o->hw_used = 0;
|
|
}
|
|
// void nvnCommandBufferAddControlMemory(NVNcommandBuffer *cmd, void *memory, size_t size): the SDK encodes handles in
|
|
// control memory (sdk+0x444530), so reusing that storage retires the copies recorded there
|
|
NVN_API(nvnCommandBufferAddControlMemory) {
|
|
cmd_t *o = cmd_of(c);
|
|
void *m = arg_p(c, 1);
|
|
size_t n = arg_x(c, 2);
|
|
release_copies((uintptr_t)m, (uintptr_t)m + n);
|
|
o->ctl = m;
|
|
o->ctl_size = n;
|
|
o->ctl_used = 0;
|
|
}
|
|
// size_t nvnCommandBufferGet{Control,Command}Memory{Size,Used,Free}(const NVNcommandBuffer *cmd): usage is bounded by
|
|
// the memory the game gave, so the game never sees more used than it provided
|
|
static size_t min_size(size_t a, size_t b) { return a < b ? a : b; }
|
|
NVN_API(nvnCommandBufferGetControlMemorySize) { ret_x(c, cmd_of(c)->ctl_size); }
|
|
NVN_API(nvnCommandBufferGetControlMemoryUsed) { const cmd_t *o = cmd_of(c); ret_x(c, min_size(o->ctl_used, o->ctl_size)); }
|
|
NVN_API(nvnCommandBufferGetControlMemoryFree) { const cmd_t *o = cmd_of(c); ret_x(c, o->ctl_size - min_size(o->ctl_used, o->ctl_size)); }
|
|
NVN_API(nvnCommandBufferGetCommandMemorySize) { ret_x(c, cmd_of(c)->game_size); }
|
|
NVN_API(nvnCommandBufferGetCommandMemoryUsed) { const cmd_t *o = cmd_of(c); ret_x(c, min_size(o->hw_used, o->game_size)); }
|
|
NVN_API(nvnCommandBufferGetCommandMemoryFree) { const cmd_t *o = cmd_of(c); ret_x(c, o->game_size - min_size(o->hw_used, o->game_size)); }
|
|
|
|
// ---------- CommandBuffer: recording
|
|
// void nvnCommandBufferBeginRecording(NVNcommandBuffer *cmd)
|
|
NVN_API(nvnCommandBufferBeginRecording) {
|
|
cmd_t *o = cmd_of(c);
|
|
if (o->recording) rt_log("nvn: BeginRecording on command buffer %p while it records", (void *)o);
|
|
o->recording = 1;
|
|
o->cur = o->seg; // the previous recording was copied at EndRecording
|
|
o->rec_start = o->cmd_used;
|
|
put_hdr(cmd_alloc(o, 8), OP_BEGIN, 8);
|
|
o->hw_used -= 8; // BEGIN and END have no hardware equivalent
|
|
o->hw_start = o->hw_used;
|
|
if (!o->game_size) memory_event(c, o, NVN_COMMAND_BUFFER_MEMORY_EVENT_OUT_OF_COMMAND_MEMORY, MIN_COMMAND_SIZE);
|
|
}
|
|
// NVNcommandHandle nvnCommandBufferEndRecording(NVNcommandBuffer *cmd)
|
|
NVN_API(nvnCommandBufferEndRecording) {
|
|
cmd_t *o = cmd_of(c);
|
|
if (!o->recording || !o->seg) rt_fatal("nvn: EndRecording on command buffer %p, which does not record", (void *)o);
|
|
if (!o->ctl_size) memory_event(c, o, NVN_COMMAND_BUFFER_MEMORY_EVENT_OUT_OF_CONTROL_MEMORY, MIN_CONTROL_SIZE);
|
|
put_hdr(cmd_alloc(o, 8), OP_END, 8);
|
|
o->hw_used -= 8;
|
|
o->recording = 0;
|
|
uint8_t *copy = flatten_recording(o->seg, (size_t)(o->cur - o->seg));
|
|
// Finalizing the recorder does not retire its handle; only reusing the control memory it was recorded in does.
|
|
uintptr_t lo = (uintptr_t)o->ctl + (o->ctl_used < o->ctl_size ? o->ctl_used : 0);
|
|
uintptr_t hi = o->ctl ? lo + 1 : 0;
|
|
o->ctl_used += CTL_PER_RECORDING;
|
|
copy_add(lo, hi, copy);
|
|
ret_x(c, (uintptr_t)copy);
|
|
}
|
|
// NVNboolean nvnCommandBufferIsRecording(const NVNcommandBuffer *cmd)
|
|
NVN_API(nvnCommandBufferIsRecording) { ret_x(c, cmd_of(c)->recording != 0); }
|
|
|
|
// The called recording is copied into the caller at the call position: the game recycles command memory in small steps,
|
|
// so a handle may be gone by the time the caller is submitted. Deferred binds stay deferred. A copy is OP_BEGIN, its
|
|
// records and OP_END (flatten_recording checked them), so the records go over as one block; usage grows by the same
|
|
// bytes as record by record.
|
|
static void inline_recording(cmd_t *o, uint64_t handle, const char *what) {
|
|
if (!handle) rt_fatal("nvn: %s with a null handle", what);
|
|
uint64_t bytes = copy_touch(handle, __atomic_load_n(&nsubmits, __ATOMIC_RELAXED), what);
|
|
if (bytes <= 16) return;
|
|
size_t body = (size_t)bytes - 16;
|
|
cmd_reserve(o, body);
|
|
memcpy(o->cur, (const uint8_t *)(uintptr_t)handle + 8, body);
|
|
o->cur += body;
|
|
o->cmd_used += body;
|
|
o->hw_used += body;
|
|
o->last_rec = 0;
|
|
}
|
|
// void nvnCommandBuffer{Call,Copy}Commands(NVNcommandBuffer *cmd, int n, const NVNcommandHandle *handles)
|
|
NVN_API(nvnCommandBufferCallCommands) {
|
|
cmd_t *o = cmd_of(c);
|
|
int n = (int)arg_x(c, 1);
|
|
const uint64_t *h = arg_p(c, 2);
|
|
for (int i = 0; i < n; i++) inline_recording(o, h[i], "CallCommands");
|
|
}
|
|
NVN_API(nvnCommandBufferCopyCommands) {
|
|
cmd_t *o = cmd_of(c);
|
|
int n = (int)arg_x(c, 1);
|
|
const uint64_t *h = arg_p(c, 2);
|
|
for (int i = 0; i < n; i++) inline_recording(o, h[i], "CopyCommands");
|
|
}
|
|
|
|
// ---------- CommandBuffer: commands with CPU-visible results (performed by the back end in GPU order)
|
|
// void nvnCommandBufferSignalEvent(NVNcommandBuffer *cmd, const NVNevent *event, NVNeventSignalMode mode,
|
|
// NVNeventSignalLocation location, int flags, uint32_t value)
|
|
NVN_API(nvnCommandBufferSignalEvent) {
|
|
const ev_t *e = arg_p(c, 1);
|
|
*REC(cmd_of(c), OP_SIGNAL_EVENT, p_signal_event) = (p_signal_event){e->val, (uint32_t)arg_x(c, 2), (uint32_t)arg_x(c, 5)};
|
|
}
|
|
// void nvnCommandBufferResetCounter(NVNcommandBuffer *cmd, NVNcounterType type)
|
|
NVN_API(nvnCommandBufferResetCounter) { REC(cmd_of(c), OP_RESET_COUNTER, p_reset_counter)->type = (uint32_t)arg_x(c, 1); }
|
|
// void nvnCommandBufferReportCounter(NVNcommandBuffer *cmd, NVNcounterType type, NVNbufferAddress address)
|
|
NVN_API(nvnCommandBufferReportCounter) {
|
|
*REC(cmd_of(c), OP_REPORT_COUNTER, p_report_counter) = (p_report_counter){(uint32_t)arg_x(c, 1), arg_x(c, 2)};
|
|
}
|
|
// void nvnCommandBufferCopyBufferToBuffer(NVNcommandBuffer *cmd, NVNbufferAddress src, NVNbufferAddress dst,
|
|
// size_t size, int flags)
|
|
NVN_API(nvnCommandBufferCopyBufferToBuffer) {
|
|
*REC(cmd_of(c), OP_COPY_B2B, p_copy_b2b) = (p_copy_b2b){arg_x(c, 1), arg_x(c, 2), arg_x(c, 3)};
|
|
}
|
|
// void nvnCommandBufferUpdateUniformBuffer(NVNcommandBuffer *cmd, NVNbufferAddress buffer, size_t buffer_size,
|
|
// ptrdiff_t offset, size_t size, const void *data): the data is copied at record time (the caller may change it
|
|
// afterwards) and written into the buffer at replay
|
|
NVN_API(nvnCommandBufferUpdateUniformBuffer) {
|
|
uint64_t dst = arg_x(c, 1) + arg_x(c, 3), n = arg_x(c, 4);
|
|
const void *data = arg_p(c, 5);
|
|
if (n > (16u << 20)) rt_fatal("nvn: UpdateUniformBuffer of %#llx bytes", (unsigned long long)n);
|
|
p_update_ub *r = rec(cmd_of(c), OP_UPDATE_UB, offsetof(p_update_ub, data) + n);
|
|
r->dst = dst;
|
|
r->size = n;
|
|
memcpy(r->data, data, n);
|
|
}
|
|
|
|
// ---------- CommandBuffer: render state, payloads of nvn_obj.h
|
|
// void nvnCommandBufferSetRenderTargets(NVNcommandBuffer *cmd, int n, const NVNtexture *const *colors,
|
|
// const NVNtextureView *const *color_views, const NVNtexture *depth, const NVNtextureView *depth_view)
|
|
NVN_API(nvnCommandBufferSetRenderTargets) {
|
|
int n = (int)arg_x(c, 1);
|
|
const NVNtexture *const *ct = arg_p(c, 2);
|
|
const NVNtextureView *const *cv = arg_p(c, 3);
|
|
const NVNtexture *dt = arg_p(c, 4);
|
|
const NVNtextureView *dv = arg_p(c, 5);
|
|
if (n < 0 || n > 8) {
|
|
rt_log("nvn: SetRenderTargets with %d colour targets", n);
|
|
n = n < 0 ? 0 : 8;
|
|
}
|
|
p_set_rt *r = REC(cmd_of(c), OP_SET_RT, p_set_rt);
|
|
r->n = n;
|
|
for (int i = 0; i < n; i++) {
|
|
r->color[i] = ct ? tex_pin(ct[i]) : NULL;
|
|
copy_view(&r->cview[i], &r->has_cview[i], cv ? cv[i] : NULL);
|
|
}
|
|
r->depth = tex_pin(dt);
|
|
copy_view(&r->dview, &r->has_dview, dv);
|
|
}
|
|
// void nvnCommandBufferClearColor(NVNcommandBuffer *cmd, int index, const float *rgba, int mask)
|
|
NVN_API(nvnCommandBufferClearColor) {
|
|
p_clear_color *r = REC(cmd_of(c), OP_CLEAR_COLOR, p_clear_color);
|
|
r->index = (int)arg_x(c, 1);
|
|
r->mask = (int)arg_x(c, 3);
|
|
memcpy(r->rgba, arg_p(c, 2), 16);
|
|
}
|
|
// void nvnCommandBufferClearDepthStencil(NVNcommandBuffer *cmd, float depth, NVNboolean depth_mask, int stencil,
|
|
// int stencil_mask)
|
|
NVN_API(nvnCommandBufferClearDepthStencil) {
|
|
*REC(cmd_of(c), OP_CLEAR_DS, p_clear_ds) = (p_clear_ds){arg_s(c, 0), nvn_arg_bool(c, 1), (int)arg_x(c, 2), (int)arg_x(c, 3)};
|
|
}
|
|
static void set_viewports(cmd_t *o, int first, int n, const float *v) {
|
|
if (first < 0 || n < 0 || first + n > 16) {
|
|
rt_log("nvn: SetViewports first %d n %d", first, n);
|
|
return;
|
|
}
|
|
p_viewports *r = REC(o, OP_VIEWPORTS, p_viewports);
|
|
r->first = first;
|
|
r->n = n;
|
|
for (int i = 0; i < n; i++) memcpy(r->v[i], v + 4 * i, 16);
|
|
}
|
|
// void nvnCommandBufferSetViewports(NVNcommandBuffer *cmd, int first, int n, const float *xywh)
|
|
NVN_API(nvnCommandBufferSetViewports) { set_viewports(cmd_of(c), (int)arg_x(c, 1), (int)arg_x(c, 2), arg_p(c, 3)); }
|
|
// void nvnCommandBufferSetViewport(NVNcommandBuffer *cmd, int x, int y, int w, int h)
|
|
NVN_API(nvnCommandBufferSetViewport) {
|
|
float v[4] = {(float)(int)arg_x(c, 1), (float)(int)arg_x(c, 2), (float)(int)arg_x(c, 3), (float)(int)arg_x(c, 4)};
|
|
set_viewports(cmd_of(c), 0, 1, v);
|
|
}
|
|
static void set_scissor(cmd_t *o, int x, int y, int w, int h) { *REC(o, OP_SCISSOR, p_scissor) = (p_scissor){x, y, w, h}; }
|
|
// void nvnCommandBufferSetScissor(NVNcommandBuffer *cmd, int x, int y, int w, int h)
|
|
NVN_API(nvnCommandBufferSetScissor) {
|
|
set_scissor(cmd_of(c), (int)arg_x(c, 1), (int)arg_x(c, 2), (int)arg_x(c, 3), (int)arg_x(c, 4));
|
|
}
|
|
// void nvnCommandBufferSetScissors(NVNcommandBuffer *cmd, int first, int n, const int *xywh): the replay keeps one
|
|
// scissor (index 0); other indices matter only to shaders that select a viewport index, so differing ones are reported
|
|
NVN_API(nvnCommandBufferSetScissors) {
|
|
int first = (int)arg_x(c, 1), n = (int)arg_x(c, 2);
|
|
const int *r = arg_p(c, 3);
|
|
if (first < 0 || n < 0 || first + n > 16) rt_fatal("nvn: SetScissors first %d n %d", first, n);
|
|
for (int i = 1; i < n; i++)
|
|
if (memcmp(r + 4 * i, r, 16)) {
|
|
static int once;
|
|
if (!once++) rt_log("nvn: SetScissors: differing scissor %d ignored (single-scissor replay)", first + i);
|
|
}
|
|
if (first == 0 && n > 0) set_scissor(cmd_of(c), r[0], r[1], r[2], r[3]);
|
|
}
|
|
static void set_depth_range(cmd_t *o, float n, float f) { *REC(o, OP_DEPTH_RANGE, p_depth_range) = (p_depth_range){n, f}; }
|
|
// void nvnCommandBufferSetDepthRange(NVNcommandBuffer *cmd, float near, float far)
|
|
NVN_API(nvnCommandBufferSetDepthRange) { set_depth_range(cmd_of(c), arg_s(c, 0), arg_s(c, 1)); }
|
|
// void nvnCommandBufferSetDepthRanges(NVNcommandBuffer *cmd, int first, int n, const float *ranges)
|
|
NVN_API(nvnCommandBufferSetDepthRanges) {
|
|
int first = (int)arg_x(c, 1), n = (int)arg_x(c, 2);
|
|
const float *r = arg_p(c, 3);
|
|
if (first < 0 || n < 0 || first + n > 16) rt_fatal("nvn: SetDepthRanges first %d n %d", first, n);
|
|
for (int i = 1; i < n; i++)
|
|
if (memcmp(r + 2 * i, r, 8)) {
|
|
static int once;
|
|
if (!once++) rt_log("nvn: SetDepthRanges: differing range %d ignored (single-range replay)", first + i);
|
|
}
|
|
if (first == 0 && n > 0) set_depth_range(cmd_of(c), r[0], r[1]);
|
|
}
|
|
// void nvnCommandBufferSetDepthClamp / SetRenderEnable / SetRasterizerDiscard(NVNcommandBuffer *cmd, NVNboolean enable)
|
|
NVN_API(nvnCommandBufferSetDepthClamp) { REC(cmd_of(c), OP_DEPTH_CLAMP, p_flag)->value = nvn_arg_bool(c, 1) != 0; }
|
|
NVN_API(nvnCommandBufferSetRenderEnable) { REC(cmd_of(c), OP_RENDER_ENABLE, p_flag)->value = nvn_arg_bool(c, 1) != 0; }
|
|
// Rasterizer discard runs vertex work without rasterising; with no transform feedback that equals not drawing.
|
|
NVN_API(nvnCommandBufferSetRasterizerDiscard) { REC(cmd_of(c), OP_RASTER_DISCARD, p_flag)->value = nvn_arg_bool(c, 1) != 0; }
|
|
// void nvnCommandBufferSetPrimitiveRestart(NVNcommandBuffer *cmd, NVNboolean enable, int index): TOTK draws its
|
|
// terrain and cliff strips with 16-bit indices and restart at 0xffff; the replay applies it per indexed draw (op_draw_idx)
|
|
NVN_API(nvnCommandBufferSetPrimitiveRestart) {
|
|
*REC(cmd_of(c), OP_PRIM_RESTART, p_prim_restart) = (p_prim_restart){nvn_arg_bool(c, 1) != 0, (uint32_t)arg_x(c, 2)};
|
|
}
|
|
// void nvnCommandBufferSetRenderEnableConditional(NVNcommandBuffer *cmd, NVNconditionalRenderMode mode,
|
|
// NVNbufferAddress counters): rendering is enabled when the two NVNcounterData.counter values at addr / addr+16
|
|
// compare equal (mode 0) or not equal (mode 1). TOTK gates most of its ClearColor commands with it; the replay
|
|
// evaluates it in GPU order.
|
|
NVN_API(nvnCommandBufferSetRenderEnableConditional) {
|
|
int m = (int)arg_x(c, 1);
|
|
if (m != 0 && m != 1) rt_fatal("nvn: SetRenderEnableConditional: unknown mode %d", m);
|
|
p_render_cond *r = REC(cmd_of(c), OP_RENDER_COND, p_render_cond);
|
|
r->mode = m;
|
|
r->addr = arg_x(c, 2);
|
|
}
|
|
// void nvnCommandBufferBindProgram(NVNcommandBuffer *cmd, const NVNprogram *program, int stages)
|
|
NVN_API(nvnCommandBufferBindProgram) {
|
|
*REC(cmd_of(c), OP_BIND_PROGRAM, p_bind_program) = (p_bind_program){arg_p(c, 1), (int)arg_x(c, 2)};
|
|
}
|
|
// void nvnCommandBufferBind{Blend,Color,ChannelMask,Multisample,Polygon,DepthStencil}State(NVNcommandBuffer *cmd,
|
|
// const NVN*State *state): the state object's bytes are copied (p_bind_state.kind order)
|
|
static void bind_state(const Cpu *c, int kind, size_t n) {
|
|
p_bind_state *r = REC(cmd_of(c), OP_BIND_STATE, p_bind_state);
|
|
r->kind = kind;
|
|
memcpy(r->bytes, arg_p(c, 1), n);
|
|
}
|
|
NVN_API(nvnCommandBufferBindBlendState) { bind_state(c, 0, sizeof(blend_st)); }
|
|
NVN_API(nvnCommandBufferBindColorState) { bind_state(c, 1, sizeof(color_st)); }
|
|
NVN_API(nvnCommandBufferBindChannelMaskState) { bind_state(c, 2, sizeof(chmask_st)); }
|
|
NVN_API(nvnCommandBufferBindMultisampleState) { bind_state(c, 3, sizeof(ms_st)); }
|
|
NVN_API(nvnCommandBufferBindPolygonState) { bind_state(c, 4, sizeof(poly_st)); }
|
|
NVN_API(nvnCommandBufferBindDepthStencilState) { bind_state(c, 5, sizeof(ds_st)); }
|
|
// void nvnCommandBufferSetPointSize(NVNcommandBuffer *cmd, float size): recorded as bind-state kind 6 with the float as payload
|
|
NVN_API(nvnCommandBufferSetPointSize) {
|
|
p_bind_state *r = REC(cmd_of(c), OP_BIND_STATE, p_bind_state);
|
|
float s = arg_s(c, 0);
|
|
r->kind = 6;
|
|
memcpy(r->bytes, &s, sizeof s);
|
|
}
|
|
// void nvnCommandBufferBindVertexAttribState(NVNcommandBuffer *cmd, int n, const NVNvertexAttribState *states)
|
|
NVN_API(nvnCommandBufferBindVertexAttribState) {
|
|
cmd_t *o = cmd_of(c);
|
|
int n = (int)arg_x(c, 1);
|
|
if (n < 0 || n > 16) { rt_log("nvn: BindVertexAttribState n %d", n); return; }
|
|
p_bind_vattr *r = RECN(o, OP_BIND_VATTR, p_bind_vattr, st, n);
|
|
r->first = 0;
|
|
r->n = n;
|
|
memcpy(r->st, arg_p(c, 2), 4 * (size_t)n);
|
|
hw_fix(o, 68); // the driver always writes all 16 attribute formats: 17 words
|
|
}
|
|
// void nvnCommandBufferBindVertexStreamState(NVNcommandBuffer *cmd, int n, const NVNvertexStreamState *states)
|
|
NVN_API(nvnCommandBufferBindVertexStreamState) {
|
|
cmd_t *o = cmd_of(c);
|
|
int n = (int)arg_x(c, 1);
|
|
if (n < 0 || n > 16) { rt_log("nvn: BindVertexStreamState n %d", n); return; }
|
|
p_bind_vstream *r = RECN(o, OP_BIND_VSTREAM, p_bind_vstream, st, n);
|
|
r->first = 0;
|
|
r->n = n;
|
|
memcpy(r->st, arg_p(c, 2), 8 * (size_t)n);
|
|
hw_fix(o, 4 * (16 + (size_t)n));
|
|
}
|
|
static void bind_vertex_buffers(cmd_t *o, int first, int n, const NVNbufferRange *rg) {
|
|
if (first < 0 || n < 0 || first + n > 16) { rt_log("nvn: BindVertexBuffers first %d n %d", first, n); return; }
|
|
p_bind_vbuf *r = RECN(o, OP_BIND_VBUF, p_bind_vbuf, r, n);
|
|
r->first = first;
|
|
r->n = n;
|
|
memcpy(r->r, rg, sizeof *rg * (size_t)n);
|
|
hw_fix(o, 24 * (size_t)n);
|
|
}
|
|
// void nvnCommandBufferBindVertexBuffers(NVNcommandBuffer *cmd, int first, int n, const NVNbufferRange *ranges)
|
|
NVN_API(nvnCommandBufferBindVertexBuffers) { bind_vertex_buffers(cmd_of(c), (int)arg_x(c, 1), (int)arg_x(c, 2), arg_p(c, 3)); }
|
|
// void nvnCommandBufferBindVertexBuffer(NVNcommandBuffer *cmd, int index, NVNbufferAddress address, size_t size)
|
|
NVN_API(nvnCommandBufferBindVertexBuffer) {
|
|
NVNbufferRange r = {arg_x(c, 2), arg_x(c, 3)};
|
|
bind_vertex_buffers(cmd_of(c), (int)arg_x(c, 1), 1, &r);
|
|
}
|
|
// Uniform and storage buffers: one record per binding (CB_SIZE/ADDRESS 4 words + CB_BIND 1 on hardware).
|
|
static void bind_buffers(const Cpu *c, uint32_t op) { // (cmd, stage, first, n, const NVNbufferRange *ranges)
|
|
cmd_t *o = cmd_of(c);
|
|
int st = (int)arg_x(c, 1), first = (int)arg_x(c, 2), n = (int)arg_x(c, 3);
|
|
const NVNbufferRange *rg = arg_p(c, 4);
|
|
for (int i = 0; i < n; i++) {
|
|
*REC(o, op, p_bind_buf) = (p_bind_buf){st, first + i, rg[i].address, rg[i].size};
|
|
hw_fix(o, 20);
|
|
}
|
|
}
|
|
static void bind_buffer(const Cpu *c, uint32_t op) { // (cmd, stage, index, NVNbufferAddress address, size_t size)
|
|
cmd_t *o = cmd_of(c);
|
|
*REC(o, op, p_bind_buf) = (p_bind_buf){(int)arg_x(c, 1), (int)arg_x(c, 2), arg_x(c, 3), arg_x(c, 4)};
|
|
hw_fix(o, 20);
|
|
}
|
|
NVN_API(nvnCommandBufferBindUniformBuffer) { bind_buffer(c, OP_BIND_UBO); }
|
|
NVN_API(nvnCommandBufferBindUniformBuffers) { bind_buffers(c, OP_BIND_UBO); }
|
|
NVN_API(nvnCommandBufferBindStorageBuffer) { bind_buffer(c, OP_BIND_SSBO); }
|
|
NVN_API(nvnCommandBufferBindStorageBuffers) { bind_buffers(c, OP_BIND_SSBO); }
|
|
// Texture, image and separate handle binds. A deferred bind (NVN 55 *Deferred entry points, the fallback of the inline
|
|
// fast paths) keeps the pointer to the game's handle array: the original driver (sdk+0x46cc00 BindTexturesDeferred)
|
|
// reads the handles when the command buffer is submitted, and the game fills its material handle tables after
|
|
// recording and before the submit (nvn_submit_prepare). hw_bytes: the hardware size of the plain variant.
|
|
static void bind_handles(cmd_t *o, uint32_t op, int st, int first, int n, const uint64_t *h, int deferred, size_t hw_bytes) {
|
|
if (n < 0 || n > 32) { rt_log("nvn: bind of %d handles", n); return; }
|
|
p_bind_handles *r = RECN(o, op, p_bind_handles, h, n);
|
|
r->stage = st;
|
|
r->first = first;
|
|
r->n = n;
|
|
r->src = deferred ? (uint64_t)(uintptr_t)h : 0;
|
|
memcpy(r->h, h, 8 * (size_t)n);
|
|
hw_fix(o, hw_bytes ? hw_bytes : 16 + 4 * (size_t)n); // macro 7 + method 0x1a2c + CB_POS + one word per binding
|
|
}
|
|
static void bind_handle_array(const Cpu *c, uint32_t op, int deferred, int wide) { // (cmd, stage, first, n, handles)
|
|
int n = (int)arg_x(c, 3);
|
|
bind_handles(cmd_of(c), op, (int)arg_x(c, 1), (int)arg_x(c, 2), n, arg_p(c, 4), deferred,
|
|
wide && n >= 0 && n <= 32 ? 16 + 8 * (size_t)n : 0);
|
|
}
|
|
static void bind_one_handle(const Cpu *c, uint32_t op, size_t hw_bytes) { // (cmd, stage, index, handle)
|
|
uint64_t v = arg_x(c, 3);
|
|
bind_handles(cmd_of(c), op, (int)arg_x(c, 1), (int)arg_x(c, 2), 1, &v, 0, hw_bytes);
|
|
}
|
|
// Combined texture handles share the texture slots with separate textures (sampler id in bits 20 and up). Images
|
|
// (writable textures): handle = texture pool index; the game does not call RegisterImage, images and textures share
|
|
// their registration.
|
|
NVN_API(nvnCommandBufferBindTexture) { bind_one_handle(c, OP_BIND_TEX, 0); }
|
|
NVN_API(nvnCommandBufferBindTextures) { bind_handle_array(c, OP_BIND_TEX, 0, 0); }
|
|
NVN_API(nvnCommandBufferBindImage) { bind_one_handle(c, OP_BIND_IMG, 24); }
|
|
NVN_API(nvnCommandBufferBindImages) { bind_handle_array(c, OP_BIND_IMG, 0, 1); }
|
|
NVN_API(nvnCommandBufferBindSeparateTexture) { bind_one_handle(c, OP_BIND_TEX, 24); }
|
|
NVN_API(nvnCommandBufferBindSeparateTextures) { bind_handle_array(c, OP_BIND_TEX, 0, 1); }
|
|
NVN_API(nvnCommandBufferBindSeparateSampler) { bind_one_handle(c, OP_BIND_SAMP, 24); }
|
|
NVN_API(nvnCommandBufferBindSeparateSamplers) { bind_handle_array(c, OP_BIND_SAMP, 0, 1); }
|
|
// void nvnCommandBufferBind{Textures,Images,SeparateTextures,SeparateSamplers}Deferred(NVNcommandBuffer *cmd,
|
|
// NVNshaderStage stage, int first, int n, const T *handles) (NVN 55, no prototype in nvn.h)
|
|
NVN_API(nvnCommandBufferBindTexturesDeferred) { bind_handle_array(c, OP_BIND_TEX, 1, 0); }
|
|
NVN_API(nvnCommandBufferBindImagesDeferred) { bind_handle_array(c, OP_BIND_IMG, 1, 1); }
|
|
NVN_API(nvnCommandBufferBindSeparateTexturesDeferred) { bind_handle_array(c, OP_BIND_TEX, 1, 1); }
|
|
NVN_API(nvnCommandBufferBindSeparateSamplersDeferred) { bind_handle_array(c, OP_BIND_SAMP, 1, 1); }
|
|
// void nvnCommandBufferBind{Uniform,Storage}BuffersDeferred(NVNcommandBuffer *cmd, NVNshaderStage stage, int first,
|
|
// int n, const NVNbufferRange *ranges): buffer ranges still bind at record time
|
|
NVN_API(nvnCommandBufferBindUniformBuffersDeferred) {
|
|
static int once;
|
|
if (!once++) NVN_LOG("BindUniformBuffersDeferred: buffer ranges bind at record time");
|
|
bind_buffers(c, OP_BIND_UBO);
|
|
}
|
|
NVN_API(nvnCommandBufferBindStorageBuffersDeferred) {
|
|
static int once;
|
|
if (!once++) NVN_LOG("BindStorageBuffersDeferred: buffer ranges bind at record time");
|
|
bind_buffers(c, OP_BIND_SSBO);
|
|
}
|
|
// void nvnCommandBufferSetStencil{ValueMask,Mask,Ref}(NVNcommandBuffer *cmd, NVNface face, int value)
|
|
NVN_API(nvnCommandBufferSetStencilValueMask) {
|
|
*REC(cmd_of(c), OP_STENCIL, p_stencil) = (p_stencil){(int)arg_x(c, 1), (int)arg_x(c, 2), 0, 0, 0};
|
|
}
|
|
NVN_API(nvnCommandBufferSetStencilMask) {
|
|
*REC(cmd_of(c), OP_STENCIL, p_stencil) = (p_stencil){(int)arg_x(c, 1), 0, (int)arg_x(c, 2), 0, 1};
|
|
}
|
|
NVN_API(nvnCommandBufferSetStencilRef) {
|
|
*REC(cmd_of(c), OP_STENCIL, p_stencil) = (p_stencil){(int)arg_x(c, 1), 0, 0, (int)arg_x(c, 2), 2};
|
|
}
|
|
// void nvnCommandBufferSetPolygonOffsetClamp(NVNcommandBuffer *cmd, float factor, float units, float clamp)
|
|
NVN_API(nvnCommandBufferSetPolygonOffsetClamp) {
|
|
*REC(cmd_of(c), OP_POLY_OFFSET, p_poly_offset) = (p_poly_offset){arg_s(c, 0), arg_s(c, 1), arg_s(c, 2)};
|
|
}
|
|
// void nvnCommandBufferDrawArrays(NVNcommandBuffer *cmd, NVNdrawPrimitive prim, int first, int count)
|
|
NVN_API(nvnCommandBufferDrawArrays) {
|
|
*REC(cmd_of(c), OP_DRAW, p_draw) = (p_draw){(int)arg_x(c, 1), (int)arg_x(c, 2), (int)arg_x(c, 3), 0, 1};
|
|
}
|
|
// void nvnCommandBufferDrawArraysInstanced(NVNcommandBuffer *cmd, NVNdrawPrimitive prim, int first, int count,
|
|
// int base_instance, int instances): zero instances draw nothing
|
|
NVN_API(nvnCommandBufferDrawArraysInstanced) {
|
|
int inst = (int)arg_x(c, 5);
|
|
if (inst <= 0) return;
|
|
*REC(cmd_of(c), OP_DRAW, p_draw) = (p_draw){(int)arg_x(c, 1), (int)arg_x(c, 2), (int)arg_x(c, 3), (int)arg_x(c, 4), inst};
|
|
}
|
|
// void nvnCommandBufferDrawElements(NVNcommandBuffer *cmd, NVNdrawPrimitive prim, NVNindexType type, int count,
|
|
// NVNbufferAddress indices)
|
|
NVN_API(nvnCommandBufferDrawElements) {
|
|
cmd_t *o = cmd_of(c);
|
|
*REC(o, OP_DRAW_IDX, p_draw_idx) = (p_draw_idx){(int)arg_x(c, 1), (int)arg_x(c, 2), (int)arg_x(c, 3), 0, 0, 1, arg_x(c, 4)};
|
|
hw_fix(o, 48);
|
|
}
|
|
// void nvnCommandBufferDrawElementsBaseVertex(..., NVNbufferAddress indices, int base_vertex)
|
|
NVN_API(nvnCommandBufferDrawElementsBaseVertex) {
|
|
cmd_t *o = cmd_of(c);
|
|
*REC(o, OP_DRAW_IDX, p_draw_idx) =
|
|
(p_draw_idx){(int)arg_x(c, 1), (int)arg_x(c, 2), (int)arg_x(c, 3), (int)arg_x(c, 5), 0, 1, arg_x(c, 4)};
|
|
hw_fix(o, 44);
|
|
}
|
|
// void nvnCommandBufferDrawElementsInstanced(..., NVNbufferAddress indices, int base_vertex, int base_instance,
|
|
// int instances): assumed to use the 6-parameter draw macro of the base-vertex form (unverified)
|
|
NVN_API(nvnCommandBufferDrawElementsInstanced) {
|
|
cmd_t *o = cmd_of(c);
|
|
*REC(o, OP_DRAW_IDX, p_draw_idx) = (p_draw_idx){(int)arg_x(c, 1), (int)arg_x(c, 2), (int)arg_x(c, 3),
|
|
(int)arg_x(c, 5), (int)arg_x(c, 6), (int)arg_x(c, 7), arg_x(c, 4)};
|
|
hw_fix(o, 44);
|
|
}
|
|
// void nvnCommandBufferDrawElementsIndirect(NVNcommandBuffer *cmd, NVNdrawPrimitive prim, NVNindexType type,
|
|
// NVNbufferAddress indices, NVNbufferAddress indirect): the arguments at indirect are often GPU-written, so the replay
|
|
// draws with vkCmdDrawIndexedIndirect from the pool buffer (nxnative hle_nvn.c, SavageWindRecomp cmdbuf.c)
|
|
NVN_API(nvnCommandBufferDrawElementsIndirect) {
|
|
*REC(cmd_of(c), OP_DRAW_IDX_IND, p_draw_idx_ind) = (p_draw_idx_ind){(int)arg_x(c, 1), (int)arg_x(c, 2), arg_x(c, 3), arg_x(c, 4)};
|
|
}
|
|
// void nvnCommandBufferDispatchCompute(NVNcommandBuffer *cmd, int x, int y, int z): caller = the game's return address
|
|
NVN_API(nvnCommandBufferDispatchCompute) {
|
|
*REC(cmd_of(c), OP_DISPATCH, p_dispatch) = (p_dispatch){(int)arg_x(c, 1), (int)arg_x(c, 2), (int)arg_x(c, 3), c->x[30]};
|
|
}
|
|
// void nvnCommandBufferDispatchComputeIndirect(NVNcommandBuffer *cmd, NVNbufferAddress indirect): vkCmdDispatchIndirect
|
|
NVN_API(nvnCommandBufferDispatchComputeIndirect) {
|
|
*REC(cmd_of(c), OP_DISPATCH_IND, p_dispatch_ind) = (p_dispatch_ind){arg_x(c, 1), c->x[30]};
|
|
}
|
|
// void nvnCommandBufferClearBuffer(NVNcommandBuffer *cmd, NVNbufferAddress addr, size_t size, uint32_t value)
|
|
NVN_API(nvnCommandBufferClearBuffer) {
|
|
*REC(cmd_of(c), OP_CLEAR_BUF, p_clear_buf) = (p_clear_buf){arg_x(c, 1), arg_x(c, 2), (uint32_t)arg_x(c, 3), 0};
|
|
}
|
|
// void nvnCommandBufferReportValue(NVNcommandBuffer *cmd, uint32_t value, NVNbufferAddress addr): the value lands when
|
|
// the GPU reaches this point. Recorded as a 4-byte ClearBuffer, so it stays in GPU order with the ClearBuffer resets of
|
|
// the same counter (main+0x7b9274 modes 1-3) and works in CPU-invisible pools; the CPU sees it after the fence, as a
|
|
// counter report.
|
|
NVN_API(nvnCommandBufferReportValue) {
|
|
*REC(cmd_of(c), OP_CLEAR_BUF, p_clear_buf) = (p_clear_buf){arg_x(c, 2), 4, (uint32_t)arg_x(c, 1), 0};
|
|
}
|
|
// Colour reduction is a Maxwell bandwidth hint for blending into compressed targets: no effect on the result.
|
|
// void nvnCommandBufferSetColorReductionEnable(cmd, NVNboolean enable), SetColorReductionThresholds(cmd, class, const int *)
|
|
NVN_API(nvnCommandBufferSetColorReductionEnable) { (void)c; }
|
|
NVN_API(nvnCommandBufferSetColorReductionThresholds) { (void)c; }
|
|
// void nvnCommandBufferBarrier(NVNcommandBuffer *cmd, int bits)
|
|
NVN_API(nvnCommandBufferBarrier) { REC(cmd_of(c), OP_BARRIER, p_barrier)->bits = (int)arg_x(c, 1); }
|
|
// void nvnCommandBufferSetTexturePool / SetSamplerPool(NVNcommandBuffer *cmd, const NVNtexturePool / NVNsamplerPool *)
|
|
NVN_API(nvnCommandBufferSetTexturePool) { REC(cmd_of(c), OP_SET_TEX_POOL, p_set_pool)->pool = arg_p(c, 1); }
|
|
NVN_API(nvnCommandBufferSetSamplerPool) { REC(cmd_of(c), OP_SET_SAMP_POOL, p_set_pool)->pool = arg_p(c, 1); }
|
|
// void nvnCommandBufferCopyBufferToTexture(NVNcommandBuffer *cmd, NVNbufferAddress src, const NVNtexture *dst,
|
|
// const NVNtextureView *view, const NVNcopyRegion *region, int flags)
|
|
NVN_API(nvnCommandBufferCopyBufferToTexture) {
|
|
p_copy_b2t *o = REC(cmd_of(c), OP_COPY_B2T, p_copy_b2t);
|
|
o->src = arg_x(c, 1);
|
|
o->tex = tex_pin(arg_p(c, 2));
|
|
copy_view(&o->view, &o->has_view, arg_p(c, 3));
|
|
o->r = *(const NVNcopyRegion *)arg_p(c, 4);
|
|
o->flags = (int)arg_x(c, 5);
|
|
}
|
|
// void nvnCommandBufferCopyTextureToBuffer(NVNcommandBuffer *cmd, const NVNtexture *src, const NVNtextureView *view,
|
|
// const NVNcopyRegion *region, NVNbufferAddress dst, int flags): p_copy_b2t with src = the destination address
|
|
NVN_API(nvnCommandBufferCopyTextureToBuffer) {
|
|
p_copy_b2t *o = REC(cmd_of(c), OP_COPY_T2B, p_copy_b2t);
|
|
o->src = arg_x(c, 4);
|
|
o->tex = tex_pin(arg_p(c, 1));
|
|
copy_view(&o->view, &o->has_view, arg_p(c, 2));
|
|
o->r = *(const NVNcopyRegion *)arg_p(c, 3);
|
|
o->flags = (int)arg_x(c, 5);
|
|
}
|
|
// void nvnCommandBufferClearTexture{,ui}(NVNcommandBuffer *cmd, const NVNtexture *tex, const NVNtextureView *view,
|
|
// const NVNcopyRegion *region, const float / uint32_t *rgba, int mask)
|
|
static void clear_texture(const Cpu *c, int is_uint) {
|
|
p_clear_tex *o = REC(cmd_of(c), OP_CLEAR_TEX, p_clear_tex);
|
|
o->tex = tex_pin(arg_p(c, 1));
|
|
copy_view(&o->view, &o->has_view, arg_p(c, 2));
|
|
o->r = *(const NVNcopyRegion *)arg_p(c, 3);
|
|
o->mask = (int)arg_x(c, 5);
|
|
o->is_uint = (uint8_t)is_uint;
|
|
memcpy(o->c.u, arg_p(c, 4), 16);
|
|
}
|
|
NVN_API(nvnCommandBufferClearTexture) { clear_texture(c, 0); }
|
|
NVN_API(nvnCommandBufferClearTextureui) { clear_texture(c, 1); }
|
|
// void nvnCommandBufferCopyTextureToTexture(NVNcommandBuffer *cmd, const NVNtexture *src, const NVNtextureView *src_view,
|
|
// const NVNcopyRegion *src_region, const NVNtexture *dst, const NVNtextureView *dst_view,
|
|
// const NVNcopyRegion *dst_region, int flags)
|
|
NVN_API(nvnCommandBufferCopyTextureToTexture) {
|
|
p_copy_t2t *o = REC(cmd_of(c), OP_COPY_T2T, p_copy_t2t);
|
|
o->src = tex_pin(arg_p(c, 1));
|
|
o->dst = tex_pin(arg_p(c, 4));
|
|
copy_view(&o->sv, &o->has_sv, arg_p(c, 2));
|
|
copy_view(&o->dv, &o->has_dv, arg_p(c, 5));
|
|
o->sr = *(const NVNcopyRegion *)arg_p(c, 3);
|
|
o->dr = *(const NVNcopyRegion *)arg_p(c, 6);
|
|
o->flags = (int)arg_x(c, 7);
|
|
}
|
|
// void nvnCommandBufferDownsample(NVNcommandBuffer *cmd, const NVNtexture *src, const NVNtexture *dst): level 0 of src
|
|
// into level 0 of dst, whole extents, filtered. Recorded as a linear-filter texture copy: copy_t2t resolves a
|
|
// multisample source into an equal-size target and blits (2x2 average at half size) between sizes. As nxnative
|
|
// hle_nvn.c and SavageWindRecomp cmdbuf.c record it; docs/maxwell.md 1.19.
|
|
NVN_API(nvnCommandBufferDownsample) {
|
|
p_copy_t2t *o = REC(cmd_of(c), OP_COPY_T2T, p_copy_t2t);
|
|
o->src = tex_pin(arg_p(c, 1));
|
|
o->dst = tex_pin(arg_p(c, 2));
|
|
if (o->src) o->sr = (NVNcopyRegion){0, 0, 0, (int)o->src->d.width, (int)o->src->d.height, 1};
|
|
if (o->dst) o->dr = (NVNcopyRegion){0, 0, 0, (int)o->dst->d.width, (int)o->dst->d.height, 1};
|
|
o->flags = NVN_COPY_FLAGS_LINEAR_FILTER_BIT;
|
|
}
|
|
// void nvnCommandBufferSetCopyRowStride(NVNcommandBuffer *cmd, ptrdiff_t stride)
|
|
NVN_API(nvnCommandBufferSetCopyRowStride) { REC(cmd_of(c), OP_COPY_STRIDE, p_copy_stride)->stride = (int64_t)arg_x(c, 1); }
|
|
// void nvnCommandBufferSetBlendColor(NVNcommandBuffer *cmd, const float *rgba)
|
|
NVN_API(nvnCommandBufferSetBlendColor) { memcpy(REC(cmd_of(c), OP_BLEND_COLOR, p_blend_color)->rgba, arg_p(c, 1), 16); }
|
|
|
|
// Commands without an effect on this replay (hle_nvn.c g_nops): WaitSync and WaitEvent order work against other queues
|
|
// or the CPU, and every submit replays in order on one queue; shader scratch memory and the tessellation patch size
|
|
// have no host counterpart the replay needs.
|
|
// void nvnCommandBufferWaitSync(NVNcommandBuffer *cmd, const NVNsync *sync), WaitEvent(cmd, const NVNevent *event,
|
|
// NVNeventWaitMode mode, uint32_t value), SetShaderScratchMemory(cmd, const NVNmemoryPool *pool, ptrdiff_t offset,
|
|
// size_t size), SetPatchSize(cmd, int size)
|
|
NVN_API(nvnCommandBufferWaitSync) { (void)c; }
|
|
NVN_API(nvnCommandBufferWaitEvent) { (void)c; }
|
|
NVN_API(nvnCommandBufferSetShaderScratchMemory) { (void)c; }
|
|
NVN_API(nvnCommandBufferSetPatchSize) { (void)c; }
|
|
|
|
// ---------- submit
|
|
// What the console reads at the submit call is read here, on the game thread: every handle must be a live recording,
|
|
// and deferred binds take the handles the game's arrays hold now (the game fills its tables for the next frame as soon
|
|
// as the submit returns). The result is written into the recording copy.
|
|
static uint64_t def_reads, def_changed, n_clones;
|
|
|
|
static int resolve_deferred(const uint8_t *p, int depth) { // returns the number of deferred binds
|
|
int nd = 0;
|
|
for (;;) {
|
|
op_hdr h;
|
|
memcpy(&h, p, sizeof h);
|
|
if (h.size < 8 || (h.size & 7) || h.op >= OP_COUNT_)
|
|
rt_fatal("nvn: corrupt recording at %p while resolving deferred binds (op %u size %u)", (const void *)p,
|
|
h.op, h.size);
|
|
if (h.op == OP_END) return nd;
|
|
if (h.op == OP_JUMP) {
|
|
memcpy(&p, p + 8, 8);
|
|
continue;
|
|
}
|
|
if (h.op == OP_CALL) {
|
|
uint64_t t;
|
|
memcpy(&t, p + 8, 8);
|
|
if (depth >= 16) rt_fatal("nvn: deferred bind resolution: call depth over 16");
|
|
nd += resolve_deferred((const uint8_t *)(uintptr_t)t, depth + 1);
|
|
} else if (h.op == OP_BIND_TEX || h.op == OP_BIND_IMG || h.op == OP_BIND_SAMP) {
|
|
p_bind_handles *b = (p_bind_handles *)(uintptr_t)(p + 8);
|
|
if (b->src && b->n > 0 && b->n <= 32) {
|
|
nd++;
|
|
uint64_t cur[32];
|
|
memcpy(cur, (const void *)(uintptr_t)b->src, 8 * (size_t)b->n);
|
|
__atomic_fetch_add(&def_reads, 1, __ATOMIC_RELAXED);
|
|
if (memcmp(cur, b->h, 8 * (size_t)b->n)) {
|
|
if (__atomic_fetch_add(&def_changed, 1, __ATOMIC_RELAXED) < 6)
|
|
NVN_LOG("deferred bind read new handles at submit: stage %d slot %d, %#llx at record -> %#llx",
|
|
b->stage, b->first, (unsigned long long)b->h[0], (unsigned long long)cur[0]);
|
|
memcpy(b->h, cur, 8 * (size_t)b->n);
|
|
}
|
|
}
|
|
}
|
|
p += h.size;
|
|
}
|
|
}
|
|
|
|
void nvn_submit_prepare(const uint64_t *handles, int n) {
|
|
uint64_t now = __atomic_add_fetch(&nsubmits, 1, __ATOMIC_RELAXED);
|
|
for (int i = 0; i < n; i++) copy_touch(handles[i], now, "QueueSubmitCommands");
|
|
for (int i = 0; i < n; i++) resolve_deferred((const uint8_t *)(uintptr_t)handles[i], 0);
|
|
}
|
|
|
|
// Asynchronous replay. A copy a queued submission still reads keeps that submission's deferred binds: when it holds
|
|
// deferred binds (or is not known to hold none), this submission gets a clone, resolved here and freed after its replay.
|
|
void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) {
|
|
uint64_t now = __atomic_add_fetch(&nsubmits, 1, __ATOMIC_RELAXED);
|
|
uint64_t replayed = gfx_replayed_submits();
|
|
plat_mutex_lock(©_lock);
|
|
if (nzombies) zombie_sweep(replayed);
|
|
for (int i = 0; i < n; i++) {
|
|
copy_prefix *pf = copy_find(handles[i]);
|
|
if (!pf) {
|
|
plat_mutex_unlock(©_lock);
|
|
rt_fatal("nvn: QueueSubmitCommands: command handle %#llx is not a live recording (released or never recorded)",
|
|
(unsigned long long)handles[i]);
|
|
}
|
|
out[i] = handles[i];
|
|
if (pf->used_at > replayed && pf->used_at != now && pf->deferred != 1) {
|
|
copy_prefix *cl = copy_block(pf->bytes, 0, "recording clone"); // copy_lock held
|
|
size_t cap = cl->cap;
|
|
memcpy(cl, pf, sizeof *pf + pf->bytes);
|
|
cl->cap = cap;
|
|
cl->pin_v = NULL;
|
|
if (pf->pin_v && (cl->pin_v = malloc(sizeof *cl->pin_v * pf->pins)))
|
|
memcpy((void *)cl->pin_v, (const void *)pf->pin_v, sizeof *cl->pin_v * pf->pins);
|
|
copy_pins_ref(cl, 1);
|
|
cl->magic = 0; // not a live handle: the game never sees it
|
|
cl->used_at = now;
|
|
zombie_add(cl);
|
|
out[i] = (uintptr_t)(cl + 1);
|
|
__atomic_fetch_add(&n_clones, 1, __ATOMIC_RELAXED);
|
|
} else {
|
|
pf->used_at = now;
|
|
}
|
|
}
|
|
plat_mutex_unlock(©_lock);
|
|
dead_free();
|
|
for (int i = 0; i < n; i++) {
|
|
int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0);
|
|
if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1;
|
|
}
|
|
if (now % 7200 == 0)
|
|
rt_log("nvn: asynchronous replay at submit %llu: %llu recording clones so far, %u copies waiting for the replay",
|
|
(unsigned long long)now, (unsigned long long)__atomic_load_n(&n_clones, __ATOMIC_RELAXED), nzombies);
|
|
}
|
|
|
|
uint64_t nvn_submit_count(void) { return __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); }
|
|
|
|
const char *nvn_op_name(uint32_t op) {
|
|
static const char *const names[] = {
|
|
"END", "JUMP", "BEGIN", "CALL", "SIGNAL_EVENT", "REPORT_COUNTER", "COPY_B2B", "UPDATE_UB", "SET_RT",
|
|
"CLEAR_COLOR", "CLEAR_DS", "VIEWPORTS", "SCISSOR", "DEPTH_RANGE", "BIND_PROGRAM", "BIND_STATE", "BIND_VATTR",
|
|
"BIND_VSTREAM", "BIND_VBUF", "BIND_UBO", "BIND_SSBO", "BIND_TEX", "BIND_SAMP", "STENCIL", "POLY_OFFSET", "DRAW",
|
|
"DRAW_IDX", "DISPATCH", "BARRIER", "SET_TEX_POOL", "SET_SAMP_POOL", "COPY_B2T", "COPY_STRIDE", "BLEND_COLOR",
|
|
"COPY_T2B", "BIND_IMG", "COPY_T2T", "RESET_COUNTER", "CLEAR_TEX", "DEPTH_CLAMP", "RENDER_ENABLE",
|
|
"RASTER_DISCARD", "RENDER_COND", "PRIM_RESTART", "DRAW_IDX_IND", "DISPATCH_IND", "CLEAR_BUF",
|
|
};
|
|
_Static_assert(sizeof names / sizeof *names == OP_COUNT_, "one name per op");
|
|
return op < OP_COUNT_ ? names[op] : "?";
|
|
}
|