A draw whose pipeline state no op changed takes the previous draw's pipeline and state without building and comparing them (TOTK_BIND_INC)

From task/fastrep 18eab597 (TOTK_PIPE_GEN), now under TOTK_BIND_INC: the pipeline memo keeps the pipe_gen its state
was built under; a draw with that generation and the same shaders, primitive, restart, feedback flags and depth mode
takes the memo's state and pipeline and skips the three 360-byte compares (memo, cache entry, recorded state). The
shader memo skips its compare of pipe the same way. check builds and compares as before and counts a state the
generation called unchanged that differs (bi_mismatch logs the first of each kind with its marks and generations).
The census runs only in check mode. Lookout Landing check, 60 s: 67% of draws keep the state, 0 mismatches.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-13 23:42:11 +03:00
1 parent 2abfa23a67
commit ff3a95d0f1
2 files changed
+117 -30

No files matched your search

+1 -1
View File
@@ -122,7 +122,7 @@ Many rows below are developer tools that only act together with another setting
| Setting | What it does | Effect of changing it | Safety |
| --- | --- | --- | --- |
| `bind_inc` | `on` prepares a draw from what changed since the draw before (the bind ops mark the slots whose bytes they changed) instead of from the whole bound state; `check` prepares every draw as before, takes every decision of `on` beside it, logs each kind of difference once and counts them in a `bind_inc:` line every 300 frames; `off` or unset: as before. Both modes also log what changed between draws. | Less work per draw on the render thread; a change the marks missed would leave a stale descriptor, a wrong picture or a lost device, which `check` would show first. | Caution |
| `bind_inc` | `on` prepares a draw from what changed since the draw before (the bind ops mark the slots whose bytes they changed) instead of from the whole bound state; `check` prepares every draw as before, takes every decision of `on` beside it, logs each kind of difference once and counts them in a `bind_inc:` line every 300 frames; `off` or unset: as before. `check` also counts what changed between draws in that line. | Less work per draw on the render thread; a change the marks missed would leave a stale descriptor, a wrong picture or a lost device, which `check` would show first. | Caution |
| `block_views_3d` (default `0`) | `1` gives 3D block-compressed images block views too. | On some AMD cards a GPU write past the end of the image's memory loses the Vulkan device, and the game stops. | Risky |
| `deswizzle` (default `auto`) | Where block-linear textures are deswizzled: `auto` or `gpu` uses a compute pass where the texture's storage is an imported pool and the CPU elsewhere, `cpu` always uses the CPU, `check` uses the GPU and also deswizzles every upload on the CPU and compares the two. | `cpu` brings back the long frames on entering new areas (the replay thread deswizzles every texture); `check` waits for the GPU at every upload and is for testing only. The log names the mode in use. | Caution |
| `dyn_delta` (default `1`) | Records only the pipeline dynamic states (cull mode, depth and stencil tests, vertex input, blend enables, equations and write masks, ...) whose values changed since the last draw in the command buffer, also across pipeline changes; `0` records the full set at every change of pipeline state, as before; `ab` alternates with the `ab` blocks. | Fewer commands per pipeline change on the render and recorder threads; a driver that loses dynamic state across pipeline binds would draw with stale state. `TOTK_TRACE=perf` prints the calls and commands per submission. | Caution |
+116 -29
View File
@@ -97,6 +97,8 @@ typedef struct {
VkPipeline pipe[2]; // bound per bind point: 0 graphics, 1 compute
VkPipeState st; // the state vk_pipeline_set_dynamic last recorded
int st_valid;
int st_pm; // st equals PipeMemo.s (set where both were the draw's state, cleared at
// every write of PipeMemo): TOTK_BIND_INC skips the compare of st
int dyn_valid; // vk_pipeline_set_dynamic_delta's record of this command buffer's dynamic state
// holds (TOTK_DYN_DELTA); pipeline binds keep it, the memset clears it
uint32_t vmask; // the vertex bindings st's vertex input reads (valid with st_valid)
@@ -269,13 +271,15 @@ typedef struct {
const prog_t *prog;
uint64_t serial;
uint32_t gen, tex_serial;
uint64_t pipe_gen; // VkExec.pipe_gen when pipe was copied
uint64_t tex[2][32], img[2][32];
VkPipeState pipe;
} MemoKey;
typedef struct { int valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } ShaderMemo;
// The previous draw's pipeline and the state it was found for (prepare_draw).
typedef struct { int valid; VkPipeState s; const VkPipeInfo *info; } PipeMemo;
// The previous draw's pipeline and the state it was found for (prepare_draw). gen: VkExec.pipe_gen when s was built
// from VkExec.pipe.
typedef struct { int valid; VkPipeState s; const VkPipeInfo *info; uint64_t gen; } PipeMemo;
// The pipelines found before, by state: vk_pipeline_graphics_async takes the pipeline cache's lock, which the contexts
// of a chunked replay would take on every draw whose state differs from the draw before. Direct-mapped.
#define PC_N 1024
@@ -300,8 +304,12 @@ typedef struct {
uint32_t frames; // frames since the last line
uint64_t draws, cls[4], b_gen, desc, rec[4], always, mv[BI_MV], slots[5];
uint64_t sh_keep, sh_memo, fb_n, fb_keep, fb_memo, vb_skip, pg_hit;
uint64_t mism, mism_total;
uint64_t mism, mism_total, checked;
uint32_t mism_frame, frame_lines;
} BiCensus;
enum { BIM_PIPE, BIM_DYN, BIM_SHADER, BIM_FB, BIM_VB, BIM_DESC, BIM_SET, BIM_N }; // check mode's kinds of difference
static int bind_inc_mode(void);
static void bi_mismatch(struct VkExec *ex, int kind, const char *fmt, ...);
// The memos and command buffer state of one recording context (VkExec.draw). Nothing in it is shared: the context's
// thread reads and writes it without a lock.
@@ -2508,14 +2516,25 @@ static int pipe_skip_mode(void) {
static void memo_key(const VkExec *ex, uint32_t gen, MemoKey *k) {
k->prog = ex->prog, k->serial = ex->prog->serial, k->gen = gen, k->tex_serial = ex->tex_serial;
k->pipe_gen = ex->pipe_gen;
memcpy(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]), memcpy(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]);
memcpy(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]), memcpy(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]);
memcpy(&k->pipe, &ex->pipe, sizeof k->pipe);
}
static int memo_match(const VkExec *ex, uint32_t gen, MemoKey *k) {
if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen || memcmp(&k->pipe, &ex->pipe, sizeof k->pipe))
static int bind_inc_mode(void);
static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...);
// TOTK_BIND_INC: pipe equals the key's while pipe_gen does (no write changed it since the copy), so the 360-byte
// compare is skipped; check compares anyway and counts a difference the generation missed.
static int memo_match(VkExec *ex, uint32_t gen, MemoKey *k) {
if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen) return 0;
int bim = bind_inc_mode(), same_gen = k->pipe_gen == ex->pipe_gen;
if (!(bim == 1 && same_gen && !ex->chunked) && memcmp(&k->pipe, &ex->pipe, sizeof k->pipe)) {
if (bim == 2 && same_gen && !ex->chunked) bi_mismatch(ex, BIM_PIPE, "shader memo: bound state changed without its generation");
return 0;
}
k->pipe_gen = ex->pipe_gen;
if (k->tex_serial == ex->tex_serial) return 1;
if (memcmp(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]) || memcmp(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]) ||
memcmp(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]) || memcmp(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]))
@@ -2549,7 +2568,7 @@ static int pipe_memo_on(void) {
// mark, the CbState memset drops BL, and a chunk of a chunked replay never keeps anything (its slots resolve per op).
// off: as before. on: the kept results. check: prepares every draw as before, takes every decision of on beside it,
// compares what on would have kept with what the full path gave, logs each kind of difference once with its marks and
// counts them. Both modes count the decisions (bi_census) and log them every 300 frames.
// counts them; it also counts the decisions (bi_census) and logs them every 300 frames.
static int bind_inc_mode(void) { // 0 off, 1 on, 2 check
static int mode = -1;
if (mode < 0) {
@@ -2573,6 +2592,40 @@ static void bi_gb(const VkExec *ex, uint64_t g[4]) {
g[0] = vk_mem_gen(), g[1] = vk_usnap_gen(), g[2] = OV.gen, g[3] = ex->fence;
}
// check mode: a decision of on would have kept a result that differs from the full path's. Counted; the first of each
// kind is logged with the marks, generations and counters it was taken under.
static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...) {
static const char *const name[BIM_N] = {"pipeline state", "recorded dynamic state", "shaders", "feedback scan",
"vertex buffers", "descriptor", "descriptor set"};
static volatile int32_t seen[BIM_N];
VkExecDraw *D = ex->draw;
D->BC.mism++, D->BC.mism_total++, D->BC.mism_frame++;
if (__atomic_exchange_n(&seen[kind], 1, __ATOMIC_RELAXED)) return;
char msg[320];
va_list ap;
va_start(ap, fmt);
vsnprintf(msg, sizeof msg, fmt, ap);
va_end(ap);
const VkExecDirty *d = &ex->dirt;
const BiLast *L = &D->BL;
uint64_t gt[4], gb[4];
bi_gt(ex, gt), bi_gb(ex, gb);
rt_log("bind_inc check: kept %s differs from the full path: %s; program %llu (last %llu), marks prog %u ubo %#x/%#x "
"ssbo %#x/%#x tex %016llx:%016llx/%016llx:%016llx img %#x/%#x vbuf %#x, pipe_gen %llu (last %llu); G_tex "
"%llx %llx %llx %llx (last %llx %llx %llx %llx), G_buf %llx %llx %llx %llx (last %llx %llx %llx %llx); "
"pass %u (last %u), draw %llu of the census window, bi_valid %d",
name[kind], msg, ex->prog ? (unsigned long long)ex->prog->serial : 0ull, (unsigned long long)L->serial,
d->prog, d->ubo[0], d->ubo[1], d->ssbo[0], d->ssbo[1], (unsigned long long)d->tex[0][1],
(unsigned long long)d->tex[0][0], (unsigned long long)d->tex[1][1], (unsigned long long)d->tex[1][0], d->img[0],
d->img[1], d->vbuf, (unsigned long long)ex->pipe_gen, (unsigned long long)L->pipe_gen,
(unsigned long long)gt[0], (unsigned long long)gt[1], (unsigned long long)gt[2], (unsigned long long)gt[3],
(unsigned long long)L->gt[0], (unsigned long long)L->gt[1], (unsigned long long)L->gt[2],
(unsigned long long)L->gt[3], (unsigned long long)gb[0], (unsigned long long)gb[1], (unsigned long long)gb[2],
(unsigned long long)gb[3], (unsigned long long)L->gb[0], (unsigned long long)L->gb[1],
(unsigned long long)L->gb[2], (unsigned long long)L->gb[3], ex->pass_serial, L->pass_serial,
(unsigned long long)D->BC.draws, D->B.bi_valid);
}
static double bi_pct(uint64_t n, uint64_t of) { return of ? 100.0 * (double)n / (double)of : 0; }
static void bi_log(BiCensus *c) {
@@ -2592,13 +2645,14 @@ static void bi_log(BiCensus *c) {
bi_pct(c->mv[8], c->draws), bi_pct(c->mv[9], c->draws), c->slots[0] / d, c->slots[1] / d, c->slots[2] / d,
c->slots[3] / d, c->slots[4] / d);
rt_log("bind_inc: kept (%% of draws): shaders %.1f (memo today %.1f), pipeline state %.1f, vertex buffers %.1f, "
"feedback scan %.1f of %.1f (today %.1f); mismatches %.2f a frame, %llu in all",
"feedback scan %.1f of %.1f (today %.1f); mismatches %.2f a frame, %llu in all, %.0f kept results checked a frame",
bi_pct(c->sh_keep, c->draws), bi_pct(c->sh_memo, c->draws), bi_pct(c->pg_hit, c->draws),
bi_pct(c->vb_skip, c->draws), bi_pct(c->fb_keep, c->draws), bi_pct(c->fb_n, c->draws),
bi_pct(c->fb_memo, c->draws), c->mism / fr, (unsigned long long)c->mism_total);
bi_pct(c->fb_memo, c->draws), c->mism / fr, (unsigned long long)c->mism_total, c->checked / fr);
uint64_t total = c->mism_total;
uint32_t lines = c->frame_lines;
memset(c, 0, sizeof *c);
c->mism_total = total;
c->mism_total = total, c->frame_lines = lines;
}
static int popc64(uint64_t v) { return __builtin_popcountll(v); }
@@ -2612,6 +2666,9 @@ static void bi_census(VkExec *ex, const VkPipeInfo *info, const VkPipeState *s,
const BiLast *L = &D->BL;
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
if (f != c->frame) {
if (c->mism_frame && c->frame_lines < 200)
c->frame_lines++, rt_log("bind_inc check: frame %llu: %u mismatches", (unsigned long long)c->frame, c->mism_frame);
c->mism_frame = 0;
if (c->frames >= 300) bi_log(c);
c->frame = f, c->frames++;
}
@@ -2752,13 +2809,32 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
if (!vs || (has_fs && !fs)) return 0;
if (vk_dlaa_skip_draw(ex, fs)) return 0; // the game's FXAA off, or its scaling replaced by DLSS (vk_dlaa.c)
uint32_t fbl = feedback_prepare(ex, fs);
VkPipeState s = ex->pipe;
s.vs = vs, s.fs = fs;
s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0));
s.prim = (uint8_t)prim;
s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0);
s.y_flip = 1;
s.prim_restart = (uint8_t)(restart != 0);
uint8_t fb = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0)), dmw = (uint8_t)(nvn_depth_minus_w() != 0);
// TOTK_BIND_INC: the pipeline memo's state is this draw's when pipe was not written since the memo's state was built
// from it and the fields a draw sets are the same (y_flip is always 1): its pipeline is taken without building the
// state and comparing it with the memo's and the recorded one (3 x 360 bytes). check builds and compares as before.
int pg = pipe_memo_on() && !ex->chunked ? bim : 0;
int pg_hit = pg && D->PM.valid && D->PM.gen == ex->pipe_gen && D->PM.s.vs == vs && D->PM.s.fs == fs &&
D->PM.s.feedback == fb && D->PM.s.prim == (uint8_t)prim && D->PM.s.depth_minus_w == dmw &&
D->PM.s.prim_restart == (uint8_t)(restart != 0);
VkPipeState s; // built from pipe unless the generation hit
const VkPipeState *sp = &s;
uint64_t pgen = ex->pipe_gen;
if (pg_hit && pg == 1) {
sp = &D->PM.s;
} else {
s = ex->pipe;
s.vs = vs, s.fs = fs;
s.feedback = fb;
s.prim = (uint8_t)prim;
s.depth_minus_w = dmw;
s.y_flip = 1;
s.prim_restart = (uint8_t)(restart != 0);
if (pg_hit) { // check
D->BC.checked++;
if (memcmp(&s, &D->PM.s, sizeof s)) bi_mismatch(ex, BIM_PIPE, "pipeline memo: state differs from the memo its generation matched");
}
}
const VkPipeInfo *info = NULL;
int pending;
// A draw of at most 6 vertices over half the render area or more (a full-screen triangle or quad, a tile) waits for
@@ -2782,12 +2858,17 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
const char *why = NULL;
uint64_t w0 = 0;
PipeCache *pc = NULL;
if (pipe_memo_on() && !(D->PM.valid && !memcmp(&D->PM.s, &s, sizeof s)))
int pg_take = pg_hit && pg == 1; // the memo's pipeline without building or comparing the state
if (!pg_take && pipe_memo_on() && !(D->PM.valid && !memcmp(&D->PM.s, &s, sizeof s)))
pc = &D->PC[nvn_hm_hash(&s, sizeof s) & (PC_N - 1)];
if (pipe_memo_on() && (!pc || (pc->info && !memcmp(&pc->s, &s, sizeof s)))) {
if (pg_take) {
info = D->PM.info, pending = 0;
p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE);
} else if (pipe_memo_on() && (!pc || (pc->info && !memcmp(&pc->s, &s, sizeof s)))) {
info = pc ? pc->info : D->PM.info, pending = 0;
p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE);
if (pc) D->PM.valid = 1, D->PM.s = s, D->PM.info = info;
if (pc) D->PM.valid = 1, D->PM.s = s, D->PM.info = info, D->B.st_pm = 0;
D->PM.gen = pgen; // s equals PM.s: built from pipe as it is now
} else {
w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait
p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
@@ -2802,8 +2883,8 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
if (!w0) w0 = kern_now_ns();
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
}
D->PM.valid = !pending && p && info;
if (D->PM.valid) D->PM.s = s, D->PM.info = info;
D->PM.valid = !pending && p && info, D->B.st_pm = 0;
if (D->PM.valid) D->PM.s = s, D->PM.info = info, D->PM.gen = pgen;
if (D->PM.valid && pc) pc->s = s, pc->info = info;
}
uint64_t waited = w0 ? kern_now_ns() - w0 : 0;
@@ -2819,7 +2900,11 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
if (pending) return 0; // a background worker is building it: the draw is skipped until it is ready
if (!p || !info) return warn_once(W_PIPELINE, "a graphics pipeline could not be created: draws with it are skipped"), 0;
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p);
if (!D->B.st_valid || memcmp(&D->B.st, &s, sizeof s)) {
// On a generation hit the state recorded last is the memo's when st_pm says so (both were the last draw's).
int st_same = D->B.st_valid && (pg_take && D->B.st_pm ? 1 : !memcmp(&D->B.st, sp, sizeof *sp));
if (pg_hit && pg == 2 && D->B.st_valid && D->B.st_pm && !st_same)
bi_mismatch(ex, BIM_DYN, "the recorded dynamic state differs from the pipeline memo's");
if (!st_same) {
// TOTK_DYN_DELTA: only the dynamic states whose values changed since the last call in this command buffer. A
// pipeline bind clears st_valid but not dyn_valid: every pipeline here declares the same dynamic states, which
// the bind leaves as they were (the viewport, scissor and stencil values of record_fixed rely on that too).
@@ -2827,18 +2912,19 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
if (DD.mode < 0) DD.mode = rt_knob("DYN_DELTA", "1");
uint32_t cmds = 1;
if (RT_KNOB_ON(DD.mode)) {
D->B.vmask = vk_pipeline_set_dynamic_delta(ex->cb, &s, D->B.dyn_valid, &cmds);
D->B.vmask = vk_pipeline_set_dynamic_delta(ex->cb, sp, D->B.dyn_valid, &cmds);
D->B.dyn_valid = 1;
} else {
D->B.vmask = vk_pipeline_set_dynamic(ex->cb, &s);
D->B.vmask = vk_pipeline_set_dynamic(ex->cb, sp);
D->B.dyn_valid = 0; // the delta's record is stale from here on
}
__atomic_fetch_add(&DD.calls, 1, __ATOMIC_RELAXED), __atomic_fetch_add(&DD.cmds, cmds, __ATOMIC_RELAXED);
if (!cmds) __atomic_fetch_add(&DD.empty, 1, __ATOMIC_RELAXED);
ex->recorded += cmds != 0;
D->B.st = s, D->B.st_valid = 1;
D->B.st = *sp, D->B.st_valid = 1;
if (ex->mark && ex->fr.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs;
}
D->B.st_pm = D->PM.valid; // st is this draw's state, which PM holds while valid
vk_dlaa_draw(ex);
int mo = ex->chunked && vk_dlaa_available();
if (mo) plat_mutex_lock(&mo_lock);
@@ -2847,15 +2933,16 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
if (mo) plat_mutex_unlock(&mo_lock);
record_fixed(ex);
repeat_stats(ex, info);
if (bim) bi_gt(ex, bgt), bi_gb(ex, bgb), bi_census(ex, info, &s, bgt, bgb, bsg, sm_hit);
if (bim) bi_gt(ex, bgt), bi_gb(ex, bgb);
if (bim == 2) bi_census(ex, info, sp, bgt, bgb, bsg, sm_hit); // on counts nothing: it is what gets timed
bind_resources(ex, info);
if (fbl & ex->fb_dirty) feedback_barrier(ex);
feedback_written(ex, &s);
feedback_written(ex, sp);
bind_vertex_buffers(ex);
if (vk_dump_armed) vk_dump_draw(ex, &s, p, info);
if (vk_dump_armed) vk_dump_draw(ex, sp, p, info);
if (vk_hdr_track) vk_hdr_draw(ex, info);
vk_exec_oq_draw(ex);
if (bim) bi_done(ex, info, &s, bgt, bgb, bsg);
if (bim) bi_done(ex, info, sp, bgt, bgb, bsg);
return 1;
}