A graphics draw with the previous draw's pipeline rebuilds only the descriptors whose inputs changed (TOTK_BIND_INC)

With TOTK_BIND_INC the graphics build writes into VkExecDraw.BI, which keeps the packed descriptors of the last build
and, per descriptor, what it counted for the traffic census and whether it copied guest bytes. bi_plan names the
bindings to build again: support always; c0 when a storage slot of the stage or the base instance changed; the
compiler constants when the program, its serial or G_buf changed; uniform and storage slots when marked or when G_buf
(pools and mirrors, snapshots, overlays, the submission) moved; textures and images when a handle was marked or G_tex
(slot registrations, samplers, the storage epoch, the feedback layout, DLSS's swap) moved; storage addresses held in
a uniform block, and any descriptor whose build copied guest bytes, every time. A pipeline with a written storage
binding is built in full. Kept descriptors count their census again; a set none of whose descriptors changed and that
is still pushed or bound records nothing, without the psig or sig compare. check builds everything as before and
compares each kept descriptor and its census with the build. Lookout Landing check, 60 s: about 17,000 kept bindings
compared a frame, 0 mismatches.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-13 23:49:21 +03:00
1 parent c86feefadc
commit c26ed9ba7b
1 file changed
+145 -9
+145 -9
View File
@@ -215,6 +215,7 @@ typedef struct {
uint8_t c0[2], c1[2]; // reads c0 (every storage slot and the base instance), the compiler constants
uint8_t gfx; // every binding is a vertex or fragment one
uint16_t always, ndesc;
uint8_t wr; // a storage binding the shaders write: its build has side effects, nothing is kept
} BiMask;
typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; BiMask m; RsRange r[]; } RsList;
// (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the
@@ -314,6 +315,21 @@ enum { BIM_PIPE, BIM_DYN, BIM_SHADER, BIM_FB, BIM_VB, BIM_DESC, BIM_SET, BIM_N }
static int bind_inc_mode(void);
static void bi_mismatch(struct VkExec *ex, int kind, const char *fmt, ...);
// TOTK_BIND_INC: what building one descriptor counted (count_use; n 0: nothing, more than 1: never kept) and whether it
// copied guest bytes into transient memory (vol: a rebuild may differ, so it is never kept).
typedef struct { uint8_t vol, n; int16_t cls; int32_t use; uint64_t bytes; } BiMeta;
// The packed descriptors of the last graphics build (bind_resources writes them here while TOTK_BIND_INC is set), the
// pipeline they are for (valid with CbState.bi_valid) and, per descriptor, BiMeta.
typedef struct {
const VkPipeInfo *info;
BiMeta *cur; // count_use and the transient copies record here while a descriptor is built
VkDescSlot d[MAX_DESC];
BiMeta meta[MAX_DESC];
VkDescSlot old[MAX_DESC]; // check mode: the descriptors before the build
BiMeta mold[MAX_DESC];
uint8_t plan[MAX_BINDINGS]; // bindings to build again (bi_plan)
} BiDesc;
// The memos and command buffer state of one recording context (VkExec.draw). Nothing in it is shared: the context's
// thread reads and writes it without a lock.
typedef struct VkExecDraw {
@@ -325,6 +341,7 @@ typedef struct VkExecDraw {
RsState RS;
BiLast BL;
BiCensus BC;
BiDesc BI;
nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown
const VkPipeInfo *rs_last; // the last lookup
const RsList *rs_last_l;
@@ -1069,6 +1086,11 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) {
// vk_mem_count, also kept per (class, use) while a graphics build runs (RS).
static void count_use(VkExecDraw *D, int cls, int use, uint64_t n) {
vk_mem_count(cls, use, n);
if (D->BI.cur) {
BiMeta *m = D->BI.cur;
if (m->n++) m->vol = 1; // a second count: not kept (one is all a descriptor build makes)
m->cls = (int16_t)cls, m->use = use, m->bytes = n;
}
if (!D->RS.rec) return;
for (int i = 0; i < D->RS.ncnt; i++)
if (D->RS.cnt[i].cls == cls && D->RS.cnt[i].use == use) {
@@ -1109,6 +1131,7 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align,
VkBuffer b;
VkDeviceSize o;
D->RS.volatile_ = 1;
if (D->BI.cur) D->BI.cur->vol = 1;
memcpy(scratch(ex, n, &b, &o), (const void *)(uintptr_t)addr, n);
vk_mem_count(sp.cls, VK_MUSE_CPU_READ, n);
vk_mem_count(VK_MCLS_SCRATCH, use, n);
@@ -2036,6 +2059,7 @@ static void bi_mask(const VkPipeInfo *info, BiMask *m) {
}
} else if (b->kind == GFX_RES_SSBO) {
int ubo, slot = ssbo_slot(r, st, &ubo);
if (r->flags & 1) m->wr = 1;
if ((r->flags & 1) || slot < 0 || slot >= 16 || ubo) m->always = (uint16_t)(m->always + cnt);
else m->ssbo[st] |= 1u << slot;
} else if ((b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER &&
@@ -2206,15 +2230,79 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c
// graphics push equal to the last one at its set number, with no other layout or bind between, is not recorded.
// The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update
// template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template.
static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
static int bind_inc_mode(void);
static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...);
// TOTK_BIND_INC: which bindings of a graphics draw with the last prepared draw's pipeline must be built again (BI.plan,
// 1 each), from the marks since that draw and the generations G_tex and G_buf read right before this build (gt, gb)
// against those read before its build. 0 when nothing can be kept: no last draw in this command buffer, another
// pipeline, a chunk, a storage binding the shaders write (its build restarts passes and notes writes).
// - support: always (its inputs are compared inside, 12 bytes);
// - uniform slot 0 (c0): a storage slot of the stage or the base instance changed; slot 1 (the compiler constants):
// the program or its serial changed, or G_buf moved; slots 3..20: the slot changed or G_buf moved (snapshots,
// overlays, pools and mirrors); other slots bind the zero buffer;
// - storage: as a uniform slot; an address held in a uniform block (ubo_ptr): always, the block changes without an op;
// - textures and images: a handle changed or G_tex moved (slot resolution, samplers, the storage check once per
// submission, the feedback layout, DLSS's swap); buffer textures bind the zero view;
// - any descriptor whose last build copied guest bytes or counted more than once (BiMeta): always.
static int bi_plan(VkExec *ex, const VkPipeInfo *info, const uint64_t gt[4], const uint64_t gb[4]) {
VkExecDraw *D = ex->draw;
const BiLast *L = &D->BL;
if (!D->B.bi_valid || ex->chunked || L->info != info || D->BI.info != info) return 0;
const BiMask *m = &rs_list(ex, info)->m;
if (!m->gfx || m->wr) return 0;
const VkExecDirty *dt = &ex->dirt;
int tmv = memcmp(gt, L->gt, 4 * sizeof *gt) != 0, bmv = memcmp(gb, L->gb, 4 * sizeof *gb) != 0;
int prog = dt->prog || ex->prog != L->prog || ex->prog->serial != L->serial, binst = D->c0_binst != L->binst;
uint32_t nd = 0;
for (uint32_t i = 0; i < info->n_bindings; i++) {
const VkPipeBinding *b = &info->bindings[i];
const GfxShaderRes *r = b->res;
uint32_t cnt = b->count ? b->count : 1;
int st = nvn_stage_of(b->stage), again = 0;
for (uint32_t k = 0; k < cnt && !again; k++) again = D->BI.meta[nd + k].vol;
if (b->kind == VK_PIPE_SUPPORT) {
again = 1;
} else if (b->kind == GFX_RES_UBO) {
for (uint32_t k = 0; k < cnt && !again; k++) {
int slot = r->slot + (int)k;
if (slot == 0) again = dt->ssbo[st] || binst;
else if (slot == 1) again = prog || bmv;
else if (slot >= 3 && slot - 3 < 18) again = bmv || (dt->ubo[st] >> (slot - 3) & 1);
}
} else if (b->kind == GFX_RES_SSBO) {
int ubo, slot = ssbo_slot(r, st, &ubo);
again = again || ubo || slot < 0 || slot >= 16 || bmv || (dt->ssbo[st] >> slot & 1);
} else if ((b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER &&
b->type != VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) {
int img = b->kind == GFX_RES_IMG;
again = again || tmv;
for (uint32_t k = 0; k < cnt && !again; k++) {
int slot = r->slot + (int)k;
if (img) again = slot >= 0 && slot < 32 && (dt->img[st] >> slot & 1);
else again = slot >= 0 && slot < 128 && (dt->tex[st][slot >> 6] >> (slot & 63) & 1);
}
}
D->BI.plan[i] = (uint8_t)again;
nd += cnt;
}
return 1;
}
static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *bgt, const uint64_t *bgb) {
VkExecDraw *D = ex->draw;
VkWriteDescriptorSet *w = ex->draw->w;
VkDescSlot *d = ex->draw->d;
VkBufferView *bv = ex->draw->bv;
if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS);
bind_stats(ex->draw, info);
int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS;
int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = D->B.res_valid;
// TOTK_BIND_INC: a graphics build writes BI's array and per-descriptor census (bgt: a prepared draw's), on builds
// only the bindings bi_plan names, check builds everything and compares what on would have kept.
int bim = gfx && bgt ? bind_inc_mode() : 0;
int inc = bim ? bi_plan(ex, info, bgt, bgb) : 0;
VkDescSlot *d = bim ? D->BI.d : ex->draw->d;
BiMeta *meta = bim ? D->BI.meta : NULL;
int skip = gfx && bim != 1 ? bind_skip_on() : 0, same = 0, was_valid = D->B.res_valid;
if (skip) {
same = bind_sig(ex, info);
if (same && was_valid && skip == 1) {
@@ -2236,6 +2324,13 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
lo[s] = i0, hi[s] = i1, i0 = i1;
if (i1 > lo[s] && !info->tpl[s]) need_w = 1;
}
if (inc && need_w) inc = 0; // a set without a template reads the write structs, which only a full build fills
int changed[2] = {!inc, !inc}; // inc: a descriptor of the set differs from the last build's
if (bim == 2 && inc) { // check: the descriptors before the build, to compare those on would have kept
uint32_t n = 0;
for (uint32_t i = 0; i < info->n_bindings; i++) n += info->bindings[i].count ? info->bindings[i].count : 1;
memcpy(D->BI.old, d, n * sizeof *d), memcpy(D->BI.mold, meta, n * sizeof *meta);
}
uint32_t nd = 0;
for (uint32_t i = 0; i < info->n_bindings; i++) {
const VkPipeBinding *b = &info->bindings[i];
@@ -2245,6 +2340,19 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
if (n_sets == 2 && i == lo[1]) d0[1] = nd;
VkDescSlot *o = &d[nd];
int st = nvn_stage_of(b->stage), kind = 0; // kind: 0 buffer, 1 image, 2 texel buffer view
uint32_t set_of = n_sets == 2 && i >= lo[1] ? 1 : 0;
if (inc && bim == 1 && !D->BI.plan[i]) { // kept: what it counted is counted again
for (uint32_t k = 0; k < cnt; k++)
if (meta[nd + k].n) vk_mem_count(meta[nd + k].cls, meta[nd + k].use, meta[nd + k].bytes);
nd += cnt;
continue;
}
VkDescSlot prev[4]; // on: the descriptors before the build, to tell whether the set changed
int cmp = inc && bim == 1 && cnt <= 4 && !changed[set_of];
if (cmp) memcpy(prev, o, cnt * sizeof *o);
else if (inc && bim == 1) changed[set_of] = 1;
if (meta) memset(&meta[nd], 0, cnt * sizeof *meta);
#define BI_CUR(k) (D->BI.cur = meta ? &meta[nd + (k)] : NULL)
if (b->kind == VK_PIPE_SUPPORT) {
float in[3];
support_in(ex, in);
@@ -2262,9 +2370,9 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
}
o->buffer = (VkDescriptorBufferInfo){D->B.sup_buf, D->B.sup_off, SUPPORT_BYTES};
} else if (b->kind == GFX_RES_UBO) {
for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ubo_info(ex, st, r->slot + (int)k);
for (uint32_t k = 0; k < cnt; k++) BI_CUR(k), o[k].buffer = ubo_info(ex, st, r->slot + (int)k);
} else if (b->kind == GFX_RES_SSBO) {
for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ssbo_info(ex, st, r, r->flags & 1);
for (uint32_t k = 0; k < cnt; k++) BI_CUR(k), o[k].buffer = ssbo_info(ex, st, r, r->flags & 1);
} else if (b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) {
int img = b->kind == GFX_RES_IMG;
if (b->type == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER || b->type == VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) {
@@ -2282,6 +2390,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
uint64_t h = 0;
if (img) h = slot >= 0 && slot < 32 ? ex->img[st][slot] : 0;
else h = slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0;
BI_CUR(k);
tex_info(ex, r, h, img, r->separate, &o[k].image, &locked);
o[k].w[2] = (uint32_t)o[k].image.imageLayout; // zero the tail padding
}
@@ -2290,6 +2399,26 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
} else {
rt_fatal("vk: binding %u has an unknown kind %d", (unsigned)b->binding, (int)b->kind);
}
D->BI.cur = NULL;
#undef BI_CUR
if (cmp && memcmp(prev, o, cnt * sizeof *o)) changed[set_of] = 1;
if (bim == 2 && inc && !D->BI.plan[i]) { // check: on would have kept these
D->BC.checked++;
for (uint32_t k = 0; k < cnt; k++) {
const BiMeta *mo = &D->BI.mold[nd + k], *mn = &meta[nd + k];
if (!memcmp(&D->BI.old[nd + k], &o[k], sizeof *o) && mo->n == mn->n &&
(!mn->n || (mo->cls == mn->cls && mo->use == mn->use && mo->bytes == mn->bytes)))
continue;
bi_mismatch(ex, BIM_DESC, "binding %u (stage %d, kind %d, slot %d + %u, set %u): kept %016llx %016llx %016llx, "
"built %016llx %016llx %016llx; census %d/%d/%llu x%u, now %d/%d/%llu x%u", (unsigned)b->binding,
st, (int)b->kind, r ? r->slot : -1, k, set_of, (unsigned long long)D->BI.old[nd + k].w[0],
(unsigned long long)D->BI.old[nd + k].w[1], (unsigned long long)D->BI.old[nd + k].w[2],
(unsigned long long)o[k].w[0], (unsigned long long)o[k].w[1], (unsigned long long)o[k].w[2],
mo->cls, mo->use, (unsigned long long)mo->bytes, mo->n, mn->cls, mn->use,
(unsigned long long)mn->bytes, mn->n);
break;
}
}
if (need_w) {
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
.dstBinding = b->binding, .descriptorCount = cnt, .descriptorType = b->type};
@@ -2326,6 +2455,10 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
}
if (s ? info->push1 : info->push) {
uint32_t n = gfx && push_skip_on() ? dbytes : 0;
if (inc && bim == 1 && !changed[s] && n && D->B.ds_pushed[s] && D->DS.pn[s] == n) { // psig holds ds
D->BS.push_skip++;
continue;
}
if (n && D->B.ds_pushed[s] && D->DS.pn[s] == n && !memcmp(D->DS.psig[s], ds, n)) {
D->BS.push_skip++;
continue;
@@ -2345,7 +2478,8 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
} else {
VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout;
VkDescriptorSet set = D->B.ds_last[s];
if (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || memcmp(D->DS.sig[s], ds, dbytes)) {
int kept = inc && bim == 1 && !changed[s] && set && D->B.ds_last_layout[s] == sl && D->DS.n[s] == dbytes;
if (!kept && (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || memcmp(D->DS.sig[s], ds, dbytes))) {
set = alloc_set(ex, sl);
write_set(info, s, set, ds, w, i0, i1);
D->B.ds_last[s] = set, D->B.ds_last_layout[s] = sl, D->DS.n[s] = dbytes;
@@ -2383,6 +2517,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
}
D->B.res_valid = valid;
}
if (bim) D->BI.info = info;
ex->recorded++;
}
@@ -2999,7 +3134,7 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
repeat_stats(ex, info);
if (bim) bi_gt(ex, bgt), bi_gb(ex, bgb);
if (bim == 2) bi_census(ex, info, sp, bgt, bgb, bsg, sm_hit); // on counts nothing: it is what gets timed
bind_resources(ex, info);
bind_resources(ex, info, bim ? bgt : NULL, bgb);
if (fbl & ex->fb_dirty) feedback_barrier(ex);
feedback_written(ex, sp);
bind_vertex_buffers(ex);
@@ -3919,7 +4054,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) {
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
D->c0_dim[0] = (uint32_t)d->x, D->c0_dim[1] = (uint32_t)d->y, D->c0_dim[2] = (uint32_t)d->z;
D->twin_dispatch = 1;
bind_resources(ex, info);
bind_resources(ex, info, NULL, NULL);
D->twin_dispatch = 0;
VkPassMark *m = dispatch_mark(ex, cs, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z);
vkCmdDispatch(ex->cb, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z);
@@ -3950,7 +4085,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) {
D->c0_dim[0] = D->c0_dim[1] = D->c0_dim[2] = 0;
D->B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads
D->twin_dispatch = 1;
bind_resources(ex, info);
bind_resources(ex, info, NULL, NULL);
D->twin_dispatch = 0;
if (D->B.c0_valid[2]) { // the shader binds c0
VkBufferCopy cp = {ip.offset, D->B.c0_off[2] + 0x200, sizeof(VkDispatchIndirectCommand)};
@@ -4158,6 +4293,7 @@ static int ub_bind(VkExec *ex, uint64_t addr, uint64_t n, uint64_t align, VkDesc
VkDeviceSize o;
uint8_t *dst = scratch(ex, n, &b, &o);
ex->draw->RS.volatile_ = 1;
if (ex->draw->BI.cur) ex->draw->BI.cur->vol = 1;
const uint8_t *base = vk_usnap_bytes(addr, n);
VkMemSpan sp;
if (base) memcpy(dst, base, n);