From 2abfa23a67cb37971da22ce521d357dd9058ac63 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 13 Nov 2026 23:40:01 +0300 Subject: [PATCH 01/10] Bind ops mark the slots whose bytes they changed and a per-draw census counts what the next draw could keep (TOTK_BIND_INC) vk_exec_bind compares before it writes: VkExec.dirt gets a bit per program, uniform, storage, texture, image and vertex buffer slot that changed, pipe_gen moves with every change of the pipeline state (also depth clamp, raster discard and the target formats), fbl_serial with the feedback-layout images, OV.gen with the uniform overlays. vk_exec_draw_new_cb, copy_state and state_reset mark everything. With TOTK_BIND_INC=on or check, bi_census sorts each prepared draw by what changed since the draw before (nothing, uniform slots only, other slots, another pipeline) and counts the descriptors to rebuild, the generation moves and the shader, pipeline, vertex buffer and feedback keeps; a bind_inc line every 300 frames. No draw is prepared differently yet. Lookout Landing, 4500 draws a frame: nothing changed 8.7%, uniforms only 55.5%, other slots 22.9%, another pipeline 12.8%; 3.4 of 11.2 descriptors a draw to rebuild; uniform overlays move on 2.3% of draws, snapshots 0.1%. Co-Authored-By: Claude Opus 5.5 --- docs/settings.md | 1 + runtime/cfg.c | 2 +- runtime/vk_exec.c | 21 ++- runtime/vk_exec.h | 27 +++- runtime/vk_exec_draw.c | 327 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 352 insertions(+), 26 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index fb381e2a..4b1225e5 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -122,6 +122,7 @@ Many rows below are developer tools that only act together with another setting | Setting | What it does | Effect of changing it | Safety | | --- | --- | --- | --- | +| `bind_inc` | `on` prepares a draw from what changed since the draw before (the bind ops mark the slots whose bytes they changed) instead of from the whole bound state; `check` prepares every draw as before, takes every decision of `on` beside it, logs each kind of difference once and counts them in a `bind_inc:` line every 300 frames; `off` or unset: as before. Both modes also log what changed between draws. | Less work per draw on the render thread; a change the marks missed would leave a stale descriptor, a wrong picture or a lost device, which `check` would show first. | Caution | | `block_views_3d` (default `0`) | `1` gives 3D block-compressed images block views too. | On some AMD cards a GPU write past the end of the image's memory loses the Vulkan device, and the game stops. | Risky | | `deswizzle` (default `auto`) | Where block-linear textures are deswizzled: `auto` or `gpu` uses a compute pass where the texture's storage is an imported pool and the CPU elsewhere, `cpu` always uses the CPU, `check` uses the GPU and also deswizzles every upload on the CPU and compares the two. | `cpu` brings back the long frames on entering new areas (the replay thread deswizzles every texture); `check` waits for the GPU at every upload and is for testing only. The log names the mode in use. | Caution | | `dyn_delta` (default `1`) | Records only the pipeline dynamic states (cull mode, depth and stencil tests, vertex input, blend enables, equations and write masks, ...) whose values changed since the last draw in the command buffer, also across pipeline changes; `0` records the full set at every change of pipeline state, as before; `ab` alternates with the `ab` blocks. | Fewer commands per pipeline change on the render and recorder threads; a driver that loses dynamic state across pipeline binds would draw with stale state. `TOTK_TRACE=perf` prints the calls and commands per submission. | Caution | diff --git a/runtime/cfg.c b/runtime/cfg.c index 9488381b..305db8ea 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -37,7 +37,7 @@ static const char *const known[] = { "BIND_LSTICK_RIGHT", "BIND_LSTICK_UP", "BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_RSTICK_DOWN", "BIND_RSTICK_LEFT", "BIND_RSTICK_RIGHT", "BIND_RSTICK_UP", "BIND_SHAKE", - "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", + "BIND_INC", "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLIMB_FIX", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DESWIZZLE", "DEVICE_PROFILE", "DIALOGUE_CLICK", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 957f1eb3..c2592b48 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -136,6 +136,7 @@ static void state_reset(VkExec *ex) { vk_pipe_state_defaults(&ex->pipe); ex->barrier_at = -1; ex->oq_open = -1; + vk_exec_dirty_all(ex); } // ---- render targets @@ -410,6 +411,8 @@ void vk_exec_noatt_dims(const VkExec *ex, uint32_t *w, uint32_t *h) { static void resolve_rt_as(VkExec *ex, int peek) { if (ex->rt_resolved) return; ex->rt_resolved = 1; + int32_t fmt0[8], ds0[2] = {ex->pipe.depth_fmt, ex->pipe.samples}; + memcpy(fmt0, ex->pipe.color_fmt, sizeof fmt0); const p_set_rt *r = &ex->rt; int n = r->n < 0 ? 0 : r->n > 8 ? 8 : r->n; uint32_t w = UINT32_MAX, h = UINT32_MAX, layers = UINT32_MAX, samples = 0, scale = 1; @@ -451,6 +454,8 @@ static void resolve_rt_as(VkExec *ex, int peek) { ex->pass_ncolor = (uint32_t)n; ex->pass_samples = samples ? samples : 1; ex->pipe.samples = (uint8_t)ex->pass_samples; + if (memcmp(fmt0, ex->pipe.color_fmt, sizeof fmt0) || ds0[0] != ex->pipe.depth_fmt || ds0[1] != ex->pipe.samples) + ex->pipe_gen++; // the pipeline's target formats changed } static void resolve_rt(VkExec *ex) { resolve_rt_as(ex, 0); } @@ -649,6 +654,7 @@ static void barrier_to(VkExec *ex, const VkImage *want, const VkImageAspectFlags for (int i = 0; i < nwant; i++) if (!has_image(ex->fbl_img, ex->nfbl, want[i])) ib[n++] = layout_barrier(want[i], wasp[i], VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT); + if (n || nwant != ex->nfbl) ex->fbl_serial++; // the images in the feedback layout changed for (int i = 0; i < nwant; i++) ex->fbl_img[i] = want[i], ex->fbl_aspect[i] = wasp[i]; ex->nfbl = nwant; if (ex->barrier_at == ex->recorded && !n) return; @@ -967,8 +973,16 @@ static int state_op(VkExec *ex, uint32_t op, const void *a) { memcpy(ex->blend_color, ((const p_blend_color *)a)->rgba, sizeof ex->blend_color); ex->dirty |= VK_EXEC_DIRTY_BLEND_CONST; break; - case OP_DEPTH_CLAMP: ex->pipe.depth_clamp = ((const p_flag *)a)->value != 0; break; - case OP_RASTER_DISCARD: ex->pipe.raster_discard = ((const p_flag *)a)->value != 0; break; + case OP_DEPTH_CLAMP: { + uint8_t v = ((const p_flag *)a)->value != 0; + if (ex->pipe.depth_clamp != v) ex->pipe.depth_clamp = v, ex->pipe_gen++; + break; + } + case OP_RASTER_DISCARD: { + uint8_t v = ((const p_flag *)a)->value != 0; + if (ex->pipe.raster_discard != v) ex->pipe.raster_discard = v, ex->pipe_gen++; + break; + } case OP_PRIM_RESTART: ex->prim_restart = *(const p_prim_restart *)a; break; case OP_COPY_STRIDE: ex->copy_stride = ((const p_copy_stride *)a)->stride; break; case OP_SET_TEX_POOL: ex->tex_pool = ((const p_set_pool *)a)->pool, ex->tex_serial++; break; @@ -1170,6 +1184,7 @@ static void *grow_n(void *p, uint32_t *cap, size_t elem) { static void copy_state(VkExec *dst, const VkExec *src) { memcpy((char *)dst + VK_EXEC_STATE_BEGIN, (const char *)src + VK_EXEC_STATE_BEGIN, VK_EXEC_STATE_END - VK_EXEC_STATE_BEGIN); + vk_exec_dirty_all(dst); // the bound state was replaced: no draw memo may take it for what it held } static int draw_op(uint32_t op) { return op == OP_DRAW || op == OP_DRAW_IDX || op == OP_DRAW_IDX_IND; } @@ -1336,6 +1351,7 @@ static void chunk_record(Rec *r, int k) { const Chunk *next = k + 1 < __atomic_load_n(&SC.nch, __ATOMIC_ACQUIRE) ? &SC.ch[k + 1] : NULL; ex->cb = rec_cb(r, RP.fence); ex->fence = RP.fence, ex->recorded = 0, ex->in_pass = 0, ex->nviews = 0, ex->barrier_at = -1, ex->nfbl = 0; + ex->fbl_serial++; ex->part_start = 0; x_cur = ex; chunk_k = k; @@ -1701,6 +1717,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t ex->nviews = 0; ex->barrier_at = -1; ex->nfbl = 0; // the previous command buffer ended with a full barrier: every image is in GENERAL + ex->fbl_serial++; ex->fr.submits++; ex->split_draws = ex->part_start = 0; ex->part_t0 = kern_now_ns(); diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index bc98dcd1..865d89db 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -1,3 +1,14 @@ +// What the bind ops changed in a context's bound slots since its last prepared draw (VkExec.dirt, TOTK_BIND_INC in +// vk_exec_draw.c): a bit per slot, per NVN stage, set only when the bytes an op writes differ from those it replaces. +typedef struct { + uint32_t prog; // the program pointer or its stage mask + uint32_t ubo[6]; // uniform slots 0..17 (OP_BIND_UBO index) + uint32_t ssbo[6]; // storage slots 0..15 + uint64_t tex[6][2]; // texture handles 0..127 + uint32_t img[6]; // image handles 0..31 + uint32_t vbuf; // vertex buffer ranges 0..15 +} VkExecDirty; + // The op-stream replay of the Vulkan back end. vk_exec.c walks the command // handles of a submission, keeps render targets, passes, clears and fixed-function state; draws, binds and dispatches // (vk_exec_draw.c) and copies, counters, events and conditional rendering (vk_exec_xfer.c) plug in through the two @@ -110,7 +121,7 @@ typedef struct VkExec { const dpool_t *tex_pool, *samp_pool; // OP_SET_TEX_POOL, OP_SET_SAMP_POOL // Pipeline inputs. vk_exec.c writes depth_clamp (OP_DEPTH_CLAMP), raster_discard (OP_RASTER_DISCARD) and, at pass // begin, color_fmt, depth_fmt and samples; vk_exec_draw.c owns every other field (shaders, prim, y_flip, state - // objects, vertex input). + // objects, vertex input). Every write that changes it bumps pipe_gen. VkPipeState pipe; // ---- vk_exec_draw.c slots (draws, binds, dispatch): left empty by vk_exec.c @@ -148,6 +159,14 @@ typedef struct VkExec { const void *fb_fs; // feedback_scan's memo: fragment shader, texture bind serial, pass serial uint32_t fb_serial, fb_pass, fb_sampled; uint32_t tex_serial, pass_serial; // bumped by every texture or image bind and pass begin + // TOTK_BIND_INC (vk_exec_draw.c): the slots the bind ops changed since this context's last prepared draw, cleared + // by that draw; vk_exec_draw_new_cb and copy_state set every bit. pipe_gen: bumped by every write that changes + // pipe (the state object and vertex input binds, OP_DEPTH_CLAMP, OP_RASTER_DISCARD, the target formats of + // resolve_rt, copy_state); an equal value says pipe holds what it held then. fbl_serial: bumped whenever the images + // in the feedback layout (fbl_img, nfbl) may have changed. Outside the copied state: each context counts its own. + VkExecDirty dirt; + uint64_t pipe_gen; + uint32_t fbl_serial; // ---- vk_exec_xfer.c: the occlusion query of the open pass and the queries closed since the last report or reset int oq_open; // slot of the active query, -1 none @@ -195,6 +214,12 @@ void vk_exec_inline_end(int reopen); // vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex // buffers): the replay continues in a new one (vk_sync_split). void vk_exec_draw_new_cb(VkExec *ex); +// Everything a context's next draw reads may differ from its last prepared draw's (VkExec.dirt, pipe_gen, fbl_serial): +// its bound state was replaced or its command buffer state lost. +static inline void vk_exec_dirty_all(VkExec *ex) { + __builtin_memset(&ex->dirt, 0xff, sizeof ex->dirt); + ex->pipe_gen++, ex->fbl_serial++; +} // vk_exec_draw.c: the fragment stage of the draw being recorded samples img through one of its bound combined // textures (OP_BIND_TEX, the handles tex_info swaps; vk_dlaa.c vk_dlaa_skip_draw). Takes the texture lock. int vk_exec_draw_samples(VkExec *ex, const GfxShader *fs, const VkTexImage *img); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index cfd23688..3b53be23 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -124,6 +124,7 @@ typedef struct { VkDescriptorSetLayout ds_last_layout[2]; int ds_pushed[2]; // set number holds the push of DS.psig under ds_layout int res_valid; // the graphics sets bound hold what the inputs in RS give (bind_resources) + int bi_valid; // TOTK_BIND_INC: VkExecDraw.BL holds this command buffer's last prepared draw } CbState; // Contents of CbState.ds_last: the set's packed descriptors (VkDescSlot), which with its layout determine every write. A draw @@ -170,6 +171,7 @@ static struct { int mode; // rt_knob: 0 off, 1 on, 2 ab; -1 unread uint64_t fence; // the submission the lists belong to uint64_t serial; // numbers the overlays, so bind_sig tells one from the next at the same range + uint64_t gen; // bumped by every change of the list o (TOTK_BIND_INC: a binding may read other bytes) uint64_t lo, hi; // the span of the overlays: most bindings lie outside it uint32_t n, np, nf; int f_over; // more foreign writes than f holds: no more overlays in this submission @@ -200,7 +202,18 @@ static uint64_t ov_at(const VkExec *ex, uint64_t addr, uint64_t n) { #define RS_PTRS 4 typedef struct { uint32_t off, len; } RsRange; typedef struct { int st; const GfxShaderRes *res; } RsPtr; -typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; RsRange r[]; } RsList; +// TOTK_BIND_INC: the bound slots a graphics pipeline's descriptors read, per stage (0 vertex, 1 fragment). always: +// descriptors rebuilt on every draw (written storage buffers and those whose address a uniform block holds). +typedef struct { + uint32_t ubo[2]; // uniform slots 3..20 (bit slot - 3: the OP_BIND_UBO index) + uint32_t ssbo[2]; // storage slots read through c0's table + uint64_t tex[2][2]; // texture handle slots 0..127 + uint32_t img[2]; // image handle slots 0..31 + uint8_t c0[2], c1[2]; // reads c0 (every storage slot and the base instance), the compiler constants + uint8_t gfx; // every binding is a vertex or fragment one + uint16_t always, ndesc; +} BiMask; +typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; BiMask m; RsRange r[]; } RsList; // (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the // whole shader code (gfx_shader.c req_init), slow on CPUs without SHA instructions: a field frame looks up ~3000 and // the 2048-slot direct-mapped table this replaced missed 3% of them on conflicts. @@ -268,6 +281,28 @@ typedef struct { int valid; VkPipeState s; const VkPipeInfo *info; } PipeMemo; #define PC_N 1024 typedef struct { const VkPipeInfo *info; VkPipeState s; } PipeCache; +// TOTK_BIND_INC (section before prepare_draw): the last prepared graphics draw of the command buffer, as far as the +// next draw's decisions compare it (valid with CbState.bi_valid), and the census of those decisions. +#define BI_MV 10 +typedef struct { + const VkPipeInfo *info; + const prog_t *prog; + uint64_t serial, pipe_gen; + uint64_t gt[4], gb[4]; // bi_gt, bi_gb before its build + uint32_t binst, slots_gen, pass_serial, tex_serial; + const GfxShader *vs, *fs; + uint8_t sx[4]; // the state fields a draw sets: prim, restart, feedback, depth_minus_w + uint32_t vmask; + uint64_t vgen; +} BiLast; +typedef struct { + uint64_t frame; // gfx_counters.frames at the last draw counted + uint32_t frames; // frames since the last line + uint64_t draws, cls[4], b_gen, desc, rec[4], always, mv[BI_MV], slots[5]; + uint64_t sh_keep, sh_memo, fb_n, fb_keep, fb_memo, vb_skip, pg_hit; + uint64_t mism, mism_total; +} BiCensus; + // The memos and command buffer state of one recording context (VkExec.draw). Nothing in it is shared: the context's // thread reads and writes it without a lock. typedef struct VkExecDraw { @@ -277,6 +312,8 @@ typedef struct VkExecDraw { DescSigs DS; BindStats BS; RsState RS; + BiLast BL; + BiCensus BC; nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown const VkPipeInfo *rs_last; // the last lookup const RsList *rs_last_l; @@ -331,6 +368,7 @@ static void cb_done(void *arg) { // completion thread // Also after the bound state was replaced (a chunk's start): bind_sig compares the bound slots again. void vk_exec_draw_new_cb(VkExec *ex) { + vk_exec_dirty_all(ex); if (!ex->draw) return; memset(&ex->draw->B, 0, sizeof ex->draw->B); ex->draw->bind_gen++; @@ -1945,6 +1983,42 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si else r[(*n)++] = (RsRange){off, (uint32_t)len}; } +static void bi_mask(const VkPipeInfo *info, BiMask *m) { + memset(m, 0, sizeof *m); + m->gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS; + for (uint32_t i = 0; i < info->n_bindings; i++) { + const VkPipeBinding *b = &info->bindings[i]; + const GfxShaderRes *r = b->res; + uint32_t cnt = b->count ? b->count : 1; + int st = nvn_stage_of(b->stage); + if (st != NVN_STAGE_VS && st != NVN_STAGE_FS && b->kind != VK_PIPE_SUPPORT) { + m->gfx = 0; + continue; + } + m->ndesc = (uint16_t)(m->ndesc + cnt); + if (b->kind == GFX_RES_UBO) { + for (uint32_t k = 0; k < cnt; k++) { + int slot = r->slot + (int)k; + if (slot == 0) m->c0[st] = 1; + else if (slot == 1) m->c1[st] = 1; + else if (slot >= 3 && slot - 3 < 18) m->ubo[st] |= 1u << (slot - 3); + } + } else if (b->kind == GFX_RES_SSBO) { + int ubo, slot = ssbo_slot(r, st, &ubo); + if ((r->flags & 1) || slot < 0 || slot >= 16 || ubo) m->always = (uint16_t)(m->always + cnt); + else m->ssbo[st] |= 1u << slot; + } else if ((b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER && + b->type != VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) { + int img = b->kind == GFX_RES_IMG; + for (uint32_t k = 0; k < cnt; k++) { + int slot = r->slot + (int)k; + if (img && slot >= 0 && slot < 32) m->img[st] |= 1u << slot; + else if (!img && slot >= 0 && slot < 128) m->tex[st][slot >> 6] |= 1ull << (slot & 63); + } + } + } +} + static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { VkExecDraw *D = ex->draw; if (info == D->rs_last) return D->rs_last_l; @@ -1989,11 +2063,14 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { } uint64_t bytes = 0; for (uint32_t i = 0; i < n; i++) bytes += tmp[i].len; + BiMask m; + bi_mask(info, &m); if (bytes > RS_BYTES) ok = 0; l = malloc(sizeof *l + sizeof *tmp * n); if (!l) rt_fatal("vk: out of memory"); l->n = n, l->bytes = (uint32_t)bytes, l->sup = (uint8_t)sup, l->c0 = (uint8_t)c0, l->ok = (uint8_t)ok; l->np = (uint8_t)np; + l->m = m; memcpy(l->p, ptr, sizeof *ptr * np); memcpy(l->r, tmp, sizeof *tmp * n); nvn_hm_put(&D->rs_lists, &info, l); @@ -2462,6 +2539,165 @@ static int pipe_memo_on(void) { return on; } +// ---- incremental draw preparation (TOTK_BIND_INC) +// A draw is prepared from what changed since the context's last prepared draw in the same command buffer: the bind +// ops mark the slots whose bytes they changed (VkExec.dirt, vk_exec_bind), every write that changes the pipeline state +// bumps VkExec.pipe_gen, and the global inputs are read as two groups of generations, G_tex (bi_gt: texture slots, +// samplers, the submission's storage epoch, the images in the feedback layout, DLSS's swap) and G_buf (bi_gb: pool +// index and mirrors, uniform snapshots, uniform overlays, the submission). A step whose inputs are all unchanged keeps +// the last draw's result. A new command buffer or a replaced bound state (vk_exec_draw_new_cb, copy_state) sets every +// mark, the CbState memset drops BL, and a chunk of a chunked replay never keeps anything (its slots resolve per op). +// off: as before. on: the kept results. check: prepares every draw as before, takes every decision of on beside it, +// compares what on would have kept with what the full path gave, logs each kind of difference once with its marks and +// counts them. Both modes count the decisions (bi_census) and log them every 300 frames. +static int bind_inc_mode(void) { // 0 off, 1 on, 2 check + static int mode = -1; + if (mode < 0) { + const char *e = cfg_get("TOTK_BIND_INC"); + int m = !e || !*e || !strcmp(e, "off") || !strcmp(e, "0") ? 0 : !strcmp(e, "check") ? 2 : 1; + if (m) rt_log("vk_exec: incremental draw preparation (TOTK_BIND_INC) %s", m == 2 ? "check" : "on"); + mode = m; + } + return mode; +} + +static void bi_gt(const VkExec *ex, uint64_t g[4]) { + g[0] = (uint64_t)vk_texture_slots_gen() << 32 | vk_sampler_gen(); + g[1] = (uint64_t)vk_texture_epoch() << 32 | ex->fbl_serial; + g[2] = (uintptr_t)vk_dlaa_swap_img; + int swap = vk_dlaa_swap_img && ex->in_pass && ex->pass_w > vk_dlaa_swap_img->width + 1; // tex_info's swap test + g[3] = swap ? (uint64_t)(uintptr_t)vk_dlaa_swap_view | 1 : 0; +} + +static void bi_gb(const VkExec *ex, uint64_t g[4]) { + g[0] = vk_mem_gen(), g[1] = vk_usnap_gen(), g[2] = OV.gen, g[3] = ex->fence; +} + +static double bi_pct(uint64_t n, uint64_t of) { return of ? 100.0 * (double)n / (double)of : 0; } + +static void bi_log(BiCensus *c) { + double fr = c->frames ? (double)c->frames : 1, d = c->draws ? (double)c->draws : 1; + rt_log("bind_inc: %u frames, %.0f draws a frame: nothing changed %.1f%%, uniforms only %.1f%% (%.1f%% of draws with a " + "buffer generation moved), other slots %.1f%%, another pipeline or nothing kept %.1f%%; descriptors %.2f a draw, " + "to rebuild per draw of each class %.2f %.2f %.2f %.2f, always %.2f", + c->frames, d / fr, bi_pct(c->cls[0], c->draws), bi_pct(c->cls[1], c->draws), bi_pct(c->b_gen, c->draws), + bi_pct(c->cls[2], c->draws), bi_pct(c->cls[3], c->draws), c->desc / d, + c->cls[0] ? c->rec[0] / (double)c->cls[0] : 0, c->cls[1] ? c->rec[1] / (double)c->cls[1] : 0, + c->cls[2] ? c->rec[2] / (double)c->cls[2] : 0, c->cls[3] ? c->rec[3] / (double)c->cls[3] : 0, c->always / d); + rt_log("bind_inc: moved since the draw before (%% of draws): slots %.1f, samplers %.1f, epoch %.1f, feedback layout " + "%.1f, dlss swap %.1f, pools %.1f, uniform snapshots %.1f, overlays %.1f, submission %.1f, pass %.1f; changed " + "slots read per draw: uniform %.2f, storage %.2f, texture %.2f, image %.2f, vertex %.2f", + bi_pct(c->mv[0], c->draws), bi_pct(c->mv[1], c->draws), bi_pct(c->mv[2], c->draws), bi_pct(c->mv[3], c->draws), + bi_pct(c->mv[4], c->draws), bi_pct(c->mv[5], c->draws), bi_pct(c->mv[6], c->draws), bi_pct(c->mv[7], c->draws), + bi_pct(c->mv[8], c->draws), bi_pct(c->mv[9], c->draws), c->slots[0] / d, c->slots[1] / d, c->slots[2] / d, + c->slots[3] / d, c->slots[4] / d); + rt_log("bind_inc: kept (%% of draws): shaders %.1f (memo today %.1f), pipeline state %.1f, vertex buffers %.1f, " + "feedback scan %.1f of %.1f (today %.1f); mismatches %.2f a frame, %llu in all", + bi_pct(c->sh_keep, c->draws), bi_pct(c->sh_memo, c->draws), bi_pct(c->pg_hit, c->draws), + bi_pct(c->vb_skip, c->draws), bi_pct(c->fb_keep, c->draws), bi_pct(c->fb_n, c->draws), + bi_pct(c->fb_memo, c->draws), c->mism / fr, (unsigned long long)c->mism_total); + uint64_t total = c->mism_total; + memset(c, 0, sizeof *c); + c->mism_total = total; +} + +static int popc64(uint64_t v) { return __builtin_popcountll(v); } + +// Counts what the draw about to be bound (state s, pipeline info) could keep from the last prepared draw, before +// bind_resources; gt and gb were read right before it, slots_gen before shader_for. sm_hit: today's shader memo hit. +static void bi_census(VkExec *ex, const VkPipeInfo *info, const VkPipeState *s, const uint64_t gt[4], + const uint64_t gb[4], uint32_t slots_gen, int sm_hit) { + VkExecDraw *D = ex->draw; + BiCensus *c = &D->BC; + const BiLast *L = &D->BL; + uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (f != c->frame) { + if (c->frames >= 300) bi_log(c); + c->frame = f, c->frames++; + } + c->draws++; + c->sh_memo += sm_hit; + const RsList *l = rs_list(ex, info); + const BiMask *m = &l->m; + const VkExecDirty *dt = &ex->dirt; + if (!D->B.bi_valid || ex->chunked) { + c->cls[3]++, c->desc += m->ndesc, c->rec[3] += m->ndesc, c->always += m->always; + return; + } + int mv[BI_MV] = {(uint32_t)(gt[0] >> 32) != (uint32_t)(L->gt[0] >> 32), (uint32_t)gt[0] != (uint32_t)L->gt[0], + (uint32_t)(gt[1] >> 32) != (uint32_t)(L->gt[1] >> 32), (uint32_t)gt[1] != (uint32_t)L->gt[1], + gt[2] != L->gt[2] || gt[3] != L->gt[3], gb[0] != L->gb[0], gb[1] != L->gb[1], gb[2] != L->gb[2], + gb[3] != L->gb[3], ex->pass_serial != L->pass_serial}; + for (int i = 0; i < BI_MV; i++) c->mv[i] += (uint64_t)mv[i]; + int tmv = mv[0] || mv[1] || mv[2] || mv[3] || mv[4], bmv = mv[5] || mv[6] || mv[7] || mv[8]; + int prog = dt->prog || ex->prog != L->prog || ex->prog->serial != L->serial; + int binst = D->c0_binst != L->binst; + uint32_t du = 0, ds = 0, di = 0, c0d = 0; + int ndt = 0; + for (int k = 0; k < 2; k++) { + du += (uint32_t)__builtin_popcount(dt->ubo[k] & m->ubo[k]); + ds += (uint32_t)__builtin_popcount(dt->ssbo[k] & m->ssbo[k]); + di += (uint32_t)__builtin_popcount(dt->img[k] & m->img[k]); + ndt += popc64(dt->tex[k][0] & m->tex[k][0]) + popc64(dt->tex[k][1] & m->tex[k][1]); + c0d += m->c0[k] && (dt->ssbo[k] || binst); + } + c->slots[0] += du, c->slots[1] += ds, c->slots[2] += (uint64_t)ndt, c->slots[3] += di; + c->slots[4] += (uint64_t)__builtin_popcount(dt->vbuf & D->B.vmask); + int same = L->info == info && m->gfx; + int cls = !same ? 3 : !prog && !tmv && !bmv && !du && !ds && !di && !ndt && !c0d ? 0 + : !prog && !tmv && !ds && !di && !ndt && !c0d ? 1 : 2; + c->cls[cls]++; + if (cls == 1 && bmv) c->b_gen++; + c->desc += m->ndesc; + c->always += m->always; + uint32_t rec = m->ndesc; + if (same) { + int nubo = 0, nssbo = 0, ntex = 0, nimg = 0; + for (int k = 0; k < 2; k++) + nubo += __builtin_popcount(m->ubo[k]), nssbo += __builtin_popcount(m->ssbo[k]), + nimg += __builtin_popcount(m->img[k]), ntex += popc64(m->tex[k][0]) + popc64(m->tex[k][1]); + rec = m->always + c0d + (uint32_t)(prog || bmv) * (uint32_t)(m->c1[0] + m->c1[1]) + + (bmv ? (uint32_t)(nubo + nssbo) : du + ds) + (tmv ? (uint32_t)(ntex + nimg) : (uint32_t)ndt + di); + } + c->rec[cls] += rec; + // Step candidates: shaders (the masked slots of both stages), pipeline state (the generation and the per-draw + // fields), vertex buffers (the streams the state reads), feedback scan (the fragment stage's textures). + if (!dt->prog && ex->prog == L->prog && ex->prog->serial == L->serial && ex->pipe_gen == L->pipe_gen && + slots_gen == L->slots_gen) { + int keep = 1; + for (int k = 0; k < 2 && keep; k++) { + uint32_t sm = slot_mask_of(D, ex->prog->serial, k); + keep = !((uint32_t)dt->tex[k][0] & sm) && !(dt->img[k] & sm); + } + c->sh_keep += (uint64_t)keep; + } + uint8_t sx[4] = {s->prim, s->prim_restart, s->feedback, s->depth_minus_w}; + c->pg_hit += ex->pipe_gen == L->pipe_gen && s->vs == L->vs && s->fs == L->fs && !memcmp(sx, L->sx, 4); + c->vb_skip += !(dt->vbuf & D->B.vmask) && D->B.vmask == L->vmask && vk_mem_gen() == L->vgen && + ex->pipe_gen == L->pipe_gen && D->B.vb_valid; + if (s->fs && ex->in_pass) { + c->fb_n++; + int fsame = s->fs == L->fs && ex->pass_serial == L->pass_serial; + c->fb_keep += fsame && !(dt->tex[1][0] & m->tex[1][0]) && !(dt->tex[1][1] & m->tex[1][1]) && slots_gen == L->slots_gen; + c->fb_memo += fsame && ex->tex_serial == L->tex_serial; + } +} + +// A graphics draw was prepared: it becomes the one the next draw compares with, and the marks start again. +static void bi_done(VkExec *ex, const VkPipeInfo *info, const VkPipeState *s, const uint64_t gt[4], const uint64_t gb[4], + uint32_t slots_gen) { + VkExecDraw *D = ex->draw; + BiLast *L = &D->BL; + L->info = info, L->prog = ex->prog, L->serial = ex->prog->serial, L->pipe_gen = ex->pipe_gen; + memcpy(L->gt, gt, sizeof L->gt), memcpy(L->gb, gb, sizeof L->gb); + L->binst = D->c0_binst, L->slots_gen = slots_gen, L->pass_serial = ex->pass_serial, L->tex_serial = ex->tex_serial; + L->vs = s->vs, L->fs = s->fs; + L->sx[0] = s->prim, L->sx[1] = s->prim_restart, L->sx[2] = s->feedback, L->sx[3] = s->depth_minus_w; + L->vmask = D->B.vmask, L->vgen = D->B.vgen; + memset(&ex->dirt, 0, sizeof ex->dirt); + D->B.bi_valid = 1; +} + // Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped. static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { VkExecDraw *D = ex->draw; @@ -2483,8 +2719,12 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { const GfxShader *vs, *fs; // Before shader_for, whose lookups may bump it. In a chunk per pass too, as in bind_sig. uint32_t gen = memo ? vk_texture_slots_gen() + (ex->chunked ? ex->pass_serial : 0) : 0; + int bim = bind_inc_mode(), sm_hit = 0; + uint64_t bgt[4], bgb[4]; // TOTK_BIND_INC: G_tex and G_buf right before the descriptors are built + uint32_t bsg = bim ? vk_texture_slots_gen() : 0; // the shaders' slot generation, before shader_for if (memo && D->SM.valid && memo_match(ex, gen, &D->SM.k)) { vs = D->SM.vs, fs = D->SM.fs, has_vs = 1, has_fs = D->SM.has_fs; + sm_hit = 1; if (memo == 2) { int hv, hf, wv, wf; const GfxShader *v2 = shader_for(ex, NVN_STAGE_VS, &hv, &wv, 0), *f2 = shader_for(ex, NVN_STAGE_FS, &hf, &wf, 0); @@ -2607,6 +2847,7 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (mo) plat_mutex_unlock(&mo_lock); record_fixed(ex); repeat_stats(ex, info); + if (bim) bi_gt(ex, bgt), bi_gb(ex, bgb), bi_census(ex, info, &s, bgt, bgb, bsg, sm_hit); bind_resources(ex, info); if (fbl & ex->fb_dirty) feedback_barrier(ex); feedback_written(ex, &s); @@ -2614,6 +2855,7 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (vk_dump_armed) vk_dump_draw(ex, &s, p, info); if (vk_hdr_track) vk_hdr_draw(ex, info); vk_exec_oq_draw(ex); + if (bim) bi_done(ex, info, &s, bgt, bgb, bsg); return 1; } @@ -3619,6 +3861,7 @@ static void ov_sub(const VkExec *ex) { // the submission being replayed rt_log("vk: %u overlaid uniform updates of submission %llu were never copied into their buffers", OV.np, (unsigned long long)OV.fence); OV.fence = ex->fence, OV.n = OV.np = OV.nf = 0, OV.f_over = 0, OV.lo = UINT64_MAX, OV.hi = 0; + OV.gen++; } static void ov_span(void) { @@ -3645,6 +3888,7 @@ static void ov_add(uint64_t lo, uint64_t hi, VkBuffer b, VkDeviceSize o, const V break; } OV.o[OV.n++] = (typeof(OV.o[0])){lo, hi, ++OV.serial, b, o}; + OV.gen++; OV.p[OV.np++] = (typeof(OV.p[0])){lo, b, o, *sp, hi - lo}; if (lo < OV.lo) OV.lo = lo; if (hi > OV.hi) OV.hi = hi; @@ -3666,6 +3910,7 @@ static void ov_retire(uint64_t lo, uint64_t hi) { if (!(lo < OV.o[i].hi && hi > OV.o[i].lo)) OV.o[k++] = OV.o[i]; if (k == OV.n) return; OV.n = k; + OV.gen++; ov_span(); } @@ -3709,6 +3954,7 @@ void vk_exec_ub_flush(VkExec *ex) { ex->recorded++; } OV.np = 0; + OV.gen++; // The full barrier barrier_to records next makes the copied bytes visible: later bindings read the real buffer, // as after an update copied at once, and need no overlay lookups or merged copies (ov_at answers 0 for them, so // the descriptor build skip rebuilds the draws that bound an overlay) @@ -3832,21 +4078,48 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { // ---- bind ops -static void bind_handles(uint64_t *dst, int cap, const p_bind_handles *b) { +// Writes the handles of b into dst; returns the slots whose handle changed (bit i: slot b->first + i). +static uint32_t bind_handles(uint64_t *dst, int cap, const p_bind_handles *b) { if (b->stage < 0 || b->stage >= 6 || b->first < 0 || b->n < 0 || b->n > 32 || b->first + b->n > cap) { warn_once(W_BIND, "a bind of %d handles at %d of stage %d is out of range: ignored", b->n, b->first, b->stage); - return; + return 0; } - memcpy(&dst[b->first], b->h, sizeof(uint64_t) * (size_t)b->n); + uint32_t chg = 0; + for (int i = 0; i < b->n; i++) + if (dst[b->first + i] != b->h[i]) dst[b->first + i] = b->h[i], chg |= 1u << i; + return chg; +} + +// Writes n bytes at dst; 1 when they differ from what they replace. +static int put_changed(void *dst, const void *src, size_t n) { + if (!memcmp(dst, src, n)) return 0; + memcpy(dst, src, n); + return 1; +} + +// A write of n bytes of ex->pipe; one that changes them bumps ex->pipe_gen, so a state object bound again with the same +// values leaves the draws after it on the generation's fast path. +static void pipe_put(VkExec *ex, void *dst, const void *src, size_t n) { + if (put_changed(dst, src, n)) ex->pipe_gen++; +} + +// The changed handle slots chg of a bind at first into a 128-bit dirty mask. +static void dirty_slots(uint64_t m[2], int first, uint32_t chg) { + if (!chg) return; + uint64_t c = (uint64_t)chg; + if (first < 64) m[0] |= c << first, m[1] |= first ? c >> (64 - first) : 0; + else m[1] |= c << (first - 64); } // The bind ops: writes into ex alone, so the state scan (vk_exec.c) applies them too. 1 when the op wrote ex->prog, -// ubo, ssbo, tex or img (VkExecDraw.bind_gen). +// ubo, ssbo, tex or img (VkExecDraw.bind_gen). Each op marks in ex->dirt the slots whose bytes it changed and bumps +// ex->pipe_gen when it changed pipe (TOTK_BIND_INC); a rebind of the same values marks nothing. int vk_exec_bind(VkExec *ex, uint32_t op, const void *payload) { int gen = 0; switch (op) { case OP_BIND_PROGRAM: { const p_bind_program *b = payload; + if (ex->prog != b->prog || ex->prog_stages != b->stages) ex->dirt.prog = 1; ex->prog = b->prog, ex->prog_stages = b->stages; gen = 1; break; @@ -3857,64 +4130,74 @@ int vk_exec_bind(VkExec *ex, uint32_t op, const void *payload) { switch (b->kind) { case 0: memcpy(&bs, b->bytes, sizeof bs); - if (bs.target < 8) ex->pipe.blend[bs.target] = bs; + if (bs.target < 8) pipe_put(ex, &ex->pipe.blend[bs.target], &bs, sizeof bs); break; - case 1: memcpy(&ex->pipe.color, b->bytes, sizeof ex->pipe.color); break; - case 2: memcpy(&ex->pipe.chmask, b->bytes, sizeof ex->pipe.chmask); break; - case 3: memcpy(&ex->pipe.ms, b->bytes, sizeof ex->pipe.ms); break; - case 4: memcpy(&ex->pipe.poly, b->bytes, sizeof ex->pipe.poly); break; - case 5: memcpy(&ex->pipe.ds, b->bytes, sizeof ex->pipe.ds); break; - case 6: memcpy(&ex->point_size, b->bytes, sizeof ex->point_size); break; + case 1: pipe_put(ex, &ex->pipe.color, b->bytes, sizeof ex->pipe.color); break; + case 2: pipe_put(ex, &ex->pipe.chmask, b->bytes, sizeof ex->pipe.chmask); break; + case 3: pipe_put(ex, &ex->pipe.ms, b->bytes, sizeof ex->pipe.ms); break; + case 4: pipe_put(ex, &ex->pipe.poly, b->bytes, sizeof ex->pipe.poly); break; + case 5: pipe_put(ex, &ex->pipe.ds, b->bytes, sizeof ex->pipe.ds); break; + case 6: memcpy(&ex->point_size, b->bytes, sizeof ex->point_size); break; // a support input: compared per draw } break; } case OP_BIND_VATTR: { const p_bind_vattr *b = payload; if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break; - memset(ex->pipe.vattr, 0, sizeof ex->pipe.vattr); // MNG: the new list replaces the old one - memcpy(&ex->pipe.vattr[b->first], b->st, sizeof(uint32_t) * (size_t)b->n); + uint32_t v[16] = {0}; // MNG: the new list replaces the old one + memcpy(&v[b->first], b->st, sizeof(uint32_t) * (size_t)b->n); + pipe_put(ex, ex->pipe.vattr, v, sizeof v); break; } case OP_BIND_VSTREAM: { const p_bind_vstream *b = payload; if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break; - memset(ex->pipe.vstream, 0, sizeof ex->pipe.vstream); - memcpy(&ex->pipe.vstream[b->first], b->st, sizeof(vs_st) * (size_t)b->n); + vs_st v[16]; + memset(v, 0, sizeof v); + memcpy(&v[b->first], b->st, sizeof(vs_st) * (size_t)b->n); + pipe_put(ex, ex->pipe.vstream, v, sizeof v); break; } case OP_BIND_VBUF: { const p_bind_vbuf *b = payload; if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break; - memcpy(&ex->vbuf[b->first], b->r, sizeof(NVNbufferRange) * (size_t)b->n); + for (int i = 0; i < b->n; i++) + if (put_changed(&ex->vbuf[b->first + i], &b->r[i], sizeof(NVNbufferRange))) ex->dirt.vbuf |= 1u << (b->first + i); break; } case OP_BIND_UBO: { const p_bind_buf *b = payload; - if (b->stage >= 0 && b->stage < 6 && b->index >= 0 && b->index < 18) ex->ubo[b->stage][b->index] = *b; + if (b->stage >= 0 && b->stage < 6 && b->index >= 0 && b->index < 18 && + put_changed(&ex->ubo[b->stage][b->index], b, sizeof *b)) + ex->dirt.ubo[b->stage] |= 1u << b->index; gen = 1; break; } case OP_BIND_SSBO: { const p_bind_buf *b = payload; - if (b->stage >= 0 && b->stage < 6 && b->index >= 0 && b->index < 16) ex->ssbo[b->stage][b->index] = *b; + if (b->stage >= 0 && b->stage < 6 && b->index >= 0 && b->index < 16 && + put_changed(&ex->ssbo[b->stage][b->index], b, sizeof *b)) + ex->dirt.ssbo[b->stage] |= 1u << b->index; gen = 1; break; } case OP_BIND_TEX: { const p_bind_handles *b = payload; - bind_handles(b->stage >= 0 && b->stage < 6 ? ex->tex[b->stage] : NULL, 128, b); + uint32_t chg = bind_handles(b->stage >= 0 && b->stage < 6 ? ex->tex[b->stage] : NULL, 128, b); + if (chg) dirty_slots(ex->dirt.tex[b->stage], b->first, chg); ex->tex_serial++; gen = 1; break; } - case OP_BIND_SAMP: { + case OP_BIND_SAMP: { // not read by the bind path: a texture handle carries its sampler id (h >> 20) const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 6 ? ex->samp[b->stage] : NULL, 128, b); break; } case OP_BIND_IMG: { const p_bind_handles *b = payload; - bind_handles(b->stage >= 0 && b->stage < 6 ? ex->img[b->stage] : NULL, 32, b); + uint32_t chg = bind_handles(b->stage >= 0 && b->stage < 6 ? ex->img[b->stage] : NULL, 32, b); + if (chg) ex->dirt.img[b->stage] |= chg << b->first; ex->tex_serial++; // the shader memo's handle check (feedback_scan reads only textures: a rescan at most) gen = 1; break; From ff3a95d0f1734635155ca090452bb11fddfed3d4 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 13 Nov 2026 23:42:11 +0300 Subject: [PATCH 02/10] A draw whose pipeline state no op changed takes the previous draw's pipeline and state without building and comparing them (TOTK_BIND_INC) From task/fastrep 18eab597 (TOTK_PIPE_GEN), now under TOTK_BIND_INC: the pipeline memo keeps the pipe_gen its state was built under; a draw with that generation and the same shaders, primitive, restart, feedback flags and depth mode takes the memo's state and pipeline and skips the three 360-byte compares (memo, cache entry, recorded state). The shader memo skips its compare of pipe the same way. check builds and compares as before and counts a state the generation called unchanged that differs (bi_mismatch logs the first of each kind with its marks and generations). The census runs only in check mode. Lookout Landing check, 60 s: 67% of draws keep the state, 0 mismatches. Co-Authored-By: Claude Opus 5.5 --- docs/settings.md | 2 +- runtime/vk_exec_draw.c | 145 ++++++++++++++++++++++++++++++++--------- 2 files changed, 117 insertions(+), 30 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 4b1225e5..005cc347 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -122,7 +122,7 @@ Many rows below are developer tools that only act together with another setting | Setting | What it does | Effect of changing it | Safety | | --- | --- | --- | --- | -| `bind_inc` | `on` prepares a draw from what changed since the draw before (the bind ops mark the slots whose bytes they changed) instead of from the whole bound state; `check` prepares every draw as before, takes every decision of `on` beside it, logs each kind of difference once and counts them in a `bind_inc:` line every 300 frames; `off` or unset: as before. Both modes also log what changed between draws. | Less work per draw on the render thread; a change the marks missed would leave a stale descriptor, a wrong picture or a lost device, which `check` would show first. | Caution | +| `bind_inc` | `on` prepares a draw from what changed since the draw before (the bind ops mark the slots whose bytes they changed) instead of from the whole bound state; `check` prepares every draw as before, takes every decision of `on` beside it, logs each kind of difference once and counts them in a `bind_inc:` line every 300 frames; `off` or unset: as before. `check` also counts what changed between draws in that line. | Less work per draw on the render thread; a change the marks missed would leave a stale descriptor, a wrong picture or a lost device, which `check` would show first. | Caution | | `block_views_3d` (default `0`) | `1` gives 3D block-compressed images block views too. | On some AMD cards a GPU write past the end of the image's memory loses the Vulkan device, and the game stops. | Risky | | `deswizzle` (default `auto`) | Where block-linear textures are deswizzled: `auto` or `gpu` uses a compute pass where the texture's storage is an imported pool and the CPU elsewhere, `cpu` always uses the CPU, `check` uses the GPU and also deswizzles every upload on the CPU and compares the two. | `cpu` brings back the long frames on entering new areas (the replay thread deswizzles every texture); `check` waits for the GPU at every upload and is for testing only. The log names the mode in use. | Caution | | `dyn_delta` (default `1`) | Records only the pipeline dynamic states (cull mode, depth and stencil tests, vertex input, blend enables, equations and write masks, ...) whose values changed since the last draw in the command buffer, also across pipeline changes; `0` records the full set at every change of pipeline state, as before; `ab` alternates with the `ab` blocks. | Fewer commands per pipeline change on the render and recorder threads; a driver that loses dynamic state across pipeline binds would draw with stale state. `TOTK_TRACE=perf` prints the calls and commands per submission. | Caution | diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 3b53be23..a522dc64 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -97,6 +97,8 @@ typedef struct { VkPipeline pipe[2]; // bound per bind point: 0 graphics, 1 compute VkPipeState st; // the state vk_pipeline_set_dynamic last recorded int st_valid; + int st_pm; // st equals PipeMemo.s (set where both were the draw's state, cleared at + // every write of PipeMemo): TOTK_BIND_INC skips the compare of st int dyn_valid; // vk_pipeline_set_dynamic_delta's record of this command buffer's dynamic state // holds (TOTK_DYN_DELTA); pipeline binds keep it, the memset clears it uint32_t vmask; // the vertex bindings st's vertex input reads (valid with st_valid) @@ -269,13 +271,15 @@ typedef struct { const prog_t *prog; uint64_t serial; uint32_t gen, tex_serial; + uint64_t pipe_gen; // VkExec.pipe_gen when pipe was copied uint64_t tex[2][32], img[2][32]; VkPipeState pipe; } MemoKey; typedef struct { int valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } ShaderMemo; -// The previous draw's pipeline and the state it was found for (prepare_draw). -typedef struct { int valid; VkPipeState s; const VkPipeInfo *info; } PipeMemo; +// The previous draw's pipeline and the state it was found for (prepare_draw). gen: VkExec.pipe_gen when s was built +// from VkExec.pipe. +typedef struct { int valid; VkPipeState s; const VkPipeInfo *info; uint64_t gen; } PipeMemo; // The pipelines found before, by state: vk_pipeline_graphics_async takes the pipeline cache's lock, which the contexts // of a chunked replay would take on every draw whose state differs from the draw before. Direct-mapped. #define PC_N 1024 @@ -300,8 +304,12 @@ typedef struct { uint32_t frames; // frames since the last line uint64_t draws, cls[4], b_gen, desc, rec[4], always, mv[BI_MV], slots[5]; uint64_t sh_keep, sh_memo, fb_n, fb_keep, fb_memo, vb_skip, pg_hit; - uint64_t mism, mism_total; + uint64_t mism, mism_total, checked; + uint32_t mism_frame, frame_lines; } BiCensus; +enum { BIM_PIPE, BIM_DYN, BIM_SHADER, BIM_FB, BIM_VB, BIM_DESC, BIM_SET, BIM_N }; // check mode's kinds of difference +static int bind_inc_mode(void); +static void bi_mismatch(struct VkExec *ex, int kind, const char *fmt, ...); // The memos and command buffer state of one recording context (VkExec.draw). Nothing in it is shared: the context's // thread reads and writes it without a lock. @@ -2508,14 +2516,25 @@ static int pipe_skip_mode(void) { static void memo_key(const VkExec *ex, uint32_t gen, MemoKey *k) { k->prog = ex->prog, k->serial = ex->prog->serial, k->gen = gen, k->tex_serial = ex->tex_serial; + k->pipe_gen = ex->pipe_gen; memcpy(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]), memcpy(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]); memcpy(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]), memcpy(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]); memcpy(&k->pipe, &ex->pipe, sizeof k->pipe); } -static int memo_match(const VkExec *ex, uint32_t gen, MemoKey *k) { - if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen || memcmp(&k->pipe, &ex->pipe, sizeof k->pipe)) +static int bind_inc_mode(void); +static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...); + +// TOTK_BIND_INC: pipe equals the key's while pipe_gen does (no write changed it since the copy), so the 360-byte +// compare is skipped; check compares anyway and counts a difference the generation missed. +static int memo_match(VkExec *ex, uint32_t gen, MemoKey *k) { + if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen) return 0; + int bim = bind_inc_mode(), same_gen = k->pipe_gen == ex->pipe_gen; + if (!(bim == 1 && same_gen && !ex->chunked) && memcmp(&k->pipe, &ex->pipe, sizeof k->pipe)) { + if (bim == 2 && same_gen && !ex->chunked) bi_mismatch(ex, BIM_PIPE, "shader memo: bound state changed without its generation"); return 0; + } + k->pipe_gen = ex->pipe_gen; if (k->tex_serial == ex->tex_serial) return 1; if (memcmp(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]) || memcmp(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]) || memcmp(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]) || memcmp(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1])) @@ -2549,7 +2568,7 @@ static int pipe_memo_on(void) { // mark, the CbState memset drops BL, and a chunk of a chunked replay never keeps anything (its slots resolve per op). // off: as before. on: the kept results. check: prepares every draw as before, takes every decision of on beside it, // compares what on would have kept with what the full path gave, logs each kind of difference once with its marks and -// counts them. Both modes count the decisions (bi_census) and log them every 300 frames. +// counts them; it also counts the decisions (bi_census) and logs them every 300 frames. static int bind_inc_mode(void) { // 0 off, 1 on, 2 check static int mode = -1; if (mode < 0) { @@ -2573,6 +2592,40 @@ static void bi_gb(const VkExec *ex, uint64_t g[4]) { g[0] = vk_mem_gen(), g[1] = vk_usnap_gen(), g[2] = OV.gen, g[3] = ex->fence; } +// check mode: a decision of on would have kept a result that differs from the full path's. Counted; the first of each +// kind is logged with the marks, generations and counters it was taken under. +static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...) { + static const char *const name[BIM_N] = {"pipeline state", "recorded dynamic state", "shaders", "feedback scan", + "vertex buffers", "descriptor", "descriptor set"}; + static volatile int32_t seen[BIM_N]; + VkExecDraw *D = ex->draw; + D->BC.mism++, D->BC.mism_total++, D->BC.mism_frame++; + if (__atomic_exchange_n(&seen[kind], 1, __ATOMIC_RELAXED)) return; + char msg[320]; + va_list ap; + va_start(ap, fmt); + vsnprintf(msg, sizeof msg, fmt, ap); + va_end(ap); + const VkExecDirty *d = &ex->dirt; + const BiLast *L = &D->BL; + uint64_t gt[4], gb[4]; + bi_gt(ex, gt), bi_gb(ex, gb); + rt_log("bind_inc check: kept %s differs from the full path: %s; program %llu (last %llu), marks prog %u ubo %#x/%#x " + "ssbo %#x/%#x tex %016llx:%016llx/%016llx:%016llx img %#x/%#x vbuf %#x, pipe_gen %llu (last %llu); G_tex " + "%llx %llx %llx %llx (last %llx %llx %llx %llx), G_buf %llx %llx %llx %llx (last %llx %llx %llx %llx); " + "pass %u (last %u), draw %llu of the census window, bi_valid %d", + name[kind], msg, ex->prog ? (unsigned long long)ex->prog->serial : 0ull, (unsigned long long)L->serial, + d->prog, d->ubo[0], d->ubo[1], d->ssbo[0], d->ssbo[1], (unsigned long long)d->tex[0][1], + (unsigned long long)d->tex[0][0], (unsigned long long)d->tex[1][1], (unsigned long long)d->tex[1][0], d->img[0], + d->img[1], d->vbuf, (unsigned long long)ex->pipe_gen, (unsigned long long)L->pipe_gen, + (unsigned long long)gt[0], (unsigned long long)gt[1], (unsigned long long)gt[2], (unsigned long long)gt[3], + (unsigned long long)L->gt[0], (unsigned long long)L->gt[1], (unsigned long long)L->gt[2], + (unsigned long long)L->gt[3], (unsigned long long)gb[0], (unsigned long long)gb[1], (unsigned long long)gb[2], + (unsigned long long)gb[3], (unsigned long long)L->gb[0], (unsigned long long)L->gb[1], + (unsigned long long)L->gb[2], (unsigned long long)L->gb[3], ex->pass_serial, L->pass_serial, + (unsigned long long)D->BC.draws, D->B.bi_valid); +} + static double bi_pct(uint64_t n, uint64_t of) { return of ? 100.0 * (double)n / (double)of : 0; } static void bi_log(BiCensus *c) { @@ -2592,13 +2645,14 @@ static void bi_log(BiCensus *c) { bi_pct(c->mv[8], c->draws), bi_pct(c->mv[9], c->draws), c->slots[0] / d, c->slots[1] / d, c->slots[2] / d, c->slots[3] / d, c->slots[4] / d); rt_log("bind_inc: kept (%% of draws): shaders %.1f (memo today %.1f), pipeline state %.1f, vertex buffers %.1f, " - "feedback scan %.1f of %.1f (today %.1f); mismatches %.2f a frame, %llu in all", + "feedback scan %.1f of %.1f (today %.1f); mismatches %.2f a frame, %llu in all, %.0f kept results checked a frame", bi_pct(c->sh_keep, c->draws), bi_pct(c->sh_memo, c->draws), bi_pct(c->pg_hit, c->draws), bi_pct(c->vb_skip, c->draws), bi_pct(c->fb_keep, c->draws), bi_pct(c->fb_n, c->draws), - bi_pct(c->fb_memo, c->draws), c->mism / fr, (unsigned long long)c->mism_total); + bi_pct(c->fb_memo, c->draws), c->mism / fr, (unsigned long long)c->mism_total, c->checked / fr); uint64_t total = c->mism_total; + uint32_t lines = c->frame_lines; memset(c, 0, sizeof *c); - c->mism_total = total; + c->mism_total = total, c->frame_lines = lines; } static int popc64(uint64_t v) { return __builtin_popcountll(v); } @@ -2612,6 +2666,9 @@ static void bi_census(VkExec *ex, const VkPipeInfo *info, const VkPipeState *s, const BiLast *L = &D->BL; uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (f != c->frame) { + if (c->mism_frame && c->frame_lines < 200) + c->frame_lines++, rt_log("bind_inc check: frame %llu: %u mismatches", (unsigned long long)c->frame, c->mism_frame); + c->mism_frame = 0; if (c->frames >= 300) bi_log(c); c->frame = f, c->frames++; } @@ -2752,13 +2809,32 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (!vs || (has_fs && !fs)) return 0; if (vk_dlaa_skip_draw(ex, fs)) return 0; // the game's FXAA off, or its scaling replaced by DLSS (vk_dlaa.c) uint32_t fbl = feedback_prepare(ex, fs); - VkPipeState s = ex->pipe; - s.vs = vs, s.fs = fs; - s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0)); - s.prim = (uint8_t)prim; - s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0); - s.y_flip = 1; - s.prim_restart = (uint8_t)(restart != 0); + uint8_t fb = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0)), dmw = (uint8_t)(nvn_depth_minus_w() != 0); + // TOTK_BIND_INC: the pipeline memo's state is this draw's when pipe was not written since the memo's state was built + // from it and the fields a draw sets are the same (y_flip is always 1): its pipeline is taken without building the + // state and comparing it with the memo's and the recorded one (3 x 360 bytes). check builds and compares as before. + int pg = pipe_memo_on() && !ex->chunked ? bim : 0; + int pg_hit = pg && D->PM.valid && D->PM.gen == ex->pipe_gen && D->PM.s.vs == vs && D->PM.s.fs == fs && + D->PM.s.feedback == fb && D->PM.s.prim == (uint8_t)prim && D->PM.s.depth_minus_w == dmw && + D->PM.s.prim_restart == (uint8_t)(restart != 0); + VkPipeState s; // built from pipe unless the generation hit + const VkPipeState *sp = &s; + uint64_t pgen = ex->pipe_gen; + if (pg_hit && pg == 1) { + sp = &D->PM.s; + } else { + s = ex->pipe; + s.vs = vs, s.fs = fs; + s.feedback = fb; + s.prim = (uint8_t)prim; + s.depth_minus_w = dmw; + s.y_flip = 1; + s.prim_restart = (uint8_t)(restart != 0); + if (pg_hit) { // check + D->BC.checked++; + if (memcmp(&s, &D->PM.s, sizeof s)) bi_mismatch(ex, BIM_PIPE, "pipeline memo: state differs from the memo its generation matched"); + } + } const VkPipeInfo *info = NULL; int pending; // A draw of at most 6 vertices over half the render area or more (a full-screen triangle or quad, a tile) waits for @@ -2782,12 +2858,17 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { const char *why = NULL; uint64_t w0 = 0; PipeCache *pc = NULL; - if (pipe_memo_on() && !(D->PM.valid && !memcmp(&D->PM.s, &s, sizeof s))) + int pg_take = pg_hit && pg == 1; // the memo's pipeline without building or comparing the state + if (!pg_take && pipe_memo_on() && !(D->PM.valid && !memcmp(&D->PM.s, &s, sizeof s))) pc = &D->PC[nvn_hm_hash(&s, sizeof s) & (PC_N - 1)]; - if (pipe_memo_on() && (!pc || (pc->info && !memcmp(&pc->s, &s, sizeof s)))) { + if (pg_take) { + info = D->PM.info, pending = 0; + p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); + } else if (pipe_memo_on() && (!pc || (pc->info && !memcmp(&pc->s, &s, sizeof s)))) { info = pc ? pc->info : D->PM.info, pending = 0; p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); - if (pc) D->PM.valid = 1, D->PM.s = s, D->PM.info = info; + if (pc) D->PM.valid = 1, D->PM.s = s, D->PM.info = info, D->B.st_pm = 0; + D->PM.gen = pgen; // s equals PM.s: built from pipe as it is now } else { w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); @@ -2802,8 +2883,8 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (!w0) w0 = kern_now_ns(); p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS); } - D->PM.valid = !pending && p && info; - if (D->PM.valid) D->PM.s = s, D->PM.info = info; + D->PM.valid = !pending && p && info, D->B.st_pm = 0; + if (D->PM.valid) D->PM.s = s, D->PM.info = info, D->PM.gen = pgen; if (D->PM.valid && pc) pc->s = s, pc->info = info; } uint64_t waited = w0 ? kern_now_ns() - w0 : 0; @@ -2819,7 +2900,11 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (pending) return 0; // a background worker is building it: the draw is skipped until it is ready if (!p || !info) return warn_once(W_PIPELINE, "a graphics pipeline could not be created: draws with it are skipped"), 0; bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p); - if (!D->B.st_valid || memcmp(&D->B.st, &s, sizeof s)) { + // On a generation hit the state recorded last is the memo's when st_pm says so (both were the last draw's). + int st_same = D->B.st_valid && (pg_take && D->B.st_pm ? 1 : !memcmp(&D->B.st, sp, sizeof *sp)); + if (pg_hit && pg == 2 && D->B.st_valid && D->B.st_pm && !st_same) + bi_mismatch(ex, BIM_DYN, "the recorded dynamic state differs from the pipeline memo's"); + if (!st_same) { // TOTK_DYN_DELTA: only the dynamic states whose values changed since the last call in this command buffer. A // pipeline bind clears st_valid but not dyn_valid: every pipeline here declares the same dynamic states, which // the bind leaves as they were (the viewport, scissor and stencil values of record_fixed rely on that too). @@ -2827,18 +2912,19 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (DD.mode < 0) DD.mode = rt_knob("DYN_DELTA", "1"); uint32_t cmds = 1; if (RT_KNOB_ON(DD.mode)) { - D->B.vmask = vk_pipeline_set_dynamic_delta(ex->cb, &s, D->B.dyn_valid, &cmds); + D->B.vmask = vk_pipeline_set_dynamic_delta(ex->cb, sp, D->B.dyn_valid, &cmds); D->B.dyn_valid = 1; } else { - D->B.vmask = vk_pipeline_set_dynamic(ex->cb, &s); + D->B.vmask = vk_pipeline_set_dynamic(ex->cb, sp); D->B.dyn_valid = 0; // the delta's record is stale from here on } __atomic_fetch_add(&DD.calls, 1, __ATOMIC_RELAXED), __atomic_fetch_add(&DD.cmds, cmds, __ATOMIC_RELAXED); if (!cmds) __atomic_fetch_add(&DD.empty, 1, __ATOMIC_RELAXED); ex->recorded += cmds != 0; - D->B.st = s, D->B.st_valid = 1; + D->B.st = *sp, D->B.st_valid = 1; if (ex->mark && ex->fr.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; } + D->B.st_pm = D->PM.valid; // st is this draw's state, which PM holds while valid vk_dlaa_draw(ex); int mo = ex->chunked && vk_dlaa_available(); if (mo) plat_mutex_lock(&mo_lock); @@ -2847,15 +2933,16 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (mo) plat_mutex_unlock(&mo_lock); record_fixed(ex); repeat_stats(ex, info); - if (bim) bi_gt(ex, bgt), bi_gb(ex, bgb), bi_census(ex, info, &s, bgt, bgb, bsg, sm_hit); + if (bim) bi_gt(ex, bgt), bi_gb(ex, bgb); + if (bim == 2) bi_census(ex, info, sp, bgt, bgb, bsg, sm_hit); // on counts nothing: it is what gets timed bind_resources(ex, info); if (fbl & ex->fb_dirty) feedback_barrier(ex); - feedback_written(ex, &s); + feedback_written(ex, sp); bind_vertex_buffers(ex); - if (vk_dump_armed) vk_dump_draw(ex, &s, p, info); + if (vk_dump_armed) vk_dump_draw(ex, sp, p, info); if (vk_hdr_track) vk_hdr_draw(ex, info); vk_exec_oq_draw(ex); - if (bim) bi_done(ex, info, &s, bgt, bgb, bsg); + if (bim) bi_done(ex, info, sp, bgt, bgb, bsg); return 1; } From dcd5dc3cf496b8906b073ab856137ebdbaad8de8 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 13 Nov 2026 23:44:01 +0300 Subject: [PATCH 03/10] A draw keeps the previous draw's shaders while no texture or image slot their translation reads changed (TOTK_BIND_INC) The shader memo already keeps them while the program, the bound state and the first 32 handles of both stages are equal. With the marks it also keeps them when other slots were rebound: no program or serial change, the same pipe_gen and slot generation, and no marked texture or image slot inside the program's slot mask (slot_mask_of, ~0u while unknown), which are all answers_for reads. BiLast.sm_ok says the memo holds the last prepared draw's shaders, so a mark cleared by a later draw never hides a change. check looks the shaders up and compares. Lookout Landing check, 60 s: shaders kept on 65% of draws (the memo alone 55%), 0 mismatches. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a522dc64..77b159f1 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -296,6 +296,7 @@ typedef struct { uint32_t binst, slots_gen, pass_serial, tex_serial; const GfxShader *vs, *fs; uint8_t sx[4]; // the state fields a draw sets: prim, restart, feedback, depth_minus_w + uint8_t sm_ok; // the shader memo (SM) holds this draw's shaders and lookup inputs uint32_t vmask; uint64_t vgen; } BiLast; @@ -2751,6 +2752,7 @@ static void bi_done(VkExec *ex, const VkPipeInfo *info, const VkPipeState *s, co L->vs = s->vs, L->fs = s->fs; L->sx[0] = s->prim, L->sx[1] = s->prim_restart, L->sx[2] = s->feedback, L->sx[3] = s->depth_minus_w; L->vmask = D->B.vmask, L->vgen = D->B.vgen; + L->sm_ok = (uint8_t)(D->SM.valid && D->SM.vs == s->vs && D->SM.fs == s->fs && D->SM.k.prog == ex->prog); memset(&ex->dirt, 0, sizeof ex->dirt); D->B.bi_valid = 1; } @@ -2779,7 +2781,21 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { int bim = bind_inc_mode(), sm_hit = 0; uint64_t bgt[4], bgb[4]; // TOTK_BIND_INC: G_tex and G_buf right before the descriptors are built uint32_t bsg = bim ? vk_texture_slots_gen() : 0; // the shaders' slot generation, before shader_for - if (memo && D->SM.valid && memo_match(ex, gen, &D->SM.k)) { + // TOTK_BIND_INC: the shader memo holds the last prepared draw's shaders (BL.sm_ok), and since then no op changed the + // program, its serial, the bound state, the slot generation, or a texture or image slot of either stage whose type + // the program's translation reads (slot_mask_of; ~0u while unknown): shader_for would answer the same. + int sh_keep = 0; + if (bim && memo && D->SM.valid && D->B.bi_valid && D->BL.sm_ok && !ex->chunked && !ex->dirt.prog && + ex->prog == D->BL.prog && ex->prog->serial == D->BL.serial && ex->pipe_gen == D->BL.pipe_gen && + bsg == D->BL.slots_gen) { + uint32_t m0 = slot_mask_of(D, ex->prog->serial, NVN_STAGE_VS), m1 = slot_mask_of(D, ex->prog->serial, NVN_STAGE_FS); + sh_keep = !((uint32_t)ex->dirt.tex[NVN_STAGE_VS][0] & m0) && !(ex->dirt.img[NVN_STAGE_VS] & m0) && + !((uint32_t)ex->dirt.tex[NVN_STAGE_FS][0] & m1) && !(ex->dirt.img[NVN_STAGE_FS] & m1); + } + if (sh_keep && bim == 1) { + vs = D->SM.vs, fs = D->SM.fs, has_vs = 1, has_fs = D->SM.has_fs; + sm_hit = 1; + } else if (memo && D->SM.valid && memo_match(ex, gen, &D->SM.k)) { vs = D->SM.vs, fs = D->SM.fs, has_vs = 1, has_fs = D->SM.has_fs; sm_hit = 1; if (memo == 2) { @@ -2795,6 +2811,12 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs)) memo_key(ex, gen, &D->SM.k), D->SM.vs = vs, D->SM.fs = fs, D->SM.has_fs = has_fs, D->SM.valid = 1; } + if (sh_keep && bim == 2) { + D->BC.checked++; + if (vs != D->BL.vs || fs != D->BL.fs || vs_wait || fs_wait) + bi_mismatch(ex, BIM_SHADER, "vs %p fs %p (kept %p %p), pending %d %d", (const void *)vs, (const void *)fs, + (const void *)D->BL.vs, (const void *)D->BL.fs, vs_wait, fs_wait); + } if (!has_vs) return warn_once(W_NO_VS, "a draw with a program that has no vertex stage is skipped"), 0; // A draw into a target the previous frame did not render (or into none) waits for a pending shader or pipeline: // one-shot draws (volumes and LUTs generated once, later only sampled) would leave their target empty for the From 772d9dea669c19f1094ffd05969f3ecc4b10494c Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 13 Nov 2026 23:45:44 +0300 Subject: [PATCH 04/10] The feedback scan keeps its answer across texture binds that leave the fragment shader's slots alone (TOTK_BIND_INC) feedback_scan's memo is keyed by tex_serial, which every texture or image bind of either stage moves. When the memo holds the last prepared draw's scan (BiLast.fb_ok) of the same fragment shader in the same pass, no slot was registered since, and no marked fragment texture slot is one the shader reads (BiMask.ftex, buffer textures included), the answer stands. check scans and compares. Lookout Landing check, 60 s: kept on 68% of draws (the memo alone 60%), 0 mismatches. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 77b159f1..643ed34c 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -210,6 +210,7 @@ typedef struct { uint32_t ubo[2]; // uniform slots 3..20 (bit slot - 3: the OP_BIND_UBO index) uint32_t ssbo[2]; // storage slots read through c0's table uint64_t tex[2][2]; // texture handle slots 0..127 + uint64_t ftex[2]; // every fragment texture slot, buffer textures too (feedback_scan reads them) uint32_t img[2]; // image handle slots 0..31 uint8_t c0[2], c1[2]; // reads c0 (every storage slot and the base instance), the compiler constants uint8_t gfx; // every binding is a vertex or fragment one @@ -297,6 +298,7 @@ typedef struct { const GfxShader *vs, *fs; uint8_t sx[4]; // the state fields a draw sets: prim, restart, feedback, depth_minus_w uint8_t sm_ok; // the shader memo (SM) holds this draw's shaders and lookup inputs + uint8_t fb_ok; // VkExec's feedback scan memo (fb_fs ...) holds this draw's scan uint32_t vmask; uint64_t vgen; } BiLast; @@ -1432,9 +1434,25 @@ static SubRange sampled_sub(uint64_t h) { // texture lock held // Attachments of the open pass the fragment shader fs samples; kept while the shader, the texture bindings and the // pass stay the same. +static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info); + static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) { VkExecDraw *D = ex->draw; if (ex->fb_fs == fs && ex->fb_serial == ex->tex_serial && ex->fb_pass == ex->pass_serial) return ex->fb_sampled; + // TOTK_BIND_INC: the memo holds the last prepared draw's scan (BL.fb_ok) of this shader in this pass, and since then + // no op changed a fragment texture slot the shader reads (the pipeline's BiMask.ftex) and no slot was registered: + // a texture bind elsewhere (tex_serial) no longer forces the scan. + int bim = bind_inc_mode(), keep = 0; + if (bim && D->B.bi_valid && D->BL.fb_ok && !ex->chunked && ex->fb_fs == fs && D->BL.fs == fs && + ex->fb_pass == ex->pass_serial && vk_texture_slots_gen() == D->BL.slots_gen) { + const BiMask *bm = &rs_list(ex, D->BL.info)->m; + keep = !(ex->dirt.tex[NVN_STAGE_FS][0] & bm->ftex[0]) && !(ex->dirt.tex[NVN_STAGE_FS][1] & bm->ftex[1]); + if (keep && bim == 1) { + ex->fb_serial = ex->tex_serial; + return ex->fb_sampled; + } + } + uint32_t kept = ex->fb_sampled; const GfxShaderRes *res; int n = gfx_shader_resources(fs, GFX_RES_TEX, &res); uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup @@ -1463,6 +1481,10 @@ static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) { else f->h = 0; } if (locked) vk_texture_unlock(); + if (keep) { + D->BC.checked++; + if (m != kept) bi_mismatch(ex, BIM_FB, "sampled attachments %#x, kept %#x", m, kept); + } ex->fb_fs = fs, ex->fb_serial = ex->tex_serial, ex->fb_pass = ex->pass_serial, ex->fb_sampled = m; return m; } @@ -2025,6 +2047,11 @@ static void bi_mask(const VkPipeInfo *info, BiMask *m) { else if (!img && slot >= 0 && slot < 128) m->tex[st][slot >> 6] |= 1ull << (slot & 63); } } + if (b->kind == GFX_RES_TEX && st == NVN_STAGE_FS) + for (uint32_t k = 0; k < cnt; k++) { + int slot = r->slot + (int)k; + if (slot >= 0 && slot < 128) m->ftex[slot >> 6] |= 1ull << (slot & 63); + } } } @@ -2753,6 +2780,7 @@ static void bi_done(VkExec *ex, const VkPipeInfo *info, const VkPipeState *s, co L->sx[0] = s->prim, L->sx[1] = s->prim_restart, L->sx[2] = s->feedback, L->sx[3] = s->depth_minus_w; L->vmask = D->B.vmask, L->vgen = D->B.vgen; L->sm_ok = (uint8_t)(D->SM.valid && D->SM.vs == s->vs && D->SM.fs == s->fs && D->SM.k.prog == ex->prog); + L->fb_ok = (uint8_t)(s->fs && ex->fb_fs == s->fs && ex->fb_serial == ex->tex_serial && ex->fb_pass == ex->pass_serial); memset(&ex->dirt, 0, sizeof ex->dirt); D->B.bi_valid = 1; } From c86feefadc331ba3938daaf8a8a24bc21392733d Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 13 Nov 2026 23:47:24 +0300 Subject: [PATCH 05/10] A draw skips the vertex stream loop while none of the streams it reads changed (TOTK_BIND_INC) bind_vertex_buffers resolves and compares the 17 streams the vertex input reads on every draw. When the last prepared draw bound the same set of streams in this command buffer, no op changed one of their ranges, vk_mem_gen stayed, and with static vertex input the strides did (pipe_gen), every stream would resolve to what is bound: the loop is skipped. check runs the loop and counts a stream it would bind again. Lookout Landing check, 60 s: 62% of draws skip it, 0 mismatches. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 643ed34c..a28c5fe4 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -299,7 +299,7 @@ typedef struct { uint8_t sx[4]; // the state fields a draw sets: prim, restart, feedback, depth_minus_w uint8_t sm_ok; // the shader memo (SM) holds this draw's shaders and lookup inputs uint8_t fb_ok; // VkExec's feedback scan memo (fb_fs ...) holds this draw's scan - uint32_t vmask; + uint32_t vmask, vused; // vused: the streams its bind_vertex_buffers resolved uint64_t vgen; } BiLast; typedef struct { @@ -2461,6 +2461,15 @@ static void bind_vertex_buffers(VkExec *ex) { // submission is done. uint32_t used = vb_used_on() && D->B.st_valid ? D->B.vmask : VK_PIPE_ALL_BINDINGS; uint32_t chg = 0; + // TOTK_BIND_INC: the last prepared draw resolved and bound these streams, and since then no op changed one of their + // ranges, the pools and mirrors stayed (vk_mem_gen) and, with static vertex input, the strides (pipe_gen): every + // stream would resolve to what is bound, so the loop is skipped. + int bim = bind_inc_mode(), keep = 0; + if (bim && D->B.bi_valid && D->B.vb_valid && !ex->chunked && used == D->BL.vused && !(ex->dirt.vbuf & used) && + D->B.vgen == vk_mem_gen() && (dyn || ex->pipe_gen == D->BL.pipe_gen)) { + if (bim == 1) return; + keep = 1; + } if (!D->B.vb_valid) { // the command buffer's first bind sets all 17 slots, the zero buffer where nothing resolves for (int b = 0; b < 17; b++) D->B.vb[b] = Z.zbuf, D->B.vo[b] = 0, D->B.vsz[b] = ZERO_BYTES, D->B.vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0, @@ -2496,6 +2505,10 @@ static void bind_vertex_buffers(VkExec *ex) { if (nb != D->B.vb[b] || no != D->B.vo[b] || nsz != D->B.vsz[b] || (!dyn && nstr != D->B.vstr[b])) chg |= 1u << b; D->B.vb[b] = nb, D->B.vo[b] = no, D->B.vsz[b] = nsz, D->B.vstr[b] = nstr, D->B.vcls[b] = ncls, D->B.vpool[b] = npool; } + if (keep) { + D->BC.checked++; + if (chg) bi_mismatch(ex, BIM_VB, "streams %#x would be bound again (used %#x)", chg, used); + } if (!chg) return; // One bind over the changed slots; the slots between them get what they hold again. int lo = __builtin_ctz(chg), hi = 31 - __builtin_clz(chg); @@ -2779,6 +2792,7 @@ static void bi_done(VkExec *ex, const VkPipeInfo *info, const VkPipeState *s, co L->vs = s->vs, L->fs = s->fs; L->sx[0] = s->prim, L->sx[1] = s->prim_restart, L->sx[2] = s->feedback, L->sx[3] = s->depth_minus_w; L->vmask = D->B.vmask, L->vgen = D->B.vgen; + L->vused = vb_used_on() && D->B.st_valid ? D->B.vmask : VK_PIPE_ALL_BINDINGS; L->sm_ok = (uint8_t)(D->SM.valid && D->SM.vs == s->vs && D->SM.fs == s->fs && D->SM.k.prog == ex->prog); L->fb_ok = (uint8_t)(s->fs && ex->fb_fs == s->fs && ex->fb_serial == ex->tex_serial && ex->fb_pass == ex->pass_serial); memset(&ex->dirt, 0, sizeof ex->dirt); From c26ed9ba7b05a8beb0a87dc669568778a722ef70 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 13 Nov 2026 23:49:21 +0300 Subject: [PATCH 06/10] A graphics draw with the previous draw's pipeline rebuilds only the descriptors whose inputs changed (TOTK_BIND_INC) With TOTK_BIND_INC the graphics build writes into VkExecDraw.BI, which keeps the packed descriptors of the last build and, per descriptor, what it counted for the traffic census and whether it copied guest bytes. bi_plan names the bindings to build again: support always; c0 when a storage slot of the stage or the base instance changed; the compiler constants when the program, its serial or G_buf changed; uniform and storage slots when marked or when G_buf (pools and mirrors, snapshots, overlays, the submission) moved; textures and images when a handle was marked or G_tex (slot registrations, samplers, the storage epoch, the feedback layout, DLSS's swap) moved; storage addresses held in a uniform block, and any descriptor whose build copied guest bytes, every time. A pipeline with a written storage binding is built in full. Kept descriptors count their census again; a set none of whose descriptors changed and that is still pushed or bound records nothing, without the psig or sig compare. check builds everything as before and compares each kept descriptor and its census with the build. Lookout Landing check, 60 s: about 17,000 kept bindings compared a frame, 0 mismatches. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 154 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 9 deletions(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a28c5fe4..19fab425 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -215,6 +215,7 @@ typedef struct { uint8_t c0[2], c1[2]; // reads c0 (every storage slot and the base instance), the compiler constants uint8_t gfx; // every binding is a vertex or fragment one uint16_t always, ndesc; + uint8_t wr; // a storage binding the shaders write: its build has side effects, nothing is kept } BiMask; typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; BiMask m; RsRange r[]; } RsList; // (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the @@ -314,6 +315,21 @@ enum { BIM_PIPE, BIM_DYN, BIM_SHADER, BIM_FB, BIM_VB, BIM_DESC, BIM_SET, BIM_N } static int bind_inc_mode(void); static void bi_mismatch(struct VkExec *ex, int kind, const char *fmt, ...); +// TOTK_BIND_INC: what building one descriptor counted (count_use; n 0: nothing, more than 1: never kept) and whether it +// copied guest bytes into transient memory (vol: a rebuild may differ, so it is never kept). +typedef struct { uint8_t vol, n; int16_t cls; int32_t use; uint64_t bytes; } BiMeta; +// The packed descriptors of the last graphics build (bind_resources writes them here while TOTK_BIND_INC is set), the +// pipeline they are for (valid with CbState.bi_valid) and, per descriptor, BiMeta. +typedef struct { + const VkPipeInfo *info; + BiMeta *cur; // count_use and the transient copies record here while a descriptor is built + VkDescSlot d[MAX_DESC]; + BiMeta meta[MAX_DESC]; + VkDescSlot old[MAX_DESC]; // check mode: the descriptors before the build + BiMeta mold[MAX_DESC]; + uint8_t plan[MAX_BINDINGS]; // bindings to build again (bi_plan) +} BiDesc; + // The memos and command buffer state of one recording context (VkExec.draw). Nothing in it is shared: the context's // thread reads and writes it without a lock. typedef struct VkExecDraw { @@ -325,6 +341,7 @@ typedef struct VkExecDraw { RsState RS; BiLast BL; BiCensus BC; + BiDesc BI; nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown const VkPipeInfo *rs_last; // the last lookup const RsList *rs_last_l; @@ -1069,6 +1086,11 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) { // vk_mem_count, also kept per (class, use) while a graphics build runs (RS). static void count_use(VkExecDraw *D, int cls, int use, uint64_t n) { vk_mem_count(cls, use, n); + if (D->BI.cur) { + BiMeta *m = D->BI.cur; + if (m->n++) m->vol = 1; // a second count: not kept (one is all a descriptor build makes) + m->cls = (int16_t)cls, m->use = use, m->bytes = n; + } if (!D->RS.rec) return; for (int i = 0; i < D->RS.ncnt; i++) if (D->RS.cnt[i].cls == cls && D->RS.cnt[i].use == use) { @@ -1109,6 +1131,7 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, VkBuffer b; VkDeviceSize o; D->RS.volatile_ = 1; + if (D->BI.cur) D->BI.cur->vol = 1; memcpy(scratch(ex, n, &b, &o), (const void *)(uintptr_t)addr, n); vk_mem_count(sp.cls, VK_MUSE_CPU_READ, n); vk_mem_count(VK_MCLS_SCRATCH, use, n); @@ -2036,6 +2059,7 @@ static void bi_mask(const VkPipeInfo *info, BiMask *m) { } } else if (b->kind == GFX_RES_SSBO) { int ubo, slot = ssbo_slot(r, st, &ubo); + if (r->flags & 1) m->wr = 1; if ((r->flags & 1) || slot < 0 || slot >= 16 || ubo) m->always = (uint16_t)(m->always + cnt); else m->ssbo[st] |= 1u << slot; } else if ((b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER && @@ -2206,15 +2230,79 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c // graphics push equal to the last one at its set number, with no other layout or bind between, is not recorded. // The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update // template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. -static void bind_resources(VkExec *ex, const VkPipeInfo *info) { +static int bind_inc_mode(void); +static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...); + +// TOTK_BIND_INC: which bindings of a graphics draw with the last prepared draw's pipeline must be built again (BI.plan, +// 1 each), from the marks since that draw and the generations G_tex and G_buf read right before this build (gt, gb) +// against those read before its build. 0 when nothing can be kept: no last draw in this command buffer, another +// pipeline, a chunk, a storage binding the shaders write (its build restarts passes and notes writes). +// - support: always (its inputs are compared inside, 12 bytes); +// - uniform slot 0 (c0): a storage slot of the stage or the base instance changed; slot 1 (the compiler constants): +// the program or its serial changed, or G_buf moved; slots 3..20: the slot changed or G_buf moved (snapshots, +// overlays, pools and mirrors); other slots bind the zero buffer; +// - storage: as a uniform slot; an address held in a uniform block (ubo_ptr): always, the block changes without an op; +// - textures and images: a handle changed or G_tex moved (slot resolution, samplers, the storage check once per +// submission, the feedback layout, DLSS's swap); buffer textures bind the zero view; +// - any descriptor whose last build copied guest bytes or counted more than once (BiMeta): always. +static int bi_plan(VkExec *ex, const VkPipeInfo *info, const uint64_t gt[4], const uint64_t gb[4]) { + VkExecDraw *D = ex->draw; + const BiLast *L = &D->BL; + if (!D->B.bi_valid || ex->chunked || L->info != info || D->BI.info != info) return 0; + const BiMask *m = &rs_list(ex, info)->m; + if (!m->gfx || m->wr) return 0; + const VkExecDirty *dt = &ex->dirt; + int tmv = memcmp(gt, L->gt, 4 * sizeof *gt) != 0, bmv = memcmp(gb, L->gb, 4 * sizeof *gb) != 0; + int prog = dt->prog || ex->prog != L->prog || ex->prog->serial != L->serial, binst = D->c0_binst != L->binst; + uint32_t nd = 0; + for (uint32_t i = 0; i < info->n_bindings; i++) { + const VkPipeBinding *b = &info->bindings[i]; + const GfxShaderRes *r = b->res; + uint32_t cnt = b->count ? b->count : 1; + int st = nvn_stage_of(b->stage), again = 0; + for (uint32_t k = 0; k < cnt && !again; k++) again = D->BI.meta[nd + k].vol; + if (b->kind == VK_PIPE_SUPPORT) { + again = 1; + } else if (b->kind == GFX_RES_UBO) { + for (uint32_t k = 0; k < cnt && !again; k++) { + int slot = r->slot + (int)k; + if (slot == 0) again = dt->ssbo[st] || binst; + else if (slot == 1) again = prog || bmv; + else if (slot >= 3 && slot - 3 < 18) again = bmv || (dt->ubo[st] >> (slot - 3) & 1); + } + } else if (b->kind == GFX_RES_SSBO) { + int ubo, slot = ssbo_slot(r, st, &ubo); + again = again || ubo || slot < 0 || slot >= 16 || bmv || (dt->ssbo[st] >> slot & 1); + } else if ((b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER && + b->type != VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) { + int img = b->kind == GFX_RES_IMG; + again = again || tmv; + for (uint32_t k = 0; k < cnt && !again; k++) { + int slot = r->slot + (int)k; + if (img) again = slot >= 0 && slot < 32 && (dt->img[st] >> slot & 1); + else again = slot >= 0 && slot < 128 && (dt->tex[st][slot >> 6] >> (slot & 63) & 1); + } + } + D->BI.plan[i] = (uint8_t)again; + nd += cnt; + } + return 1; +} + +static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *bgt, const uint64_t *bgb) { VkExecDraw *D = ex->draw; VkWriteDescriptorSet *w = ex->draw->w; - VkDescSlot *d = ex->draw->d; VkBufferView *bv = ex->draw->bv; if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS); bind_stats(ex->draw, info); int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS; - int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = D->B.res_valid; + // TOTK_BIND_INC: a graphics build writes BI's array and per-descriptor census (bgt: a prepared draw's), on builds + // only the bindings bi_plan names, check builds everything and compares what on would have kept. + int bim = gfx && bgt ? bind_inc_mode() : 0; + int inc = bim ? bi_plan(ex, info, bgt, bgb) : 0; + VkDescSlot *d = bim ? D->BI.d : ex->draw->d; + BiMeta *meta = bim ? D->BI.meta : NULL; + int skip = gfx && bim != 1 ? bind_skip_on() : 0, same = 0, was_valid = D->B.res_valid; if (skip) { same = bind_sig(ex, info); if (same && was_valid && skip == 1) { @@ -2236,6 +2324,13 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { lo[s] = i0, hi[s] = i1, i0 = i1; if (i1 > lo[s] && !info->tpl[s]) need_w = 1; } + if (inc && need_w) inc = 0; // a set without a template reads the write structs, which only a full build fills + int changed[2] = {!inc, !inc}; // inc: a descriptor of the set differs from the last build's + if (bim == 2 && inc) { // check: the descriptors before the build, to compare those on would have kept + uint32_t n = 0; + for (uint32_t i = 0; i < info->n_bindings; i++) n += info->bindings[i].count ? info->bindings[i].count : 1; + memcpy(D->BI.old, d, n * sizeof *d), memcpy(D->BI.mold, meta, n * sizeof *meta); + } uint32_t nd = 0; for (uint32_t i = 0; i < info->n_bindings; i++) { const VkPipeBinding *b = &info->bindings[i]; @@ -2245,6 +2340,19 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { if (n_sets == 2 && i == lo[1]) d0[1] = nd; VkDescSlot *o = &d[nd]; int st = nvn_stage_of(b->stage), kind = 0; // kind: 0 buffer, 1 image, 2 texel buffer view + uint32_t set_of = n_sets == 2 && i >= lo[1] ? 1 : 0; + if (inc && bim == 1 && !D->BI.plan[i]) { // kept: what it counted is counted again + for (uint32_t k = 0; k < cnt; k++) + if (meta[nd + k].n) vk_mem_count(meta[nd + k].cls, meta[nd + k].use, meta[nd + k].bytes); + nd += cnt; + continue; + } + VkDescSlot prev[4]; // on: the descriptors before the build, to tell whether the set changed + int cmp = inc && bim == 1 && cnt <= 4 && !changed[set_of]; + if (cmp) memcpy(prev, o, cnt * sizeof *o); + else if (inc && bim == 1) changed[set_of] = 1; + if (meta) memset(&meta[nd], 0, cnt * sizeof *meta); +#define BI_CUR(k) (D->BI.cur = meta ? &meta[nd + (k)] : NULL) if (b->kind == VK_PIPE_SUPPORT) { float in[3]; support_in(ex, in); @@ -2262,9 +2370,9 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } o->buffer = (VkDescriptorBufferInfo){D->B.sup_buf, D->B.sup_off, SUPPORT_BYTES}; } else if (b->kind == GFX_RES_UBO) { - for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ubo_info(ex, st, r->slot + (int)k); + for (uint32_t k = 0; k < cnt; k++) BI_CUR(k), o[k].buffer = ubo_info(ex, st, r->slot + (int)k); } else if (b->kind == GFX_RES_SSBO) { - for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ssbo_info(ex, st, r, r->flags & 1); + for (uint32_t k = 0; k < cnt; k++) BI_CUR(k), o[k].buffer = ssbo_info(ex, st, r, r->flags & 1); } else if (b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) { int img = b->kind == GFX_RES_IMG; if (b->type == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER || b->type == VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) { @@ -2282,6 +2390,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { uint64_t h = 0; if (img) h = slot >= 0 && slot < 32 ? ex->img[st][slot] : 0; else h = slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0; + BI_CUR(k); tex_info(ex, r, h, img, r->separate, &o[k].image, &locked); o[k].w[2] = (uint32_t)o[k].image.imageLayout; // zero the tail padding } @@ -2290,6 +2399,26 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } else { rt_fatal("vk: binding %u has an unknown kind %d", (unsigned)b->binding, (int)b->kind); } + D->BI.cur = NULL; +#undef BI_CUR + if (cmp && memcmp(prev, o, cnt * sizeof *o)) changed[set_of] = 1; + if (bim == 2 && inc && !D->BI.plan[i]) { // check: on would have kept these + D->BC.checked++; + for (uint32_t k = 0; k < cnt; k++) { + const BiMeta *mo = &D->BI.mold[nd + k], *mn = &meta[nd + k]; + if (!memcmp(&D->BI.old[nd + k], &o[k], sizeof *o) && mo->n == mn->n && + (!mn->n || (mo->cls == mn->cls && mo->use == mn->use && mo->bytes == mn->bytes))) + continue; + bi_mismatch(ex, BIM_DESC, "binding %u (stage %d, kind %d, slot %d + %u, set %u): kept %016llx %016llx %016llx, " + "built %016llx %016llx %016llx; census %d/%d/%llu x%u, now %d/%d/%llu x%u", (unsigned)b->binding, + st, (int)b->kind, r ? r->slot : -1, k, set_of, (unsigned long long)D->BI.old[nd + k].w[0], + (unsigned long long)D->BI.old[nd + k].w[1], (unsigned long long)D->BI.old[nd + k].w[2], + (unsigned long long)o[k].w[0], (unsigned long long)o[k].w[1], (unsigned long long)o[k].w[2], + mo->cls, mo->use, (unsigned long long)mo->bytes, mo->n, mn->cls, mn->use, + (unsigned long long)mn->bytes, mn->n); + break; + } + } if (need_w) { w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = b->binding, .descriptorCount = cnt, .descriptorType = b->type}; @@ -2326,6 +2455,10 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } if (s ? info->push1 : info->push) { uint32_t n = gfx && push_skip_on() ? dbytes : 0; + if (inc && bim == 1 && !changed[s] && n && D->B.ds_pushed[s] && D->DS.pn[s] == n) { // psig holds ds + D->BS.push_skip++; + continue; + } if (n && D->B.ds_pushed[s] && D->DS.pn[s] == n && !memcmp(D->DS.psig[s], ds, n)) { D->BS.push_skip++; continue; @@ -2345,7 +2478,8 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } else { VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout; VkDescriptorSet set = D->B.ds_last[s]; - if (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || memcmp(D->DS.sig[s], ds, dbytes)) { + int kept = inc && bim == 1 && !changed[s] && set && D->B.ds_last_layout[s] == sl && D->DS.n[s] == dbytes; + if (!kept && (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || memcmp(D->DS.sig[s], ds, dbytes))) { set = alloc_set(ex, sl); write_set(info, s, set, ds, w, i0, i1); D->B.ds_last[s] = set, D->B.ds_last_layout[s] = sl, D->DS.n[s] = dbytes; @@ -2383,6 +2517,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } D->B.res_valid = valid; } + if (bim) D->BI.info = info; ex->recorded++; } @@ -2999,7 +3134,7 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { repeat_stats(ex, info); if (bim) bi_gt(ex, bgt), bi_gb(ex, bgb); if (bim == 2) bi_census(ex, info, sp, bgt, bgb, bsg, sm_hit); // on counts nothing: it is what gets timed - bind_resources(ex, info); + bind_resources(ex, info, bim ? bgt : NULL, bgb); if (fbl & ex->fb_dirty) feedback_barrier(ex); feedback_written(ex, sp); bind_vertex_buffers(ex); @@ -3919,7 +4054,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); D->c0_dim[0] = (uint32_t)d->x, D->c0_dim[1] = (uint32_t)d->y, D->c0_dim[2] = (uint32_t)d->z; D->twin_dispatch = 1; - bind_resources(ex, info); + bind_resources(ex, info, NULL, NULL); D->twin_dispatch = 0; VkPassMark *m = dispatch_mark(ex, cs, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z); vkCmdDispatch(ex->cb, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z); @@ -3950,7 +4085,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { D->c0_dim[0] = D->c0_dim[1] = D->c0_dim[2] = 0; D->B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads D->twin_dispatch = 1; - bind_resources(ex, info); + bind_resources(ex, info, NULL, NULL); D->twin_dispatch = 0; if (D->B.c0_valid[2]) { // the shader binds c0 VkBufferCopy cp = {ip.offset, D->B.c0_off[2] + 0x200, sizeof(VkDispatchIndirectCommand)}; @@ -4158,6 +4293,7 @@ static int ub_bind(VkExec *ex, uint64_t addr, uint64_t n, uint64_t align, VkDesc VkDeviceSize o; uint8_t *dst = scratch(ex, n, &b, &o); ex->draw->RS.volatile_ = 1; + if (ex->draw->BI.cur) ex->draw->BI.cur->vol = 1; const uint8_t *base = vk_usnap_bytes(addr, n); VkMemSpan sp; if (base) memcpy(dst, base, n); From dba054110be41666b33e92caea4a4fac1b6a3839 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 13 Nov 2026 23:52:37 +0300 Subject: [PATCH 07/10] Graphics descriptor sets are shared within a submission and allocated in batches (TOTK_BIND_INC) Adapted from Savage Wind Recomp's set_finish and vk_alloc_set (MIT, row in THIRD_PARTY_NOTICES.md). A graphics set that would be allocated and written is first looked up in a per-context table keyed by two independent 64-bit hashes of its layout and packed descriptors and stamped with the submission's fence: an equal set of the same submission is bound instead. New sets come from the submission's last pool in batches of one layout that grow 1, 2, 4, 8, 16 while the layout repeats, one vkAllocateDescriptorSets per batch instead of per set. check stores each set's contents, compares every hit with them and allocates as before. Lookout Landing: about 2,000 allocated graphics sets a frame, 6.7% shared; check 0 mismatches; on ran 60 s without errors. Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 1 + runtime/vk_exec_draw.c | 95 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 164e6a75..e56e73f2 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -31,6 +31,7 @@ Code copied or adapted from another project is listed here with its source, the | libopus (MSYS2 `mingw-w64-ucrt-x86_64-opus`), dynamic link, `libopus-0.dll` copied next to `totk.exe` | https://opus-codec.org, Copyright (c) 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon | BSD-3-Clause | `runtime/ov_media.c` (nn::codec hardware Opus decoder) | | Dear ImGui 1.91.9 (core and Vulkan back end, unmodified), as vendored by LAS Recomp | https://github.com/ocornut/imgui, via https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f` (`third_party/imgui/`), Copyright (c) 2014-2025 Omar Cornut | MIT | `runtime/third_party/imgui/` (compiled into `totk.exe` for the in-game menu, `runtime/ui/`) | | SavageWindRecomp settings menu: two-pane option rows (row look, left / right stepping, hold speed-up, description pane), the menu style and tab bar | https://github.com/Revivedx/SavageWindRecomp @ `575ad88e02f9a89a9a885d90039d48af33c982f4` (`runtime/ui/option_rows.mm`, `runtime/ui/menu.mm`), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/ui/option_rows.cpp`, `runtime/ui/option_rows.h`, `runtime/ui/menu.cpp` | +| SavageWindRecomp descriptor sets shared within a submission (two 64-bit hashes of layout and contents) and allocated in growing batches per layout | https://github.com/Revivedx/SavageWindRecomp @ `740b18025fe11ba03c117dbb56668db304b5043e` (`runtime/nvn/vulkan/exec.c` set_finish, vk_alloc_set), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/vk_exec_draw.c` (bi_set_find, bi_set_put, alloc_set_batched) | | SavageWindRecomp performance overlay: the frame ring and its one-second summary, the NVML calls, the sampling loop, the full panel's layout | https://github.com/Revivedx/SavageWindRecomp @ `575ad88e02f9a89a9a885d90039d48af33c982f4` (`runtime/perfmon.c`, `runtime/perfmon.h`, `perf_overlay_draw` in `runtime/ui/menu.mm`), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/perfmon.c`, `runtime/perfmon.h`, `runtime/ui/menu.cpp` | | Roboto Medium font (Dear ImGui's `misc/fonts/Roboto-Medium.ttf`, unmodified) | https://fonts.google.com/specimen/Roboto, via https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f` (`third_party/imgui/misc/fonts/`), Copyright 2011 Google Inc. | Apache-2.0 | `runtime/ui/fonts/Roboto-Medium.ttf` (licence text in `runtime/ui/fonts/LICENSE-Roboto.txt`), embedded by `runtime/ui/menu.cpp` | | Vulkan Memory Allocator 3.4.0 (MSYS2 `mingw-w64-ucrt-x86_64-vulkan-memory-allocator`), header-only, compiled into `totk.exe` | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator, Copyright (c) 2017-2026 Advanced Micro Devices, Inc. | MIT | `runtime/vk_vma.cpp` (the allocator behind `runtime/vk_memory.c`) | diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 19fab425..aec977c4 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -307,7 +307,7 @@ typedef struct { uint64_t frame; // gfx_counters.frames at the last draw counted uint32_t frames; // frames since the last line uint64_t draws, cls[4], b_gen, desc, rec[4], always, mv[BI_MV], slots[5]; - uint64_t sh_keep, sh_memo, fb_n, fb_keep, fb_memo, vb_skip, pg_hit; + uint64_t sh_keep, sh_memo, fb_n, fb_keep, fb_memo, vb_skip, pg_hit, sets, set_shared; uint64_t mism, mism_total, checked; uint32_t mism_frame, frame_lines; } BiCensus; @@ -330,6 +330,23 @@ typedef struct { uint8_t plan[MAX_BINDINGS]; // bindings to build again (bi_plan) } BiDesc; +// TOTK_BIND_INC on: graphics descriptor sets shared within a submission and allocated in batches (bi_set_find, +// alloc_set_batched). Adapted from Savage Wind Recomp runtime/nvn/vulkan/exec.c set_finish and vk_alloc_set +// (https://github.com/Revivedx/SavageWindRecomp @ 740b18025fe11ba03c117dbb56668db304b5043e, MIT, Copyright (c) 2026 +// Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai). +#define SETC_TAB 4096 +typedef struct { + uint64_t fence, hash, hash2; // the submission whose pools hold set; two hashes of layout and contents + VkDescriptorSetLayout layout; + VkDescriptorSet set; + uint32_t n; // content bytes + uint32_t ckcap; // check mode: the contents, to compare a hit with + uint8_t *ck; +} SetEnt; +#define ASET_CACHE 16 +#define ASET_BATCH 16 +typedef struct { VkDescriptorSetLayout layout; uint64_t fence; int n, next; VkDescriptorSet set[ASET_BATCH]; } ASetCache; + // The memos and command buffer state of one recording context (VkExec.draw). Nothing in it is shared: the context's // thread reads and writes it without a lock. typedef struct VkExecDraw { @@ -342,6 +359,8 @@ typedef struct VkExecDraw { BiLast BL; BiCensus BC; BiDesc BI; + SetEnt *setc; // SETC_TAB entries, made on first use + ASetCache aset[ASET_CACHE]; nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown const VkPipeInfo *rs_last; // the last lookup const RsList *rs_last_l; @@ -1874,6 +1893,67 @@ static VkDescriptorSet alloc_set(VkExec *ex, VkDescriptorSetLayout layout) { return VK_NULL_HANDLE; } +// Sets come from the submission's pools in batches of one layout that grow 1, 2, 4, 8, 16 while the layout repeats, +// and are handed out one by one: a vkAllocateDescriptorSets per set costs the driver a lock and a free-list walk. A +// batch belongs to its submission's pools (fence) and dies with their reset; sets it leaves unused stay in the pool. +static VkDescriptorSet alloc_set_batched(VkExec *ex, VkDescriptorSetLayout layout) { + VkExecDraw *D = ex->draw; + CbRes *r = cb_res(ex); + ASetCache *c = &D->aset[(uint32_t)(((uintptr_t)layout >> 4) * 2654435761u) >> 28]; + if (c->layout == layout && c->fence == ex->fence && c->n > 0) return c->set[--c->n]; + if (c->layout != layout) c->layout = layout, c->next = 1; + else if (c->next < ASET_BATCH) c->next *= 2; + c->n = 0; + int want = c->next; + if (want > 1 && r->npools) { + VkDescriptorSetLayout ls[ASET_BATCH]; + VkDescriptorSet got[ASET_BATCH]; + for (int i = 0; i < want; i++) ls[i] = layout; + VkDescriptorSetAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, + .descriptorPool = r->pools[r->npools - 1], .descriptorSetCount = (uint32_t)want, + .pSetLayouts = ls}; + if (vkAllocateDescriptorSets(vk_ctx()->device, &ai, got) == VK_SUCCESS) { + for (int i = 1; i < want; i++) c->set[i - 1] = got[i]; + c->n = want - 1, c->fence = ex->fence; + return got[0]; + } // the pool is nearly full: one set, from a new pool when needed + } + return alloc_set(ex, layout); +} + +// A set of this submission with layout sl and the contents ds (dbytes) written already, VK_NULL_HANDLE when none is +// known; *e is the table entry a new set goes into (bi_set_put). Entries are told apart by two independent 64-bit +// hashes of the layout and the packed descriptors. check: a hit is compared with the stored contents and not taken. +static VkDescriptorSet bi_set_find(VkExec *ex, VkDescriptorSetLayout sl, const VkDescSlot *ds, uint32_t dbytes, SetEnt **e, + uint64_t hh[2]) { + VkExecDraw *D = ex->draw; + if (!D->setc && !(D->setc = calloc(SETC_TAB, sizeof *D->setc))) rt_fatal("vk: out of memory"); + uint64_t h = (uint64_t)(uintptr_t)sl, h2 = h ^ 0x6A09E667F3BCC909ull; + const uint64_t *v = (const uint64_t *)ds; + for (uint32_t i = 0; i < dbytes / 8; i++) { + h = (h ^ v[i]) * 0x9E3779B97F4A7C15ull, h ^= h >> 29; + h2 = (h2 ^ v[i]) * 0xC2B2AE3D27D4EB4Full, h2 ^= h2 >> 31; + } + h ^= h >> 32; + SetEnt *x = *e = &D->setc[(h ^ (h >> 31)) & (SETC_TAB - 1)]; + hh[0] = h, hh[1] = h2; + D->BC.sets++; + if (x->fence != ex->fence || x->hash != h || x->hash2 != h2 || x->layout != sl || x->n != dbytes) return VK_NULL_HANDLE; + D->BC.set_shared++; + if (bind_inc_mode() == 1) return x->set; + D->BC.checked++; + if (!x->ck || memcmp(x->ck, ds, dbytes)) bi_mismatch(ex, BIM_SET, "a shared set's contents differ (%u bytes)", dbytes); + return VK_NULL_HANDLE; +} + +static void bi_set_put(VkExec *ex, SetEnt *e, const uint64_t hh[2], VkDescriptorSetLayout sl, VkDescriptorSet set, + const VkDescSlot *ds, uint32_t dbytes) { + e->fence = ex->fence, e->hash = hh[0], e->hash2 = hh[1], e->layout = sl, e->set = set, e->n = dbytes; + if (bind_inc_mode() != 2) return; + if (e->ckcap < dbytes && !(e->ck = realloc(e->ck, e->ckcap = dbytes))) rt_fatal("vk: out of memory"); + memcpy(e->ck, ds, dbytes); +} + static void support_in(const VkExec *ex, float in[3]) { // what support_fill reads: viewport size, point size float k = ex->pass_scale > 1 ? (float)ex->pass_scale : 1.0f; in[0] = ex->in_pass && ex->has_vp ? ex->vp[0][2] * k : (float)ex->pass_w; @@ -2480,8 +2560,14 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b VkDescriptorSet set = D->B.ds_last[s]; int kept = inc && bim == 1 && !changed[s] && set && D->B.ds_last_layout[s] == sl && D->DS.n[s] == dbytes; if (!kept && (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || memcmp(D->DS.sig[s], ds, dbytes))) { - set = alloc_set(ex, sl); - write_set(info, s, set, ds, w, i0, i1); + SetEnt *e = NULL; + uint64_t hh[2]; + set = bim && !ex->chunked ? bi_set_find(ex, sl, ds, dbytes, &e, hh) : VK_NULL_HANDLE; + if (!set) { + set = bim == 1 && !ex->chunked ? alloc_set_batched(ex, sl) : alloc_set(ex, sl); + write_set(info, s, set, ds, w, i0, i1); + if (e) bi_set_put(ex, e, hh, sl, set, ds, dbytes); + } D->B.ds_last[s] = set, D->B.ds_last_layout[s] = sl, D->DS.n[s] = dbytes; memcpy(D->DS.sig[s], ds, dbytes); D->BS.set_new++; @@ -2825,6 +2911,9 @@ static void bi_log(BiCensus *c) { bi_pct(c->sh_keep, c->draws), bi_pct(c->sh_memo, c->draws), bi_pct(c->pg_hit, c->draws), bi_pct(c->vb_skip, c->draws), bi_pct(c->fb_keep, c->draws), bi_pct(c->fb_n, c->draws), bi_pct(c->fb_memo, c->draws), c->mism / fr, (unsigned long long)c->mism_total, c->checked / fr); + if (c->sets) + rt_log("bind_inc: allocated graphics sets %.1f a frame, %.1f%% of them shared with an equal one of the submission", + c->sets / fr, bi_pct(c->set_shared, c->sets)); uint64_t total = c->mism_total; uint32_t lines = c->frame_lines; memset(c, 0, sizeof *c); From ecd0f004cc2984862c4052c181efed27c0dc0c5d Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 14 Nov 2026 00:18:52 +0300 Subject: [PATCH 08/10] A kept descriptor is built again when a build earlier in the same draw moved the generations it depends on (TOTK_BIND_INC) Check mode on the island save caught one kept texture descriptor whose view differed from the full build: a lookup of an earlier binding of the same draw made a texture entry (vk_texture_slots_gen moved during the build), and the full build resolved the later slot under the new generation. bi_plan now records per kept binding whether it depends on G_tex or G_buf; after a binding is built, the loop reads both again and builds every later binding that depends on one that moved. Island and Lookout Landing check, 60 s each: 0 mismatches. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index aec977c4..f634e1bd 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -327,7 +327,7 @@ typedef struct { BiMeta meta[MAX_DESC]; VkDescSlot old[MAX_DESC]; // check mode: the descriptors before the build BiMeta mold[MAX_DESC]; - uint8_t plan[MAX_BINDINGS]; // bindings to build again (bi_plan) + uint8_t plan[MAX_BINDINGS]; // per binding (bi_plan): BI_AGAIN, or the generations it depends on } BiDesc; // TOTK_BIND_INC on: graphics descriptor sets shared within a submission and allocated in batches (bi_set_find, @@ -2313,6 +2313,10 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c static int bind_inc_mode(void); static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...); +static void bi_gt(const VkExec *ex, uint64_t g[4]); +static void bi_gb(const VkExec *ex, uint64_t g[4]); +enum { BI_AGAIN = 1, BI_TEX = 2, BI_BUF = 4 }; // BiDesc.plan: build again; kept while G_tex / G_buf stay + // TOTK_BIND_INC: which bindings of a graphics draw with the last prepared draw's pipeline must be built again (BI.plan, // 1 each), from the marks since that draw and the generations G_tex and G_buf read right before this build (gt, gb) // against those read before its build. 0 when nothing can be kept: no last draw in this command buffer, another @@ -2363,7 +2367,11 @@ static int bi_plan(VkExec *ex, const VkPipeInfo *info, const uint64_t gt[4], con else again = slot >= 0 && slot < 128 && (dt->tex[st][slot >> 6] >> (slot & 63) & 1); } } - D->BI.plan[i] = (uint8_t)again; + // A kept binding also depends on the generations its build reads: a lookup or resolve of a binding built again + // in this draw may move them (an entry made, a mirror created), and the bindings after it are then built too. + int dep = (b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) ? BI_TEX + : b->kind == GFX_RES_UBO || b->kind == GFX_RES_SSBO ? BI_BUF : 0; + D->BI.plan[i] = (uint8_t)(again ? BI_AGAIN : dep); nd += cnt; } return 1; @@ -2412,6 +2420,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b memcpy(D->BI.old, d, n * sizeof *d), memcpy(D->BI.mold, meta, n * sizeof *meta); } uint32_t nd = 0; + int midmv = 0, built = 0; // inc: generations a build in this draw moved; a binding was built since the last read for (uint32_t i = 0; i < info->n_bindings; i++) { const VkPipeBinding *b = &info->bindings[i]; const GfxShaderRes *r = b->res; @@ -2421,7 +2430,20 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b VkDescSlot *o = &d[nd]; int st = nvn_stage_of(b->stage), kind = 0; // kind: 0 buffer, 1 image, 2 texel buffer view uint32_t set_of = n_sets == 2 && i >= lo[1] ? 1 : 0; - if (inc && bim == 1 && !D->BI.plan[i]) { // kept: what it counted is counted again + int keepb = 0; + if (inc && !(D->BI.plan[i] & BI_AGAIN)) { + if (built && (D->BI.plan[i] & ~midmv)) { + uint64_t g[4]; + bi_gt(ex, g); + if (memcmp(g, bgt, sizeof g)) midmv |= BI_TEX; + bi_gb(ex, g); + if (memcmp(g, bgb, sizeof g)) midmv |= BI_BUF; + built = 0; + } + keepb = !(D->BI.plan[i] & midmv); + } + if (!keepb) built = 1; + if (bim == 1 && keepb) { // kept: what it counted is counted again for (uint32_t k = 0; k < cnt; k++) if (meta[nd + k].n) vk_mem_count(meta[nd + k].cls, meta[nd + k].use, meta[nd + k].bytes); nd += cnt; @@ -2482,7 +2504,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b D->BI.cur = NULL; #undef BI_CUR if (cmp && memcmp(prev, o, cnt * sizeof *o)) changed[set_of] = 1; - if (bim == 2 && inc && !D->BI.plan[i]) { // check: on would have kept these + if (bim == 2 && keepb) { // check: on would have kept these D->BC.checked++; for (uint32_t k = 0; k < cnt; k++) { const BiMeta *mo = &D->BI.mold[nd + k], *mn = &meta[nd + k]; From e7dbe74f4454b06001ae15922183847570e44d48 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 14 Nov 2026 07:58:16 +0300 Subject: [PATCH 09/10] Graphics descriptor sets are no longer shared within a submission, and the per-draw keep tests are made once per pipeline (TOTK_BIND_INC) The set table probe cost about 54 ns a draw on Lookout Landing (a cache miss on the 4096-entry table plus two hashes of every set) and shared 6.7% of the allocated sets. Batched allocation stays. bi_plan's walk over every binding is replaced by per-binding tests (bi_tests, kept in the pipeline's RsList) that bind_resources evaluates in its own loop, with the volatile flag kept per binding. A kept binding after a build rereads only the generation group it depends on. Check mode: 0 mismatches in town, Kakariko, fight2, island and Depths, 60 s each. Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 2 +- runtime/vk_exec_draw.c | 244 +++++++++++++++++++---------------------- 2 files changed, 114 insertions(+), 132 deletions(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index e56e73f2..47fdd0fc 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -31,7 +31,7 @@ Code copied or adapted from another project is listed here with its source, the | libopus (MSYS2 `mingw-w64-ucrt-x86_64-opus`), dynamic link, `libopus-0.dll` copied next to `totk.exe` | https://opus-codec.org, Copyright (c) 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon | BSD-3-Clause | `runtime/ov_media.c` (nn::codec hardware Opus decoder) | | Dear ImGui 1.91.9 (core and Vulkan back end, unmodified), as vendored by LAS Recomp | https://github.com/ocornut/imgui, via https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f` (`third_party/imgui/`), Copyright (c) 2014-2025 Omar Cornut | MIT | `runtime/third_party/imgui/` (compiled into `totk.exe` for the in-game menu, `runtime/ui/`) | | SavageWindRecomp settings menu: two-pane option rows (row look, left / right stepping, hold speed-up, description pane), the menu style and tab bar | https://github.com/Revivedx/SavageWindRecomp @ `575ad88e02f9a89a9a885d90039d48af33c982f4` (`runtime/ui/option_rows.mm`, `runtime/ui/menu.mm`), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/ui/option_rows.cpp`, `runtime/ui/option_rows.h`, `runtime/ui/menu.cpp` | -| SavageWindRecomp descriptor sets shared within a submission (two 64-bit hashes of layout and contents) and allocated in growing batches per layout | https://github.com/Revivedx/SavageWindRecomp @ `740b18025fe11ba03c117dbb56668db304b5043e` (`runtime/nvn/vulkan/exec.c` set_finish, vk_alloc_set), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/vk_exec_draw.c` (bi_set_find, bi_set_put, alloc_set_batched) | +| SavageWindRecomp descriptor sets allocated in growing batches per layout | https://github.com/Revivedx/SavageWindRecomp @ `740b18025fe11ba03c117dbb56668db304b5043e` (`runtime/nvn/vulkan/exec.c` vk_alloc_set), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/vk_exec_draw.c` (alloc_set_batched) | | SavageWindRecomp performance overlay: the frame ring and its one-second summary, the NVML calls, the sampling loop, the full panel's layout | https://github.com/Revivedx/SavageWindRecomp @ `575ad88e02f9a89a9a885d90039d48af33c982f4` (`runtime/perfmon.c`, `runtime/perfmon.h`, `perf_overlay_draw` in `runtime/ui/menu.mm`), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/perfmon.c`, `runtime/perfmon.h`, `runtime/ui/menu.cpp` | | Roboto Medium font (Dear ImGui's `misc/fonts/Roboto-Medium.ttf`, unmodified) | https://fonts.google.com/specimen/Roboto, via https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f` (`third_party/imgui/misc/fonts/`), Copyright 2011 Google Inc. | Apache-2.0 | `runtime/ui/fonts/Roboto-Medium.ttf` (licence text in `runtime/ui/fonts/LICENSE-Roboto.txt`), embedded by `runtime/ui/menu.cpp` | | Vulkan Memory Allocator 3.4.0 (MSYS2 `mingw-w64-ucrt-x86_64-vulkan-memory-allocator`), header-only, compiled into `totk.exe` | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator, Copyright (c) 2017-2026 Advanced Micro Devices, Inc. | MIT | `runtime/vk_vma.cpp` (the allocator behind `runtime/vk_memory.c`) | diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index f634e1bd..68cb8a40 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -217,7 +217,11 @@ typedef struct { uint16_t always, ndesc; uint8_t wr; // a storage binding the shaders write: its build has side effects, nothing is kept } BiMask; -typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; BiMask m; RsRange r[]; } RsList; +// TOTK_BIND_INC: per binding of a graphics pipeline, the marks and generations whose change makes bind_resources build +// it again (bi_tests). f: BT_* flags; st: the stage the masks are of; the masks: the slots of that stage it reads. +enum { BT_ALWAYS = 1, BT_C0 = 2, BT_C1 = 4, BT_BUF = 8, BT_TEX = 16 }; +typedef struct { uint8_t f, st, dep; uint32_t ubo, ssbo, img; uint64_t tex[2]; } BiTest; +typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; BiMask m; BiTest *t; RsRange r[]; } RsList; // (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the // whole shader code (gfx_shader.c req_init), slow on CPUs without SHA instructions: a field frame looks up ~3000 and // the 2048-slot direct-mapped table this replaced missed 3% of them on conflicts. @@ -307,11 +311,11 @@ typedef struct { uint64_t frame; // gfx_counters.frames at the last draw counted uint32_t frames; // frames since the last line uint64_t draws, cls[4], b_gen, desc, rec[4], always, mv[BI_MV], slots[5]; - uint64_t sh_keep, sh_memo, fb_n, fb_keep, fb_memo, vb_skip, pg_hit, sets, set_shared; + uint64_t sh_keep, sh_memo, fb_n, fb_keep, fb_memo, vb_skip, pg_hit; uint64_t mism, mism_total, checked; uint32_t mism_frame, frame_lines; } BiCensus; -enum { BIM_PIPE, BIM_DYN, BIM_SHADER, BIM_FB, BIM_VB, BIM_DESC, BIM_SET, BIM_N }; // check mode's kinds of difference +enum { BIM_PIPE, BIM_DYN, BIM_SHADER, BIM_FB, BIM_VB, BIM_DESC, BIM_N }; // check mode's kinds of difference static int bind_inc_mode(void); static void bi_mismatch(struct VkExec *ex, int kind, const char *fmt, ...); @@ -327,22 +331,13 @@ typedef struct { BiMeta meta[MAX_DESC]; VkDescSlot old[MAX_DESC]; // check mode: the descriptors before the build BiMeta mold[MAX_DESC]; - uint8_t plan[MAX_BINDINGS]; // per binding (bi_plan): BI_AGAIN, or the generations it depends on + uint8_t vol[MAX_BINDINGS]; // per binding: a descriptor of its last build has BiMeta.vol } BiDesc; -// TOTK_BIND_INC on: graphics descriptor sets shared within a submission and allocated in batches (bi_set_find, -// alloc_set_batched). Adapted from Savage Wind Recomp runtime/nvn/vulkan/exec.c set_finish and vk_alloc_set -// (https://github.com/Revivedx/SavageWindRecomp @ 740b18025fe11ba03c117dbb56668db304b5043e, MIT, Copyright (c) 2026 -// Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai). -#define SETC_TAB 4096 -typedef struct { - uint64_t fence, hash, hash2; // the submission whose pools hold set; two hashes of layout and contents - VkDescriptorSetLayout layout; - VkDescriptorSet set; - uint32_t n; // content bytes - uint32_t ckcap; // check mode: the contents, to compare a hit with - uint8_t *ck; -} SetEnt; +// TOTK_BIND_INC on: graphics descriptor sets allocated in batches (alloc_set_batched). Adapted from Savage Wind Recomp +// runtime/nvn/vulkan/exec.c vk_alloc_set (https://github.com/Revivedx/SavageWindRecomp @ +// 740b18025fe11ba03c117dbb56668db304b5043e, MIT, Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, +// Copyright (c) 2026 Peiikai). #define ASET_CACHE 16 #define ASET_BATCH 16 typedef struct { VkDescriptorSetLayout layout; uint64_t fence; int n, next; VkDescriptorSet set[ASET_BATCH]; } ASetCache; @@ -359,7 +354,6 @@ typedef struct VkExecDraw { BiLast BL; BiCensus BC; BiDesc BI; - SetEnt *setc; // SETC_TAB entries, made on first use ASetCache aset[ASET_CACHE]; nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown const VkPipeInfo *rs_last; // the last lookup @@ -1921,39 +1915,6 @@ static VkDescriptorSet alloc_set_batched(VkExec *ex, VkDescriptorSetLayout layou return alloc_set(ex, layout); } -// A set of this submission with layout sl and the contents ds (dbytes) written already, VK_NULL_HANDLE when none is -// known; *e is the table entry a new set goes into (bi_set_put). Entries are told apart by two independent 64-bit -// hashes of the layout and the packed descriptors. check: a hit is compared with the stored contents and not taken. -static VkDescriptorSet bi_set_find(VkExec *ex, VkDescriptorSetLayout sl, const VkDescSlot *ds, uint32_t dbytes, SetEnt **e, - uint64_t hh[2]) { - VkExecDraw *D = ex->draw; - if (!D->setc && !(D->setc = calloc(SETC_TAB, sizeof *D->setc))) rt_fatal("vk: out of memory"); - uint64_t h = (uint64_t)(uintptr_t)sl, h2 = h ^ 0x6A09E667F3BCC909ull; - const uint64_t *v = (const uint64_t *)ds; - for (uint32_t i = 0; i < dbytes / 8; i++) { - h = (h ^ v[i]) * 0x9E3779B97F4A7C15ull, h ^= h >> 29; - h2 = (h2 ^ v[i]) * 0xC2B2AE3D27D4EB4Full, h2 ^= h2 >> 31; - } - h ^= h >> 32; - SetEnt *x = *e = &D->setc[(h ^ (h >> 31)) & (SETC_TAB - 1)]; - hh[0] = h, hh[1] = h2; - D->BC.sets++; - if (x->fence != ex->fence || x->hash != h || x->hash2 != h2 || x->layout != sl || x->n != dbytes) return VK_NULL_HANDLE; - D->BC.set_shared++; - if (bind_inc_mode() == 1) return x->set; - D->BC.checked++; - if (!x->ck || memcmp(x->ck, ds, dbytes)) bi_mismatch(ex, BIM_SET, "a shared set's contents differ (%u bytes)", dbytes); - return VK_NULL_HANDLE; -} - -static void bi_set_put(VkExec *ex, SetEnt *e, const uint64_t hh[2], VkDescriptorSetLayout sl, VkDescriptorSet set, - const VkDescSlot *ds, uint32_t dbytes) { - e->fence = ex->fence, e->hash = hh[0], e->hash2 = hh[1], e->layout = sl, e->set = set, e->n = dbytes; - if (bind_inc_mode() != 2) return; - if (e->ckcap < dbytes && !(e->ck = realloc(e->ck, e->ckcap = dbytes))) rt_fatal("vk: out of memory"); - memcpy(e->ck, ds, dbytes); -} - static void support_in(const VkExec *ex, float in[3]) { // what support_fill reads: viewport size, point size float k = ex->pass_scale > 1 ? (float)ex->pass_scale : 1.0f; in[0] = ex->in_pass && ex->has_vp ? ex->vp[0][2] * k : (float)ex->pass_w; @@ -2159,6 +2120,57 @@ static void bi_mask(const VkPipeInfo *info, BiMask *m) { } } +enum { BI_TEX = 1, BI_BUF = 2 }; // BiTest.dep: a kept binding stays valid while G_tex / G_buf stay + +// The rules of bi_plan per binding, made once per pipeline info: +// - support: always (its inputs are compared inside, 12 bytes); +// - uniform slot 0 (c0): a storage slot of the stage or the base instance changed; slot 1 (the compiler constants): +// the program or its serial changed, or G_buf moved; slots 3..20: the slot changed or G_buf moved (snapshots, +// overlays, pools and mirrors); other slots bind the zero buffer; +// - storage: as a uniform slot; an address held in a uniform block (ubo_ptr): always, the block changes without an op; +// - textures and images: a handle changed or G_tex moved (slot resolution, samplers, the storage check once per +// submission, the feedback layout, DLSS's swap); buffer textures bind the zero view. +// dep: the generations the binding's build reads, which a build of an earlier binding in the same draw may move. +static BiTest *bi_tests(const VkPipeInfo *info) { + BiTest *t = calloc(info->n_bindings ? info->n_bindings : 1, sizeof *t); + if (!t) rt_fatal("vk: out of memory"); + for (uint32_t i = 0; i < info->n_bindings; i++) { + const VkPipeBinding *b = &info->bindings[i]; + const GfxShaderRes *r = b->res; + uint32_t cnt = b->count ? b->count : 1; + int st = nvn_stage_of(b->stage); + BiTest *x = &t[i]; + x->st = (uint8_t)(st >= 0 && st < 6 ? st : 0); + if (b->kind == VK_PIPE_SUPPORT) { + x->f = BT_ALWAYS; + } else if (b->kind == GFX_RES_UBO) { + x->dep = BI_BUF; + for (uint32_t k = 0; k < cnt; k++) { + int slot = r->slot + (int)k; + if (slot == 0) x->f |= BT_C0; + else if (slot == 1) x->f |= BT_C1 | BT_BUF; + else if (slot >= 3 && slot - 3 < 18) x->f |= BT_BUF, x->ubo |= 1u << (slot - 3); + } + } else if (b->kind == GFX_RES_SSBO) { + x->dep = BI_BUF; + int ubo, slot = ssbo_slot(r, st, &ubo); + if (ubo || slot < 0 || slot >= 16) x->f = BT_ALWAYS; + else x->f = BT_BUF, x->ssbo = 1u << slot; + } else if (b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) { + x->dep = BI_TEX; + if (b->type == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER || b->type == VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) continue; + int img = b->kind == GFX_RES_IMG; + x->f = BT_TEX; + for (uint32_t k = 0; k < cnt; k++) { + int slot = r->slot + (int)k; + if (img && slot >= 0 && slot < 32) x->img |= 1u << slot; + else if (!img && slot >= 0 && slot < 128) x->tex[slot >> 6] |= 1ull << (slot & 63); + } + } + } + return t; +} + static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { VkExecDraw *D = ex->draw; if (info == D->rs_last) return D->rs_last_l; @@ -2211,6 +2223,7 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { l->n = n, l->bytes = (uint32_t)bytes, l->sup = (uint8_t)sup, l->c0 = (uint8_t)c0, l->ok = (uint8_t)ok; l->np = (uint8_t)np; l->m = m; + l->t = bi_tests(info); memcpy(l->p, ptr, sizeof *ptr * np); memcpy(l->r, tmp, sizeof *tmp * n); nvn_hm_put(&D->rs_lists, &info, l); @@ -2315,66 +2328,34 @@ static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...); static void bi_gt(const VkExec *ex, uint64_t g[4]); static void bi_gb(const VkExec *ex, uint64_t g[4]); -enum { BI_AGAIN = 1, BI_TEX = 2, BI_BUF = 4 }; // BiDesc.plan: build again; kept while G_tex / G_buf stay -// TOTK_BIND_INC: which bindings of a graphics draw with the last prepared draw's pipeline must be built again (BI.plan, -// 1 each), from the marks since that draw and the generations G_tex and G_buf read right before this build (gt, gb) -// against those read before its build. 0 when nothing can be kept: no last draw in this command buffer, another -// pipeline, a chunk, a storage binding the shaders write (its build restarts passes and notes writes). -// - support: always (its inputs are compared inside, 12 bytes); -// - uniform slot 0 (c0): a storage slot of the stage or the base instance changed; slot 1 (the compiler constants): -// the program or its serial changed, or G_buf moved; slots 3..20: the slot changed or G_buf moved (snapshots, -// overlays, pools and mirrors); other slots bind the zero buffer; -// - storage: as a uniform slot; an address held in a uniform block (ubo_ptr): always, the block changes without an op; -// - textures and images: a handle changed or G_tex moved (slot resolution, samplers, the storage check once per -// submission, the feedback layout, DLSS's swap); buffer textures bind the zero view; -// - any descriptor whose last build copied guest bytes or counted more than once (BiMeta): always. -static int bi_plan(VkExec *ex, const VkPipeInfo *info, const uint64_t gt[4], const uint64_t gb[4]) { +// TOTK_BIND_INC: whether a graphics draw with the last prepared draw's pipeline may keep descriptors of its last build, +// and the BiTest flags that hold for every binding of this draw (*fl), from the marks since that draw and the +// generations G_tex and G_buf read right before this build (gt, gb) against those read before its build. The tests +// (bi_tests, per binding) are evaluated in bind_resources' loop. NULL when nothing can be kept: no last draw in this +// command buffer, another pipeline, a chunk, a storage binding the shaders write (its build restarts passes and notes +// writes). A descriptor whose last build copied guest bytes or counted more than once (BiMeta, BiDesc.vol) is always +// built again. +static const BiTest *bi_plan(VkExec *ex, const VkPipeInfo *info, const uint64_t gt[4], const uint64_t gb[4], int *fl) { VkExecDraw *D = ex->draw; const BiLast *L = &D->BL; - if (!D->B.bi_valid || ex->chunked || L->info != info || D->BI.info != info) return 0; - const BiMask *m = &rs_list(ex, info)->m; - if (!m->gfx || m->wr) return 0; - const VkExecDirty *dt = &ex->dirt; - int tmv = memcmp(gt, L->gt, 4 * sizeof *gt) != 0, bmv = memcmp(gb, L->gb, 4 * sizeof *gb) != 0; - int prog = dt->prog || ex->prog != L->prog || ex->prog->serial != L->serial, binst = D->c0_binst != L->binst; - uint32_t nd = 0; - for (uint32_t i = 0; i < info->n_bindings; i++) { - const VkPipeBinding *b = &info->bindings[i]; - const GfxShaderRes *r = b->res; - uint32_t cnt = b->count ? b->count : 1; - int st = nvn_stage_of(b->stage), again = 0; - for (uint32_t k = 0; k < cnt && !again; k++) again = D->BI.meta[nd + k].vol; - if (b->kind == VK_PIPE_SUPPORT) { - again = 1; - } else if (b->kind == GFX_RES_UBO) { - for (uint32_t k = 0; k < cnt && !again; k++) { - int slot = r->slot + (int)k; - if (slot == 0) again = dt->ssbo[st] || binst; - else if (slot == 1) again = prog || bmv; - else if (slot >= 3 && slot - 3 < 18) again = bmv || (dt->ubo[st] >> (slot - 3) & 1); - } - } else if (b->kind == GFX_RES_SSBO) { - int ubo, slot = ssbo_slot(r, st, &ubo); - again = again || ubo || slot < 0 || slot >= 16 || bmv || (dt->ssbo[st] >> slot & 1); - } else if ((b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER && - b->type != VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) { - int img = b->kind == GFX_RES_IMG; - again = again || tmv; - for (uint32_t k = 0; k < cnt && !again; k++) { - int slot = r->slot + (int)k; - if (img) again = slot >= 0 && slot < 32 && (dt->img[st] >> slot & 1); - else again = slot >= 0 && slot < 128 && (dt->tex[st][slot >> 6] >> (slot & 63) & 1); - } - } - // A kept binding also depends on the generations its build reads: a lookup or resolve of a binding built again - // in this draw may move them (an entry made, a mirror created), and the bindings after it are then built too. - int dep = (b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) ? BI_TEX - : b->kind == GFX_RES_UBO || b->kind == GFX_RES_SSBO ? BI_BUF : 0; - D->BI.plan[i] = (uint8_t)(again ? BI_AGAIN : dep); - nd += cnt; - } - return 1; + if (!D->B.bi_valid || ex->chunked || L->info != info || D->BI.info != info) return NULL; + const RsList *l = rs_list(ex, info); + if (!l->m.gfx || l->m.wr) return NULL; + int f = BT_ALWAYS; + if (memcmp(gt, L->gt, 4 * sizeof *gt)) f |= BT_TEX; + if (memcmp(gb, L->gb, 4 * sizeof *gb)) f |= BT_BUF; + if (ex->dirt.prog || ex->prog != L->prog || ex->prog->serial != L->serial) f |= BT_C1; + if (D->c0_binst != L->binst) f |= BT_C0; + *fl = f; + return l->t; +} + +static inline int bi_again(const BiTest *t, const VkExecDirty *dt, int fl, uint8_t vol) { + int st = t->st; + return vol | ((t->f & fl) != 0) | ((t->f & BT_C0) && dt->ssbo[st]) | ((t->ubo & dt->ubo[st]) != 0) | + ((t->ssbo & dt->ssbo[st]) != 0) | ((t->img & dt->img[st]) != 0) | + (((t->tex[0] & dt->tex[st][0]) | (t->tex[1] & dt->tex[st][1])) != 0); } static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *bgt, const uint64_t *bgb) { @@ -2387,7 +2368,9 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b // TOTK_BIND_INC: a graphics build writes BI's array and per-descriptor census (bgt: a prepared draw's), on builds // only the bindings bi_plan names, check builds everything and compares what on would have kept. int bim = gfx && bgt ? bind_inc_mode() : 0; - int inc = bim ? bi_plan(ex, info, bgt, bgb) : 0; + int fl = 0; + const BiTest *bt = bim ? bi_plan(ex, info, bgt, bgb, &fl) : NULL; + int inc = bt != NULL; VkDescSlot *d = bim ? D->BI.d : ex->draw->d; BiMeta *meta = bim ? D->BI.meta : NULL; int skip = gfx && bim != 1 ? bind_skip_on() : 0, same = 0, was_valid = D->B.res_valid; @@ -2420,7 +2403,11 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b memcpy(D->BI.old, d, n * sizeof *d), memcpy(D->BI.mold, meta, n * sizeof *meta); } uint32_t nd = 0; - int midmv = 0, built = 0; // inc: generations a build in this draw moved; a binding was built since the last read + // inc: the generations a build in this draw moved (BI_TEX, BI_BUF); per group, a binding was built since it was + // last read. A kept binding also depends on the generations its build reads: a lookup or resolve of a binding built + // again in this draw may move them (an entry made, a mirror created), and the bindings after it are then built too. + int midmv = 0, built = 0; + const VkExecDirty *dt = &ex->dirt; for (uint32_t i = 0; i < info->n_bindings; i++) { const VkPipeBinding *b = &info->bindings[i]; const GfxShaderRes *r = b->res; @@ -2431,18 +2418,17 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b int st = nvn_stage_of(b->stage), kind = 0; // kind: 0 buffer, 1 image, 2 texel buffer view uint32_t set_of = n_sets == 2 && i >= lo[1] ? 1 : 0; int keepb = 0; - if (inc && !(D->BI.plan[i] & BI_AGAIN)) { - if (built && (D->BI.plan[i] & ~midmv)) { + if (inc && !bi_again(&bt[i], dt, fl, D->BI.vol[i])) { + int dep = bt[i].dep, rd = dep & built & ~midmv; + if (rd) { uint64_t g[4]; - bi_gt(ex, g); - if (memcmp(g, bgt, sizeof g)) midmv |= BI_TEX; - bi_gb(ex, g); - if (memcmp(g, bgb, sizeof g)) midmv |= BI_BUF; - built = 0; + if (rd & BI_TEX) bi_gt(ex, g), midmv |= memcmp(g, bgt, sizeof g) ? BI_TEX : 0; + else bi_gb(ex, g), midmv |= memcmp(g, bgb, sizeof g) ? BI_BUF : 0; + built &= ~rd; } - keepb = !(D->BI.plan[i] & midmv); + keepb = !(dep & midmv); } - if (!keepb) built = 1; + if (!keepb) built = BI_TEX | BI_BUF; if (bim == 1 && keepb) { // kept: what it counted is counted again for (uint32_t k = 0; k < cnt; k++) if (meta[nd + k].n) vk_mem_count(meta[nd + k].cls, meta[nd + k].use, meta[nd + k].bytes); @@ -2503,6 +2489,11 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b } D->BI.cur = NULL; #undef BI_CUR + if (meta) { + uint8_t v = 0; + for (uint32_t k = 0; k < cnt; k++) v |= meta[nd + k].vol; + D->BI.vol[i] = v; + } if (cmp && memcmp(prev, o, cnt * sizeof *o)) changed[set_of] = 1; if (bim == 2 && keepb) { // check: on would have kept these D->BC.checked++; @@ -2582,14 +2573,8 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info, const uint64_t *b VkDescriptorSet set = D->B.ds_last[s]; int kept = inc && bim == 1 && !changed[s] && set && D->B.ds_last_layout[s] == sl && D->DS.n[s] == dbytes; if (!kept && (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || memcmp(D->DS.sig[s], ds, dbytes))) { - SetEnt *e = NULL; - uint64_t hh[2]; - set = bim && !ex->chunked ? bi_set_find(ex, sl, ds, dbytes, &e, hh) : VK_NULL_HANDLE; - if (!set) { - set = bim == 1 && !ex->chunked ? alloc_set_batched(ex, sl) : alloc_set(ex, sl); - write_set(info, s, set, ds, w, i0, i1); - if (e) bi_set_put(ex, e, hh, sl, set, ds, dbytes); - } + set = bim == 1 && !ex->chunked ? alloc_set_batched(ex, sl) : alloc_set(ex, sl); + write_set(info, s, set, ds, w, i0, i1); D->B.ds_last[s] = set, D->B.ds_last_layout[s] = sl, D->DS.n[s] = dbytes; memcpy(D->DS.sig[s], ds, dbytes); D->BS.set_new++; @@ -2880,7 +2865,7 @@ static void bi_gb(const VkExec *ex, uint64_t g[4]) { // kind is logged with the marks, generations and counters it was taken under. static void bi_mismatch(VkExec *ex, int kind, const char *fmt, ...) { static const char *const name[BIM_N] = {"pipeline state", "recorded dynamic state", "shaders", "feedback scan", - "vertex buffers", "descriptor", "descriptor set"}; + "vertex buffers", "descriptor"}; static volatile int32_t seen[BIM_N]; VkExecDraw *D = ex->draw; D->BC.mism++, D->BC.mism_total++, D->BC.mism_frame++; @@ -2933,9 +2918,6 @@ static void bi_log(BiCensus *c) { bi_pct(c->sh_keep, c->draws), bi_pct(c->sh_memo, c->draws), bi_pct(c->pg_hit, c->draws), bi_pct(c->vb_skip, c->draws), bi_pct(c->fb_keep, c->draws), bi_pct(c->fb_n, c->draws), bi_pct(c->fb_memo, c->draws), c->mism / fr, (unsigned long long)c->mism_total, c->checked / fr); - if (c->sets) - rt_log("bind_inc: allocated graphics sets %.1f a frame, %.1f%% of them shared with an equal one of the submission", - c->sets / fr, bi_pct(c->set_shared, c->sets)); uint64_t total = c->mism_total; uint32_t lines = c->frame_lines; memset(c, 0, sizeof *c); From 2f32f21f5c7e2872745f253fbd2721355397910e Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 14 Nov 2026 20:03:24 +0300 Subject: [PATCH 10/10] The incremental draw preparation is on by default, and bind_inc and memx are listed in totk.ini.example (#381) TOTK_BIND_INC had no settings row, so unset meant off. It now has one with default 1, like TOTK_MEMX: 0 or off turns it off, check keeps the comparison mode. The check mode found 0 differences on 13 scenes (town, fight2, Kakariko, Great Sky Island, Depths lit and dark, a shrine, rain, a cave, evening, a field, the menus, entering a building). Co-Authored-By: Claude Opus 5.5 --- docs/settings.md | 2 +- runtime/cfg.c | 4 ++-- runtime/vk_exec_draw.c | 6 +++--- totk.ini.example | 6 +++++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 005cc347..d43c98a9 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -122,7 +122,7 @@ Many rows below are developer tools that only act together with another setting | Setting | What it does | Effect of changing it | Safety | | --- | --- | --- | --- | -| `bind_inc` | `on` prepares a draw from what changed since the draw before (the bind ops mark the slots whose bytes they changed) instead of from the whole bound state; `check` prepares every draw as before, takes every decision of `on` beside it, logs each kind of difference once and counts them in a `bind_inc:` line every 300 frames; `off` or unset: as before. `check` also counts what changed between draws in that line. | Less work per draw on the render thread; a change the marks missed would leave a stale descriptor, a wrong picture or a lost device, which `check` would show first. | Caution | +| `bind_inc` (default `1`) | `1` (or `on`) prepares a draw from what changed since the draw before (the bind ops mark the slots whose bytes they changed) instead of from the whole bound state; `check` prepares every draw as before, takes every decision of `on` beside it, logs each kind of difference once and counts them in a `bind_inc:` line every 300 frames; `0` or `off`: as before. `check` also counts what changed between draws in that line. | Less work per draw on the render thread; a change the marks missed would leave a stale descriptor, a wrong picture or a lost device, which `check` would show first. | Caution | | `block_views_3d` (default `0`) | `1` gives 3D block-compressed images block views too. | On some AMD cards a GPU write past the end of the image's memory loses the Vulkan device, and the game stops. | Risky | | `deswizzle` (default `auto`) | Where block-linear textures are deswizzled: `auto` or `gpu` uses a compute pass where the texture's storage is an imported pool and the CPU elsewhere, `cpu` always uses the CPU, `check` uses the GPU and also deswizzles every upload on the CPU and compares the two. | `cpu` brings back the long frames on entering new areas (the replay thread deswizzles every texture); `check` waits for the GPU at every upload and is for testing only. The log names the mode in use. | Caution | | `dyn_delta` (default `1`) | Records only the pipeline dynamic states (cull mode, depth and stencil tests, vertex input, blend enables, equations and write masks, ...) whose values changed since the last draw in the command buffer, also across pipeline changes; `0` records the full set at every change of pipeline state, as before; `ab` alternates with the `ab` blocks. | Fewer commands per pipeline change on the render and recorder threads; a driver that loses dynamic state across pipeline binds would draw with stale state. `TOTK_TRACE=perf` prints the calls and commands per submission. | Caution | diff --git a/runtime/cfg.c b/runtime/cfg.c index 248b009c..bcbaadc3 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -106,7 +106,7 @@ static const struct { const char *key, *section; } sections[] = { {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_RELEASED_MIB", "performance"}, {"TEX_BLOCK_MIB", "performance"}, {"VMA_BLOCK_MIB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, - {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"MEMX", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"}, + {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"MEMX", "performance"}, {"BIND_INC", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"}, {"SYNC_AHEAD", "performance"}, {"SHADOW_DEFER", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"OPTREF_GUARD", "performance"}, {"ABILITY_GUARD", "performance"}, {"REPORT_GUARD", "performance"}, @@ -209,7 +209,7 @@ static const CfgMeta metas[] = { {"OPTREF_GUARD", "1", "0|1", RESTART}, {"ABILITY_GUARD", "1", "0|1", RESTART}, {"REPORT_GUARD", "1", "0|1", RESTART}, // render fixes {"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART}, - {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"MEMX", "1", "0|1|check", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART}, + {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"MEMX", "1", "0|1|check", RESTART}, {"BIND_INC", "1", "0|1|check", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART}, {"VB_USED", "1", "0|1", RESTART}, {"DESC_TEMPLATE", "1", "0|1", RESTART}, {"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART}, diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 95a4d959..7af5df4e 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -2863,9 +2863,9 @@ static int pipe_memo_on(void) { static int bind_inc_mode(void) { // 0 off, 1 on, 2 check static int mode = -1; if (mode < 0) { - const char *e = cfg_get("TOTK_BIND_INC"); - int m = !e || !*e || !strcmp(e, "off") || !strcmp(e, "0") ? 0 : !strcmp(e, "check") ? 2 : 1; - if (m) rt_log("vk_exec: incremental draw preparation (TOTK_BIND_INC) %s", m == 2 ? "check" : "on"); + const char *e = cfg_value("TOTK_BIND_INC"); + int m = !strcmp(e, "off") || !strcmp(e, "0") ? 0 : !strcmp(e, "check") ? 2 : 1; + rt_log("vk_exec: incremental draw preparation (TOTK_BIND_INC) %s", m == 2 ? "check" : m ? "on" : "off"); mode = m; } return mode; diff --git a/totk.ini.example b/totk.ini.example index 1ad08738..255ecc0e 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -125,7 +125,9 @@ # fix_zero = 1 # Optimisations, 0 turns each off: skip bindings that did not change, remember shader lookups, reuse the previous # draw's pipeline when its state repeats, remember texture descriptors between draws, update descriptors in bulk, -# skip push constants that did not change, bind only the vertex buffers a draw reads. +# skip push constants that did not change, bind only the vertex buffers a draw reads, compare and copy each draw's +# state inline, and rebuild only the parts of a draw's bindings, shaders and pipeline whose inputs changed since the +# draw before (check: build everything and log any difference from what would have been kept). # bind_skip = 1 # shader_memo = 1 # pipe_memo = 1 @@ -133,6 +135,8 @@ # desc_template = 1 # push_skip = 1 # vb_used = 1 +# memx = 1 +# bind_inc = 1 # Submit GPU work in parts every N draws between passes so the GPU starts earlier (0 to 1000; 0 = off), each part at # least split_us microseconds long (0 to 20000). # split_draws = 64