From b2730059012d2de69930fc3ce0f10c80b11e54dc Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 26 Oct 2026 22:31:53 +0300 Subject: [PATCH 1/8] Look up texture types only for the slots a shader's translation reads, and reuse the previous draw's pipeline while its state repeats answers_for looked up the type of all 64 texture and image handles of a stage whenever one changed; the use mask of the first shader of each program stage now limits it to the slots that select the variant, with a generation per slot and the texture lock taken only on a miss (texture type recomputations per draw 0.39 to 0.05 in fight). prepare_draw keeps the previous draw's pipeline info while the pipeline state is equal (TOTK_PIPE_MEMO), and reads the clock only for draws that may wait. Fight on the 7900X: replay 15.7 to 14.0 ms per frame, 53.3 to 58.7 fps; field 11.1 to 9.5 ms. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 7 ++- runtime/gfx.h | 1 + runtime/gfx_shader.c | 4 ++ runtime/ui/menu.cpp | 1 + runtime/ui_text.c | 2 + runtime/ui_text.h | 1 + runtime/vk_exec_draw.c | 135 ++++++++++++++++++++++++----------------- 7 files changed, 94 insertions(+), 57 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index 828dbf16..e15c1f07 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -39,7 +39,7 @@ static const char *const known[] = { "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MENU_LANG", "MENU_TEST", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", - "OPERATION_MODE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", + "OPERATION_MODE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", @@ -65,7 +65,7 @@ static const struct { const char *key, *section; } sections[] = { {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, - {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, + {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, @@ -104,7 +104,8 @@ static const CfgMeta metas[] = { {"TEX_ALIAS", "1", "0|1", RESTART}, // render fixes {"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART}, - {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"DESC_TEMPLATE", "1", "0|1", RESTART}, + {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, + {"DESC_TEMPLATE", "1", "0|1", RESTART}, {"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART}, // Vulkan {"VK_VALIDATION", "0", "0|1", RESTART}, {"VK_DYN_VERTEX", "", "0|1", RESTART}, diff --git a/runtime/gfx.h b/runtime/gfx.h index 2ef5c446..daa91179 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -93,6 +93,7 @@ typedef struct { typedef struct { uint8_t type, comps, packed; } GfxShaderIo; // GFX_TYPE_*, components, 1 with a Component decoration int gfx_shader_stage(const GfxShader *s); // GFX_STAGE_VS, GFX_STAGE_FS or GFX_STAGE_CS int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res); // count of GFX_RES_* kind; array in *res +uint32_t gfx_shader_slot_mask(const GfxShader *s); // texture/image slots whose type its translation read uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]); // Input Locations (VS attributes, FS varyings) as a mask uint32_t gfx_shader_outputs(const GfxShader *s, GfxShaderIo io[32]); // Output Locations (VS varyings, FS colour targets) void gfx_shader_local_size(const GfxShader *s, uint32_t local[3]); // CS: the LocalSize execution mode; 1, 1, 1 otherwise diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index ebe2b488..099218db 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1418,6 +1418,10 @@ int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res) return s->nres[kind]; } +// The slots of the use mask: answers outside it select the same translation (mask_answers), so a caller may leave +// their texture and image types zero. Parses the reflection: callers keep the result. +uint32_t gfx_shader_slot_mask(const GfxShader *s) { return use_of(s).slots; } + uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]) { if (io) memcpy(io, s->in, sizeof s->in); return s->in_mask; diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index fb4e7624..9771d416 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -608,6 +608,7 @@ static void tab_advanced(float scale) { opt_toggle("TOTK_FIX_ZERO", UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC); opt_toggle("TOTK_BIND_SKIP", UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC); opt_toggle("TOTK_SHADER_MEMO", UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC); + opt_toggle("TOTK_PIPE_MEMO", UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC); opt_toggle("TOTK_DESC_TEMPLATE", UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC); opt_toggle("TOTK_PUSH_SKIP", UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC); opt_slider("TOTK_SPLIT_DRAWS", UI_MENU_SPLIT_DRAWS, 0, 1000, UI_MENU_SPLIT_DRAWS_DESC, 8, "%.0f"); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 69af8480..084fef38 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -127,6 +127,7 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_BIND_SKIP] = "Skip repeated binds", [UI_MENU_BIND_SKIP_DESC] = "Optimisation: skips bindings that did not change.", [UI_MENU_SHADER_MEMO] = "Shader memo", [UI_MENU_SHADER_MEMO_DESC] = "Optimisation: remembers shader lookups.", + [UI_MENU_PIPE_MEMO] = "Pipeline memo", [UI_MENU_PIPE_MEMO_DESC] = "Optimisation: reuses the previous draw's pipeline when its state repeats.", [UI_MENU_DESC_TEMPLATE] = "Descriptor templates", [UI_MENU_DESC_TEMPLATE_DESC] = "Optimisation: updates descriptors in bulk.", [UI_MENU_PUSH_SKIP] = "Skip repeated push constants", @@ -271,6 +272,7 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_BIND_SKIP] = "Пропуск повторных привязок", [UI_MENU_BIND_SKIP_DESC] = "Оптимизация: пропускает неизменившиеся привязки.", [UI_MENU_SHADER_MEMO] = "Память шейдеров", [UI_MENU_SHADER_MEMO_DESC] = "Оптимизация: запоминает поиск шейдеров.", + [UI_MENU_PIPE_MEMO] = "Память конвейера", [UI_MENU_PIPE_MEMO_DESC] = "Оптимизация: повторяет конвейер прошлой отрисовки, если её состояние не изменилось.", [UI_MENU_DESC_TEMPLATE] = "Шаблоны дескрипторов", [UI_MENU_DESC_TEMPLATE_DESC] = "Оптимизация: обновляет дескрипторы пачками.", [UI_MENU_PUSH_SKIP] = "Пропуск повторных push-констант", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index e391aeb3..7860bbd7 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -84,6 +84,7 @@ typedef enum { UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC, UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC, UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC, + UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC, UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC, UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC, UI_MENU_SPLIT_DRAWS, UI_MENU_SPLIT_DRAWS_DESC, diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 30b61352..21e5414d 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -548,12 +548,38 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const return 0; } -// Texture and image types of a stage's first 32 slots, kept while every slot's resolution (vk_texture_slots_gen) -// stays the same: only slots whose handle changed are looked up again, each lookup being a hash probe. -static struct { int valid; uint32_t gen; uint64_t tex[32], img[32]; uint8_t tt[32], it[32]; } AC[6]; +// Texture and image types of a stage's first 32 slots, each kept with the handle and the resolution generation +// (vk_texture_slots_gen) it was looked up under: a slot is looked up again, a hash probe, only when either changed. +static struct { uint64_t tex[32], img[32]; uint32_t tgen[32], igen[32]; uint8_t tv[32], iv[32], tt[32], it[32]; } AC[6]; static int shader_memo_on(void); -static void answers_for(VkExec *ex, int st, const uint8_t *ctl, GfxShaderAnswers *a) { +// The slots of (program, stage) whose types its translation reads (gfx_shader_slot_mask), learned from its first +// shader; ~0u while unknown. The types of other slots select the same shader, so answers_for leaves them zero instead +// of looking up all 64 handles. Direct-mapped by serial and stage; a collision only relearns. +#define SLOTM_N 4096 +static struct { uint64_t serial; int st; uint32_t m; } slotm[SLOTM_N]; + +static uint32_t slotm_idx(uint64_t serial, int st) { return (uint32_t)((serial * 8 + (uint64_t)st) * 0x9E3779B97F4A7C15ull >> 52); } + +static uint32_t slot_mask_of(uint64_t serial, int st) { + const typeof(slotm[0]) *e = &slotm[slotm_idx(serial, st)]; + return serial && e->serial == serial && e->st == st ? e->m : ~0u; +} + +static void slot_mask_learn(uint64_t serial, int st, const GfxShader *s) { + if (!serial || slot_mask_of(serial, st) != ~0u) return; + slotm[slotm_idx(serial, st)] = (typeof(slotm[0])){serial, st, gfx_shader_slot_mask(s)}; +} + +static uint8_t type_of_handle(uint64_t h) { // texture lock held + view_t v; + int hv; + const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; + return img ? (uint8_t)gfx_type_of(img, &v, hv) : 0; +} + +// The answers of stage st for the slots in mask; the types of other slots stay zero. +static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, GfxShaderAnswers *a) { memset(a, 0, sizeof *a); if (st == NVN_STAGE_VS) vk_pipeline_answers(&ex->pipe, a, NULL); else if (st == NVN_STAGE_FS) vk_pipeline_answers(&ex->pipe, NULL, a); @@ -564,44 +590,23 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, GfxShaderAnswers a->local[k] = (uint16_t)(l ? l : 1); } typeof(AC[0]) *c = &AC[st]; - vk_texture_lock(); - uint32_t gen = vk_texture_slots_gen(); // before the lookups: one that creates an entry bumps it - int all = !c->valid || c->gen != gen; - if (all || memcmp(c->tex, ex->tex[st], sizeof c->tex) || memcmp(c->img, ex->img[st], sizeof c->img)) { - for (int slot = 0; slot < 32; slot++) { - view_t v; - int hv; - uint64_t h = ex->tex[st][slot]; - if (all || h != c->tex[slot]) { - const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; - c->tt[slot] = img ? (uint8_t)gfx_type_of(img, &v, hv) : 0; - } - h = ex->img[st][slot]; - if (all || h != c->img[slot]) { - const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; - c->it[slot] = img ? (uint8_t)gfx_type_of(img, &v, hv) : 0; - } - } - memcpy(c->tex, ex->tex[st], sizeof c->tex); - memcpy(c->img, ex->img[st], sizeof c->img); - c->gen = gen, c->valid = 1; - BS.ac_miss++; + // Read before the lookups (one that creates an entry bumps it) and unlocked: a registration racing with it is + // ordered after this draw, and a slot stored under an older generation is only looked up again. + uint32_t gen = vk_texture_slots_gen(); + int locked = 0, check = shader_memo_on() == 2; + for (uint32_t m = mask; m; m &= m - 1) { + int slot = __builtin_ctz(m); + uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot]; + int tmiss = !c->tv[slot] || c->tgen[slot] != gen || c->tex[slot] != h; + int imiss = !c->iv[slot] || c->igen[slot] != gen || c->img[slot] != hi; + if ((tmiss || imiss || check) && !locked) vk_texture_lock(), locked = 1, BS.ac_miss++; + if (tmiss) c->tt[slot] = type_of_handle(h), c->tex[slot] = h, c->tgen[slot] = gen, c->tv[slot] = 1; + if (imiss) c->it[slot] = type_of_handle(hi), c->img[slot] = hi, c->igen[slot] = gen, c->iv[slot] = 1; + if (check && (type_of_handle(h) != c->tt[slot] || type_of_handle(hi) != c->it[slot])) // TOTK_SHADER_MEMO=check + rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot); + a->tex_type[slot] = c->tt[slot], a->img_type[slot] = c->it[slot]; } - if (shader_memo_on() == 2) // check mode: every slot looked up again - for (int slot = 0; slot < 32; slot++) { - view_t v; - int hv; - uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot]; - const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; - int tt = img ? gfx_type_of(img, &v, hv) : 0; - img = hi ? vk_texture_by_id_locked((int)(hi & 0xfffff), &v, &hv) : NULL; - int it = img ? gfx_type_of(img, &v, hv) : 0; - if (tt != c->tt[slot] || it != c->it[slot]) - rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot); - } - vk_texture_unlock(); - memcpy(a->tex_type, c->tt, sizeof c->tt); - memcpy(a->img_type, c->it, sizeof c->it); + if (locked) vk_texture_unlock(); } // *present: the program has this stage. Returns NULL when it has not or the translation failed. pending (graphics @@ -615,8 +620,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (!ex->prog || !find_stage(ex->prog, st, &ctl, &code, &len)) return NULL; *present = 1; GfxShaderAnswers a; - answers_for(ex, st, ctl, &a); uint64_t serial = ex->prog->serial; + answers_for(ex, st, ctl, slot_mask_of(serial, st), &a); ShaderEnt *set = NULL; if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded @@ -637,6 +642,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin : pending && vk_pipeline_async_on() ? gfx_shader_get_async(gs, ctl, code, len, &a, pending) : gfx_shader_get(gs, ctl, code, len, NULL, &a); if (pending && *pending) return NULL; + if (s) slot_mask_learn(serial, st, s); if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st, (unsigned long long)serial); else if (set) { // in front, the least recently used way drops out @@ -1580,6 +1586,13 @@ static int shader_memo_on(void) { return SM.on; } +static struct { int on, valid; VkPipeState s; const VkPipeInfo *info; } PM = {-1}; + +static int pipe_memo_on(void) { + if (PM.on < 0) PM.on = cfg_bool("TOTK_PIPE_MEMO"); + return PM.on; +} + // Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped. static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (ex->render_disabled) return 0; @@ -1652,19 +1665,33 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { wait_ms = (uint32_t)(wait_left_ns / 1000000); if (wait_ms) t0 = kern_now_ns(); } - uint64_t w0 = kern_now_ns(); - VkPipeline p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); - if (t0) { - uint64_t spent = kern_now_ns() - t0; - wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0; + // The previous draw's pipeline while its state repeats (PM, TOTK_PIPE_MEMO=0 turns it off): the cache key is made + // from the state alone, and a pipeline once found only changes from its fast link to its optimised build, read + // from the info the cache entry keeps. + VkPipeline p; + const char *why = NULL; + uint64_t w0 = 0; + if (pipe_memo_on() && PM.valid && !memcmp(&PM.s, &s, sizeof s)) { + info = PM.info, pending = 0; + p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); + } else { + w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait + p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); + if (t0) { + uint64_t spent = kern_now_ns() - t0; + wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0; + } + why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL; + if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) { + perf_count(PERF_CNT_ONESHOT_WAIT); + why = "one-shot"; + if (!w0) w0 = kern_now_ns(); + p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS); + } + PM.valid = !pending && p && info; + if (PM.valid) PM.s = s, PM.info = info; } - const char *why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL; - if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) { - perf_count(PERF_CNT_ONESHOT_WAIT); - why = "one-shot"; - p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS); - } - uint64_t waited = kern_now_ns() - w0; + uint64_t waited = w0 ? kern_now_ns() - w0 : 0; static int wait_logs; if (why && waited >= 2000000 && wait_logs < 200) { char tg[512]; From 1836d5c5d0e1f2227510e2bf264a0144e55e112d Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 26 Oct 2026 22:42:47 +0300 Subject: [PATCH 2/8] Keep texture image views for the image's lifetime instead of creating them again in every command buffer Views made per command buffer were created anew every frame: vkCreateImageView under get_view took about 8% of the fight replay on the 7900X. They now live in one table keyed as before, used under the texture lock, and destroy_tex hands an image's views to the deferred release with the image (vk_exec_image_gone). The buffer range limits ubo_info and ssbo_info read on every binding are read once. Fight replay 14.0 to 12.5 ms per frame (15.7 before the branch), field 9.5 to 9.2 ms; a fight run under the validation layer shows only the known cp_t_tcb_20 message. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 1 + runtime/vk_exec_draw.c | 56 ++++++++++++++++++++++++++++++------------ runtime/vk_texture.c | 1 + 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/runtime/vk.h b/runtime/vk.h index db2507e4..69144a01 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -198,6 +198,7 @@ uint64_t vk_upload_flush(int wait); // Deferred destruction: freed once the upload batches recorded so far and the timeline value last noted have completed. void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestroyImage only void vk_defer_view(VkImageView view); +void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc); // ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c. enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK }; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 21e5414d..9c9f25a1 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -67,7 +67,6 @@ typedef struct { VkImageViewType type; uint32_t aspect, level0, levels, layer0, layers, swz; } ViewKey; -typedef struct { ViewKey k; VkImageView view; } ViewEnt; _Static_assert(sizeof(ViewKey) == 40, "ViewKey is hashed as bytes: no padding"); typedef struct { uint64_t addr, size; } Range; typedef struct { @@ -75,9 +74,6 @@ typedef struct { int nchunks, cchunks; VkDescriptorPool *pools; int npools, cpools; - ViewEnt *views; - int nviews, cviews; - nvn_hmap view_of; // ViewKey -> the VkImageView in views Range *written; int nwritten, cwritten; } CbRes; @@ -175,7 +171,6 @@ static void cb_done(void *arg) { // completion thread CbRes *r = arg; VkCtx *c = vk_ctx(); for (int i = 0; i < r->nwritten; i++) vk_guest_written(r->written[i].addr, r->written[i].size); - for (int i = 0; i < r->nviews; i++) vkDestroyImageView(c->device, r->views[i].view, NULL); for (int i = 0; i < r->npools; i++) { vkResetDescriptorPool(c->device, r->pools[i], 0); AcquireSRWLockExclusive(&spare_lock); @@ -191,8 +186,7 @@ static void cb_done(void *arg) { // completion thread ReleaseSRWLockExclusive(&spare_lock); if (!kept) vmaDestroyBuffer(c->vma, r->chunks[i].buf, r->chunks[i].alloc); } - free(r->view_of.keys), free(r->view_of.vals); - free(r->written), free(r->views), free(r->pools), free(r->chunks), free(r); + free(r->written), free(r->pools), free(r->chunks), free(r); } void vk_exec_draw_new_cb(void) { memset(&B, 0, sizeof B); } @@ -201,7 +195,6 @@ static CbRes *cb_res(VkExec *ex) { if (cur && cur_fence == ex->fence) return cur; cur = calloc(1, sizeof *cur); if (!cur) rt_fatal("vk: out of memory"); - nvn_hm_init(&cur->view_of, sizeof(ViewKey)); cur_fence = ex->fence; memset(&B, 0, sizeof B); vk_completion_add(ex->fence, cb_done, cur); @@ -455,8 +448,15 @@ VkImageAspectFlags vk_exec_view_aspect(const VkTexImage *img, const view_t *v, i : VK_IMAGE_ASPECT_COLOR_BIT; } +// Image views of registered textures, kept for their image's lifetime: made per command buffer they were created again +// every frame (vkCreateImageView took about 8% of the fight replay on the 7900X). Used under the texture lock, as +// vk_exec_image_gone is (destroy_tex), which hands the views of a destroyed image to the deferred release with it. +typedef struct ViewNode { struct ViewNode *next; ViewKey k; VkImageView view; } ViewNode; +static nvn_hmap view_of; // ViewKey -> VkImageView +static nvn_hmap views_by_image; // VkImage -> its first ViewNode + // An image view of a registered texture for a shader resource of type `rtype` (GfxShaderRes.type); VK_NULL_HANDLE when -// the image cannot serve it (the caller binds a stand-in). +// the image cannot serve it (the caller binds a stand-in). Texture lock held. static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, int hv, uint16_t rtype, int storage) { VkImageViewType vt = res_view_type(rtype); int ms = (rtype & 0x200) != 0; @@ -505,9 +505,9 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, } } uint32_t swz = (uint32_t)sw[0] | (uint32_t)sw[1] << 8 | (uint32_t)sw[2] << 16 | (uint32_t)sw[3] << 24; - CbRes *r = cb_res(ex); ViewKey k = {img->image, img->format, vt, aspect, l0, nl, y0, ny, swz}; - VkImageView found = (VkImageView)nvn_hm_get(&r->view_of, &k); + if (!view_of.klen) nvn_hm_init(&view_of, sizeof(ViewKey)), nvn_hm_init(&views_by_image, sizeof(VkImage)); + VkImageView found = (VkImageView)nvn_hm_get(&view_of, &k); if (found) return found; VkImageViewCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = img->image, .viewType = vt, .format = img->format, @@ -518,12 +518,24 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c) VkImageView view; if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE; - if (r->nviews == r->cviews) r->views = grow(r->views, &r->cviews, sizeof *r->views); - r->views[r->nviews++] = (ViewEnt){k, view}; - nvn_hm_put(&r->view_of, &k, (void *)view); + ViewNode *n = malloc(sizeof *n); + if (!n) rt_fatal("vk: out of memory"); + *n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view}; + nvn_hm_put(&views_by_image, &img->image, n); + nvn_hm_put(&view_of, &k, (void *)view); return view; } +void vk_exec_image_gone(VkImage image) { + if (!views_by_image.klen) return; + for (ViewNode *n = nvn_hm_del(&views_by_image, &image), *next; n; n = next) { + next = n->next; + nvn_hm_del(&view_of, &n->k); + vk_defer_view(n->view); + free(n); + } +} + // ---- program to shaders // (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the @@ -766,8 +778,19 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; } +// The buffer range limits, read once: the device's properties are a cold cache line on every binding of every draw. +static struct { uint64_t ubo_max, ubo_align, sbo_max, sbo_align; } LIM; + +static void limits_init(void) { + const VkPhysicalDeviceLimits *l = &vk_ctx()->props.limits; + LIM.ubo_align = l->minUniformBufferOffsetAlignment, LIM.sbo_max = l->maxStorageBufferRange; + LIM.sbo_align = l->minStorageBufferOffsetAlignment; + __atomic_store_n(&LIM.ubo_max, l->maxUniformBufferRange, __ATOMIC_RELEASE); +} + static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { - uint64_t max = vk_ctx()->props.limits.maxUniformBufferRange, align = vk_ctx()->props.limits.minUniformBufferOffsetAlignment; + if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); + uint64_t max = LIM.ubo_max, align = LIM.ubo_align; VkDescriptorBufferInfo bi; if (slot == 0 && c0_on()) return c0_info(ex, st); if (slot == 1 && ex->prog) { // compiler constants stored after the shader code @@ -851,7 +874,8 @@ static uint64_t ubo_ptr(const VkExec *ex, int st, const GfxShaderRes *r) { } static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *r, int written) { - uint64_t max = vk_ctx()->props.limits.maxStorageBufferRange, align = vk_ctx()->props.limits.minStorageBufferOffsetAlignment; + if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); + uint64_t max = LIM.sbo_max, align = LIM.sbo_align; VkDescriptorBufferInfo bi; int ubo, slot = ssbo_slot(r, st, &ubo); p_bind_buf up = {0}; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 75352880..70c858ae 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -426,6 +426,7 @@ static void destroy_tex(Tex *e) { if (p->next_at == e) { p->next_at = e->next_at; break; } } if (e->released) nreleased--; + vk_exec_image_gone(e->pub.image); vk_defer_image(e->pub.image, e->alloc); free(e); } From 6b65fb6c0e2529ca149ed1d2a0615f64b4c00f2f Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 26 Oct 2026 22:50:22 +0300 Subject: [PATCH 3/8] Keep sampled-texture descriptors between draws while the slot, sampler, submission and feedback pass stay the same tex_info resolved the slot, built a view key, probed the view table and looked up the sampler for every texture binding of every draw. A direct-mapped memo keyed by handle, resource type and separate returns the descriptor while vk_texture_slots_gen, vk_sampler_gen, the submission epoch and (with feedback-loop images) the pass are unchanged; storage images are not kept. TOTK_TEX_MEMO=0 turns it off, =check recomputes every hit (a fight run with it and TOTK_SHADER_MEMO=check stopped on nothing). Against main, profiler on, no bind stats: fight replay 13.9 to 11.9 ms per frame, render thread busy 16.3 to 14.6 ms, CPU-bound windows 15 to 9; field replay 9.5 to 8.2 ms. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 6 +++--- runtime/ui/menu.cpp | 1 + runtime/ui_text.c | 2 ++ runtime/ui_text.h | 1 + runtime/vk_exec_draw.c | 41 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 48 insertions(+), 3 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index e15c1f07..b607d728 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -45,7 +45,7 @@ static const char *const known[] = { "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS", "TRACE", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VS_OUTS", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", }; @@ -65,7 +65,7 @@ static const struct { const char *key, *section; } sections[] = { {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, - {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, + {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, @@ -104,7 +104,7 @@ static const CfgMeta metas[] = { {"TEX_ALIAS", "1", "0|1", RESTART}, // render fixes {"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART}, - {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, + {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART}, {"DESC_TEMPLATE", "1", "0|1", RESTART}, {"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART}, // Vulkan diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 9771d416..29417698 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -609,6 +609,7 @@ static void tab_advanced(float scale) { opt_toggle("TOTK_BIND_SKIP", UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC); opt_toggle("TOTK_SHADER_MEMO", UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC); opt_toggle("TOTK_PIPE_MEMO", UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC); + opt_toggle("TOTK_TEX_MEMO", UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC); opt_toggle("TOTK_DESC_TEMPLATE", UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC); opt_toggle("TOTK_PUSH_SKIP", UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC); opt_slider("TOTK_SPLIT_DRAWS", UI_MENU_SPLIT_DRAWS, 0, 1000, UI_MENU_SPLIT_DRAWS_DESC, 8, "%.0f"); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 084fef38..2347af64 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -128,6 +128,7 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_BIND_SKIP_DESC] = "Optimisation: skips bindings that did not change.", [UI_MENU_SHADER_MEMO] = "Shader memo", [UI_MENU_SHADER_MEMO_DESC] = "Optimisation: remembers shader lookups.", [UI_MENU_PIPE_MEMO] = "Pipeline memo", [UI_MENU_PIPE_MEMO_DESC] = "Optimisation: reuses the previous draw's pipeline when its state repeats.", + [UI_MENU_TEX_MEMO] = "Texture binding memo", [UI_MENU_TEX_MEMO_DESC] = "Optimisation: remembers texture descriptors between draws.", [UI_MENU_DESC_TEMPLATE] = "Descriptor templates", [UI_MENU_DESC_TEMPLATE_DESC] = "Optimisation: updates descriptors in bulk.", [UI_MENU_PUSH_SKIP] = "Skip repeated push constants", @@ -273,6 +274,7 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_BIND_SKIP_DESC] = "Оптимизация: пропускает неизменившиеся привязки.", [UI_MENU_SHADER_MEMO] = "Память шейдеров", [UI_MENU_SHADER_MEMO_DESC] = "Оптимизация: запоминает поиск шейдеров.", [UI_MENU_PIPE_MEMO] = "Память конвейера", [UI_MENU_PIPE_MEMO_DESC] = "Оптимизация: повторяет конвейер прошлой отрисовки, если её состояние не изменилось.", + [UI_MENU_TEX_MEMO] = "Память привязок текстур", [UI_MENU_TEX_MEMO_DESC] = "Оптимизация: запоминает дескрипторы текстур между отрисовками.", [UI_MENU_DESC_TEMPLATE] = "Шаблоны дескрипторов", [UI_MENU_DESC_TEMPLATE_DESC] = "Оптимизация: обновляет дескрипторы пачками.", [UI_MENU_PUSH_SKIP] = "Пропуск повторных push-констант", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 7860bbd7..3a8dc234 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -85,6 +85,7 @@ typedef enum { UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC, UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC, UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC, + UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC, UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC, UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC, UI_MENU_SPLIT_DRAWS, UI_MENU_SPLIT_DRAWS_DESC, diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 9c9f25a1..3c74c7c4 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -999,7 +999,48 @@ static void feedback_written(VkExec *ex, const VkPipeState *s) { if (ex->color[i].image && !(ex->fb_dirty >> i & 1) && vk_pipeline_target_wmask(s, (int)i)) ex->fb_dirty |= 1u << i; } +static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, + VkDescriptorImageInfo *out); + +// Sampled-texture descriptors per (handle, resource type, separate), kept while nothing they derive from changed: every +// slot's resolution and texture entry (vk_texture_slots_gen), the samplers (vk_sampler_gen), the submission +// (vk_texture_epoch: the first binding of a slot in a submission checks its storage) and, in a pass with images in the +// feedback layout, the pass. Draws bind the same textures thousands of times a frame; a hit skips the slot lookup, the +// view key with its hash probe and the sampler lookup. Storage images, which note their binding per submission, are +// not kept. Direct-mapped; texture lock held. TOTK_TEX_MEMO=0 turns it off; =check computes every hit again and stops +// on a difference. +#define TD_N 4096 +static struct { uint64_t h; uint32_t gen, epoch, sgen, pass; uint16_t type; uint8_t separate, valid; VkDescriptorImageInfo out; } TD[TD_N]; +static int td_on = -1; + static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) { + if (td_on < 0) { + const char *e = cfg_value("TOTK_TEX_MEMO"); + td_on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; + } + if (!h || storage || !td_on) return tex_info_full(ex, r, h, storage, separate, out); + uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup + uint32_t pass = ex->nfbl ? ex->pass_serial | 0x80000000u : 0; + typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 52]; + if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->gen == gen && m->epoch == ep && + m->sgen == sg && m->pass == pass) { + *out = m->out; + if (td_on == 2) { + VkDescriptorImageInfo full; + tex_info_full(ex, r, h, storage, separate, &full); + if (full.sampler != out->sampler || full.imageView != out->imageView || full.imageLayout != out->imageLayout) + rt_fatal("vk_exec: texture descriptor memo of handle %#llx type %#x differs from a lookup", + (unsigned long long)h, r->type); + } + return; + } + tex_info_full(ex, r, h, storage, separate, out); + *m = (typeof(*m)){.h = h, .gen = gen, .epoch = ep, .sgen = sg, .pass = pass, .type = r->type, + .separate = (uint8_t)separate, .valid = 1, .out = *out}; +} + +static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, + VkDescriptorImageInfo *out) { int ms = (r->type & 0x200) != 0; VkImageView view = VK_NULL_HANDLE; VkSampler samp = VK_NULL_HANDLE; From 353c23720841356a996e6f67b364631a5c39c35f Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 00:55:17 +0300 Subject: [PATCH 4/8] Count the replay's buffer traffic with plain adds: the replay is the census's only writer, and the locked add was 2% of fight replay Co-Authored-By: Claude Opus 5.5 --- runtime/vk_memory.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index c1103b4a..e66ada31 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -467,8 +467,12 @@ int vk_mem_class_of(uint64_t addr) { // ---------------------------------------------------------------- census static volatile LONG64 traffic[VK_MCLS_N][VK_MUSE_N]; +// Every caller is the replay (vk_exec*, under vk_sync_submit's lock), about ten adds per draw: one writer at a time, so +// a plain add, not a locked one. An add racing the census's reset may be lost. void vk_mem_count(int cls, int use, uint64_t bytes) { - if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N) InterlockedAdd64(&traffic[cls][use], (LONG64)bytes); + if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N) + __atomic_store_n(&traffic[cls][use], __atomic_load_n(&traffic[cls][use], __ATOMIC_RELAXED) + (LONG64)bytes, + __ATOMIC_RELAXED); } static void flag_names(uint32_t f, char *buf, size_t n) { From bcbbcb0b1a615d2811c2461fb9b68a5af97b3e0b Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 00:55:17 +0300 Subject: [PATCH 5/8] Keep texture lookups per slot instead of per global texture generation, and vertex buffer spans while their ranges repeat TotK finalizes and initializes transient textures every frame: the global tex_gen and slots_gen moved about every 16 draws in the monsters fight, so every cache keyed on them (the slot cache in vk_texture_by_id_locked, the texture types of answers_for, the texture descriptor memo) lived for a few draws. A slot resolves through the entries at its own storage address only, so each address hash now has its own generation and each slot a token (vk_texture_slot_token: registration serial and address generation, lock-free). The memos check the global generation first and the slot's token only when it moved. The descriptor memo no longer drops an entry at each submission (the first binding still makes the storage check; a re-upload writes the same image) or pass (the feedback layout is set on each hit from the kept image), and is twice the size. feedback_scan keeps each handle's sampled subresource across passes instead of looking up every fragment texture after every texture bind. bind_vertex_buffers keeps each stream's resolved span while its range and vk_mem_gen stay the same. Monsters fight on the RX 9070: replay 12.0 -> 10.8 ms per frame; field 8.1 -> 7.4 ms. TOTK_TEX_MEMO=check and TOTK_SHADER_MEMO=check runs found no difference. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 4 + runtime/vk_exec_draw.c | 186 ++++++++++++++++++++++++++++++----------- runtime/vk_texture.c | 62 ++++++++++---- 3 files changed, 189 insertions(+), 63 deletions(-) diff --git a/runtime/vk.h b/runtime/vk.h index 69144a01..17e579f2 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -264,6 +264,10 @@ VkSampler vk_sampler_by_id_locked(int id); // on the Switch although the game binds a linear sampler (MacNativeGames 6283599). VK_NULL_HANDLE for an empty slot. VkSampler vk_sampler_point_by_id_locked(int id); uint32_t vk_texture_slots_gen(void); +// What vk_texture_by_id(id) returns can change only when this does: the slot is registered again, or a texture entry at +// its storage address is created, dropped, released or taken back into use. Unlocked; read it before the lookup it +// guards (a lookup may create an entry). 0 when the slot was never registered: nothing is kept under 0. +uint64_t vk_texture_slot_token(int id); uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 3c74c7c4..2da0075d 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -98,6 +98,10 @@ static struct { int st_valid; VkBuffer vb[17]; // vertex buffers last bound VkDeviceSize vo[17], vsz[17], vstr[17]; + int vcls[17]; // their pool class and pool record, for the census + void *vpool[17]; + NVNbufferRange vin[16]; // the ranges they were resolved from, under vk_mem_gen vgen + uint64_t vgen; int vb_valid; uint8_t support[SUPPORT_BYTES]; // last support buffer and where it lives float sup_in[3]; // the support_fill arguments it was filled from @@ -560,9 +564,24 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const return 0; } -// Texture and image types of a stage's first 32 slots, each kept with the handle and the resolution generation -// (vk_texture_slots_gen) it was looked up under: a slot is looked up again, a hash probe, only when either changed. -static struct { uint64_t tex[32], img[32]; uint32_t tgen[32], igen[32]; uint8_t tv[32], iv[32], tt[32], it[32]; } AC[6]; +// Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it +// was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve +// otherwise (slot_still). +static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32]; } AC[6]; + +// vk_texture_slot_token of the slot a texture or image handle names; an empty handle always resolves to nothing. +static uint64_t token_of(uint64_t h) { return h ? vk_texture_slot_token((int)(h & 0xfffff)) : 1; } + +// Whether what handle h resolved to under the global generation *egen and its slot token etok (both read before that +// lookup) still holds at global generation gen: nothing at all changed since, or not its slot (the token, read only when +// the global generation moved, about every 16 draws). A token hit moves *egen to gen. +static int slot_still(uint32_t *egen, uint64_t etok, uint64_t h, uint32_t gen) { + if (*egen == gen) return 1; + uint64_t t = token_of(h); + if (!t || t != etok) return 0; + *egen = gen; + return 1; +} static int shader_memo_on(void); // The slots of (program, stage) whose types its translation reads (gfx_shader_slot_mask), learned from its first @@ -602,18 +621,20 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G a->local[k] = (uint16_t)(l ? l : 1); } typeof(AC[0]) *c = &AC[st]; - // Read before the lookups (one that creates an entry bumps it) and unlocked: a registration racing with it is - // ordered after this draw, and a slot stored under an older generation is only looked up again. + // The generation and tokens are read before the lookups (one that creates an entry changes them) and unlocked: a + // registration racing with them is ordered after this draw, and a slot stored under older ones is only looked up + // again. uint32_t gen = vk_texture_slots_gen(); int locked = 0, check = shader_memo_on() == 2; for (uint32_t m = mask; m; m &= m - 1) { int slot = __builtin_ctz(m); uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot]; - int tmiss = !c->tv[slot] || c->tgen[slot] != gen || c->tex[slot] != h; - int imiss = !c->iv[slot] || c->igen[slot] != gen || c->img[slot] != hi; + int tmiss = c->tex[slot] != h || !c->ttok[slot] || !slot_still(&c->tgen[slot], c->ttok[slot], h, gen); + int imiss = c->img[slot] != hi || !c->itok[slot] || !slot_still(&c->igen[slot], c->itok[slot], hi, gen); + uint64_t tt = tmiss ? token_of(h) : 0, it = imiss ? token_of(hi) : 0; if ((tmiss || imiss || check) && !locked) vk_texture_lock(), locked = 1, BS.ac_miss++; - if (tmiss) c->tt[slot] = type_of_handle(h), c->tex[slot] = h, c->tgen[slot] = gen, c->tv[slot] = 1; - if (imiss) c->it[slot] = type_of_handle(hi), c->img[slot] = hi, c->igen[slot] = gen, c->iv[slot] = 1; + if (tmiss) c->tt[slot] = type_of_handle(h), c->tex[slot] = h, c->ttok[slot] = tt, c->tgen[slot] = gen; + if (imiss) c->it[slot] = type_of_handle(hi), c->img[slot] = hi, c->itok[slot] = it, c->igen[slot] = gen; if (check && (type_of_handle(h) != c->tt[slot] || type_of_handle(hi) != c->it[slot])) // TOTK_SHADER_MEMO=check rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot); a->tex_type[slot] = c->tt[slot], a->img_type[slot] = c->it[slot]; @@ -916,39 +937,85 @@ static int in_fbl(const VkExec *ex, VkImage img) { return 0; } -// Attachment slots of the open pass (bit 8 depth) whose subresource the view of img covers. -static uint32_t attached_as(const VkExec *ex, const VkTexImage *img, const view_t *v, int hv) { - uint32_t l0 = 0, l1 = img->levels, a0 = 0, a1 = img->layers, m = 0; - if (hv && v->set_levels) l0 = v->base_level, l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels); - if (hv && v->set_layers) a0 = v->min_layer, a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers); +// The image and the levels [l0, l1) and layers [a0, a1) a texture's view covers. +typedef struct { VkImage image; uint32_t l0, l1, a0, a1; } SubRange; + +static SubRange sub_of(const VkTexImage *img, const view_t *v, int hv) { + SubRange r = {img->image, 0, img->levels, 0, img->layers}; + if (hv && v->set_levels) r.l0 = v->base_level, r.l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels); + if (hv && v->set_layers) r.a0 = v->min_layer, r.a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers); + return r; +} + +// Attachment slots of the open pass (bit 8 depth) whose subresource r covers. +static uint32_t attached_as(const VkExec *ex, const SubRange *r) { + uint32_t m = 0; + if (!r->image) return 0; for (uint32_t i = 0; i < 9; i++) { const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth; - if ((i < 8 && i >= ex->pass_ncolor) || t->image != img->image) continue; - if (t->level < l0 || t->level >= l1) continue; - if (!t->is3d && (t->base_layer + t->layers <= a0 || t->base_layer >= a1)) continue; + if ((i < 8 && i >= ex->pass_ncolor) || t->image != r->image) continue; + if (t->level < r->l0 || t->level >= r->l1) continue; + if (!t->is3d && (t->base_layer + t->layers <= r->a0 || t->base_layer >= r->a1)) continue; m |= 1u << i; } return m; } +// TOTK_TEX_MEMO (tex_info, feedback_scan): 0 off, 1 on, 2 (=check) computes every hit again and stops on a difference. +static int td_mode(void) { + static int on = -1; + if (on < 0) { + const char *e = cfg_value("TOTK_TEX_MEMO"); + on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; + } + return on; +} + +// The subresource a texture handle samples, per handle, kept while its slot resolves the same (slot_still): a draw's +// texture bindings change the whole-scan memo below, but the handles they bind were mostly looked up before, in this +// pass or an earlier one. Direct-mapped. +#define FM_N 8192 +static struct { uint64_t h, tok; uint32_t gen; SubRange r; } FM[FM_N]; + +static SubRange sampled_sub(uint64_t h) { // texture lock held + view_t v; + int hv; + const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv); + return img ? sub_of(img, &v, hv) : (SubRange){0}; +} + // Attachments of the open pass the fragment shader fs samples; kept while the shader, the texture bindings and the // pass stay the same. static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) { if (ex->fb_fs == fs && ex->fb_serial == ex->tex_serial && ex->fb_pass == ex->pass_serial) return ex->fb_sampled; const GfxShaderRes *res; int n = gfx_shader_resources(fs, GFX_RES_TEX, &res); - uint32_t m = 0; - vk_texture_lock(); + uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup + int memo = td_mode(), locked = 0; for (int r = 0; r < n; r++) for (int k = 0; k < (res[r].count ? res[r].count : 1); k++) { int slot = res[r].slot + k; uint64_t h = slot >= 0 && slot < 128 ? ex->tex[NVN_STAGE_FS][slot] : 0; - view_t v; - int hv; - const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; - if (img) m |= attached_as(ex, img, &v, hv); + if (!h) continue; + typeof(FM[0]) *f = &FM[(h * 0x9E3779B97F4A7C15ull) >> 51]; + if (memo && f->h == h && slot_still(&f->gen, f->tok, h, gen)) { + if (memo == 2) { + if (!locked) vk_texture_lock(), locked = 1; + SubRange c = sampled_sub(h); + if (memcmp(&c, &f->r, sizeof c)) + rt_fatal("vk_exec: feedback memo of handle %#llx differs from a lookup", (unsigned long long)h); + } + m |= attached_as(ex, &f->r); + continue; + } + uint64_t tok = token_of(h); // before the lookup + if (!locked) vk_texture_lock(), locked = 1; + SubRange sr = sampled_sub(h); + m |= attached_as(ex, &sr); + if (tok) *f = (typeof(*f)){.h = h, .tok = tok, .gen = gen, .r = sr}; + else f->h = 0; } - vk_texture_unlock(); + if (locked) vk_texture_unlock(); ex->fb_fs = fs, ex->fb_serial = ex->tex_serial, ex->fb_pass = ex->pass_serial, ex->fb_sampled = m; return m; } @@ -1000,48 +1067,60 @@ static void feedback_written(VkExec *ex, const VkPipeState *s) { } static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, - VkDescriptorImageInfo *out); + VkDescriptorImageInfo *out, VkImage *image); -// Sampled-texture descriptors per (handle, resource type, separate), kept while nothing they derive from changed: every -// slot's resolution and texture entry (vk_texture_slots_gen), the samplers (vk_sampler_gen), the submission -// (vk_texture_epoch: the first binding of a slot in a submission checks its storage) and, in a pass with images in the -// feedback layout, the pass. Draws bind the same textures thousands of times a frame; a hit skips the slot lookup, the -// view key with its hash probe and the sampler lookup. Storage images, which note their binding per submission, are +// Sampled-texture descriptors per (handle, resource type, separate), kept while nothing they derive from changed: the +// slot's resolution and texture entry (slot_still) and the samplers (vk_sampler_gen); the layout is set again on every +// hit from the images in the feedback layout (in_fbl of the image kept with it). The first binding of a slot in a submission (vk_texture_epoch) still makes the slot lookup +// for its storage check; a re-upload writes the same image, so the descriptor stays. Draws bind the same textures +// thousands of times a frame (15,000 lookups a frame in the monsters fight); a hit skips the slot lookup, the view key +// with its hash probe and the sampler lookup. Storage images, which note their binding per submission, are // not kept. Direct-mapped; texture lock held. TOTK_TEX_MEMO=0 turns it off; =check computes every hit again and stops // on a difference. -#define TD_N 4096 -static struct { uint64_t h; uint32_t gen, epoch, sgen, pass; uint16_t type; uint8_t separate, valid; VkDescriptorImageInfo out; } TD[TD_N]; -static int td_on = -1; +#define TD_N 8192 +static struct { + uint64_t h, tok; + uint32_t gen, epoch, sgen; + uint16_t type; + uint8_t separate, valid; + VkImage image; + VkDescriptorImageInfo out; +} TD[TD_N]; static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) { - if (td_on < 0) { - const char *e = cfg_value("TOTK_TEX_MEMO"); - td_on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; - } - if (!h || storage || !td_on) return tex_info_full(ex, r, h, storage, separate, out); + int td_on = td_mode(); + VkImage image; + if (!h || storage || !td_on) return tex_info_full(ex, r, h, storage, separate, out, &image); uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup - uint32_t pass = ex->nfbl ? ex->pass_serial | 0x80000000u : 0; - typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 52]; - if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->gen == gen && m->epoch == ep && - m->sgen == sg && m->pass == pass) { + typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 51]; + if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->sgen == sg && + slot_still(&m->gen, m->tok, h, gen)) { + if (m->epoch != ep) { // the slot's first binding in this submission: only its storage check is due + vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL); + m->epoch = ep; + } *out = m->out; + out->imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT + : VK_IMAGE_LAYOUT_GENERAL; if (td_on == 2) { VkDescriptorImageInfo full; - tex_info_full(ex, r, h, storage, separate, &full); + tex_info_full(ex, r, h, storage, separate, &full, &image); if (full.sampler != out->sampler || full.imageView != out->imageView || full.imageLayout != out->imageLayout) rt_fatal("vk_exec: texture descriptor memo of handle %#llx type %#x differs from a lookup", (unsigned long long)h, r->type); } return; } - tex_info_full(ex, r, h, storage, separate, out); - *m = (typeof(*m)){.h = h, .gen = gen, .epoch = ep, .sgen = sg, .pass = pass, .type = r->type, - .separate = (uint8_t)separate, .valid = 1, .out = *out}; + uint64_t tok = token_of(h); // before the lookup + tex_info_full(ex, r, h, storage, separate, out, &image); + *m = (typeof(*m)){.h = h, .tok = tok, .gen = gen, .epoch = ep, .sgen = sg, .type = r->type, + .separate = (uint8_t)separate, .valid = 1, .image = image, .out = *out}; } static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, - VkDescriptorImageInfo *out) { + VkDescriptorImageInfo *out, VkImage *image) { int ms = (r->type & 0x200) != 0; + *image = VK_NULL_HANDLE; VkImageView view = VK_NULL_HANDLE; VkSampler samp = VK_NULL_HANDLE; VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL; @@ -1049,7 +1128,7 @@ static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int sto view_t v; int hv; const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv); - if (img) view = get_view(ex, img, &v, hv, r->type, storage); + if (img) view = get_view(ex, img, &v, hv, r->type, storage), *image = img->image; if (img && storage) vk_texture_storage_bound_locked(img); if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT; if (!storage && !separate) { @@ -1570,16 +1649,27 @@ static void bind_vertex_buffers(VkExec *ex) { int vcls[17] = {0}; void *vpool[17] = {0}; int dyn = vk_pipeline_vertex_input_dynamic(); + // A stream whose range is the one its bound buffer was resolved from, with no pool or mirror change since + // (vk_mem_gen, read before any resolve), resolves to that buffer again: most draws change one stream or none. + uint64_t gen = vk_mem_gen(); + int same_gen = B.vb_valid && B.vgen == gen; for (int b = 0; b < 17; b++) { vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0; if (b == VK_PIPE_ZERO_BINDING) continue; const NVNbufferRange *r = &ex->vbuf[b]; + if (same_gen && B.vin[b].address == r->address && B.vin[b].size == r->size) { + vb[b] = B.vb[b], vo[b] = B.vo[b], vsz[b] = B.vsz[b], vcls[b] = B.vcls[b], vpool[b] = B.vpool[b]; + continue; + } VkMemSpan sp; if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue; VK_MEM_USE(sp); uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail; vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool; } + memcpy(B.vcls, vcls, sizeof vcls), memcpy(B.vpool, vpool, sizeof vpool); + memcpy(B.vin, ex->vbuf, sizeof B.vin); + B.vgen = gen; if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) && (dyn || !memcmp(B.vstr, vstr, sizeof vstr))) return; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 70c858ae..f381e912 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -59,11 +59,12 @@ typedef struct Tex { } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the -// entry (ce), valid while tex_gen is still cgen, the view and has_view handed out with it, and the submit_epoch of the -// last freshness check through the slot (cepoch). Hot fields first: a lookup that hits touches one or two cache lines. +// entry (ce), valid while the generation of its address (agen) is still cgen, the view and has_view handed out with it, +// and the submit_epoch of the last freshness check through the slot (cepoch). Hot fields first: a lookup that hits +// touches one or two cache lines. typedef struct { - struct Tex *ce; - uint32_t cgen; // tex_gen when ce was resolved; 0: nothing cached (registered since, or never resolved) + struct Tex *ce; // NULL: nothing cached (registered since, or never resolved) + uint32_t cgen; // agen of the slot's address when ce was resolved uint32_t cepoch; view_t cview; uint8_t chas_view, used, has_view; @@ -105,10 +106,25 @@ static uint32_t tex_gen = 1; static uint32_t slots_gen = 1; static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line +// tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so +// a slot's resolution stays valid while its address's generation does. TotK finalizes and initializes transient +// textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws, +// which emptied every cache keyed on the global generations. +#define AGEN_N 4096 +static uint32_t agen[AGEN_N]; +// Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered) +// and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are +// 20 bits), unlike tslots, which moves when it grows. +static uint64_t *sinfo; +static uint32_t reg_serial; -static void tex_gen_bump(void) { +static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); } + +static void tex_gen_bump(uint64_t addr) { if (!++tex_gen) tex_gen = 1; slots_gen++; + uint32_t *g = &agen[agen_idx(addr)]; + __atomic_store_n(g, *g + 1, __ATOMIC_RELEASE); } static volatile long tex_ready; @@ -415,7 +431,7 @@ static void own_remove(Tex *e) { static void destroy_tex(Tex *e) { own_remove(e); - tex_gen_bump(); + tex_gen_bump(e->key.addr); nvn_hm_del(&by_key, &e->key); Tex *head = nvn_hm_get(&by_addr, &e->key.addr); if (head == e) { @@ -782,9 +798,9 @@ static Tex *entry_of(const tex_t *t) { e->next_at = nvn_hm_get(&by_addr, &k.addr); nvn_hm_put(&by_addr, &k.addr, e); nvn_hm_put(&by_key, &k, e); - tex_gen_bump(); + tex_gen_bump(k.addr); } - if (e->released) e->released = 0, nreleased--, tex_gen_bump(); + if (e->released) e->released = 0, nreleased--, tex_gen_bump(k.addr); e->t = *t; return e; } @@ -846,7 +862,7 @@ void vk_texture_release(const tex_t *t) { if (e && !e->released) { e->released = 1; nreleased++; - tex_gen_bump(); + tex_gen_bump(k.addr); sweep_released(); } unlock(); @@ -877,11 +893,18 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { memset(s + ntslots, 0, sizeof *s * (size_t)(n - ntslots)); tslots = s, ntslots = n; } + if (!sinfo) { + uint64_t *p = calloc(1 << 20, sizeof *p); + if (!p) rt_fatal("vk: out of memory for texture slot tokens"); + __atomic_store_n(&sinfo, p, __ATOMIC_RELEASE); + } TexSlot *s = &tslots[id]; s->t = *t; s->used = 1; - s->cgen = 0; + s->ce = NULL; slots_gen++; + if (!++reg_serial) reg_serial = 1; + __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); s->has_view = v != NULL; if (v) s->v = *v; else memset(&s->v, 0, sizeof s->v); @@ -932,6 +955,13 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { void vk_texture_lock(void) { lock(); } void vk_texture_unlock(void) { unlock(); } uint32_t vk_texture_slots_gen(void) { return slots_gen; } + +uint64_t vk_texture_slot_token(int id) { + const uint64_t *si = __atomic_load_n(&sinfo, __ATOMIC_ACQUIRE); + if (!si || id < 0 || id >= (1 << 20)) return 0; + uint64_t v = __atomic_load_n(&si[id], __ATOMIC_ACQUIRE); + return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0; +} uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); } const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { @@ -944,12 +974,13 @@ const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { if (id < 0 || id >= ntslots || !tslots[id].used) return NULL; TexSlot *s = &tslots[id]; - if (s->cgen != tex_gen) { // resolved again: an entry was created, dropped or released, or the slot registered - view_t v; - int hv; - s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry: tex_gen is read after it + uint32_t g = agen[sinfo[id] & 0xffff]; + if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the + view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may + int hv; // succeed later + s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it s->cview = v, s->chas_view = (uint8_t)hv; - s->cgen = s->ce ? tex_gen : 0; // an empty result is not kept: image creation may succeed later + s->cgen = g; s->cepoch = 0; } Tex *e = s->ce; @@ -1562,5 +1593,6 @@ void vk_texture_shutdown(void) { samplers = (nvn_hmap){.klen = sizeof(samp_par)}; free(tslots), free(sslots); tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0; + if (sinfo) memset(sinfo, 0, sizeof *sinfo << 20); // no slot is registered: tokens 0 unlock(); } From 895aa43ab90c49d77bd5c72b05412f2cb13ceb2e Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 00:56:48 +0300 Subject: [PATCH 6/8] Wake the completion thread for a new callback only when its value is already reached: the replay's callbacks wait for the submission it records, whose submit wakes the thread anyway Co-Authored-By: Claude Opus 5.5 --- runtime/vk_sync.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index fbbf465e..d65a4976 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -381,7 +381,9 @@ void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) { } } queue[qhead + qn++] = (Pending){fence, fn, arg}; - WakeAllConditionVariable(&cv_work); + // The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's, + // for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it. + if (fence <= done) WakeAllConditionVariable(&cv_work); ReleaseSRWLockExclusive(&st_lock); } From b92f1130734729243e3ef1046da20ab581f91956 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 01:01:54 +0300 Subject: [PATCH 7/8] Check a texture's storage pages only in the 2 MiB regions stamped since its last check: an unchanged 4 MiB texture read 1,024 page generations, at about 1,100 checks a frame Co-Authored-By: Claude Opus 5.5 --- runtime/vk_memory.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index e66ada31..b6b2e669 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -728,9 +728,15 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e } int untracked = 0, changed = 0; for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r]; - for (uint64_t a = lo; !untracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) { - uint32_t *s = gen_slot(a, 0); - if (s && *s > since) changed = 1; + // Pages only in regions stamped after since (reg_max; REG_SHIFT is SNAP_SHIFT): an unchanged texture of 4 MiB was + // 1,024 page reads, at about 1,100 texture checks a frame. + for (uint64_t r = r0; !untracked && r <= r1 && !changed; r++) { + if (reg_max[r] <= since) continue; + uint64_t a = r << REG_SHIFT > lo ? r << REG_SHIFT : lo, b = (r + 1) << REG_SHIFT < hi ? (r + 1) << REG_SHIFT : hi; + for (; a < b && !changed; a += 1ull << GEN_PAGE_SHIFT) { + uint32_t *s = gen_slot(a, 0); + if (s && *s > since) changed = 1; + } } *now = gen_now; // every observation so far is in the table __atomic_add_fetch(&n_snap_checks, 1, __ATOMIC_RELAXED); From 2bcba6bc3fa633937126cbf9874ffaee33f0300e Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 01:01:54 +0300 Subject: [PATCH 8/8] Keep which shader of a program serves each stage per program serial instead of reading every stage's control block several times a draw Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 2da0075d..a9db2c5d 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -550,7 +550,7 @@ void vk_exec_image_gone(VkImage image) { typedef struct { uint64_t serial; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt; static ShaderEnt shader_tab[SHADER_SETS][SHADER_WAYS]; -static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { +static int find_stage_scan(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { for (uint32_t i = 0; i < p->nstage && i < 6; i++) { const uint8_t *c = p->sd[i].control; if (!c || !p->sd[i].data) continue; @@ -564,6 +564,32 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const return 0; } +// Which shader of a program serves each NVN stage, and its code length, per program serial (new at every +// nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs +// several times a draw. Direct-mapped; a collision only scans again. +#define STAGEM_N 512 +static struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } stagem[STAGEM_N]; + +static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { + if (!p->serial || (unsigned)nvn_stage >= 6) return find_stage_scan(p, nvn_stage, ctl, code, len); + typeof(stagem[0]) *m = &stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55]; + if (m->serial != p->serial) { + m->serial = p->serial; + memset(m->idx, -1, sizeof m->idx); + for (uint32_t i = 0; i < p->nstage && i < 6; i++) { + const uint8_t *c = p->sd[i].control; + if (!c || !p->sd[i].data) continue; + uint32_t st; + memcpy(&st, c + 0x714, 4); + if (st < 6 && m->idx[st] < 0) m->idx[st] = (int8_t)i, memcpy(&m->len[st], c + 0x704, 4); + } + } + int i = m->idx[nvn_stage]; + if (i < 0) return 0; + *ctl = p->sd[i].control, *code = (const uint8_t *)(uintptr_t)p->sd[i].data, *len = m->len[nvn_stage]; + return 1; +} + // Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it // was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve // otherwise (slot_still).