diff --git a/runtime/cfg.c b/runtime/cfg.c index 828dbf16..b607d728 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -39,13 +39,13 @@ static const char *const known[] = { "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MENU_LANG", "MENU_TEST", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", - "OPERATION_MODE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", + "OPERATION_MODE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC", "REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS", "TRACE", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VS_OUTS", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", }; @@ -65,7 +65,7 @@ static const struct { const char *key, *section; } sections[] = { {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, - {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, + {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, @@ -104,7 +104,8 @@ static const CfgMeta metas[] = { {"TEX_ALIAS", "1", "0|1", RESTART}, // render fixes {"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART}, - {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"DESC_TEMPLATE", "1", "0|1", RESTART}, + {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART}, + {"DESC_TEMPLATE", "1", "0|1", RESTART}, {"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART}, // Vulkan {"VK_VALIDATION", "0", "0|1", RESTART}, {"VK_DYN_VERTEX", "", "0|1", RESTART}, diff --git a/runtime/gfx.h b/runtime/gfx.h index 2ef5c446..daa91179 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -93,6 +93,7 @@ typedef struct { typedef struct { uint8_t type, comps, packed; } GfxShaderIo; // GFX_TYPE_*, components, 1 with a Component decoration int gfx_shader_stage(const GfxShader *s); // GFX_STAGE_VS, GFX_STAGE_FS or GFX_STAGE_CS int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res); // count of GFX_RES_* kind; array in *res +uint32_t gfx_shader_slot_mask(const GfxShader *s); // texture/image slots whose type its translation read uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]); // Input Locations (VS attributes, FS varyings) as a mask uint32_t gfx_shader_outputs(const GfxShader *s, GfxShaderIo io[32]); // Output Locations (VS varyings, FS colour targets) void gfx_shader_local_size(const GfxShader *s, uint32_t local[3]); // CS: the LocalSize execution mode; 1, 1, 1 otherwise diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index ebe2b488..099218db 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1418,6 +1418,10 @@ int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res) return s->nres[kind]; } +// The slots of the use mask: answers outside it select the same translation (mask_answers), so a caller may leave +// their texture and image types zero. Parses the reflection: callers keep the result. +uint32_t gfx_shader_slot_mask(const GfxShader *s) { return use_of(s).slots; } + uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]) { if (io) memcpy(io, s->in, sizeof s->in); return s->in_mask; diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index fb4e7624..29417698 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -608,6 +608,8 @@ static void tab_advanced(float scale) { opt_toggle("TOTK_FIX_ZERO", UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC); opt_toggle("TOTK_BIND_SKIP", UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC); opt_toggle("TOTK_SHADER_MEMO", UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC); + opt_toggle("TOTK_PIPE_MEMO", UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC); + opt_toggle("TOTK_TEX_MEMO", UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC); opt_toggle("TOTK_DESC_TEMPLATE", UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC); opt_toggle("TOTK_PUSH_SKIP", UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC); opt_slider("TOTK_SPLIT_DRAWS", UI_MENU_SPLIT_DRAWS, 0, 1000, UI_MENU_SPLIT_DRAWS_DESC, 8, "%.0f"); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 69af8480..2347af64 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -127,6 +127,8 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_BIND_SKIP] = "Skip repeated binds", [UI_MENU_BIND_SKIP_DESC] = "Optimisation: skips bindings that did not change.", [UI_MENU_SHADER_MEMO] = "Shader memo", [UI_MENU_SHADER_MEMO_DESC] = "Optimisation: remembers shader lookups.", + [UI_MENU_PIPE_MEMO] = "Pipeline memo", [UI_MENU_PIPE_MEMO_DESC] = "Optimisation: reuses the previous draw's pipeline when its state repeats.", + [UI_MENU_TEX_MEMO] = "Texture binding memo", [UI_MENU_TEX_MEMO_DESC] = "Optimisation: remembers texture descriptors between draws.", [UI_MENU_DESC_TEMPLATE] = "Descriptor templates", [UI_MENU_DESC_TEMPLATE_DESC] = "Optimisation: updates descriptors in bulk.", [UI_MENU_PUSH_SKIP] = "Skip repeated push constants", @@ -271,6 +273,8 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_BIND_SKIP] = "Пропуск повторных привязок", [UI_MENU_BIND_SKIP_DESC] = "Оптимизация: пропускает неизменившиеся привязки.", [UI_MENU_SHADER_MEMO] = "Память шейдеров", [UI_MENU_SHADER_MEMO_DESC] = "Оптимизация: запоминает поиск шейдеров.", + [UI_MENU_PIPE_MEMO] = "Память конвейера", [UI_MENU_PIPE_MEMO_DESC] = "Оптимизация: повторяет конвейер прошлой отрисовки, если её состояние не изменилось.", + [UI_MENU_TEX_MEMO] = "Память привязок текстур", [UI_MENU_TEX_MEMO_DESC] = "Оптимизация: запоминает дескрипторы текстур между отрисовками.", [UI_MENU_DESC_TEMPLATE] = "Шаблоны дескрипторов", [UI_MENU_DESC_TEMPLATE_DESC] = "Оптимизация: обновляет дескрипторы пачками.", [UI_MENU_PUSH_SKIP] = "Пропуск повторных push-констант", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index e391aeb3..3a8dc234 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -84,6 +84,8 @@ typedef enum { UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC, UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC, UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC, + UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC, + UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC, UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC, UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC, UI_MENU_SPLIT_DRAWS, UI_MENU_SPLIT_DRAWS_DESC, diff --git a/runtime/vk.h b/runtime/vk.h index db2507e4..17e579f2 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -198,6 +198,7 @@ uint64_t vk_upload_flush(int wait); // Deferred destruction: freed once the upload batches recorded so far and the timeline value last noted have completed. void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestroyImage only void vk_defer_view(VkImageView view); +void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc); // ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c. enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK }; @@ -263,6 +264,10 @@ VkSampler vk_sampler_by_id_locked(int id); // on the Switch although the game binds a linear sampler (MacNativeGames 6283599). VK_NULL_HANDLE for an empty slot. VkSampler vk_sampler_point_by_id_locked(int id); uint32_t vk_texture_slots_gen(void); +// What vk_texture_by_id(id) returns can change only when this does: the slot is registered again, or a texture entry at +// its storage address is created, dropped, released or taken back into use. Unlocked; read it before the lookup it +// guards (a lookup may create an entry). 0 when the slot was never registered: nothing is kept under 0. +uint64_t vk_texture_slot_token(int id); uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 30b61352..a9db2c5d 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -67,7 +67,6 @@ typedef struct { VkImageViewType type; uint32_t aspect, level0, levels, layer0, layers, swz; } ViewKey; -typedef struct { ViewKey k; VkImageView view; } ViewEnt; _Static_assert(sizeof(ViewKey) == 40, "ViewKey is hashed as bytes: no padding"); typedef struct { uint64_t addr, size; } Range; typedef struct { @@ -75,9 +74,6 @@ typedef struct { int nchunks, cchunks; VkDescriptorPool *pools; int npools, cpools; - ViewEnt *views; - int nviews, cviews; - nvn_hmap view_of; // ViewKey -> the VkImageView in views Range *written; int nwritten, cwritten; } CbRes; @@ -102,6 +98,10 @@ static struct { int st_valid; VkBuffer vb[17]; // vertex buffers last bound VkDeviceSize vo[17], vsz[17], vstr[17]; + int vcls[17]; // their pool class and pool record, for the census + void *vpool[17]; + NVNbufferRange vin[16]; // the ranges they were resolved from, under vk_mem_gen vgen + uint64_t vgen; int vb_valid; uint8_t support[SUPPORT_BYTES]; // last support buffer and where it lives float sup_in[3]; // the support_fill arguments it was filled from @@ -175,7 +175,6 @@ static void cb_done(void *arg) { // completion thread CbRes *r = arg; VkCtx *c = vk_ctx(); for (int i = 0; i < r->nwritten; i++) vk_guest_written(r->written[i].addr, r->written[i].size); - for (int i = 0; i < r->nviews; i++) vkDestroyImageView(c->device, r->views[i].view, NULL); for (int i = 0; i < r->npools; i++) { vkResetDescriptorPool(c->device, r->pools[i], 0); AcquireSRWLockExclusive(&spare_lock); @@ -191,8 +190,7 @@ static void cb_done(void *arg) { // completion thread ReleaseSRWLockExclusive(&spare_lock); if (!kept) vmaDestroyBuffer(c->vma, r->chunks[i].buf, r->chunks[i].alloc); } - free(r->view_of.keys), free(r->view_of.vals); - free(r->written), free(r->views), free(r->pools), free(r->chunks), free(r); + free(r->written), free(r->pools), free(r->chunks), free(r); } void vk_exec_draw_new_cb(void) { memset(&B, 0, sizeof B); } @@ -201,7 +199,6 @@ static CbRes *cb_res(VkExec *ex) { if (cur && cur_fence == ex->fence) return cur; cur = calloc(1, sizeof *cur); if (!cur) rt_fatal("vk: out of memory"); - nvn_hm_init(&cur->view_of, sizeof(ViewKey)); cur_fence = ex->fence; memset(&B, 0, sizeof B); vk_completion_add(ex->fence, cb_done, cur); @@ -455,8 +452,15 @@ VkImageAspectFlags vk_exec_view_aspect(const VkTexImage *img, const view_t *v, i : VK_IMAGE_ASPECT_COLOR_BIT; } +// Image views of registered textures, kept for their image's lifetime: made per command buffer they were created again +// every frame (vkCreateImageView took about 8% of the fight replay on the 7900X). Used under the texture lock, as +// vk_exec_image_gone is (destroy_tex), which hands the views of a destroyed image to the deferred release with it. +typedef struct ViewNode { struct ViewNode *next; ViewKey k; VkImageView view; } ViewNode; +static nvn_hmap view_of; // ViewKey -> VkImageView +static nvn_hmap views_by_image; // VkImage -> its first ViewNode + // An image view of a registered texture for a shader resource of type `rtype` (GfxShaderRes.type); VK_NULL_HANDLE when -// the image cannot serve it (the caller binds a stand-in). +// the image cannot serve it (the caller binds a stand-in). Texture lock held. static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, int hv, uint16_t rtype, int storage) { VkImageViewType vt = res_view_type(rtype); int ms = (rtype & 0x200) != 0; @@ -505,9 +509,9 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, } } uint32_t swz = (uint32_t)sw[0] | (uint32_t)sw[1] << 8 | (uint32_t)sw[2] << 16 | (uint32_t)sw[3] << 24; - CbRes *r = cb_res(ex); ViewKey k = {img->image, img->format, vt, aspect, l0, nl, y0, ny, swz}; - VkImageView found = (VkImageView)nvn_hm_get(&r->view_of, &k); + if (!view_of.klen) nvn_hm_init(&view_of, sizeof(ViewKey)), nvn_hm_init(&views_by_image, sizeof(VkImage)); + VkImageView found = (VkImageView)nvn_hm_get(&view_of, &k); if (found) return found; VkImageViewCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = img->image, .viewType = vt, .format = img->format, @@ -518,12 +522,24 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c) VkImageView view; if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE; - if (r->nviews == r->cviews) r->views = grow(r->views, &r->cviews, sizeof *r->views); - r->views[r->nviews++] = (ViewEnt){k, view}; - nvn_hm_put(&r->view_of, &k, (void *)view); + ViewNode *n = malloc(sizeof *n); + if (!n) rt_fatal("vk: out of memory"); + *n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view}; + nvn_hm_put(&views_by_image, &img->image, n); + nvn_hm_put(&view_of, &k, (void *)view); return view; } +void vk_exec_image_gone(VkImage image) { + if (!views_by_image.klen) return; + for (ViewNode *n = nvn_hm_del(&views_by_image, &image), *next; n; n = next) { + next = n->next; + nvn_hm_del(&view_of, &n->k); + vk_defer_view(n->view); + free(n); + } +} + // ---- program to shaders // (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the @@ -534,7 +550,7 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, typedef struct { uint64_t serial; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt; static ShaderEnt shader_tab[SHADER_SETS][SHADER_WAYS]; -static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { +static int find_stage_scan(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { for (uint32_t i = 0; i < p->nstage && i < 6; i++) { const uint8_t *c = p->sd[i].control; if (!c || !p->sd[i].data) continue; @@ -548,12 +564,79 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const return 0; } -// Texture and image types of a stage's first 32 slots, kept while every slot's resolution (vk_texture_slots_gen) -// stays the same: only slots whose handle changed are looked up again, each lookup being a hash probe. -static struct { int valid; uint32_t gen; uint64_t tex[32], img[32]; uint8_t tt[32], it[32]; } AC[6]; +// Which shader of a program serves each NVN stage, and its code length, per program serial (new at every +// nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs +// several times a draw. Direct-mapped; a collision only scans again. +#define STAGEM_N 512 +static struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } stagem[STAGEM_N]; + +static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { + if (!p->serial || (unsigned)nvn_stage >= 6) return find_stage_scan(p, nvn_stage, ctl, code, len); + typeof(stagem[0]) *m = &stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55]; + if (m->serial != p->serial) { + m->serial = p->serial; + memset(m->idx, -1, sizeof m->idx); + for (uint32_t i = 0; i < p->nstage && i < 6; i++) { + const uint8_t *c = p->sd[i].control; + if (!c || !p->sd[i].data) continue; + uint32_t st; + memcpy(&st, c + 0x714, 4); + if (st < 6 && m->idx[st] < 0) m->idx[st] = (int8_t)i, memcpy(&m->len[st], c + 0x704, 4); + } + } + int i = m->idx[nvn_stage]; + if (i < 0) return 0; + *ctl = p->sd[i].control, *code = (const uint8_t *)(uintptr_t)p->sd[i].data, *len = m->len[nvn_stage]; + return 1; +} + +// Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it +// was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve +// otherwise (slot_still). +static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32]; } AC[6]; + +// vk_texture_slot_token of the slot a texture or image handle names; an empty handle always resolves to nothing. +static uint64_t token_of(uint64_t h) { return h ? vk_texture_slot_token((int)(h & 0xfffff)) : 1; } + +// Whether what handle h resolved to under the global generation *egen and its slot token etok (both read before that +// lookup) still holds at global generation gen: nothing at all changed since, or not its slot (the token, read only when +// the global generation moved, about every 16 draws). A token hit moves *egen to gen. +static int slot_still(uint32_t *egen, uint64_t etok, uint64_t h, uint32_t gen) { + if (*egen == gen) return 1; + uint64_t t = token_of(h); + if (!t || t != etok) return 0; + *egen = gen; + return 1; +} static int shader_memo_on(void); -static void answers_for(VkExec *ex, int st, const uint8_t *ctl, GfxShaderAnswers *a) { +// The slots of (program, stage) whose types its translation reads (gfx_shader_slot_mask), learned from its first +// shader; ~0u while unknown. The types of other slots select the same shader, so answers_for leaves them zero instead +// of looking up all 64 handles. Direct-mapped by serial and stage; a collision only relearns. +#define SLOTM_N 4096 +static struct { uint64_t serial; int st; uint32_t m; } slotm[SLOTM_N]; + +static uint32_t slotm_idx(uint64_t serial, int st) { return (uint32_t)((serial * 8 + (uint64_t)st) * 0x9E3779B97F4A7C15ull >> 52); } + +static uint32_t slot_mask_of(uint64_t serial, int st) { + const typeof(slotm[0]) *e = &slotm[slotm_idx(serial, st)]; + return serial && e->serial == serial && e->st == st ? e->m : ~0u; +} + +static void slot_mask_learn(uint64_t serial, int st, const GfxShader *s) { + if (!serial || slot_mask_of(serial, st) != ~0u) return; + slotm[slotm_idx(serial, st)] = (typeof(slotm[0])){serial, st, gfx_shader_slot_mask(s)}; +} + +static uint8_t type_of_handle(uint64_t h) { // texture lock held + view_t v; + int hv; + const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; + return img ? (uint8_t)gfx_type_of(img, &v, hv) : 0; +} + +// The answers of stage st for the slots in mask; the types of other slots stay zero. +static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, GfxShaderAnswers *a) { memset(a, 0, sizeof *a); if (st == NVN_STAGE_VS) vk_pipeline_answers(&ex->pipe, a, NULL); else if (st == NVN_STAGE_FS) vk_pipeline_answers(&ex->pipe, NULL, a); @@ -564,44 +647,25 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, GfxShaderAnswers a->local[k] = (uint16_t)(l ? l : 1); } typeof(AC[0]) *c = &AC[st]; - vk_texture_lock(); - uint32_t gen = vk_texture_slots_gen(); // before the lookups: one that creates an entry bumps it - int all = !c->valid || c->gen != gen; - if (all || memcmp(c->tex, ex->tex[st], sizeof c->tex) || memcmp(c->img, ex->img[st], sizeof c->img)) { - for (int slot = 0; slot < 32; slot++) { - view_t v; - int hv; - uint64_t h = ex->tex[st][slot]; - if (all || h != c->tex[slot]) { - const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; - c->tt[slot] = img ? (uint8_t)gfx_type_of(img, &v, hv) : 0; - } - h = ex->img[st][slot]; - if (all || h != c->img[slot]) { - const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; - c->it[slot] = img ? (uint8_t)gfx_type_of(img, &v, hv) : 0; - } - } - memcpy(c->tex, ex->tex[st], sizeof c->tex); - memcpy(c->img, ex->img[st], sizeof c->img); - c->gen = gen, c->valid = 1; - BS.ac_miss++; + // The generation and tokens are read before the lookups (one that creates an entry changes them) and unlocked: a + // registration racing with them is ordered after this draw, and a slot stored under older ones is only looked up + // again. + uint32_t gen = vk_texture_slots_gen(); + int locked = 0, check = shader_memo_on() == 2; + for (uint32_t m = mask; m; m &= m - 1) { + int slot = __builtin_ctz(m); + uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot]; + int tmiss = c->tex[slot] != h || !c->ttok[slot] || !slot_still(&c->tgen[slot], c->ttok[slot], h, gen); + int imiss = c->img[slot] != hi || !c->itok[slot] || !slot_still(&c->igen[slot], c->itok[slot], hi, gen); + uint64_t tt = tmiss ? token_of(h) : 0, it = imiss ? token_of(hi) : 0; + if ((tmiss || imiss || check) && !locked) vk_texture_lock(), locked = 1, BS.ac_miss++; + if (tmiss) c->tt[slot] = type_of_handle(h), c->tex[slot] = h, c->ttok[slot] = tt, c->tgen[slot] = gen; + if (imiss) c->it[slot] = type_of_handle(hi), c->img[slot] = hi, c->itok[slot] = it, c->igen[slot] = gen; + if (check && (type_of_handle(h) != c->tt[slot] || type_of_handle(hi) != c->it[slot])) // TOTK_SHADER_MEMO=check + rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot); + a->tex_type[slot] = c->tt[slot], a->img_type[slot] = c->it[slot]; } - if (shader_memo_on() == 2) // check mode: every slot looked up again - for (int slot = 0; slot < 32; slot++) { - view_t v; - int hv; - uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot]; - const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; - int tt = img ? gfx_type_of(img, &v, hv) : 0; - img = hi ? vk_texture_by_id_locked((int)(hi & 0xfffff), &v, &hv) : NULL; - int it = img ? gfx_type_of(img, &v, hv) : 0; - if (tt != c->tt[slot] || it != c->it[slot]) - rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot); - } - vk_texture_unlock(); - memcpy(a->tex_type, c->tt, sizeof c->tt); - memcpy(a->img_type, c->it, sizeof c->it); + if (locked) vk_texture_unlock(); } // *present: the program has this stage. Returns NULL when it has not or the translation failed. pending (graphics @@ -615,8 +679,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (!ex->prog || !find_stage(ex->prog, st, &ctl, &code, &len)) return NULL; *present = 1; GfxShaderAnswers a; - answers_for(ex, st, ctl, &a); uint64_t serial = ex->prog->serial; + answers_for(ex, st, ctl, slot_mask_of(serial, st), &a); ShaderEnt *set = NULL; if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded @@ -637,6 +701,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin : pending && vk_pipeline_async_on() ? gfx_shader_get_async(gs, ctl, code, len, &a, pending) : gfx_shader_get(gs, ctl, code, len, NULL, &a); if (pending && *pending) return NULL; + if (s) slot_mask_learn(serial, st, s); if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st, (unsigned long long)serial); else if (set) { // in front, the least recently used way drops out @@ -760,8 +825,19 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; } +// The buffer range limits, read once: the device's properties are a cold cache line on every binding of every draw. +static struct { uint64_t ubo_max, ubo_align, sbo_max, sbo_align; } LIM; + +static void limits_init(void) { + const VkPhysicalDeviceLimits *l = &vk_ctx()->props.limits; + LIM.ubo_align = l->minUniformBufferOffsetAlignment, LIM.sbo_max = l->maxStorageBufferRange; + LIM.sbo_align = l->minStorageBufferOffsetAlignment; + __atomic_store_n(&LIM.ubo_max, l->maxUniformBufferRange, __ATOMIC_RELEASE); +} + static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { - uint64_t max = vk_ctx()->props.limits.maxUniformBufferRange, align = vk_ctx()->props.limits.minUniformBufferOffsetAlignment; + if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); + uint64_t max = LIM.ubo_max, align = LIM.ubo_align; VkDescriptorBufferInfo bi; if (slot == 0 && c0_on()) return c0_info(ex, st); if (slot == 1 && ex->prog) { // compiler constants stored after the shader code @@ -845,7 +921,8 @@ static uint64_t ubo_ptr(const VkExec *ex, int st, const GfxShaderRes *r) { } static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *r, int written) { - uint64_t max = vk_ctx()->props.limits.maxStorageBufferRange, align = vk_ctx()->props.limits.minStorageBufferOffsetAlignment; + if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); + uint64_t max = LIM.sbo_max, align = LIM.sbo_align; VkDescriptorBufferInfo bi; int ubo, slot = ssbo_slot(r, st, &ubo); p_bind_buf up = {0}; @@ -886,39 +963,85 @@ static int in_fbl(const VkExec *ex, VkImage img) { return 0; } -// Attachment slots of the open pass (bit 8 depth) whose subresource the view of img covers. -static uint32_t attached_as(const VkExec *ex, const VkTexImage *img, const view_t *v, int hv) { - uint32_t l0 = 0, l1 = img->levels, a0 = 0, a1 = img->layers, m = 0; - if (hv && v->set_levels) l0 = v->base_level, l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels); - if (hv && v->set_layers) a0 = v->min_layer, a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers); +// The image and the levels [l0, l1) and layers [a0, a1) a texture's view covers. +typedef struct { VkImage image; uint32_t l0, l1, a0, a1; } SubRange; + +static SubRange sub_of(const VkTexImage *img, const view_t *v, int hv) { + SubRange r = {img->image, 0, img->levels, 0, img->layers}; + if (hv && v->set_levels) r.l0 = v->base_level, r.l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels); + if (hv && v->set_layers) r.a0 = v->min_layer, r.a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers); + return r; +} + +// Attachment slots of the open pass (bit 8 depth) whose subresource r covers. +static uint32_t attached_as(const VkExec *ex, const SubRange *r) { + uint32_t m = 0; + if (!r->image) return 0; for (uint32_t i = 0; i < 9; i++) { const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth; - if ((i < 8 && i >= ex->pass_ncolor) || t->image != img->image) continue; - if (t->level < l0 || t->level >= l1) continue; - if (!t->is3d && (t->base_layer + t->layers <= a0 || t->base_layer >= a1)) continue; + if ((i < 8 && i >= ex->pass_ncolor) || t->image != r->image) continue; + if (t->level < r->l0 || t->level >= r->l1) continue; + if (!t->is3d && (t->base_layer + t->layers <= r->a0 || t->base_layer >= r->a1)) continue; m |= 1u << i; } return m; } +// TOTK_TEX_MEMO (tex_info, feedback_scan): 0 off, 1 on, 2 (=check) computes every hit again and stops on a difference. +static int td_mode(void) { + static int on = -1; + if (on < 0) { + const char *e = cfg_value("TOTK_TEX_MEMO"); + on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; + } + return on; +} + +// The subresource a texture handle samples, per handle, kept while its slot resolves the same (slot_still): a draw's +// texture bindings change the whole-scan memo below, but the handles they bind were mostly looked up before, in this +// pass or an earlier one. Direct-mapped. +#define FM_N 8192 +static struct { uint64_t h, tok; uint32_t gen; SubRange r; } FM[FM_N]; + +static SubRange sampled_sub(uint64_t h) { // texture lock held + view_t v; + int hv; + const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv); + return img ? sub_of(img, &v, hv) : (SubRange){0}; +} + // Attachments of the open pass the fragment shader fs samples; kept while the shader, the texture bindings and the // pass stay the same. static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) { if (ex->fb_fs == fs && ex->fb_serial == ex->tex_serial && ex->fb_pass == ex->pass_serial) return ex->fb_sampled; const GfxShaderRes *res; int n = gfx_shader_resources(fs, GFX_RES_TEX, &res); - uint32_t m = 0; - vk_texture_lock(); + uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup + int memo = td_mode(), locked = 0; for (int r = 0; r < n; r++) for (int k = 0; k < (res[r].count ? res[r].count : 1); k++) { int slot = res[r].slot + k; uint64_t h = slot >= 0 && slot < 128 ? ex->tex[NVN_STAGE_FS][slot] : 0; - view_t v; - int hv; - const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; - if (img) m |= attached_as(ex, img, &v, hv); + if (!h) continue; + typeof(FM[0]) *f = &FM[(h * 0x9E3779B97F4A7C15ull) >> 51]; + if (memo && f->h == h && slot_still(&f->gen, f->tok, h, gen)) { + if (memo == 2) { + if (!locked) vk_texture_lock(), locked = 1; + SubRange c = sampled_sub(h); + if (memcmp(&c, &f->r, sizeof c)) + rt_fatal("vk_exec: feedback memo of handle %#llx differs from a lookup", (unsigned long long)h); + } + m |= attached_as(ex, &f->r); + continue; + } + uint64_t tok = token_of(h); // before the lookup + if (!locked) vk_texture_lock(), locked = 1; + SubRange sr = sampled_sub(h); + m |= attached_as(ex, &sr); + if (tok) *f = (typeof(*f)){.h = h, .tok = tok, .gen = gen, .r = sr}; + else f->h = 0; } - vk_texture_unlock(); + if (locked) vk_texture_unlock(); ex->fb_fs = fs, ex->fb_serial = ex->tex_serial, ex->fb_pass = ex->pass_serial, ex->fb_sampled = m; return m; } @@ -969,8 +1092,61 @@ static void feedback_written(VkExec *ex, const VkPipeState *s) { if (ex->color[i].image && !(ex->fb_dirty >> i & 1) && vk_pipeline_target_wmask(s, (int)i)) ex->fb_dirty |= 1u << i; } +static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, + VkDescriptorImageInfo *out, VkImage *image); + +// Sampled-texture descriptors per (handle, resource type, separate), kept while nothing they derive from changed: the +// slot's resolution and texture entry (slot_still) and the samplers (vk_sampler_gen); the layout is set again on every +// hit from the images in the feedback layout (in_fbl of the image kept with it). The first binding of a slot in a submission (vk_texture_epoch) still makes the slot lookup +// for its storage check; a re-upload writes the same image, so the descriptor stays. Draws bind the same textures +// thousands of times a frame (15,000 lookups a frame in the monsters fight); a hit skips the slot lookup, the view key +// with its hash probe and the sampler lookup. Storage images, which note their binding per submission, are +// not kept. Direct-mapped; texture lock held. TOTK_TEX_MEMO=0 turns it off; =check computes every hit again and stops +// on a difference. +#define TD_N 8192 +static struct { + uint64_t h, tok; + uint32_t gen, epoch, sgen; + uint16_t type; + uint8_t separate, valid; + VkImage image; + VkDescriptorImageInfo out; +} TD[TD_N]; + static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) { + int td_on = td_mode(); + VkImage image; + if (!h || storage || !td_on) return tex_info_full(ex, r, h, storage, separate, out, &image); + uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup + typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 51]; + if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->sgen == sg && + slot_still(&m->gen, m->tok, h, gen)) { + if (m->epoch != ep) { // the slot's first binding in this submission: only its storage check is due + vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL); + m->epoch = ep; + } + *out = m->out; + out->imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT + : VK_IMAGE_LAYOUT_GENERAL; + if (td_on == 2) { + VkDescriptorImageInfo full; + tex_info_full(ex, r, h, storage, separate, &full, &image); + if (full.sampler != out->sampler || full.imageView != out->imageView || full.imageLayout != out->imageLayout) + rt_fatal("vk_exec: texture descriptor memo of handle %#llx type %#x differs from a lookup", + (unsigned long long)h, r->type); + } + return; + } + uint64_t tok = token_of(h); // before the lookup + tex_info_full(ex, r, h, storage, separate, out, &image); + *m = (typeof(*m)){.h = h, .tok = tok, .gen = gen, .epoch = ep, .sgen = sg, .type = r->type, + .separate = (uint8_t)separate, .valid = 1, .image = image, .out = *out}; +} + +static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, + VkDescriptorImageInfo *out, VkImage *image) { int ms = (r->type & 0x200) != 0; + *image = VK_NULL_HANDLE; VkImageView view = VK_NULL_HANDLE; VkSampler samp = VK_NULL_HANDLE; VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL; @@ -978,7 +1154,7 @@ static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, view_t v; int hv; const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv); - if (img) view = get_view(ex, img, &v, hv, r->type, storage); + if (img) view = get_view(ex, img, &v, hv, r->type, storage), *image = img->image; if (img && storage) vk_texture_storage_bound_locked(img); if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT; if (!storage && !separate) { @@ -1499,16 +1675,27 @@ static void bind_vertex_buffers(VkExec *ex) { int vcls[17] = {0}; void *vpool[17] = {0}; int dyn = vk_pipeline_vertex_input_dynamic(); + // A stream whose range is the one its bound buffer was resolved from, with no pool or mirror change since + // (vk_mem_gen, read before any resolve), resolves to that buffer again: most draws change one stream or none. + uint64_t gen = vk_mem_gen(); + int same_gen = B.vb_valid && B.vgen == gen; for (int b = 0; b < 17; b++) { vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0; if (b == VK_PIPE_ZERO_BINDING) continue; const NVNbufferRange *r = &ex->vbuf[b]; + if (same_gen && B.vin[b].address == r->address && B.vin[b].size == r->size) { + vb[b] = B.vb[b], vo[b] = B.vo[b], vsz[b] = B.vsz[b], vcls[b] = B.vcls[b], vpool[b] = B.vpool[b]; + continue; + } VkMemSpan sp; if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue; VK_MEM_USE(sp); uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail; vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool; } + memcpy(B.vcls, vcls, sizeof vcls), memcpy(B.vpool, vpool, sizeof vpool); + memcpy(B.vin, ex->vbuf, sizeof B.vin); + B.vgen = gen; if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) && (dyn || !memcmp(B.vstr, vstr, sizeof vstr))) return; @@ -1580,6 +1767,13 @@ static int shader_memo_on(void) { return SM.on; } +static struct { int on, valid; VkPipeState s; const VkPipeInfo *info; } PM = {-1}; + +static int pipe_memo_on(void) { + if (PM.on < 0) PM.on = cfg_bool("TOTK_PIPE_MEMO"); + return PM.on; +} + // Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped. static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (ex->render_disabled) return 0; @@ -1652,19 +1846,33 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { wait_ms = (uint32_t)(wait_left_ns / 1000000); if (wait_ms) t0 = kern_now_ns(); } - uint64_t w0 = kern_now_ns(); - VkPipeline p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); - if (t0) { - uint64_t spent = kern_now_ns() - t0; - wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0; + // The previous draw's pipeline while its state repeats (PM, TOTK_PIPE_MEMO=0 turns it off): the cache key is made + // from the state alone, and a pipeline once found only changes from its fast link to its optimised build, read + // from the info the cache entry keeps. + VkPipeline p; + const char *why = NULL; + uint64_t w0 = 0; + if (pipe_memo_on() && PM.valid && !memcmp(&PM.s, &s, sizeof s)) { + info = PM.info, pending = 0; + p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); + } else { + w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait + p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); + if (t0) { + uint64_t spent = kern_now_ns() - t0; + wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0; + } + why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL; + if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) { + perf_count(PERF_CNT_ONESHOT_WAIT); + why = "one-shot"; + if (!w0) w0 = kern_now_ns(); + p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS); + } + PM.valid = !pending && p && info; + if (PM.valid) PM.s = s, PM.info = info; } - const char *why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL; - if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) { - perf_count(PERF_CNT_ONESHOT_WAIT); - why = "one-shot"; - p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS); - } - uint64_t waited = kern_now_ns() - w0; + uint64_t waited = w0 ? kern_now_ns() - w0 : 0; static int wait_logs; if (why && waited >= 2000000 && wait_logs < 200) { char tg[512]; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index c1103b4a..b6b2e669 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -467,8 +467,12 @@ int vk_mem_class_of(uint64_t addr) { // ---------------------------------------------------------------- census static volatile LONG64 traffic[VK_MCLS_N][VK_MUSE_N]; +// Every caller is the replay (vk_exec*, under vk_sync_submit's lock), about ten adds per draw: one writer at a time, so +// a plain add, not a locked one. An add racing the census's reset may be lost. void vk_mem_count(int cls, int use, uint64_t bytes) { - if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N) InterlockedAdd64(&traffic[cls][use], (LONG64)bytes); + if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N) + __atomic_store_n(&traffic[cls][use], __atomic_load_n(&traffic[cls][use], __ATOMIC_RELAXED) + (LONG64)bytes, + __ATOMIC_RELAXED); } static void flag_names(uint32_t f, char *buf, size_t n) { @@ -724,9 +728,15 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e } int untracked = 0, changed = 0; for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r]; - for (uint64_t a = lo; !untracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) { - uint32_t *s = gen_slot(a, 0); - if (s && *s > since) changed = 1; + // Pages only in regions stamped after since (reg_max; REG_SHIFT is SNAP_SHIFT): an unchanged texture of 4 MiB was + // 1,024 page reads, at about 1,100 texture checks a frame. + for (uint64_t r = r0; !untracked && r <= r1 && !changed; r++) { + if (reg_max[r] <= since) continue; + uint64_t a = r << REG_SHIFT > lo ? r << REG_SHIFT : lo, b = (r + 1) << REG_SHIFT < hi ? (r + 1) << REG_SHIFT : hi; + for (; a < b && !changed; a += 1ull << GEN_PAGE_SHIFT) { + uint32_t *s = gen_slot(a, 0); + if (s && *s > since) changed = 1; + } } *now = gen_now; // every observation so far is in the table __atomic_add_fetch(&n_snap_checks, 1, __ATOMIC_RELAXED); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index fbbf465e..d65a4976 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -381,7 +381,9 @@ void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) { } } queue[qhead + qn++] = (Pending){fence, fn, arg}; - WakeAllConditionVariable(&cv_work); + // The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's, + // for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it. + if (fence <= done) WakeAllConditionVariable(&cv_work); ReleaseSRWLockExclusive(&st_lock); } diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 75352880..f381e912 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -59,11 +59,12 @@ typedef struct Tex { } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the -// entry (ce), valid while tex_gen is still cgen, the view and has_view handed out with it, and the submit_epoch of the -// last freshness check through the slot (cepoch). Hot fields first: a lookup that hits touches one or two cache lines. +// entry (ce), valid while the generation of its address (agen) is still cgen, the view and has_view handed out with it, +// and the submit_epoch of the last freshness check through the slot (cepoch). Hot fields first: a lookup that hits +// touches one or two cache lines. typedef struct { - struct Tex *ce; - uint32_t cgen; // tex_gen when ce was resolved; 0: nothing cached (registered since, or never resolved) + struct Tex *ce; // NULL: nothing cached (registered since, or never resolved) + uint32_t cgen; // agen of the slot's address when ce was resolved uint32_t cepoch; view_t cview; uint8_t chas_view, used, has_view; @@ -105,10 +106,25 @@ static uint32_t tex_gen = 1; static uint32_t slots_gen = 1; static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line +// tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so +// a slot's resolution stays valid while its address's generation does. TotK finalizes and initializes transient +// textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws, +// which emptied every cache keyed on the global generations. +#define AGEN_N 4096 +static uint32_t agen[AGEN_N]; +// Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered) +// and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are +// 20 bits), unlike tslots, which moves when it grows. +static uint64_t *sinfo; +static uint32_t reg_serial; -static void tex_gen_bump(void) { +static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); } + +static void tex_gen_bump(uint64_t addr) { if (!++tex_gen) tex_gen = 1; slots_gen++; + uint32_t *g = &agen[agen_idx(addr)]; + __atomic_store_n(g, *g + 1, __ATOMIC_RELEASE); } static volatile long tex_ready; @@ -415,7 +431,7 @@ static void own_remove(Tex *e) { static void destroy_tex(Tex *e) { own_remove(e); - tex_gen_bump(); + tex_gen_bump(e->key.addr); nvn_hm_del(&by_key, &e->key); Tex *head = nvn_hm_get(&by_addr, &e->key.addr); if (head == e) { @@ -426,6 +442,7 @@ static void destroy_tex(Tex *e) { if (p->next_at == e) { p->next_at = e->next_at; break; } } if (e->released) nreleased--; + vk_exec_image_gone(e->pub.image); vk_defer_image(e->pub.image, e->alloc); free(e); } @@ -781,9 +798,9 @@ static Tex *entry_of(const tex_t *t) { e->next_at = nvn_hm_get(&by_addr, &k.addr); nvn_hm_put(&by_addr, &k.addr, e); nvn_hm_put(&by_key, &k, e); - tex_gen_bump(); + tex_gen_bump(k.addr); } - if (e->released) e->released = 0, nreleased--, tex_gen_bump(); + if (e->released) e->released = 0, nreleased--, tex_gen_bump(k.addr); e->t = *t; return e; } @@ -845,7 +862,7 @@ void vk_texture_release(const tex_t *t) { if (e && !e->released) { e->released = 1; nreleased++; - tex_gen_bump(); + tex_gen_bump(k.addr); sweep_released(); } unlock(); @@ -876,11 +893,18 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { memset(s + ntslots, 0, sizeof *s * (size_t)(n - ntslots)); tslots = s, ntslots = n; } + if (!sinfo) { + uint64_t *p = calloc(1 << 20, sizeof *p); + if (!p) rt_fatal("vk: out of memory for texture slot tokens"); + __atomic_store_n(&sinfo, p, __ATOMIC_RELEASE); + } TexSlot *s = &tslots[id]; s->t = *t; s->used = 1; - s->cgen = 0; + s->ce = NULL; slots_gen++; + if (!++reg_serial) reg_serial = 1; + __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); s->has_view = v != NULL; if (v) s->v = *v; else memset(&s->v, 0, sizeof s->v); @@ -931,6 +955,13 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { void vk_texture_lock(void) { lock(); } void vk_texture_unlock(void) { unlock(); } uint32_t vk_texture_slots_gen(void) { return slots_gen; } + +uint64_t vk_texture_slot_token(int id) { + const uint64_t *si = __atomic_load_n(&sinfo, __ATOMIC_ACQUIRE); + if (!si || id < 0 || id >= (1 << 20)) return 0; + uint64_t v = __atomic_load_n(&si[id], __ATOMIC_ACQUIRE); + return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0; +} uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); } const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { @@ -943,12 +974,13 @@ const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { if (id < 0 || id >= ntslots || !tslots[id].used) return NULL; TexSlot *s = &tslots[id]; - if (s->cgen != tex_gen) { // resolved again: an entry was created, dropped or released, or the slot registered - view_t v; - int hv; - s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry: tex_gen is read after it + uint32_t g = agen[sinfo[id] & 0xffff]; + if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the + view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may + int hv; // succeed later + s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it s->cview = v, s->chas_view = (uint8_t)hv; - s->cgen = s->ce ? tex_gen : 0; // an empty result is not kept: image creation may succeed later + s->cgen = g; s->cepoch = 0; } Tex *e = s->ce; @@ -1561,5 +1593,6 @@ void vk_texture_shutdown(void) { samplers = (nvn_hmap){.klen = sizeof(samp_par)}; free(tslots), free(sslots); tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0; + if (sinfo) memset(sinfo, 0, sizeof *sinfo << 20); // no slot is registered: tokens 0 unlock(); }