diff --git a/runtime/cfg.c b/runtime/cfg.c index 6dfad3af..c6836ff6 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -24,9 +24,9 @@ static int n_entries; // logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line. static const char *const known[] = { "APP_POOL", "ASSUME_FOCUS", "AUDIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", - "BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", + "BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", - "CLOTH_REALTIME", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", + "CLOTH_REALTIME", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "FREEZE_IDLE", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", diff --git a/runtime/vk.h b/runtime/vk.h index a2df85ed..692eb319 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -170,6 +170,9 @@ void vk_mem_census(uint64_t frame); // at each present: pool censu // 0: this allocator's own allocations against the heap sizes. 0 when unknown. int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact); int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed +// Changes whenever vk_mem_resolve plus VK_MEM_USE may give another span for an address: a pool added, finalized or +// mapped, a mirror created, refused or forgotten, a submission started. +uint64_t vk_mem_gen(void); // Page write generations over guest memory (mem_written_pages plus vk_guest_written). Returns 1 when a page of the // range was written after generation `since`, 0 when not, -1 when the range is not write-watched (treat as written); // *now is the generation to pass next time. @@ -234,6 +237,7 @@ void vk_texture_unlock(void); const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view); VkSampler vk_sampler_by_id_locked(int id); uint32_t vk_texture_slots_gen(void); +uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage int vk_texture_small_dirty(void); // a small mirrored target holds GPU texels its storage lacks void vk_texture_write_back_small(uint64_t fence); // writes those rendered by work up to fence back to storage @@ -343,7 +347,13 @@ typedef struct { VkDescriptorSetLayout set_layout1; int push1; uint32_t n_sets; + // Descriptor update template per set (vk_pipeline.c make_tpl): reads one VkDescSlot per descriptor, in binding order, + // from the start of the set's descriptors. VK_NULL_HANDLE: the set is written with VkWriteDescriptorSet structs. + VkDescriptorUpdateTemplate tpl[2]; } VkPipeInfo; +// One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the +// bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte. +typedef union { VkDescriptorImageInfo image; VkDescriptorBufferInfo buffer; VkBufferView view; uint64_t w[3]; } VkDescSlot; // Both return the cached pipeline (created on first use; failures are cached and logged once). info: may be NULL. // Valid until vk_pipeline_shutdown; thread-safe. VkPipeline vk_pipeline_graphics(const VkPipeState *s, const VkPipeInfo **info); @@ -377,6 +387,10 @@ VkPrimitiveTopology vk_pipeline_topology(int nvn_prim); // quads -> triangle l // rewrites those draws; polygon -> fan VkFormat vk_vertex_format(int nvn_format, int *type); // attribute format (supported fallback), GFX_TYPE_* in *type PFN_vkCmdPushDescriptorSetKHR vk_pipeline_push_fn(void); // after the first vk_pipeline_* call +// Descriptor update templates (after the first vk_pipeline_* call): NULL when templates are off (TOTK_DESC_TEMPLATE=0, +// a device below Vulkan 1.1) or, for the push variant, vkCmdPushDescriptorSetWithTemplateKHR is missing. +PFN_vkCmdPushDescriptorSetWithTemplateKHR vk_pipeline_push_tpl_fn(void); +PFN_vkUpdateDescriptorSetWithTemplate vk_pipeline_update_tpl_fn(void); // Probe only: 1 selects stage-independent layouts for pipelines looked up from now on, 0 the shared set 0; returns // the previous mode. 1 needs vk_caps.gpl (the independent-sets flag comes with the extension). int vk_pipeline_set_indep(int on); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 74d0609f..a751ea01 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -118,19 +118,54 @@ static struct { VkDescriptorSet ds_last[2]; // last allocated and written set per set number, contents in DS VkDescriptorSetLayout ds_last_layout[2]; int ds_pushed[2]; // set number holds the push of DS.psig under ds_layout + int res_valid; // the graphics sets bound hold what the inputs in RS give (bind_resources) } B; -// Contents of B.ds_last: the writes' bindings, types, counts and descriptor infos. A draw whose allocated set would -// hold the same reuses the set instead of allocating and writing one (Task 2.2's per-draw cost). psig: the same for -// the last push per set number; a push equal to it under the same layout is not recorded again (the descriptors stay -// bound across pipeline binds of that layout). -#define DS_SIG_BYTES (MAX_DESC * 24 + MAX_BINDINGS * 12) -static struct { uint32_t n[2], pn[2]; uint8_t sig[2][DS_SIG_BYTES], psig[2][DS_SIG_BYTES], tmp[DS_SIG_BYTES]; } DS; +// Contents of B.ds_last: the set's packed descriptors (VkDescSlot), which with its layout determine every write. A draw +// whose allocated set would hold the same reuses the set instead of allocating and writing one (Task 2.2's per-draw +// cost). psig: the same for the last push per set number; a push equal to it under the same layout is not recorded +// again (the descriptors stay bound across pipeline binds of that layout). +#define DS_SIG_BYTES (MAX_DESC * sizeof(VkDescSlot)) +static struct { uint32_t n[2], pn[2]; uint8_t sig[2][DS_SIG_BYTES], psig[2][DS_SIG_BYTES]; } DS; // TOTK_BIND_STATS counters (bind_stats). -static struct { int on; uint64_t draws, bind[2], desc[2], push, push_skip, set_new, set_reuse, ac_miss, same_prog, same_all; } BS = {-1}; +static struct { + int on; + uint64_t draws, bind[2], desc[2], push, push_skip, set_new, set_reuse, ac_miss, same_prog, same_all, bind_skip; +} BS = {-1}; + +// The inputs of the last graphics bind_resources (bind_sig) and what its build counted for the traffic census, which a +// skipped build counts again. RS_BYTES bounds the bound-slot bytes a pipeline reads (RsList): per binding at most one +// c0 storage table, otherwise one bound range or handle per descriptor. +#define RS_BYTES (MAX_BINDINGS * sizeof(((VkExec *)0)->ssbo[0]) + MAX_DESC * sizeof(p_bind_buf)) +typedef struct { int cls, use; uint64_t bytes; } RsCount; +static struct { + int on; // TOTK_BIND_SKIP: 1 skip (default), 2 check, 0 off + uint32_t hn; // words of h the last signature filled; 0: none + uint64_t h[24]; // generations, feedback images, support inputs, base instance + const VkPipeInfo *info; // the pipeline whose RsList ranges body holds, as of bind op generation gen + uint64_t gen; + uint8_t body[RS_BYTES]; + int rec; // a graphics build is running: count_use records, scratch copies mark volatile + int volatile_; // the build copied guest bytes into transient memory: a rebuild differs + RsCount cnt[2 * VK_MCLS_N], pcnt[2 * VK_MCLS_N]; + int ncnt, pncnt, cnt_over; + int cmds; // check mode: pushes, binds and set writes the build recorded + uint32_t ckn; // check mode: the last build's packed descriptors (bytes in ck) + uint8_t ck[DS_SIG_BYTES]; +} RS = {-1}; + +// The bound-slot bytes of VkExec a pipeline's descriptors read, as ranges merged where they touch, built once per +// pipeline info: c0's storage table, the uniform and storage ranges of the bound slots and the texture and image +// handles. sup and c0: the support inputs and the base instance count too. ok 0: never skipped (too many descriptors). +typedef struct { uint32_t off, len; } RsRange; +typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok; RsRange r[]; } RsList; +static nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown +static const VkPipeInfo *rs_last; // the last lookup +static const RsList *rs_last_l; static uint32_t c0_dim[3], c0_binst; // the dispatch size and base instance of the draw or dispatch being bound +static uint64_t bind_gen = 1; // bumped by every bind op that writes ex->prog, ubo, ssbo, tex or img static void cb_done(void *arg) { // completion thread CbRes *r = arg; @@ -345,6 +380,8 @@ void vk_exec_draw_shutdown(void) { vmaDestroyImage(c->vma, SD[i].image, SD[i].alloc); } memset(SD, 0, sizeof SD); + nvn_hm_clear(&rs_lists, free); + rs_last = NULL, rs_last_l = NULL; for (int i = 0; i < nspare_pool; i++) vkDestroyDescriptorPool(c->device, spare_pool[i], NULL); for (int i = 0; i < nspare_chunk; i++) vmaDestroyBuffer(c->vma, spare_chunk[i].buf, spare_chunk[i].alloc); nspare_pool = nspare_chunk = 0; @@ -612,6 +649,19 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) { return (VkDescriptorBufferInfo){Z.zbuf, 0, max < ZERO_BYTES ? max : ZERO_BYTES}; } +// vk_mem_count, also kept per (class, use) while a graphics build runs (RS). +static void count_use(int cls, int use, uint64_t n) { + vk_mem_count(cls, use, n); + if (!RS.rec) return; + for (int i = 0; i < RS.ncnt; i++) + if (RS.cnt[i].cls == cls && RS.cnt[i].use == use) { + RS.cnt[i].bytes += n; + return; + } + if (RS.ncnt < (int)(sizeof RS.cnt / sizeof RS.cnt[0])) RS.cnt[RS.ncnt++] = (RsCount){cls, use, n}; + else RS.cnt_over = 1; +} + // A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when // aligned, a copy in transient memory when the backing is host memory with an unaligned offset. 0 when neither works. // written: the shader stores into it (a mirror's range goes back to guest memory at the end of the submission). @@ -626,13 +676,14 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, VK_MEM_USE(sp); *out = (VkDescriptorBufferInfo){sp.buffer, sp.offset, n}; if (written) vk_mem_gpu_wrote(&sp, n); - vk_mem_count(sp.cls, use, n); + count_use(sp.cls, use, n); VK_MEM_BOUND(sp); return 1; } if (sp.alloc) return 0; VkBuffer b; VkDeviceSize o; + RS.volatile_ = 1; memcpy(scratch(ex, n, &b, &o), (const void *)(uintptr_t)addr, n); vk_mem_count(sp.cls, VK_MUSE_CPU_READ, n); vk_mem_count(VK_MCLS_SCRATCH, use, n); @@ -914,6 +965,12 @@ static VkDescriptorSet alloc_set(VkExec *ex, VkDescriptorSetLayout layout) { return VK_NULL_HANDLE; } +static void support_in(const VkExec *ex, float in[3]) { // what support_fill reads: viewport size, point size + in[0] = ex->in_pass && ex->has_vp ? ex->vp[0][2] : (float)ex->pass_w; + in[1] = ex->in_pass && ex->has_vp ? ex->vp[0][3] : (float)ex->pass_h; + in[2] = ex->point_size > 0 ? ex->point_size : 1.0f; +} + static void support_fill(uint8_t *b, float w, float h, float point_size) { memset(b, 0, SUPPORT_BYTES); float inv[4] = {w > 0 ? 2.0f / w : 0, h > 0 ? 2.0f / h : 0, 0, 0}, size[4] = {w, h, 0, 0}; @@ -949,9 +1006,9 @@ static void bind_stats(const VkPipeInfo *info) { rt_log("vk_exec: per draw over %llu draws: vertex %.2f bindings %.2f descriptors, fragment %.2f bindings %.2f descriptors", (unsigned long long)BS.draws, BS.bind[0] / d, BS.desc[0] / d, BS.bind[1] / d, BS.desc[1] / d); rt_log("vk_exec: per draw: pushes %.3f skipped %.3f, sets written %.3f reused %.3f, texture types %.3f; " - "same program and state as the draw before %.3f, also same buffers %.3f", + "same program and state as the draw before %.3f, also same buffers %.3f; descriptor builds skipped %.3f", BS.push / d, BS.push_skip / d, BS.set_new / d, BS.set_reuse / d, BS.ac_miss / d, BS.same_prog / d, - BS.same_all / d); + BS.same_all / d, BS.bind_skip / d); } } @@ -982,30 +1039,6 @@ static void repeat_stats(const VkExec *ex, const VkPipeInfo *info) { last = cur; } -// The signature of a set's writes into out (DS layout): bindings, types, counts and descriptor infos. Returns its size. -static uint32_t ds_sig(const VkWriteDescriptorSet *w, uint32_t nw, uint8_t *out) { - uint32_t n = 0; - for (uint32_t k = 0; k < nw; k++) { - const VkWriteDescriptorSet *x = &w[k]; - uint32_t hdr[3] = {x->dstBinding, (uint32_t)x->descriptorType, x->descriptorCount}; - memcpy(out + n, hdr, sizeof hdr), n += sizeof hdr; - if (x->pImageInfo) { // field by field: the struct's tail padding is not written - for (uint32_t d = 0; d < x->descriptorCount; d++) { - const VkDescriptorImageInfo *ii1 = &x->pImageInfo[d]; - memcpy(out + n, &ii1->sampler, 8), memcpy(out + n + 8, &ii1->imageView, 8); - memcpy(out + n + 16, &ii1->imageLayout, 4), n += 20; - } - } else if (x->pBufferInfo) { - memcpy(out + n, x->pBufferInfo, sizeof *x->pBufferInfo * x->descriptorCount); - n += (uint32_t)(sizeof *x->pBufferInfo * x->descriptorCount); - } else { - memcpy(out + n, x->pTexelBufferView, sizeof *x->pTexelBufferView * x->descriptorCount); - n += (uint32_t)(sizeof *x->pTexelBufferView * x->descriptorCount); - } - } - return n; -} - static int push_skip_on(void) { // TOTK_PUSH_SKIP=0: every graphics draw pushes its set again static int on = -1; if (on < 0) { @@ -1015,30 +1048,190 @@ static int push_skip_on(void) { // TOTK_PUSH_SKIP=0: every graphics draw pushe return on; } +// A graphics draw whose descriptor inputs equal the previous graphics draw's in this command buffer skips building its +// sets: every write would be the same, the push would be skipped and the allocated set reused without a bind (about a +// quarter of field and monsters2 draws, TOTK_BIND_STATS). The inputs, compared and stored in place, are what +// bind_resources and its helpers read beyond the pipeline info's fixed bindings and Z's stand-ins: +// - the info (bindings, layouts), the program and its serial (the compiler-constant range of UBO slot 1); +// - vk_texture_slots_gen, vk_texture_epoch (a slot's image, view and once-per-submit storage check), vk_sampler_gen; +// - vk_mem_gen (buffer and offset of a span, mirror creation and per-submission sync); +// - the images in the feedback layout (the sampled layout); +// - per binding: the support inputs, c0's storage table and base instance, the bound uniform and storage ranges, the +// texture and image handles (RsList; compared only when a bind op ran since the last draw, bind_gen). +// The side effects of a build repeat as no-ops in the same command buffer and submission: c0 and support buffers kept +// in B, note_written's ranges, vk_mem_gpu_wrote's write-backs, the per-submission mirror sync and dirty census note, +// the views cached per submission. The census counts are added again. A build that copied guest bytes into transient +// memory (an unaligned host range) is never skipped after: the copy may differ. B.res_valid says the sets bound hold +// the last build; vk_exec_draw_new_cb's memset clears it, and every build clears it until it ends. +// TOTK_BIND_SKIP=0 builds every draw; =check builds every draw and stops when a draw with equal inputs gets other +// writes, census counts or descriptor commands. +static int bind_skip_on(void) { + if (RS.on < 0) { + const char *e = cfg_get("TOTK_BIND_SKIP"); + RS.on = e && !strcmp(e, "check") ? 2 : !(e && !strcmp(e, "0")); + if (RS.on != 1) rt_log("vk_exec: descriptor build skip %s", RS.on ? "check" : "off"); + } + return RS.on; +} + +static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, size_t len) { + uint32_t off = (uint32_t)((const uint8_t *)at - (const uint8_t *)ex); + if (*n && r[*n - 1].off + r[*n - 1].len == off) r[*n - 1].len += (uint32_t)len; + else r[(*n)++] = (RsRange){off, (uint32_t)len}; +} + +static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { + if (info == rs_last) return rs_last_l; + if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info); + RsList *l = nvn_hm_get(&rs_lists, &info); + if (!l) { + static RsRange tmp[MAX_DESC]; + uint32_t n = 0, nd = 0; + int sup = 0, c0 = 0, ok = 1; + for (uint32_t i = 0; i < info->n_bindings && ok; i++) { + const VkPipeBinding *b = &info->bindings[i]; + const GfxShaderRes *r = b->res; + uint32_t cnt = b->count ? b->count : 1; + if ((nd += cnt) > MAX_DESC) { ok = 0; break; } // the build stops on it + int st = nvn_stage_of(b->stage); + if (b->kind == VK_PIPE_SUPPORT) { + sup = 1; + } else if (b->kind == GFX_RES_UBO) { + for (uint32_t k = 0; k < cnt; k++) { + int slot = r->slot + (int)k; + if (slot == 0 && c0_on()) c0 = 1, rs_add(tmp, &n, ex, ex->ssbo[st], sizeof ex->ssbo[st]); + else if (slot >= 3 && slot - 3 < 18) rs_add(tmp, &n, ex, &ex->ubo[st][slot - 3], sizeof(p_bind_buf)); + } // slot 1: the program (bind_sig); others: the zero buffer + } else if (b->kind == GFX_RES_SSBO) { // every descriptor of the binding is the same range + int ubo, slot = ssbo_slot(r, st, &ubo); + if (slot >= 0 && slot < 16) rs_add(tmp, &n, ex, ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot], sizeof(p_bind_buf)); + } else if ((b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER && + b->type != VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) { + int img = b->kind == GFX_RES_IMG, cap = img ? 32 : 128; + for (uint32_t k = 0; k < cnt; k++) { + int slot = r->slot + (int)k; + if (slot >= 0 && slot < cap) rs_add(tmp, &n, ex, img ? &ex->img[st][slot] : &ex->tex[st][slot], 8); + } + } + } + uint64_t bytes = 0; + for (uint32_t i = 0; i < n; i++) bytes += tmp[i].len; + if (bytes > RS_BYTES) ok = 0; + l = malloc(sizeof *l + sizeof *tmp * n); + if (!l) rt_fatal("vk: out of memory"); + l->n = n, l->bytes = (uint32_t)bytes, l->sup = (uint8_t)sup, l->c0 = (uint8_t)c0, l->ok = (uint8_t)ok; + memcpy(l->r, tmp, sizeof *tmp * n); + nvn_hm_put(&rs_lists, &info, l); + } + rs_last = info, rs_last_l = l; + return l; +} + +// The signature of a graphics draw's descriptor inputs into RS; 1 when it equals the stored one. The bound-slot bytes +// are compared only when a bind op ran since they were stored or the pipeline changed: otherwise they are the same. +// Check mode compares them anyway and stops when the bind op generation said they could not differ but they do. +static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { + const RsList *l = rs_list(ex, info); + if (!l->ok) return RS.hn = 0, RS.info = NULL, 0; + uint64_t *h = RS.h, diff = 0; + uint32_t n = 0; +#define PUT(v) do { uint64_t v_ = (uint64_t)(v); diff |= h[n] ^ v_; h[n++] = v_; } while (0) + PUT((uintptr_t)info); + PUT((uintptr_t)ex->prog); + PUT(ex->prog->serial); + PUT((uint64_t)vk_texture_slots_gen() << 32 | vk_texture_epoch()); + PUT((uint64_t)vk_sampler_gen() << 32 | (uint32_t)ex->nfbl); + PUT(vk_mem_gen()); + for (int i = 0; i < ex->nfbl && i < 9; i++) PUT((uintptr_t)ex->fbl_img[i]); + if (l->sup) { + float in[3]; + support_in(ex, in); + uint64_t a; + uint32_t ps; + memcpy(&a, in, 8), memcpy(&ps, &in[2], 4); + PUT(a); + PUT(ps); + } + if (l->c0) PUT(c0_binst); +#undef PUT + diff |= n != RS.hn; + RS.hn = n; + int fast = RS.info == info && RS.gen == bind_gen; + if (fast && RS.on != 2) return !diff; + const uint8_t *e = (const uint8_t *)ex; + uint8_t *d = RS.body; + int bd = 0; + for (uint32_t i = 0; i < l->n; i++) { + const RsRange *r = &l->r[i]; + if (!bd && memcmp(d, e + r->off, r->len)) bd = 1; + if (bd) memcpy(d, e + r->off, r->len); + d += r->len; + } + if (fast && bd) rt_fatal("vk_exec: descriptor build skip check: bound slots changed without a bind op"); + RS.info = info, RS.gen = bind_gen; + return !diff && !bd; +} + +// Writes allocated set s from its packed descriptors d: through the layout's update template, or with the write +// structs w[i0..i1) when it has none. +static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, const VkDescSlot *d, VkWriteDescriptorSet *w, + uint32_t i0, uint32_t i1) { + if (i1 == i0) return; + if (info->tpl[s]) { + vk_pipeline_update_tpl_fn()(vk_ctx()->device, set, info->tpl[s], d); + return; + } + for (uint32_t k = i0; k < i1; k++) w[k].dstSet = set; + vkUpdateDescriptorSets(vk_ctx()->device, i1 - i0, w + i0, 0, NULL); +} + // Writes the pipeline's sets from the bound state: set 0, and set 1 with independent layouts. Each is pushed into the // command buffer, or allocated, written and bound; a graphics set whose contents equal the last one allocated for that // set number in this command buffer is reused, and bound again only when another layout or a push came between. A // graphics push equal to the last one at its set number, with no other layout or bind between, is not recorded. +// The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update +// template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. static void bind_resources(VkExec *ex, const VkPipeInfo *info) { static VkWriteDescriptorSet w[MAX_BINDINGS]; - static VkDescriptorBufferInfo bi[MAX_DESC]; - static VkDescriptorImageInfo ii[MAX_DESC]; + static VkDescSlot d[MAX_DESC]; static VkBufferView bv[MAX_DESC]; if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS); bind_stats(info); + int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS; + int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = B.res_valid; + if (skip) { + same = bind_sig(ex, info); + if (same && was_valid && skip == 1) { + for (int i = 0; i < RS.ncnt; i++) vk_mem_count(RS.cnt[i].cls, RS.cnt[i].use, RS.cnt[i].bytes); + BS.bind_skip++; + ex->recorded++; + return; + } + B.res_valid = 0; // until the build below ends + if (skip == 2) memcpy(RS.pcnt, RS.cnt, sizeof RS.cnt), RS.pncnt = RS.ncnt; + RS.rec = 1, RS.ncnt = 0, RS.cnt_over = 0, RS.volatile_ = 0, RS.cmds = 0; + } + uint32_t n_sets = info->n_sets ? info->n_sets : 1; + uint32_t lo[2] = {0, 0}, hi[2] = {0, 0}, d0[2] = {0, 0}; // bindings [lo, hi) and first descriptor per set + int need_w = 0; // a set without a template: fill the write structs + for (uint32_t s = 0, i0 = 0; s < n_sets; s++) { + uint32_t i1 = i0; + while (i1 < info->n_bindings && (n_sets == 1 || info->bindings[i1].set == s)) i1++; + lo[s] = i0, hi[s] = i1, i0 = i1; + if (i1 > lo[s] && !info->tpl[s]) need_w = 1; + } uint32_t nd = 0; for (uint32_t i = 0; i < info->n_bindings; i++) { const VkPipeBinding *b = &info->bindings[i]; const GfxShaderRes *r = b->res; uint32_t cnt = b->count ? b->count : 1; if (nd + cnt > MAX_DESC) rt_fatal("vk: %u descriptors exceed the replay's %d", nd + cnt, MAX_DESC); - w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, - .dstBinding = b->binding, .descriptorCount = cnt, .descriptorType = b->type}; - int st = nvn_stage_of(b->stage); + if (n_sets == 2 && i == lo[1]) d0[1] = nd; + VkDescSlot *o = &d[nd]; + int st = nvn_stage_of(b->stage), kind = 0; // kind: 0 buffer, 1 image, 2 texel buffer view if (b->kind == VK_PIPE_SUPPORT) { - float vw = ex->in_pass && ex->has_vp ? ex->vp[0][2] : (float)ex->pass_w; - float vh = ex->in_pass && ex->has_vp ? ex->vp[0][3] : (float)ex->pass_h; - float in[3] = {vw, vh, ex->point_size > 0 ? ex->point_size : 1.0f}; + float in[3]; + support_in(ex, in); if (!B.sup_valid || memcmp(in, B.sup_in, sizeof in)) { // support_fill reads only these uint8_t tmp[SUPPORT_BYTES]; support_fill(tmp, in[0], in[1], in[2]); @@ -1051,38 +1244,45 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { B.sup_buf = sb, B.sup_off = so, B.sup_valid = 1; } } - bi[nd] = (VkDescriptorBufferInfo){B.sup_buf, B.sup_off, SUPPORT_BYTES}; - w[i].pBufferInfo = &bi[nd]; + o->buffer = (VkDescriptorBufferInfo){B.sup_buf, B.sup_off, SUPPORT_BYTES}; } else if (b->kind == GFX_RES_UBO) { - for (uint32_t k = 0; k < cnt; k++) bi[nd + k] = ubo_info(ex, st, r->slot + (int)k); - w[i].pBufferInfo = &bi[nd]; + for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ubo_info(ex, st, r->slot + (int)k); } else if (b->kind == GFX_RES_SSBO) { - for (uint32_t k = 0; k < cnt; k++) bi[nd + k] = ssbo_info(ex, st, r, r->flags & 1); - w[i].pBufferInfo = &bi[nd]; + for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ssbo_info(ex, st, r, r->flags & 1); } else if (b->kind == GFX_RES_TEX || b->kind == GFX_RES_IMG) { int img = b->kind == GFX_RES_IMG; if (b->type == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER || b->type == VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER) { - for (uint32_t k = 0; k < cnt; k++) bv[nd + k] = Z.zview; // buffer textures are not replayed yet - w[i].pTexelBufferView = &bv[nd]; + kind = 2; + for (uint32_t k = 0; k < cnt; k++) { // buffer textures are not replayed yet + o[k].w[0] = 0, o[k].w[1] = 0, o[k].w[2] = 0; + o[k].view = Z.zview; + if (need_w) bv[nd + k] = Z.zview; + } } else { + kind = 1; vk_texture_lock(); // tex_info looks up with the texture lock held for (uint32_t k = 0; k < cnt; k++) { int slot = r->slot + (int)k; uint64_t h = 0; if (img) h = slot >= 0 && slot < 32 ? ex->img[st][slot] : 0; else h = slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0; - tex_info(ex, r, h, img, r->separate, &ii[nd + k]); + tex_info(ex, r, h, img, r->separate, &o[k].image); + o[k].w[2] = (uint32_t)o[k].image.imageLayout; // zero the tail padding } vk_texture_unlock(); - w[i].pImageInfo = &ii[nd]; } } else { rt_fatal("vk: binding %u has an unknown kind %d", (unsigned)b->binding, (int)b->kind); } + if (need_w) { + w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .dstBinding = b->binding, .descriptorCount = cnt, .descriptorType = b->type}; + if (kind == 0) w[i].pBufferInfo = &o->buffer; // a VkDescSlot is as large as either info struct + else if (kind == 1) w[i].pImageInfo = &o->image; + else w[i].pTexelBufferView = &bv[nd]; + } nd += cnt; } - uint32_t n_sets = info->n_sets ? info->n_sets : 1; - int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS; // Under the validation layer, a new graphics layout binds set 1 before set 0 is pushed or bound. The layer (SDK // 1.4.363, PipelineLayoutCompatDef::operator==) takes any two INDEPENDENT_SETS layouts as compatible, so // vkCmdPushDescriptorSetKHR keeps the push set it made for an earlier layout and drops writes to bindings that @@ -1095,59 +1295,78 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { B.ds_pushed[0] = B.ds_pushed[1] = 0; set1_first = n_sets == 2 && vk_ctx()->validation; } - uint32_t lo[2] = {0, 0}, hi[2] = {0, 0}; - for (uint32_t s = 0, i0 = 0; s < n_sets; s++) { - uint32_t i1 = i0; - while (i1 < info->n_bindings && (n_sets == 1 || info->bindings[i1].set == s)) i1++; - lo[s] = i0, hi[s] = i1, i0 = i1; - } for (uint32_t k = 0; k < n_sets; k++) { uint32_t s = set1_first ? 1 - k : k, i0 = lo[s], i1 = hi[s]; + const VkDescSlot *ds = &d[d0[s]]; + uint32_t dbytes = (uint32_t)(((s + 1 < n_sets ? d0[s + 1] : nd) - d0[s]) * sizeof *d); if (s && i1 == i0) { // no fragment stage: set 1 is empty and unused if (set1_first) { // bound only to drop the layer's push set (comment above) VkDescriptorSet set = alloc_set(ex, info->set_layout1); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 1, 1, &set, 0, NULL); B.ds_bound[1] = set, B.ds_pushed[1] = 0; + RS.cmds++; } continue; } if (s ? info->push1 : info->push) { - uint32_t n = gfx && push_skip_on() ? ds_sig(w + i0, i1 - i0, DS.tmp) : 0; - if (n && B.ds_pushed[s] && DS.pn[s] == n && !memcmp(DS.psig[s], DS.tmp, n)) { + uint32_t n = gfx && push_skip_on() ? dbytes : 0; + if (n && B.ds_pushed[s] && DS.pn[s] == n && !memcmp(DS.psig[s], ds, n)) { BS.push_skip++; continue; } - vk_pipeline_push_fn()(ex->cb, info->bind_point, info->layout, s, i1 - i0, w + i0); + if (info->tpl[s]) vk_pipeline_push_tpl_fn()(ex->cb, info->tpl[s], info->layout, s, ds); + else vk_pipeline_push_fn()(ex->cb, info->bind_point, info->layout, s, i1 - i0, w + i0); BS.push++; + RS.cmds++; if (gfx) { B.ds_bound[s] = VK_NULL_HANDLE, B.ds_pushed[s] = n != 0; - if (n) DS.pn[s] = n, memcpy(DS.psig[s], DS.tmp, n); + if (n) DS.pn[s] = n, memcpy(DS.psig[s], ds, n); } } else if (!gfx || s > 1) { VkDescriptorSet set = alloc_set(ex, s ? info->set_layout1 : info->set_layout); - for (uint32_t k = i0; k < i1; k++) w[k].dstSet = set; - if (i1 > i0) vkUpdateDescriptorSets(vk_ctx()->device, i1 - i0, w + i0, 0, NULL); + write_set(info, s, set, ds, w, i0, i1); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL); } else { VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout; - uint32_t n = ds_sig(w + i0, i1 - i0, DS.tmp); VkDescriptorSet set = B.ds_last[s]; - if (!set || B.ds_last_layout[s] != sl || DS.n[s] != n || memcmp(DS.sig[s], DS.tmp, n)) { + if (!set || B.ds_last_layout[s] != sl || DS.n[s] != dbytes || memcmp(DS.sig[s], ds, dbytes)) { set = alloc_set(ex, sl); - for (uint32_t k = i0; k < i1; k++) w[k].dstSet = set; - if (i1 > i0) vkUpdateDescriptorSets(vk_ctx()->device, i1 - i0, w + i0, 0, NULL); - B.ds_last[s] = set, B.ds_last_layout[s] = sl, DS.n[s] = n; - memcpy(DS.sig[s], DS.tmp, n); + write_set(info, s, set, ds, w, i0, i1); + B.ds_last[s] = set, B.ds_last_layout[s] = sl, DS.n[s] = dbytes; + memcpy(DS.sig[s], ds, dbytes); BS.set_new++; + RS.cmds++; } else { BS.set_reuse++; } if (B.ds_bound[s] != set) { vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL); B.ds_bound[s] = set, B.ds_pushed[s] = 0; + RS.cmds++; } } } + if (skip) { + RS.rec = 0; + // An equal next draw records nothing when every push set holds its push (TOTK_PUSH_SKIP on) and nothing made + // the build unrepeatable; allocated sets are always bound to the one written last. + int valid = !RS.volatile_ && !RS.cnt_over && RS.hn; + for (uint32_t s = 0; s < n_sets; s++) + if (hi[s] > lo[s] && (s ? info->push1 : info->push) && !B.ds_pushed[s]) valid = 0; + if (skip == 2) { // with the pipeline info equal (bind_sig), the packed descriptors determine every write + uint32_t n = (uint32_t)(nd * sizeof *d); + int wr = n != RS.ckn || memcmp(RS.ck, d, n); + int cn = RS.ncnt != RS.pncnt || memcmp(RS.cnt, RS.pcnt, sizeof RS.cnt[0] * (size_t)RS.ncnt); + if (same && was_valid && (wr || cn || RS.cmds)) + rt_fatal("vk_exec: descriptor build skip check: a draw of program %llu with the inputs of the previous one " + "gets %s", (unsigned long long)ex->prog->serial, + RS.cmds ? "descriptor commands" : wr ? "other writes" : "other census counts"); + if (same && was_valid) BS.bind_skip++; // the draws a skip would have spared, compared instead + RS.ckn = n; + memcpy(RS.ck, d, n); + } + B.res_valid = valid; + } ex->recorded++; } @@ -1565,6 +1784,7 @@ void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) { case OP_BIND_PROGRAM: { const p_bind_program *b = payload; ex->prog = b->prog, ex->prog_stages = b->stages; + bind_gen++; break; } case OP_BIND_STATE: { @@ -1607,17 +1827,20 @@ void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) { case OP_BIND_UBO: { const p_bind_buf *b = payload; if (b->stage >= 0 && b->stage < 6 && b->index >= 0 && b->index < 18) ex->ubo[b->stage][b->index] = *b; + bind_gen++; break; } case OP_BIND_SSBO: { const p_bind_buf *b = payload; if (b->stage >= 0 && b->stage < 6 && b->index >= 0 && b->index < 16) ex->ssbo[b->stage][b->index] = *b; + bind_gen++; break; } case OP_BIND_TEX: { const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 6 ? ex->tex[b->stage] : NULL, 128, b); ex->tex_serial++; + bind_gen++; break; } case OP_BIND_SAMP: { @@ -1628,6 +1851,7 @@ void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) { case OP_BIND_IMG: { const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 6 ? ex->img[b->stage] : NULL, 32, b); + bind_gen++; break; } case OP_UPDATE_UB: op_update_ub(ex, payload); break; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 5ee24bdf..f7b5e1c0 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -82,6 +82,9 @@ static SRWLOCK pool_lock = SRWLOCK_INIT; // Bumped under pool_lock (exclusive) by every change to the index or a page table: vk_mem_resolve's per-thread hits // stay valid while it is unchanged. static uint64_t pool_gen = 1; +// Bumped when a buffer use may get another buffer for the same span (a mirror created, refused or forgotten) and at +// every submission's start (mirror syncs are per submission). vk_mem_gen: both together. +static uint64_t mirror_gen; static pidx pool_index; // POOL_IMPORT, POOL_PHYS and POOL_VIRT records by GPU address static PoolRec **phys_list; // physical pools by key, for MapVirtual requests static int nphys, cap_phys; @@ -451,6 +454,10 @@ static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) { return ok; } +uint64_t vk_mem_gen(void) { + return __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE) + __atomic_load_n(&mirror_gen, __ATOMIC_ACQUIRE); +} + int vk_mem_class_of(uint64_t addr) { VkMemSpan m; return vk_mem_resolve(addr, 1, &m) ? m.cls : VK_MCLS_NONE; @@ -1054,6 +1061,7 @@ static void mirror_create(PoolRec *r) { VkCtx *c = vk_ctx(); uint64_t n = r->import_hi - r->import_lo; r->mstate = MS_REFUSED; + __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); if (r->kind != POOL_IMPORT || !r->buf || !n) return; AcquireSRWLockShared(&pool_lock); int i = pidx_upper(&pool_index, r->lo) - 1; @@ -1108,6 +1116,7 @@ static void mirror_create(PoolRec *r) { } static void mirror_forget(PoolRec *r) { + __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); for (int i = 0; i < nm; i++) if (mlist[i] == r) { memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist); @@ -1268,6 +1277,7 @@ void vk_mem_mirror_begin(void) { uint64_t t0 = qpc(); uint32_t prev = msub; msub = msub + 1 ? msub + 1 : 1; + __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); mepoch = vk_texture_epoch(); uint64_t q = 0, run0 = 0, run1 = 0; int have = 0; diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 50b299b6..7ea6c2d5 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -88,6 +88,11 @@ static struct { VkDynamicState dyn[32]; uint32_t n_dyn; PFN_vkCmdPushDescriptorSetKHR push; + int tpl; // descriptor update templates on (make_tpl) + PFN_vkCreateDescriptorUpdateTemplate tpl_create; + PFN_vkDestroyDescriptorUpdateTemplate tpl_destroy; + PFN_vkUpdateDescriptorSetWithTemplate tpl_update; + PFN_vkCmdPushDescriptorSetWithTemplateKHR tpl_push; PFN_vkCmdSetVertexInputEXT set_vi; PFN_vkCmdSetDepthClampEnableEXT set_clamp; PFN_vkCmdSetPolygonModeEXT set_poly; @@ -110,6 +115,7 @@ typedef struct Layout { int push1; // set 1 is a push-descriptor set uint32_t n0; // independent layouts: b[0..n0) are set 0, the rest set 1 VkPipelineLayout half[2]; // GPL libraries: {set 0, null}, {null, set 1} (made on first use) + VkDescriptorUpdateTemplate tpl[2]; // per set (make_tpl); VK_NULL_HANDLE: written with write structs VkDescriptorSetLayoutBinding b[]; } Layout; @@ -242,6 +248,20 @@ static int ensure_init(void) { P.f_dual_src = f2.features.dualSrcBlend; #define PROC(field, name) (P.field = (void *)vkGetDeviceProcAddr(c->device, name)) PROC(push, "vkCmdPushDescriptorSetKHR"); + // Descriptor update templates (core 1.1; the push variant comes with VK_KHR_push_descriptor on 1.1): the driver + // walks a layout prepared when the layout is made instead of parsing write structs every draw (DXVK does so for the + // same reason). TOTK_DESC_TEMPLATE=0 keeps the write structs. + const char *dt = cfg_get("TOTK_DESC_TEMPLATE"); + if (dt && !strcmp(dt, "0")) { + rt_log("vk_pipeline: descriptor update templates off (TOTK_DESC_TEMPLATE=0)"); + } else if (c->props.apiVersion >= VK_API_VERSION_1_1 && PROC(tpl_create, "vkCreateDescriptorUpdateTemplate") && + PROC(tpl_destroy, "vkDestroyDescriptorUpdateTemplate") && PROC(tpl_update, "vkUpdateDescriptorSetWithTemplate")) { + P.tpl = 1; + if (!PROC(tpl_push, "vkCmdPushDescriptorSetWithTemplateKHR")) + rt_log("vk_pipeline: vkCmdPushDescriptorSetWithTemplateKHR missing: push sets are written with write structs"); + } else { + rt_log("vk_pipeline: descriptor update templates unavailable: sets are written with write structs"); + } // TOTK_VK_DYN_VERTEX=0/1 forces dynamic vertex input off or on. Default off on Intel's frozen Windows driver for its // older integrated GPUs (UHD 630, Ice Lake: Vulkan 1.3.215 and older), which draws nothing with it (LAS Recomp // 61fe44ff, its issues #8 and #357). @@ -588,6 +608,8 @@ static void vertex_input(const VkPipeState *s, VtxInput *v) { int vk_pipeline_vertex_input_dynamic(void) { return ensure_init() && P.dyn_vi; } PFN_vkCmdPushDescriptorSetKHR vk_pipeline_push_fn(void) { return ensure_init() ? P.push : NULL; } +PFN_vkCmdPushDescriptorSetWithTemplateKHR vk_pipeline_push_tpl_fn(void) { return ensure_init() && P.tpl ? P.tpl_push : NULL; } +PFN_vkUpdateDescriptorSetWithTemplate vk_pipeline_update_tpl_fn(void) { return ensure_init() && P.tpl ? P.tpl_update : NULL; } int vk_pipeline_set_indep(int on) { if (!ensure_init()) return 0; @@ -954,6 +976,37 @@ static int layout_covers(const VkPipeState *s, const Layout *l, uint32_t fs_off, return m < 0; } +// The update template of a set whose bindings are b[0..n): one entry per binding, in order, over the packed array the +// replay fills from the set's first descriptor, a VkDescSlot per descriptor. The bind point follows the stages (compute +// layouts hold the compute stage only). VK_NULL_HANDLE when templates are off, for an empty set, for a push set +// without vkCmdPushDescriptorSetWithTemplateKHR or when creation fails: the replay then writes the set with write +// structs. +static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl, + int push, VkPipelineLayout pl, uint32_t set) { + if (!P.tpl || !n || (push && !P.tpl_push)) return VK_NULL_HANDLE; + VkDescriptorUpdateTemplateEntry e[MAX_BINDINGS]; + size_t off = 0; + VkShaderStageFlags stages = 0; + for (uint32_t i = 0; i < n; i++) { + e[i] = (VkDescriptorUpdateTemplateEntry){b[i].binding, 0, b[i].descriptorCount, b[i].descriptorType, off, sizeof(VkDescSlot)}; + off += sizeof(VkDescSlot) * b[i].descriptorCount; + stages |= b[i].stageFlags; + } + VkDescriptorUpdateTemplateCreateInfo ci = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_UPDATE_TEMPLATE_CREATE_INFO, .descriptorUpdateEntryCount = n, + .pDescriptorUpdateEntries = e, + .templateType = push ? VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR : VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET, + .descriptorSetLayout = sl, + .pipelineBindPoint = stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, + .pipelineLayout = pl, .set = set}; + VkDescriptorUpdateTemplate t; + if (P.tpl_create(P.device, &ci, NULL, &t) != VK_SUCCESS) { + LOG_ONCE("vk_pipeline: descriptor update template creation failed: that set is written with write structs"); + return VK_NULL_HANDLE; + } + return t; +} + // Finds or creates the layout of a binding list (sorted here); NULL on failure. Caller holds P.lock. // Sorts a binding list by binding and hashes it into *h; the descriptor total, or -1 when a binding repeats. static int64_t bindings_sort(Bindings *B, uint64_t *h) { @@ -998,6 +1051,7 @@ static Layout *layout_get(Bindings *B) { free(l); return NULL; } + l->tpl[0] = make_tpl(l->b, l->n, l->set, l->push, l->layout, 0); l->next = *head; *head = l; P.stats.layouts++; @@ -1055,6 +1109,8 @@ static Layout *layout_pair(Bindings *B, Bindings *F) { free(l); return NULL; } + l->tpl[0] = make_tpl(l->b, n0, l->set, l->push, l->layout, 0); + l->tpl[1] = make_tpl(l->b + n0, n - n0, l->set1, l->push1, l->layout, 1); l->next = *head; *head = l; P.stats.layouts++; @@ -1127,6 +1183,7 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel p->info.set_layout1 = l ? l->set1 : VK_NULL_HANDLE; p->info.push1 = l ? l->push1 : 0; p->info.n_sets = l && l->set1 ? 2 : 1; + if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1]; p->lay = l; Pipe **head = &pipes[h % PIPE_BUCKETS]; p->next = *head; @@ -2823,6 +2880,8 @@ void vk_pipeline_shutdown(void) { for (int i = 0; i < LAYOUT_BUCKETS; i++) { for (Layout *l = layouts[i], *n; l; l = n) { n = l->next; + for (int k = 0; k < 2; k++) + if (l->tpl[k]) P.tpl_destroy(P.device, l->tpl[k], NULL); vkDestroyPipelineLayout(P.device, l->layout, NULL); for (int h = 0; h < 2; h++) if (l->half[h]) vkDestroyPipelineLayout(P.device, l->half[h], NULL); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 9d621593..9e7c4b38 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -97,6 +97,7 @@ static uint64_t gpu_writes; // GPU writes into mirrored textures so far: Tex.gp static uint32_t tex_gen = 1; // Bumped with tex_gen and whenever a slot is registered: what vk_texture_by_id returns for any slot may have changed. static uint32_t slots_gen = 1; +static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line static void tex_gen_bump(void) { @@ -840,6 +841,7 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { void vk_texture_lock(void) { lock(); } void vk_texture_unlock(void) { unlock(); } uint32_t vk_texture_slots_gen(void) { return slots_gen; } +uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); } const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { lock(); @@ -1316,6 +1318,7 @@ void vk_sampler_register(int id, const samp_par *s) { sslots[id].p = *s; sslots[id].s = VK_NULL_HANDLE; sslots[id].used = 1; + samp_gen++; unlock(); }