Merge the replay CPU work: texture lookups and descriptors kept per slot, image views per image, texture types only for the slots a shader reads, the previous draw's pipeline reused, vertex buffer spans, plain-add traffic census, completion wakes only when due, storage checks per stamped 2 MiB region and shader stages per program serial; fight replay 13.8 -> 9.2 ms a frame on A, field 9.5 -> 7.2 ms

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-27 08:14:21 +03:00
commit ad3fa470d1
11 files changed
+383 -111

No files matched your search

+5 -4
View File
@@ -39,13 +39,13 @@ static const char *const known[] = {
"GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MENU_LANG", "MENU_TEST",
"MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",
"OPERATION_MODE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
"OPERATION_MODE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
"PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES",
"PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC",
"REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
"SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER",
"SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS", "TRACE", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
"SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VS_OUTS",
"WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS",
};
@@ -65,7 +65,7 @@ static const struct { const char *key, *section; } sections[] = {
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"},
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
{"TEX_ALIAS", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
{"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
{"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"VK_VALIDATION", "performance"},
{"VK_DYN_VERTEX", "performance"}, {"VK_SUBGROUP_PIN", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
@@ -104,7 +104,8 @@ static const CfgMeta metas[] = {
{"TEX_ALIAS", "1", "0|1", RESTART},
// render fixes
{"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART},
{"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"DESC_TEMPLATE", "1", "0|1", RESTART},
{"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART},
{"DESC_TEMPLATE", "1", "0|1", RESTART},
{"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART},
// Vulkan
{"VK_VALIDATION", "0", "0|1", RESTART}, {"VK_DYN_VERTEX", "", "0|1", RESTART},
+1
View File
@@ -93,6 +93,7 @@ typedef struct {
typedef struct { uint8_t type, comps, packed; } GfxShaderIo; // GFX_TYPE_*, components, 1 with a Component decoration
int gfx_shader_stage(const GfxShader *s); // GFX_STAGE_VS, GFX_STAGE_FS or GFX_STAGE_CS
int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res); // count of GFX_RES_* kind; array in *res
uint32_t gfx_shader_slot_mask(const GfxShader *s); // texture/image slots whose type its translation read
uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]); // Input Locations (VS attributes, FS varyings) as a mask
uint32_t gfx_shader_outputs(const GfxShader *s, GfxShaderIo io[32]); // Output Locations (VS varyings, FS colour targets)
void gfx_shader_local_size(const GfxShader *s, uint32_t local[3]); // CS: the LocalSize execution mode; 1, 1, 1 otherwise
+4
View File
@@ -1418,6 +1418,10 @@ int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res)
return s->nres[kind];
}
// The slots of the use mask: answers outside it select the same translation (mask_answers), so a caller may leave
// their texture and image types zero. Parses the reflection: callers keep the result.
uint32_t gfx_shader_slot_mask(const GfxShader *s) { return use_of(s).slots; }
uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]) {
if (io) memcpy(io, s->in, sizeof s->in);
return s->in_mask;
+2
View File
@@ -608,6 +608,8 @@ static void tab_advanced(float scale) {
opt_toggle("TOTK_FIX_ZERO", UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC);
opt_toggle("TOTK_BIND_SKIP", UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC);
opt_toggle("TOTK_SHADER_MEMO", UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC);
opt_toggle("TOTK_PIPE_MEMO", UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC);
opt_toggle("TOTK_TEX_MEMO", UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC);
opt_toggle("TOTK_DESC_TEMPLATE", UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC);
opt_toggle("TOTK_PUSH_SKIP", UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC);
opt_slider("TOTK_SPLIT_DRAWS", UI_MENU_SPLIT_DRAWS, 0, 1000, UI_MENU_SPLIT_DRAWS_DESC, 8, "%.0f");
+4
View File
@@ -127,6 +127,8 @@ static const char *const text_en[UI_TEXT_COUNT] = {
[UI_MENU_BIND_SKIP] = "Skip repeated binds",
[UI_MENU_BIND_SKIP_DESC] = "Optimisation: skips bindings that did not change.",
[UI_MENU_SHADER_MEMO] = "Shader memo", [UI_MENU_SHADER_MEMO_DESC] = "Optimisation: remembers shader lookups.",
[UI_MENU_PIPE_MEMO] = "Pipeline memo", [UI_MENU_PIPE_MEMO_DESC] = "Optimisation: reuses the previous draw's pipeline when its state repeats.",
[UI_MENU_TEX_MEMO] = "Texture binding memo", [UI_MENU_TEX_MEMO_DESC] = "Optimisation: remembers texture descriptors between draws.",
[UI_MENU_DESC_TEMPLATE] = "Descriptor templates",
[UI_MENU_DESC_TEMPLATE_DESC] = "Optimisation: updates descriptors in bulk.",
[UI_MENU_PUSH_SKIP] = "Skip repeated push constants",
@@ -271,6 +273,8 @@ static const char *const text_ru[UI_TEXT_COUNT] = {
[UI_MENU_BIND_SKIP] = "Пропуск повторных привязок",
[UI_MENU_BIND_SKIP_DESC] = "Оптимизация: пропускает неизменившиеся привязки.",
[UI_MENU_SHADER_MEMO] = "Память шейдеров", [UI_MENU_SHADER_MEMO_DESC] = "Оптимизация: запоминает поиск шейдеров.",
[UI_MENU_PIPE_MEMO] = "Память конвейера", [UI_MENU_PIPE_MEMO_DESC] = "Оптимизация: повторяет конвейер прошлой отрисовки, если её состояние не изменилось.",
[UI_MENU_TEX_MEMO] = "Память привязок текстур", [UI_MENU_TEX_MEMO_DESC] = "Оптимизация: запоминает дескрипторы текстур между отрисовками.",
[UI_MENU_DESC_TEMPLATE] = "Шаблоны дескрипторов",
[UI_MENU_DESC_TEMPLATE_DESC] = "Оптимизация: обновляет дескрипторы пачками.",
[UI_MENU_PUSH_SKIP] = "Пропуск повторных push-констант",
+2
View File
@@ -84,6 +84,8 @@ typedef enum {
UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC,
UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC,
UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC,
UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC,
UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC,
UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC,
UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC,
UI_MENU_SPLIT_DRAWS, UI_MENU_SPLIT_DRAWS_DESC,
+5
View File
@@ -198,6 +198,7 @@ uint64_t vk_upload_flush(int wait);
// Deferred destruction: freed once the upload batches recorded so far and the timeline value last noted have completed.
void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestroyImage only
void vk_defer_view(VkImageView view);
void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc);
// ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c.
enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK };
@@ -263,6 +264,10 @@ VkSampler vk_sampler_by_id_locked(int id);
// on the Switch although the game binds a linear sampler (MacNativeGames 6283599). VK_NULL_HANDLE for an empty slot.
VkSampler vk_sampler_point_by_id_locked(int id);
uint32_t vk_texture_slots_gen(void);
// What vk_texture_by_id(id) returns can change only when this does: the slot is registered again, or a texture entry at
// its storage address is created, dropped, released or taken back into use. Unlocked; read it before the lookup it
// guards (a lookup may create an entry). 0 when the slot was never registered: nothing is kept under 0.
uint64_t vk_texture_slot_token(int id);
uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered
void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage
void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission
+295 -87
View File
@@ -67,7 +67,6 @@ typedef struct {
VkImageViewType type;
uint32_t aspect, level0, levels, layer0, layers, swz;
} ViewKey;
typedef struct { ViewKey k; VkImageView view; } ViewEnt;
_Static_assert(sizeof(ViewKey) == 40, "ViewKey is hashed as bytes: no padding");
typedef struct { uint64_t addr, size; } Range;
typedef struct {
@@ -75,9 +74,6 @@ typedef struct {
int nchunks, cchunks;
VkDescriptorPool *pools;
int npools, cpools;
ViewEnt *views;
int nviews, cviews;
nvn_hmap view_of; // ViewKey -> the VkImageView in views
Range *written;
int nwritten, cwritten;
} CbRes;
@@ -102,6 +98,10 @@ static struct {
int st_valid;
VkBuffer vb[17]; // vertex buffers last bound
VkDeviceSize vo[17], vsz[17], vstr[17];
int vcls[17]; // their pool class and pool record, for the census
void *vpool[17];
NVNbufferRange vin[16]; // the ranges they were resolved from, under vk_mem_gen vgen
uint64_t vgen;
int vb_valid;
uint8_t support[SUPPORT_BYTES]; // last support buffer and where it lives
float sup_in[3]; // the support_fill arguments it was filled from
@@ -175,7 +175,6 @@ static void cb_done(void *arg) { // completion thread
CbRes *r = arg;
VkCtx *c = vk_ctx();
for (int i = 0; i < r->nwritten; i++) vk_guest_written(r->written[i].addr, r->written[i].size);
for (int i = 0; i < r->nviews; i++) vkDestroyImageView(c->device, r->views[i].view, NULL);
for (int i = 0; i < r->npools; i++) {
vkResetDescriptorPool(c->device, r->pools[i], 0);
AcquireSRWLockExclusive(&spare_lock);
@@ -191,8 +190,7 @@ static void cb_done(void *arg) { // completion thread
ReleaseSRWLockExclusive(&spare_lock);
if (!kept) vmaDestroyBuffer(c->vma, r->chunks[i].buf, r->chunks[i].alloc);
}
free(r->view_of.keys), free(r->view_of.vals);
free(r->written), free(r->views), free(r->pools), free(r->chunks), free(r);
free(r->written), free(r->pools), free(r->chunks), free(r);
}
void vk_exec_draw_new_cb(void) { memset(&B, 0, sizeof B); }
@@ -201,7 +199,6 @@ static CbRes *cb_res(VkExec *ex) {
if (cur && cur_fence == ex->fence) return cur;
cur = calloc(1, sizeof *cur);
if (!cur) rt_fatal("vk: out of memory");
nvn_hm_init(&cur->view_of, sizeof(ViewKey));
cur_fence = ex->fence;
memset(&B, 0, sizeof B);
vk_completion_add(ex->fence, cb_done, cur);
@@ -455,8 +452,15 @@ VkImageAspectFlags vk_exec_view_aspect(const VkTexImage *img, const view_t *v, i
: VK_IMAGE_ASPECT_COLOR_BIT;
}
// Image views of registered textures, kept for their image's lifetime: made per command buffer they were created again
// every frame (vkCreateImageView took about 8% of the fight replay on the 7900X). Used under the texture lock, as
// vk_exec_image_gone is (destroy_tex), which hands the views of a destroyed image to the deferred release with it.
typedef struct ViewNode { struct ViewNode *next; ViewKey k; VkImageView view; } ViewNode;
static nvn_hmap view_of; // ViewKey -> VkImageView
static nvn_hmap views_by_image; // VkImage -> its first ViewNode
// An image view of a registered texture for a shader resource of type `rtype` (GfxShaderRes.type); VK_NULL_HANDLE when
// the image cannot serve it (the caller binds a stand-in).
// the image cannot serve it (the caller binds a stand-in). Texture lock held.
static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, int hv, uint16_t rtype, int storage) {
VkImageViewType vt = res_view_type(rtype);
int ms = (rtype & 0x200) != 0;
@@ -505,9 +509,9 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v,
}
}
uint32_t swz = (uint32_t)sw[0] | (uint32_t)sw[1] << 8 | (uint32_t)sw[2] << 16 | (uint32_t)sw[3] << 24;
CbRes *r = cb_res(ex);
ViewKey k = {img->image, img->format, vt, aspect, l0, nl, y0, ny, swz};
VkImageView found = (VkImageView)nvn_hm_get(&r->view_of, &k);
if (!view_of.klen) nvn_hm_init(&view_of, sizeof(ViewKey)), nvn_hm_init(&views_by_image, sizeof(VkImage));
VkImageView found = (VkImageView)nvn_hm_get(&view_of, &k);
if (found) return found;
VkImageViewCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = img->image, .viewType = vt,
.format = img->format,
@@ -518,12 +522,24 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v,
if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c)
VkImageView view;
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE;
if (r->nviews == r->cviews) r->views = grow(r->views, &r->cviews, sizeof *r->views);
r->views[r->nviews++] = (ViewEnt){k, view};
nvn_hm_put(&r->view_of, &k, (void *)view);
ViewNode *n = malloc(sizeof *n);
if (!n) rt_fatal("vk: out of memory");
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
nvn_hm_put(&views_by_image, &img->image, n);
nvn_hm_put(&view_of, &k, (void *)view);
return view;
}
void vk_exec_image_gone(VkImage image) {
if (!views_by_image.klen) return;
for (ViewNode *n = nvn_hm_del(&views_by_image, &image), *next; n; n = next) {
next = n->next;
nvn_hm_del(&view_of, &n->k);
vk_defer_view(n->view);
free(n);
}
}
// ---- program to shaders
// (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the
@@ -534,7 +550,7 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v,
typedef struct { uint64_t serial; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt;
static ShaderEnt shader_tab[SHADER_SETS][SHADER_WAYS];
static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) {
static int find_stage_scan(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) {
for (uint32_t i = 0; i < p->nstage && i < 6; i++) {
const uint8_t *c = p->sd[i].control;
if (!c || !p->sd[i].data) continue;
@@ -548,12 +564,79 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const
return 0;
}
// Texture and image types of a stage's first 32 slots, kept while every slot's resolution (vk_texture_slots_gen)
// stays the same: only slots whose handle changed are looked up again, each lookup being a hash probe.
static struct { int valid; uint32_t gen; uint64_t tex[32], img[32]; uint8_t tt[32], it[32]; } AC[6];
// Which shader of a program serves each NVN stage, and its code length, per program serial (new at every
// nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs
// several times a draw. Direct-mapped; a collision only scans again.
#define STAGEM_N 512
static struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } stagem[STAGEM_N];
static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) {
if (!p->serial || (unsigned)nvn_stage >= 6) return find_stage_scan(p, nvn_stage, ctl, code, len);
typeof(stagem[0]) *m = &stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55];
if (m->serial != p->serial) {
m->serial = p->serial;
memset(m->idx, -1, sizeof m->idx);
for (uint32_t i = 0; i < p->nstage && i < 6; i++) {
const uint8_t *c = p->sd[i].control;
if (!c || !p->sd[i].data) continue;
uint32_t st;
memcpy(&st, c + 0x714, 4);
if (st < 6 && m->idx[st] < 0) m->idx[st] = (int8_t)i, memcpy(&m->len[st], c + 0x704, 4);
}
}
int i = m->idx[nvn_stage];
if (i < 0) return 0;
*ctl = p->sd[i].control, *code = (const uint8_t *)(uintptr_t)p->sd[i].data, *len = m->len[nvn_stage];
return 1;
}
// Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it
// was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve
// otherwise (slot_still).
static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32]; } AC[6];
// vk_texture_slot_token of the slot a texture or image handle names; an empty handle always resolves to nothing.
static uint64_t token_of(uint64_t h) { return h ? vk_texture_slot_token((int)(h & 0xfffff)) : 1; }
// Whether what handle h resolved to under the global generation *egen and its slot token etok (both read before that
// lookup) still holds at global generation gen: nothing at all changed since, or not its slot (the token, read only when
// the global generation moved, about every 16 draws). A token hit moves *egen to gen.
static int slot_still(uint32_t *egen, uint64_t etok, uint64_t h, uint32_t gen) {
if (*egen == gen) return 1;
uint64_t t = token_of(h);
if (!t || t != etok) return 0;
*egen = gen;
return 1;
}
static int shader_memo_on(void);
static void answers_for(VkExec *ex, int st, const uint8_t *ctl, GfxShaderAnswers *a) {
// The slots of (program, stage) whose types its translation reads (gfx_shader_slot_mask), learned from its first
// shader; ~0u while unknown. The types of other slots select the same shader, so answers_for leaves them zero instead
// of looking up all 64 handles. Direct-mapped by serial and stage; a collision only relearns.
#define SLOTM_N 4096
static struct { uint64_t serial; int st; uint32_t m; } slotm[SLOTM_N];
static uint32_t slotm_idx(uint64_t serial, int st) { return (uint32_t)((serial * 8 + (uint64_t)st) * 0x9E3779B97F4A7C15ull >> 52); }
static uint32_t slot_mask_of(uint64_t serial, int st) {
const typeof(slotm[0]) *e = &slotm[slotm_idx(serial, st)];
return serial && e->serial == serial && e->st == st ? e->m : ~0u;
}
static void slot_mask_learn(uint64_t serial, int st, const GfxShader *s) {
if (!serial || slot_mask_of(serial, st) != ~0u) return;
slotm[slotm_idx(serial, st)] = (typeof(slotm[0])){serial, st, gfx_shader_slot_mask(s)};
}
static uint8_t type_of_handle(uint64_t h) { // texture lock held
view_t v;
int hv;
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
return img ? (uint8_t)gfx_type_of(img, &v, hv) : 0;
}
// The answers of stage st for the slots in mask; the types of other slots stay zero.
static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, GfxShaderAnswers *a) {
memset(a, 0, sizeof *a);
if (st == NVN_STAGE_VS) vk_pipeline_answers(&ex->pipe, a, NULL);
else if (st == NVN_STAGE_FS) vk_pipeline_answers(&ex->pipe, NULL, a);
@@ -564,44 +647,25 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, GfxShaderAnswers
a->local[k] = (uint16_t)(l ? l : 1);
}
typeof(AC[0]) *c = &AC[st];
vk_texture_lock();
uint32_t gen = vk_texture_slots_gen(); // before the lookups: one that creates an entry bumps it
int all = !c->valid || c->gen != gen;
if (all || memcmp(c->tex, ex->tex[st], sizeof c->tex) || memcmp(c->img, ex->img[st], sizeof c->img)) {
for (int slot = 0; slot < 32; slot++) {
view_t v;
int hv;
uint64_t h = ex->tex[st][slot];
if (all || h != c->tex[slot]) {
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
c->tt[slot] = img ? (uint8_t)gfx_type_of(img, &v, hv) : 0;
}
h = ex->img[st][slot];
if (all || h != c->img[slot]) {
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
c->it[slot] = img ? (uint8_t)gfx_type_of(img, &v, hv) : 0;
}
}
memcpy(c->tex, ex->tex[st], sizeof c->tex);
memcpy(c->img, ex->img[st], sizeof c->img);
c->gen = gen, c->valid = 1;
BS.ac_miss++;
// The generation and tokens are read before the lookups (one that creates an entry changes them) and unlocked: a
// registration racing with them is ordered after this draw, and a slot stored under older ones is only looked up
// again.
uint32_t gen = vk_texture_slots_gen();
int locked = 0, check = shader_memo_on() == 2;
for (uint32_t m = mask; m; m &= m - 1) {
int slot = __builtin_ctz(m);
uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot];
int tmiss = c->tex[slot] != h || !c->ttok[slot] || !slot_still(&c->tgen[slot], c->ttok[slot], h, gen);
int imiss = c->img[slot] != hi || !c->itok[slot] || !slot_still(&c->igen[slot], c->itok[slot], hi, gen);
uint64_t tt = tmiss ? token_of(h) : 0, it = imiss ? token_of(hi) : 0;
if ((tmiss || imiss || check) && !locked) vk_texture_lock(), locked = 1, BS.ac_miss++;
if (tmiss) c->tt[slot] = type_of_handle(h), c->tex[slot] = h, c->ttok[slot] = tt, c->tgen[slot] = gen;
if (imiss) c->it[slot] = type_of_handle(hi), c->img[slot] = hi, c->itok[slot] = it, c->igen[slot] = gen;
if (check && (type_of_handle(h) != c->tt[slot] || type_of_handle(hi) != c->it[slot])) // TOTK_SHADER_MEMO=check
rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot);
a->tex_type[slot] = c->tt[slot], a->img_type[slot] = c->it[slot];
}
if (shader_memo_on() == 2) // check mode: every slot looked up again
for (int slot = 0; slot < 32; slot++) {
view_t v;
int hv;
uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot];
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
int tt = img ? gfx_type_of(img, &v, hv) : 0;
img = hi ? vk_texture_by_id_locked((int)(hi & 0xfffff), &v, &hv) : NULL;
int it = img ? gfx_type_of(img, &v, hv) : 0;
if (tt != c->tt[slot] || it != c->it[slot])
rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot);
}
vk_texture_unlock();
memcpy(a->tex_type, c->tt, sizeof c->tt);
memcpy(a->img_type, c->it, sizeof c->it);
if (locked) vk_texture_unlock();
}
// *present: the program has this stage. Returns NULL when it has not or the translation failed. pending (graphics
@@ -615,8 +679,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
if (!ex->prog || !find_stage(ex->prog, st, &ctl, &code, &len)) return NULL;
*present = 1;
GfxShaderAnswers a;
answers_for(ex, st, ctl, &a);
uint64_t serial = ex->prog->serial;
answers_for(ex, st, ctl, slot_mask_of(serial, st), &a);
ShaderEnt *set = NULL;
if (serial) {
uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded
@@ -637,6 +701,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
: pending && vk_pipeline_async_on() ? gfx_shader_get_async(gs, ctl, code, len, &a, pending)
: gfx_shader_get(gs, ctl, code, len, NULL, &a);
if (pending && *pending) return NULL;
if (s) slot_mask_learn(serial, st, s);
if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st,
(unsigned long long)serial);
else if (set) { // in front, the least recently used way drops out
@@ -760,8 +825,19 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) {
return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES};
}
// The buffer range limits, read once: the device's properties are a cold cache line on every binding of every draw.
static struct { uint64_t ubo_max, ubo_align, sbo_max, sbo_align; } LIM;
static void limits_init(void) {
const VkPhysicalDeviceLimits *l = &vk_ctx()->props.limits;
LIM.ubo_align = l->minUniformBufferOffsetAlignment, LIM.sbo_max = l->maxStorageBufferRange;
LIM.sbo_align = l->minStorageBufferOffsetAlignment;
__atomic_store_n(&LIM.ubo_max, l->maxUniformBufferRange, __ATOMIC_RELEASE);
}
static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) {
uint64_t max = vk_ctx()->props.limits.maxUniformBufferRange, align = vk_ctx()->props.limits.minUniformBufferOffsetAlignment;
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
uint64_t max = LIM.ubo_max, align = LIM.ubo_align;
VkDescriptorBufferInfo bi;
if (slot == 0 && c0_on()) return c0_info(ex, st);
if (slot == 1 && ex->prog) { // compiler constants stored after the shader code
@@ -845,7 +921,8 @@ static uint64_t ubo_ptr(const VkExec *ex, int st, const GfxShaderRes *r) {
}
static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *r, int written) {
uint64_t max = vk_ctx()->props.limits.maxStorageBufferRange, align = vk_ctx()->props.limits.minStorageBufferOffsetAlignment;
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
uint64_t max = LIM.sbo_max, align = LIM.sbo_align;
VkDescriptorBufferInfo bi;
int ubo, slot = ssbo_slot(r, st, &ubo);
p_bind_buf up = {0};
@@ -886,39 +963,85 @@ static int in_fbl(const VkExec *ex, VkImage img) {
return 0;
}
// Attachment slots of the open pass (bit 8 depth) whose subresource the view of img covers.
static uint32_t attached_as(const VkExec *ex, const VkTexImage *img, const view_t *v, int hv) {
uint32_t l0 = 0, l1 = img->levels, a0 = 0, a1 = img->layers, m = 0;
if (hv && v->set_levels) l0 = v->base_level, l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels);
if (hv && v->set_layers) a0 = v->min_layer, a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers);
// The image and the levels [l0, l1) and layers [a0, a1) a texture's view covers.
typedef struct { VkImage image; uint32_t l0, l1, a0, a1; } SubRange;
static SubRange sub_of(const VkTexImage *img, const view_t *v, int hv) {
SubRange r = {img->image, 0, img->levels, 0, img->layers};
if (hv && v->set_levels) r.l0 = v->base_level, r.l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels);
if (hv && v->set_layers) r.a0 = v->min_layer, r.a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers);
return r;
}
// Attachment slots of the open pass (bit 8 depth) whose subresource r covers.
static uint32_t attached_as(const VkExec *ex, const SubRange *r) {
uint32_t m = 0;
if (!r->image) return 0;
for (uint32_t i = 0; i < 9; i++) {
const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth;
if ((i < 8 && i >= ex->pass_ncolor) || t->image != img->image) continue;
if (t->level < l0 || t->level >= l1) continue;
if (!t->is3d && (t->base_layer + t->layers <= a0 || t->base_layer >= a1)) continue;
if ((i < 8 && i >= ex->pass_ncolor) || t->image != r->image) continue;
if (t->level < r->l0 || t->level >= r->l1) continue;
if (!t->is3d && (t->base_layer + t->layers <= r->a0 || t->base_layer >= r->a1)) continue;
m |= 1u << i;
}
return m;
}
// TOTK_TEX_MEMO (tex_info, feedback_scan): 0 off, 1 on, 2 (=check) computes every hit again and stops on a difference.
static int td_mode(void) {
static int on = -1;
if (on < 0) {
const char *e = cfg_value("TOTK_TEX_MEMO");
on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0;
}
return on;
}
// The subresource a texture handle samples, per handle, kept while its slot resolves the same (slot_still): a draw's
// texture bindings change the whole-scan memo below, but the handles they bind were mostly looked up before, in this
// pass or an earlier one. Direct-mapped.
#define FM_N 8192
static struct { uint64_t h, tok; uint32_t gen; SubRange r; } FM[FM_N];
static SubRange sampled_sub(uint64_t h) { // texture lock held
view_t v;
int hv;
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
return img ? sub_of(img, &v, hv) : (SubRange){0};
}
// Attachments of the open pass the fragment shader fs samples; kept while the shader, the texture bindings and the
// pass stay the same.
static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) {
if (ex->fb_fs == fs && ex->fb_serial == ex->tex_serial && ex->fb_pass == ex->pass_serial) return ex->fb_sampled;
const GfxShaderRes *res;
int n = gfx_shader_resources(fs, GFX_RES_TEX, &res);
uint32_t m = 0;
vk_texture_lock();
uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup
int memo = td_mode(), locked = 0;
for (int r = 0; r < n; r++)
for (int k = 0; k < (res[r].count ? res[r].count : 1); k++) {
int slot = res[r].slot + k;
uint64_t h = slot >= 0 && slot < 128 ? ex->tex[NVN_STAGE_FS][slot] : 0;
view_t v;
int hv;
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
if (img) m |= attached_as(ex, img, &v, hv);
if (!h) continue;
typeof(FM[0]) *f = &FM[(h * 0x9E3779B97F4A7C15ull) >> 51];
if (memo && f->h == h && slot_still(&f->gen, f->tok, h, gen)) {
if (memo == 2) {
if (!locked) vk_texture_lock(), locked = 1;
SubRange c = sampled_sub(h);
if (memcmp(&c, &f->r, sizeof c))
rt_fatal("vk_exec: feedback memo of handle %#llx differs from a lookup", (unsigned long long)h);
}
m |= attached_as(ex, &f->r);
continue;
}
uint64_t tok = token_of(h); // before the lookup
if (!locked) vk_texture_lock(), locked = 1;
SubRange sr = sampled_sub(h);
m |= attached_as(ex, &sr);
if (tok) *f = (typeof(*f)){.h = h, .tok = tok, .gen = gen, .r = sr};
else f->h = 0;
}
vk_texture_unlock();
if (locked) vk_texture_unlock();
ex->fb_fs = fs, ex->fb_serial = ex->tex_serial, ex->fb_pass = ex->pass_serial, ex->fb_sampled = m;
return m;
}
@@ -969,8 +1092,61 @@ static void feedback_written(VkExec *ex, const VkPipeState *s) {
if (ex->color[i].image && !(ex->fb_dirty >> i & 1) && vk_pipeline_target_wmask(s, (int)i)) ex->fb_dirty |= 1u << i;
}
static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate,
VkDescriptorImageInfo *out, VkImage *image);
// Sampled-texture descriptors per (handle, resource type, separate), kept while nothing they derive from changed: the
// slot's resolution and texture entry (slot_still) and the samplers (vk_sampler_gen); the layout is set again on every
// hit from the images in the feedback layout (in_fbl of the image kept with it). The first binding of a slot in a submission (vk_texture_epoch) still makes the slot lookup
// for its storage check; a re-upload writes the same image, so the descriptor stays. Draws bind the same textures
// thousands of times a frame (15,000 lookups a frame in the monsters fight); a hit skips the slot lookup, the view key
// with its hash probe and the sampler lookup. Storage images, which note their binding per submission, are
// not kept. Direct-mapped; texture lock held. TOTK_TEX_MEMO=0 turns it off; =check computes every hit again and stops
// on a difference.
#define TD_N 8192
static struct {
uint64_t h, tok;
uint32_t gen, epoch, sgen;
uint16_t type;
uint8_t separate, valid;
VkImage image;
VkDescriptorImageInfo out;
} TD[TD_N];
static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) {
int td_on = td_mode();
VkImage image;
if (!h || storage || !td_on) return tex_info_full(ex, r, h, storage, separate, out, &image);
uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup
typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 51];
if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->sgen == sg &&
slot_still(&m->gen, m->tok, h, gen)) {
if (m->epoch != ep) { // the slot's first binding in this submission: only its storage check is due
vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL);
m->epoch = ep;
}
*out = m->out;
out->imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT
: VK_IMAGE_LAYOUT_GENERAL;
if (td_on == 2) {
VkDescriptorImageInfo full;
tex_info_full(ex, r, h, storage, separate, &full, &image);
if (full.sampler != out->sampler || full.imageView != out->imageView || full.imageLayout != out->imageLayout)
rt_fatal("vk_exec: texture descriptor memo of handle %#llx type %#x differs from a lookup",
(unsigned long long)h, r->type);
}
return;
}
uint64_t tok = token_of(h); // before the lookup
tex_info_full(ex, r, h, storage, separate, out, &image);
*m = (typeof(*m)){.h = h, .tok = tok, .gen = gen, .epoch = ep, .sgen = sg, .type = r->type,
.separate = (uint8_t)separate, .valid = 1, .image = image, .out = *out};
}
static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate,
VkDescriptorImageInfo *out, VkImage *image) {
int ms = (r->type & 0x200) != 0;
*image = VK_NULL_HANDLE;
VkImageView view = VK_NULL_HANDLE;
VkSampler samp = VK_NULL_HANDLE;
VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
@@ -978,7 +1154,7 @@ static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage,
view_t v;
int hv;
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
if (img) view = get_view(ex, img, &v, hv, r->type, storage);
if (img) view = get_view(ex, img, &v, hv, r->type, storage), *image = img->image;
if (img && storage) vk_texture_storage_bound_locked(img);
if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT;
if (!storage && !separate) {
@@ -1499,16 +1675,27 @@ static void bind_vertex_buffers(VkExec *ex) {
int vcls[17] = {0};
void *vpool[17] = {0};
int dyn = vk_pipeline_vertex_input_dynamic();
// A stream whose range is the one its bound buffer was resolved from, with no pool or mirror change since
// (vk_mem_gen, read before any resolve), resolves to that buffer again: most draws change one stream or none.
uint64_t gen = vk_mem_gen();
int same_gen = B.vb_valid && B.vgen == gen;
for (int b = 0; b < 17; b++) {
vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0;
if (b == VK_PIPE_ZERO_BINDING) continue;
const NVNbufferRange *r = &ex->vbuf[b];
if (same_gen && B.vin[b].address == r->address && B.vin[b].size == r->size) {
vb[b] = B.vb[b], vo[b] = B.vo[b], vsz[b] = B.vsz[b], vcls[b] = B.vcls[b], vpool[b] = B.vpool[b];
continue;
}
VkMemSpan sp;
if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue;
VK_MEM_USE(sp);
uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail;
vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool;
}
memcpy(B.vcls, vcls, sizeof vcls), memcpy(B.vpool, vpool, sizeof vpool);
memcpy(B.vin, ex->vbuf, sizeof B.vin);
B.vgen = gen;
if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) &&
(dyn || !memcmp(B.vstr, vstr, sizeof vstr)))
return;
@@ -1580,6 +1767,13 @@ static int shader_memo_on(void) {
return SM.on;
}
static struct { int on, valid; VkPipeState s; const VkPipeInfo *info; } PM = {-1};
static int pipe_memo_on(void) {
if (PM.on < 0) PM.on = cfg_bool("TOTK_PIPE_MEMO");
return PM.on;
}
// Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped.
static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
if (ex->render_disabled) return 0;
@@ -1652,19 +1846,33 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
wait_ms = (uint32_t)(wait_left_ns / 1000000);
if (wait_ms) t0 = kern_now_ns();
}
uint64_t w0 = kern_now_ns();
VkPipeline p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
if (t0) {
uint64_t spent = kern_now_ns() - t0;
wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0;
// The previous draw's pipeline while its state repeats (PM, TOTK_PIPE_MEMO=0 turns it off): the cache key is made
// from the state alone, and a pipeline once found only changes from its fast link to its optimised build, read
// from the info the cache entry keeps.
VkPipeline p;
const char *why = NULL;
uint64_t w0 = 0;
if (pipe_memo_on() && PM.valid && !memcmp(&PM.s, &s, sizeof s)) {
info = PM.info, pending = 0;
p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE);
} else {
w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait
p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
if (t0) {
uint64_t spent = kern_now_ns() - t0;
wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0;
}
why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL;
if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) {
perf_count(PERF_CNT_ONESHOT_WAIT);
why = "one-shot";
if (!w0) w0 = kern_now_ns();
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
}
PM.valid = !pending && p && info;
if (PM.valid) PM.s = s, PM.info = info;
}
const char *why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL;
if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) {
perf_count(PERF_CNT_ONESHOT_WAIT);
why = "one-shot";
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
}
uint64_t waited = kern_now_ns() - w0;
uint64_t waited = w0 ? kern_now_ns() - w0 : 0;
static int wait_logs;
if (why && waited >= 2000000 && wait_logs < 200) {
char tg[512];
+14 -4
View File
@@ -467,8 +467,12 @@ int vk_mem_class_of(uint64_t addr) {
// ---------------------------------------------------------------- census
static volatile LONG64 traffic[VK_MCLS_N][VK_MUSE_N];
// Every caller is the replay (vk_exec*, under vk_sync_submit's lock), about ten adds per draw: one writer at a time, so
// a plain add, not a locked one. An add racing the census's reset may be lost.
void vk_mem_count(int cls, int use, uint64_t bytes) {
if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N) InterlockedAdd64(&traffic[cls][use], (LONG64)bytes);
if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N)
__atomic_store_n(&traffic[cls][use], __atomic_load_n(&traffic[cls][use], __ATOMIC_RELAXED) + (LONG64)bytes,
__ATOMIC_RELAXED);
}
static void flag_names(uint32_t f, char *buf, size_t n) {
@@ -724,9 +728,15 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e
}
int untracked = 0, changed = 0;
for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r];
for (uint64_t a = lo; !untracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) {
uint32_t *s = gen_slot(a, 0);
if (s && *s > since) changed = 1;
// Pages only in regions stamped after since (reg_max; REG_SHIFT is SNAP_SHIFT): an unchanged texture of 4 MiB was
// 1,024 page reads, at about 1,100 texture checks a frame.
for (uint64_t r = r0; !untracked && r <= r1 && !changed; r++) {
if (reg_max[r] <= since) continue;
uint64_t a = r << REG_SHIFT > lo ? r << REG_SHIFT : lo, b = (r + 1) << REG_SHIFT < hi ? (r + 1) << REG_SHIFT : hi;
for (; a < b && !changed; a += 1ull << GEN_PAGE_SHIFT) {
uint32_t *s = gen_slot(a, 0);
if (s && *s > since) changed = 1;
}
}
*now = gen_now; // every observation so far is in the table
__atomic_add_fetch(&n_snap_checks, 1, __ATOMIC_RELAXED);
+3 -1
View File
@@ -381,7 +381,9 @@ void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
}
}
queue[qhead + qn++] = (Pending){fence, fn, arg};
WakeAllConditionVariable(&cv_work);
// The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's,
// for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it.
if (fence <= done) WakeAllConditionVariable(&cv_work);
ReleaseSRWLockExclusive(&st_lock);
}
+48 -15
View File
@@ -59,11 +59,12 @@ typedef struct Tex {
} Tex;
// A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the
// entry (ce), valid while tex_gen is still cgen, the view and has_view handed out with it, and the submit_epoch of the
// last freshness check through the slot (cepoch). Hot fields first: a lookup that hits touches one or two cache lines.
// entry (ce), valid while the generation of its address (agen) is still cgen, the view and has_view handed out with it,
// and the submit_epoch of the last freshness check through the slot (cepoch). Hot fields first: a lookup that hits
// touches one or two cache lines.
typedef struct {
struct Tex *ce;
uint32_t cgen; // tex_gen when ce was resolved; 0: nothing cached (registered since, or never resolved)
struct Tex *ce; // NULL: nothing cached (registered since, or never resolved)
uint32_t cgen; // agen of the slot's address when ce was resolved
uint32_t cepoch;
view_t cview;
uint8_t chas_view, used, has_view;
@@ -105,10 +106,25 @@ static uint32_t tex_gen = 1;
static uint32_t slots_gen = 1;
static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise
static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line
// tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so
// a slot's resolution stays valid while its address's generation does. TotK finalizes and initializes transient
// textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws,
// which emptied every cache keyed on the global generations.
#define AGEN_N 4096
static uint32_t agen[AGEN_N];
// Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered)
// and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are
// 20 bits), unlike tslots, which moves when it grows.
static uint64_t *sinfo;
static uint32_t reg_serial;
static void tex_gen_bump(void) {
static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); }
static void tex_gen_bump(uint64_t addr) {
if (!++tex_gen) tex_gen = 1;
slots_gen++;
uint32_t *g = &agen[agen_idx(addr)];
__atomic_store_n(g, *g + 1, __ATOMIC_RELEASE);
}
static volatile long tex_ready;
@@ -415,7 +431,7 @@ static void own_remove(Tex *e) {
static void destroy_tex(Tex *e) {
own_remove(e);
tex_gen_bump();
tex_gen_bump(e->key.addr);
nvn_hm_del(&by_key, &e->key);
Tex *head = nvn_hm_get(&by_addr, &e->key.addr);
if (head == e) {
@@ -426,6 +442,7 @@ static void destroy_tex(Tex *e) {
if (p->next_at == e) { p->next_at = e->next_at; break; }
}
if (e->released) nreleased--;
vk_exec_image_gone(e->pub.image);
vk_defer_image(e->pub.image, e->alloc);
free(e);
}
@@ -781,9 +798,9 @@ static Tex *entry_of(const tex_t *t) {
e->next_at = nvn_hm_get(&by_addr, &k.addr);
nvn_hm_put(&by_addr, &k.addr, e);
nvn_hm_put(&by_key, &k, e);
tex_gen_bump();
tex_gen_bump(k.addr);
}
if (e->released) e->released = 0, nreleased--, tex_gen_bump();
if (e->released) e->released = 0, nreleased--, tex_gen_bump(k.addr);
e->t = *t;
return e;
}
@@ -845,7 +862,7 @@ void vk_texture_release(const tex_t *t) {
if (e && !e->released) {
e->released = 1;
nreleased++;
tex_gen_bump();
tex_gen_bump(k.addr);
sweep_released();
}
unlock();
@@ -876,11 +893,18 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
memset(s + ntslots, 0, sizeof *s * (size_t)(n - ntslots));
tslots = s, ntslots = n;
}
if (!sinfo) {
uint64_t *p = calloc(1 << 20, sizeof *p);
if (!p) rt_fatal("vk: out of memory for texture slot tokens");
__atomic_store_n(&sinfo, p, __ATOMIC_RELEASE);
}
TexSlot *s = &tslots[id];
s->t = *t;
s->used = 1;
s->cgen = 0;
s->ce = NULL;
slots_gen++;
if (!++reg_serial) reg_serial = 1;
__atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE);
s->has_view = v != NULL;
if (v) s->v = *v;
else memset(&s->v, 0, sizeof s->v);
@@ -931,6 +955,13 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) {
void vk_texture_lock(void) { lock(); }
void vk_texture_unlock(void) { unlock(); }
uint32_t vk_texture_slots_gen(void) { return slots_gen; }
uint64_t vk_texture_slot_token(int id) {
const uint64_t *si = __atomic_load_n(&sinfo, __ATOMIC_ACQUIRE);
if (!si || id < 0 || id >= (1 << 20)) return 0;
uint64_t v = __atomic_load_n(&si[id], __ATOMIC_ACQUIRE);
return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0;
}
uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); }
const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) {
@@ -943,12 +974,13 @@ const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) {
const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
if (id < 0 || id >= ntslots || !tslots[id].used) return NULL;
TexSlot *s = &tslots[id];
if (s->cgen != tex_gen) { // resolved again: an entry was created, dropped or released, or the slot registered
view_t v;
int hv;
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry: tex_gen is read after it
uint32_t g = agen[sinfo[id] & 0xffff];
if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the
view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may
int hv; // succeed later
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it
s->cview = v, s->chas_view = (uint8_t)hv;
s->cgen = s->ce ? tex_gen : 0; // an empty result is not kept: image creation may succeed later
s->cgen = g;
s->cepoch = 0;
}
Tex *e = s->ce;
@@ -1561,5 +1593,6 @@ void vk_texture_shutdown(void) {
samplers = (nvn_hmap){.klen = sizeof(samp_par)};
free(tslots), free(sslots);
tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0;
if (sinfo) memset(sinfo, 0, sizeof *sinfo << 20); // no slot is registered: tokens 0
unlock();
}