mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 04:35:57 +00:00
Merge the replay CPU work: texture lookups and descriptors kept per slot, image views per image, texture types only for the slots a shader reads, the previous draw's pipeline reused, vertex buffer spans, plain-add traffic census, completion wakes only when due, storage checks per stamped 2 MiB region and shader stages per program serial; fight replay 13.8 -> 9.2 ms a frame on A, field 9.5 -> 7.2 ms
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
ad3fa470d1
11 files changed
+383
-111
No files matched your search
+5
-4
@@ -39,13 +39,13 @@ static const char *const known[] = {
|
||||
"GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
|
||||
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MENU_LANG", "MENU_TEST",
|
||||
"MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",
|
||||
"OPERATION_MODE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
|
||||
"OPERATION_MODE", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT",
|
||||
"PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES",
|
||||
"PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "REPLAY_ASYNC",
|
||||
"REPLAY_CHECK", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
|
||||
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
|
||||
"SHADER_MEMO", "SHARED_FONTS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SWKBD_ANSWER",
|
||||
"SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TIMER_1MS", "TRACE", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
|
||||
"SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "VK_DYN_VERTEX", "VK_NO_FLOAT64",
|
||||
"VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VS_OUTS",
|
||||
"WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS",
|
||||
};
|
||||
@@ -65,7 +65,7 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"},
|
||||
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
|
||||
{"TEX_ALIAS", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"VK_VALIDATION", "performance"},
|
||||
{"VK_DYN_VERTEX", "performance"}, {"VK_SUBGROUP_PIN", "performance"},
|
||||
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
|
||||
@@ -104,7 +104,8 @@ static const CfgMeta metas[] = {
|
||||
{"TEX_ALIAS", "1", "0|1", RESTART},
|
||||
// render fixes
|
||||
{"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART},
|
||||
{"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"DESC_TEMPLATE", "1", "0|1", RESTART},
|
||||
{"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART},
|
||||
{"DESC_TEMPLATE", "1", "0|1", RESTART},
|
||||
{"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART},
|
||||
// Vulkan
|
||||
{"VK_VALIDATION", "0", "0|1", RESTART}, {"VK_DYN_VERTEX", "", "0|1", RESTART},
|
||||
|
||||
@@ -93,6 +93,7 @@ typedef struct {
|
||||
typedef struct { uint8_t type, comps, packed; } GfxShaderIo; // GFX_TYPE_*, components, 1 with a Component decoration
|
||||
int gfx_shader_stage(const GfxShader *s); // GFX_STAGE_VS, GFX_STAGE_FS or GFX_STAGE_CS
|
||||
int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res); // count of GFX_RES_* kind; array in *res
|
||||
uint32_t gfx_shader_slot_mask(const GfxShader *s); // texture/image slots whose type its translation read
|
||||
uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]); // Input Locations (VS attributes, FS varyings) as a mask
|
||||
uint32_t gfx_shader_outputs(const GfxShader *s, GfxShaderIo io[32]); // Output Locations (VS varyings, FS colour targets)
|
||||
void gfx_shader_local_size(const GfxShader *s, uint32_t local[3]); // CS: the LocalSize execution mode; 1, 1, 1 otherwise
|
||||
|
||||
@@ -1418,6 +1418,10 @@ int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res)
|
||||
return s->nres[kind];
|
||||
}
|
||||
|
||||
// The slots of the use mask: answers outside it select the same translation (mask_answers), so a caller may leave
|
||||
// their texture and image types zero. Parses the reflection: callers keep the result.
|
||||
uint32_t gfx_shader_slot_mask(const GfxShader *s) { return use_of(s).slots; }
|
||||
|
||||
uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]) {
|
||||
if (io) memcpy(io, s->in, sizeof s->in);
|
||||
return s->in_mask;
|
||||
|
||||
@@ -608,6 +608,8 @@ static void tab_advanced(float scale) {
|
||||
opt_toggle("TOTK_FIX_ZERO", UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC);
|
||||
opt_toggle("TOTK_BIND_SKIP", UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC);
|
||||
opt_toggle("TOTK_SHADER_MEMO", UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC);
|
||||
opt_toggle("TOTK_PIPE_MEMO", UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC);
|
||||
opt_toggle("TOTK_TEX_MEMO", UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC);
|
||||
opt_toggle("TOTK_DESC_TEMPLATE", UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC);
|
||||
opt_toggle("TOTK_PUSH_SKIP", UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC);
|
||||
opt_slider("TOTK_SPLIT_DRAWS", UI_MENU_SPLIT_DRAWS, 0, 1000, UI_MENU_SPLIT_DRAWS_DESC, 8, "%.0f");
|
||||
|
||||
@@ -127,6 +127,8 @@ static const char *const text_en[UI_TEXT_COUNT] = {
|
||||
[UI_MENU_BIND_SKIP] = "Skip repeated binds",
|
||||
[UI_MENU_BIND_SKIP_DESC] = "Optimisation: skips bindings that did not change.",
|
||||
[UI_MENU_SHADER_MEMO] = "Shader memo", [UI_MENU_SHADER_MEMO_DESC] = "Optimisation: remembers shader lookups.",
|
||||
[UI_MENU_PIPE_MEMO] = "Pipeline memo", [UI_MENU_PIPE_MEMO_DESC] = "Optimisation: reuses the previous draw's pipeline when its state repeats.",
|
||||
[UI_MENU_TEX_MEMO] = "Texture binding memo", [UI_MENU_TEX_MEMO_DESC] = "Optimisation: remembers texture descriptors between draws.",
|
||||
[UI_MENU_DESC_TEMPLATE] = "Descriptor templates",
|
||||
[UI_MENU_DESC_TEMPLATE_DESC] = "Optimisation: updates descriptors in bulk.",
|
||||
[UI_MENU_PUSH_SKIP] = "Skip repeated push constants",
|
||||
@@ -271,6 +273,8 @@ static const char *const text_ru[UI_TEXT_COUNT] = {
|
||||
[UI_MENU_BIND_SKIP] = "Пропуск повторных привязок",
|
||||
[UI_MENU_BIND_SKIP_DESC] = "Оптимизация: пропускает неизменившиеся привязки.",
|
||||
[UI_MENU_SHADER_MEMO] = "Память шейдеров", [UI_MENU_SHADER_MEMO_DESC] = "Оптимизация: запоминает поиск шейдеров.",
|
||||
[UI_MENU_PIPE_MEMO] = "Память конвейера", [UI_MENU_PIPE_MEMO_DESC] = "Оптимизация: повторяет конвейер прошлой отрисовки, если её состояние не изменилось.",
|
||||
[UI_MENU_TEX_MEMO] = "Память привязок текстур", [UI_MENU_TEX_MEMO_DESC] = "Оптимизация: запоминает дескрипторы текстур между отрисовками.",
|
||||
[UI_MENU_DESC_TEMPLATE] = "Шаблоны дескрипторов",
|
||||
[UI_MENU_DESC_TEMPLATE_DESC] = "Оптимизация: обновляет дескрипторы пачками.",
|
||||
[UI_MENU_PUSH_SKIP] = "Пропуск повторных push-констант",
|
||||
|
||||
@@ -84,6 +84,8 @@ typedef enum {
|
||||
UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC,
|
||||
UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC,
|
||||
UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC,
|
||||
UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC,
|
||||
UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC,
|
||||
UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC,
|
||||
UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC,
|
||||
UI_MENU_SPLIT_DRAWS, UI_MENU_SPLIT_DRAWS_DESC,
|
||||
|
||||
@@ -198,6 +198,7 @@ uint64_t vk_upload_flush(int wait);
|
||||
// Deferred destruction: freed once the upload batches recorded so far and the timeline value last noted have completed.
|
||||
void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestroyImage only
|
||||
void vk_defer_view(VkImageView view);
|
||||
void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it
|
||||
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc);
|
||||
// ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c.
|
||||
enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK };
|
||||
@@ -263,6 +264,10 @@ VkSampler vk_sampler_by_id_locked(int id);
|
||||
// on the Switch although the game binds a linear sampler (MacNativeGames 6283599). VK_NULL_HANDLE for an empty slot.
|
||||
VkSampler vk_sampler_point_by_id_locked(int id);
|
||||
uint32_t vk_texture_slots_gen(void);
|
||||
// What vk_texture_by_id(id) returns can change only when this does: the slot is registered again, or a texture entry at
|
||||
// its storage address is created, dropped, released or taken back into use. Unlocked; read it before the lookup it
|
||||
// guards (a lookup may create an entry). 0 when the slot was never registered: nothing is kept under 0.
|
||||
uint64_t vk_texture_slot_token(int id);
|
||||
uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered
|
||||
void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage
|
||||
void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission
|
||||
|
||||
+295
-87
@@ -67,7 +67,6 @@ typedef struct {
|
||||
VkImageViewType type;
|
||||
uint32_t aspect, level0, levels, layer0, layers, swz;
|
||||
} ViewKey;
|
||||
typedef struct { ViewKey k; VkImageView view; } ViewEnt;
|
||||
_Static_assert(sizeof(ViewKey) == 40, "ViewKey is hashed as bytes: no padding");
|
||||
typedef struct { uint64_t addr, size; } Range;
|
||||
typedef struct {
|
||||
@@ -75,9 +74,6 @@ typedef struct {
|
||||
int nchunks, cchunks;
|
||||
VkDescriptorPool *pools;
|
||||
int npools, cpools;
|
||||
ViewEnt *views;
|
||||
int nviews, cviews;
|
||||
nvn_hmap view_of; // ViewKey -> the VkImageView in views
|
||||
Range *written;
|
||||
int nwritten, cwritten;
|
||||
} CbRes;
|
||||
@@ -102,6 +98,10 @@ static struct {
|
||||
int st_valid;
|
||||
VkBuffer vb[17]; // vertex buffers last bound
|
||||
VkDeviceSize vo[17], vsz[17], vstr[17];
|
||||
int vcls[17]; // their pool class and pool record, for the census
|
||||
void *vpool[17];
|
||||
NVNbufferRange vin[16]; // the ranges they were resolved from, under vk_mem_gen vgen
|
||||
uint64_t vgen;
|
||||
int vb_valid;
|
||||
uint8_t support[SUPPORT_BYTES]; // last support buffer and where it lives
|
||||
float sup_in[3]; // the support_fill arguments it was filled from
|
||||
@@ -175,7 +175,6 @@ static void cb_done(void *arg) { // completion thread
|
||||
CbRes *r = arg;
|
||||
VkCtx *c = vk_ctx();
|
||||
for (int i = 0; i < r->nwritten; i++) vk_guest_written(r->written[i].addr, r->written[i].size);
|
||||
for (int i = 0; i < r->nviews; i++) vkDestroyImageView(c->device, r->views[i].view, NULL);
|
||||
for (int i = 0; i < r->npools; i++) {
|
||||
vkResetDescriptorPool(c->device, r->pools[i], 0);
|
||||
AcquireSRWLockExclusive(&spare_lock);
|
||||
@@ -191,8 +190,7 @@ static void cb_done(void *arg) { // completion thread
|
||||
ReleaseSRWLockExclusive(&spare_lock);
|
||||
if (!kept) vmaDestroyBuffer(c->vma, r->chunks[i].buf, r->chunks[i].alloc);
|
||||
}
|
||||
free(r->view_of.keys), free(r->view_of.vals);
|
||||
free(r->written), free(r->views), free(r->pools), free(r->chunks), free(r);
|
||||
free(r->written), free(r->pools), free(r->chunks), free(r);
|
||||
}
|
||||
|
||||
void vk_exec_draw_new_cb(void) { memset(&B, 0, sizeof B); }
|
||||
@@ -201,7 +199,6 @@ static CbRes *cb_res(VkExec *ex) {
|
||||
if (cur && cur_fence == ex->fence) return cur;
|
||||
cur = calloc(1, sizeof *cur);
|
||||
if (!cur) rt_fatal("vk: out of memory");
|
||||
nvn_hm_init(&cur->view_of, sizeof(ViewKey));
|
||||
cur_fence = ex->fence;
|
||||
memset(&B, 0, sizeof B);
|
||||
vk_completion_add(ex->fence, cb_done, cur);
|
||||
@@ -455,8 +452,15 @@ VkImageAspectFlags vk_exec_view_aspect(const VkTexImage *img, const view_t *v, i
|
||||
: VK_IMAGE_ASPECT_COLOR_BIT;
|
||||
}
|
||||
|
||||
// Image views of registered textures, kept for their image's lifetime: made per command buffer they were created again
|
||||
// every frame (vkCreateImageView took about 8% of the fight replay on the 7900X). Used under the texture lock, as
|
||||
// vk_exec_image_gone is (destroy_tex), which hands the views of a destroyed image to the deferred release with it.
|
||||
typedef struct ViewNode { struct ViewNode *next; ViewKey k; VkImageView view; } ViewNode;
|
||||
static nvn_hmap view_of; // ViewKey -> VkImageView
|
||||
static nvn_hmap views_by_image; // VkImage -> its first ViewNode
|
||||
|
||||
// An image view of a registered texture for a shader resource of type `rtype` (GfxShaderRes.type); VK_NULL_HANDLE when
|
||||
// the image cannot serve it (the caller binds a stand-in).
|
||||
// the image cannot serve it (the caller binds a stand-in). Texture lock held.
|
||||
static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, int hv, uint16_t rtype, int storage) {
|
||||
VkImageViewType vt = res_view_type(rtype);
|
||||
int ms = (rtype & 0x200) != 0;
|
||||
@@ -505,9 +509,9 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v,
|
||||
}
|
||||
}
|
||||
uint32_t swz = (uint32_t)sw[0] | (uint32_t)sw[1] << 8 | (uint32_t)sw[2] << 16 | (uint32_t)sw[3] << 24;
|
||||
CbRes *r = cb_res(ex);
|
||||
ViewKey k = {img->image, img->format, vt, aspect, l0, nl, y0, ny, swz};
|
||||
VkImageView found = (VkImageView)nvn_hm_get(&r->view_of, &k);
|
||||
if (!view_of.klen) nvn_hm_init(&view_of, sizeof(ViewKey)), nvn_hm_init(&views_by_image, sizeof(VkImage));
|
||||
VkImageView found = (VkImageView)nvn_hm_get(&view_of, &k);
|
||||
if (found) return found;
|
||||
VkImageViewCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = img->image, .viewType = vt,
|
||||
.format = img->format,
|
||||
@@ -518,12 +522,24 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v,
|
||||
if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c)
|
||||
VkImageView view;
|
||||
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE;
|
||||
if (r->nviews == r->cviews) r->views = grow(r->views, &r->cviews, sizeof *r->views);
|
||||
r->views[r->nviews++] = (ViewEnt){k, view};
|
||||
nvn_hm_put(&r->view_of, &k, (void *)view);
|
||||
ViewNode *n = malloc(sizeof *n);
|
||||
if (!n) rt_fatal("vk: out of memory");
|
||||
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
|
||||
nvn_hm_put(&views_by_image, &img->image, n);
|
||||
nvn_hm_put(&view_of, &k, (void *)view);
|
||||
return view;
|
||||
}
|
||||
|
||||
void vk_exec_image_gone(VkImage image) {
|
||||
if (!views_by_image.klen) return;
|
||||
for (ViewNode *n = nvn_hm_del(&views_by_image, &image), *next; n; n = next) {
|
||||
next = n->next;
|
||||
nvn_hm_del(&view_of, &n->k);
|
||||
vk_defer_view(n->view);
|
||||
free(n);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- program to shaders
|
||||
|
||||
// (program, stage, answers) -> shader, 4-way set-associative, most recently used first. A miss costs a SHA-256 of the
|
||||
@@ -534,7 +550,7 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v,
|
||||
typedef struct { uint64_t serial; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt;
|
||||
static ShaderEnt shader_tab[SHADER_SETS][SHADER_WAYS];
|
||||
|
||||
static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) {
|
||||
static int find_stage_scan(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) {
|
||||
for (uint32_t i = 0; i < p->nstage && i < 6; i++) {
|
||||
const uint8_t *c = p->sd[i].control;
|
||||
if (!c || !p->sd[i].data) continue;
|
||||
@@ -548,12 +564,79 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Texture and image types of a stage's first 32 slots, kept while every slot's resolution (vk_texture_slots_gen)
|
||||
// stays the same: only slots whose handle changed are looked up again, each lookup being a hash probe.
|
||||
static struct { int valid; uint32_t gen; uint64_t tex[32], img[32]; uint8_t tt[32], it[32]; } AC[6];
|
||||
// Which shader of a program serves each NVN stage, and its code length, per program serial (new at every
|
||||
// nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs
|
||||
// several times a draw. Direct-mapped; a collision only scans again.
|
||||
#define STAGEM_N 512
|
||||
static struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } stagem[STAGEM_N];
|
||||
|
||||
static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) {
|
||||
if (!p->serial || (unsigned)nvn_stage >= 6) return find_stage_scan(p, nvn_stage, ctl, code, len);
|
||||
typeof(stagem[0]) *m = &stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55];
|
||||
if (m->serial != p->serial) {
|
||||
m->serial = p->serial;
|
||||
memset(m->idx, -1, sizeof m->idx);
|
||||
for (uint32_t i = 0; i < p->nstage && i < 6; i++) {
|
||||
const uint8_t *c = p->sd[i].control;
|
||||
if (!c || !p->sd[i].data) continue;
|
||||
uint32_t st;
|
||||
memcpy(&st, c + 0x714, 4);
|
||||
if (st < 6 && m->idx[st] < 0) m->idx[st] = (int8_t)i, memcpy(&m->len[st], c + 0x704, 4);
|
||||
}
|
||||
}
|
||||
int i = m->idx[nvn_stage];
|
||||
if (i < 0) return 0;
|
||||
*ctl = p->sd[i].control, *code = (const uint8_t *)(uintptr_t)p->sd[i].data, *len = m->len[nvn_stage];
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it
|
||||
// was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve
|
||||
// otherwise (slot_still).
|
||||
static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32]; } AC[6];
|
||||
|
||||
// vk_texture_slot_token of the slot a texture or image handle names; an empty handle always resolves to nothing.
|
||||
static uint64_t token_of(uint64_t h) { return h ? vk_texture_slot_token((int)(h & 0xfffff)) : 1; }
|
||||
|
||||
// Whether what handle h resolved to under the global generation *egen and its slot token etok (both read before that
|
||||
// lookup) still holds at global generation gen: nothing at all changed since, or not its slot (the token, read only when
|
||||
// the global generation moved, about every 16 draws). A token hit moves *egen to gen.
|
||||
static int slot_still(uint32_t *egen, uint64_t etok, uint64_t h, uint32_t gen) {
|
||||
if (*egen == gen) return 1;
|
||||
uint64_t t = token_of(h);
|
||||
if (!t || t != etok) return 0;
|
||||
*egen = gen;
|
||||
return 1;
|
||||
}
|
||||
static int shader_memo_on(void);
|
||||
|
||||
static void answers_for(VkExec *ex, int st, const uint8_t *ctl, GfxShaderAnswers *a) {
|
||||
// The slots of (program, stage) whose types its translation reads (gfx_shader_slot_mask), learned from its first
|
||||
// shader; ~0u while unknown. The types of other slots select the same shader, so answers_for leaves them zero instead
|
||||
// of looking up all 64 handles. Direct-mapped by serial and stage; a collision only relearns.
|
||||
#define SLOTM_N 4096
|
||||
static struct { uint64_t serial; int st; uint32_t m; } slotm[SLOTM_N];
|
||||
|
||||
static uint32_t slotm_idx(uint64_t serial, int st) { return (uint32_t)((serial * 8 + (uint64_t)st) * 0x9E3779B97F4A7C15ull >> 52); }
|
||||
|
||||
static uint32_t slot_mask_of(uint64_t serial, int st) {
|
||||
const typeof(slotm[0]) *e = &slotm[slotm_idx(serial, st)];
|
||||
return serial && e->serial == serial && e->st == st ? e->m : ~0u;
|
||||
}
|
||||
|
||||
static void slot_mask_learn(uint64_t serial, int st, const GfxShader *s) {
|
||||
if (!serial || slot_mask_of(serial, st) != ~0u) return;
|
||||
slotm[slotm_idx(serial, st)] = (typeof(slotm[0])){serial, st, gfx_shader_slot_mask(s)};
|
||||
}
|
||||
|
||||
static uint8_t type_of_handle(uint64_t h) { // texture lock held
|
||||
view_t v;
|
||||
int hv;
|
||||
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
|
||||
return img ? (uint8_t)gfx_type_of(img, &v, hv) : 0;
|
||||
}
|
||||
|
||||
// The answers of stage st for the slots in mask; the types of other slots stay zero.
|
||||
static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, GfxShaderAnswers *a) {
|
||||
memset(a, 0, sizeof *a);
|
||||
if (st == NVN_STAGE_VS) vk_pipeline_answers(&ex->pipe, a, NULL);
|
||||
else if (st == NVN_STAGE_FS) vk_pipeline_answers(&ex->pipe, NULL, a);
|
||||
@@ -564,44 +647,25 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, GfxShaderAnswers
|
||||
a->local[k] = (uint16_t)(l ? l : 1);
|
||||
}
|
||||
typeof(AC[0]) *c = &AC[st];
|
||||
vk_texture_lock();
|
||||
uint32_t gen = vk_texture_slots_gen(); // before the lookups: one that creates an entry bumps it
|
||||
int all = !c->valid || c->gen != gen;
|
||||
if (all || memcmp(c->tex, ex->tex[st], sizeof c->tex) || memcmp(c->img, ex->img[st], sizeof c->img)) {
|
||||
for (int slot = 0; slot < 32; slot++) {
|
||||
view_t v;
|
||||
int hv;
|
||||
uint64_t h = ex->tex[st][slot];
|
||||
if (all || h != c->tex[slot]) {
|
||||
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
|
||||
c->tt[slot] = img ? (uint8_t)gfx_type_of(img, &v, hv) : 0;
|
||||
}
|
||||
h = ex->img[st][slot];
|
||||
if (all || h != c->img[slot]) {
|
||||
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
|
||||
c->it[slot] = img ? (uint8_t)gfx_type_of(img, &v, hv) : 0;
|
||||
}
|
||||
}
|
||||
memcpy(c->tex, ex->tex[st], sizeof c->tex);
|
||||
memcpy(c->img, ex->img[st], sizeof c->img);
|
||||
c->gen = gen, c->valid = 1;
|
||||
BS.ac_miss++;
|
||||
// The generation and tokens are read before the lookups (one that creates an entry changes them) and unlocked: a
|
||||
// registration racing with them is ordered after this draw, and a slot stored under older ones is only looked up
|
||||
// again.
|
||||
uint32_t gen = vk_texture_slots_gen();
|
||||
int locked = 0, check = shader_memo_on() == 2;
|
||||
for (uint32_t m = mask; m; m &= m - 1) {
|
||||
int slot = __builtin_ctz(m);
|
||||
uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot];
|
||||
int tmiss = c->tex[slot] != h || !c->ttok[slot] || !slot_still(&c->tgen[slot], c->ttok[slot], h, gen);
|
||||
int imiss = c->img[slot] != hi || !c->itok[slot] || !slot_still(&c->igen[slot], c->itok[slot], hi, gen);
|
||||
uint64_t tt = tmiss ? token_of(h) : 0, it = imiss ? token_of(hi) : 0;
|
||||
if ((tmiss || imiss || check) && !locked) vk_texture_lock(), locked = 1, BS.ac_miss++;
|
||||
if (tmiss) c->tt[slot] = type_of_handle(h), c->tex[slot] = h, c->ttok[slot] = tt, c->tgen[slot] = gen;
|
||||
if (imiss) c->it[slot] = type_of_handle(hi), c->img[slot] = hi, c->itok[slot] = it, c->igen[slot] = gen;
|
||||
if (check && (type_of_handle(h) != c->tt[slot] || type_of_handle(hi) != c->it[slot])) // TOTK_SHADER_MEMO=check
|
||||
rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot);
|
||||
a->tex_type[slot] = c->tt[slot], a->img_type[slot] = c->it[slot];
|
||||
}
|
||||
if (shader_memo_on() == 2) // check mode: every slot looked up again
|
||||
for (int slot = 0; slot < 32; slot++) {
|
||||
view_t v;
|
||||
int hv;
|
||||
uint64_t h = ex->tex[st][slot], hi = ex->img[st][slot];
|
||||
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
|
||||
int tt = img ? gfx_type_of(img, &v, hv) : 0;
|
||||
img = hi ? vk_texture_by_id_locked((int)(hi & 0xfffff), &v, &hv) : NULL;
|
||||
int it = img ? gfx_type_of(img, &v, hv) : 0;
|
||||
if (tt != c->tt[slot] || it != c->it[slot])
|
||||
rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot);
|
||||
}
|
||||
vk_texture_unlock();
|
||||
memcpy(a->tex_type, c->tt, sizeof c->tt);
|
||||
memcpy(a->img_type, c->it, sizeof c->it);
|
||||
if (locked) vk_texture_unlock();
|
||||
}
|
||||
|
||||
// *present: the program has this stage. Returns NULL when it has not or the translation failed. pending (graphics
|
||||
@@ -615,8 +679,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
if (!ex->prog || !find_stage(ex->prog, st, &ctl, &code, &len)) return NULL;
|
||||
*present = 1;
|
||||
GfxShaderAnswers a;
|
||||
answers_for(ex, st, ctl, &a);
|
||||
uint64_t serial = ex->prog->serial;
|
||||
answers_for(ex, st, ctl, slot_mask_of(serial, st), &a);
|
||||
ShaderEnt *set = NULL;
|
||||
if (serial) {
|
||||
uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded
|
||||
@@ -637,6 +701,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
: pending && vk_pipeline_async_on() ? gfx_shader_get_async(gs, ctl, code, len, &a, pending)
|
||||
: gfx_shader_get(gs, ctl, code, len, NULL, &a);
|
||||
if (pending && *pending) return NULL;
|
||||
if (s) slot_mask_learn(serial, st, s);
|
||||
if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st,
|
||||
(unsigned long long)serial);
|
||||
else if (set) { // in front, the least recently used way drops out
|
||||
@@ -760,8 +825,19 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) {
|
||||
return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES};
|
||||
}
|
||||
|
||||
// The buffer range limits, read once: the device's properties are a cold cache line on every binding of every draw.
|
||||
static struct { uint64_t ubo_max, ubo_align, sbo_max, sbo_align; } LIM;
|
||||
|
||||
static void limits_init(void) {
|
||||
const VkPhysicalDeviceLimits *l = &vk_ctx()->props.limits;
|
||||
LIM.ubo_align = l->minUniformBufferOffsetAlignment, LIM.sbo_max = l->maxStorageBufferRange;
|
||||
LIM.sbo_align = l->minStorageBufferOffsetAlignment;
|
||||
__atomic_store_n(&LIM.ubo_max, l->maxUniformBufferRange, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) {
|
||||
uint64_t max = vk_ctx()->props.limits.maxUniformBufferRange, align = vk_ctx()->props.limits.minUniformBufferOffsetAlignment;
|
||||
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
|
||||
uint64_t max = LIM.ubo_max, align = LIM.ubo_align;
|
||||
VkDescriptorBufferInfo bi;
|
||||
if (slot == 0 && c0_on()) return c0_info(ex, st);
|
||||
if (slot == 1 && ex->prog) { // compiler constants stored after the shader code
|
||||
@@ -845,7 +921,8 @@ static uint64_t ubo_ptr(const VkExec *ex, int st, const GfxShaderRes *r) {
|
||||
}
|
||||
|
||||
static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *r, int written) {
|
||||
uint64_t max = vk_ctx()->props.limits.maxStorageBufferRange, align = vk_ctx()->props.limits.minStorageBufferOffsetAlignment;
|
||||
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
|
||||
uint64_t max = LIM.sbo_max, align = LIM.sbo_align;
|
||||
VkDescriptorBufferInfo bi;
|
||||
int ubo, slot = ssbo_slot(r, st, &ubo);
|
||||
p_bind_buf up = {0};
|
||||
@@ -886,39 +963,85 @@ static int in_fbl(const VkExec *ex, VkImage img) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Attachment slots of the open pass (bit 8 depth) whose subresource the view of img covers.
|
||||
static uint32_t attached_as(const VkExec *ex, const VkTexImage *img, const view_t *v, int hv) {
|
||||
uint32_t l0 = 0, l1 = img->levels, a0 = 0, a1 = img->layers, m = 0;
|
||||
if (hv && v->set_levels) l0 = v->base_level, l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels);
|
||||
if (hv && v->set_layers) a0 = v->min_layer, a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers);
|
||||
// The image and the levels [l0, l1) and layers [a0, a1) a texture's view covers.
|
||||
typedef struct { VkImage image; uint32_t l0, l1, a0, a1; } SubRange;
|
||||
|
||||
static SubRange sub_of(const VkTexImage *img, const view_t *v, int hv) {
|
||||
SubRange r = {img->image, 0, img->levels, 0, img->layers};
|
||||
if (hv && v->set_levels) r.l0 = v->base_level, r.l1 = v->base_level + (v->nlevels ? v->nlevels : img->levels);
|
||||
if (hv && v->set_layers) r.a0 = v->min_layer, r.a1 = v->min_layer + (v->nlayers ? v->nlayers : img->layers);
|
||||
return r;
|
||||
}
|
||||
|
||||
// Attachment slots of the open pass (bit 8 depth) whose subresource r covers.
|
||||
static uint32_t attached_as(const VkExec *ex, const SubRange *r) {
|
||||
uint32_t m = 0;
|
||||
if (!r->image) return 0;
|
||||
for (uint32_t i = 0; i < 9; i++) {
|
||||
const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth;
|
||||
if ((i < 8 && i >= ex->pass_ncolor) || t->image != img->image) continue;
|
||||
if (t->level < l0 || t->level >= l1) continue;
|
||||
if (!t->is3d && (t->base_layer + t->layers <= a0 || t->base_layer >= a1)) continue;
|
||||
if ((i < 8 && i >= ex->pass_ncolor) || t->image != r->image) continue;
|
||||
if (t->level < r->l0 || t->level >= r->l1) continue;
|
||||
if (!t->is3d && (t->base_layer + t->layers <= r->a0 || t->base_layer >= r->a1)) continue;
|
||||
m |= 1u << i;
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
// TOTK_TEX_MEMO (tex_info, feedback_scan): 0 off, 1 on, 2 (=check) computes every hit again and stops on a difference.
|
||||
static int td_mode(void) {
|
||||
static int on = -1;
|
||||
if (on < 0) {
|
||||
const char *e = cfg_value("TOTK_TEX_MEMO");
|
||||
on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0;
|
||||
}
|
||||
return on;
|
||||
}
|
||||
|
||||
// The subresource a texture handle samples, per handle, kept while its slot resolves the same (slot_still): a draw's
|
||||
// texture bindings change the whole-scan memo below, but the handles they bind were mostly looked up before, in this
|
||||
// pass or an earlier one. Direct-mapped.
|
||||
#define FM_N 8192
|
||||
static struct { uint64_t h, tok; uint32_t gen; SubRange r; } FM[FM_N];
|
||||
|
||||
static SubRange sampled_sub(uint64_t h) { // texture lock held
|
||||
view_t v;
|
||||
int hv;
|
||||
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
|
||||
return img ? sub_of(img, &v, hv) : (SubRange){0};
|
||||
}
|
||||
|
||||
// Attachments of the open pass the fragment shader fs samples; kept while the shader, the texture bindings and the
|
||||
// pass stay the same.
|
||||
static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) {
|
||||
if (ex->fb_fs == fs && ex->fb_serial == ex->tex_serial && ex->fb_pass == ex->pass_serial) return ex->fb_sampled;
|
||||
const GfxShaderRes *res;
|
||||
int n = gfx_shader_resources(fs, GFX_RES_TEX, &res);
|
||||
uint32_t m = 0;
|
||||
vk_texture_lock();
|
||||
uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup
|
||||
int memo = td_mode(), locked = 0;
|
||||
for (int r = 0; r < n; r++)
|
||||
for (int k = 0; k < (res[r].count ? res[r].count : 1); k++) {
|
||||
int slot = res[r].slot + k;
|
||||
uint64_t h = slot >= 0 && slot < 128 ? ex->tex[NVN_STAGE_FS][slot] : 0;
|
||||
view_t v;
|
||||
int hv;
|
||||
const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL;
|
||||
if (img) m |= attached_as(ex, img, &v, hv);
|
||||
if (!h) continue;
|
||||
typeof(FM[0]) *f = &FM[(h * 0x9E3779B97F4A7C15ull) >> 51];
|
||||
if (memo && f->h == h && slot_still(&f->gen, f->tok, h, gen)) {
|
||||
if (memo == 2) {
|
||||
if (!locked) vk_texture_lock(), locked = 1;
|
||||
SubRange c = sampled_sub(h);
|
||||
if (memcmp(&c, &f->r, sizeof c))
|
||||
rt_fatal("vk_exec: feedback memo of handle %#llx differs from a lookup", (unsigned long long)h);
|
||||
}
|
||||
m |= attached_as(ex, &f->r);
|
||||
continue;
|
||||
}
|
||||
uint64_t tok = token_of(h); // before the lookup
|
||||
if (!locked) vk_texture_lock(), locked = 1;
|
||||
SubRange sr = sampled_sub(h);
|
||||
m |= attached_as(ex, &sr);
|
||||
if (tok) *f = (typeof(*f)){.h = h, .tok = tok, .gen = gen, .r = sr};
|
||||
else f->h = 0;
|
||||
}
|
||||
vk_texture_unlock();
|
||||
if (locked) vk_texture_unlock();
|
||||
ex->fb_fs = fs, ex->fb_serial = ex->tex_serial, ex->fb_pass = ex->pass_serial, ex->fb_sampled = m;
|
||||
return m;
|
||||
}
|
||||
@@ -969,8 +1092,61 @@ static void feedback_written(VkExec *ex, const VkPipeState *s) {
|
||||
if (ex->color[i].image && !(ex->fb_dirty >> i & 1) && vk_pipeline_target_wmask(s, (int)i)) ex->fb_dirty |= 1u << i;
|
||||
}
|
||||
|
||||
static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate,
|
||||
VkDescriptorImageInfo *out, VkImage *image);
|
||||
|
||||
// Sampled-texture descriptors per (handle, resource type, separate), kept while nothing they derive from changed: the
|
||||
// slot's resolution and texture entry (slot_still) and the samplers (vk_sampler_gen); the layout is set again on every
|
||||
// hit from the images in the feedback layout (in_fbl of the image kept with it). The first binding of a slot in a submission (vk_texture_epoch) still makes the slot lookup
|
||||
// for its storage check; a re-upload writes the same image, so the descriptor stays. Draws bind the same textures
|
||||
// thousands of times a frame (15,000 lookups a frame in the monsters fight); a hit skips the slot lookup, the view key
|
||||
// with its hash probe and the sampler lookup. Storage images, which note their binding per submission, are
|
||||
// not kept. Direct-mapped; texture lock held. TOTK_TEX_MEMO=0 turns it off; =check computes every hit again and stops
|
||||
// on a difference.
|
||||
#define TD_N 8192
|
||||
static struct {
|
||||
uint64_t h, tok;
|
||||
uint32_t gen, epoch, sgen;
|
||||
uint16_t type;
|
||||
uint8_t separate, valid;
|
||||
VkImage image;
|
||||
VkDescriptorImageInfo out;
|
||||
} TD[TD_N];
|
||||
|
||||
static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) {
|
||||
int td_on = td_mode();
|
||||
VkImage image;
|
||||
if (!h || storage || !td_on) return tex_info_full(ex, r, h, storage, separate, out, &image);
|
||||
uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup
|
||||
typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 51];
|
||||
if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->sgen == sg &&
|
||||
slot_still(&m->gen, m->tok, h, gen)) {
|
||||
if (m->epoch != ep) { // the slot's first binding in this submission: only its storage check is due
|
||||
vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL);
|
||||
m->epoch = ep;
|
||||
}
|
||||
*out = m->out;
|
||||
out->imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT
|
||||
: VK_IMAGE_LAYOUT_GENERAL;
|
||||
if (td_on == 2) {
|
||||
VkDescriptorImageInfo full;
|
||||
tex_info_full(ex, r, h, storage, separate, &full, &image);
|
||||
if (full.sampler != out->sampler || full.imageView != out->imageView || full.imageLayout != out->imageLayout)
|
||||
rt_fatal("vk_exec: texture descriptor memo of handle %#llx type %#x differs from a lookup",
|
||||
(unsigned long long)h, r->type);
|
||||
}
|
||||
return;
|
||||
}
|
||||
uint64_t tok = token_of(h); // before the lookup
|
||||
tex_info_full(ex, r, h, storage, separate, out, &image);
|
||||
*m = (typeof(*m)){.h = h, .tok = tok, .gen = gen, .epoch = ep, .sgen = sg, .type = r->type,
|
||||
.separate = (uint8_t)separate, .valid = 1, .image = image, .out = *out};
|
||||
}
|
||||
|
||||
static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate,
|
||||
VkDescriptorImageInfo *out, VkImage *image) {
|
||||
int ms = (r->type & 0x200) != 0;
|
||||
*image = VK_NULL_HANDLE;
|
||||
VkImageView view = VK_NULL_HANDLE;
|
||||
VkSampler samp = VK_NULL_HANDLE;
|
||||
VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
|
||||
@@ -978,7 +1154,7 @@ static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage,
|
||||
view_t v;
|
||||
int hv;
|
||||
const VkTexImage *img = vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv);
|
||||
if (img) view = get_view(ex, img, &v, hv, r->type, storage);
|
||||
if (img) view = get_view(ex, img, &v, hv, r->type, storage), *image = img->image;
|
||||
if (img && storage) vk_texture_storage_bound_locked(img);
|
||||
if (img && !storage && ex->nfbl && in_fbl(ex, img->image)) layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT;
|
||||
if (!storage && !separate) {
|
||||
@@ -1499,16 +1675,27 @@ static void bind_vertex_buffers(VkExec *ex) {
|
||||
int vcls[17] = {0};
|
||||
void *vpool[17] = {0};
|
||||
int dyn = vk_pipeline_vertex_input_dynamic();
|
||||
// A stream whose range is the one its bound buffer was resolved from, with no pool or mirror change since
|
||||
// (vk_mem_gen, read before any resolve), resolves to that buffer again: most draws change one stream or none.
|
||||
uint64_t gen = vk_mem_gen();
|
||||
int same_gen = B.vb_valid && B.vgen == gen;
|
||||
for (int b = 0; b < 17; b++) {
|
||||
vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0;
|
||||
if (b == VK_PIPE_ZERO_BINDING) continue;
|
||||
const NVNbufferRange *r = &ex->vbuf[b];
|
||||
if (same_gen && B.vin[b].address == r->address && B.vin[b].size == r->size) {
|
||||
vb[b] = B.vb[b], vo[b] = B.vo[b], vsz[b] = B.vsz[b], vcls[b] = B.vcls[b], vpool[b] = B.vpool[b];
|
||||
continue;
|
||||
}
|
||||
VkMemSpan sp;
|
||||
if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue;
|
||||
VK_MEM_USE(sp);
|
||||
uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail;
|
||||
vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool;
|
||||
}
|
||||
memcpy(B.vcls, vcls, sizeof vcls), memcpy(B.vpool, vpool, sizeof vpool);
|
||||
memcpy(B.vin, ex->vbuf, sizeof B.vin);
|
||||
B.vgen = gen;
|
||||
if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) &&
|
||||
(dyn || !memcmp(B.vstr, vstr, sizeof vstr)))
|
||||
return;
|
||||
@@ -1580,6 +1767,13 @@ static int shader_memo_on(void) {
|
||||
return SM.on;
|
||||
}
|
||||
|
||||
static struct { int on, valid; VkPipeState s; const VkPipeInfo *info; } PM = {-1};
|
||||
|
||||
static int pipe_memo_on(void) {
|
||||
if (PM.on < 0) PM.on = cfg_bool("TOTK_PIPE_MEMO");
|
||||
return PM.on;
|
||||
}
|
||||
|
||||
// Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped.
|
||||
static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
|
||||
if (ex->render_disabled) return 0;
|
||||
@@ -1652,19 +1846,33 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
|
||||
wait_ms = (uint32_t)(wait_left_ns / 1000000);
|
||||
if (wait_ms) t0 = kern_now_ns();
|
||||
}
|
||||
uint64_t w0 = kern_now_ns();
|
||||
VkPipeline p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
|
||||
if (t0) {
|
||||
uint64_t spent = kern_now_ns() - t0;
|
||||
wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0;
|
||||
// The previous draw's pipeline while its state repeats (PM, TOTK_PIPE_MEMO=0 turns it off): the cache key is made
|
||||
// from the state alone, and a pipeline once found only changes from its fast link to its optimised build, read
|
||||
// from the info the cache entry keeps.
|
||||
VkPipeline p;
|
||||
const char *why = NULL;
|
||||
uint64_t w0 = 0;
|
||||
if (pipe_memo_on() && PM.valid && !memcmp(&PM.s, &s, sizeof s)) {
|
||||
info = PM.info, pending = 0;
|
||||
p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE);
|
||||
} else {
|
||||
w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait
|
||||
p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
|
||||
if (t0) {
|
||||
uint64_t spent = kern_now_ns() - t0;
|
||||
wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0;
|
||||
}
|
||||
why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL;
|
||||
if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) {
|
||||
perf_count(PERF_CNT_ONESHOT_WAIT);
|
||||
why = "one-shot";
|
||||
if (!w0) w0 = kern_now_ns();
|
||||
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
|
||||
}
|
||||
PM.valid = !pending && p && info;
|
||||
if (PM.valid) PM.s = s, PM.info = info;
|
||||
}
|
||||
const char *why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL;
|
||||
if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) {
|
||||
perf_count(PERF_CNT_ONESHOT_WAIT);
|
||||
why = "one-shot";
|
||||
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
|
||||
}
|
||||
uint64_t waited = kern_now_ns() - w0;
|
||||
uint64_t waited = w0 ? kern_now_ns() - w0 : 0;
|
||||
static int wait_logs;
|
||||
if (why && waited >= 2000000 && wait_logs < 200) {
|
||||
char tg[512];
|
||||
|
||||
+14
-4
@@ -467,8 +467,12 @@ int vk_mem_class_of(uint64_t addr) {
|
||||
// ---------------------------------------------------------------- census
|
||||
static volatile LONG64 traffic[VK_MCLS_N][VK_MUSE_N];
|
||||
|
||||
// Every caller is the replay (vk_exec*, under vk_sync_submit's lock), about ten adds per draw: one writer at a time, so
|
||||
// a plain add, not a locked one. An add racing the census's reset may be lost.
|
||||
void vk_mem_count(int cls, int use, uint64_t bytes) {
|
||||
if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N) InterlockedAdd64(&traffic[cls][use], (LONG64)bytes);
|
||||
if ((unsigned)cls < VK_MCLS_N && (unsigned)use < VK_MUSE_N)
|
||||
__atomic_store_n(&traffic[cls][use], __atomic_load_n(&traffic[cls][use], __ATOMIC_RELAXED) + (LONG64)bytes,
|
||||
__ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static void flag_names(uint32_t f, char *buf, size_t n) {
|
||||
@@ -724,9 +728,15 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e
|
||||
}
|
||||
int untracked = 0, changed = 0;
|
||||
for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r];
|
||||
for (uint64_t a = lo; !untracked && a < hi && !changed; a += 1ull << GEN_PAGE_SHIFT) {
|
||||
uint32_t *s = gen_slot(a, 0);
|
||||
if (s && *s > since) changed = 1;
|
||||
// Pages only in regions stamped after since (reg_max; REG_SHIFT is SNAP_SHIFT): an unchanged texture of 4 MiB was
|
||||
// 1,024 page reads, at about 1,100 texture checks a frame.
|
||||
for (uint64_t r = r0; !untracked && r <= r1 && !changed; r++) {
|
||||
if (reg_max[r] <= since) continue;
|
||||
uint64_t a = r << REG_SHIFT > lo ? r << REG_SHIFT : lo, b = (r + 1) << REG_SHIFT < hi ? (r + 1) << REG_SHIFT : hi;
|
||||
for (; a < b && !changed; a += 1ull << GEN_PAGE_SHIFT) {
|
||||
uint32_t *s = gen_slot(a, 0);
|
||||
if (s && *s > since) changed = 1;
|
||||
}
|
||||
}
|
||||
*now = gen_now; // every observation so far is in the table
|
||||
__atomic_add_fetch(&n_snap_checks, 1, __ATOMIC_RELAXED);
|
||||
|
||||
+3
-1
@@ -381,7 +381,9 @@ void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
|
||||
}
|
||||
}
|
||||
queue[qhead + qn++] = (Pending){fence, fn, arg};
|
||||
WakeAllConditionVariable(&cv_work);
|
||||
// The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's,
|
||||
// for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it.
|
||||
if (fence <= done) WakeAllConditionVariable(&cv_work);
|
||||
ReleaseSRWLockExclusive(&st_lock);
|
||||
}
|
||||
|
||||
|
||||
+48
-15
@@ -59,11 +59,12 @@ typedef struct Tex {
|
||||
} Tex;
|
||||
|
||||
// A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the
|
||||
// entry (ce), valid while tex_gen is still cgen, the view and has_view handed out with it, and the submit_epoch of the
|
||||
// last freshness check through the slot (cepoch). Hot fields first: a lookup that hits touches one or two cache lines.
|
||||
// entry (ce), valid while the generation of its address (agen) is still cgen, the view and has_view handed out with it,
|
||||
// and the submit_epoch of the last freshness check through the slot (cepoch). Hot fields first: a lookup that hits
|
||||
// touches one or two cache lines.
|
||||
typedef struct {
|
||||
struct Tex *ce;
|
||||
uint32_t cgen; // tex_gen when ce was resolved; 0: nothing cached (registered since, or never resolved)
|
||||
struct Tex *ce; // NULL: nothing cached (registered since, or never resolved)
|
||||
uint32_t cgen; // agen of the slot's address when ce was resolved
|
||||
uint32_t cepoch;
|
||||
view_t cview;
|
||||
uint8_t chas_view, used, has_view;
|
||||
@@ -105,10 +106,25 @@ static uint32_t tex_gen = 1;
|
||||
static uint32_t slots_gen = 1;
|
||||
static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise
|
||||
static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line
|
||||
// tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so
|
||||
// a slot's resolution stays valid while its address's generation does. TotK finalizes and initializes transient
|
||||
// textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws,
|
||||
// which emptied every cache keyed on the global generations.
|
||||
#define AGEN_N 4096
|
||||
static uint32_t agen[AGEN_N];
|
||||
// Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered)
|
||||
// and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are
|
||||
// 20 bits), unlike tslots, which moves when it grows.
|
||||
static uint64_t *sinfo;
|
||||
static uint32_t reg_serial;
|
||||
|
||||
static void tex_gen_bump(void) {
|
||||
static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); }
|
||||
|
||||
static void tex_gen_bump(uint64_t addr) {
|
||||
if (!++tex_gen) tex_gen = 1;
|
||||
slots_gen++;
|
||||
uint32_t *g = &agen[agen_idx(addr)];
|
||||
__atomic_store_n(g, *g + 1, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
static volatile long tex_ready;
|
||||
@@ -415,7 +431,7 @@ static void own_remove(Tex *e) {
|
||||
|
||||
static void destroy_tex(Tex *e) {
|
||||
own_remove(e);
|
||||
tex_gen_bump();
|
||||
tex_gen_bump(e->key.addr);
|
||||
nvn_hm_del(&by_key, &e->key);
|
||||
Tex *head = nvn_hm_get(&by_addr, &e->key.addr);
|
||||
if (head == e) {
|
||||
@@ -426,6 +442,7 @@ static void destroy_tex(Tex *e) {
|
||||
if (p->next_at == e) { p->next_at = e->next_at; break; }
|
||||
}
|
||||
if (e->released) nreleased--;
|
||||
vk_exec_image_gone(e->pub.image);
|
||||
vk_defer_image(e->pub.image, e->alloc);
|
||||
free(e);
|
||||
}
|
||||
@@ -781,9 +798,9 @@ static Tex *entry_of(const tex_t *t) {
|
||||
e->next_at = nvn_hm_get(&by_addr, &k.addr);
|
||||
nvn_hm_put(&by_addr, &k.addr, e);
|
||||
nvn_hm_put(&by_key, &k, e);
|
||||
tex_gen_bump();
|
||||
tex_gen_bump(k.addr);
|
||||
}
|
||||
if (e->released) e->released = 0, nreleased--, tex_gen_bump();
|
||||
if (e->released) e->released = 0, nreleased--, tex_gen_bump(k.addr);
|
||||
e->t = *t;
|
||||
return e;
|
||||
}
|
||||
@@ -845,7 +862,7 @@ void vk_texture_release(const tex_t *t) {
|
||||
if (e && !e->released) {
|
||||
e->released = 1;
|
||||
nreleased++;
|
||||
tex_gen_bump();
|
||||
tex_gen_bump(k.addr);
|
||||
sweep_released();
|
||||
}
|
||||
unlock();
|
||||
@@ -876,11 +893,18 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
|
||||
memset(s + ntslots, 0, sizeof *s * (size_t)(n - ntslots));
|
||||
tslots = s, ntslots = n;
|
||||
}
|
||||
if (!sinfo) {
|
||||
uint64_t *p = calloc(1 << 20, sizeof *p);
|
||||
if (!p) rt_fatal("vk: out of memory for texture slot tokens");
|
||||
__atomic_store_n(&sinfo, p, __ATOMIC_RELEASE);
|
||||
}
|
||||
TexSlot *s = &tslots[id];
|
||||
s->t = *t;
|
||||
s->used = 1;
|
||||
s->cgen = 0;
|
||||
s->ce = NULL;
|
||||
slots_gen++;
|
||||
if (!++reg_serial) reg_serial = 1;
|
||||
__atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE);
|
||||
s->has_view = v != NULL;
|
||||
if (v) s->v = *v;
|
||||
else memset(&s->v, 0, sizeof s->v);
|
||||
@@ -931,6 +955,13 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) {
|
||||
void vk_texture_lock(void) { lock(); }
|
||||
void vk_texture_unlock(void) { unlock(); }
|
||||
uint32_t vk_texture_slots_gen(void) { return slots_gen; }
|
||||
|
||||
uint64_t vk_texture_slot_token(int id) {
|
||||
const uint64_t *si = __atomic_load_n(&sinfo, __ATOMIC_ACQUIRE);
|
||||
if (!si || id < 0 || id >= (1 << 20)) return 0;
|
||||
uint64_t v = __atomic_load_n(&si[id], __ATOMIC_ACQUIRE);
|
||||
return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0;
|
||||
}
|
||||
uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); }
|
||||
|
||||
const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) {
|
||||
@@ -943,12 +974,13 @@ const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) {
|
||||
const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
|
||||
if (id < 0 || id >= ntslots || !tslots[id].used) return NULL;
|
||||
TexSlot *s = &tslots[id];
|
||||
if (s->cgen != tex_gen) { // resolved again: an entry was created, dropped or released, or the slot registered
|
||||
view_t v;
|
||||
int hv;
|
||||
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry: tex_gen is read after it
|
||||
uint32_t g = agen[sinfo[id] & 0xffff];
|
||||
if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the
|
||||
view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may
|
||||
int hv; // succeed later
|
||||
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it
|
||||
s->cview = v, s->chas_view = (uint8_t)hv;
|
||||
s->cgen = s->ce ? tex_gen : 0; // an empty result is not kept: image creation may succeed later
|
||||
s->cgen = g;
|
||||
s->cepoch = 0;
|
||||
}
|
||||
Tex *e = s->ce;
|
||||
@@ -1561,5 +1593,6 @@ void vk_texture_shutdown(void) {
|
||||
samplers = (nvn_hmap){.klen = sizeof(samp_par)};
|
||||
free(tslots), free(sslots);
|
||||
tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0;
|
||||
if (sinfo) memset(sinfo, 0, sizeof *sinfo << 20); // no slot is registered: tokens 0
|
||||
unlock();
|
||||
}
|
||||
Reference in new issue
Block a user