From 354887a2a3e970769df618597cb603d6dbe93ea8 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 14 Nov 2026 10:51:50 +0300 Subject: [PATCH] The replay's per-draw compares and copies run inline instead of calling ucrtbase memcmp and memcpy (TOTK_MEMX) runtime/memx.h: MX_EQ and MX_CPY compare and copy with 32-byte loads in the caller; a constant size unrolls, a variable one loops without a call. Used for the pipeline memo state and the recorded dynamic state (prepare_draw), the shader answers (shader_for, which also compares the stored hash first), descriptor signatures and push sets (bind_resources), c0 and support buffers, the uniform update copies, the vertex attribute and stream binds, pipe_find, the dynamic state delta, nvn_hm_get and the recorder's tail_copy. TOTK_MEMX=1 (default) takes the inline forms, 0 keeps memcmp/memcpy, check runs memcmp and logs any compare the inline form answers differently. Machine A, 4 cores, 1080p, bindinc on both sides: Lookout Landing replay 9.64 -> 9.16 ms over 4+4 runs (each ABBA block -0.40 and -0.55 ms), Kakariko 9.04 -> 8.38 ms over 2+2. Check mode on dev: Lookout Landing pan, 0 mismatches. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 6 +-- runtime/memx.h | 120 +++++++++++++++++++++++++++++++++++++++++ runtime/nvn_hmap.h | 7 ++- runtime/vk_device.c | 2 + runtime/vk_exec_draw.c | 99 +++++++++++++++++++++------------- runtime/vk_pipeline.c | 15 +++--- runtime/vk_rec.c | 3 +- 7 files changed, 201 insertions(+), 51 deletions(-) create mode 100644 runtime/memx.h diff --git a/runtime/cfg.c b/runtime/cfg.c index 96465f60..2680628d 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -51,7 +51,7 @@ static const char *const known[] = { "GAMEPLAY_WEAPON_WHEEL", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "GRASS_RANGE", "HDR", "HDR_GAME_NITS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "KSPIN_US", "LANG", "LIB_PREBUILD", - "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MAP_CLICK", "MEMORY", "MENU_LANG", "MENU_TEST", "MENU_FONT_FONTCONFIG", + "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MAP_CLICK", "MEMORY", "MEMX", "MENU_LANG", "MENU_TEST", "MENU_FONT_FONTCONFIG", "MEM_RESERVE_MIB", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_AIM_SCALE", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_ASPECT", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", @@ -106,7 +106,7 @@ static const struct { const char *key, *section; } sections[] = { {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_RELEASED_MIB", "performance"}, {"TEX_BLOCK_MIB", "performance"}, {"VMA_BLOCK_MIB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, - {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"}, + {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"MEMX", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"}, {"SYNC_AHEAD", "performance"}, {"SHADOW_DEFER", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"OPTREF_GUARD", "performance"}, {"ABILITY_GUARD", "performance"}, {"REPORT_GUARD", "performance"}, @@ -209,7 +209,7 @@ static const CfgMeta metas[] = { {"OPTREF_GUARD", "1", "0|1", RESTART}, {"ABILITY_GUARD", "1", "0|1", RESTART}, {"REPORT_GUARD", "1", "0|1", RESTART}, // render fixes {"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART}, - {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART}, + {"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"MEMX", "1", "0|1|check", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART}, {"VB_USED", "1", "0|1", RESTART}, {"DESC_TEMPLATE", "1", "0|1", RESTART}, {"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART}, diff --git a/runtime/memx.h b/runtime/memx.h new file mode 100644 index 00000000..69635d82 --- /dev/null +++ b/runtime/memx.h @@ -0,0 +1,120 @@ +// Inline equality compares and copies for the replay's per-draw paths (TOTK_MEMX=0|1|check, default 1). +// ucrtbase's memcmp walks 8 bytes a step with a call around it, and the replay compares and copies a few hundred +// bytes at a time thousands of times a frame (pipeline states, shader answers, descriptor signatures): MX_EQ and +// MX_CPY do the same work with 32-byte loads in the caller, so a constant size unrolls and a variable one loops +// without a call. MX_EQ answers only equal or not, never an order. MX_CPY is memcpy: source and destination never +// overlap. 1: the inline forms. 0: memcmp/memcpy as before. check: memcmp/memcpy, and every MX_EQ also runs the +// inline compare and logs a different answer (vk_memx_bad). The mode is read once per process (vk_memx_init); calls +// before that take the off path. +#pragma once +#include +#include +#include + +extern int vk_memx_mode; // -1 before vk_memx_init, 0 off, 1 on, 2 check +void vk_memx_init(void); +void vk_memx_bad(const char *file, int line, size_t n); + +typedef uint64_t mx_v4 __attribute__((vector_size(32), aligned(1))); +typedef uint64_t mx_v2 __attribute__((vector_size(16), aligned(1))); + +static inline __attribute__((always_inline)) mx_v4 mx_ld4(const uint8_t *p) { + mx_v4 v; + __builtin_memcpy(&v, p, 32); + return v; +} + +static inline __attribute__((always_inline)) mx_v2 mx_ld2(const uint8_t *p) { + mx_v2 v; + __builtin_memcpy(&v, p, 16); + return v; +} + +static inline __attribute__((always_inline)) uint64_t mx_ld1(const uint8_t *p) { + uint64_t v; + __builtin_memcpy(&v, p, 8); + return v; +} + +static inline __attribute__((always_inline)) uint32_t mx_ldw(const uint8_t *p) { + uint32_t v; + __builtin_memcpy(&v, p, 4); + return v; +} + +// 1 when the n bytes at a and b are equal. Sizes of 32 and up compare in 128-byte steps with one branch each, the +// last partial step as an overlapping 32-byte load from the end; smaller sizes take two overlapping loads. +static inline __attribute__((always_inline)) int mx_eq(const void *a, const void *b, size_t n) { + const uint8_t *p = a, *q = b; + if (n >= 32) { + size_t i = 0; + for (; i + 128 <= n; i += 128) { + mx_v4 d = (mx_ld4(p + i) ^ mx_ld4(q + i)) | (mx_ld4(p + i + 32) ^ mx_ld4(q + i + 32)) | + (mx_ld4(p + i + 64) ^ mx_ld4(q + i + 64)) | (mx_ld4(p + i + 96) ^ mx_ld4(q + i + 96)); + if (d[0] | d[1] | d[2] | d[3]) return 0; + } + mx_v4 d = {0, 0, 0, 0}; + for (; i + 32 <= n; i += 32) d |= mx_ld4(p + i) ^ mx_ld4(q + i); + if (i < n) d |= mx_ld4(p + n - 32) ^ mx_ld4(q + n - 32); + return !(d[0] | d[1] | d[2] | d[3]); + } + if (n >= 16) { + mx_v2 d = (mx_ld2(p) ^ mx_ld2(q)) | (mx_ld2(p + n - 16) ^ mx_ld2(q + n - 16)); + return !(d[0] | d[1]); + } + if (n >= 8) return !((mx_ld1(p) ^ mx_ld1(q)) | (mx_ld1(p + n - 8) ^ mx_ld1(q + n - 8))); + if (n >= 4) return !((mx_ldw(p) ^ mx_ldw(q)) | (mx_ldw(p + n - 4) ^ mx_ldw(q + n - 4))); + for (size_t i = 0; i < n; i++) + if (p[i] != q[i]) return 0; + return 1; +} + +// memcpy of n bytes between buffers that do not overlap: 32-byte moves, the last partial one overlapping the one +// before it (it rewrites bytes it already wrote with the same values). +static inline __attribute__((always_inline)) void mx_cpy(void *d, const void *s, size_t n) { + uint8_t *p = d; + const uint8_t *q = s; + if (n >= 32) { + mx_v4 last = mx_ld4(q + n - 32); + for (size_t i = 0; i + 32 <= n; i += 32) { + __asm__("" : "+r"(p)); // opaque to the loop-idiom pass, which would turn the loop back into a memcpy call + mx_v4 v = mx_ld4(q + i); + __builtin_memcpy(p + i, &v, 32); + } + __builtin_memcpy(p + n - 32, &last, 32); + return; + } + if (n >= 16) { + mx_v2 x = mx_ld2(q), y = mx_ld2(q + n - 16); + __builtin_memcpy(p, &x, 16), __builtin_memcpy(p + n - 16, &y, 16); + return; + } + if (n >= 8) { + uint64_t x = mx_ld1(q), y = mx_ld1(q + n - 8); + __builtin_memcpy(p, &x, 8), __builtin_memcpy(p + n - 8, &y, 8); + return; + } + if (n >= 4) { + uint32_t x = mx_ldw(q), y = mx_ldw(q + n - 4); + __builtin_memcpy(p, &x, 4), __builtin_memcpy(p + n - 4, &y, 4); + return; + } + for (size_t i = 0; i < n; i++) p[i] = q[i]; +} + +static inline __attribute__((always_inline)) int mx_eq_at(const void *a, const void *b, size_t n, const char *file, + int line) { + if (vk_memx_mode == 1) return mx_eq(a, b, n); + int r = !memcmp(a, b, n); + if (vk_memx_mode == 2 && mx_eq(a, b, n) != r) vk_memx_bad(file, line, n); + return r; +} + +static inline __attribute__((always_inline)) void mx_cpy_sw(void *d, const void *s, size_t n) { + if (vk_memx_mode == 1) mx_cpy(d, s, n); + else memcpy(d, s, n); +} + +#define MX_EQ(a, b, n) mx_eq_at((a), (b), (n), __FILE__, __LINE__) +#define MX_NE(a, b, n) (!MX_EQ(a, b, n)) +#define MX_CPY(d, s, n) mx_cpy_sw((d), (s), (n)) diff --git a/runtime/nvn_hmap.h b/runtime/nvn_hmap.h index 73bf7859..54387971 100644 --- a/runtime/nvn_hmap.h +++ b/runtime/nvn_hmap.h @@ -1,5 +1,6 @@ // Ported from MacNativeGames games/totk/third_party/d2r_gfx/gfx/nvn/nvn_hmap.h @ b75be8cf (MIT, Copyright (c) 2026 -// jhaizhou; origin d2r-mobile); Chinese duplicate comments dropped, code unchanged. +// jhaizhou; origin d2r-mobile); Chinese duplicate comments dropped, code unchanged but for +// nvn_hm_get's key compare (MX_EQ, memx.h). // // Open-addressing hash table from fixed-length byte keys -> pointers (linear probing + backward shift on deletion, no // tombstones). The replay hot path uses it instead of a boxed dictionary: keys are not boxed, no allocation, a lookup @@ -11,6 +12,8 @@ #include #include +#include "memx.h" + typedef struct { uint8_t *keys; void **vals; uint32_t klen, cap, n; } nvn_hmap; // vals[i] == NULL means an empty slot static inline uint64_t nvn_hm_hash(const void *k, uint32_t len) { @@ -25,7 +28,7 @@ static inline void *nvn_hm_get(const nvn_hmap *m, const void *k) { if (!m->n) return NULL; for (uint32_t i = (uint32_t)nvn_hm_hash(k, m->klen) & (m->cap - 1);; i = (i + 1) & (m->cap - 1)) { if (!m->vals[i]) return NULL; - if (!memcmp(m->keys + (size_t)i * m->klen, k, m->klen)) return m->vals[i]; + if (MX_EQ(m->keys + (size_t)i * m->klen, k, m->klen)) return m->vals[i]; } } static inline void *nvn_hm_put(nvn_hmap *m, const void *k, void *v); diff --git a/runtime/vk_device.c b/runtime/vk_device.c index c7f85a17..ccc16b7d 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -8,6 +8,7 @@ #include "cfg.h" #include "host.h" +#include "memx.h" #include "vk.h" #include "platform/platform.h" #include "vk_dlaa.h" @@ -497,6 +498,7 @@ long vk_validation_errors_total(void) { return g_errors_total + (g_ready ? g_vk. int vk_init(void) { plat_mutex_lock(&g_lock); + vk_memx_init(); if (!g_ready) { memset(&g_vk, 0, sizeof g_vk); if (create_instance() && pick_device() && create_device()) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index cfd23688..a8b88d03 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -30,6 +30,7 @@ #include "vk_exec.h" #include "platform/platform.h" #include "vk_dlaa.h" +#include "memx.h" #define MAX_BINDINGS 128 #define MAX_DESC 1024 @@ -206,7 +207,7 @@ typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; R // the 2048-slot direct-mapped table this replaced missed 3% of them on conflicts. #define SHADER_SETS 2048 #define SHADER_WAYS 4 -typedef struct { uint64_t serial; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt; +typedef struct { uint64_t serial, h; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt; // h: hash of a // Which shader of a program serves each NVN stage, and its code length, per program serial (new at every // nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs @@ -976,12 +977,13 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin answers_for(ex, st, ctl, slot_mask_of(D, serial, st), &a); a.mvprev = (uint8_t)(sf_mvprev && st == NVN_STAGE_VS); ShaderEnt *set = NULL; + uint64_t h = 0; if (serial) { - uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded + h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded set = D->shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)]; for (int w = 0; w < SHADER_WAYS && set[w].used; w++) { ShaderEnt *e = &set[w]; - if (e->serial != serial || e->stage != st || memcmp(&e->a, &a, sizeof a)) continue; + if (e->serial != serial || e->stage != st || (vk_memx_mode == 1 && e->h != h) || MX_NE(&e->a, &a, sizeof a)) continue; if (w) { // to the front ShaderEnt hit = *e; memmove(&set[1], &set[0], w * sizeof *set); @@ -1002,7 +1004,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin (unsigned long long)serial); else if (set) { // in front, the least recently used way drops out memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set); - set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s}; + set[0] = (ShaderEnt){.serial = serial, .h = h, .stage = st, .used = 1, .a = a, .s = s}; } return s; } @@ -1117,14 +1119,14 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { memcpy(in.ssbo, ex->ssbo[st], sizeof in.ssbo); if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, D->c0_dim, sizeof in.dim); else in.binst = D->c0_binst; - if (D->B.c0_valid[k] && !memcmp(&in, &D->B.c0_in[k], sizeof in)) + if (D->B.c0_valid[k] && MX_EQ(&in, &D->B.c0_in[k], sizeof in)) return (VkDescriptorBufferInfo){D->B.c0_buf[k], D->B.c0_off[k], VK_C0_BYTES}; D->B.c0_in[k] = in; uint8_t tmp[VK_C0_BYTES]; vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], D->c0_dim, D->c0_binst); - if (!D->B.c0_valid[k] || memcmp(tmp, D->B.c0[k], VK_C0_BYTES)) { - memcpy(scratch(ex, VK_C0_BYTES, &D->B.c0_buf[k], &D->B.c0_off[k]), tmp, VK_C0_BYTES); - memcpy(D->B.c0[k], tmp, VK_C0_BYTES); + if (!D->B.c0_valid[k] || MX_NE(tmp, D->B.c0[k], VK_C0_BYTES)) { + MX_CPY(scratch(ex, VK_C0_BYTES, &D->B.c0_buf[k], &D->B.c0_off[k]), tmp, VK_C0_BYTES); + MX_CPY(D->B.c0[k], tmp, VK_C0_BYTES); D->B.c0_valid[k] = 1; } return (VkDescriptorBufferInfo){D->B.c0_buf[k], D->B.c0_off[k], VK_C0_BYTES}; @@ -1229,7 +1231,7 @@ static int upd_read(const VkExec *ex, uint64_t a, void *out, size_t n) { const VkExecUpd *e = &u[i & mask]; if (a >= e->hi || a + n <= e->lo) continue; if (a < e->lo || a + n > e->hi) return 0; // partly covered: memory - memcpy(out, e->data + (a - e->lo), n); + MX_CPY(out, e->data + (a - e->lo), n); return 1; } return 0; @@ -1242,7 +1244,7 @@ static void upd_apply(const VkExec *ex, uint64_t a, uint8_t *out, size_t n) { for (uint32_t i = U->n > UPD_N ? U->n - UPD_N : 0; i < U->n; i++) { const VkExecUpd *e = &U->u[i % UPD_N]; uint64_t lo = e->lo > a ? e->lo : a, hi = e->hi < a + n ? e->hi : a + n; - if (lo < hi) memcpy(out + (lo - a), e->data + (lo - e->lo), hi - lo); + if (lo < hi) MX_CPY(out + (lo - a), e->data + (lo - e->lo), hi - lo); } } @@ -2135,15 +2137,15 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { if (b->kind == VK_PIPE_SUPPORT) { float in[3]; support_in(ex, in); - if (!D->B.sup_valid || memcmp(in, D->B.sup_in, sizeof in)) { // support_fill reads only these + if (!D->B.sup_valid || MX_NE(in, D->B.sup_in, sizeof in)) { // support_fill reads only these uint8_t tmp[SUPPORT_BYTES]; support_fill(tmp, in[0], in[1], in[2]); - memcpy(D->B.sup_in, in, sizeof in); - if (!D->B.sup_valid || memcmp(tmp, D->B.support, SUPPORT_BYTES)) { // a new buffer only when the contents changed + MX_CPY(D->B.sup_in, in, sizeof in); + if (!D->B.sup_valid || MX_NE(tmp, D->B.support, SUPPORT_BYTES)) { // a new buffer only when the contents changed VkBuffer sb; VkDeviceSize so; - memcpy(scratch(ex, SUPPORT_BYTES, &sb, &so), tmp, SUPPORT_BYTES); - memcpy(D->B.support, tmp, SUPPORT_BYTES); + MX_CPY(scratch(ex, SUPPORT_BYTES, &sb, &so), tmp, SUPPORT_BYTES); + MX_CPY(D->B.support, tmp, SUPPORT_BYTES); D->B.sup_buf = sb, D->B.sup_off = so, D->B.sup_valid = 1; } } @@ -2213,7 +2215,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } if (s ? info->push1 : info->push) { uint32_t n = gfx && push_skip_on() ? dbytes : 0; - if (n && D->B.ds_pushed[s] && D->DS.pn[s] == n && !memcmp(D->DS.psig[s], ds, n)) { + if (n && D->B.ds_pushed[s] && D->DS.pn[s] == n && MX_EQ(D->DS.psig[s], ds, n)) { D->BS.push_skip++; continue; } @@ -2223,7 +2225,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { D->RS.cmds++; if (gfx) { D->B.ds_bound[s] = VK_NULL_HANDLE, D->B.ds_pushed[s] = n != 0; - if (n) D->DS.pn[s] = n, memcpy(D->DS.psig[s], ds, n); + if (n) D->DS.pn[s] = n, MX_CPY(D->DS.psig[s], ds, n); } } else if (!gfx || s > 1) { VkDescriptorSet set = alloc_set(ex, s ? info->set_layout1 : info->set_layout); @@ -2232,11 +2234,11 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } else { VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout; VkDescriptorSet set = D->B.ds_last[s]; - if (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || memcmp(D->DS.sig[s], ds, dbytes)) { + if (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || MX_NE(D->DS.sig[s], ds, dbytes)) { set = alloc_set(ex, sl); write_set(info, s, set, ds, w, i0, i1); D->B.ds_last[s] = set, D->B.ds_last_layout[s] = sl, D->DS.n[s] = dbytes; - memcpy(D->DS.sig[s], ds, dbytes); + MX_CPY(D->DS.sig[s], ds, dbytes); D->BS.set_new++; D->RS.cmds++; } else { @@ -2433,15 +2435,15 @@ static void memo_key(const VkExec *ex, uint32_t gen, MemoKey *k) { k->prog = ex->prog, k->serial = ex->prog->serial, k->gen = gen, k->tex_serial = ex->tex_serial; memcpy(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]), memcpy(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]); memcpy(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]), memcpy(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]); - memcpy(&k->pipe, &ex->pipe, sizeof k->pipe); + MX_CPY(&k->pipe, &ex->pipe, sizeof k->pipe); } static int memo_match(const VkExec *ex, uint32_t gen, MemoKey *k) { - if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen || memcmp(&k->pipe, &ex->pipe, sizeof k->pipe)) + if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen || MX_NE(&k->pipe, &ex->pipe, sizeof k->pipe)) return 0; if (k->tex_serial == ex->tex_serial) return 1; - if (memcmp(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]) || memcmp(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]) || - memcmp(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]) || memcmp(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1])) + if (MX_NE(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]) || MX_NE(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]) || + MX_NE(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]) || MX_NE(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1])) return 0; k->tex_serial = ex->tex_serial; return 1; @@ -2456,6 +2458,26 @@ static int shader_memo_on(void) { return on; } +// ---- inline compares and copies (memx.h, TOTK_MEMX) +int vk_memx_mode = -1; + +void vk_memx_init(void) { + if (vk_memx_mode >= 0) return; + const char *e = cfg_value("TOTK_MEMX"); + int m = !strcmp(e, "check") ? 2 : strcmp(e, "0") && strcmp(e, "off") ? 1 : 0; + rt_log("vk: inline compares and copies (TOTK_MEMX) %s", m == 2 ? "check" : m ? "on" : "off"); + __atomic_store_n(&vk_memx_mode, m, __ATOMIC_RELAXED); +} + +void vk_memx_bad(const char *file, int line, size_t n) { + static int logged; + static long total; + long t = __atomic_add_fetch(&total, 1, __ATOMIC_RELAXED); + if (__atomic_fetch_add(&logged, 1, __ATOMIC_RELAXED) < 20 || t % 100000 == 0) + rt_log("vk: TOTK_MEMX check: the inline compare of %zu bytes at %s:%d differs from memcmp (%ld so far)", n, file, + line, t); +} + static int pipe_memo_on(void) { static int on = -1; if (on < 0) on = cfg_bool("TOTK_PIPE_MEMO"); @@ -2512,7 +2534,8 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (!vs || (has_fs && !fs)) return 0; if (vk_dlaa_skip_draw(ex, fs)) return 0; // the game's FXAA off, or its scaling replaced by DLSS (vk_dlaa.c) uint32_t fbl = feedback_prepare(ex, fs); - VkPipeState s = ex->pipe; + VkPipeState s; + MX_CPY(&s, &ex->pipe, sizeof s); s.vs = vs, s.fs = fs; s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0)); s.prim = (uint8_t)prim; @@ -2542,12 +2565,12 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { const char *why = NULL; uint64_t w0 = 0; PipeCache *pc = NULL; - if (pipe_memo_on() && !(D->PM.valid && !memcmp(&D->PM.s, &s, sizeof s))) + if (pipe_memo_on() && !(D->PM.valid && MX_EQ(&D->PM.s, &s, sizeof s))) pc = &D->PC[nvn_hm_hash(&s, sizeof s) & (PC_N - 1)]; - if (pipe_memo_on() && (!pc || (pc->info && !memcmp(&pc->s, &s, sizeof s)))) { + if (pipe_memo_on() && (!pc || (pc->info && MX_EQ(&pc->s, &s, sizeof s)))) { info = pc ? pc->info : D->PM.info, pending = 0; p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); - if (pc) D->PM.valid = 1, D->PM.s = s, D->PM.info = info; + if (pc) D->PM.valid = 1, MX_CPY(&D->PM.s, &s, sizeof s), D->PM.info = info; } else { w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); @@ -2563,8 +2586,8 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS); } D->PM.valid = !pending && p && info; - if (D->PM.valid) D->PM.s = s, D->PM.info = info; - if (D->PM.valid && pc) pc->s = s, pc->info = info; + if (D->PM.valid) MX_CPY(&D->PM.s, &s, sizeof s), D->PM.info = info; + if (D->PM.valid && pc) MX_CPY(&pc->s, &s, sizeof s), pc->info = info; } uint64_t waited = w0 ? kern_now_ns() - w0 : 0; static int wait_logs; @@ -2579,7 +2602,7 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (pending) return 0; // a background worker is building it: the draw is skipped until it is ready if (!p || !info) return warn_once(W_PIPELINE, "a graphics pipeline could not be created: draws with it are skipped"), 0; bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p); - if (!D->B.st_valid || memcmp(&D->B.st, &s, sizeof s)) { + if (!D->B.st_valid || MX_NE(&D->B.st, &s, sizeof s)) { // TOTK_DYN_DELTA: only the dynamic states whose values changed since the last call in this command buffer. A // pipeline bind clears st_valid but not dyn_valid: every pipeline here declares the same dynamic states, which // the bind leaves as they were (the viewport, scissor and stencil values of record_fixed rely on that too). @@ -2596,7 +2619,7 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { __atomic_fetch_add(&DD.calls, 1, __ATOMIC_RELAXED), __atomic_fetch_add(&DD.cmds, cmds, __ATOMIC_RELAXED); if (!cmds) __atomic_fetch_add(&DD.empty, 1, __ATOMIC_RELAXED); ex->recorded += cmds != 0; - D->B.st = s, D->B.st_valid = 1; + MX_CPY(&D->B.st, &s, sizeof s), D->B.st_valid = 1; if (ex->mark && ex->fr.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; } vk_dlaa_draw(ex); @@ -3389,7 +3412,7 @@ int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType type) { VkBuffer b; VkDeviceSize o; - memcpy(scratch(ex, bytes, &b, &o), data, bytes); + MX_CPY(scratch(ex, bytes, &b, &o), data, bytes); vkCmdBindIndexBuffer(ex->cb, b, o, type); ex->recorded++; } @@ -3763,8 +3786,8 @@ static int ub_bind(VkExec *ex, uint64_t addr, uint64_t n, uint64_t align, VkDesc ex->draw->RS.volatile_ = 1; const uint8_t *base = vk_usnap_bytes(addr, n); VkMemSpan sp; - if (base) memcpy(dst, base, n); - else if (vk_mem_resolve(addr, n, &sp)) memcpy(dst, (const void *)(uintptr_t)addr, n); + if (base) MX_CPY(dst, base, n); + else if (vk_mem_resolve(addr, n, &sp)) MX_CPY(dst, (const void *)(uintptr_t)addr, n); else memset(dst, 0, n); if (ov_foreign_hits(addr, hi)) OV.c_mixed++; upd_apply(ex, addr, dst, n); @@ -3801,7 +3824,7 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { int here = ex->in_pass || ov_out_on(); if (here && OV.n < OV_N && OV.np < OV_N && !ov_foreign_hits(u->dst, u->dst + u->size)) { uint8_t *d = scratch(ex, u->size, &b, &o); - memcpy(d, u->data, u->size); + MX_CPY(d, u->data, u->size); VK_MEM_PICK(sp); ov_add(u->dst, u->dst + u->size, b, o, &sp); vk_mem_gpu_wrote(&sp, u->size); @@ -3818,7 +3841,7 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { ov_retire(u->dst, u->dst + u->size); } vk_exec_barrier(ex); // earlier draws finished with the old contents; overlaid updates' copies go first - memcpy(scratch(ex, u->size, &b, &o), u->data, u->size); + MX_CPY(scratch(ex, u->size, &b, &o), u->data, u->size); VK_MEM_PICK(sp); VkBufferCopy cp = {o, sp.offset, u->size}; vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp); @@ -3872,14 +3895,14 @@ int vk_exec_bind(VkExec *ex, uint32_t op, const void *payload) { const p_bind_vattr *b = payload; if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break; memset(ex->pipe.vattr, 0, sizeof ex->pipe.vattr); // MNG: the new list replaces the old one - memcpy(&ex->pipe.vattr[b->first], b->st, sizeof(uint32_t) * (size_t)b->n); + MX_CPY(&ex->pipe.vattr[b->first], b->st, sizeof(uint32_t) * (size_t)b->n); break; } case OP_BIND_VSTREAM: { const p_bind_vstream *b = payload; if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break; memset(ex->pipe.vstream, 0, sizeof ex->pipe.vstream); - memcpy(&ex->pipe.vstream[b->first], b->st, sizeof(vs_st) * (size_t)b->n); + MX_CPY(&ex->pipe.vstream[b->first], b->st, sizeof(vs_st) * (size_t)b->n); break; } case OP_BIND_VBUF: { diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 19e71d23..6d37cc81 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -49,6 +49,7 @@ #include "cfg.h" #include "gfx.h" #include "host.h" +#include "memx.h" #include "nvn_hmap.h" #include "platform/platform.h" #include "tex.h" @@ -1187,7 +1188,7 @@ static uint32_t vs_outs(const GfxShader *vs, const GfxShader *fs, GfxShaderIo io static Pipe *pipe_find(const void *key, size_t size, uint64_t h) { for (Pipe *p = pipes[h % PIPE_BUCKETS]; p; p = p->next) - if (p->hash == h && p->key_size == size && !memcmp(p->key, key, size)) return p; + if (p->hash == h && p->key_size == size && MX_EQ(p->key, key, size)) return p; return NULL; } @@ -3278,7 +3279,7 @@ uint32_t vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s, if (v.restart != o->restart) vkCmdSetPrimitiveRestartEnable(cb, v.restart), k++; // The vertex input is made from the vertex shader's inputs and the attribute and stream states alone // (vertex_input): a draw that keeps all three keeps it, and the bindings it reads. - if (s->vs != DC.vs || memcmp(s->vattr, DC.vattr, sizeof DC.vattr) || memcmp(s->vstream, DC.vstream, sizeof DC.vstream)) { + if (s->vs != DC.vs || MX_NE(s->vattr, DC.vattr, sizeof DC.vattr) || MX_NE(s->vstream, DC.vstream, sizeof DC.vstream)) { uint32_t streams = VK_PIPE_ALL_BINDINGS; if (s->vs) { VtxInput vi; @@ -3288,7 +3289,7 @@ uint32_t vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s, for (uint32_t i = 0; i < vi.n_bind; i++) streams |= 1u << vi.b[i].binding; } DC.vs = s->vs, DC.streams = streams; - memcpy(DC.vattr, s->vattr, sizeof DC.vattr), memcpy(DC.vstream, s->vstream, sizeof DC.vstream); + MX_CPY(DC.vattr, s->vattr, sizeof DC.vattr), MX_CPY(DC.vstream, s->vstream, sizeof DC.vstream); } if (P.dyn_clamp && v.clamp != o->clamp) rec_bool(P.set_clamp, cb, v.clamp), k++; if (P.dyn_poly && v.poly != o->poly) rec_polygon_mode(P.set_poly, cb, v.poly), k++; @@ -3297,10 +3298,10 @@ uint32_t vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s, // One with fewer leaves the ones past its count as they were, which no draw of it reads. uint32_t n = (uint32_t)v.n; int all = v.n > o->n; - if (n && P.dyn_blend_en && (all || memcmp(v.en, o->en, n * sizeof *v.en))) rec_blend_enable(P.set_blend_en, cb, 0, n, v.en), k++; - if (n && P.dyn_blend_eq && (all || memcmp(v.eq, o->eq, n * sizeof *v.eq))) rec_blend_equation(P.set_blend_eq, cb, 0, n, v.eq), k++; - if (n && P.dyn_wmask && (all || memcmp(v.wm, o->wm, n * sizeof *v.wm))) rec_write_mask(P.set_wmask, cb, 0, n, v.wm), k++; - DC.v = v; // the invariant now covers [0, v.n) only + if (n && P.dyn_blend_en && (all || MX_NE(v.en, o->en, n * sizeof *v.en))) rec_blend_enable(P.set_blend_en, cb, 0, n, v.en), k++; + if (n && P.dyn_blend_eq && (all || MX_NE(v.eq, o->eq, n * sizeof *v.eq))) rec_blend_equation(P.set_blend_eq, cb, 0, n, v.eq), k++; + if (n && P.dyn_wmask && (all || MX_NE(v.wm, o->wm, n * sizeof *v.wm))) rec_write_mask(P.set_wmask, cb, 0, n, v.wm), k++; + MX_CPY(&DC.v, &v, sizeof v); // the invariant now covers [0, v.n) only *cmds = k; return DC.streams; } diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 56dc5361..f55fdd39 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -14,6 +14,7 @@ #include "cfg.h" #include "host.h" +#include "memx.h" #include "vk.h" #include "platform/platform.h" @@ -478,7 +479,7 @@ int rec_owns(VkCommandBuffer cb) { return OWNED(cb); } static void *tail_copy(uint8_t **at, const void *arr, size_t bytes) { if (!arr || !bytes) return NULL; void *d = *at; - memcpy(d, arr, bytes); + MX_CPY(d, arr, bytes); *at += (bytes + 7) & ~(size_t)7; return d; }