mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 12:25:56 +00:00
Merge task/memx: the replay compares and copies its per-draw state inline instead of calling the C library, about 5% less render-thread time on the CPU (#381)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
b2355be341
7 files changed
+201
-51
No files matched your search
+3
-3
@@ -51,7 +51,7 @@ static const char *const known[] = {
|
||||
"GAMEPLAY_WEAPON_WHEEL",
|
||||
"GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "GRASS_RANGE", "HDR", "HDR_GAME_NITS", "HDR_HUE",
|
||||
"HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "KSPIN_US", "LANG", "LIB_PREBUILD",
|
||||
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MAP_CLICK", "MEMORY", "MENU_LANG", "MENU_TEST", "MENU_FONT_FONTCONFIG",
|
||||
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MAP_CLICK", "MEMORY", "MEMX", "MENU_LANG", "MENU_TEST", "MENU_FONT_FONTCONFIG",
|
||||
"MEM_RESERVE_MIB", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_AIM_SCALE", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_ASPECT", "MOVIE_HOST", "MOVIE_PACE",
|
||||
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
|
||||
"MV_SURVEY",
|
||||
@@ -106,7 +106,7 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"},
|
||||
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
|
||||
{"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_RELEASED_MIB", "performance"}, {"TEX_BLOCK_MIB", "performance"}, {"VMA_BLOCK_MIB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"MEMX", "performance"}, {"TEX_MEMO", "performance"}, {"VB_USED", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"SYNC_AHEAD", "performance"}, {"SHADOW_DEFER", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
|
||||
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"},
|
||||
{"OPTREF_GUARD", "performance"}, {"ABILITY_GUARD", "performance"}, {"REPORT_GUARD", "performance"},
|
||||
@@ -209,7 +209,7 @@ static const CfgMeta metas[] = {
|
||||
{"OPTREF_GUARD", "1", "0|1", RESTART}, {"ABILITY_GUARD", "1", "0|1", RESTART}, {"REPORT_GUARD", "1", "0|1", RESTART},
|
||||
// render fixes
|
||||
{"LOD_GUARD", "1", "0|1", RESTART}, {"FIX_C0", "1", "0|1", RESTART}, {"FIX_ZERO", "1", "0|1", RESTART},
|
||||
{"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART},
|
||||
{"BIND_SKIP", "1", "0|1", RESTART}, {"SHADER_MEMO", "1", "0|1", RESTART}, {"PIPE_MEMO", "1", "0|1", RESTART}, {"MEMX", "1", "0|1|check", RESTART}, {"TEX_MEMO", "1", "0|1", RESTART},
|
||||
{"VB_USED", "1", "0|1", RESTART},
|
||||
{"DESC_TEMPLATE", "1", "0|1", RESTART},
|
||||
{"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART},
|
||||
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
// Inline equality compares and copies for the replay's per-draw paths (TOTK_MEMX=0|1|check, default 1).
|
||||
// ucrtbase's memcmp walks 8 bytes a step with a call around it, and the replay compares and copies a few hundred
|
||||
// bytes at a time thousands of times a frame (pipeline states, shader answers, descriptor signatures): MX_EQ and
|
||||
// MX_CPY do the same work with 32-byte loads in the caller, so a constant size unrolls and a variable one loops
|
||||
// without a call. MX_EQ answers only equal or not, never an order. MX_CPY is memcpy: source and destination never
|
||||
// overlap. 1: the inline forms. 0: memcmp/memcpy as before. check: memcmp/memcpy, and every MX_EQ also runs the
|
||||
// inline compare and logs a different answer (vk_memx_bad). The mode is read once per process (vk_memx_init); calls
|
||||
// before that take the off path.
|
||||
#pragma once
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
extern int vk_memx_mode; // -1 before vk_memx_init, 0 off, 1 on, 2 check
|
||||
void vk_memx_init(void);
|
||||
void vk_memx_bad(const char *file, int line, size_t n);
|
||||
|
||||
typedef uint64_t mx_v4 __attribute__((vector_size(32), aligned(1)));
|
||||
typedef uint64_t mx_v2 __attribute__((vector_size(16), aligned(1)));
|
||||
|
||||
static inline __attribute__((always_inline)) mx_v4 mx_ld4(const uint8_t *p) {
|
||||
mx_v4 v;
|
||||
__builtin_memcpy(&v, p, 32);
|
||||
return v;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline)) mx_v2 mx_ld2(const uint8_t *p) {
|
||||
mx_v2 v;
|
||||
__builtin_memcpy(&v, p, 16);
|
||||
return v;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline)) uint64_t mx_ld1(const uint8_t *p) {
|
||||
uint64_t v;
|
||||
__builtin_memcpy(&v, p, 8);
|
||||
return v;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline)) uint32_t mx_ldw(const uint8_t *p) {
|
||||
uint32_t v;
|
||||
__builtin_memcpy(&v, p, 4);
|
||||
return v;
|
||||
}
|
||||
|
||||
// 1 when the n bytes at a and b are equal. Sizes of 32 and up compare in 128-byte steps with one branch each, the
|
||||
// last partial step as an overlapping 32-byte load from the end; smaller sizes take two overlapping loads.
|
||||
static inline __attribute__((always_inline)) int mx_eq(const void *a, const void *b, size_t n) {
|
||||
const uint8_t *p = a, *q = b;
|
||||
if (n >= 32) {
|
||||
size_t i = 0;
|
||||
for (; i + 128 <= n; i += 128) {
|
||||
mx_v4 d = (mx_ld4(p + i) ^ mx_ld4(q + i)) | (mx_ld4(p + i + 32) ^ mx_ld4(q + i + 32)) |
|
||||
(mx_ld4(p + i + 64) ^ mx_ld4(q + i + 64)) | (mx_ld4(p + i + 96) ^ mx_ld4(q + i + 96));
|
||||
if (d[0] | d[1] | d[2] | d[3]) return 0;
|
||||
}
|
||||
mx_v4 d = {0, 0, 0, 0};
|
||||
for (; i + 32 <= n; i += 32) d |= mx_ld4(p + i) ^ mx_ld4(q + i);
|
||||
if (i < n) d |= mx_ld4(p + n - 32) ^ mx_ld4(q + n - 32);
|
||||
return !(d[0] | d[1] | d[2] | d[3]);
|
||||
}
|
||||
if (n >= 16) {
|
||||
mx_v2 d = (mx_ld2(p) ^ mx_ld2(q)) | (mx_ld2(p + n - 16) ^ mx_ld2(q + n - 16));
|
||||
return !(d[0] | d[1]);
|
||||
}
|
||||
if (n >= 8) return !((mx_ld1(p) ^ mx_ld1(q)) | (mx_ld1(p + n - 8) ^ mx_ld1(q + n - 8)));
|
||||
if (n >= 4) return !((mx_ldw(p) ^ mx_ldw(q)) | (mx_ldw(p + n - 4) ^ mx_ldw(q + n - 4)));
|
||||
for (size_t i = 0; i < n; i++)
|
||||
if (p[i] != q[i]) return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// memcpy of n bytes between buffers that do not overlap: 32-byte moves, the last partial one overlapping the one
|
||||
// before it (it rewrites bytes it already wrote with the same values).
|
||||
static inline __attribute__((always_inline)) void mx_cpy(void *d, const void *s, size_t n) {
|
||||
uint8_t *p = d;
|
||||
const uint8_t *q = s;
|
||||
if (n >= 32) {
|
||||
mx_v4 last = mx_ld4(q + n - 32);
|
||||
for (size_t i = 0; i + 32 <= n; i += 32) {
|
||||
__asm__("" : "+r"(p)); // opaque to the loop-idiom pass, which would turn the loop back into a memcpy call
|
||||
mx_v4 v = mx_ld4(q + i);
|
||||
__builtin_memcpy(p + i, &v, 32);
|
||||
}
|
||||
__builtin_memcpy(p + n - 32, &last, 32);
|
||||
return;
|
||||
}
|
||||
if (n >= 16) {
|
||||
mx_v2 x = mx_ld2(q), y = mx_ld2(q + n - 16);
|
||||
__builtin_memcpy(p, &x, 16), __builtin_memcpy(p + n - 16, &y, 16);
|
||||
return;
|
||||
}
|
||||
if (n >= 8) {
|
||||
uint64_t x = mx_ld1(q), y = mx_ld1(q + n - 8);
|
||||
__builtin_memcpy(p, &x, 8), __builtin_memcpy(p + n - 8, &y, 8);
|
||||
return;
|
||||
}
|
||||
if (n >= 4) {
|
||||
uint32_t x = mx_ldw(q), y = mx_ldw(q + n - 4);
|
||||
__builtin_memcpy(p, &x, 4), __builtin_memcpy(p + n - 4, &y, 4);
|
||||
return;
|
||||
}
|
||||
for (size_t i = 0; i < n; i++) p[i] = q[i];
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline)) int mx_eq_at(const void *a, const void *b, size_t n, const char *file,
|
||||
int line) {
|
||||
if (vk_memx_mode == 1) return mx_eq(a, b, n);
|
||||
int r = !memcmp(a, b, n);
|
||||
if (vk_memx_mode == 2 && mx_eq(a, b, n) != r) vk_memx_bad(file, line, n);
|
||||
return r;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline)) void mx_cpy_sw(void *d, const void *s, size_t n) {
|
||||
if (vk_memx_mode == 1) mx_cpy(d, s, n);
|
||||
else memcpy(d, s, n);
|
||||
}
|
||||
|
||||
#define MX_EQ(a, b, n) mx_eq_at((a), (b), (n), __FILE__, __LINE__)
|
||||
#define MX_NE(a, b, n) (!MX_EQ(a, b, n))
|
||||
#define MX_CPY(d, s, n) mx_cpy_sw((d), (s), (n))
|
||||
+5
-2
@@ -1,5 +1,6 @@
|
||||
// Ported from MacNativeGames games/totk/third_party/d2r_gfx/gfx/nvn/nvn_hmap.h @ b75be8cf (MIT, Copyright (c) 2026
|
||||
// jhaizhou; origin d2r-mobile); Chinese duplicate comments dropped, code unchanged.
|
||||
// jhaizhou; origin d2r-mobile); Chinese duplicate comments dropped, code unchanged but for
|
||||
// nvn_hm_get's key compare (MX_EQ, memx.h).
|
||||
//
|
||||
// Open-addressing hash table from fixed-length byte keys -> pointers (linear probing + backward shift on deletion, no
|
||||
// tombstones). The replay hot path uses it instead of a boxed dictionary: keys are not boxed, no allocation, a lookup
|
||||
@@ -11,6 +12,8 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "memx.h"
|
||||
|
||||
typedef struct { uint8_t *keys; void **vals; uint32_t klen, cap, n; } nvn_hmap; // vals[i] == NULL means an empty slot
|
||||
|
||||
static inline uint64_t nvn_hm_hash(const void *k, uint32_t len) {
|
||||
@@ -25,7 +28,7 @@ static inline void *nvn_hm_get(const nvn_hmap *m, const void *k) {
|
||||
if (!m->n) return NULL;
|
||||
for (uint32_t i = (uint32_t)nvn_hm_hash(k, m->klen) & (m->cap - 1);; i = (i + 1) & (m->cap - 1)) {
|
||||
if (!m->vals[i]) return NULL;
|
||||
if (!memcmp(m->keys + (size_t)i * m->klen, k, m->klen)) return m->vals[i];
|
||||
if (MX_EQ(m->keys + (size_t)i * m->klen, k, m->klen)) return m->vals[i];
|
||||
}
|
||||
}
|
||||
static inline void *nvn_hm_put(nvn_hmap *m, const void *k, void *v);
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
#include "cfg.h"
|
||||
#include "host.h"
|
||||
#include "memx.h"
|
||||
#include "vk.h"
|
||||
#include "platform/platform.h"
|
||||
#include "vk_dlaa.h"
|
||||
@@ -497,6 +498,7 @@ long vk_validation_errors_total(void) { return g_errors_total + (g_ready ? g_vk.
|
||||
|
||||
int vk_init(void) {
|
||||
plat_mutex_lock(&g_lock);
|
||||
vk_memx_init();
|
||||
if (!g_ready) {
|
||||
memset(&g_vk, 0, sizeof g_vk);
|
||||
if (create_instance() && pick_device() && create_device())
|
||||
|
||||
+61
-38
@@ -30,6 +30,7 @@
|
||||
#include "vk_exec.h"
|
||||
#include "platform/platform.h"
|
||||
#include "vk_dlaa.h"
|
||||
#include "memx.h"
|
||||
|
||||
#define MAX_BINDINGS 128
|
||||
#define MAX_DESC 1024
|
||||
@@ -206,7 +207,7 @@ typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; R
|
||||
// the 2048-slot direct-mapped table this replaced missed 3% of them on conflicts.
|
||||
#define SHADER_SETS 2048
|
||||
#define SHADER_WAYS 4
|
||||
typedef struct { uint64_t serial; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt;
|
||||
typedef struct { uint64_t serial, h; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt; // h: hash of a
|
||||
|
||||
// Which shader of a program serves each NVN stage, and its code length, per program serial (new at every
|
||||
// nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs
|
||||
@@ -976,12 +977,13 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
answers_for(ex, st, ctl, slot_mask_of(D, serial, st), &a);
|
||||
a.mvprev = (uint8_t)(sf_mvprev && st == NVN_STAGE_VS);
|
||||
ShaderEnt *set = NULL;
|
||||
uint64_t h = 0;
|
||||
if (serial) {
|
||||
uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded
|
||||
h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded
|
||||
set = D->shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)];
|
||||
for (int w = 0; w < SHADER_WAYS && set[w].used; w++) {
|
||||
ShaderEnt *e = &set[w];
|
||||
if (e->serial != serial || e->stage != st || memcmp(&e->a, &a, sizeof a)) continue;
|
||||
if (e->serial != serial || e->stage != st || (vk_memx_mode == 1 && e->h != h) || MX_NE(&e->a, &a, sizeof a)) continue;
|
||||
if (w) { // to the front
|
||||
ShaderEnt hit = *e;
|
||||
memmove(&set[1], &set[0], w * sizeof *set);
|
||||
@@ -1002,7 +1004,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
(unsigned long long)serial);
|
||||
else if (set) { // in front, the least recently used way drops out
|
||||
memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set);
|
||||
set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s};
|
||||
set[0] = (ShaderEnt){.serial = serial, .h = h, .stage = st, .used = 1, .a = a, .s = s};
|
||||
}
|
||||
return s;
|
||||
}
|
||||
@@ -1117,14 +1119,14 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) {
|
||||
memcpy(in.ssbo, ex->ssbo[st], sizeof in.ssbo);
|
||||
if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, D->c0_dim, sizeof in.dim);
|
||||
else in.binst = D->c0_binst;
|
||||
if (D->B.c0_valid[k] && !memcmp(&in, &D->B.c0_in[k], sizeof in))
|
||||
if (D->B.c0_valid[k] && MX_EQ(&in, &D->B.c0_in[k], sizeof in))
|
||||
return (VkDescriptorBufferInfo){D->B.c0_buf[k], D->B.c0_off[k], VK_C0_BYTES};
|
||||
D->B.c0_in[k] = in;
|
||||
uint8_t tmp[VK_C0_BYTES];
|
||||
vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], D->c0_dim, D->c0_binst);
|
||||
if (!D->B.c0_valid[k] || memcmp(tmp, D->B.c0[k], VK_C0_BYTES)) {
|
||||
memcpy(scratch(ex, VK_C0_BYTES, &D->B.c0_buf[k], &D->B.c0_off[k]), tmp, VK_C0_BYTES);
|
||||
memcpy(D->B.c0[k], tmp, VK_C0_BYTES);
|
||||
if (!D->B.c0_valid[k] || MX_NE(tmp, D->B.c0[k], VK_C0_BYTES)) {
|
||||
MX_CPY(scratch(ex, VK_C0_BYTES, &D->B.c0_buf[k], &D->B.c0_off[k]), tmp, VK_C0_BYTES);
|
||||
MX_CPY(D->B.c0[k], tmp, VK_C0_BYTES);
|
||||
D->B.c0_valid[k] = 1;
|
||||
}
|
||||
return (VkDescriptorBufferInfo){D->B.c0_buf[k], D->B.c0_off[k], VK_C0_BYTES};
|
||||
@@ -1229,7 +1231,7 @@ static int upd_read(const VkExec *ex, uint64_t a, void *out, size_t n) {
|
||||
const VkExecUpd *e = &u[i & mask];
|
||||
if (a >= e->hi || a + n <= e->lo) continue;
|
||||
if (a < e->lo || a + n > e->hi) return 0; // partly covered: memory
|
||||
memcpy(out, e->data + (a - e->lo), n);
|
||||
MX_CPY(out, e->data + (a - e->lo), n);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
@@ -1242,7 +1244,7 @@ static void upd_apply(const VkExec *ex, uint64_t a, uint8_t *out, size_t n) {
|
||||
for (uint32_t i = U->n > UPD_N ? U->n - UPD_N : 0; i < U->n; i++) {
|
||||
const VkExecUpd *e = &U->u[i % UPD_N];
|
||||
uint64_t lo = e->lo > a ? e->lo : a, hi = e->hi < a + n ? e->hi : a + n;
|
||||
if (lo < hi) memcpy(out + (lo - a), e->data + (lo - e->lo), hi - lo);
|
||||
if (lo < hi) MX_CPY(out + (lo - a), e->data + (lo - e->lo), hi - lo);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2135,15 +2137,15 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
|
||||
if (b->kind == VK_PIPE_SUPPORT) {
|
||||
float in[3];
|
||||
support_in(ex, in);
|
||||
if (!D->B.sup_valid || memcmp(in, D->B.sup_in, sizeof in)) { // support_fill reads only these
|
||||
if (!D->B.sup_valid || MX_NE(in, D->B.sup_in, sizeof in)) { // support_fill reads only these
|
||||
uint8_t tmp[SUPPORT_BYTES];
|
||||
support_fill(tmp, in[0], in[1], in[2]);
|
||||
memcpy(D->B.sup_in, in, sizeof in);
|
||||
if (!D->B.sup_valid || memcmp(tmp, D->B.support, SUPPORT_BYTES)) { // a new buffer only when the contents changed
|
||||
MX_CPY(D->B.sup_in, in, sizeof in);
|
||||
if (!D->B.sup_valid || MX_NE(tmp, D->B.support, SUPPORT_BYTES)) { // a new buffer only when the contents changed
|
||||
VkBuffer sb;
|
||||
VkDeviceSize so;
|
||||
memcpy(scratch(ex, SUPPORT_BYTES, &sb, &so), tmp, SUPPORT_BYTES);
|
||||
memcpy(D->B.support, tmp, SUPPORT_BYTES);
|
||||
MX_CPY(scratch(ex, SUPPORT_BYTES, &sb, &so), tmp, SUPPORT_BYTES);
|
||||
MX_CPY(D->B.support, tmp, SUPPORT_BYTES);
|
||||
D->B.sup_buf = sb, D->B.sup_off = so, D->B.sup_valid = 1;
|
||||
}
|
||||
}
|
||||
@@ -2213,7 +2215,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
|
||||
}
|
||||
if (s ? info->push1 : info->push) {
|
||||
uint32_t n = gfx && push_skip_on() ? dbytes : 0;
|
||||
if (n && D->B.ds_pushed[s] && D->DS.pn[s] == n && !memcmp(D->DS.psig[s], ds, n)) {
|
||||
if (n && D->B.ds_pushed[s] && D->DS.pn[s] == n && MX_EQ(D->DS.psig[s], ds, n)) {
|
||||
D->BS.push_skip++;
|
||||
continue;
|
||||
}
|
||||
@@ -2223,7 +2225,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
|
||||
D->RS.cmds++;
|
||||
if (gfx) {
|
||||
D->B.ds_bound[s] = VK_NULL_HANDLE, D->B.ds_pushed[s] = n != 0;
|
||||
if (n) D->DS.pn[s] = n, memcpy(D->DS.psig[s], ds, n);
|
||||
if (n) D->DS.pn[s] = n, MX_CPY(D->DS.psig[s], ds, n);
|
||||
}
|
||||
} else if (!gfx || s > 1) {
|
||||
VkDescriptorSet set = alloc_set(ex, s ? info->set_layout1 : info->set_layout);
|
||||
@@ -2232,11 +2234,11 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
|
||||
} else {
|
||||
VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout;
|
||||
VkDescriptorSet set = D->B.ds_last[s];
|
||||
if (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || memcmp(D->DS.sig[s], ds, dbytes)) {
|
||||
if (!set || D->B.ds_last_layout[s] != sl || D->DS.n[s] != dbytes || MX_NE(D->DS.sig[s], ds, dbytes)) {
|
||||
set = alloc_set(ex, sl);
|
||||
write_set(info, s, set, ds, w, i0, i1);
|
||||
D->B.ds_last[s] = set, D->B.ds_last_layout[s] = sl, D->DS.n[s] = dbytes;
|
||||
memcpy(D->DS.sig[s], ds, dbytes);
|
||||
MX_CPY(D->DS.sig[s], ds, dbytes);
|
||||
D->BS.set_new++;
|
||||
D->RS.cmds++;
|
||||
} else {
|
||||
@@ -2433,15 +2435,15 @@ static void memo_key(const VkExec *ex, uint32_t gen, MemoKey *k) {
|
||||
k->prog = ex->prog, k->serial = ex->prog->serial, k->gen = gen, k->tex_serial = ex->tex_serial;
|
||||
memcpy(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]), memcpy(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]);
|
||||
memcpy(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]), memcpy(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]);
|
||||
memcpy(&k->pipe, &ex->pipe, sizeof k->pipe);
|
||||
MX_CPY(&k->pipe, &ex->pipe, sizeof k->pipe);
|
||||
}
|
||||
|
||||
static int memo_match(const VkExec *ex, uint32_t gen, MemoKey *k) {
|
||||
if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen || memcmp(&k->pipe, &ex->pipe, sizeof k->pipe))
|
||||
if (k->prog != ex->prog || k->serial != ex->prog->serial || k->gen != gen || MX_NE(&k->pipe, &ex->pipe, sizeof k->pipe))
|
||||
return 0;
|
||||
if (k->tex_serial == ex->tex_serial) return 1;
|
||||
if (memcmp(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]) || memcmp(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]) ||
|
||||
memcmp(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]) || memcmp(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]))
|
||||
if (MX_NE(k->tex[0], ex->tex[NVN_STAGE_VS], sizeof k->tex[0]) || MX_NE(k->tex[1], ex->tex[NVN_STAGE_FS], sizeof k->tex[1]) ||
|
||||
MX_NE(k->img[0], ex->img[NVN_STAGE_VS], sizeof k->img[0]) || MX_NE(k->img[1], ex->img[NVN_STAGE_FS], sizeof k->img[1]))
|
||||
return 0;
|
||||
k->tex_serial = ex->tex_serial;
|
||||
return 1;
|
||||
@@ -2456,6 +2458,26 @@ static int shader_memo_on(void) {
|
||||
return on;
|
||||
}
|
||||
|
||||
// ---- inline compares and copies (memx.h, TOTK_MEMX)
|
||||
int vk_memx_mode = -1;
|
||||
|
||||
void vk_memx_init(void) {
|
||||
if (vk_memx_mode >= 0) return;
|
||||
const char *e = cfg_value("TOTK_MEMX");
|
||||
int m = !strcmp(e, "check") ? 2 : strcmp(e, "0") && strcmp(e, "off") ? 1 : 0;
|
||||
rt_log("vk: inline compares and copies (TOTK_MEMX) %s", m == 2 ? "check" : m ? "on" : "off");
|
||||
__atomic_store_n(&vk_memx_mode, m, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
void vk_memx_bad(const char *file, int line, size_t n) {
|
||||
static int logged;
|
||||
static long total;
|
||||
long t = __atomic_add_fetch(&total, 1, __ATOMIC_RELAXED);
|
||||
if (__atomic_fetch_add(&logged, 1, __ATOMIC_RELAXED) < 20 || t % 100000 == 0)
|
||||
rt_log("vk: TOTK_MEMX check: the inline compare of %zu bytes at %s:%d differs from memcmp (%ld so far)", n, file,
|
||||
line, t);
|
||||
}
|
||||
|
||||
static int pipe_memo_on(void) {
|
||||
static int on = -1;
|
||||
if (on < 0) on = cfg_bool("TOTK_PIPE_MEMO");
|
||||
@@ -2512,7 +2534,8 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
|
||||
if (!vs || (has_fs && !fs)) return 0;
|
||||
if (vk_dlaa_skip_draw(ex, fs)) return 0; // the game's FXAA off, or its scaling replaced by DLSS (vk_dlaa.c)
|
||||
uint32_t fbl = feedback_prepare(ex, fs);
|
||||
VkPipeState s = ex->pipe;
|
||||
VkPipeState s;
|
||||
MX_CPY(&s, &ex->pipe, sizeof s);
|
||||
s.vs = vs, s.fs = fs;
|
||||
s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0));
|
||||
s.prim = (uint8_t)prim;
|
||||
@@ -2542,12 +2565,12 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
|
||||
const char *why = NULL;
|
||||
uint64_t w0 = 0;
|
||||
PipeCache *pc = NULL;
|
||||
if (pipe_memo_on() && !(D->PM.valid && !memcmp(&D->PM.s, &s, sizeof s)))
|
||||
if (pipe_memo_on() && !(D->PM.valid && MX_EQ(&D->PM.s, &s, sizeof s)))
|
||||
pc = &D->PC[nvn_hm_hash(&s, sizeof s) & (PC_N - 1)];
|
||||
if (pipe_memo_on() && (!pc || (pc->info && !memcmp(&pc->s, &s, sizeof s)))) {
|
||||
if (pipe_memo_on() && (!pc || (pc->info && MX_EQ(&pc->s, &s, sizeof s)))) {
|
||||
info = pc ? pc->info : D->PM.info, pending = 0;
|
||||
p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE);
|
||||
if (pc) D->PM.valid = 1, D->PM.s = s, D->PM.info = info;
|
||||
if (pc) D->PM.valid = 1, MX_CPY(&D->PM.s, &s, sizeof s), D->PM.info = info;
|
||||
} else {
|
||||
w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait
|
||||
p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
|
||||
@@ -2563,8 +2586,8 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
|
||||
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
|
||||
}
|
||||
D->PM.valid = !pending && p && info;
|
||||
if (D->PM.valid) D->PM.s = s, D->PM.info = info;
|
||||
if (D->PM.valid && pc) pc->s = s, pc->info = info;
|
||||
if (D->PM.valid) MX_CPY(&D->PM.s, &s, sizeof s), D->PM.info = info;
|
||||
if (D->PM.valid && pc) MX_CPY(&pc->s, &s, sizeof s), pc->info = info;
|
||||
}
|
||||
uint64_t waited = w0 ? kern_now_ns() - w0 : 0;
|
||||
static int wait_logs;
|
||||
@@ -2579,7 +2602,7 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
|
||||
if (pending) return 0; // a background worker is building it: the draw is skipped until it is ready
|
||||
if (!p || !info) return warn_once(W_PIPELINE, "a graphics pipeline could not be created: draws with it are skipped"), 0;
|
||||
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p);
|
||||
if (!D->B.st_valid || memcmp(&D->B.st, &s, sizeof s)) {
|
||||
if (!D->B.st_valid || MX_NE(&D->B.st, &s, sizeof s)) {
|
||||
// TOTK_DYN_DELTA: only the dynamic states whose values changed since the last call in this command buffer. A
|
||||
// pipeline bind clears st_valid but not dyn_valid: every pipeline here declares the same dynamic states, which
|
||||
// the bind leaves as they were (the viewport, scissor and stencil values of record_fixed rely on that too).
|
||||
@@ -2596,7 +2619,7 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
|
||||
__atomic_fetch_add(&DD.calls, 1, __ATOMIC_RELAXED), __atomic_fetch_add(&DD.cmds, cmds, __ATOMIC_RELAXED);
|
||||
if (!cmds) __atomic_fetch_add(&DD.empty, 1, __ATOMIC_RELAXED);
|
||||
ex->recorded += cmds != 0;
|
||||
D->B.st = s, D->B.st_valid = 1;
|
||||
MX_CPY(&D->B.st, &s, sizeof s), D->B.st_valid = 1;
|
||||
if (ex->mark && ex->fr.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs;
|
||||
}
|
||||
vk_dlaa_draw(ex);
|
||||
@@ -3389,7 +3412,7 @@ int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth
|
||||
static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType type) {
|
||||
VkBuffer b;
|
||||
VkDeviceSize o;
|
||||
memcpy(scratch(ex, bytes, &b, &o), data, bytes);
|
||||
MX_CPY(scratch(ex, bytes, &b, &o), data, bytes);
|
||||
vkCmdBindIndexBuffer(ex->cb, b, o, type);
|
||||
ex->recorded++;
|
||||
}
|
||||
@@ -3763,8 +3786,8 @@ static int ub_bind(VkExec *ex, uint64_t addr, uint64_t n, uint64_t align, VkDesc
|
||||
ex->draw->RS.volatile_ = 1;
|
||||
const uint8_t *base = vk_usnap_bytes(addr, n);
|
||||
VkMemSpan sp;
|
||||
if (base) memcpy(dst, base, n);
|
||||
else if (vk_mem_resolve(addr, n, &sp)) memcpy(dst, (const void *)(uintptr_t)addr, n);
|
||||
if (base) MX_CPY(dst, base, n);
|
||||
else if (vk_mem_resolve(addr, n, &sp)) MX_CPY(dst, (const void *)(uintptr_t)addr, n);
|
||||
else memset(dst, 0, n);
|
||||
if (ov_foreign_hits(addr, hi)) OV.c_mixed++;
|
||||
upd_apply(ex, addr, dst, n);
|
||||
@@ -3801,7 +3824,7 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) {
|
||||
int here = ex->in_pass || ov_out_on();
|
||||
if (here && OV.n < OV_N && OV.np < OV_N && !ov_foreign_hits(u->dst, u->dst + u->size)) {
|
||||
uint8_t *d = scratch(ex, u->size, &b, &o);
|
||||
memcpy(d, u->data, u->size);
|
||||
MX_CPY(d, u->data, u->size);
|
||||
VK_MEM_PICK(sp);
|
||||
ov_add(u->dst, u->dst + u->size, b, o, &sp);
|
||||
vk_mem_gpu_wrote(&sp, u->size);
|
||||
@@ -3818,7 +3841,7 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) {
|
||||
ov_retire(u->dst, u->dst + u->size);
|
||||
}
|
||||
vk_exec_barrier(ex); // earlier draws finished with the old contents; overlaid updates' copies go first
|
||||
memcpy(scratch(ex, u->size, &b, &o), u->data, u->size);
|
||||
MX_CPY(scratch(ex, u->size, &b, &o), u->data, u->size);
|
||||
VK_MEM_PICK(sp);
|
||||
VkBufferCopy cp = {o, sp.offset, u->size};
|
||||
vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp);
|
||||
@@ -3872,14 +3895,14 @@ int vk_exec_bind(VkExec *ex, uint32_t op, const void *payload) {
|
||||
const p_bind_vattr *b = payload;
|
||||
if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break;
|
||||
memset(ex->pipe.vattr, 0, sizeof ex->pipe.vattr); // MNG: the new list replaces the old one
|
||||
memcpy(&ex->pipe.vattr[b->first], b->st, sizeof(uint32_t) * (size_t)b->n);
|
||||
MX_CPY(&ex->pipe.vattr[b->first], b->st, sizeof(uint32_t) * (size_t)b->n);
|
||||
break;
|
||||
}
|
||||
case OP_BIND_VSTREAM: {
|
||||
const p_bind_vstream *b = payload;
|
||||
if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break;
|
||||
memset(ex->pipe.vstream, 0, sizeof ex->pipe.vstream);
|
||||
memcpy(&ex->pipe.vstream[b->first], b->st, sizeof(vs_st) * (size_t)b->n);
|
||||
MX_CPY(&ex->pipe.vstream[b->first], b->st, sizeof(vs_st) * (size_t)b->n);
|
||||
break;
|
||||
}
|
||||
case OP_BIND_VBUF: {
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
#include "cfg.h"
|
||||
#include "gfx.h"
|
||||
#include "host.h"
|
||||
#include "memx.h"
|
||||
#include "nvn_hmap.h"
|
||||
#include "platform/platform.h"
|
||||
#include "tex.h"
|
||||
@@ -1187,7 +1188,7 @@ static uint32_t vs_outs(const GfxShader *vs, const GfxShader *fs, GfxShaderIo io
|
||||
|
||||
static Pipe *pipe_find(const void *key, size_t size, uint64_t h) {
|
||||
for (Pipe *p = pipes[h % PIPE_BUCKETS]; p; p = p->next)
|
||||
if (p->hash == h && p->key_size == size && !memcmp(p->key, key, size)) return p;
|
||||
if (p->hash == h && p->key_size == size && MX_EQ(p->key, key, size)) return p;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -3278,7 +3279,7 @@ uint32_t vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s,
|
||||
if (v.restart != o->restart) vkCmdSetPrimitiveRestartEnable(cb, v.restart), k++;
|
||||
// The vertex input is made from the vertex shader's inputs and the attribute and stream states alone
|
||||
// (vertex_input): a draw that keeps all three keeps it, and the bindings it reads.
|
||||
if (s->vs != DC.vs || memcmp(s->vattr, DC.vattr, sizeof DC.vattr) || memcmp(s->vstream, DC.vstream, sizeof DC.vstream)) {
|
||||
if (s->vs != DC.vs || MX_NE(s->vattr, DC.vattr, sizeof DC.vattr) || MX_NE(s->vstream, DC.vstream, sizeof DC.vstream)) {
|
||||
uint32_t streams = VK_PIPE_ALL_BINDINGS;
|
||||
if (s->vs) {
|
||||
VtxInput vi;
|
||||
@@ -3288,7 +3289,7 @@ uint32_t vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s,
|
||||
for (uint32_t i = 0; i < vi.n_bind; i++) streams |= 1u << vi.b[i].binding;
|
||||
}
|
||||
DC.vs = s->vs, DC.streams = streams;
|
||||
memcpy(DC.vattr, s->vattr, sizeof DC.vattr), memcpy(DC.vstream, s->vstream, sizeof DC.vstream);
|
||||
MX_CPY(DC.vattr, s->vattr, sizeof DC.vattr), MX_CPY(DC.vstream, s->vstream, sizeof DC.vstream);
|
||||
}
|
||||
if (P.dyn_clamp && v.clamp != o->clamp) rec_bool(P.set_clamp, cb, v.clamp), k++;
|
||||
if (P.dyn_poly && v.poly != o->poly) rec_polygon_mode(P.set_poly, cb, v.poly), k++;
|
||||
@@ -3297,10 +3298,10 @@ uint32_t vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s,
|
||||
// One with fewer leaves the ones past its count as they were, which no draw of it reads.
|
||||
uint32_t n = (uint32_t)v.n;
|
||||
int all = v.n > o->n;
|
||||
if (n && P.dyn_blend_en && (all || memcmp(v.en, o->en, n * sizeof *v.en))) rec_blend_enable(P.set_blend_en, cb, 0, n, v.en), k++;
|
||||
if (n && P.dyn_blend_eq && (all || memcmp(v.eq, o->eq, n * sizeof *v.eq))) rec_blend_equation(P.set_blend_eq, cb, 0, n, v.eq), k++;
|
||||
if (n && P.dyn_wmask && (all || memcmp(v.wm, o->wm, n * sizeof *v.wm))) rec_write_mask(P.set_wmask, cb, 0, n, v.wm), k++;
|
||||
DC.v = v; // the invariant now covers [0, v.n) only
|
||||
if (n && P.dyn_blend_en && (all || MX_NE(v.en, o->en, n * sizeof *v.en))) rec_blend_enable(P.set_blend_en, cb, 0, n, v.en), k++;
|
||||
if (n && P.dyn_blend_eq && (all || MX_NE(v.eq, o->eq, n * sizeof *v.eq))) rec_blend_equation(P.set_blend_eq, cb, 0, n, v.eq), k++;
|
||||
if (n && P.dyn_wmask && (all || MX_NE(v.wm, o->wm, n * sizeof *v.wm))) rec_write_mask(P.set_wmask, cb, 0, n, v.wm), k++;
|
||||
MX_CPY(&DC.v, &v, sizeof v); // the invariant now covers [0, v.n) only
|
||||
*cmds = k;
|
||||
return DC.streams;
|
||||
}
|
||||
|
||||
+2
-1
@@ -14,6 +14,7 @@
|
||||
|
||||
#include "cfg.h"
|
||||
#include "host.h"
|
||||
#include "memx.h"
|
||||
#include "vk.h"
|
||||
#include "platform/platform.h"
|
||||
|
||||
@@ -478,7 +479,7 @@ int rec_owns(VkCommandBuffer cb) { return OWNED(cb); }
|
||||
static void *tail_copy(uint8_t **at, const void *arr, size_t bytes) {
|
||||
if (!arr || !bytes) return NULL;
|
||||
void *d = *at;
|
||||
memcpy(d, arr, bytes);
|
||||
MX_CPY(d, arr, bytes);
|
||||
*at += (bytes + 7) & ~(size_t)7;
|
||||
return d;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user