mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 04:35:57 +00:00
Keep the recorder queue's head, tail and producer state on separate cache lines and publish head in batches
The recorder spins on head while it waits, and tail, put_h, head and the producer's statistics shared one cache line, so every packet moved that line between the two cores: put took 6% of the replay thread in the fort with the recorder on. head, tail and sleeping now have a line each, the producer keeps its own state and the last tail it read, and pub publishes head once 64 words wait (TOTK_REC_PUB; 0 publishes every packet). Every wait publishes everything first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df9ba65a91
commit
b578d26628
1 file changed
+59
-29
+59
-29
@@ -19,10 +19,20 @@
|
||||
|
||||
#define RING_WORDS (1u << 21) // 16 MiB: a part of a submission is ~1-2 MiB of packets in the town
|
||||
static uint64_t ring[RING_WORDS];
|
||||
static volatile uint64_t head, tail; // words published by the producer / consumed by the recorder
|
||||
static volatile int32_t sleeping; // the recorder waits for wake_ev
|
||||
// The queue state, a cache line per writer. The recorder spins on head while it waits for packets, so a line the
|
||||
// producer writes on every packet and the recorder reads or writes goes back and forth between their cores each time:
|
||||
// with tail, Q.put_h, head and the producer's statistics in one line (the linker's order of these statics), handing
|
||||
// packets over took 6% of the replay thread in the fort, more than building the descriptors.
|
||||
static struct {
|
||||
_Alignas(64) volatile uint64_t head; // words published by the producer; the recorder reads it
|
||||
_Alignas(64) volatile uint64_t tail; // words consumed by the recorder; the producer reads it when it seems full
|
||||
_Alignas(64) volatile int32_t sleeping; // the recorder waits for wake_ev; written only around a sleep
|
||||
_Alignas(64) uint64_t Q.put_h; // producer only: words written
|
||||
uint64_t pub_h; // the last head published
|
||||
uint64_t tail_seen; // tail as last read, at most the real one: the room it leaves is there
|
||||
uint64_t pub_words; // TOTK_REC_PUB: publish head once this many words wait (pub)
|
||||
} Q;
|
||||
static PlatEvent wake_ev; // auto-reset
|
||||
static uint64_t put_h; // producer: words written (published up to head)
|
||||
static VkCommandBuffer rec_cb; // producer: the part being recorded, whose vkCmd* become packets
|
||||
static VkCommandBuffer r_cb; // recorder: the command buffer its packets go into (set by a BeginCb packet)
|
||||
static int rec_live; // producer: rec_begin started the recorder for rec_cb
|
||||
@@ -33,11 +43,11 @@ static PlatThread *rec_thread;
|
||||
static int rec_prio; // the submitting thread's priority, which the recorder takes
|
||||
static int rec_chk; // TOTK_REC_CHECK (rec_check_handle)
|
||||
static uint64_t rec_delay_ns; // TOTK_REC_DELAY_US (rec_main)
|
||||
static volatile uint64_t part_h0; // put_h at rec_begin: where the packets of the part being recorded start
|
||||
static volatile uint64_t part_h0; // Q.put_h at rec_begin: where the packets of the part being recorded start
|
||||
static volatile uint64_t part_fence; // the value of the part being recorded (rec_begin)
|
||||
|
||||
// Producer statistics, logged by rec_stats_log.
|
||||
static struct { uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_;
|
||||
static struct { _Alignas(64) uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on, submits; } RS_;
|
||||
|
||||
enum {
|
||||
R_PAD, R_BeginQuery, R_BeginRendering, R_BindDescriptorSets, R_BindIndexBuffer, R_BindPipeline, R_BindVertexBuffers2,
|
||||
@@ -63,14 +73,14 @@ static void rec_main(void *arg) {
|
||||
plat_thread_priority(rec_prio);
|
||||
uint64_t t = 0;
|
||||
for (;;) {
|
||||
uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE);
|
||||
uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE);
|
||||
if (t == h) {
|
||||
for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&head, __ATOMIC_ACQUIRE);
|
||||
for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE);
|
||||
if (t == h) {
|
||||
__atomic_store_n(&sleeping, 1, __ATOMIC_SEQ_CST);
|
||||
h = __atomic_load_n(&head, __ATOMIC_SEQ_CST);
|
||||
__atomic_store_n(&Q.sleeping, 1, __ATOMIC_SEQ_CST);
|
||||
h = __atomic_load_n(&Q.head, __ATOMIC_SEQ_CST);
|
||||
if (t == h) plat_event_wait_ns(&wake_ev, 2000000); // a missed wake costs at most 2 ms
|
||||
__atomic_store_n(&sleeping, 0, __ATOMIC_RELAXED);
|
||||
__atomic_store_n(&Q.sleeping, 0, __ATOMIC_RELAXED);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -82,9 +92,9 @@ static void rec_main(void *arg) {
|
||||
uint32_t op = (uint32_t)(w & 0xffff), n = (uint32_t)(w >> 32);
|
||||
if (op != R_PAD) exec_packet(op, &ring[t % RING_WORDS + 1]);
|
||||
t += n;
|
||||
if ((k & 63) == 63) __atomic_store_n(&tail, t, __ATOMIC_RELEASE);
|
||||
if ((k & 63) == 63) __atomic_store_n(&Q.tail, t, __ATOMIC_RELEASE);
|
||||
}
|
||||
__atomic_store_n(&tail, t, __ATOMIC_RELEASE);
|
||||
__atomic_store_n(&Q.tail, t, __ATOMIC_RELEASE);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -93,48 +103,63 @@ static void rec_main(void *arg) {
|
||||
|
||||
static void wake(void) {
|
||||
__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev);
|
||||
if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev);
|
||||
}
|
||||
|
||||
// Room for need words: the tail read last leaves it in all but about one packet in 16 MiB of them.
|
||||
static void wait_space(uint64_t need) {
|
||||
if (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) <= RING_WORDS) return;
|
||||
if (Q.put_h + need - Q.tail_seen <= RING_WORDS) return;
|
||||
Q.tail_seen = __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE);
|
||||
if (Q.put_h + need - Q.tail_seen <= RING_WORDS) return;
|
||||
RS_.full_waits++;
|
||||
__atomic_store_n(&head, put_h, __ATOMIC_RELEASE);
|
||||
Q.pub_h = Q.put_h;
|
||||
__atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE);
|
||||
wake();
|
||||
while (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) > RING_WORDS) _mm_pause();
|
||||
while (Q.put_h + need - (Q.tail_seen = __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE)) > RING_WORDS) _mm_pause();
|
||||
}
|
||||
|
||||
// Room for a packet of bytes of arguments; the caller fills it and calls pub().
|
||||
static void *put(uint32_t op, size_t bytes) {
|
||||
uint64_t n = 1 + (bytes + 7) / 8, pos = put_h % RING_WORDS;
|
||||
uint64_t n = 1 + (bytes + 7) / 8, pos = Q.put_h % RING_WORDS;
|
||||
if (n > RING_WORDS / 4) rt_fatal("vk_rec: a packet of %llu bytes", (unsigned long long)bytes);
|
||||
if (pos + n > RING_WORDS) {
|
||||
uint64_t pad = RING_WORDS - pos;
|
||||
wait_space(pad);
|
||||
ring[pos] = (uint64_t)R_PAD | pad << 32;
|
||||
put_h += pad, pos = 0;
|
||||
Q.put_h += pad, pos = 0;
|
||||
}
|
||||
wait_space(n);
|
||||
ring[pos] = (uint64_t)op | n << 32;
|
||||
// Packers reserve room for optional arrays they may leave unwritten (BeginRendering's depth and stencil); with
|
||||
// TOTK_REC_CHECK those words would still hold handles from the ring's previous lap.
|
||||
if (rec_chk) memset(&ring[pos + 1], 0, (n - 1) * 8);
|
||||
put_h += n;
|
||||
Q.put_h += n;
|
||||
RS_.packets++, RS_.words += n;
|
||||
return &ring[pos + 1];
|
||||
}
|
||||
|
||||
// After a packet: head moves once Q.pub_words words wait, so the recorder reads a line the producer is still filling
|
||||
// once per batch instead of once per packet. Whatever the producer waits for publishes everything first (wait_space,
|
||||
// rec_drain), and the packets that begin and end a part go out at once (pub_now), so the recorder begins and ends
|
||||
// command buffers as early as before; TOTK_REC_CHECK publishes every packet, since rec_check_handle searches the ring
|
||||
// up to head.
|
||||
static void pub_now(void) {
|
||||
Q.pub_h = Q.put_h;
|
||||
__atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE);
|
||||
if (__atomic_load_n(&Q.sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev);
|
||||
}
|
||||
|
||||
static void pub(void) {
|
||||
__atomic_store_n(&head, put_h, __ATOMIC_RELEASE);
|
||||
if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev);
|
||||
if (Q.put_h - Q.pub_h >= Q.pub_words) pub_now();
|
||||
}
|
||||
|
||||
void rec_drain(void) {
|
||||
if (!rec_thread || __atomic_load_n(&tail, __ATOMIC_ACQUIRE) == put_h) return;
|
||||
if (!rec_thread || __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE) == Q.put_h) return;
|
||||
uint64_t t0 = now_ns();
|
||||
__atomic_store_n(&head, put_h, __ATOMIC_RELEASE);
|
||||
Q.pub_h = Q.put_h;
|
||||
__atomic_store_n(&Q.head, Q.put_h, __ATOMIC_RELEASE);
|
||||
wake();
|
||||
for (int i = 0; __atomic_load_n(&tail, __ATOMIC_ACQUIRE) != put_h; i++)
|
||||
for (int i = 0; __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE) != Q.put_h; i++)
|
||||
if (i < 20000) _mm_pause();
|
||||
else plat_yield();
|
||||
RS_.drains++, RS_.drain_ns += now_ns() - t0;
|
||||
@@ -175,6 +200,9 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) {
|
||||
rec_chk = rt_knob("REC_CHECK", "0"); // debug switches, read before the recorder starts
|
||||
const char *d = cfg_get("TOTK_REC_DELAY_US");
|
||||
rec_delay_ns = d && *d && atoi(d) > 0 ? (uint64_t)atoi(d) * 1000 : 0;
|
||||
const char *pw = cfg_get("TOTK_REC_PUB"); // words per publish (pub); 0: every packet
|
||||
Q.pub_words = rec_chk ? 0 : pw && *pw ? (uint64_t)atoi(pw) : 64;
|
||||
if (Q.pub_words > RING_WORDS / 8) Q.pub_words = RING_WORDS / 8;
|
||||
rec_thread = plat_thread_new(rec_main, NULL, "vk recorder");
|
||||
if (!rec_thread) {
|
||||
rt_log("vk_rec: no recorder thread: the replay makes its Vulkan calls itself");
|
||||
@@ -183,6 +211,8 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) {
|
||||
rt_log("vk_rec: the replay's Vulkan commands are made by a recorder thread%s",
|
||||
mode == 2 ? " in the B blocks (TOTK_AB)" : "");
|
||||
if (rec_chk) rt_log("vk_rec: TOTK_REC_CHECK: deferred and destroyed objects are searched in the ring");
|
||||
if (Q.pub_words != 64) rt_log("vk_rec: TOTK_REC_PUB: head published every %llu words",
|
||||
(unsigned long long)Q.pub_words);
|
||||
if (rec_delay_ns) rt_log("vk_rec: TOTK_REC_DELAY_US: the recorder waits %llu us before each batch",
|
||||
(unsigned long long)(rec_delay_ns / 1000));
|
||||
}
|
||||
@@ -198,11 +228,11 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence, int crumb, uint64_t n) {
|
||||
if (!on) return;
|
||||
}
|
||||
RS_.parts_on++;
|
||||
__atomic_store_n(&part_h0, put_h, __ATOMIC_RELEASE);
|
||||
__atomic_store_n(&part_h0, Q.put_h, __ATOMIC_RELEASE);
|
||||
__atomic_store_n(&part_fence, fence, __ATOMIC_RELEASE);
|
||||
A_BeginCb *a = put(R_BeginCb, sizeof *a);
|
||||
*a = (A_BeginCb){cb, own, crumb, n};
|
||||
pub();
|
||||
pub_now();
|
||||
rec_cb = cb;
|
||||
rec_live = 1;
|
||||
}
|
||||
@@ -241,7 +271,7 @@ int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n) {
|
||||
}
|
||||
A_EndCb *a = put(R_EndCb, sizeof *a);
|
||||
*a = (A_EndCb){crumb, n};
|
||||
pub();
|
||||
pub_now();
|
||||
rec_live = 0;
|
||||
sub_pending = 1;
|
||||
return 1;
|
||||
@@ -306,7 +336,7 @@ static const char *const kind_name[REC_KINDS] = {"image", "view", "buffer"};
|
||||
static volatile int32_t chk_seen[2], chk_named, chk_logged, chk_hz[2][2][REC_KINDS][64]; // [destroy][made][kind][op]
|
||||
void rec_check_handle(int kind, uint64_t handle, int destroy, uint64_t tl, uint64_t gpu) {
|
||||
if (!rec_chk || !handle || kind < 0 || kind >= REC_KINDS || !__atomic_load_n(&rec_live, __ATOMIC_RELAXED)) return;
|
||||
uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE), p = __atomic_load_n(&part_h0, __ATOMIC_ACQUIRE);
|
||||
uint64_t h = __atomic_load_n(&Q.head, __ATOMIC_ACQUIRE), p = __atomic_load_n(&part_h0, __ATOMIC_ACQUIRE);
|
||||
uint64_t fence = __atomic_load_n(&part_fence, __ATOMIC_ACQUIRE);
|
||||
if (p > h || h - p > RING_WORDS / 2) return; // a part this large is not searched
|
||||
plat_inc32(&chk_seen[destroy != 0]);
|
||||
@@ -317,7 +347,7 @@ void rec_check_handle(int kind, uint64_t handle, int destroy, uint64_t tl, uint6
|
||||
if (op != R_PAD)
|
||||
for (uint32_t i = 1; i < n; i++) {
|
||||
if (ring[(p + i) % RING_WORDS] != handle) continue;
|
||||
int made = p < __atomic_load_n(&tail, __ATOMIC_ACQUIRE);
|
||||
int made = p < __atomic_load_n(&Q.tail, __ATOMIC_ACQUIRE);
|
||||
if (!destroy && tl >= fence) { plat_inc32(&chk_named); return; }
|
||||
plat_inc32(&chk_hz[destroy][made][kind][op & 63]);
|
||||
if (plat_inc32(&chk_logged) <= 64)
|
||||
|
||||
Reference in new issue
Block a user