diff --git a/docs/architecture.md b/docs/architecture.md index f4b4e3d6..e2486bbe 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -74,6 +74,7 @@ Every guest thread is a host thread. TotK assumes Horizon's scheduler: three app - Vulkan 1.3 back end in place of MacNativeGames' Metal one. At `QueueSubmitCommands` the recorded NVN commands are replayed into Vulkan command buffers (`vk_exec*.c`). - Memory pools. Apple's unified memory let MacNativeGames use the game's pools zero-copy. The back end imports the pools as host memory (`VK_EXT_external_memory_host`). On a discrete GPU, pools bound as buffers are mirrored in device-local memory and the pages the CPU wrote (write watch, above) are uploaded before a replay; idle pool regions are made read-only so the write-watch queries skip them (`vk_memory.c`). GPU-rendered textures are written back when the CPU reads them: on a pool invalidate, on a CPU copy, and for small targets once the CPU waits for a fence (`vk_texture.c`). - Replay thread. `TOTK_REPLAY_ASYNC=1` moves the replay off the submitting thread onto a queue at most one frame behind the game; reads of GPU results wait for the newest writer instead of draining the queue (`vk_backend.c`). It is off by default because no timing has shown a gain yet. +- Recorder thread. The Vulkan calls the replay makes into its command buffer are made by a thread of their own (`vk_rec.c`, after Savage Wind Recomp and LAS Recomp): each `vkCmd*` copies its arguments into a packet of a ring and the recorder makes the calls in order; calls into other command buffers are made at once, and the submit waits until the ring is empty. In the town on an RX 9070 the replay takes 17% less time on the submitting thread. `TOTK_VK_RECORDER=0` makes every call on the submitting thread. - Driver data. The NVN driver keeps a per-stage constant bank (c0) that shaders read for things like point size, dispatch size and base instance. The back end fills it the way the driver does. - Shaders: the Ryujinx translator 1.3.2 with MacNativeGames' patches and ours (`tools/xlate/patches/`), built with .NET 10 NativeAOT as `totk_xlate.exe` and run as a separate process. Its SPIR-V goes to Vulkan directly. Translated shaders are cached on disk. - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it. Shader stages are compiled ahead into libraries; a draw that meets a new state links them at once (fast link), and an optimised pipeline replaces the linked one when a background thread finishes it. Pipeline keys seen in play are saved and built at boot. A draw whose pipeline is not ready is skipped when one of its pass's targets was rendered in the previous frame, and waits otherwise, so one-shot passes are never lost; a full-screen draw is skipped as well rather than waiting (`TOTK_PIPE_SKIP`); the stage libraries a waiting draw needs are built on idle workers in parallel (`TOTK_PIPE_WAIT`). Without the extension, a missing pipeline is built on a worker and the draw waits or is skipped by the same rule. diff --git a/runtime/cfg.c b/runtime/cfg.c index bee6a0ff..a43c2a49 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -47,7 +47,7 @@ static const char *const known[] = { "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_MEMO", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPLIT_DRAWS", "SPLIT_US", "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TIMER_1MS", "TRACE", "UPSCALER", "VK_DYN_VERTEX", "VK_NO_FLOAT64", - "VK_PRESENT", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", + "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", }; diff --git a/runtime/vk.h b/runtime/vk.h index 74cccc6a..597e0e02 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -452,3 +452,5 @@ void vk_pipeline_autosave(void); // vk_pipeline_save at most every 20 void vk_pipeline_shutdown(void); // waits for background builds, saves, destroys pipelines, layouts and the // cache; before vk_shutdown int vk_pipeline_selftest(void); // one graphics and one compute pipeline from census shaders; 0 on success + +#include "vk_rec.h" // last: its macros route the vkCmd* calls of every file that includes vk.h diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index 5d04fb84..70ba539d 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -466,7 +466,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons } if (slot >= 0) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, tpool, 2 * (uint32_t)slot); vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); - G.push(cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 3, w); + rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 3, w); VkDeviceSize ib = 0, ob = 0; for (int i = i0; i < i1; i++) { const VkAstcRegion *r = &rg[i]; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 3d8946c1..63d4ee5d 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1594,8 +1594,8 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { BS.push_skip++; continue; } - if (info->tpl[s]) vk_pipeline_push_tpl_fn()(ex->cb, info->tpl[s], info->layout, s, ds); - else vk_pipeline_push_fn()(ex->cb, info->bind_point, info->layout, s, i1 - i0, w + i0); + if (info->tpl[s]) rec_push_tpl(vk_pipeline_push_tpl_fn(), ex->cb, info->tpl[s], info->layout, s, ds, dbytes); + else rec_push(vk_pipeline_push_fn(), ex->cb, info->bind_point, info->layout, s, i1 - i0, w + i0); BS.push++; RS.cmds++; if (gfx) { diff --git a/runtime/vk_fsr.c b/runtime/vk_fsr.c index f9d6a456..ad6fd601 100644 --- a/runtime/vk_fsr.c +++ b/runtime/vk_fsr.c @@ -147,7 +147,7 @@ static void bind_pass(VkCommandBuffer cb, VkPipeline p, VkImageView in, VkImageV {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 1, .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .pImageInfo = &oi}}; vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, p); - F.push(cb, VK_PIPELINE_BIND_POINT_COMPUTE, F.layout, 0, 2, w); + rec_push(F.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, F.layout, 0, 2, w); vkCmdPushConstants(cb, F.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, 32, pc); } diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 79c3f34a..52cfaafe 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2979,11 +2979,11 @@ void vk_pipeline_set_dynamic(VkCommandBuffer cb, const VkPipeState *s) { if (P.dyn_vi && s->vs) { VtxInput v; vertex_input(s, &v); - P.set_vi(cb, v.n_bind, v.b, v.n_attr, v.a); + rec_vertex_input(P.set_vi, cb, v.n_bind, v.b, v.n_attr, v.a); } - if (P.dyn_clamp) P.set_clamp(cb, s->depth_clamp != 0); - if (P.dyn_poly) P.set_poly(cb, polygon_mode(pl->mode)); - if (P.dyn_a2c) P.set_a2c(cb, s->ms.a2c != 0); + if (P.dyn_clamp) rec_bool(P.set_clamp, cb, s->depth_clamp != 0); + if (P.dyn_poly) rec_polygon_mode(P.set_poly, cb, polygon_mode(pl->mode)); + if (P.dyn_a2c) rec_bool((PFN_vkCmdSetDepthClampEnableEXT)P.set_a2c, cb, s->ms.a2c != 0); // the same signature int n = target_count(s); if (!n) return; VkBool32 en[8]; @@ -2996,9 +2996,9 @@ void vk_pipeline_set_dynamic(VkCommandBuffer cb, const VkPipeState *s) { blend_factor(b->src_a), blend_factor(b->dst_a), blend_op(b->eq_a)}; wm[i] = target_wmask(s, i); } - if (P.dyn_blend_en) P.set_blend_en(cb, 0, (uint32_t)n, en); - if (P.dyn_blend_eq) P.set_blend_eq(cb, 0, (uint32_t)n, eq); - if (P.dyn_wmask) P.set_wmask(cb, 0, (uint32_t)n, wm); + if (P.dyn_blend_en) rec_blend_enable(P.set_blend_en, cb, 0, (uint32_t)n, en); + if (P.dyn_blend_eq) rec_blend_equation(P.set_blend_eq, cb, 0, (uint32_t)n, eq); + if (P.dyn_wmask) rec_write_mask(P.set_wmask, cb, 0, (uint32_t)n, wm); } // ---- statistics, persistence, teardown diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c new file mode 100644 index 00000000..2329be22 --- /dev/null +++ b/runtime/vk_rec.c @@ -0,0 +1,586 @@ +// The recorder thread: see vk_rec.h. One producer (the submitting thread, between rec_begin and rec_end) and one +// consumer (the recorder) share a ring of 64-bit words. A packet is a header word (op, length in words) and the +// call's arguments; its pointers point into the packet itself, so the recorder passes the payload to the driver as +// it is. A packet never wraps: one that does not fit before the end is preceded by a pad. The producer publishes head +// after each packet; the recorder makes the calls and advances tail, which the producer waits on when the ring is +// full and in rec_drain. While a part is recorded the recorder spins on head; idle, it sleeps on an event. +// Design: Savage Wind Recomp #189 and LAS Recomp runtime/nvn/vulkan/record.c (https://github.com/peiikai/LASRecomp +// @ a4b53a3, MIT, Copyright (c) 2026 Peiikai); the code is ours. +#define VK_REC_IMPL +#include +#include +#include +#include + +#include "host.h" +#include "vk.h" + +#define RING_WORDS (1u << 21) // 16 MiB: a part of a submission is ~1-2 MiB of packets in the town +static uint64_t ring[RING_WORDS]; +static volatile uint64_t head, tail; // words published by the producer / consumed by the recorder +static volatile LONG sleeping; // the recorder waits for wake_ev +static HANDLE wake_ev; // auto-reset +static uint64_t put_h; // producer: words written (published up to head) +static VkCommandBuffer rec_cb; // the part being recorded (set while the recorder is idle) +static int rec_live; // producer: rec_begin started the recorder for rec_cb +static HANDLE rec_thread; + +// Producer statistics, logged by rec_stats_log. +static struct { uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on; } RS_; + +enum { + R_PAD, R_BeginQuery, R_BeginRendering, R_BindDescriptorSets, R_BindIndexBuffer, R_BindPipeline, R_BindVertexBuffers2, + R_BlitImage, R_ClearAttachments, R_ClearColorImage, R_ClearDepthStencilImage, R_CopyBuffer, R_CopyBufferToImage, + R_CopyImage, R_CopyImageToBuffer, R_Dispatch, R_DispatchIndirect, R_Draw, R_DrawIndexed, R_DrawIndexedIndirect, + R_EndQuery, R_EndRendering, R_FillBuffer, R_PipelineBarrier2, R_PushConstants, R_ResetQueryPool, R_ResolveImage, + R_SetBlendConstants, R_SetCullMode, R_SetDepthBias, R_SetDepthBiasEnable, R_SetDepthCompareOp, R_SetDepthTestEnable, + R_SetDepthWriteEnable, R_SetFrontFace, R_SetPrimitiveRestartEnable, R_SetPrimitiveTopology, + R_SetRasterizerDiscardEnable, R_SetScissorWithCount, R_SetStencilCompareMask, R_SetStencilOp, R_SetStencilReference, + R_SetStencilTestEnable, R_SetStencilWriteMask, R_SetViewportWithCount, R_WriteTimestamp, R_WriteTimestamp2, + R_PushTpl, R_Push, R_VertexInput, R_Bool, R_PolygonMode, R_BlendEnable, R_BlendEquation, R_WriteMask, +}; + +static uint64_t now_ns(void) { + LARGE_INTEGER c, f; + QueryPerformanceCounter(&c), QueryPerformanceFrequency(&f); + return (uint64_t)((double)c.QuadPart * 1e9 / (double)f.QuadPart); +} + +// ---- consumer + +static void exec_packet(uint32_t op, void *p); + +static DWORD WINAPI rec_main(void *arg) { + (void)arg; + uint64_t t = 0; + for (;;) { + uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); + if (t == h) { + for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); + if (t == h) { + __atomic_store_n(&sleeping, 1, __ATOMIC_SEQ_CST); + h = __atomic_load_n(&head, __ATOMIC_SEQ_CST); + if (t == h) WaitForSingleObject(wake_ev, 2); // a missed wake costs at most 2 ms + __atomic_store_n(&sleeping, 0, __ATOMIC_RELAXED); + continue; + } + } + for (int k = 0; t != h; k++) { + uint64_t w = ring[t % RING_WORDS]; + uint32_t op = (uint32_t)(w & 0xffff), n = (uint32_t)(w >> 32); + if (op != R_PAD) exec_packet(op, &ring[t % RING_WORDS + 1]); + t += n; + if ((k & 63) == 63) __atomic_store_n(&tail, t, __ATOMIC_RELEASE); + } + __atomic_store_n(&tail, t, __ATOMIC_RELEASE); + } + return 0; +} + +// ---- producer + +static void wake(void) { + __atomic_thread_fence(__ATOMIC_SEQ_CST); + if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) SetEvent(wake_ev); +} + +static void wait_space(uint64_t need) { + if (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) <= RING_WORDS) return; + RS_.full_waits++; + __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); + wake(); + while (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) > RING_WORDS) _mm_pause(); +} + +// Room for a packet of bytes of arguments; the caller fills it and calls pub(). +static void *put(uint32_t op, size_t bytes) { + uint64_t n = 1 + (bytes + 7) / 8, pos = put_h % RING_WORDS; + if (n > RING_WORDS / 4) rt_fatal("vk_rec: a packet of %llu bytes", (unsigned long long)bytes); + if (pos + n > RING_WORDS) { + uint64_t pad = RING_WORDS - pos; + wait_space(pad); + ring[pos] = (uint64_t)R_PAD | pad << 32; + put_h += pad, pos = 0; + } + wait_space(n); + ring[pos] = (uint64_t)op | n << 32; + put_h += n; + RS_.packets++, RS_.words += n; + return &ring[pos + 1]; +} + +static void pub(void) { + __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); + if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) SetEvent(wake_ev); +} + +void rec_drain(void) { + if (!rec_thread || __atomic_load_n(&tail, __ATOMIC_ACQUIRE) == put_h) return; + uint64_t t0 = now_ns(); + __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); + wake(); + for (int i = 0; __atomic_load_n(&tail, __ATOMIC_ACQUIRE) != put_h; i++) + if (i < 20000) _mm_pause(); + else SwitchToThread(); + RS_.drains++, RS_.drain_ns += now_ns() - t0; +} + +void rec_begin(VkCommandBuffer cb) { + static int mode = -1; + if (mode < 0) { + mode = rt_knob("VK_RECORDER", "1"); + if (mode) { + wake_ev = CreateEventA(NULL, FALSE, FALSE, NULL); + rec_thread = wake_ev ? CreateThread(NULL, 0, rec_main, NULL, 0, NULL) : NULL; + if (!rec_thread) { + rt_log("vk_rec: no recorder thread (error %lu): the replay makes its Vulkan calls itself", GetLastError()); + mode = 0; + } else { + SetThreadPriority(rec_thread, GetThreadPriority(GetCurrentThread())); + rt_log("vk_rec: the replay's Vulkan commands are made by a recorder thread%s", + mode == 2 ? " in the B blocks (TOTK_AB)" : ""); + } + } + } + RS_.parts++; + if (!RT_KNOB_ON(mode)) return; + RS_.parts_on++; + rec_cb = cb; // the recorder is idle: rec_end drained it + rec_live = 1; +} + +void rec_end(void) { + if (!rec_live) return; + rec_drain(); + rec_live = 0; +} + +void rec_stats_log(void) { + if (!(rt_trace & RT_TRACE_PERF) || !RS_.parts) return; + rt_log("vk_rec: %llu of %llu parts recorded by the recorder: %llu packets (%.1f MiB), %llu drains (%llu for a call " + "without a packet) waited %.2f ms in all, ring full %llu times", + (unsigned long long)RS_.parts_on, (unsigned long long)RS_.parts, (unsigned long long)RS_.packets, + RS_.words * 8 / 1048576.0, (unsigned long long)RS_.drains, (unsigned long long)RS_.fallbacks, + RS_.drain_ns / 1e6, (unsigned long long)RS_.full_waits); + memset(&RS_, 0, sizeof RS_); +} + +#define OWNED(cb) (rec_live && (cb) == rec_cb) +// A call the ring takes no packet for: everything queued first, then the call here. +#define FALLBACK() (rec_drain(), RS_.fallbacks++) + +// Copies n elements of arr after the fixed part at *at and returns where they went (NULL for none). +static void *tail_copy(uint8_t **at, const void *arr, size_t bytes) { + if (!arr || !bytes) return NULL; + void *d = *at; + memcpy(d, arr, bytes); + *at += (bytes + 7) & ~(size_t)7; + return d; +} +#define A8(x) (((x) + 7) & ~(size_t)7) + +// ---- fixed-size commands: the arguments after cb, as a struct + +#define FIXED(name, params, call_args, ...) \ + typedef struct { __VA_ARGS__; } A_##name; \ + void rc_vkCmd##name params { \ + if (!OWNED(cb)) { vkCmd##name call_args; return; } \ + A_##name *a = put(R_##name, sizeof *a); \ + PACK_##name; \ + pub(); \ + } + +#define PACK_BeginQuery *a = (A_BeginQuery){p, q, f} +FIXED(BeginQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q, VkQueryControlFlags f), (cb, p, q, f), + VkQueryPool p; uint32_t q; VkQueryControlFlags f) +#define PACK_BindIndexBuffer *a = (A_BindIndexBuffer){b, o, t} +FIXED(BindIndexBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkIndexType t), (cb, b, o, t), VkBuffer b; + VkDeviceSize o; VkIndexType t) +#define PACK_BindPipeline *a = (A_BindPipeline){bp, p} +FIXED(BindPipeline, (VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipeline p), (cb, bp, p), VkPipelineBindPoint bp; + VkPipeline p) +#define PACK_Dispatch *a = (A_Dispatch){x, y, z} +FIXED(Dispatch, (VkCommandBuffer cb, uint32_t x, uint32_t y, uint32_t z), (cb, x, y, z), uint32_t x, y, z) +#define PACK_DispatchIndirect *a = (A_DispatchIndirect){b, o} +FIXED(DispatchIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o), (cb, b, o), VkBuffer b; VkDeviceSize o) +#define PACK_Draw *a = (A_Draw){n, i, f, fi} +FIXED(Draw, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, uint32_t fi), (cb, n, i, f, fi), uint32_t n, i, f, fi) +#define PACK_DrawIndexed *a = (A_DrawIndexed){n, i, f, v, fi} +FIXED(DrawIndexed, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, int32_t v, uint32_t fi), (cb, n, i, f, v, fi), + uint32_t n, i, f; int32_t v; uint32_t fi) +#define PACK_DrawIndexedIndirect *a = (A_DrawIndexedIndirect){b, o, n, s} +FIXED(DrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s), + VkBuffer b; VkDeviceSize o; uint32_t n, s) +#define PACK_EndQuery *a = (A_EndQuery){p, q} +FIXED(EndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), (cb, p, q), VkQueryPool p; uint32_t q) +#define PACK_EndRendering (void)a +FIXED(EndRendering, (VkCommandBuffer cb), (cb), int unused) +#define PACK_FillBuffer *a = (A_FillBuffer){b, o, n, v} +FIXED(FillBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkDeviceSize n, uint32_t v), (cb, b, o, n, v), + VkBuffer b; VkDeviceSize o, n; uint32_t v) +#define PACK_ResetQueryPool *a = (A_ResetQueryPool){p, f, n} +FIXED(ResetQueryPool, (VkCommandBuffer cb, VkQueryPool p, uint32_t f, uint32_t n), (cb, p, f, n), VkQueryPool p; + uint32_t f, n) +#define PACK_SetBlendConstants memcpy(a->c, c, sizeof a->c) +FIXED(SetBlendConstants, (VkCommandBuffer cb, const float c[4]), (cb, c), float c[4]) +#define PACK_SetCullMode a->v = m +FIXED(SetCullMode, (VkCommandBuffer cb, VkCullModeFlags m), (cb, m), VkCullModeFlags v) +#define PACK_SetDepthBias *a = (A_SetDepthBias){c, cl, s} +FIXED(SetDepthBias, (VkCommandBuffer cb, float c, float cl, float s), (cb, c, cl, s), float c, cl, s) +#define PACK_SetDepthBiasEnable a->v = v +FIXED(SetDepthBiasEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v), VkBool32 v) +#define PACK_SetDepthCompareOp a->v = v +FIXED(SetDepthCompareOp, (VkCommandBuffer cb, VkCompareOp v), (cb, v), VkCompareOp v) +#define PACK_SetDepthTestEnable a->v = v +FIXED(SetDepthTestEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v), VkBool32 v) +#define PACK_SetDepthWriteEnable a->v = v +FIXED(SetDepthWriteEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v), VkBool32 v) +#define PACK_SetFrontFace a->v = v +FIXED(SetFrontFace, (VkCommandBuffer cb, VkFrontFace v), (cb, v), VkFrontFace v) +#define PACK_SetPrimitiveRestartEnable a->v = v +FIXED(SetPrimitiveRestartEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v), VkBool32 v) +#define PACK_SetPrimitiveTopology a->v = v +FIXED(SetPrimitiveTopology, (VkCommandBuffer cb, VkPrimitiveTopology v), (cb, v), VkPrimitiveTopology v) +#define PACK_SetRasterizerDiscardEnable a->v = v +FIXED(SetRasterizerDiscardEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v), VkBool32 v) +#define PACK_SetStencilCompareMask *a = (A_SetStencilCompareMask){f, v} +FIXED(SetStencilCompareMask, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), (cb, f, v), VkStencilFaceFlags f; + uint32_t v) +#define PACK_SetStencilOp *a = (A_SetStencilOp){f, x, y, z, d} +FIXED(SetStencilOp, (VkCommandBuffer cb, VkStencilFaceFlags f, VkStencilOp x, VkStencilOp y, VkStencilOp z, VkCompareOp d), + (cb, f, x, y, z, d), VkStencilFaceFlags f; VkStencilOp x, y, z; VkCompareOp d) +#define PACK_SetStencilReference *a = (A_SetStencilReference){f, v} +FIXED(SetStencilReference, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), (cb, f, v), VkStencilFaceFlags f; + uint32_t v) +#define PACK_SetStencilTestEnable a->v = v +FIXED(SetStencilTestEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v), VkBool32 v) +#define PACK_SetStencilWriteMask *a = (A_SetStencilWriteMask){f, v} +FIXED(SetStencilWriteMask, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), (cb, f, v), VkStencilFaceFlags f; + uint32_t v) +#define PACK_WriteTimestamp *a = (A_WriteTimestamp){s, p, q} +FIXED(WriteTimestamp, (VkCommandBuffer cb, VkPipelineStageFlagBits s, VkQueryPool p, uint32_t q), (cb, s, p, q), + VkPipelineStageFlagBits s; VkQueryPool p; uint32_t q) +#define PACK_WriteTimestamp2 *a = (A_WriteTimestamp2){s, p, q} +FIXED(WriteTimestamp2, (VkCommandBuffer cb, VkPipelineStageFlags2 s, VkQueryPool p, uint32_t q), (cb, s, p, q), + VkPipelineStageFlags2 s; VkQueryPool p; uint32_t q) + +// ---- commands with arrays: a struct with the call's arguments, the arrays after it, pointers into the packet + +typedef struct { uint32_t n; const VkViewport *v; } A_SetViewportWithCount; +void rc_vkCmdSetViewportWithCount(VkCommandBuffer cb, uint32_t n, const VkViewport *v) { + if (!OWNED(cb)) { vkCmdSetViewportWithCount(cb, n, v); return; } + A_SetViewportWithCount *a = put(R_SetViewportWithCount, sizeof *a + sizeof *v * n); + uint8_t *at = (uint8_t *)(a + 1); + a->n = n, a->v = tail_copy(&at, v, sizeof *v * n); + pub(); +} + +typedef struct { uint32_t n; const VkRect2D *r; } A_SetScissorWithCount; +void rc_vkCmdSetScissorWithCount(VkCommandBuffer cb, uint32_t n, const VkRect2D *r) { + if (!OWNED(cb)) { vkCmdSetScissorWithCount(cb, n, r); return; } + A_SetScissorWithCount *a = put(R_SetScissorWithCount, sizeof *a + sizeof *r * n); + uint8_t *at = (uint8_t *)(a + 1); + a->n = n, a->r = tail_copy(&at, r, sizeof *r * n); + pub(); +} + +typedef struct { + VkPipelineBindPoint bp; + VkPipelineLayout l; + uint32_t first, n, nd; + const VkDescriptorSet *s; + const uint32_t *d; +} A_BindDescriptorSets; +void rc_vkCmdBindDescriptorSets(VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipelineLayout l, uint32_t first, + uint32_t n, const VkDescriptorSet *s, uint32_t nd, const uint32_t *d) { + if (!OWNED(cb)) { vkCmdBindDescriptorSets(cb, bp, l, first, n, s, nd, d); return; } + A_BindDescriptorSets *a = put(R_BindDescriptorSets, sizeof *a + A8(sizeof *s * n) + A8(sizeof *d * nd)); + uint8_t *at = (uint8_t *)(a + 1); + *a = (A_BindDescriptorSets){bp, l, first, n, nd, NULL, NULL}; + a->s = tail_copy(&at, s, sizeof *s * n), a->d = tail_copy(&at, d, sizeof *d * nd); + pub(); +} + +typedef struct { uint32_t first, n; const VkBuffer *b; const VkDeviceSize *o, *sz, *st; } A_BindVertexBuffers2; +void rc_vkCmdBindVertexBuffers2(VkCommandBuffer cb, uint32_t first, uint32_t n, const VkBuffer *b, const VkDeviceSize *o, + const VkDeviceSize *sz, const VkDeviceSize *st) { + if (!OWNED(cb)) { vkCmdBindVertexBuffers2(cb, first, n, b, o, sz, st); return; } + size_t e = sizeof(VkDeviceSize) * n; + A_BindVertexBuffers2 *a = put(R_BindVertexBuffers2, sizeof *a + A8(sizeof *b * n) + 3 * A8(e)); + uint8_t *at = (uint8_t *)(a + 1); + a->first = first, a->n = n; + a->b = tail_copy(&at, b, sizeof *b * n), a->o = tail_copy(&at, o, e); + a->sz = sz ? tail_copy(&at, sz, e) : NULL, a->st = st ? tail_copy(&at, st, e) : NULL; + pub(); +} + +typedef struct { VkImage s; VkImageLayout sl; VkImage d; VkImageLayout dl; uint32_t n; const void *r; VkFilter f; } A_Img; +#define IMG_CMD(name, rtype, params, call, ...) \ + void rc_vkCmd##name params { \ + if (!OWNED(cb)) { vkCmd##name call; return; } \ + A_Img *a = put(R_##name, sizeof *a + sizeof(rtype) * n); \ + uint8_t *at = (uint8_t *)(a + 1); \ + *a = (A_Img){__VA_ARGS__}; \ + a->r = tail_copy(&at, r, sizeof(rtype) * n); \ + pub(); \ + } +IMG_CMD(BlitImage, VkImageBlit, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, uint32_t n, + const VkImageBlit *r, VkFilter f), (cb, s, sl, d, dl, n, r, f), s, sl, d, dl, n, NULL, f) +IMG_CMD(CopyImage, VkImageCopy, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, uint32_t n, + const VkImageCopy *r), (cb, s, sl, d, dl, n, r), s, sl, d, dl, n, NULL, 0) +IMG_CMD(ResolveImage, VkImageResolve, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, + uint32_t n, const VkImageResolve *r), (cb, s, sl, d, dl, n, r), s, sl, d, dl, n, NULL, 0) + +typedef struct { uint64_t s, d; VkImageLayout l; uint32_t n; const void *r; } A_Xfer; // buffer or image handles +#define XFER_CMD(name, rtype, params, call, S, D, L) \ + void rc_vkCmd##name params { \ + if (!OWNED(cb)) { vkCmd##name call; return; } \ + A_Xfer *a = put(R_##name, sizeof *a + sizeof(rtype) * n); \ + uint8_t *at = (uint8_t *)(a + 1); \ + *a = (A_Xfer){(uint64_t)(S), (uint64_t)(D), L, n, NULL}; \ + a->r = tail_copy(&at, r, sizeof(rtype) * n); \ + pub(); \ + } +XFER_CMD(CopyBuffer, VkBufferCopy, (VkCommandBuffer cb, VkBuffer s, VkBuffer d, uint32_t n, const VkBufferCopy *r), + (cb, s, d, n, r), s, d, 0) +XFER_CMD(CopyBufferToImage, VkBufferImageCopy, (VkCommandBuffer cb, VkBuffer s, VkImage d, VkImageLayout dl, uint32_t n, + const VkBufferImageCopy *r), (cb, s, d, dl, n, r), s, d, dl) +XFER_CMD(CopyImageToBuffer, VkBufferImageCopy, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkBuffer d, uint32_t n, + const VkBufferImageCopy *r), (cb, s, sl, d, n, r), s, d, sl) + +typedef struct { uint32_t na, nr; const VkClearAttachment *a; const VkClearRect *r; } A_ClearAttachments; +void rc_vkCmdClearAttachments(VkCommandBuffer cb, uint32_t na, const VkClearAttachment *ca, uint32_t nr, const VkClearRect *r) { + if (!OWNED(cb)) { vkCmdClearAttachments(cb, na, ca, nr, r); return; } + A_ClearAttachments *a = put(R_ClearAttachments, sizeof *a + A8(sizeof *ca * na) + A8(sizeof *r * nr)); + uint8_t *at = (uint8_t *)(a + 1); + a->na = na, a->nr = nr; + a->a = tail_copy(&at, ca, sizeof *ca * na), a->r = tail_copy(&at, r, sizeof *r * nr); + pub(); +} + +typedef struct { VkImage i; VkImageLayout l; union { VkClearColorValue c; VkClearDepthStencilValue ds; } v; uint32_t n; + const VkImageSubresourceRange *r; } A_ClearImage; +void rc_vkCmdClearColorImage(VkCommandBuffer cb, VkImage i, VkImageLayout l, const VkClearColorValue *c, uint32_t n, + const VkImageSubresourceRange *r) { + if (!OWNED(cb)) { vkCmdClearColorImage(cb, i, l, c, n, r); return; } + A_ClearImage *a = put(R_ClearColorImage, sizeof *a + sizeof *r * n); + uint8_t *at = (uint8_t *)(a + 1); + a->i = i, a->l = l, a->v.c = *c, a->n = n, a->r = tail_copy(&at, r, sizeof *r * n); + pub(); +} +void rc_vkCmdClearDepthStencilImage(VkCommandBuffer cb, VkImage i, VkImageLayout l, const VkClearDepthStencilValue *c, + uint32_t n, const VkImageSubresourceRange *r) { + if (!OWNED(cb)) { vkCmdClearDepthStencilImage(cb, i, l, c, n, r); return; } + A_ClearImage *a = put(R_ClearDepthStencilImage, sizeof *a + sizeof *r * n); + uint8_t *at = (uint8_t *)(a + 1); + a->i = i, a->l = l, a->v.ds = *c, a->n = n, a->r = tail_copy(&at, r, sizeof *r * n); + pub(); +} + +typedef struct { VkPipelineLayout l; VkShaderStageFlags st; uint32_t o, n; const void *v; } A_PushConstants; +void rc_vkCmdPushConstants(VkCommandBuffer cb, VkPipelineLayout l, VkShaderStageFlags st, uint32_t o, uint32_t n, + const void *v) { + if (!OWNED(cb)) { vkCmdPushConstants(cb, l, st, o, n, v); return; } + A_PushConstants *a = put(R_PushConstants, sizeof *a + n); + uint8_t *at = (uint8_t *)(a + 1); + *a = (A_PushConstants){l, st, o, n, NULL}; + a->v = tail_copy(&at, v, n); + pub(); +} + +void rc_vkCmdPipelineBarrier2(VkCommandBuffer cb, const VkDependencyInfo *di) { + if (!OWNED(cb)) { vkCmdPipelineBarrier2(cb, di); return; } + int chained = di->pNext != NULL; + for (uint32_t i = 0; i < di->memoryBarrierCount; i++) chained |= di->pMemoryBarriers[i].pNext != NULL; + for (uint32_t i = 0; i < di->bufferMemoryBarrierCount; i++) chained |= di->pBufferMemoryBarriers[i].pNext != NULL; + for (uint32_t i = 0; i < di->imageMemoryBarrierCount; i++) chained |= di->pImageMemoryBarriers[i].pNext != NULL; + if (chained) { FALLBACK(); vkCmdPipelineBarrier2(cb, di); return; } + size_t m = sizeof *di->pMemoryBarriers * di->memoryBarrierCount; + size_t b = sizeof *di->pBufferMemoryBarriers * di->bufferMemoryBarrierCount; + size_t im = sizeof *di->pImageMemoryBarriers * di->imageMemoryBarrierCount; + VkDependencyInfo *a = put(R_PipelineBarrier2, sizeof *a + A8(m) + A8(b) + A8(im)); + uint8_t *at = (uint8_t *)(a + 1); + *a = *di; + a->pMemoryBarriers = tail_copy(&at, di->pMemoryBarriers, m); + a->pBufferMemoryBarriers = tail_copy(&at, di->pBufferMemoryBarriers, b); + a->pImageMemoryBarriers = tail_copy(&at, di->pImageMemoryBarriers, im); + pub(); +} + +void rc_vkCmdBeginRendering(VkCommandBuffer cb, const VkRenderingInfo *ri) { + if (!OWNED(cb)) { vkCmdBeginRendering(cb, ri); return; } + int chained = ri->pNext != NULL || (ri->pDepthAttachment && ri->pDepthAttachment->pNext) || + (ri->pStencilAttachment && ri->pStencilAttachment->pNext); + for (uint32_t i = 0; i < ri->colorAttachmentCount; i++) chained |= ri->pColorAttachments[i].pNext != NULL; + if (chained) { FALLBACK(); vkCmdBeginRendering(cb, ri); return; } + size_t c = sizeof *ri->pColorAttachments * ri->colorAttachmentCount, one = sizeof(VkRenderingAttachmentInfo); + VkRenderingInfo *a = put(R_BeginRendering, sizeof *a + A8(c) + 2 * A8(one)); + uint8_t *at = (uint8_t *)(a + 1); + *a = *ri; + a->pColorAttachments = tail_copy(&at, ri->pColorAttachments, c); + a->pDepthAttachment = ri->pDepthAttachment ? tail_copy(&at, ri->pDepthAttachment, one) : NULL; + a->pStencilAttachment = ri->pStencilAttachment ? tail_copy(&at, ri->pStencilAttachment, one) : NULL; + pub(); +} + +// ---- extension commands through function pointers + +typedef struct { PFN_vkCmdPushDescriptorSetWithTemplateKHR fn; VkDescriptorUpdateTemplate tpl; VkPipelineLayout l; + uint32_t set; const void *data; } A_PushTpl; +void rec_push_tpl(PFN_vkCmdPushDescriptorSetWithTemplateKHR fn, VkCommandBuffer cb, VkDescriptorUpdateTemplate tpl, + VkPipelineLayout layout, uint32_t set, const void *data, uint32_t bytes) { + if (!OWNED(cb)) { fn(cb, tpl, layout, set, data); return; } + A_PushTpl *a = put(R_PushTpl, sizeof *a + bytes); + uint8_t *at = (uint8_t *)(a + 1); + *a = (A_PushTpl){fn, tpl, layout, set, NULL}; + a->data = tail_copy(&at, data, bytes); + pub(); +} + +typedef struct { PFN_vkCmdPushDescriptorSetKHR fn; VkPipelineBindPoint bp; VkPipelineLayout l; uint32_t set, n; + VkWriteDescriptorSet *w; } A_Push; +void rec_push(PFN_vkCmdPushDescriptorSetKHR fn, VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipelineLayout layout, + uint32_t set, uint32_t n, const VkWriteDescriptorSet *w) { + if (!OWNED(cb)) { fn(cb, bp, layout, set, n, w); return; } + size_t bytes = A8(sizeof *w * n); + int chained = 0; + for (uint32_t i = 0; i < n; i++) { + chained |= w[i].pNext != NULL; + if (w[i].pBufferInfo) bytes += A8(sizeof *w[i].pBufferInfo * w[i].descriptorCount); + if (w[i].pImageInfo) bytes += A8(sizeof *w[i].pImageInfo * w[i].descriptorCount); + if (w[i].pTexelBufferView) bytes += A8(sizeof *w[i].pTexelBufferView * w[i].descriptorCount); + } + if (chained) { FALLBACK(); fn(cb, bp, layout, set, n, w); return; } + A_Push *a = put(R_Push, sizeof *a + bytes); + uint8_t *at = (uint8_t *)(a + 1); + *a = (A_Push){fn, bp, layout, set, n, NULL}; + a->w = tail_copy(&at, w, sizeof *w * n); + for (uint32_t i = 0; i < n; i++) { + uint32_t k = w[i].descriptorCount; + a->w[i].pBufferInfo = tail_copy(&at, w[i].pBufferInfo, w[i].pBufferInfo ? sizeof *w[i].pBufferInfo * k : 0); + a->w[i].pImageInfo = tail_copy(&at, w[i].pImageInfo, w[i].pImageInfo ? sizeof *w[i].pImageInfo * k : 0); + a->w[i].pTexelBufferView = + tail_copy(&at, w[i].pTexelBufferView, w[i].pTexelBufferView ? sizeof *w[i].pTexelBufferView * k : 0); + } + pub(); +} + +typedef struct { PFN_vkCmdSetVertexInputEXT fn; uint32_t nb, na; const VkVertexInputBindingDescription2EXT *b; + const VkVertexInputAttributeDescription2EXT *v; } A_VertexInput; +void rec_vertex_input(PFN_vkCmdSetVertexInputEXT fn, VkCommandBuffer cb, uint32_t nb, + const VkVertexInputBindingDescription2EXT *b, uint32_t na, + const VkVertexInputAttributeDescription2EXT *v) { + if (!OWNED(cb)) { fn(cb, nb, b, na, v); return; } + A_VertexInput *a = put(R_VertexInput, sizeof *a + A8(sizeof *b * nb) + A8(sizeof *v * na)); + uint8_t *at = (uint8_t *)(a + 1); + *a = (A_VertexInput){fn, nb, na, NULL, NULL}; + a->b = tail_copy(&at, b, sizeof *b * nb), a->v = tail_copy(&at, v, sizeof *v * na); // pNext: always NULL there + pub(); +} + +typedef struct { PFN_vkCmdSetDepthClampEnableEXT fn; VkBool32 v; } A_Bool; +void rec_bool(PFN_vkCmdSetDepthClampEnableEXT fn, VkCommandBuffer cb, VkBool32 v) { + if (!OWNED(cb)) { fn(cb, v); return; } + A_Bool *a = put(R_Bool, sizeof *a); + *a = (A_Bool){fn, v}; + pub(); +} + +typedef struct { PFN_vkCmdSetPolygonModeEXT fn; VkPolygonMode m; } A_PolygonMode; +void rec_polygon_mode(PFN_vkCmdSetPolygonModeEXT fn, VkCommandBuffer cb, VkPolygonMode m) { + if (!OWNED(cb)) { fn(cb, m); return; } + A_PolygonMode *a = put(R_PolygonMode, sizeof *a); + *a = (A_PolygonMode){fn, m}; + pub(); +} + +typedef struct { void *fn; uint32_t first, n; const void *v; } A_Targets; +#define TARGET_CMD(fname, R, PFN, T) \ + void fname(PFN fn, VkCommandBuffer cb, uint32_t first, uint32_t n, const T *v) { \ + if (!OWNED(cb)) { fn(cb, first, n, v); return; } \ + A_Targets *a = put(R, sizeof *a + sizeof *v * n); \ + uint8_t *at = (uint8_t *)(a + 1); \ + *a = (A_Targets){(void *)fn, first, n, NULL}; \ + a->v = tail_copy(&at, v, sizeof *v * n); \ + pub(); \ + } +TARGET_CMD(rec_blend_enable, R_BlendEnable, PFN_vkCmdSetColorBlendEnableEXT, VkBool32) +TARGET_CMD(rec_blend_equation, R_BlendEquation, PFN_vkCmdSetColorBlendEquationEXT, VkColorBlendEquationEXT) +TARGET_CMD(rec_write_mask, R_WriteMask, PFN_vkCmdSetColorWriteMaskEXT, VkColorComponentFlags) + +// ---- the recorder's side + +static void exec_packet(uint32_t op, void *p) { + VkCommandBuffer cb = rec_cb; + switch (op) { +#define F(name, ...) case R_##name: { A_##name *a = p; vkCmd##name(cb, __VA_ARGS__); break; } + F(BeginQuery, a->p, a->q, a->f) + F(BindIndexBuffer, a->b, a->o, a->t) + F(BindPipeline, a->bp, a->p) + F(Dispatch, a->x, a->y, a->z) + F(DispatchIndirect, a->b, a->o) + F(Draw, a->n, a->i, a->f, a->fi) + F(DrawIndexed, a->n, a->i, a->f, a->v, a->fi) + F(DrawIndexedIndirect, a->b, a->o, a->n, a->s) + F(EndQuery, a->p, a->q) + F(FillBuffer, a->b, a->o, a->n, a->v) + F(ResetQueryPool, a->p, a->f, a->n) + F(SetBlendConstants, a->c) + F(SetCullMode, a->v) + F(SetDepthBias, a->c, a->cl, a->s) + F(SetDepthBiasEnable, a->v) + F(SetDepthCompareOp, a->v) + F(SetDepthTestEnable, a->v) + F(SetDepthWriteEnable, a->v) + F(SetFrontFace, a->v) + F(SetPrimitiveRestartEnable, a->v) + F(SetPrimitiveTopology, a->v) + F(SetRasterizerDiscardEnable, a->v) + F(SetStencilCompareMask, a->f, a->v) + F(SetStencilOp, a->f, a->x, a->y, a->z, a->d) + F(SetStencilReference, a->f, a->v) + F(SetStencilTestEnable, a->v) + F(SetStencilWriteMask, a->f, a->v) + F(WriteTimestamp, a->s, a->p, a->q) + F(WriteTimestamp2, a->s, a->p, a->q) + F(SetViewportWithCount, a->n, a->v) + F(SetScissorWithCount, a->n, a->r) + F(BindDescriptorSets, a->bp, a->l, a->first, a->n, a->s, a->nd, a->d) + F(BindVertexBuffers2, a->first, a->n, a->b, a->o, a->sz, a->st) + F(ClearAttachments, a->na, a->a, a->nr, a->r) + case R_ClearColorImage: { A_ClearImage *a = p; vkCmdClearColorImage(cb, a->i, a->l, &a->v.c, a->n, a->r); break; } + case R_ClearDepthStencilImage: { + A_ClearImage *a = p; + vkCmdClearDepthStencilImage(cb, a->i, a->l, &a->v.ds, a->n, a->r); + break; + } + F(PushConstants, a->l, a->st, a->o, a->n, a->v) +#undef F + case R_EndRendering: vkCmdEndRendering(cb); break; + case R_BlitImage: { A_Img *a = p; vkCmdBlitImage(cb, a->s, a->sl, a->d, a->dl, a->n, a->r, a->f); break; } + case R_CopyImage: { A_Img *a = p; vkCmdCopyImage(cb, a->s, a->sl, a->d, a->dl, a->n, a->r); break; } + case R_ResolveImage: { A_Img *a = p; vkCmdResolveImage(cb, a->s, a->sl, a->d, a->dl, a->n, a->r); break; } + case R_CopyBuffer: { A_Xfer *a = p; vkCmdCopyBuffer(cb, (VkBuffer)a->s, (VkBuffer)a->d, a->n, a->r); break; } + case R_CopyBufferToImage: { + A_Xfer *a = p; + vkCmdCopyBufferToImage(cb, (VkBuffer)a->s, (VkImage)a->d, a->l, a->n, a->r); + break; + } + case R_CopyImageToBuffer: { + A_Xfer *a = p; + vkCmdCopyImageToBuffer(cb, (VkImage)a->s, a->l, (VkBuffer)a->d, a->n, a->r); + break; + } + case R_PipelineBarrier2: vkCmdPipelineBarrier2(cb, p); break; + case R_BeginRendering: vkCmdBeginRendering(cb, p); break; + case R_PushTpl: { A_PushTpl *a = p; a->fn(cb, a->tpl, a->l, a->set, a->data); break; } + case R_Push: { A_Push *a = p; a->fn(cb, a->bp, a->l, a->set, a->n, a->w); break; } + case R_VertexInput: { A_VertexInput *a = p; a->fn(cb, a->nb, a->b, a->na, a->v); break; } + case R_Bool: { A_Bool *a = p; a->fn(cb, a->v); break; } + case R_PolygonMode: { A_PolygonMode *a = p; a->fn(cb, a->m); break; } + case R_BlendEnable: { A_Targets *a = p; ((PFN_vkCmdSetColorBlendEnableEXT)a->fn)(cb, a->first, a->n, a->v); break; } + case R_BlendEquation: { A_Targets *a = p; ((PFN_vkCmdSetColorBlendEquationEXT)a->fn)(cb, a->first, a->n, a->v); break; } + case R_WriteMask: { A_Targets *a = p; ((PFN_vkCmdSetColorWriteMaskEXT)a->fn)(cb, a->first, a->n, a->v); break; } + default: rt_fatal("vk_rec: unknown packet %u", op); + } +} diff --git a/runtime/vk_rec.h b/runtime/vk_rec.h new file mode 100644 index 00000000..33e5f15c --- /dev/null +++ b/runtime/vk_rec.h @@ -0,0 +1,149 @@ +// The recorder thread (vk_rec.c): the Vulkan calls the replay makes into its command buffer are made by a thread of +// their own. vk_sync.c brackets each part of a submission with rec_begin (after vkBeginCommandBuffer) and rec_end +// (before vkEndCommandBuffer). In between, every vkCmd* on that command buffer copies its arguments into a packet of a +// ring, in call order, and the recorder makes the real call in that order; calls into any other command buffer (uploads, +// the window, probes) are made at once. A call the ring has no packet for (a structure with a pNext chain) waits for +// the recorder to empty the ring and is then made at once on the calling thread. rec_end returns once the recorder made +// every call, so ending, submitting and the command pool stay on the submitting thread. +// +// Every vkCmd* the runtime calls through vk.h goes through the rc_* wrappers below (the macros at the end); the +// extension commands called through function pointers have rec_* wrappers of their own. TOTK_VK_RECORDER=0 makes every +// call at once (the same calls in the same order), =ab switches it with the A/B blocks (rt_ab), per submission part. +// +// Design from Savage Wind Recomp #189 and LAS Recomp runtime/nvn/vulkan/record.c (bb52932, c2bdc17, 76f4f21; MIT, +// Copyright (c) 2026 Peiikai): the driver's work behind the commands depends on nothing the replay decides next. +#pragma once + +void rec_begin(VkCommandBuffer cb); // the submitting thread, sub_lock held +void rec_end(void); // waits for the recorder; no-op when rec_begin did not start it +void rec_drain(void); // waits until the recorder made every call queued so far +void rec_stats_log(void); // TOTK_TRACE=perf: packets, bytes, drains and waits since the last call + +// Extension commands called through device function pointers. +void rec_push_tpl(PFN_vkCmdPushDescriptorSetWithTemplateKHR fn, VkCommandBuffer cb, VkDescriptorUpdateTemplate tpl, + VkPipelineLayout layout, uint32_t set, const void *data, uint32_t bytes); // data: bytes read by tpl +void rec_push(PFN_vkCmdPushDescriptorSetKHR fn, VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipelineLayout layout, + uint32_t set, uint32_t n, const VkWriteDescriptorSet *w); +void rec_vertex_input(PFN_vkCmdSetVertexInputEXT fn, VkCommandBuffer cb, uint32_t nb, + const VkVertexInputBindingDescription2EXT *b, uint32_t na, + const VkVertexInputAttributeDescription2EXT *a); +void rec_bool(PFN_vkCmdSetDepthClampEnableEXT fn, VkCommandBuffer cb, VkBool32 v); // also alpha to coverage +void rec_polygon_mode(PFN_vkCmdSetPolygonModeEXT fn, VkCommandBuffer cb, VkPolygonMode m); +void rec_blend_enable(PFN_vkCmdSetColorBlendEnableEXT fn, VkCommandBuffer cb, uint32_t first, uint32_t n, const VkBool32 *v); +void rec_blend_equation(PFN_vkCmdSetColorBlendEquationEXT fn, VkCommandBuffer cb, uint32_t first, uint32_t n, + const VkColorBlendEquationEXT *v); +void rec_write_mask(PFN_vkCmdSetColorWriteMaskEXT fn, VkCommandBuffer cb, uint32_t first, uint32_t n, + const VkColorComponentFlags *v); + +#define REC_CMDS(X) \ + X(BeginQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q, VkQueryControlFlags f), (cb, p, q, f)) \ + X(BeginRendering, (VkCommandBuffer cb, const VkRenderingInfo *ri), (cb, ri)) \ + X(BindDescriptorSets, (VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipelineLayout l, uint32_t first, uint32_t n, \ + const VkDescriptorSet *s, uint32_t nd, const uint32_t *d), (cb, bp, l, first, n, s, nd, d)) \ + X(BindIndexBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkIndexType t), (cb, b, o, t)) \ + X(BindPipeline, (VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipeline p), (cb, bp, p)) \ + X(BindVertexBuffers2, (VkCommandBuffer cb, uint32_t first, uint32_t n, const VkBuffer *b, const VkDeviceSize *o, \ + const VkDeviceSize *sz, const VkDeviceSize *st), (cb, first, n, b, o, sz, st)) \ + X(BlitImage, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, uint32_t n, \ + const VkImageBlit *r, VkFilter f), (cb, s, sl, d, dl, n, r, f)) \ + X(ClearAttachments, (VkCommandBuffer cb, uint32_t na, const VkClearAttachment *a, uint32_t nr, const VkClearRect *r), \ + (cb, na, a, nr, r)) \ + X(ClearColorImage, (VkCommandBuffer cb, VkImage i, VkImageLayout l, const VkClearColorValue *c, uint32_t n, \ + const VkImageSubresourceRange *r), (cb, i, l, c, n, r)) \ + X(ClearDepthStencilImage, (VkCommandBuffer cb, VkImage i, VkImageLayout l, const VkClearDepthStencilValue *c, \ + uint32_t n, const VkImageSubresourceRange *r), (cb, i, l, c, n, r)) \ + X(CopyBuffer, (VkCommandBuffer cb, VkBuffer s, VkBuffer d, uint32_t n, const VkBufferCopy *r), (cb, s, d, n, r)) \ + X(CopyBufferToImage, (VkCommandBuffer cb, VkBuffer s, VkImage d, VkImageLayout dl, uint32_t n, \ + const VkBufferImageCopy *r), (cb, s, d, dl, n, r)) \ + X(CopyImage, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, uint32_t n, \ + const VkImageCopy *r), (cb, s, sl, d, dl, n, r)) \ + X(CopyImageToBuffer, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkBuffer d, uint32_t n, \ + const VkBufferImageCopy *r), (cb, s, sl, d, n, r)) \ + X(Dispatch, (VkCommandBuffer cb, uint32_t x, uint32_t y, uint32_t z), (cb, x, y, z)) \ + X(DispatchIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o), (cb, b, o)) \ + X(Draw, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, uint32_t fi), (cb, n, i, f, fi)) \ + X(DrawIndexed, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, int32_t v, uint32_t fi), (cb, n, i, f, v, fi)) \ + X(DrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s)) \ + X(EndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), (cb, p, q)) \ + X(EndRendering, (VkCommandBuffer cb), (cb)) \ + X(FillBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkDeviceSize n, uint32_t v), (cb, b, o, n, v)) \ + X(PipelineBarrier2, (VkCommandBuffer cb, const VkDependencyInfo *di), (cb, di)) \ + X(PushConstants, (VkCommandBuffer cb, VkPipelineLayout l, VkShaderStageFlags st, uint32_t o, uint32_t n, \ + const void *v), (cb, l, st, o, n, v)) \ + X(ResetQueryPool, (VkCommandBuffer cb, VkQueryPool p, uint32_t f, uint32_t n), (cb, p, f, n)) \ + X(ResolveImage, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, uint32_t n, \ + const VkImageResolve *r), (cb, s, sl, d, dl, n, r)) \ + X(SetBlendConstants, (VkCommandBuffer cb, const float c[4]), (cb, c)) \ + X(SetCullMode, (VkCommandBuffer cb, VkCullModeFlags m), (cb, m)) \ + X(SetDepthBias, (VkCommandBuffer cb, float c, float cl, float s), (cb, c, cl, s)) \ + X(SetDepthBiasEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v)) \ + X(SetDepthCompareOp, (VkCommandBuffer cb, VkCompareOp v), (cb, v)) \ + X(SetDepthTestEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v)) \ + X(SetDepthWriteEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v)) \ + X(SetFrontFace, (VkCommandBuffer cb, VkFrontFace v), (cb, v)) \ + X(SetPrimitiveRestartEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v)) \ + X(SetPrimitiveTopology, (VkCommandBuffer cb, VkPrimitiveTopology v), (cb, v)) \ + X(SetRasterizerDiscardEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v)) \ + X(SetScissorWithCount, (VkCommandBuffer cb, uint32_t n, const VkRect2D *r), (cb, n, r)) \ + X(SetStencilCompareMask, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), (cb, f, v)) \ + X(SetStencilOp, (VkCommandBuffer cb, VkStencilFaceFlags f, VkStencilOp a, VkStencilOp b, VkStencilOp c, \ + VkCompareOp d), (cb, f, a, b, c, d)) \ + X(SetStencilReference, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), (cb, f, v)) \ + X(SetStencilTestEnable, (VkCommandBuffer cb, VkBool32 v), (cb, v)) \ + X(SetStencilWriteMask, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), (cb, f, v)) \ + X(SetViewportWithCount, (VkCommandBuffer cb, uint32_t n, const VkViewport *v), (cb, n, v)) \ + X(WriteTimestamp, (VkCommandBuffer cb, VkPipelineStageFlagBits s, VkQueryPool p, uint32_t q), (cb, s, p, q)) \ + X(WriteTimestamp2, (VkCommandBuffer cb, VkPipelineStageFlags2 s, VkQueryPool p, uint32_t q), (cb, s, p, q)) + +#define REC_DECL(name, params, args) void rc_vkCmd##name params; +REC_CMDS(REC_DECL) +#undef REC_DECL + +#ifndef VK_REC_IMPL +#define vkCmdBeginQuery rc_vkCmdBeginQuery +#define vkCmdBeginRendering rc_vkCmdBeginRendering +#define vkCmdBindDescriptorSets rc_vkCmdBindDescriptorSets +#define vkCmdBindIndexBuffer rc_vkCmdBindIndexBuffer +#define vkCmdBindPipeline rc_vkCmdBindPipeline +#define vkCmdBindVertexBuffers2 rc_vkCmdBindVertexBuffers2 +#define vkCmdBlitImage rc_vkCmdBlitImage +#define vkCmdClearAttachments rc_vkCmdClearAttachments +#define vkCmdClearColorImage rc_vkCmdClearColorImage +#define vkCmdClearDepthStencilImage rc_vkCmdClearDepthStencilImage +#define vkCmdCopyBuffer rc_vkCmdCopyBuffer +#define vkCmdCopyBufferToImage rc_vkCmdCopyBufferToImage +#define vkCmdCopyImage rc_vkCmdCopyImage +#define vkCmdCopyImageToBuffer rc_vkCmdCopyImageToBuffer +#define vkCmdDispatch rc_vkCmdDispatch +#define vkCmdDispatchIndirect rc_vkCmdDispatchIndirect +#define vkCmdDraw rc_vkCmdDraw +#define vkCmdDrawIndexed rc_vkCmdDrawIndexed +#define vkCmdDrawIndexedIndirect rc_vkCmdDrawIndexedIndirect +#define vkCmdEndQuery rc_vkCmdEndQuery +#define vkCmdEndRendering rc_vkCmdEndRendering +#define vkCmdFillBuffer rc_vkCmdFillBuffer +#define vkCmdPipelineBarrier2 rc_vkCmdPipelineBarrier2 +#define vkCmdPushConstants rc_vkCmdPushConstants +#define vkCmdResetQueryPool rc_vkCmdResetQueryPool +#define vkCmdResolveImage rc_vkCmdResolveImage +#define vkCmdSetBlendConstants rc_vkCmdSetBlendConstants +#define vkCmdSetCullMode rc_vkCmdSetCullMode +#define vkCmdSetDepthBias rc_vkCmdSetDepthBias +#define vkCmdSetDepthBiasEnable rc_vkCmdSetDepthBiasEnable +#define vkCmdSetDepthCompareOp rc_vkCmdSetDepthCompareOp +#define vkCmdSetDepthTestEnable rc_vkCmdSetDepthTestEnable +#define vkCmdSetDepthWriteEnable rc_vkCmdSetDepthWriteEnable +#define vkCmdSetFrontFace rc_vkCmdSetFrontFace +#define vkCmdSetPrimitiveRestartEnable rc_vkCmdSetPrimitiveRestartEnable +#define vkCmdSetPrimitiveTopology rc_vkCmdSetPrimitiveTopology +#define vkCmdSetRasterizerDiscardEnable rc_vkCmdSetRasterizerDiscardEnable +#define vkCmdSetScissorWithCount rc_vkCmdSetScissorWithCount +#define vkCmdSetStencilCompareMask rc_vkCmdSetStencilCompareMask +#define vkCmdSetStencilOp rc_vkCmdSetStencilOp +#define vkCmdSetStencilReference rc_vkCmdSetStencilReference +#define vkCmdSetStencilTestEnable rc_vkCmdSetStencilTestEnable +#define vkCmdSetStencilWriteMask rc_vkCmdSetStencilWriteMask +#define vkCmdSetViewportWithCount rc_vkCmdSetViewportWithCount +#define vkCmdWriteTimestamp rc_vkCmdWriteTimestamp +#define vkCmdWriteTimestamp2 rc_vkCmdWriteTimestamp2 +#endif diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 3cb0be2d..f89ac1a7 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -290,6 +290,7 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { VkCommandBuffer vk_sync_split(void) { VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; + rec_end(); // every command of the part is in cb ts_end(cb, cur_fence); vkEndCommandBuffer(cb); vk_upload_flush(0); // the uploads this part asked for precede it in queue order @@ -308,6 +309,7 @@ VkCommandBuffer vk_sync_split(void) { .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(s->cb, &bi); ts_begin(s->cb); + rec_begin(s->cb); return s->cb; } @@ -335,8 +337,10 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { uint64_t t0 = perf_begin(); vkBeginCommandBuffer(s->cb, &bi); ts_begin(s->cb); + rec_begin(s->cb); int recorded = vk_exec_record(s->cb, handles, n, fence); s = &slots[cur_slot]; // the last part (vk_sync_split) + rec_end(); if (recorded) ts_end(s->cb, fence); else ts_drop(); vkEndCommandBuffer(s->cb); @@ -363,6 +367,8 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); WakeAllConditionVariable(&cv_work); ReleaseSRWLockExclusive(&st_lock); + static unsigned subs; + if (++subs % 4096 == 0) rec_stats_log(); ReleaseSRWLockExclusive(&sub_lock); return fence; }