diff --git a/docs/architecture.md b/docs/architecture.md index 12f75987..00289d1f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,7 +82,7 @@ Every guest thread is a host thread. TotK assumes Horizon's scheduler: three app - Vulkan 1.3 back end in place of MacNativeGames' Metal one. At `QueueSubmitCommands` the recorded NVN commands are replayed into Vulkan command buffers (`vk_exec*.c`). - Memory pools. Apple's unified memory let MacNativeGames use the game's pools zero-copy. The back end imports the pools as host memory (`VK_EXT_external_memory_host`). On a discrete GPU, pools bound as buffers are mirrored in device-local memory and the pages the CPU wrote (write watch, above) are uploaded before a replay; idle pool regions are made read-only so the write-watch queries skip them (`vk_memory.c`). GPU-rendered textures are written back when the CPU reads them: on a pool invalidate, on a CPU copy, and for small targets once the CPU waits for a fence (`vk_texture.c`). - Replay thread. `TOTK_REPLAY_ASYNC=1` moves the replay off the submitting thread onto a queue at most one frame behind the game; reads of GPU results wait for the newest writer instead of draining the queue (`vk_backend.c`). It is off by default because no timing has shown a gain yet. -- Recorder thread. The Vulkan calls the replay makes into its command buffer are made by a thread of their own (`vk_rec.c`, after Savage Wind Recomp and LAS Recomp): each `vkCmd*` copies its arguments into a packet of a ring and the recorder makes the calls in order; calls into other command buffers are made at once; the recorder also ends the command buffer, and the submit waits until it has. In the town on an RX 9070 the replay takes 17% less time on the submitting thread. `TOTK_VK_RECORDER=0` makes every call on the submitting thread. +- Recorder thread. The Vulkan calls the replay makes into its command buffer are made by a thread of their own (`vk_rec.c`, after Savage Wind Recomp and LAS Recomp): each `vkCmd*` copies its arguments into a packet of a ring and the recorder makes the calls in order; calls into other command buffers are made at once; the recorder also ends the command buffer, and the submit waits until it has. In the town on an RX 9070 the replay takes 17% less time on the submitting thread. It is off by default on AMD GPUs, where the driver loses the device or hangs the GPU at startup with it (`TOTK_VK_RECORDER=1` forces it on, `=0` makes every call on the submitting thread). - Uniform snapshot. The game rewrites per-object uniform blocks on other threads soon after it submits the frame that reads them, while the replay still records it. At the start of a submission's replay every uniform range it binds from CPU-written pools is copied, and the draws bind the copy (`vk_usnap.c`, after MacNativeGames' diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index eb898e33..969139c2 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -127,10 +127,16 @@ void rec_drain(void) { void rec_begin(VkCommandBuffer cb) { static int mode = -1; if (mode < 0) { - // Default on for every GPU. AMD drivers once lost the device with the recorder (issues #5 and #40); after - // vk_mem_note_recording kept objects alive until their submission completed and the recorder began ending - // its own parts (rec_end_cb), RDNA 1 and 2 cards ran with it on and off without a loss or a white scene. - mode = rt_knob("VK_RECORDER", "1"); + // Default off on AMD GPUs. With the recorder those drivers lost the device or hung the GPU at startup on + // RDNA 2, 3 and 4, under AMD's Windows driver and under RADV (issues #5, #40, #277 and #283: a black screen, + // or VK_ERROR_DEVICE_LOST with a breadcrumb naming a part of a replay submission, and vk_recorder = 0 gets + // past it). Keeping objects until their submission completed and ending a part from the recorder + // (rec_end_cb) fixed the white scene on an RX 6700 XT, but not this: 0.2.8 turned the recorder on for every + // GPU and the AMD reports came back the same day. TOTK_VK_RECORDER=1 turns it on for an A/B run. + const char *set = cfg_get("TOTK_VK_RECORDER"); + mode = rt_knob("VK_RECORDER", vk_caps.amd ? "0" : "1"); + if (!mode && vk_caps.amd && !(set && *set)) + rt_log("vk_rec: no recorder thread on AMD GPUs (TOTK_VK_RECORDER=1 turns it on)"); if (mode) { plat_event_init(&wake_ev, 0, 0); rec_prio = plat_thread_priority_get(); @@ -162,11 +168,14 @@ void rec_end(void) { // recorded and the submitting thread ended broke when a submission was split (vk_sync.c vk_sync_split): the GPU hung // or the scene turned green, then white. The handoff in rec_drain orders the two threads, so Vulkan allows the old // split; the driver did not cope with it. TOTK_REC_END=0 brings the old split back. +// The line is written only when a part was recorded by the recorder: with the recorder off (the default on AMD) the +// switch changes nothing, and reports read an unconditional line as proof that the recorder was running (#315). typedef struct { int kind; uint64_t n; } A_EndCb; int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n) { - static int mode = -1; - if (mode < 0) { - mode = rt_knob("REC_END", "1"); + static int mode = -1, said; + if (mode < 0) mode = rt_knob("REC_END", "1"); + if (rec_live && cb == rec_cb && !said) { + said = 1; rt_log("vk_rec: a part the recorder records is ended by %s (TOTK_REC_END=%d)", mode ? "the recorder" : "the submitting thread", mode); } diff --git a/totk.ini.example b/totk.ini.example index 6d3d5f1e..cf214f16 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -89,7 +89,7 @@ # Vulkan. # Vulkan validation layer: checks every call and slows the game down a lot. Needs the layer installed (Vulkan SDK). # vk_validation = 0 -# A thread of its own makes the Vulkan calls of each frame, 0 or 1. Default: on. +# A thread of its own makes the Vulkan calls of each frame, 0 or 1. Default: on, except on AMD GPUs. # vk_recorder = 1 # Dynamic vertex input, 0 or 1. Default: on, except on Intel's older integrated GPU drivers (Vulkan 1.3.215 and older). # vk_dyn_vertex = 1