diff --git a/docs/agents/running.md b/docs/agents/running.md index 926a09b4..a927be69 100644 --- a/docs/agents/running.md +++ b/docs/agents/running.md @@ -53,6 +53,7 @@ scene you expected. Title menus on a slow boot eat presses, and runs of one scri | every render pass of one frame | `TOTK_DUMP=` and `TOTK_DUMP_DIR`: a PNG per pass target plus `passes.txt` (draws, bound textures with swizzles, storage buffers, copies, clears) and `images.txt`; `TOTK_DUMP_BUF` adds a guest buffer | | Vulkan misuse | `TOTK_VK_VALIDATION=1` (needs the Vulkan SDK), run into the world; compare the message kinds with a run without your change. The layer caps repeats, so counts are not totals | | which pass the GPU was in at a device lost | the `vk: pass marks` lines; on by default on AMD, `TOTK_PASS_CRUMBS=1` elsewhere | +| which object held the address of a GPU memory fault | `TOTK_ADDR_REPORT=1` (off by default: it slows recording, on AMD's Windows driver more and more over a session); the `vk: fault address` lines after the device lost | | a crash | the crash report at the end of the log: exception, guest and host addresses, registers | | a run on a large card that acts like a small one | `TOTK_VRAM_LIMIT_MIB=` | diff --git a/docs/tester-guide.md b/docs/tester-guide.md index 2ddd4ac9..81198acd 100644 --- a/docs/tester-guide.md +++ b/docs/tester-guide.md @@ -42,6 +42,8 @@ do one run with it and one without (`set TOTK_VK_VALIDATION=`). If the game stops with a lost device (the log says `device lost` or `VK_ERROR_DEVICE_LOST`), the lines starting with `vk: pass marks` name the render pass or dispatch the GPU was running. AMD GPUs write them by default. On an NVIDIA GPU, `set TOTK_PASS_CRUMBS=1` (or GPU pass marks: On in the Advanced tab) before the run that reproduces the crash. +`set TOTK_ADDR_REPORT=1` adds `vk: fault address` lines naming the image or buffer at the faulting address; it slows +the game down over a session, so use it only for that run. The game keeps running behind other windows (`assume_focus = 0` in `totk.ini` pauses it instead). Shaders the preparation did not cover compile on first use, so the first minutes of play can stutter. diff --git a/runtime/cfg.c b/runtime/cfg.c index 12a63007..7e42c4c6 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -32,7 +32,7 @@ static int save_ok = 1; // the last cfg_save_to wrote (o // logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line, and gets a // description in totk.ini.example or docs/settings.md (tests/settings_doc_check.py checks both). static const char *const known[] = { - "AB", "ABILITY_GUARD", "AMIIBO_DIR", "ANISO", "APP_POOL", "ASPECT", "ASPECT_PATCH", "ASSUME_FOCUS", "ASTC", "ASTC_BC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_CHANNELS", "AUDIO_DEVICE", "AUDIO_OFF", "AUDIO_PREFILL_MS", + "AB", "ABILITY_GUARD", "ADDR_REPORT", "AMIIBO_DIR", "ANISO", "APP_POOL", "ASPECT", "ASPECT_PATCH", "ASSUME_FOCUS", "ASTC", "ASTC_BC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_CHANNELS", "AUDIO_DEVICE", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", "BIND_LSTICK_DOWN", "BIND_LSTICK_LEFT", "BIND_LSTICK_RIGHT", "BIND_LSTICK_UP", "BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_RSTICK_DOWN", "BIND_RSTICK_LEFT", "BIND_RSTICK_RIGHT", "BIND_RSTICK_UP", @@ -113,7 +113,7 @@ static const struct { const char *key, *section; } sections[] = { {"SYNC_AHEAD", "performance"}, {"SHADOW_DEFER", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"OPTREF_GUARD", "performance"}, {"ABILITY_GUARD", "performance"}, {"REPORT_GUARD", "performance"}, - {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, + {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"ADDR_REPORT", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"GAMEPLAY_TWEAKS_ENABLED", "gameplay tweaks"}, {"GAMEPLAY_ULTRAHAND_ROTATE", "gameplay tweaks"}, @@ -231,6 +231,7 @@ static const CfgMeta metas[] = { {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only + {"ADDR_REPORT", "0", "0|1", RESTART}, // vk_device.c GPU address map for device faults {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 0db73ba1..549b9f5a 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -22,7 +22,10 @@ static PlatMutex g_lock = PLAT_MUTEX_INIT; // the debug messenger. The newest AB_N events stay in a ring written without a lock (one atomic add each), so a device // fault names the object behind its address and whether it was unbound before the fault, which is memory freed while // a submission still used it. One event per image, buffer or memory block made or freed, none per draw. -// TOTK_ADDR_REPORT=0 turns it off. +// Off by default; TOTK_ADDR_REPORT=1 turns it on for a run that chases a device fault. Our callback is cheap, but with +// the feature on AMD's Windows driver (26.10.1) tracks every bound range in a chained hash map it searches on nearly +// every recorded command: the chains grow as textures stream in and out, and recording a submission cost 3-4 times +// more after minutes of play in a busy area (#1174). #define AB_N (1u << 15) typedef struct { uint64_t seq, base, size, handle, issued, done; int32_t type; uint32_t unbind; } AbEvent; static AbEvent *ab_ring; @@ -137,8 +140,7 @@ static int create_instance(void) { rt_log("vk: TOTK_VK_VALIDATION set but %s is not installed; continuing without", layers[0]); validate = 0; } - const char *ar = cfg_get("TOTK_ADDR_REPORT"); - ab_want = !(ar && !strcmp(ar, "0")) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + ab_want = cfg_bool("TOTK_ADDR_REPORT") && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); if (ab_want && !ab_ring && !(ab_ring = calloc(AB_N, sizeof *ab_ring))) ab_want = 0; if (validate || ab_want) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; if (vk_hdr_wanted() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats @@ -405,8 +407,8 @@ static int create_device(void) { // wherever reported; vk_pipeline.c spirv_demote emits OpDemoteToHelperInvocation only when it is. vk_caps.demote = c->f13.shaderDemoteToHelperInvocation; rt_log("vk: GPU address map for device faults: %s", - has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report; TOTK_ADDR_REPORT=0 turns it off)" - : !ab_want ? "off (TOTK_ADDR_REPORT=0 or no VK_EXT_debug_utils)" + has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report, TOTK_ADDR_REPORT=1)" + : !ab_want ? "off (TOTK_ADDR_REPORT=1 turns it on; it slows recording on some drivers)" : "unavailable (no VK_EXT_device_address_binding_report)"); g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; // VkPresentIdKHR on a present needs the presentId feature, not just the extension diff --git a/totk.ini.example b/totk.ini.example index abb8db1d..8395fb2f 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -516,6 +516,11 @@ # last one it finished. Unset: on AMD GPUs (VK_AMD_buffer_marker). 1: also on NVIDIA GPUs # (VK_NV_device_diagnostic_checkpoints), which NVIDIA says costs a little. 0: off. # pass_crumbs = 1 +# GPU address map: after a GPU memory fault (device lost) the log names the image or buffer that held the faulting +# address and whether it was freed before the fault. 1: on, where the driver has VK_EXT_device_address_binding_report. +# It slows the recording of GPU commands, on AMD's Windows driver more and more over a session, so turn it on only for +# the run that reproduces the crash. Default: 0. +# addr_report = 0 # Render targets up to this size (bytes, 0 to 1073741824) go back to their game memory whenever the game waits for a # GPU fence. A larger target goes back too unless the GPU drew it in each of the last 16 frames. A game that reads a # larger target drawn every frame without invalidating it sees stale bytes at the default. With trace = gfx the log