diff --git a/docs/building.md b/docs/building.md index 67d3086e..0c881d0c 100644 --- a/docs/building.md +++ b/docs/building.md @@ -116,6 +116,11 @@ of them. Without this step the first launch compiles shaders and pipelines during play and stutters until the caches fill. The AMD driver keeps its pipeline cache per exe path and holds a limited amount: preparing another build folder can evict this one's. `totk.exe --warm-shaders` rebuilds it from the shader cache. + When the driver's shader compiler crashes on a variant (NVIDIA 615.78.08 on an RTX 5080 does, issue #30), + `prepare_shaders.py` runs the driver step again with `--warm-shaders`, up to three times, without the variants the + driver was building at the crash. `shader-prepare-skip.txt` next to the exe lists them with the GPU and driver + version; they build at their first draw instead, another driver version builds them again, and so does deleting the + file. 7. Run. The game keeps running when its window loses the focus (`assume_focus = 1` in `totk.ini`, the default; the Game tab switches it). With 0 it pauses, as it does on the Switch when Home is pressed. diff --git a/docs/installer.md b/docs/installer.md index 546fb891..caaa13dd 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -56,6 +56,9 @@ loads the saves 1.4.3 wrote. writes the translations it makes in play as loose files; running the tool again with the game closed folds those in. The patches are written twice: `romfs-patches` and `romfs-patches-xbox` (`--xbox-prompts`, the A/B and X/Y prompts swapped for `pad_layout = xbox_prompts`). + When the graphics driver crashes while building the shaders, `prepare_shaders.py` runs that part again without the + variants it was building ([building.md](building.md), step 6), and the setup finishes with a warning that names how + many were left out. 6. **Clean up** the temp folder (generated C and objects, about 1.3 GB). A cancelled or failed setup resumes from the last finished step (`totk-install.json`). diff --git a/installer/steps.py b/installer/steps.py index c24cb1d0..5ef4874d 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -644,6 +644,9 @@ class Compile(Step): # ---- 5. prepare TRANSLATED = re.compile(r'gfx_shader: prepare: (\d+) of (\d+) pairs') +# runtime/vk_pipeline.c lb_warm_prepared: variants the driver crashed on in an earlier run (tools/prepare_shaders.py +# runs the driver step again after such a crash) +SKIPPED = re.compile(r'prepare-shaders: (\d+) variants skipped: the graphics driver crashed') # The game resolutions of another shape than 16:9 that runtime/ui/menu.cpp row_output_res lists: 21:9 and 32:9 monitors, @@ -715,10 +718,14 @@ class Prepare(Step): shader_list = ctx.temp / 'shader_list.tsv' shader_list.write_text(shader_list_text(layout), encoding='utf-8', newline='\n') + skipped = [0] + def parse(line): m = TRANSLATED.search(line) if m: rep.progress('prepare', 'translate', int(m.group(1)), int(m.group(2)), 'shaders') + elif SKIPPED.search(line): + skipped[0] = int(SKIPPED.search(line).group(1)) elif 'listed variants' in line: rep.progress('prepare', 'translate', force=True) elif 'gfx_shader: prepare:' in line and 'translated' in line and 'programs' in line: @@ -730,6 +737,8 @@ class Prepare(Step): dest / 'exefs', '--list', shader_list], parse) if code or not (dest / 'shader-prepared.bin').is_file(): raise InstallError('prepare.shaders_failed', code=code, log=str(log)) + if skipped[0]: + rep.warning('warn.shaders_skipped', count=skipped[0], log=str(log)) # One shader-cache.pack instead of four files per shader (~117,000): totk.exe reads the pack first. rep.progress('prepare', 'pack', force=True) diff --git a/installer/text/en.json b/installer/text/en.json index 16625c72..167cdcfa 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -86,6 +86,7 @@ "warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts, which the installer does not extract; some text may not show.", "warn.npdm_differs": "main.npdm differs from the reference {version} dump; only the code modules are used.", "warn.xbox_prompts_failed": "The Xbox button prompts could not be prepared (exit code {code}); the game keeps the Switch prompts. Details: {log}", + "warn.shaders_skipped": "The graphics driver crashed while building some shaders, so {count} were left out of the preparation. The game builds them when it first needs them, and the driver may crash there too; a driver update may fix it. Details: {log}", "warn.kernel_old": "Linux {version} lacks the write tracking the game relies on (6.7 or later); the game may run very slowly. Updating the system fixes this.", "ui.title": "TotK Setup", diff --git a/installer/text/ru.json b/installer/text/ru.json index 3e843b4a..f7778dee 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -86,6 +86,7 @@ "warn.lang_needs_fonts": "Для языка «{lang}» нужны системные шрифты Switch, которые установка не извлекает; часть текста может не отображаться.", "warn.npdm_differs": "main.npdm отличается от эталонного дампа {version}; используются только модули с кодом.", "warn.xbox_prompts_failed": "Не удалось подготовить подсказки кнопок Xbox (код выхода {code}); игра оставит подсказки Switch. Подробности: {log}", + "warn.shaders_skipped": "Графический драйвер аварийно завершился при сборке части шейдеров, поэтому {count} из них пропущены при подготовке. Игра соберёт их, когда они впервые понадобятся, и драйвер может упасть и там; обновление драйвера может это исправить. Подробности: {log}", "warn.kernel_old": "В Linux {version} нет отслеживания записи, на которое опирается игра (нужна версия 6.7 или новее); игра может работать очень медленно. Обновление системы это исправит.", "ui.title": "Установка TotK", diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index feba5764..b0953383 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2569,7 +2569,8 @@ static struct { uint8_t clamp, poly, minus_w, samples; // library state (GKey fields) double t0, ms[3]; // ms: build time per kind (vertex, fragment, compute) // warmed: driver cache only; nolib: vertex and fragment variants skipped with the graphics pipeline library off - uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, done, cs_draw; + // skipped: variants of the skip list (SK below) + uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, skipped, done, cs_draw; double warm_ms[3]; volatile int32_t kept; // libraries entered in libs[] so far int32_t keep_n; @@ -2577,6 +2578,134 @@ static struct { PlatMutex stat; } LB = {.stat = PLAT_MUTEX_INIT}; +// ---- preparation that outlives a driver crash. A bug in the driver's shader compiler can kill the process inside +// vkCreate*Pipelines: NVIDIA 615.78.08 on an RTX 5080 crashed on the same vertex library at the same point of every +// preparation, natively and under Wine, so preparation never finished (issue #30 and a report on PR #60). +// While lb_warm_prepared builds, slot k of /shader-prepare-journal.bin holds the variant its worker k has handed to +// the driver (input id and stage; zeros between variants). The writes go straight to the OS, so they outlast the +// process, and a run that finishes deletes the file. The next preparation or boot (skip_init) moves the variants a +// leftover journal names into /shader-prepare-skip.txt with this GPU and driver version, and preparation and the +// boot prebuild (lb_worker) skip what that file lists for them. The variants the other workers were building at the +// crash are skipped along with the culprit; like it, they build at their first draw, where the driver can still crash. +// Another driver version builds them again, and so does deleting the file. tools/prepare_shaders.py runs the build +// again (--warm-shaders) after a run that left a journal. +#define JR_SLOT 64 // bytes per journal slot: input id, stage, zeros +#define JR_SLOTS 64 // lb_warm_prepared's thread cap + +static struct { + PlatFile *journal; // while lb_warm_prepared builds + uint8_t (*iid)[32]; // the skip list's variants for this GPU and driver + size_t n; + int loaded; +} SK; + +static int cache_dir(char *dir, size_t n) { // the folder of pipeline-cache.bin and shader-prepared.bin + snprintf(dir, n, "%s", P.cache_path); + char *cut = plat_last_sep(dir); + if (!cut) return 0; + *cut = 0; + return 1; +} + +static void gpu_key(char *out, size_t n) { // vendor:device:driver version, as the skip list names a GPU and driver + const VkPhysicalDeviceProperties *p = &vk_ctx()->props; + snprintf(out, n, "%04x:%04x:%08x", p->vendorID, p->deviceID, p->driverVersion); +} + +static const char *stage_name(int st) { + return st == GFX_STAGE_VS ? "vertex" : st == GFX_STAGE_FS ? "fragment" : st == GFX_STAGE_CS ? "compute" : "other"; +} + +static int hex_digit(int c) { + return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; +} + +// A journal a crashed run left into the skip list, then the skip list's variants for this GPU and driver into SK. +static void skip_init(const char *dir) { + if (SK.loaded) return; + SK.loaded = 1; + char jp[PLAT_PATH_MAX * 2 + 48], sp[PLAT_PATH_MAX * 2 + 48], key[32]; + snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir); + snprintf(sp, sizeof sp, "%s/shader-prepare-skip.txt", dir); + gpu_key(key, sizeof key); + uint64_t jn = 0, sn = 0; + uint8_t *j = plat_file_read(jp, &jn); + char *s = (char *)plat_file_read(sp, &sn); + int keep_journal = 0; + if (j) { + static const char head[] = + "# Shader variants the graphics driver was building when the process died during preparation\n" + "# (runtime/vk_pipeline.c skip_init): input id, stage, GPU vendor:device:driver version. Preparation skips\n" + "# them on that GPU and driver. Delete this file to build them again.\n"; + size_t cap = (s ? sn : sizeof head) + (jn / JR_SLOT + 1) * 128, len = 0; + char *out = malloc(cap); + int added = 0; + if (out) { + len = s ? sn : sizeof head - 1; + memcpy(out, s ? s : head, len); + if (len && out[len - 1] != '\n') out[len++] = '\n'; + for (uint64_t o = 0; o + 33 <= jn; o += JR_SLOT) { + int set = 0; + for (int b = 0; b < 32; b++) set |= j[o + b]; + if (!set) continue; + char hex[65]; + for (int b = 0; b < 32; b++) snprintf(hex + 2 * b, 3, "%02x", j[o + b]); + len += (size_t)snprintf(out + len, cap - len, "%s %s %s\n", hex, stage_name(j[o + 32]), key); + added++; + } + out[len] = 0; // parsed below as the list's text + } + if (added && plat_file_write_atomic(sp, out, len)) { + rt_log("vk_pipeline: the last shader preparation died while the driver built %d variants; %s now lists them " + "and preparation skips them on this GPU and driver (%s)", added, sp, key); + free(s), s = out, sn = len, out = NULL; + } else if (added) { + rt_log("vk_pipeline: cannot write %s; %s stays for the next run", sp, jp); + keep_journal = 1; + } + free(out); + free(j); + } + if (!keep_journal) plat_file_delete(jp); + for (char *line = s, *next; line && *line; line = next) { + next = strchr(line, '\n'); + if (next) *next++ = 0; + char hex[65], st[16], k[32]; + uint8_t iid[32]; + int ok = line[0] != '#' && sscanf(line, "%64s %15s %31s", hex, st, k) == 3 && strlen(hex) == 64 && + !strcmp(k, key); + for (int b = 0; ok && b < 32; b++) { + int hi = hex_digit(hex[2 * b]), lo = hex_digit(hex[2 * b + 1]); + if (hi < 0 || lo < 0) ok = 0; + else iid[b] = (uint8_t)(hi << 4 | lo); + } + uint8_t (*grown)[32] = ok ? realloc(SK.iid, (SK.n + 1) * sizeof *SK.iid) : NULL; + if (grown) memcpy(grown[SK.n++], iid, 32), SK.iid = grown; + } + free(s); + if (SK.n) rt_log("vk_pipeline: %zu shader variants skipped ahead of play on this GPU and driver (%s)", SK.n, sp); +} + +static int skip_listed(const uint8_t iid[32]) { + for (size_t i = 0; i < SK.n; i++) + if (!memcmp(SK.iid[i], iid, 32)) return 1; + return 0; +} + +static void skip_count(void) { + plat_mutex_lock(&LB.stat); + LB.skipped++; + plat_mutex_unlock(&LB.stat); +} + +// Journal slot `slot`: the variant e its worker hands to the driver next, or zeros (e NULL) when the build returned. +static void journal_mark(int slot, const GfxWarmEnt *e) { + if (!SK.journal || slot < 0 || slot >= JR_SLOTS) return; + uint8_t rec[JR_SLOT] = {0}; + if (e) memcpy(rec, e->iid, 32), rec[32] = e->stage; + plat_file_pwrite(SK.journal, rec, sizeof rec, (uint64_t)slot * JR_SLOT); +} + VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { if (info) *info = NULL; if (!cs || !ensure_init()) return VK_NULL_HANDLE; @@ -2749,7 +2878,8 @@ static void lb_report(const char *how) { rt_log("vk_pipeline: stage library prebuild %s: %llu of %zu variants in %.0f ms; kept: vertex %llu built (avg %.1f ms), " "%llu there; fragment %llu built (avg %.1f ms), %llu there; compute %llu built (avg %.1f ms), %llu there; driver " "cache only: vertex %llu (avg %.1f ms), fragment %llu (avg %.1f ms), compute %llu (avg %.1f ms); %llu failed, %llu not in the shader cache, " - "%llu vertex/fragment skipped (graphics pipeline library off); process working set %.0f MB", how, (unsigned long long)LB.done, LB.n, + "%llu vertex/fragment skipped (graphics pipeline library off), %llu skipped (the driver crashed on them before, " + "shader-prepare-skip.txt); process working set %.0f MB", how, (unsigned long long)LB.done, LB.n, now_ms() - LB.t0, (unsigned long long)LB.built[0], LB.built[0] ? LB.ms[0] / LB.built[0] : 0.0, (unsigned long long)LB.have[0], (unsigned long long)LB.built[1], LB.built[1] ? LB.ms[1] / LB.built[1] : 0.0, (unsigned long long)LB.have[1], (unsigned long long)LB.built[2], LB.built[2] ? LB.ms[2] / LB.built[2] : 0.0, @@ -2757,7 +2887,7 @@ static void lb_report(const char *how) { (unsigned long long)LB.warmed[1], LB.warmed[1] ? LB.warm_ms[1] / LB.warmed[1] : 0.0, (unsigned long long)LB.warmed[2], LB.warmed[2] ? LB.warm_ms[2] / LB.warmed[2] : 0.0, (unsigned long long)(LB.fail[0] + LB.fail[1] + LB.fail[2]), - (unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (double)mem_proc); + (unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (unsigned long long)LB.skipped, (double)mem_proc); plat_mutex_unlock_shared(&LB.stat); } @@ -2768,6 +2898,13 @@ static void lb_worker(void *arg) { for (;;) { size_t i = (size_t)plat_inc64(&LB.next) - 1; if (i >= LB.n || LB.stop) break; + if (skip_listed(LB.e[i].iid)) { + skip_count(); + plat_mutex_lock(&LB.stat); + LB.done++; + plat_mutex_unlock(&LB.stat); + continue; + } plat_thread_priority(PLAT_PRIO_NORMAL); // gfx_shader's cache lock, disk reads const GfxShader *sh = gfx_shader_load_input(LB.e[i].iid, 0); plat_thread_priority(PLAT_PRIO_IDLE); @@ -2805,6 +2942,8 @@ static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { const char *e = cfg_value("TOTK_LIB_PREBUILD"), *ke = cfg_get("TOTK_LIB_PREBUILD_KEEP"); const char *te = cfg_get("TOTK_LIB_PREBUILD_THREADS"); // no fixed default: a quarter of the CPUs when unset if (!A.on || !cfg_bool("TOTK_LIB_PREBUILD")) return; + char dir[PLAT_PATH_MAX * 2]; + if (cache_dir(dir, sizeof dir)) skip_init(dir); GfxWarmEnt *all = NULL; size_t total = gfx_shader_warm_list(&all), n = 0, n_cs = 0; LB.keep_n = ke && *ke ? atoi(ke) : 0; @@ -2847,16 +2986,20 @@ static void lb_stop(void) { LB.threads = 0; } -// Preparation: one prepared variant at a time into the driver's disk cache (private shader copy, nothing kept). +// Preparation: one prepared variant at a time into the driver's disk cache (private shader copy, nothing kept). arg is +// the worker's journal slot. static void lb_warm_worker(void *arg) { - (void)arg; + int slot = (int)(intptr_t)arg; for (;;) { size_t i = (size_t)plat_inc64(&LB.next) - 1; if (i >= LB.n) return; - int owned = 0; - const GfxShader *sh = gfx_shader_load_private(LB.e[i].iid, &owned); + int owned = 0, skip = skip_listed(LB.e[i].iid); + const GfxShader *sh = skip ? NULL : gfx_shader_load_private(LB.e[i].iid, &owned); int st = sh ? gfx_shader_stage(sh) : -1; - if (!sh) { + if (sh) journal_mark(slot, &LB.e[i]); // from here to its clearing below, a crash names this variant + if (skip) { + skip_count(); + } else if (!sh) { plat_mutex_lock(&LB.stat); LB.noshader++; plat_mutex_unlock(&LB.stat); @@ -2874,6 +3017,7 @@ static void lb_warm_worker(void *arg) { if (P.gpl) lb_library(sh, st == GFX_STAGE_VS, 0); else lb_skip_lib(); } + if (sh) journal_mark(slot, NULL); if (owned) gfx_shader_release_private(sh); plat_mutex_lock(&LB.stat); uint64_t d = ++LB.done; @@ -2894,23 +3038,36 @@ static void lb_warm_prepared(const char *dir) { memcpy(LB.e[i].iid, f + 16 + 36 * i + 4, 32), LB.e[i].stage = f[16 + 36 * i], LB.e[i].prio = f[16 + 36 * i + 1]; free(f); if (!LB.e) return; + skip_init(dir); + char jp[PLAT_PATH_MAX * 2 + 48]; + snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir); + SK.journal = plat_fs_create(jp, NULL); // skip_init removed a leftover one + if (!SK.journal) rt_log("vk_pipeline: cannot create %s: a driver crash in this run will not be recorded", jp); LB.n = n, LB.idle = 0, LB.t0 = now_ms(); static const uint32_t no_pr[16], no_fs[4]; lb_state(no_pr, no_fs); int cpus = (int)plat_cpu_usable(), k = 0; - PlatThread *t[64]; - for (int i = 0; i < cpus && i < 64; i++) - if ((t[k] = plat_thread_new(lb_warm_worker, NULL, "lib warm"))) k++; - if (!k) lb_warm_worker(NULL); + PlatThread *t[JR_SLOTS]; + for (int i = 0; i < cpus && i < JR_SLOTS; i++) + if ((t[k] = plat_thread_new(lb_warm_worker, (void *)(intptr_t)k, "lib warm"))) k++; + if (!k) lb_warm_worker((void *)(intptr_t)0); for (int i = 0; i < k; i++) plat_thread_join(t[i]); + if (SK.journal) plat_file_close(SK.journal), SK.journal = NULL, plat_file_delete(jp); // every build returned lb_report("preparation done"); // with gpl off only compute variants are built: say so, rather than count the skipped ones as built + unsigned long long skipped = LB.skipped; + size_t tried = n - (size_t)skipped; if (LB.nolib) printf("prepare-shaders: %zu variants on %d threads in %.0f ms, %llu vertex/fragment variants skipped (graphics " - "pipeline library off, they build at their first draw)\n", n, k, now_ms() - LB.t0, (unsigned long long)LB.nolib); + "pipeline library off, they build at their first draw)\n", tried, k, now_ms() - LB.t0, + (unsigned long long)LB.nolib); else - printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", n, k, now_ms() - LB.t0); - free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0; + printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", tried, k, + now_ms() - LB.t0); + if (skipped) // installer/steps.py Prepare reads this line + printf("prepare-shaders: %llu variants skipped: the graphics driver crashed while building them in an earlier " + "run (shader-prepare-skip.txt); they build at their first draw\n", skipped); + free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0, LB.skipped = 0; } // totk.exe --prepare-shaders (tools/prepare_shaders.py): the device sets the translator's host answers (they are in every @@ -2922,10 +3079,7 @@ int vk_pipeline_prepare_shaders(void) { return 1; } char dir[PLAT_PATH_MAX * 2]; - snprintf(dir, sizeof dir, "%s", P.cache_path); - char *cut = plat_last_sep(dir); - if (!cut) return 1; - *cut = 0; + if (!cache_dir(dir, sizeof dir)) return 1; int cpus = (int)plat_cpu_usable(); const char *pe = cfg_get("TOTK_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2; @@ -2944,10 +3098,7 @@ int vk_pipeline_warm_shaders(void) { return 1; } char dir[PLAT_PATH_MAX * 2]; - snprintf(dir, sizeof dir, "%s", P.cache_path); - char *cut = plat_last_sep(dir); - if (!cut) return 1; - *cut = 0; + if (!cache_dir(dir, sizeof dir)) return 1; char prp[PLAT_PATH_MAX * 2 + 32]; snprintf(prp, sizeof prp, "%s/shader-prepared.bin", dir); if (!plat_file_exists(prp)) { diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index f26aa30d..f7351e99 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -18,6 +18,11 @@ shader-prepared.bin (one record per variant, in priority order) and builds every pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files can be deleted afterwards. +A driver whose shader compiler crashes on a variant crashes on it in every run (NVIDIA 615.78.08 on an RTX 5080, issue +#30). The translations and shader-prepared.bin are written by then, and the journal the run leaves names the variants +the driver was building (vk_pipeline.c skip_init), so only the driver step runs again (`totk.exe --warm-shaders`), up to +RETRIES times, and skips them. + Game-derived output stays in the build directory (ignored). The list is read at preparation time from tools/data/shader_list.tsv when that file exists (the installer archive bundles it; it stays out of the repository), else from refs/ through git with the rows its history dropped (tools/shader_list.py); nothing of it is copied into the repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14: @@ -54,6 +59,8 @@ TYPES = {'float': 0, 'sint': 1, 'uint': 2} DEFAULT = {'tex': {str(slot): '2d' for slot in range(32)}} # answers outside a shader's use mask are dropped CONTROL = 0x880 ANS_REC = 4 + 32 + 32 + 147 + 1 +JOURNAL = 'shader-prepare-journal.bin' # vk_pipeline.c skip_init: a 64-byte slot per worker, input id first +RETRIES = 3 BUNDLED_LIST = ROOT / 'tools' / 'data' / 'shader_list.tsv' # the installer archive carries a copy (no git, no refs/) @@ -128,6 +135,13 @@ def code_hash(stage, control, code, length, c1): struct.pack('