Merge pull request #278: Let shader preparation survive a driver crash (wdiasjunior)

This commit is contained in:
Valiantsin Dzerakh committed 2026-11-01 10:55:09 +03:00
commit f7e5639ab9
7 files changed
+213 -23

No files matched your search

+5
View File
@@ -116,6 +116,11 @@ of them.
Without this step the first launch compiles shaders and pipelines during play and stutters until the caches fill.
The AMD driver keeps its pipeline cache per exe path and holds a limited amount: preparing another build folder can
evict this one's. `totk.exe --warm-shaders` rebuilds it from the shader cache.
When the driver's shader compiler crashes on a variant (NVIDIA 615.78.08 on an RTX 5080 does, issue #30),
`prepare_shaders.py` runs the driver step again with `--warm-shaders`, up to three times, without the variants the
driver was building at the crash. `shader-prepare-skip.txt` next to the exe lists them with the GPU and driver
version; they build at their first draw instead, another driver version builds them again, and so does deleting the
file.
7. Run. The game keeps running when its window loses the focus (`assume_focus = 1` in `totk.ini`, the default; the
Game tab switches it). With 0 it pauses, as it does on the Switch when Home is pressed.
+3
View File
@@ -56,6 +56,9 @@ loads the saves 1.4.3 wrote.
writes the translations it makes in play as loose files; running the tool again with the game closed folds those in.
The patches are written twice: `romfs-patches` and `romfs-patches-xbox` (`--xbox-prompts`, the A/B and X/Y prompts
swapped for `pad_layout = xbox_prompts`).
When the graphics driver crashes while building the shaders, `prepare_shaders.py` runs that part again without the
variants it was building ([building.md](building.md), step 6), and the setup finishes with a warning that names how
many were left out.
6. **Clean up** the temp folder (generated C and objects, about 1.3 GB).
A cancelled or failed setup resumes from the last finished step (`totk-install.json`).
+9
View File
@@ -644,6 +644,9 @@ class Compile(Step):
# ---- 5. prepare
TRANSLATED = re.compile(r'gfx_shader: prepare: (\d+) of (\d+) pairs')
# runtime/vk_pipeline.c lb_warm_prepared: variants the driver crashed on in an earlier run (tools/prepare_shaders.py
# runs the driver step again after such a crash)
SKIPPED = re.compile(r'prepare-shaders: (\d+) variants skipped: the graphics driver crashed')
# The game resolutions of another shape than 16:9 that runtime/ui/menu.cpp row_output_res lists: 21:9 and 32:9 monitors,
@@ -715,10 +718,14 @@ class Prepare(Step):
shader_list = ctx.temp / 'shader_list.tsv'
shader_list.write_text(shader_list_text(layout), encoding='utf-8', newline='\n')
skipped = [0]
def parse(line):
m = TRANSLATED.search(line)
if m:
rep.progress('prepare', 'translate', int(m.group(1)), int(m.group(2)), 'shaders')
elif SKIPPED.search(line):
skipped[0] = int(SKIPPED.search(line).group(1))
elif 'listed variants' in line:
rep.progress('prepare', 'translate', force=True)
elif 'gfx_shader: prepare:' in line and 'translated' in line and 'programs' in line:
@@ -730,6 +737,8 @@ class Prepare(Step):
dest / 'exefs', '--list', shader_list], parse)
if code or not (dest / 'shader-prepared.bin').is_file():
raise InstallError('prepare.shaders_failed', code=code, log=str(log))
if skipped[0]:
rep.warning('warn.shaders_skipped', count=skipped[0], log=str(log))
# One shader-cache.pack instead of four files per shader (~117,000): totk.exe reads the pack first.
rep.progress('prepare', 'pack', force=True)
+1
View File
@@ -86,6 +86,7 @@
"warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts, which the installer does not extract; some text may not show.",
"warn.npdm_differs": "main.npdm differs from the reference {version} dump; only the code modules are used.",
"warn.xbox_prompts_failed": "The Xbox button prompts could not be prepared (exit code {code}); the game keeps the Switch prompts. Details: {log}",
"warn.shaders_skipped": "The graphics driver crashed while building some shaders, so {count} were left out of the preparation. The game builds them when it first needs them, and the driver may crash there too; a driver update may fix it. Details: {log}",
"warn.kernel_old": "Linux {version} lacks the write tracking the game relies on (6.7 or later); the game may run very slowly. Updating the system fixes this.",
"ui.title": "TotK Setup",
+1
View File
@@ -86,6 +86,7 @@
"warn.lang_needs_fonts": "Для языка «{lang}» нужны системные шрифты Switch, которые установка не извлекает; часть текста может не отображаться.",
"warn.npdm_differs": "main.npdm отличается от эталонного дампа {version}; используются только модули с кодом.",
"warn.xbox_prompts_failed": "Не удалось подготовить подсказки кнопок Xbox (код выхода {code}); игра оставит подсказки Switch. Подробности: {log}",
"warn.shaders_skipped": "Графический драйвер аварийно завершился при сборке части шейдеров, поэтому {count} из них пропущены при подготовке. Игра соберёт их, когда они впервые понадобятся, и драйвер может упасть и там; обновление драйвера может это исправить. Подробности: {log}",
"warn.kernel_old": "В Linux {version} нет отслеживания записи, на которое опирается игра (нужна версия 6.7 или новее); игра может работать очень медленно. Обновление системы это исправит.",
"ui.title": "Установка TotK",
+174 -23
View File
@@ -2569,7 +2569,8 @@ static struct {
uint8_t clamp, poly, minus_w, samples; // library state (GKey fields)
double t0, ms[3]; // ms: build time per kind (vertex, fragment, compute)
// warmed: driver cache only; nolib: vertex and fragment variants skipped with the graphics pipeline library off
uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, done, cs_draw;
// skipped: variants of the skip list (SK below)
uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, skipped, done, cs_draw;
double warm_ms[3];
volatile int32_t kept; // libraries entered in libs[] so far
int32_t keep_n;
@@ -2577,6 +2578,134 @@ static struct {
PlatMutex stat;
} LB = {.stat = PLAT_MUTEX_INIT};
// ---- preparation that outlives a driver crash. A bug in the driver's shader compiler can kill the process inside
// vkCreate*Pipelines: NVIDIA 615.78.08 on an RTX 5080 crashed on the same vertex library at the same point of every
// preparation, natively and under Wine, so preparation never finished (issue #30 and a report on PR #60).
// While lb_warm_prepared builds, slot k of <dir>/shader-prepare-journal.bin holds the variant its worker k has handed to
// the driver (input id and stage; zeros between variants). The writes go straight to the OS, so they outlast the
// process, and a run that finishes deletes the file. The next preparation or boot (skip_init) moves the variants a
// leftover journal names into <dir>/shader-prepare-skip.txt with this GPU and driver version, and preparation and the
// boot prebuild (lb_worker) skip what that file lists for them. The variants the other workers were building at the
// crash are skipped along with the culprit; like it, they build at their first draw, where the driver can still crash.
// Another driver version builds them again, and so does deleting the file. tools/prepare_shaders.py runs the build
// again (--warm-shaders) after a run that left a journal.
#define JR_SLOT 64 // bytes per journal slot: input id, stage, zeros
#define JR_SLOTS 64 // lb_warm_prepared's thread cap
static struct {
PlatFile *journal; // while lb_warm_prepared builds
uint8_t (*iid)[32]; // the skip list's variants for this GPU and driver
size_t n;
int loaded;
} SK;
static int cache_dir(char *dir, size_t n) { // the folder of pipeline-cache.bin and shader-prepared.bin
snprintf(dir, n, "%s", P.cache_path);
char *cut = plat_last_sep(dir);
if (!cut) return 0;
*cut = 0;
return 1;
}
static void gpu_key(char *out, size_t n) { // vendor:device:driver version, as the skip list names a GPU and driver
const VkPhysicalDeviceProperties *p = &vk_ctx()->props;
snprintf(out, n, "%04x:%04x:%08x", p->vendorID, p->deviceID, p->driverVersion);
}
static const char *stage_name(int st) {
return st == GFX_STAGE_VS ? "vertex" : st == GFX_STAGE_FS ? "fragment" : st == GFX_STAGE_CS ? "compute" : "other";
}
static int hex_digit(int c) {
return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1;
}
// A journal a crashed run left into the skip list, then the skip list's variants for this GPU and driver into SK.
static void skip_init(const char *dir) {
if (SK.loaded) return;
SK.loaded = 1;
char jp[PLAT_PATH_MAX * 2 + 48], sp[PLAT_PATH_MAX * 2 + 48], key[32];
snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir);
snprintf(sp, sizeof sp, "%s/shader-prepare-skip.txt", dir);
gpu_key(key, sizeof key);
uint64_t jn = 0, sn = 0;
uint8_t *j = plat_file_read(jp, &jn);
char *s = (char *)plat_file_read(sp, &sn);
int keep_journal = 0;
if (j) {
static const char head[] =
"# Shader variants the graphics driver was building when the process died during preparation\n"
"# (runtime/vk_pipeline.c skip_init): input id, stage, GPU vendor:device:driver version. Preparation skips\n"
"# them on that GPU and driver. Delete this file to build them again.\n";
size_t cap = (s ? sn : sizeof head) + (jn / JR_SLOT + 1) * 128, len = 0;
char *out = malloc(cap);
int added = 0;
if (out) {
len = s ? sn : sizeof head - 1;
memcpy(out, s ? s : head, len);
if (len && out[len - 1] != '\n') out[len++] = '\n';
for (uint64_t o = 0; o + 33 <= jn; o += JR_SLOT) {
int set = 0;
for (int b = 0; b < 32; b++) set |= j[o + b];
if (!set) continue;
char hex[65];
for (int b = 0; b < 32; b++) snprintf(hex + 2 * b, 3, "%02x", j[o + b]);
len += (size_t)snprintf(out + len, cap - len, "%s %s %s\n", hex, stage_name(j[o + 32]), key);
added++;
}
out[len] = 0; // parsed below as the list's text
}
if (added && plat_file_write_atomic(sp, out, len)) {
rt_log("vk_pipeline: the last shader preparation died while the driver built %d variants; %s now lists them "
"and preparation skips them on this GPU and driver (%s)", added, sp, key);
free(s), s = out, sn = len, out = NULL;
} else if (added) {
rt_log("vk_pipeline: cannot write %s; %s stays for the next run", sp, jp);
keep_journal = 1;
}
free(out);
free(j);
}
if (!keep_journal) plat_file_delete(jp);
for (char *line = s, *next; line && *line; line = next) {
next = strchr(line, '\n');
if (next) *next++ = 0;
char hex[65], st[16], k[32];
uint8_t iid[32];
int ok = line[0] != '#' && sscanf(line, "%64s %15s %31s", hex, st, k) == 3 && strlen(hex) == 64 &&
!strcmp(k, key);
for (int b = 0; ok && b < 32; b++) {
int hi = hex_digit(hex[2 * b]), lo = hex_digit(hex[2 * b + 1]);
if (hi < 0 || lo < 0) ok = 0;
else iid[b] = (uint8_t)(hi << 4 | lo);
}
uint8_t (*grown)[32] = ok ? realloc(SK.iid, (SK.n + 1) * sizeof *SK.iid) : NULL;
if (grown) memcpy(grown[SK.n++], iid, 32), SK.iid = grown;
}
free(s);
if (SK.n) rt_log("vk_pipeline: %zu shader variants skipped ahead of play on this GPU and driver (%s)", SK.n, sp);
}
static int skip_listed(const uint8_t iid[32]) {
for (size_t i = 0; i < SK.n; i++)
if (!memcmp(SK.iid[i], iid, 32)) return 1;
return 0;
}
static void skip_count(void) {
plat_mutex_lock(&LB.stat);
LB.skipped++;
plat_mutex_unlock(&LB.stat);
}
// Journal slot `slot`: the variant e its worker hands to the driver next, or zeros (e NULL) when the build returned.
static void journal_mark(int slot, const GfxWarmEnt *e) {
if (!SK.journal || slot < 0 || slot >= JR_SLOTS) return;
uint8_t rec[JR_SLOT] = {0};
if (e) memcpy(rec, e->iid, 32), rec[32] = e->stage;
plat_file_pwrite(SK.journal, rec, sizeof rec, (uint64_t)slot * JR_SLOT);
}
VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) {
if (info) *info = NULL;
if (!cs || !ensure_init()) return VK_NULL_HANDLE;
@@ -2749,7 +2878,8 @@ static void lb_report(const char *how) {
rt_log("vk_pipeline: stage library prebuild %s: %llu of %zu variants in %.0f ms; kept: vertex %llu built (avg %.1f ms), "
"%llu there; fragment %llu built (avg %.1f ms), %llu there; compute %llu built (avg %.1f ms), %llu there; driver "
"cache only: vertex %llu (avg %.1f ms), fragment %llu (avg %.1f ms), compute %llu (avg %.1f ms); %llu failed, %llu not in the shader cache, "
"%llu vertex/fragment skipped (graphics pipeline library off); process working set %.0f MB", how, (unsigned long long)LB.done, LB.n,
"%llu vertex/fragment skipped (graphics pipeline library off), %llu skipped (the driver crashed on them before, "
"shader-prepare-skip.txt); process working set %.0f MB", how, (unsigned long long)LB.done, LB.n,
now_ms() - LB.t0, (unsigned long long)LB.built[0], LB.built[0] ? LB.ms[0] / LB.built[0] : 0.0,
(unsigned long long)LB.have[0], (unsigned long long)LB.built[1], LB.built[1] ? LB.ms[1] / LB.built[1] : 0.0,
(unsigned long long)LB.have[1], (unsigned long long)LB.built[2], LB.built[2] ? LB.ms[2] / LB.built[2] : 0.0,
@@ -2757,7 +2887,7 @@ static void lb_report(const char *how) {
(unsigned long long)LB.warmed[1], LB.warmed[1] ? LB.warm_ms[1] / LB.warmed[1] : 0.0,
(unsigned long long)LB.warmed[2], LB.warmed[2] ? LB.warm_ms[2] / LB.warmed[2] : 0.0,
(unsigned long long)(LB.fail[0] + LB.fail[1] + LB.fail[2]),
(unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (double)mem_proc);
(unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (unsigned long long)LB.skipped, (double)mem_proc);
plat_mutex_unlock_shared(&LB.stat);
}
@@ -2768,6 +2898,13 @@ static void lb_worker(void *arg) {
for (;;) {
size_t i = (size_t)plat_inc64(&LB.next) - 1;
if (i >= LB.n || LB.stop) break;
if (skip_listed(LB.e[i].iid)) {
skip_count();
plat_mutex_lock(&LB.stat);
LB.done++;
plat_mutex_unlock(&LB.stat);
continue;
}
plat_thread_priority(PLAT_PRIO_NORMAL); // gfx_shader's cache lock, disk reads
const GfxShader *sh = gfx_shader_load_input(LB.e[i].iid, 0);
plat_thread_priority(PLAT_PRIO_IDLE);
@@ -2805,6 +2942,8 @@ static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) {
const char *e = cfg_value("TOTK_LIB_PREBUILD"), *ke = cfg_get("TOTK_LIB_PREBUILD_KEEP");
const char *te = cfg_get("TOTK_LIB_PREBUILD_THREADS"); // no fixed default: a quarter of the CPUs when unset
if (!A.on || !cfg_bool("TOTK_LIB_PREBUILD")) return;
char dir[PLAT_PATH_MAX * 2];
if (cache_dir(dir, sizeof dir)) skip_init(dir);
GfxWarmEnt *all = NULL;
size_t total = gfx_shader_warm_list(&all), n = 0, n_cs = 0;
LB.keep_n = ke && *ke ? atoi(ke) : 0;
@@ -2847,16 +2986,20 @@ static void lb_stop(void) {
LB.threads = 0;
}
// Preparation: one prepared variant at a time into the driver's disk cache (private shader copy, nothing kept).
// Preparation: one prepared variant at a time into the driver's disk cache (private shader copy, nothing kept). arg is
// the worker's journal slot.
static void lb_warm_worker(void *arg) {
(void)arg;
int slot = (int)(intptr_t)arg;
for (;;) {
size_t i = (size_t)plat_inc64(&LB.next) - 1;
if (i >= LB.n) return;
int owned = 0;
const GfxShader *sh = gfx_shader_load_private(LB.e[i].iid, &owned);
int owned = 0, skip = skip_listed(LB.e[i].iid);
const GfxShader *sh = skip ? NULL : gfx_shader_load_private(LB.e[i].iid, &owned);
int st = sh ? gfx_shader_stage(sh) : -1;
if (!sh) {
if (sh) journal_mark(slot, &LB.e[i]); // from here to its clearing below, a crash names this variant
if (skip) {
skip_count();
} else if (!sh) {
plat_mutex_lock(&LB.stat);
LB.noshader++;
plat_mutex_unlock(&LB.stat);
@@ -2874,6 +3017,7 @@ static void lb_warm_worker(void *arg) {
if (P.gpl) lb_library(sh, st == GFX_STAGE_VS, 0);
else lb_skip_lib();
}
if (sh) journal_mark(slot, NULL);
if (owned) gfx_shader_release_private(sh);
plat_mutex_lock(&LB.stat);
uint64_t d = ++LB.done;
@@ -2894,23 +3038,36 @@ static void lb_warm_prepared(const char *dir) {
memcpy(LB.e[i].iid, f + 16 + 36 * i + 4, 32), LB.e[i].stage = f[16 + 36 * i], LB.e[i].prio = f[16 + 36 * i + 1];
free(f);
if (!LB.e) return;
skip_init(dir);
char jp[PLAT_PATH_MAX * 2 + 48];
snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir);
SK.journal = plat_fs_create(jp, NULL); // skip_init removed a leftover one
if (!SK.journal) rt_log("vk_pipeline: cannot create %s: a driver crash in this run will not be recorded", jp);
LB.n = n, LB.idle = 0, LB.t0 = now_ms();
static const uint32_t no_pr[16], no_fs[4];
lb_state(no_pr, no_fs);
int cpus = (int)plat_cpu_usable(), k = 0;
PlatThread *t[64];
for (int i = 0; i < cpus && i < 64; i++)
if ((t[k] = plat_thread_new(lb_warm_worker, NULL, "lib warm"))) k++;
if (!k) lb_warm_worker(NULL);
PlatThread *t[JR_SLOTS];
for (int i = 0; i < cpus && i < JR_SLOTS; i++)
if ((t[k] = plat_thread_new(lb_warm_worker, (void *)(intptr_t)k, "lib warm"))) k++;
if (!k) lb_warm_worker((void *)(intptr_t)0);
for (int i = 0; i < k; i++) plat_thread_join(t[i]);
if (SK.journal) plat_file_close(SK.journal), SK.journal = NULL, plat_file_delete(jp); // every build returned
lb_report("preparation done");
// with gpl off only compute variants are built: say so, rather than count the skipped ones as built
unsigned long long skipped = LB.skipped;
size_t tried = n - (size_t)skipped;
if (LB.nolib)
printf("prepare-shaders: %zu variants on %d threads in %.0f ms, %llu vertex/fragment variants skipped (graphics "
"pipeline library off, they build at their first draw)\n", n, k, now_ms() - LB.t0, (unsigned long long)LB.nolib);
"pipeline library off, they build at their first draw)\n", tried, k, now_ms() - LB.t0,
(unsigned long long)LB.nolib);
else
printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", n, k, now_ms() - LB.t0);
free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0;
printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", tried, k,
now_ms() - LB.t0);
if (skipped) // installer/steps.py Prepare reads this line
printf("prepare-shaders: %llu variants skipped: the graphics driver crashed while building them in an earlier "
"run (shader-prepare-skip.txt); they build at their first draw\n", skipped);
free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0, LB.skipped = 0;
}
// totk.exe --prepare-shaders (tools/prepare_shaders.py): the device sets the translator's host answers (they are in every
@@ -2922,10 +3079,7 @@ int vk_pipeline_prepare_shaders(void) {
return 1;
}
char dir[PLAT_PATH_MAX * 2];
snprintf(dir, sizeof dir, "%s", P.cache_path);
char *cut = plat_last_sep(dir);
if (!cut) return 1;
*cut = 0;
if (!cache_dir(dir, sizeof dir)) return 1;
int cpus = (int)plat_cpu_usable();
const char *pe = cfg_get("TOTK_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset
int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2;
@@ -2944,10 +3098,7 @@ int vk_pipeline_warm_shaders(void) {
return 1;
}
char dir[PLAT_PATH_MAX * 2];
snprintf(dir, sizeof dir, "%s", P.cache_path);
char *cut = plat_last_sep(dir);
if (!cut) return 1;
*cut = 0;
if (!cache_dir(dir, sizeof dir)) return 1;
char prp[PLAT_PATH_MAX * 2 + 32];
snprintf(prp, sizeof prp, "%s/shader-prepared.bin", dir);
if (!plat_file_exists(prp)) {
+20
View File
@@ -18,6 +18,11 @@ shader-prepared.bin (one record per variant, in priority order) and builds every
pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files
can be deleted afterwards.
A driver whose shader compiler crashes on a variant crashes on it in every run (NVIDIA 615.78.08 on an RTX 5080, issue
#30). The translations and shader-prepared.bin are written by then, and the journal the run leaves names the variants
the driver was building (vk_pipeline.c skip_init), so only the driver step runs again (`totk.exe --warm-shaders`), up to
RETRIES times, and skips them.
Game-derived output stays in the build directory (ignored). The list is read at preparation time from tools/data/shader_list.tsv when that file exists (the installer archive
bundles it; it stays out of the repository), else from refs/ through git with the rows its history dropped
(tools/shader_list.py); nothing of it is copied into the repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14:
@@ -54,6 +59,8 @@ TYPES = {'float': 0, 'sint': 1, 'uint': 2}
DEFAULT = {'tex': {str(slot): '2d' for slot in range(32)}} # answers outside a shader's use mask are dropped
CONTROL = 0x880
ANS_REC = 4 + 32 + 32 + 147 + 1
JOURNAL = 'shader-prepare-journal.bin' # vk_pipeline.c skip_init: a 64-byte slot per worker, input id first
RETRIES = 3
BUNDLED_LIST = ROOT / 'tools' / 'data' / 'shader_list.tsv' # the installer archive carries a copy (no git, no refs/)
@@ -128,6 +135,13 @@ def code_hash(stage, control, code, length, c1):
struct.pack('<I', len(c1)) + c1).digest()
def driver_crashed(out):
"""totk died while the graphics driver built variants: the journal it left names at least one."""
p = out / JOURNAL
data = p.read_bytes() if p.is_file() else b''
return any(any(data[i:i + 32]) for i in range(0, len(data), 64))
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('build', type=Path, help='build directory holding totk.exe (build\\v121)')
@@ -193,6 +207,12 @@ def main():
return 0
exe = out / ('totk.exe' if os.name == 'nt' else 'totk')
r = subprocess.run([str(exe), '--prepare-shaders'], cwd=str(out))
for _ in range(RETRIES):
if not r.returncode or not driver_crashed(out):
break
print(f'prepare_shaders: the graphics driver crashed while building shaders (rc={r.returncode}); building them '
'again without the variants it was building (totk.exe --warm-shaders)', flush=True)
r = subprocess.run([str(exe), '--warm-shaders'], cwd=str(out))
print(f'totk.exe --prepare-shaders: rc={r.returncode}; total {time.monotonic() - t0:.1f} s')
return r.returncode