mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 20:25:57 +00:00
Warm the GPU driver's cache again when build_game relinks a prepared build: the cache is keyed to the exe, and after a relink every new stage library took 40-220 ms in play instead of about 2 ms; totk.exe --warm-shaders builds the prepared variants from the shader cache without translating
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
138a7d3a99
commit
7daeffdc08
5 files changed
+53
-2
No files matched your search
@@ -43,7 +43,7 @@ Setup and verify commands: [docs/toolchain-windows.md](docs/toolchain-windows.md
|
||||
- `tests\diff\atomic.py`: exclusive load/store sequences and ARMv8.1 atomics; needs no game files.
|
||||
- `tests\diff\func.py`: whole leaf functions as the generator emits them, loops and jump tables included.
|
||||
- `tests\excl_mt.py`: the exclusive monitor helpers of `cpu.h` under contention and ABA, on several threads; needs no game files.
|
||||
- Game code build (`.venv`): `recomp\gen.py --exefs build\extracted\v121\exefs --out generated\v121 --patches recomp\patches-v121.txt` writes the C (about 15 s). The patch table holds the 60 fps and real-time frame step code patches, switchable at run time (`recomp/patches.py`); `.venv-py314\Scripts\python tools\make_romfs_patches.py` writes their RomFS half to `build\v121\romfs-patches`. `tools\build_game.py` compiles and links `build\v121\totk.exe` with Ninja (about 200 s clean, incremental after). `recomp/layout.py` alone decides module bases. Jump tables are resolved by `recomp/jumptables.py`; after changing it, `tests\jumptables_check.py` must report 0 disagreements with its Unicorn oracle (on 1.4.3 too: `--exefs build\extracted\v143\exefs`).
|
||||
- Game code build (`.venv`): `recomp\gen.py --exefs build\extracted\v121\exefs --out generated\v121 --patches recomp\patches-v121.txt` writes the C (about 15 s). The patch table holds the 60 fps and real-time frame step code patches, switchable at run time (`recomp/patches.py`); `.venv-py314\Scripts\python tools\make_romfs_patches.py` writes their RomFS half to `build\v121\romfs-patches`. `tools\build_game.py` compiles and links `build\v121\totk.exe` with Ninja (about 200 s clean, incremental after); relinking a build that has `shader-prepared.bin` re-warms the GPU driver's cache, which is keyed to the exe (`totk.exe --warm-shaders`, about 3.5 min; `--no-warm` skips it). `recomp/layout.py` alone decides module bases. Jump tables are resolved by `recomp/jumptables.py`; after changing it, `tests\jumptables_check.py` must report 0 disagreements with its Unicorn oracle (on 1.4.3 too: `--exefs build\extracted\v143\exefs`).
|
||||
- Game version 1.4.3: the same commands with `v143` (`--patches recomp\patches-v143.txt`, `build_game.py --version v143`, `make_romfs_patches.py --version v143`). Its builds default to `build\profile-v143`; `run_game.py` runs it on copies of the 1.2.1 save backups.
|
||||
- Shader census (`.venv-py314`): `tools\shader_census.py --version v121 [--translate]` extracts every Maxwell shader to `build/v121/shaders/` and runs it through `build\xlate\totk_xlate.exe` and `spirv-val`; failure groups go to `failures.txt`. `tools\xlate_smoke.py` checks the translator alone, without game files.
|
||||
- Shader preparation (`.venv-py314`, once per build dir and translator): `tools\prepare_shaders.py build\v121` translates every game shader's default variant and the variants of the MacNativeGames shared shader list (read from `refs/` at run time, with the rows its history dropped: `tools/shader_list.py`) into the build's shader cache and builds their stage libraries into the driver cache (`totk.exe --prepare-shaders`; 1.4.3: 29,587 variants, 7 min on 24 threads, 1.7 GB); play then translates and compiles only variants neither covers.
|
||||
|
||||
@@ -260,4 +260,5 @@ int probe_shader(void); // M4 T4
|
||||
int probe_vkmem(void), probe_vkexec(void), probe_astc(void); // M4 T5, T6
|
||||
int probe_movie(void); // movie_probe.c: host VP9 movie::Decoder on one RomFS movie
|
||||
int vk_pipeline_prepare_shaders(void); // vk_pipeline.c: --prepare-shaders (tools/prepare_shaders.py)
|
||||
int vk_pipeline_warm_shaders(void); // vk_pipeline.c: --warm-shaders (tools/build_game.py)
|
||||
// gfx_stubs.c defines each of these weakly: prints "probe-x: not built yet", returns 2.
|
||||
+2
-1
@@ -388,7 +388,7 @@ static int usage(const char *argv0) {
|
||||
fprintf(stderr, "usage: %s [--exefs DIR] [--romfs DIR] [--profile DIR] [--selftest | --probe-loader | "
|
||||
"--probe-cfg | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | "
|
||||
"--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | "
|
||||
"--probe-vkexec | --probe-astc | --probe-movie | --prepare-shaders | --boot |--frames N [--shot PNG]]\n", argv0);
|
||||
"--probe-vkexec | --probe-astc | --probe-movie | --prepare-shaders | --warm-shaders | --boot |--frames N [--shot PNG]]\n", argv0);
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -488,6 +488,7 @@ int main(int argc, char **argv) {
|
||||
if (!strcmp(mode, "--probe-vkdevice")) return probe_vkdevice();
|
||||
if (!strcmp(mode, "--probe-movie")) return probe_movie();
|
||||
if (!strcmp(mode, "--prepare-shaders")) return vk_pipeline_prepare_shaders();
|
||||
if (!strcmp(mode, "--warm-shaders")) return vk_pipeline_warm_shaders();
|
||||
if (!strcmp(mode, "--boot")) { cfg.boot = 1; rt_boot(); return 0; }
|
||||
return usage(argv[0]);
|
||||
}
|
||||
@@ -2926,6 +2926,29 @@ int vk_pipeline_prepare_shaders(void) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
// totk.exe --warm-shaders (tools/build_game.py after it relinked a prepared build): the driver's disk cache is keyed to
|
||||
// the exe, so a rebuilt totk.exe finds it cold and a new library costs 40-220 ms in play instead of about 2 ms. This
|
||||
// builds every variant of shader-prepared.bin into it again from the shader cache, without translating the seed.
|
||||
int vk_pipeline_warm_shaders(void) {
|
||||
if (!ensure_init()) {
|
||||
fprintf(stderr, "warm-shaders: no Vulkan device\n");
|
||||
return 1;
|
||||
}
|
||||
char dir[MAX_PATH * 2];
|
||||
snprintf(dir, sizeof dir, "%s", P.cache_path);
|
||||
char *cut = strrchr(dir, '\\');
|
||||
if (!cut) return 1;
|
||||
*cut = 0;
|
||||
char prp[MAX_PATH * 2 + 32];
|
||||
snprintf(prp, sizeof prp, "%s\\shader-prepared.bin", dir);
|
||||
if (GetFileAttributesA(prp) == INVALID_FILE_ATTRIBUTES) {
|
||||
fprintf(stderr, "warm-shaders: no %s (tools/prepare_shaders.py writes it)\n", prp);
|
||||
return 1;
|
||||
}
|
||||
lb_warm_prepared(dir);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---- dynamic state
|
||||
|
||||
void vk_pipeline_set_dynamic(VkCommandBuffer cb, const VkPipeState *s) {
|
||||
|
||||
@@ -567,6 +567,27 @@ def mib(n):
|
||||
return f'{n / (1 << 20):,.1f} MiB'
|
||||
|
||||
|
||||
def warm_driver_cache(exe):
|
||||
"""Rebuild the GPU driver's disk cache of a prepared build after its totk.exe changed.
|
||||
|
||||
tools/prepare_shaders.py builds every prepared variant's stage library once so the driver's own disk cache holds it
|
||||
(vk_pipeline.c "stage library prebuild"). That cache is keyed to the exe: after a relink, a library a draw needs
|
||||
took 40-220 ms in play instead of about 2 ms. totk.exe --warm-shaders builds them again from the shader cache
|
||||
(about 3.5 min on 24 threads), without translating. Builds without shader-prepared.bin are left alone.
|
||||
"""
|
||||
if not (exe.parent / 'shader-prepared.bin').is_file():
|
||||
return
|
||||
print('build_game: totk.exe changed in a prepared build: warming the driver cache (--no-warm skips it)', flush=True)
|
||||
t0 = time.monotonic()
|
||||
r = subprocess.run([str(exe), '--warm-shaders'], cwd=str(exe.parent), stdout=subprocess.PIPE, text=True,
|
||||
errors='replace')
|
||||
last = (r.stdout.strip().splitlines() or [''])[-1]
|
||||
print(f'warm: rc={r.returncode}, {time.monotonic() - t0:.0f} s; {last}')
|
||||
if r.returncode:
|
||||
print(f'build_game: warning: the driver cache stays cold; run python tools/prepare_shaders.py '
|
||||
f'{below_root(exe.parent)}', file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--version', default='v121', help='game version (default v121)')
|
||||
@@ -600,6 +621,8 @@ def main():
|
||||
f'({", ".join(sorted(INSTALL_SIDE))} and modules.c still compile)')
|
||||
ap.add_argument('--expected', type=Path,
|
||||
help='expected hashes (default tools/data/expected-<display version>.json)')
|
||||
ap.add_argument('--no-warm', action='store_true',
|
||||
help='leave the driver cache cold after relinking a prepared build (see warm_driver_cache)')
|
||||
args = ap.parse_args()
|
||||
|
||||
gen = (args.gen or ROOT / 'generated' / args.version).resolve()
|
||||
@@ -706,6 +729,7 @@ def main():
|
||||
return
|
||||
if args.make_runtime_lib:
|
||||
exe.unlink(missing_ok=True) # ar rcs would keep members of deleted sources
|
||||
before = exe.stat().st_mtime_ns if exe.exists() else 0
|
||||
code, dt, peak_one, _ = run_measured(base + [flag_path(exe)], env)
|
||||
print(f'link: {dt:.1f} s, peak {mib(peak_one)}')
|
||||
if code:
|
||||
@@ -722,6 +746,8 @@ def main():
|
||||
f'image {mib(image)}; ' + ', '.join(f'{k} {mib(v)}' for k, v in sections.items() if k.startswith('.') and v))
|
||||
if image > 3 * GiB // 2:
|
||||
print('build_game: warning: image over 1.5 GiB; RIP-relative code reaches only 2 GiB', file=sys.stderr)
|
||||
if exe.stat().st_mtime_ns != before and not args.no_warm:
|
||||
warm_driver_cache(exe)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in new issue
Block a user