From d6e5b782523254f96f6ffb11b7ec414194d915c7 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 26 Oct 2026 19:53:16 +0300 Subject: [PATCH] Prepare every game shader's default variant and the shader list rows MacNativeGames pruned MacNativeGames now translates each shader's default bind state when it builds and keeps only other variants in its shared list, so the list at origin/HEAD has 134 rows of 1.4.3 shaders. prepare_shaders.py seeds the default variant of every shader it finds and reads the list through tools/shader_list.py, which adds the V rows older revisions held. On 1.4.3 that prepares 29,587 variants in 7 min with 0 translation failures, and every one of the 1,095 variants the field and intro runs met in play is among them. The installer's dev path and make_release.py read the list the same way. Also record that 1.2.1 loads saves 1.4.3 wrote. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/backlog.md | 12 +++++++----- docs/installer.md | 2 +- installer/layout.py | 1 - installer/steps.py | 9 ++++----- tools/build_game.py | 4 ++-- tools/make_release.py | 8 ++++---- tools/prepare_shaders.py | 38 +++++++++++++++++++++++--------------- tools/shader_list.py | 28 ++++++++++++++++++++++++++++ 9 files changed, 70 insertions(+), 34 deletions(-) create mode 100644 tools/shader_list.py diff --git a/AGENTS.md b/AGENTS.md index ecf3c6e0..72342a66 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ Setup and verify commands: [docs/toolchain-windows.md](docs/toolchain-windows.md - Game code build (`.venv`): `recomp\gen.py --exefs build\extracted\v121\exefs --out generated\v121 --patches recomp\patches-v121.txt` writes the C (about 15 s). The patch table holds the 60 fps and real-time frame step code patches, switchable at run time (`recomp/patches.py`); `.venv-py314\Scripts\python tools\make_romfs_patches.py` writes their RomFS half to `build\v121\romfs-patches`. `tools\build_game.py` compiles and links `build\v121\totk.exe` with Ninja (about 200 s clean, incremental after). `recomp/layout.py` alone decides module bases. Jump tables are resolved by `recomp/jumptables.py`; after changing it, `tests\jumptables_check.py` must report 0 disagreements with its Unicorn oracle (on 1.4.3 too: `--exefs build\extracted\v143\exefs`). - Game version 1.4.3: the same commands with `v143` (`--patches recomp\patches-v143.txt`, `build_game.py --version v143`, `make_romfs_patches.py --version v143`). Its builds default to `build\profile-v143`; `run_game.py` runs it on copies of the 1.2.1 save backups. - Shader census (`.venv-py314`): `tools\shader_census.py --version v121 [--translate]` extracts every Maxwell shader to `build/v121/shaders/` and runs it through `build\xlate\totk_xlate.exe` and `spirv-val`; failure groups go to `failures.txt`. `tools\xlate_smoke.py` checks the translator alone, without game files. -- Shader preparation (`.venv-py314`, once per build dir and translator): `tools\prepare_shaders.py build\v121` translates the variants of the MacNativeGames shared shader list (read from `refs/` at run time) into the build's shader cache and builds their stage libraries into the driver cache (`totk.exe --prepare-shaders`, about 5 min, 1.3 GB); play then translates and compiles only variants the list lacks. +- Shader preparation (`.venv-py314`, once per build dir and translator): `tools\prepare_shaders.py build\v121` translates every game shader's default variant and the variants of the MacNativeGames shared shader list (read from `refs/` at run time, with the rows its history dropped: `tools/shader_list.py`) into the build's shader cache and builds their stage libraries into the driver cache (`totk.exe --prepare-shaders`; 1.4.3: 29,587 variants, 7 min on 24 threads, 1.7 GB); play then translates and compiles only variants neither covers. - Open work with its evidence: [docs/backlog.md](docs/backlog.md). - Installer (`installer/`, `tools/make_release.py`), which builds the game on a player's machine: [docs/installer.md](docs/installer.md). - Running, measuring or frame-dumping the game, save backups, task worktrees and their builds: [docs/agent-playbook.md](docs/agent-playbook.md). diff --git a/docs/backlog.md b/docs/backlog.md index df2f4694..28830e8e 100644 --- a/docs/backlog.md +++ b/docs/backlog.md @@ -177,11 +177,13 @@ Each platform item gets a written design first, agreed with the maintainer. load) runs at 60 fps with game speed 1.000, REV15 audio without underruns, 0 unimplemented NVN calls, 0 validation errors; every probe passes on 1.4.3 and on 1.2.1; the jump-table check reports 0 disagreements on both; the installer takes either update. Open: - 1. Shader preparation: the MacNativeGames list at our pin has 134 keys of 1.4.3 (its compiler changed every - binary), so 1.4.3 translates nearly every shader on first sight in play. A list recorded from our own 1.4.3 - play, or a newer pin, would fill it. - 2. Not played beyond the field save and the opening cutscene: combat, abilities, cutscenes, menus, saving; whether - 1.2.1 loads saves that 1.4.3 wrote. + 1. Shader preparation is done: `prepare_shaders.py` seeds every shader's default variant, as MacNativeGames' + seed does, and reads the list with the V rows its history dropped (`tools/shader_list.py`; the current list + alone has 134 rows of 1.4.3 shaders). 29,587 variants, and all 1,095 that the field and intro runs met in play + are among them. Not ported: MacNativeGames types vertex inputs and colour outputs by the bound format, so a + variant serves any format; ours still differ by those types. + 2. Not played beyond the field save and the opening cutscene: combat, abilities, cutscenes, menus, saving. Saves + carry over both ways (the user's saves were written by 1.4.3 and load on 1.2.1). 3. The installer end-to-end test (`tests/installer_e2e.py`) runs only on 1.2.1. 4. MacNativeGames' 1.4.3 output, render and fixed resolution patches (`game_patches_v143.c`) are not ported; 1.2.1 has none of them either. diff --git a/docs/installer.md b/docs/installer.md index b3e9ce2f..fa4318f3 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -34,7 +34,7 @@ The player runs a newer installer on the same folder. It reads `totk-install.jso release that changes nothing game-derived skips the rebuild. Given another update file than the recorded one (1.2.1, then 1.4.3), the installer extracts and builds again from it and -keeps `profile/`. Whether 1.2.1 loads the saves 1.4.3 wrote is not checked: back the profile up before switching. +keeps `profile/`. Saves carry over both ways: 1.4.3 converts a 1.2.1 save on load, and 1.2.1 loads the saves 1.4.3 wrote. ## Setup steps diff --git a/installer/layout.py b/installer/layout.py index 206041cc..31f2fe1c 100644 --- a/installer/layout.py +++ b/installer/layout.py @@ -29,7 +29,6 @@ from pathlib import Path EXE = '.exe' if os.name == 'nt' else '' UCRT = Path(r'C:\msys64\ucrt64\bin') -SHADER_LIST_REF = 'origin/HEAD:games/totk/assets/shader_list.tsv' # dev: tools/prepare_shaders.py LIST_REF SUPPORTED = ('1.2.1', '1.4.3') # updates the installer builds; the check identifies the player's by hash diff --git a/installer/steps.py b/installer/steps.py index 630b9683..c7d9c40a 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -27,6 +27,7 @@ from .runner import InstallError, Step sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'tools')) import extract # noqa: E402 +import shader_list # noqa: E402 from extract_exefs import (common_tickets, container_entries, load_keys, nca_ctr_key, nca_inventory, # noqa: E402 program_nca) @@ -150,12 +151,10 @@ def expected_update(ctx): def shader_list_text(layout): if layout.shader_list: return layout.shader_list.read_text(encoding='utf-8') - refs = layout.root / 'refs' / 'MacNativeGames' - r = subprocess.run(['git', '-C', str(refs), 'show', 'origin/HEAD:games/totk/assets/shader_list.tsv'], - capture_output=True, encoding='utf-8') - if r.returncode: + try: + return shader_list.history(layout.root / 'refs' / 'MacNativeGames') + except subprocess.CalledProcessError: raise InstallError('prepare.shader_list_missing') - return r.stdout class Watch(threading.Thread): diff --git a/tools/build_game.py b/tools/build_game.py index 6c2af38c..77fc7d15 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -586,8 +586,8 @@ def main(): ap.add_argument('--romfs', type=Path, help="totk.exe's default RomFS directory with romfs.bin and index.json " '(default build/extracted//romfs)') ap.add_argument('--profile', type=Path, help="totk.exe's default profile directory (default build/profile for v121, " - "build/profile- otherwise: a newer game version rewrites the " - "saves it loads, and whether an older one reads them is not checked)") + "build/profile- otherwise, so trying another version leaves " + "the main profile alone; saves load on 1.2.1 and 1.4.3 alike)") ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of ' 'MSYS2 UCRT64 clang') ap.add_argument('--ninja', type=Path, help='ninja.exe to build with (default: MSYS2 UCRT64, else bin/ninja.exe of the ' diff --git a/tools/make_release.py b/tools/make_release.py index 0f945379..0f323f09 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -41,6 +41,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import check_content # noqa: E402 +import shader_list # noqa: E402 ROOT = Path(__file__).resolve().parent.parent MAIN_ROOT = (ROOT / 'build').resolve().parent # build/, generated/, toolchain/ and refs/ are junctions in a worktree @@ -95,7 +96,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. TOOLS_FILES = ['build_game.py', 'check_content.py', 'extract.py', 'make_romfs_patches.py', 'prepare_shaders.py', - 'romfs_index.py', 'shader_census.py'] + 'romfs_index.py', 'shader_census.py', 'shader_list.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon @@ -343,9 +344,8 @@ def build_sources(out): for d in TOOLS_DIRS: copy_tree(ROOT / 'tools' / d, out / 'tools' / d, lambda r: '__pycache__' in r or r.endswith('.md')) (out / 'data').mkdir(exist_ok=True) - text = subprocess.run(['git', '-C', MAIN_ROOT / 'refs' / 'MacNativeGames', 'show', - f'{MNG_COMMIT}:games/totk/assets/shader_list.tsv'], check=True, capture_output=True).stdout - (out / 'data' / 'shader_list.tsv').write_bytes(text) + text = shader_list.history(MAIN_ROOT / 'refs' / 'MacNativeGames', MNG_COMMIT) + (out / 'data' / 'shader_list.tsv').write_text(text, encoding='utf-8', newline='\n') copy_tree(ROOT / 'installer', out / 'installer', lambda r: '__pycache__' in r or r.startswith('launcher/')) shutil.copy2(ROOT / 'totk.ini.example', out / 'totk.ini.example') shutil.copy2(ROOT / 'THIRD_PARTY_NOTICES.md', out / 'THIRD_PARTY_NOTICES.md') diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index 1224fa87..4535f708 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -1,24 +1,26 @@ -"""Prepare the shader variants of the MacNativeGames shared shader list for a build, once, before play. +"""Prepare the default variant of every game shader and the variants of the MacNativeGames shared shader list for a +build, once, before play. The list (refs/MacNativeGames games/totk/assets/shader_list.tsv, MIT; docs/shader-list.md there) names shader variants players met: a shader key (sha256 of the NVN control section and code, as tools/shader_census.py names shaders) and the translator answers it was translated for. This tool reads the shaders from the extracted RomFS and ExeFS -(tools/shader_census.py inventory), keeps the list's variants whose shader is in this game version, and writes them -next to the build's totk.exe in the record layouts of runtime/gfx_shader.c ("answers met in play"): +(tools/shader_census.py inventory), keeps the list's variants whose shader is in this game version, adds each shader's +default variant (every texture slot a 2D texture, float vertex inputs and colour outputs, as MacNativeGames' seed +translates every shader for its default bind state: prepare_shader_seed.py all_default), and writes them next to the +build's totk.exe in the record layouts of runtime/gfx_shader.c ("answers met in play"): seed-programs.bin "TOTKSPRG": stage, length, c1 length, runtime code hash, control section, code, c1 seed-answers.bin "TOTKSANS": stage, code hash, input id (zero: the runtime computes it), answers, priority Priority (the pad byte): 0 shaders the list's warm rows flag as title/menu (t), 1 other warm rows (S, C, R), 2 the -rest. Then it runs `totk.exe --prepare-shaders`, which translates every record into the build's shader cache, writes +rest; a default variant takes its shader's priority. Then it runs `totk.exe --prepare-shaders`, which translates every record into the build's shader cache, writes shader-prepared.bin (one record per variant, in priority order) and builds every variant's stage library or compute -pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). About 5 min on -a 12-core machine (inventory 1, translation 3, library builds 0.6) and 1.3 GB of shader cache; the seed files can be -deleted afterwards. +pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files +can be deleted afterwards. Game-derived output stays in the build directory (ignored). The list is read at preparation time from tools/data/shader_list.tsv when that file exists (the installer archive -bundles it; it stays out of the repository), else from refs/ through git; nothing of it is copied into the -repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14: +bundles it; it stays out of the repository), else from refs/ through git with the rows its history dropped +(tools/shader_list.py); nothing of it is copied into the repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14: .venv-py314\\Scripts\\python tools\\prepare_shaders.py build\\v121 [--version v121] [--list ] [--no-run] [--romfs DIR --exefs DIR] # instead of build\\extracted\\ @@ -36,9 +38,9 @@ ROOT = Path(__file__).resolve().parents[1] while not (ROOT / 'tools' / 'shader_census.py').is_file() and ROOT != ROOT.parent: ROOT = ROOT.parent sys.path.insert(0, str(ROOT / 'tools')) +import shader_list # noqa: E402 from shader_census import inventory # noqa: E402 -LIST_REF = ('origin/HEAD', 'games/totk/assets/shader_list.tsv') STAGES = {'vs': 0, 'ps': 4, 'cs': 5} # runtime/gfx.h GFX_STAGE_* TEX = [None, '1d', '1darray', 'buffer', '2d', '2darray', '2dms', '2dmsarray', '3d', 'cube', 'cubearray'] FMT = [None, 'R8Unorm', 'R8Snorm', 'R8Uint', 'R8Sint', 'R16Float', 'R16Unorm', 'R16Snorm', 'R16Uint', 'R16Sint', @@ -48,6 +50,7 @@ FMT = [None, 'R8Unorm', 'R8Snorm', 'R8Uint', 'R8Sint', 'R16Float', 'R16Unorm', ' 'R16G16B16A16Uint', 'R16G16B16A16Sint', 'R32G32B32A32Float', 'R32G32B32A32Uint', 'R32G32B32A32Sint', 'R10G10B10A2Unorm', 'R10G10B10A2Uint', 'R11G11B10Float'] # gfx_shader.c fmt_names TYPES = {'float': 0, 'sint': 1, 'uint': 2} +DEFAULT = {'tex': {str(slot): '2d' for slot in range(32)}} # answers outside a shader's use mask are dropped CONTROL = 0x880 ANS_REC = 4 + 32 + 32 + 147 + 1 @@ -62,8 +65,7 @@ def read_list(path): return Path(path).read_text(encoding='utf-8') if BUNDLED_LIST.is_file(): return BUNDLED_LIST.read_text(encoding='utf-8') - return subprocess.run(['git', '-C', str(ROOT / 'refs' / 'MacNativeGames'), 'show', ':'.join(LIST_REF)], - check=True, capture_output=True, encoding='utf-8').stdout + return shader_list.history(ROOT / 'refs' / 'MacNativeGames') def parse_list(text): @@ -131,7 +133,9 @@ def main(): extracted = ROOT / 'build' / 'extracted' / a.version shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs') t1 = time.monotonic() - progs, recs, skipped = {}, [], {'not in this version': 0, 'stage': 0} + progs, recs, seen, skipped = {}, [], set(), {'not in this version': 0, 'stage': 0} + listed = len(variants) + variants += [(key, DEFAULT) for key in sorted(shaders)] for key, deps in variants: s = shaders.get(key) if not s: @@ -148,7 +152,11 @@ def main(): h = code_hash(stage, control, code, length, c1) progs[key] = (h, struct.pack('