"""Prepare the shader variants of the MacNativeGames shared shader list for a build, once, before play. The list (refs/MacNativeGames games/totk/assets/shader_list.tsv, MIT; docs/shader-list.md there) names shader variants players met: a shader key (sha256 of the NVN control section and code, as tools/shader_census.py names shaders) and the translator answers it was translated for. This tool reads the shaders from the extracted RomFS and ExeFS (tools/shader_census.py inventory), keeps the list's variants whose shader is in this game version, and writes them next to the build's totk.exe in the record layouts of runtime/gfx_shader.c ("answers met in play"): seed-programs.bin "TOTKSPRG": stage, length, c1 length, runtime code hash, control section, code, c1 seed-answers.bin "TOTKSANS": stage, code hash, input id (zero: the runtime computes it), answers, priority Priority (the pad byte): 0 shaders the list's warm rows flag as title/menu (t), 1 other warm rows (S, C, R), 2 the rest. Then it runs `totk.exe --prepare-shaders`, which translates every record into the build's shader cache, writes shader-prepared.bin (one record per variant, in priority order) and builds every variant's stage library or compute pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). About 5 min on a 12-core machine (inventory 1, translation 3, library builds 0.6) and 1.3 GB of shader cache; the seed files can be deleted afterwards. Game-derived output stays in the build directory (ignored). The list is read at preparation time from tools/data/shader_list.tsv when that file exists (the installer archive bundles it; it stays out of the repository), else from refs/ through git; nothing of it is copied into the repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14: .venv-py314\\Scripts\\python tools\\prepare_shaders.py build\\v121 [--version v121] [--list ] [--no-run] [--romfs DIR --exefs DIR] # instead of build\\extracted\\ """ import argparse import hashlib import json import struct import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] while not (ROOT / 'tools' / 'shader_census.py').is_file() and ROOT != ROOT.parent: ROOT = ROOT.parent sys.path.insert(0, str(ROOT / 'tools')) from shader_census import inventory # noqa: E402 LIST_REF = ('origin/HEAD', 'games/totk/assets/shader_list.tsv') STAGES = {'vs': 0, 'ps': 4, 'cs': 5} # runtime/gfx.h GFX_STAGE_* TEX = [None, '1d', '1darray', 'buffer', '2d', '2darray', '2dms', '2dmsarray', '3d', 'cube', 'cubearray'] FMT = [None, 'R8Unorm', 'R8Snorm', 'R8Uint', 'R8Sint', 'R16Float', 'R16Unorm', 'R16Snorm', 'R16Uint', 'R16Sint', 'R32Float', 'R32Uint', 'R32Sint', 'R8G8Unorm', 'R8G8Snorm', 'R8G8Uint', 'R8G8Sint', 'R16G16Float', 'R16G16Unorm', 'R16G16Snorm', 'R16G16Uint', 'R16G16Sint', 'R32G32Float', 'R32G32Uint', 'R32G32Sint', 'R8G8B8A8Unorm', 'R8G8B8A8Snorm', 'R8G8B8A8Uint', 'R8G8B8A8Sint', 'R16G16B16A16Float', 'R16G16B16A16Unorm', 'R16G16B16A16Snorm', 'R16G16B16A16Uint', 'R16G16B16A16Sint', 'R32G32B32A32Float', 'R32G32B32A32Uint', 'R32G32B32A32Sint', 'R10G10B10A2Unorm', 'R10G10B10A2Uint', 'R11G11B10Float'] # gfx_shader.c fmt_names TYPES = {'float': 0, 'sint': 1, 'uint': 2} CONTROL = 0x880 ANS_REC = 4 + 32 + 32 + 147 + 1 BUNDLED_LIST = ROOT / 'tools' / 'data' / 'shader_list.tsv' # the installer archive carries a copy (no git, no refs/) if not BUNDLED_LIST.is_file(): BUNDLED_LIST = ROOT / 'data' / 'shader_list.tsv' # release folder layout (tools/make_release.py) def read_list(path): if path: return Path(path).read_text(encoding='utf-8') if BUNDLED_LIST.is_file(): return BUNDLED_LIST.read_text(encoding='utf-8') return subprocess.run(['git', '-C', str(ROOT / 'refs' / 'MacNativeGames'), 'show', ':'.join(LIST_REF)], check=True, capture_output=True, encoding='utf-8').stdout def parse_list(text): """V rows as (key, deps); priority per shader key from the warm rows (variant names start with the key).""" variants, prio = [], {} def warm(key, flags): p = 0 if 't' in flags else 1 prio[key] = min(prio.get(key, 2), p) for line in text.splitlines(): f = line.split('\t') if f[0] == 'V' and len(f) == 3: variants.append((f[1], json.loads(f[2]))) elif f[0] == 'S' and len(f) == 4: warm(f[1][:64], f[3]) elif f[0] == 'C' and len(f) == 3: warm(f[1][:64], f[2]) elif f[0] in ('R', 'B') and len(f) >= 3: names = bytes.fromhex(f[1]).split(b'\0')[:2] # "\0\0" + a Metal descriptor for n in names: if len(n) >= 64: warm(n[:64].decode('ascii', 'replace'), f[-1]) return variants, prio def answers(deps, stage, control): """GfxShaderAnswers in gfx_shader.c pack_answers layout (147 bytes) from the list's deps JSON.""" tex, img, fmt, attr, fout = bytearray(32), bytearray(32), bytearray(32), bytearray(32), bytearray(8) for slot, t in deps.get('tex', {}).items(): tex[int(slot)] = TEX.index(t.removesuffix('_depth')) for slot, o in deps.get('img', {}).items(): img[int(slot)] = TEX.index(o['type']) fmt[int(slot)] = FMT.index(o['fmt']) if o.get('fmt') in FMT else 0 for loc, t in deps.get('attr', {}).items(): attr[int(loc)] = TYPES[t] for idx, t in deps.get('fout', {}).items(): fout[int(idx)] = TYPES[t] local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for graphics stages (they are in the input id) if stage == 5: # compute: the control section's local size, 0 read as 1 local = [v or 1 for v in struct.unpack_from('