From 6813b60ed9a79b4775e5f8281d5cd04f8a6c52dd Mon Sep 17 00:00:00 2001 From: shenjack <3695888@qq.com> Date: Sat, 31 Oct 2026 21:42:04 +0800 Subject: [PATCH] Find the shaders by their directories instead of reading the whole RomFS inventory() read every RomFS file into memory and decompressed it before deciding whether the file can hold a shader. On 1.1.1 that is 304,469 entries and 15.86 GiB, and the shaders live in six directories: 1,996 entries, 147 MiB, 0.9% of the bytes. The walk visits those directories and reads everything else only when the shader list names keys it did not find there. Measured on one install: 66.50 s -> 3.35 s for the same 27,576 shaders, key for key. pack_shader_cache.py also rewrote its pack after every prepare although there was nothing to fold in: 2.6 s of copying 1.37 GiB over the same entries, and a file system that then releases the old pack's blocks. It returns at once when the cache holds no loose files. --- tools/pack_shader_cache.py | 5 +++++ tools/prepare_shaders.py | 9 +++++++-- tools/shader_census.py | 10 +++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tools/pack_shader_cache.py b/tools/pack_shader_cache.py index 0b326d5d..f41e2c3f 100644 --- a/tools/pack_shader_cache.py +++ b/tools/pack_shader_cache.py @@ -79,6 +79,11 @@ def pack(cache, keep=False): out = cache / PACK packed = read_pack(out) loose = loose_files(cache) + if not loose: + # Nothing to fold in: every file of the cache is already in the pack, so writing it again copies the same + # bytes over the same entries (1.37 GiB here, 2.6 s, and a file system that then has the old pack's blocks to + # release as well). The installer runs this after every prepare. + return len(packed), 0, 0, out.stat().st_size if out.is_file() else 0 keys = sorted(set(packed) | set(loose)) tmp = out.with_name(PACK + '.tmp') index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys) diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index 90f32ad5..3ff5def2 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -140,10 +140,15 @@ def main(): t0 = time.monotonic() variants, prio = parse_list(read_list(a.list)) extracted = ROOT / 'build' / 'extracted' / a.version - shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs') + listed = len(variants) + shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs', everything=False) + missing = sum(1 for key, _ in variants if key not in shaders) + if missing > max(16, listed // 20): # the list names shaders the narrowed walk did not reach + print(f'{missing} of the {listed} listed variants are not among the {len(shaders)} shaders found: reading every ' + f'RomFS file instead', flush=True) + shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs') t1 = time.monotonic() progs, recs, seen, skipped = {}, [], set(), {'not in this version': 0, 'stage': 0} - listed = len(variants) variants += [(key, DEFAULT) for key in sorted(shaders)] for key, deps in variants: s = shaders.get(key) diff --git a/tools/shader_census.py b/tools/shader_census.py index ddc9c3d5..7f1e1790 100644 --- a/tools/shader_census.py +++ b/tools/shader_census.py @@ -172,7 +172,13 @@ def startup_package(name): name.startswith('Shader/') and name.endswith('.zs')) -def inventory(romfs, exefs): +# The RomFS directories that can hold shaders: the agl and gsys packages, the effects, terrain, the UI layout archives +# and the game's own shader packs. Every shader of 1.1.1 (27,576) comes from these six; they are 1,996 entries and +# 147 MiB, 0.9% of a RomFS whose every file the walk below used to read and decompress first (65 s against 2.8 s). +SHADER_DIRS = ('Shader/', 'Effect/', 'Lib/agl/', 'Lib/Terrain/', 'Lib/gsys/', 'UI/LayoutArchive/') + + +def inventory(romfs, exefs, everything=True): """Every unique shader: key -> dict(stage, control, code, source, origin, startup). As in MNG, a shader is a startup shader when it comes from an agl package, a game module, or a startup_package() RomFS file; the others come from effect and UI layout resources, which MNG translates only for bind states players have met. Also returns the number @@ -228,6 +234,8 @@ def inventory(romfs, exefs): pos += max(size, 8) for name in sorted(entries): + if not everything and not name.startswith(SHADER_DIRS): + continue raw = read(name) if raw[:4] not in (ZSTD, b'SARC', b'FSHA', b'BNSH', b'BAHS') and b'BNSH\0\0\0\0' not in raw: continue