Find the shaders by their directories instead of reading the whole RomFS

inventory() read every RomFS file into memory and decompressed it before deciding whether the file can hold a
shader. On 1.1.1 that is 304,469 entries and 15.86 GiB, and the shaders live in six directories: 1,996 entries,
147 MiB, 0.9% of the bytes. The walk visits those directories and reads everything else only when the shader list
names keys it did not find there.

Measured on one install: 66.50 s -> 3.35 s for the same 27,576 shaders, key for key.

pack_shader_cache.py also rewrote its pack after every prepare although there was nothing to fold in: 2.6 s of
copying 1.37 GiB over the same entries, and a file system that then releases the old pack's blocks. It returns at
once when the cache holds no loose files.
This commit is contained in:
shenjack committed 2026-10-31 21:42:04 +08:00
1 parent 7d62cbfa85
commit 6813b60ed9
3 files changed
+21 -3

No files matched your search

+5
View File
@@ -79,6 +79,11 @@ def pack(cache, keep=False):
out = cache / PACK
packed = read_pack(out)
loose = loose_files(cache)
if not loose:
# Nothing to fold in: every file of the cache is already in the pack, so writing it again copies the same
# bytes over the same entries (1.37 GiB here, 2.6 s, and a file system that then has the old pack's blocks to
# release as well). The installer runs this after every prepare.
return len(packed), 0, 0, out.stat().st_size if out.is_file() else 0
keys = sorted(set(packed) | set(loose))
tmp = out.with_name(PACK + '.tmp')
index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys)
+7 -2
View File
@@ -140,10 +140,15 @@ def main():
t0 = time.monotonic()
variants, prio = parse_list(read_list(a.list))
extracted = ROOT / 'build' / 'extracted' / a.version
shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs')
listed = len(variants)
shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs', everything=False)
missing = sum(1 for key, _ in variants if key not in shaders)
if missing > max(16, listed // 20): # the list names shaders the narrowed walk did not reach
print(f'{missing} of the {listed} listed variants are not among the {len(shaders)} shaders found: reading every '
f'RomFS file instead', flush=True)
shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs')
t1 = time.monotonic()
progs, recs, seen, skipped = {}, [], set(), {'not in this version': 0, 'stage': 0}
listed = len(variants)
variants += [(key, DEFAULT) for key in sorted(shaders)]
for key, deps in variants:
s = shaders.get(key)
+9 -1
View File
@@ -172,7 +172,13 @@ def startup_package(name):
name.startswith('Shader/') and name.endswith('.zs'))
def inventory(romfs, exefs):
# The RomFS directories that can hold shaders: the agl and gsys packages, the effects, terrain, the UI layout archives
# and the game's own shader packs. Every shader of 1.1.1 (27,576) comes from these six; they are 1,996 entries and
# 147 MiB, 0.9% of a RomFS whose every file the walk below used to read and decompress first (65 s against 2.8 s).
SHADER_DIRS = ('Shader/', 'Effect/', 'Lib/agl/', 'Lib/Terrain/', 'Lib/gsys/', 'UI/LayoutArchive/')
def inventory(romfs, exefs, everything=True):
"""Every unique shader: key -> dict(stage, control, code, source, origin, startup). As in MNG, a shader is a startup
shader when it comes from an agl package, a game module, or a startup_package() RomFS file; the others come from
effect and UI layout resources, which MNG translates only for bind states players have met. Also returns the number
@@ -228,6 +234,8 @@ def inventory(romfs, exefs):
pos += max(size, 8)
for name in sorted(entries):
if not everything and not name.startswith(SHADER_DIRS):
continue
raw = read(name)
if raw[:4] not in (ZSTD, b'SARC', b'FSHA', b'BNSH', b'BAHS') and b'BNSH\0\0\0\0' not in raw:
continue