diff --git a/tools/pack_shader_cache.py b/tools/pack_shader_cache.py new file mode 100644 index 00000000..0b326d5d --- /dev/null +++ b/tools/pack_shader_cache.py @@ -0,0 +1,140 @@ +"""Fold the loose files of a shader cache into /shader-cache.pack and delete them. + +The translator writes four files per shader into the cache (runtime/gfx_shader.c): .spv, .res.json, +.use and in/.in, about 117,000 files after an install, which are slow to copy, back up, scan and sync. +totk.exe reads the pack first and the loose files after it, and keeps writing new translations as loose files, so +this tool can run again at any time the game is closed: it merges the existing pack with the loose files (a loose file +wins over a packed one of the same name) and writes the pack anew. The installer runs it after preparing the shaders. + +Layout, little-endian (gfx_shader.c, section "cache pack"): b'TKSPACK1', u32 count, u32 0, then count entries of 48 +bytes sorted by (kind, name): name (the 32 bytes the file name's 64 hex digits spell), u32 kind (1 .spv, 2 .res.json, +3 .use, 4 in/*.in), u32 size, u64 offset; then the data. + + python tools\\pack_shader_cache.py \\shader-cache [--keep] + +--keep leaves the loose files in place. Close the game first: a running totk.exe has read the old pack's index. +""" +import argparse +import os +import re +import struct +import sys +from pathlib import Path + +MAGIC = b'TKSPACK1' +PACK = 'shader-cache.pack' +KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4} +NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)') +USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks + + +def loose_files(cache): + """{(kind, name): path} of the cache's loose files; *.in only in in/.""" + found = {} + for folder, kinds in ((cache, (1, 2, 3)), (cache / 'in', (4,))): + if not folder.is_dir(): + continue + with os.scandir(folder) as it: + for e in it: + m = NAME.fullmatch(e.name) + if not m or not e.is_file(): + continue + kind = KINDS[m.group(2)] + if kind in kinds: + found[(kind, bytes.fromhex(m.group(1)))] = Path(e.path) + return found + + +def read_pack(path): + """{(kind, name): (offset, size)} of an existing pack; {} when there is none or it fails a check.""" + try: + f = open(path, 'rb') + except FileNotFoundError: + return {} + with f: + size = os.fstat(f.fileno()).st_size + head = f.read(16) + if len(head) != 16 or head[:8] != MAGIC: + print(f'{path}: not a cache pack, rebuilt from the loose files only', flush=True) + return {} + count = struct.unpack_from(' 0 + + +def pack(cache, keep=False): + cache = Path(cache) + out = cache / PACK + packed = read_pack(out) + loose = loose_files(cache) + keys = sorted(set(packed) | set(loose)) + tmp = out.with_name(PACK + '.tmp') + index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys) + old = open(out, 'rb') if packed else None + try: + with open(tmp, 'wb') as f: + f.write(bytes(offset)) # header and index, written once the offsets are known + for key in keys: + data = None + if key in loose: # a valid loose file, else the packed copy, else nothing + try: + data = loose[key].read_bytes() + except OSError: + pass + if data is not None and valid(key[0], data): + written.append(loose[key]) + else: + data = None + if data is None and key in packed: + old.seek(packed[key][0]) + data = old.read(packed[key][1]) + if data is None or not valid(key[0], data): + skipped += 1 + continue + f.write(data) + index.append(struct.pack('<32sIIQ', key[1], key[0], len(data), offset)) + offset += len(data) + f.seek(0) + f.write(MAGIC + struct.pack('