diff --git a/docs/installer.md b/docs/installer.md index b6157b83..a3158d28 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -24,7 +24,7 @@ The install folder (`main.c` `portable_paths()` reads this layout; it stays port exefs/, romfs/ extracted from the player's files profile/ saves logs/ totk.log of the last run, totk.1.log to totk.4.log of the runs before it - shader and pipeline caches prepared by the setup + shader and pipeline caches prepared by the setup; shader-cache/shader-cache.pack holds the translations totk-install.json what built this folder: release version, input hashes, finished steps ``` @@ -50,7 +50,9 @@ keeps `profile/`. Saves carry over both ways: 1.4.3 converts a 1.2.1 save on loa hashes and the game version's patch page) and link them with the prebuilt runtime library into `totk.exe` (`build_game.py --runtime-lib`; about 3 min on 24 cores). 5. **Prepare** the RomFS patches and the shaders (`tools/make_romfs_patches.py`, `tools/prepare_shaders.py`, - `totk.exe --prepare-shaders`) in the install folder. + `totk.exe --prepare-shaders`) in the install folder, then fold the shader cache's loose files (four per shader, about + 117,000) into `shader-cache/shader-cache.pack` (`tools/pack_shader_cache.py`). totk.exe reads the pack first and + writes the translations it makes in play as loose files; running the tool again with the game closed folds those in. 6. **Clean up** the temp folder (generated C and objects, about 1.3 GB). A cancelled or failed setup resumes from the last finished step (`totk-install.json`). diff --git a/installer/layout.py b/installer/layout.py index 655c7c22..affcb63b 100644 --- a/installer/layout.py +++ b/installer/layout.py @@ -72,7 +72,8 @@ class Layout: def missing(self): """Paths the steps need that do not exist.""" need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py', - self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', self.clang, + self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', + self.clang, self.deps / 'include', self.deps / 'lib', self.runtime, self.runtime_lib, self.ninja, self.hactool, self.xlate, *self.dlls, *(f(v) for v in SUPPORTED for f in (self.patches, self.expected))] if self.shader_list: diff --git a/installer/steps.py b/installer/steps.py index 9d31fb13..b2109fa1 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -587,7 +587,8 @@ class Prepare(Step): def key(self, ctx): layout = ctx.layout xlate_id = layout.xlate.with_suffix('.id') - tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py')] + tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py', + 'pack_shader_cache.py')] return fingerprint('prepare-1', ctx.keys['compile'], layout.xlate, xlate_id, shader_list_text(layout), *tools) def current(self, ctx, record): @@ -638,6 +639,12 @@ class Prepare(Step): dest / 'exefs', '--list', shader_list], parse) if code or not (dest / 'shader-prepared.bin').is_file(): raise InstallError('prepare.shaders_failed', code=code, log=str(log)) + + # One shader-cache.pack instead of four files per shader (~117,000): totk.exe reads the pack first. + rep.progress('prepare', 'pack', force=True) + code, log = run_tool(ctx, 'prepare', layout.tools / 'pack_shader_cache.py', [dest / 'shader-cache']) + if code: + raise InstallError('prepare.pack_failed', code=code, log=str(log)) return {'xlate_sha256': file_sha256(xlate)} diff --git a/installer/text/en.json b/installer/text/en.json index 3425f5a5..5e8a5912 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -26,6 +26,7 @@ "phase.prepare.seed": "Reading the shaders from the game data", "phase.prepare.translate": "Translating the shaders", "phase.prepare.libraries": "Building the shaders for the graphics driver", + "phase.prepare.pack": "Packing the shader cache into one file", "phase.cleanup.ini": "Writing totk.ini", "phase.cleanup.temp": "Removing temporary files", @@ -76,6 +77,7 @@ "prepare.romfs_patches_failed": "Writing the game data patches failed (exit code {code}). Details: {log}", "prepare.shaders_failed": "Preparing the shaders failed (exit code {code}). Details: {log}", "cleanup.romfs_unreadable": "The game data in {path} is missing or cannot be read after the install. Restart Windows, then run the installer again on the same folder: it extracts the game data again and keeps the rest.", + "prepare.pack_failed": "Packing the shader cache failed (exit code {code}). Details: {log}", "cleanup.ini_failed": "totk.ini cannot be written: {detail}", "warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts, which the installer does not extract; some text may not show.", diff --git a/installer/text/ru.json b/installer/text/ru.json index b1dc33ae..fa4d3a76 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -26,6 +26,7 @@ "phase.prepare.seed": "Чтение шейдеров из данных игры", "phase.prepare.translate": "Трансляция шейдеров", "phase.prepare.libraries": "Сборка шейдеров для драйвера видеокарты", + "phase.prepare.pack": "Упаковка кэша шейдеров в один файл", "phase.cleanup.ini": "Запись totk.ini", "phase.cleanup.temp": "Удаление временных файлов", @@ -76,6 +77,7 @@ "prepare.romfs_patches_failed": "Не удалось записать патчи данных игры (код выхода {code}). Подробности: {log}", "prepare.shaders_failed": "Не удалось подготовить шейдеры (код выхода {code}). Подробности: {log}", "cleanup.romfs_unreadable": "Данные игры в {path} пропали или не читаются после установки. Перезагрузите Windows и запустите установщик снова на ту же папку: он заново распакует данные игры, остальное сохранится.", + "prepare.pack_failed": "Не удалось упаковать кэш шейдеров (код выхода {code}). Подробности: {log}", "cleanup.ini_failed": "Не удаётся записать totk.ini: {detail}", "warn.lang_needs_fonts": "Для языка «{lang}» нужны системные шрифты Switch, которые установка не извлекает; часть текста может не отображаться.", diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 4df23fea..15e2343a 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -18,6 +18,11 @@ // (64 hex digits) overrides it. A translator change therefore changes every cache key without a hand-made KEY_VERSION // bump; KEY_VERSION is for changes on this side (key layout, how the output is used). // +// Pack: tools/pack_shader_cache.py folds the .spv, .res.json, .use and in/*.in files into /shader-cache.pack +// (one file instead of four per shader: ~117,000 files after an install) and deletes them; the installer runs it after +// preparing the shaders. The pack is read-only here. Every read looks in the pack first and then for the loose file, +// and new translations are written as loose files as before, until the tool runs again (section "cache pack"). +// // Stored inputs: every translation also writes /in/.in (magic "TKSI"; stage, the full answers, the // control section, the code up to its length, c1), so a pipeline key record (vk_pipeline.c), which names its shaders by // input id, can be translated again under a new translator without guest memory (gfx_shader_load_input, @@ -125,6 +130,94 @@ static INIT_ONCE paths_once = INIT_ONCE_STATIC_INIT; #define read_file plat_file_read #define write_file plat_file_write +// ---- cache pack (/shader-cache.pack, written by tools/pack_shader_cache.py) +// Little-endian: "TKSPACK1", u32 count, u32 0, then count entries sorted by (kind, name), then the data. name is the +// 32 bytes the loose file's 64 hex digits spell. The index is read once (48 bytes a file: ~5.6 MB after an install); +// the data is read per lookup with positioned reads on one handle, safe from every thread. A pack that fails a check is +// ignored as a whole and the loose files are used. +enum { PACK_SPV = 1, PACK_RES = 2, PACK_USE = 3, PACK_IN = 4 }; +typedef struct { uint8_t name[32]; uint32_t kind, size; uint64_t offset; } PackEnt; +_Static_assert(sizeof(PackEnt) == 48, "shader-cache.pack entry layout"); +static HANDLE pack_file = INVALID_HANDLE_VALUE; +static PackEnt *pack_ix; +static uint32_t pack_n; + +static int pack_cmp(const PackEnt *a, uint32_t kind, const uint8_t name[32]) { + if (a->kind != kind) return a->kind < kind ? -1 : 1; + return memcmp(a->name, name, 32); +} + +static int pack_read_at(uint64_t offset, void *out, uint32_t n) { + OVERLAPPED o = {0}; + o.Offset = (DWORD)offset, o.OffsetHigh = (DWORD)(offset >> 32); + DWORD got = 0; + return ReadFile(pack_file, out, n, &got, &o) && got == n; +} + +// Opens /shader-cache.pack and reads its index (init_paths). Nothing happens when there is none. +static void pack_open(void) { + char p[PATH_CAP + 32]; + wchar_t w[PATH_CAP + 32]; + snprintf(p, sizeof p, "%s\\shader-cache.pack", cache_dir); + if (!MultiByteToWideChar(CP_UTF8, 0, p, -1, w, PATH_CAP + 32)) return; + HANDLE h = CreateFileW(w, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + if (h == INVALID_HANDLE_VALUE) return; + LARGE_INTEGER size; + uint8_t head[16]; + uint32_t n = 0; + const char *why = NULL; + PackEnt *ix = NULL; + pack_file = h; + if (!GetFileSizeEx(h, &size) || !pack_read_at(0, head, sizeof head) || memcmp(head, "TKSPACK1", 8)) why = "header"; + if (!why) { + memcpy(&n, head + 8, 4); + if (n > (1u << 24) || (uint64_t)n * sizeof *ix + sizeof head > (uint64_t)size.QuadPart || + !(ix = malloc((size_t)n * sizeof *ix + 1))) + why = "index size"; + else if (!pack_read_at(sizeof head, ix, n * (uint32_t)sizeof *ix)) why = "index read"; + } + for (uint32_t i = 0; !why && i < n; i++) { + if (ix[i].offset + ix[i].size > (uint64_t)size.QuadPart) why = "entry out of bounds"; + else if (i && pack_cmp(&ix[i - 1], ix[i].kind, ix[i].name) >= 0) why = "index order"; + } + if (why) { + rt_log("gfx_shader: %s is not a usable cache pack (%s); reading the loose files", p, why); + free(ix); + CloseHandle(h); + pack_file = INVALID_HANDLE_VALUE; + return; + } + pack_ix = ix, pack_n = n; + rt_log("gfx_shader: cache pack %s: %u files", p, n); +} + +static const PackEnt *pack_find(uint32_t kind, const uint8_t name[32]) { + size_t lo = 0, hi = pack_n; + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + int c = pack_cmp(&pack_ix[mid], kind, name); + if (!c) return &pack_ix[mid]; + if (c < 0) lo = mid + 1; + else hi = mid; + } + return NULL; +} + +// A packed file as plat_file_read returns a loose one: malloc'd, NUL after the last byte; NULL when absent or empty. +static uint8_t *pack_read(uint32_t kind, const uint8_t name[32], uint64_t *size) { + const PackEnt *e = pack_find(kind, name); + uint8_t *buf = e && e->size ? malloc((size_t)e->size + 1) : NULL; + if (!buf) return NULL; + if (!pack_read_at(e->offset, buf, e->size)) { + free(buf); + return NULL; + } + buf[e->size] = 0; + *size = e->size; + return buf; +} + // Appends a UTF-8 string as a JSON string body: ASCII as is, quotes and backslashes escaped, the rest as \uXXXX // (UTF-16 units), so the service's stdin encoding does not matter. static void json_esc(char **o, const char *s) { @@ -218,6 +311,7 @@ static BOOL CALLBACK init_paths(PINIT_ONCE once, PVOID param, PVOID *ctx) { if (cache_cfg && *cache_cfg && strlen(cache_cfg) < sizeof cache_dir) strcpy(cache_dir, cache_cfg); else if (exe_dir[0]) snprintf(cache_dir, sizeof cache_dir, "%s\\shader-cache", exe_dir); xlate_identity(); + if (cache_dir[0]) pack_open(); use_preload_start(); return TRUE; } @@ -468,15 +562,15 @@ static int spirv_complete(const uint32_t *w, uint64_t n) { return i == n; } -// Reads .spv and .res.json; NULL if either is missing or malformed (the caller translates again and -// overwrites both). iid: the input id, NULL when the caller sets it. -static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { +// Reads .spv and .res.json from the pack (packed 1) or the loose files; NULL if either is missing or +// malformed. iid: the input id, NULL when the caller sets it. +static GfxShader *disk_load_from(const uint8_t key[32], const uint8_t *iid, int packed) { static volatile LONG n_bad; char hex[65], p[PATH_CAP + 80]; key_hex(key, hex); uint64_t spv_size = 0, res_size = 0; snprintf(p, PATH_CAP + 80, "%s\\%s.res.json", cache_dir, hex); - uint8_t *res = read_file(p, &res_size); + uint8_t *res = packed ? pack_read(PACK_RES, key, &res_size) : read_file(p, &res_size); if (!res) return NULL; if (!reflection_valid((const char *)res, res_size)) { LONG n = InterlockedIncrement(&n_bad); @@ -485,7 +579,7 @@ static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { return NULL; } snprintf(p, PATH_CAP + 80, "%s\\%s.spv", cache_dir, hex); - uint8_t *spv = read_file(p, &spv_size); + uint8_t *spv = packed ? pack_read(PACK_SPV, key, &spv_size) : read_file(p, &spv_size); uint32_t magic = 0; if (spv && spv_size >= 20) memcpy(&magic, spv, 4); if (!spv || spv_size % 4 || magic != 0x07230203 || !spirv_complete((const uint32_t *)spv, spv_size / 4)) { @@ -503,6 +597,13 @@ static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { return s; } +// The pack, then the loose files (a newer translation of a key the pack holds a bad copy of); NULL when neither has a +// valid pair (the caller translates again and writes loose files). +static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { + GfxShader *s = pack_n ? disk_load_from(key, iid, 1) : NULL; + return s ? s : disk_load_from(key, iid, 0); +} + // ---- request building static const char *const tex_names[] = {NULL, "1d", "1darray", "buffer", "2d", "2darray", "2dms", "2dmsarray", "3d", @@ -751,10 +852,25 @@ static DWORD WINAPI use_preload(void *arg) { (void)arg; uint64_t t0 = plat_now_ns(); char pattern[PATH_CAP + 16], p[PATH_CAP + 80]; + int n = 0, bad = 0; + for (uint32_t i = 0; i < pack_n; i++) { // the packed masks; the loose files below add the masks found since + uint32_t u[5]; + UseFile **slot, *x; + if (pack_ix[i].kind != PACK_USE) continue; + if (pack_ix[i].size != sizeof u || !pack_read_at(pack_ix[i].offset, u, sizeof u) || u[0] != USE_MAGIC || + u[1] != USE_VERSION) { + bad++; + continue; + } + if (!*(slot = use_file_slot(pack_ix[i].name)) && (x = malloc(sizeof *x))) { + memcpy(x->name, pack_ix[i].name, 32); + x->next = NULL, x->m.slots = u[2], x->m.attrs = u[3], x->m.fouts = u[4]; + *slot = x, n++; + } + } snprintf(pattern, sizeof pattern, "%s\\*.use", cache_dir); WIN32_FIND_DATAA fd; HANDLE f = FindFirstFileExA(pattern, FindExInfoBasic, &fd, FindExSearchNameMatch, NULL, FIND_FIRST_EX_LARGE_FETCH); - int n = 0, bad = 0; if (f != INVALID_HANDLE_VALUE) { do { UseFile e = {0}; @@ -824,7 +940,8 @@ static int use_find(const uint8_t h[32], UseMask *m) { char p[PATH_CAP + 80]; use_path_of(name, p); uint64_t n = 0; - uint32_t *f = (uint32_t *)read_file(p, &n); + uint32_t *f = (uint32_t *)pack_read(PACK_USE, name, &n); + if (!f) f = (uint32_t *)read_file(p, &n); known = f && n == 20 && f[0] == USE_MAGIC && f[1] == USE_VERSION; if (known) m->slots = f[2], m->attrs = f[3], m->fouts = f[4]; free(f); @@ -1105,7 +1222,7 @@ static void in_store(const uint8_t iid[32], int stage, const uint8_t *control, c const GfxShaderAnswers *a) { char p[PATH_CAP + 80], tmp[PATH_CAP + 120]; in_path(iid, p); - if (file_exists(p)) return; + if (pack_find(PACK_IN, iid) || file_exists(p)) return; GfxShaderAnswers zero = {0}; InHeader h = {IN_MAGIC, IN_VERSION, (uint32_t)stage, r->length, r->c1_len, sizeof(GfxShaderAnswers)}; size_t n = sizeof h + sizeof zero + CONTROL_SIZE + r->length + r->c1_len; @@ -1127,7 +1244,8 @@ static int in_load(const uint8_t iid[32], Inputs *in) { char p[PATH_CAP + 80]; in_path(iid, p); uint64_t n = 0; - uint8_t *f = read_file(p, &n); + uint8_t *f = pack_read(PACK_IN, iid, &n); + if (!f) f = read_file(p, &n); InHeader h; if (!f || n < sizeof h) return free(f), 0; memcpy(&h, f, sizeof h); @@ -1425,7 +1543,7 @@ static DWORD WINAPI warm_worker(void *arg) { key_of_input(iid, key); key_hex(key, hex); snprintf(p, PATH_CAP + 80, "%s\\%s.spv", cache_dir, hex); - if (file_exists(p)) InterlockedIncrement(&w->other_cached); + if (pack_find(PACK_SPV, key) || file_exists(p)) InterlockedIncrement(&w->other_cached); else gfx_shader_load_input(iid, 1); } InterlockedIncrement64(&w->done); @@ -1463,7 +1581,14 @@ static size_t other_inputs(const uint8_t (*named)[32], size_t n_named, uint8_t ( char dir[PATH_CAP + 80]; snprintf(dir, sizeof dir, "%s\\in", cache_dir); InList l = {.named = named, .n_named = n_named}; - plat_file_list(dir, ".in", in_collect, &l); + char name[68]; + for (uint32_t i = 0; i < pack_n; i++) { // the packed inputs, under the name their loose file would have + if (pack_ix[i].kind != PACK_IN) continue; + key_hex(pack_ix[i].name, name); + memcpy(name + 64, ".in", 4); + if (!in_collect(name, &l)) break; + } + plat_file_list(dir, ".in", in_collect, &l); // in_store writes no loose file for a packed input: no duplicates return *out = l.v, l.n; } diff --git a/tests/pack_shader_cache_test.py b/tests/pack_shader_cache_test.py new file mode 100644 index 00000000..53356ef7 --- /dev/null +++ b/tests/pack_shader_cache_test.py @@ -0,0 +1,111 @@ +"""tools/pack_shader_cache.py on a fake shader cache: the pack layout gfx_shader.c reads (sorted index, bounds, data), +loose files folded in and deleted, a second run merging new loose files, a loose file winning over its packed copy, +malformed files skipped and --keep. No game files; stdlib only. + +Run: python tests/pack_shader_cache_test.py +""" +import hashlib +import struct +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +import pack_shader_cache as psc # noqa: E402 + +USE = struct.pack('<5I', 0x4d554b54, 1, 3, 0, 1) # gfx_shader.c use_store + + +def name(seed): + return hashlib.sha256(seed.encode()).hexdigest() + + +def read_pack(path): + """{(kind, name hex): bytes}, checking what gfx_shader.c pack_open checks.""" + b = path.read_bytes() + assert b[:8] == psc.MAGIC + count = struct.unpack_from('= 16 + 48 * count + assert last is None or (last[0], last[1]) < (kind, nm), 'index not sorted by (kind, name)' + last = (kind, nm) + out[(kind, nm.hex())] = b[offset:offset + size] + return out + + +class PackTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.cache = Path(self.tmp.name) / 'shader-cache' + (self.cache / 'in').mkdir(parents=True) + + def tearDown(self): + self.tmp.cleanup() + + def shader(self, seed, spv=b'\x03\x02\x23\x07spirv', res=b'{"x":1}'): + k = name(seed) + (self.cache / f'{k}.spv').write_bytes(spv + seed.encode()) + (self.cache / f'{k}.res.json').write_bytes(res) + (self.cache / f'{name("use" + seed)}.use').write_bytes(USE) + (self.cache / 'in').mkdir(exist_ok=True) # in_store makes it again after a pack removed it + (self.cache / 'in' / f'{name("in" + seed)}.in').write_bytes(b'TKSI' + seed.encode()) + return k + + def test_pack_holds_every_file_and_deletes_the_loose_ones(self): + keys = [self.shader(str(i)) for i in range(50)] + (self.cache / 'notes.txt').write_text('kept') + (self.cache / f'{keys[0]}.code').write_bytes(b'kept too') + n, new, skipped, _ = psc.pack(self.cache) + self.assertEqual((n, new, skipped), (200, 200, 0)) + got = read_pack(self.cache / psc.PACK) + self.assertEqual(got[(1, keys[3])], b'\x03\x02\x23\x07spirv3') + self.assertEqual(got[(2, keys[3])], b'{"x":1}') + self.assertEqual(got[(3, name('use3'))], USE) + self.assertEqual(got[(4, name('in3'))], b'TKSI3') + left = sorted(p.name for p in self.cache.iterdir()) + self.assertEqual(left, sorted([psc.PACK, 'notes.txt', f'{keys[0]}.code'])) # in/ removed when empty + + def test_second_run_merges_new_loose_files_and_loose_wins(self): + a = self.shader('a') + psc.pack(self.cache) + b = self.shader('b') + (self.cache / f'{a}.res.json').write_bytes(b'{"newer":1}') + n, new, _, _ = psc.pack(self.cache) + got = read_pack(self.cache / psc.PACK) + self.assertEqual(n, 8) + self.assertEqual(new, 5) + self.assertEqual(got[(2, a)], b'{"newer":1}') + self.assertEqual(got[(1, b)], b'\x03\x02\x23\x07spirvb') + + def test_malformed_files_are_skipped_or_replaced_by_the_packed_copy(self): + a = self.shader('a') + psc.pack(self.cache) + (self.cache / f'{name("usea")}.use').write_bytes(b'short') # bad loose copy: the packed one stays + (self.cache / f'{name("bad")}.use').write_bytes(b'short') # bad and nowhere else: skipped + (self.cache / f'{name("empty")}.spv').write_bytes(b'') + n, _, skipped, _ = psc.pack(self.cache) + got = read_pack(self.cache / psc.PACK) + self.assertEqual(got[(3, name('usea'))], USE) + self.assertNotIn((3, name('bad')), got) + self.assertEqual((n, skipped), (4, 2)) + self.assertTrue((self.cache / f'{name("bad")}.use').exists()) # never deleted unless packed + + def test_keep_leaves_the_loose_files(self): + k = self.shader('a') + psc.pack(self.cache, keep=True) + self.assertTrue((self.cache / f'{k}.spv').exists()) + self.assertEqual(len(read_pack(self.cache / psc.PACK)), 4) + + def test_a_broken_pack_is_rebuilt_from_the_loose_files(self): + self.shader('a') + (self.cache / psc.PACK).write_bytes(b'garbage') + n, _, _, _ = psc.pack(self.cache) + self.assertEqual(n, 4) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/make_release.py b/tools/make_release.py index 9fd9d33c..9c3c5f92 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -98,7 +98,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py', - 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py'] + 'pack_shader_cache.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon diff --git a/tools/pack_shader_cache.py b/tools/pack_shader_cache.py new file mode 100644 index 00000000..0b326d5d --- /dev/null +++ b/tools/pack_shader_cache.py @@ -0,0 +1,140 @@ +"""Fold the loose files of a shader cache into /shader-cache.pack and delete them. + +The translator writes four files per shader into the cache (runtime/gfx_shader.c): .spv, .res.json, +.use and in/.in, about 117,000 files after an install, which are slow to copy, back up, scan and sync. +totk.exe reads the pack first and the loose files after it, and keeps writing new translations as loose files, so +this tool can run again at any time the game is closed: it merges the existing pack with the loose files (a loose file +wins over a packed one of the same name) and writes the pack anew. The installer runs it after preparing the shaders. + +Layout, little-endian (gfx_shader.c, section "cache pack"): b'TKSPACK1', u32 count, u32 0, then count entries of 48 +bytes sorted by (kind, name): name (the 32 bytes the file name's 64 hex digits spell), u32 kind (1 .spv, 2 .res.json, +3 .use, 4 in/*.in), u32 size, u64 offset; then the data. + + python tools\\pack_shader_cache.py \\shader-cache [--keep] + +--keep leaves the loose files in place. Close the game first: a running totk.exe has read the old pack's index. +""" +import argparse +import os +import re +import struct +import sys +from pathlib import Path + +MAGIC = b'TKSPACK1' +PACK = 'shader-cache.pack' +KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4} +NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)') +USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks + + +def loose_files(cache): + """{(kind, name): path} of the cache's loose files; *.in only in in/.""" + found = {} + for folder, kinds in ((cache, (1, 2, 3)), (cache / 'in', (4,))): + if not folder.is_dir(): + continue + with os.scandir(folder) as it: + for e in it: + m = NAME.fullmatch(e.name) + if not m or not e.is_file(): + continue + kind = KINDS[m.group(2)] + if kind in kinds: + found[(kind, bytes.fromhex(m.group(1)))] = Path(e.path) + return found + + +def read_pack(path): + """{(kind, name): (offset, size)} of an existing pack; {} when there is none or it fails a check.""" + try: + f = open(path, 'rb') + except FileNotFoundError: + return {} + with f: + size = os.fstat(f.fileno()).st_size + head = f.read(16) + if len(head) != 16 or head[:8] != MAGIC: + print(f'{path}: not a cache pack, rebuilt from the loose files only', flush=True) + return {} + count = struct.unpack_from(' 0 + + +def pack(cache, keep=False): + cache = Path(cache) + out = cache / PACK + packed = read_pack(out) + loose = loose_files(cache) + keys = sorted(set(packed) | set(loose)) + tmp = out.with_name(PACK + '.tmp') + index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys) + old = open(out, 'rb') if packed else None + try: + with open(tmp, 'wb') as f: + f.write(bytes(offset)) # header and index, written once the offsets are known + for key in keys: + data = None + if key in loose: # a valid loose file, else the packed copy, else nothing + try: + data = loose[key].read_bytes() + except OSError: + pass + if data is not None and valid(key[0], data): + written.append(loose[key]) + else: + data = None + if data is None and key in packed: + old.seek(packed[key][0]) + data = old.read(packed[key][1]) + if data is None or not valid(key[0], data): + skipped += 1 + continue + f.write(data) + index.append(struct.pack('<32sIIQ', key[1], key[0], len(data), offset)) + offset += len(data) + f.seek(0) + f.write(MAGIC + struct.pack('