Merge task/shaderpack: shader cache in one pack file from pull request #64

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-31 07:36:02 +03:00
commit fc133df1c0
9 files changed
+406 -16

No files matched your search

+4 -2
View File
@@ -24,7 +24,7 @@ The install folder (`main.c` `portable_paths()` reads this layout; it stays port
exefs/, romfs/ extracted from the player's files
profile/ saves
logs/ totk.log of the last run, totk.1.log to totk.4.log of the runs before it
shader and pipeline caches prepared by the setup
shader and pipeline caches prepared by the setup; shader-cache/shader-cache.pack holds the translations
totk-install.json what built this folder: release version, input hashes, finished steps
```
@@ -50,7 +50,9 @@ keeps `profile/`. Saves carry over both ways: 1.4.3 converts a 1.2.1 save on loa
hashes and the game version's patch page) and link
them with the prebuilt runtime library into `totk.exe` (`build_game.py --runtime-lib`; about 3 min on 24 cores).
5. **Prepare** the RomFS patches and the shaders (`tools/make_romfs_patches.py`, `tools/prepare_shaders.py`,
`totk.exe --prepare-shaders`) in the install folder.
`totk.exe --prepare-shaders`) in the install folder, then fold the shader cache's loose files (four per shader, about
117,000) into `shader-cache/shader-cache.pack` (`tools/pack_shader_cache.py`). totk.exe reads the pack first and
writes the translations it makes in play as loose files; running the tool again with the game closed folds those in.
6. **Clean up** the temp folder (generated C and objects, about 1.3 GB).
A cancelled or failed setup resumes from the last finished step (`totk-install.json`).
+2 -1
View File
@@ -72,7 +72,8 @@ class Layout:
def missing(self):
"""Paths the steps need that do not exist."""
need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py',
self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', self.clang,
self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py',
self.clang,
self.deps / 'include', self.deps / 'lib', self.runtime, self.runtime_lib, self.ninja, self.hactool,
self.xlate, *self.dlls, *(f(v) for v in SUPPORTED for f in (self.patches, self.expected))]
if self.shader_list:
+8 -1
View File
@@ -587,7 +587,8 @@ class Prepare(Step):
def key(self, ctx):
layout = ctx.layout
xlate_id = layout.xlate.with_suffix('.id')
tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py')]
tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py',
'pack_shader_cache.py')]
return fingerprint('prepare-1', ctx.keys['compile'], layout.xlate, xlate_id, shader_list_text(layout), *tools)
def current(self, ctx, record):
@@ -638,6 +639,12 @@ class Prepare(Step):
dest / 'exefs', '--list', shader_list], parse)
if code or not (dest / 'shader-prepared.bin').is_file():
raise InstallError('prepare.shaders_failed', code=code, log=str(log))
# One shader-cache.pack instead of four files per shader (~117,000): totk.exe reads the pack first.
rep.progress('prepare', 'pack', force=True)
code, log = run_tool(ctx, 'prepare', layout.tools / 'pack_shader_cache.py', [dest / 'shader-cache'])
if code:
raise InstallError('prepare.pack_failed', code=code, log=str(log))
return {'xlate_sha256': file_sha256(xlate)}
+2
View File
@@ -26,6 +26,7 @@
"phase.prepare.seed": "Reading the shaders from the game data",
"phase.prepare.translate": "Translating the shaders",
"phase.prepare.libraries": "Building the shaders for the graphics driver",
"phase.prepare.pack": "Packing the shader cache into one file",
"phase.cleanup.ini": "Writing totk.ini",
"phase.cleanup.temp": "Removing temporary files",
@@ -76,6 +77,7 @@
"prepare.romfs_patches_failed": "Writing the game data patches failed (exit code {code}). Details: {log}",
"prepare.shaders_failed": "Preparing the shaders failed (exit code {code}). Details: {log}",
"cleanup.romfs_unreadable": "The game data in {path} is missing or cannot be read after the install. Restart Windows, then run the installer again on the same folder: it extracts the game data again and keeps the rest.",
"prepare.pack_failed": "Packing the shader cache failed (exit code {code}). Details: {log}",
"cleanup.ini_failed": "totk.ini cannot be written: {detail}",
"warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts, which the installer does not extract; some text may not show.",
+2
View File
@@ -26,6 +26,7 @@
"phase.prepare.seed": "Чтение шейдеров из данных игры",
"phase.prepare.translate": "Трансляция шейдеров",
"phase.prepare.libraries": "Сборка шейдеров для драйвера видеокарты",
"phase.prepare.pack": "Упаковка кэша шейдеров в один файл",
"phase.cleanup.ini": "Запись totk.ini",
"phase.cleanup.temp": "Удаление временных файлов",
@@ -76,6 +77,7 @@
"prepare.romfs_patches_failed": "Не удалось записать патчи данных игры (код выхода {code}). Подробности: {log}",
"prepare.shaders_failed": "Не удалось подготовить шейдеры (код выхода {code}). Подробности: {log}",
"cleanup.romfs_unreadable": "Данные игры в {path} пропали или не читаются после установки. Перезагрузите Windows и запустите установщик снова на ту же папку: он заново распакует данные игры, остальное сохранится.",
"prepare.pack_failed": "Не удалось упаковать кэш шейдеров (код выхода {code}). Подробности: {log}",
"cleanup.ini_failed": "Не удаётся записать totk.ini: {detail}",
"warn.lang_needs_fonts": "Для языка «{lang}» нужны системные шрифты Switch, которые установка не извлекает; часть текста может не отображаться.",
+136 -11
View File
@@ -18,6 +18,11 @@
// (64 hex digits) overrides it. A translator change therefore changes every cache key without a hand-made KEY_VERSION
// bump; KEY_VERSION is for changes on this side (key layout, how the output is used).
//
// Pack: tools/pack_shader_cache.py folds the .spv, .res.json, .use and in/*.in files into <cache>/shader-cache.pack
// (one file instead of four per shader: ~117,000 files after an install) and deletes them; the installer runs it after
// preparing the shaders. The pack is read-only here. Every read looks in the pack first and then for the loose file,
// and new translations are written as loose files as before, until the tool runs again (section "cache pack").
//
// Stored inputs: every translation also writes <cache>/in/<input id>.in (magic "TKSI"; stage, the full answers, the
// control section, the code up to its length, c1), so a pipeline key record (vk_pipeline.c), which names its shaders by
// input id, can be translated again under a new translator without guest memory (gfx_shader_load_input,
@@ -125,6 +130,94 @@ static INIT_ONCE paths_once = INIT_ONCE_STATIC_INIT;
#define read_file plat_file_read
#define write_file plat_file_write
// ---- cache pack (<cache>/shader-cache.pack, written by tools/pack_shader_cache.py)
// Little-endian: "TKSPACK1", u32 count, u32 0, then count entries sorted by (kind, name), then the data. name is the
// 32 bytes the loose file's 64 hex digits spell. The index is read once (48 bytes a file: ~5.6 MB after an install);
// the data is read per lookup with positioned reads on one handle, safe from every thread. A pack that fails a check is
// ignored as a whole and the loose files are used.
enum { PACK_SPV = 1, PACK_RES = 2, PACK_USE = 3, PACK_IN = 4 };
typedef struct { uint8_t name[32]; uint32_t kind, size; uint64_t offset; } PackEnt;
_Static_assert(sizeof(PackEnt) == 48, "shader-cache.pack entry layout");
static HANDLE pack_file = INVALID_HANDLE_VALUE;
static PackEnt *pack_ix;
static uint32_t pack_n;
static int pack_cmp(const PackEnt *a, uint32_t kind, const uint8_t name[32]) {
if (a->kind != kind) return a->kind < kind ? -1 : 1;
return memcmp(a->name, name, 32);
}
static int pack_read_at(uint64_t offset, void *out, uint32_t n) {
OVERLAPPED o = {0};
o.Offset = (DWORD)offset, o.OffsetHigh = (DWORD)(offset >> 32);
DWORD got = 0;
return ReadFile(pack_file, out, n, &got, &o) && got == n;
}
// Opens <cache>/shader-cache.pack and reads its index (init_paths). Nothing happens when there is none.
static void pack_open(void) {
char p[PATH_CAP + 32];
wchar_t w[PATH_CAP + 32];
snprintf(p, sizeof p, "%s\\shader-cache.pack", cache_dir);
if (!MultiByteToWideChar(CP_UTF8, 0, p, -1, w, PATH_CAP + 32)) return;
HANDLE h = CreateFileW(w, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL, NULL);
if (h == INVALID_HANDLE_VALUE) return;
LARGE_INTEGER size;
uint8_t head[16];
uint32_t n = 0;
const char *why = NULL;
PackEnt *ix = NULL;
pack_file = h;
if (!GetFileSizeEx(h, &size) || !pack_read_at(0, head, sizeof head) || memcmp(head, "TKSPACK1", 8)) why = "header";
if (!why) {
memcpy(&n, head + 8, 4);
if (n > (1u << 24) || (uint64_t)n * sizeof *ix + sizeof head > (uint64_t)size.QuadPart ||
!(ix = malloc((size_t)n * sizeof *ix + 1)))
why = "index size";
else if (!pack_read_at(sizeof head, ix, n * (uint32_t)sizeof *ix)) why = "index read";
}
for (uint32_t i = 0; !why && i < n; i++) {
if (ix[i].offset + ix[i].size > (uint64_t)size.QuadPart) why = "entry out of bounds";
else if (i && pack_cmp(&ix[i - 1], ix[i].kind, ix[i].name) >= 0) why = "index order";
}
if (why) {
rt_log("gfx_shader: %s is not a usable cache pack (%s); reading the loose files", p, why);
free(ix);
CloseHandle(h);
pack_file = INVALID_HANDLE_VALUE;
return;
}
pack_ix = ix, pack_n = n;
rt_log("gfx_shader: cache pack %s: %u files", p, n);
}
static const PackEnt *pack_find(uint32_t kind, const uint8_t name[32]) {
size_t lo = 0, hi = pack_n;
while (lo < hi) {
size_t mid = lo + (hi - lo) / 2;
int c = pack_cmp(&pack_ix[mid], kind, name);
if (!c) return &pack_ix[mid];
if (c < 0) lo = mid + 1;
else hi = mid;
}
return NULL;
}
// A packed file as plat_file_read returns a loose one: malloc'd, NUL after the last byte; NULL when absent or empty.
static uint8_t *pack_read(uint32_t kind, const uint8_t name[32], uint64_t *size) {
const PackEnt *e = pack_find(kind, name);
uint8_t *buf = e && e->size ? malloc((size_t)e->size + 1) : NULL;
if (!buf) return NULL;
if (!pack_read_at(e->offset, buf, e->size)) {
free(buf);
return NULL;
}
buf[e->size] = 0;
*size = e->size;
return buf;
}
// Appends a UTF-8 string as a JSON string body: ASCII as is, quotes and backslashes escaped, the rest as \uXXXX
// (UTF-16 units), so the service's stdin encoding does not matter.
static void json_esc(char **o, const char *s) {
@@ -218,6 +311,7 @@ static BOOL CALLBACK init_paths(PINIT_ONCE once, PVOID param, PVOID *ctx) {
if (cache_cfg && *cache_cfg && strlen(cache_cfg) < sizeof cache_dir) strcpy(cache_dir, cache_cfg);
else if (exe_dir[0]) snprintf(cache_dir, sizeof cache_dir, "%s\\shader-cache", exe_dir);
xlate_identity();
if (cache_dir[0]) pack_open();
use_preload_start();
return TRUE;
}
@@ -468,15 +562,15 @@ static int spirv_complete(const uint32_t *w, uint64_t n) {
return i == n;
}
// Reads <key>.spv and <key>.res.json; NULL if either is missing or malformed (the caller translates again and
// overwrites both). iid: the input id, NULL when the caller sets it.
static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
// Reads <key>.spv and <key>.res.json from the pack (packed 1) or the loose files; NULL if either is missing or
// malformed. iid: the input id, NULL when the caller sets it.
static GfxShader *disk_load_from(const uint8_t key[32], const uint8_t *iid, int packed) {
static volatile LONG n_bad;
char hex[65], p[PATH_CAP + 80];
key_hex(key, hex);
uint64_t spv_size = 0, res_size = 0;
snprintf(p, PATH_CAP + 80, "%s\\%s.res.json", cache_dir, hex);
uint8_t *res = read_file(p, &res_size);
uint8_t *res = packed ? pack_read(PACK_RES, key, &res_size) : read_file(p, &res_size);
if (!res) return NULL;
if (!reflection_valid((const char *)res, res_size)) {
LONG n = InterlockedIncrement(&n_bad);
@@ -485,7 +579,7 @@ static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
return NULL;
}
snprintf(p, PATH_CAP + 80, "%s\\%s.spv", cache_dir, hex);
uint8_t *spv = read_file(p, &spv_size);
uint8_t *spv = packed ? pack_read(PACK_SPV, key, &spv_size) : read_file(p, &spv_size);
uint32_t magic = 0;
if (spv && spv_size >= 20) memcpy(&magic, spv, 4);
if (!spv || spv_size % 4 || magic != 0x07230203 || !spirv_complete((const uint32_t *)spv, spv_size / 4)) {
@@ -503,6 +597,13 @@ static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
return s;
}
// The pack, then the loose files (a newer translation of a key the pack holds a bad copy of); NULL when neither has a
// valid pair (the caller translates again and writes loose files).
static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
GfxShader *s = pack_n ? disk_load_from(key, iid, 1) : NULL;
return s ? s : disk_load_from(key, iid, 0);
}
// ---- request building
static const char *const tex_names[] = {NULL, "1d", "1darray", "buffer", "2d", "2darray", "2dms", "2dmsarray", "3d",
@@ -751,10 +852,25 @@ static DWORD WINAPI use_preload(void *arg) {
(void)arg;
uint64_t t0 = plat_now_ns();
char pattern[PATH_CAP + 16], p[PATH_CAP + 80];
int n = 0, bad = 0;
for (uint32_t i = 0; i < pack_n; i++) { // the packed masks; the loose files below add the masks found since
uint32_t u[5];
UseFile **slot, *x;
if (pack_ix[i].kind != PACK_USE) continue;
if (pack_ix[i].size != sizeof u || !pack_read_at(pack_ix[i].offset, u, sizeof u) || u[0] != USE_MAGIC ||
u[1] != USE_VERSION) {
bad++;
continue;
}
if (!*(slot = use_file_slot(pack_ix[i].name)) && (x = malloc(sizeof *x))) {
memcpy(x->name, pack_ix[i].name, 32);
x->next = NULL, x->m.slots = u[2], x->m.attrs = u[3], x->m.fouts = u[4];
*slot = x, n++;
}
}
snprintf(pattern, sizeof pattern, "%s\\*.use", cache_dir);
WIN32_FIND_DATAA fd;
HANDLE f = FindFirstFileExA(pattern, FindExInfoBasic, &fd, FindExSearchNameMatch, NULL, FIND_FIRST_EX_LARGE_FETCH);
int n = 0, bad = 0;
if (f != INVALID_HANDLE_VALUE) {
do {
UseFile e = {0};
@@ -824,7 +940,8 @@ static int use_find(const uint8_t h[32], UseMask *m) {
char p[PATH_CAP + 80];
use_path_of(name, p);
uint64_t n = 0;
uint32_t *f = (uint32_t *)read_file(p, &n);
uint32_t *f = (uint32_t *)pack_read(PACK_USE, name, &n);
if (!f) f = (uint32_t *)read_file(p, &n);
known = f && n == 20 && f[0] == USE_MAGIC && f[1] == USE_VERSION;
if (known) m->slots = f[2], m->attrs = f[3], m->fouts = f[4];
free(f);
@@ -1105,7 +1222,7 @@ static void in_store(const uint8_t iid[32], int stage, const uint8_t *control, c
const GfxShaderAnswers *a) {
char p[PATH_CAP + 80], tmp[PATH_CAP + 120];
in_path(iid, p);
if (file_exists(p)) return;
if (pack_find(PACK_IN, iid) || file_exists(p)) return;
GfxShaderAnswers zero = {0};
InHeader h = {IN_MAGIC, IN_VERSION, (uint32_t)stage, r->length, r->c1_len, sizeof(GfxShaderAnswers)};
size_t n = sizeof h + sizeof zero + CONTROL_SIZE + r->length + r->c1_len;
@@ -1127,7 +1244,8 @@ static int in_load(const uint8_t iid[32], Inputs *in) {
char p[PATH_CAP + 80];
in_path(iid, p);
uint64_t n = 0;
uint8_t *f = read_file(p, &n);
uint8_t *f = pack_read(PACK_IN, iid, &n);
if (!f) f = read_file(p, &n);
InHeader h;
if (!f || n < sizeof h) return free(f), 0;
memcpy(&h, f, sizeof h);
@@ -1425,7 +1543,7 @@ static DWORD WINAPI warm_worker(void *arg) {
key_of_input(iid, key);
key_hex(key, hex);
snprintf(p, PATH_CAP + 80, "%s\\%s.spv", cache_dir, hex);
if (file_exists(p)) InterlockedIncrement(&w->other_cached);
if (pack_find(PACK_SPV, key) || file_exists(p)) InterlockedIncrement(&w->other_cached);
else gfx_shader_load_input(iid, 1);
}
InterlockedIncrement64(&w->done);
@@ -1463,7 +1581,14 @@ static size_t other_inputs(const uint8_t (*named)[32], size_t n_named, uint8_t (
char dir[PATH_CAP + 80];
snprintf(dir, sizeof dir, "%s\\in", cache_dir);
InList l = {.named = named, .n_named = n_named};
plat_file_list(dir, ".in", in_collect, &l);
char name[68];
for (uint32_t i = 0; i < pack_n; i++) { // the packed inputs, under the name their loose file would have
if (pack_ix[i].kind != PACK_IN) continue;
key_hex(pack_ix[i].name, name);
memcpy(name + 64, ".in", 4);
if (!in_collect(name, &l)) break;
}
plat_file_list(dir, ".in", in_collect, &l); // in_store writes no loose file for a packed input: no duplicates
return *out = l.v, l.n;
}
+111
View File
@@ -0,0 +1,111 @@
"""tools/pack_shader_cache.py on a fake shader cache: the pack layout gfx_shader.c reads (sorted index, bounds, data),
loose files folded in and deleted, a second run merging new loose files, a loose file winning over its packed copy,
malformed files skipped and --keep. No game files; stdlib only.
Run: python tests/pack_shader_cache_test.py
"""
import hashlib
import struct
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'tools'))
import pack_shader_cache as psc # noqa: E402
USE = struct.pack('<5I', 0x4d554b54, 1, 3, 0, 1) # gfx_shader.c use_store
def name(seed):
return hashlib.sha256(seed.encode()).hexdigest()
def read_pack(path):
"""{(kind, name hex): bytes}, checking what gfx_shader.c pack_open checks."""
b = path.read_bytes()
assert b[:8] == psc.MAGIC
count = struct.unpack_from('<I', b, 8)[0]
out, last = {}, None
for i in range(count):
nm, kind, size, offset = struct.unpack_from('<32sIIQ', b, 16 + 48 * i)
assert offset + size <= len(b) and offset >= 16 + 48 * count
assert last is None or (last[0], last[1]) < (kind, nm), 'index not sorted by (kind, name)'
last = (kind, nm)
out[(kind, nm.hex())] = b[offset:offset + size]
return out
class PackTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.cache = Path(self.tmp.name) / 'shader-cache'
(self.cache / 'in').mkdir(parents=True)
def tearDown(self):
self.tmp.cleanup()
def shader(self, seed, spv=b'\x03\x02\x23\x07spirv', res=b'{"x":1}'):
k = name(seed)
(self.cache / f'{k}.spv').write_bytes(spv + seed.encode())
(self.cache / f'{k}.res.json').write_bytes(res)
(self.cache / f'{name("use" + seed)}.use').write_bytes(USE)
(self.cache / 'in').mkdir(exist_ok=True) # in_store makes it again after a pack removed it
(self.cache / 'in' / f'{name("in" + seed)}.in').write_bytes(b'TKSI' + seed.encode())
return k
def test_pack_holds_every_file_and_deletes_the_loose_ones(self):
keys = [self.shader(str(i)) for i in range(50)]
(self.cache / 'notes.txt').write_text('kept')
(self.cache / f'{keys[0]}.code').write_bytes(b'kept too')
n, new, skipped, _ = psc.pack(self.cache)
self.assertEqual((n, new, skipped), (200, 200, 0))
got = read_pack(self.cache / psc.PACK)
self.assertEqual(got[(1, keys[3])], b'\x03\x02\x23\x07spirv3')
self.assertEqual(got[(2, keys[3])], b'{"x":1}')
self.assertEqual(got[(3, name('use3'))], USE)
self.assertEqual(got[(4, name('in3'))], b'TKSI3')
left = sorted(p.name for p in self.cache.iterdir())
self.assertEqual(left, sorted([psc.PACK, 'notes.txt', f'{keys[0]}.code'])) # in/ removed when empty
def test_second_run_merges_new_loose_files_and_loose_wins(self):
a = self.shader('a')
psc.pack(self.cache)
b = self.shader('b')
(self.cache / f'{a}.res.json').write_bytes(b'{"newer":1}')
n, new, _, _ = psc.pack(self.cache)
got = read_pack(self.cache / psc.PACK)
self.assertEqual(n, 8)
self.assertEqual(new, 5)
self.assertEqual(got[(2, a)], b'{"newer":1}')
self.assertEqual(got[(1, b)], b'\x03\x02\x23\x07spirvb')
def test_malformed_files_are_skipped_or_replaced_by_the_packed_copy(self):
a = self.shader('a')
psc.pack(self.cache)
(self.cache / f'{name("usea")}.use').write_bytes(b'short') # bad loose copy: the packed one stays
(self.cache / f'{name("bad")}.use').write_bytes(b'short') # bad and nowhere else: skipped
(self.cache / f'{name("empty")}.spv').write_bytes(b'')
n, _, skipped, _ = psc.pack(self.cache)
got = read_pack(self.cache / psc.PACK)
self.assertEqual(got[(3, name('usea'))], USE)
self.assertNotIn((3, name('bad')), got)
self.assertEqual((n, skipped), (4, 2))
self.assertTrue((self.cache / f'{name("bad")}.use').exists()) # never deleted unless packed
def test_keep_leaves_the_loose_files(self):
k = self.shader('a')
psc.pack(self.cache, keep=True)
self.assertTrue((self.cache / f'{k}.spv').exists())
self.assertEqual(len(read_pack(self.cache / psc.PACK)), 4)
def test_a_broken_pack_is_rebuilt_from_the_loose_files(self):
self.shader('a')
(self.cache / psc.PACK).write_bytes(b'garbage')
n, _, _, _ = psc.pack(self.cache)
self.assertEqual(n, 4)
if __name__ == '__main__':
unittest.main()
+1 -1
View File
@@ -98,7 +98,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
# tools/ files the installer steps run (and what they import), copied from the repo.
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py',
'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py']
'pack_shader_cache.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py']
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
+140
View File
@@ -0,0 +1,140 @@
"""Fold the loose files of a shader cache into <cache>/shader-cache.pack and delete them.
The translator writes four files per shader into the cache (runtime/gfx_shader.c): <key>.spv, <key>.res.json,
<name>.use and in/<input id>.in, about 117,000 files after an install, which are slow to copy, back up, scan and sync.
totk.exe reads the pack first and the loose files after it, and keeps writing new translations as loose files, so
this tool can run again at any time the game is closed: it merges the existing pack with the loose files (a loose file
wins over a packed one of the same name) and writes the pack anew. The installer runs it after preparing the shaders.
Layout, little-endian (gfx_shader.c, section "cache pack"): b'TKSPACK1', u32 count, u32 0, then count entries of 48
bytes sorted by (kind, name): name (the 32 bytes the file name's 64 hex digits spell), u32 kind (1 .spv, 2 .res.json,
3 .use, 4 in/*.in), u32 size, u64 offset; then the data.
python tools\\pack_shader_cache.py <install folder>\\shader-cache [--keep]
--keep leaves the loose files in place. Close the game first: a running totk.exe has read the old pack's index.
"""
import argparse
import os
import re
import struct
import sys
from pathlib import Path
MAGIC = b'TKSPACK1'
PACK = 'shader-cache.pack'
KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4}
NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)')
USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks
def loose_files(cache):
"""{(kind, name): path} of the cache's loose files; *.in only in in/."""
found = {}
for folder, kinds in ((cache, (1, 2, 3)), (cache / 'in', (4,))):
if not folder.is_dir():
continue
with os.scandir(folder) as it:
for e in it:
m = NAME.fullmatch(e.name)
if not m or not e.is_file():
continue
kind = KINDS[m.group(2)]
if kind in kinds:
found[(kind, bytes.fromhex(m.group(1)))] = Path(e.path)
return found
def read_pack(path):
"""{(kind, name): (offset, size)} of an existing pack; {} when there is none or it fails a check."""
try:
f = open(path, 'rb')
except FileNotFoundError:
return {}
with f:
size = os.fstat(f.fileno()).st_size
head = f.read(16)
if len(head) != 16 or head[:8] != MAGIC:
print(f'{path}: not a cache pack, rebuilt from the loose files only', flush=True)
return {}
count = struct.unpack_from('<I', head, 8)[0]
raw = f.read(48 * count)
if len(raw) != 48 * count:
print(f'{path}: truncated index, rebuilt from the loose files only', flush=True)
return {}
out = {}
for i in range(count):
name, kind, length, offset = struct.unpack_from('<32sIIQ', raw, 48 * i)
if offset + length <= size:
out[(kind, name)] = (offset, length)
return out
def valid(kind, data):
return len(data) == USE_SIZE if kind == 3 else len(data) > 0
def pack(cache, keep=False):
cache = Path(cache)
out = cache / PACK
packed = read_pack(out)
loose = loose_files(cache)
keys = sorted(set(packed) | set(loose))
tmp = out.with_name(PACK + '.tmp')
index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys)
old = open(out, 'rb') if packed else None
try:
with open(tmp, 'wb') as f:
f.write(bytes(offset)) # header and index, written once the offsets are known
for key in keys:
data = None
if key in loose: # a valid loose file, else the packed copy, else nothing
try:
data = loose[key].read_bytes()
except OSError:
pass
if data is not None and valid(key[0], data):
written.append(loose[key])
else:
data = None
if data is None and key in packed:
old.seek(packed[key][0])
data = old.read(packed[key][1])
if data is None or not valid(key[0], data):
skipped += 1
continue
f.write(data)
index.append(struct.pack('<32sIIQ', key[1], key[0], len(data), offset))
offset += len(data)
f.seek(0)
f.write(MAGIC + struct.pack('<II', len(index), 0) + b''.join(index))
f.flush()
os.fsync(f.fileno())
finally:
if old:
old.close()
os.replace(tmp, out)
if not keep:
for p in written:
p.unlink(missing_ok=True)
try:
(cache / 'in').rmdir() # only when empty
except OSError:
pass
return len(index), len(written), skipped, offset
def main():
ap = argparse.ArgumentParser(description='Fold a shader cache into shader-cache.pack.')
ap.add_argument('cache', type=Path, help='the shader-cache folder next to totk.exe')
ap.add_argument('--keep', action='store_true', help='keep the loose files')
a = ap.parse_args()
if not a.cache.is_dir():
sys.exit(f'{a.cache} is not a folder')
n, new, skipped, size = pack(a.cache, a.keep)
print(f'{a.cache / PACK}: {n} files, {size / 2**20:.0f} MiB ({new} loose files folded in'
f'{", " + str(skipped) + " unreadable or empty skipped" if skipped else ""})', flush=True)
if __name__ == '__main__':
main()