mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 20:25:57 +00:00
Merge task/shaderpack: shader cache in one pack file from pull request #64
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
fc133df1c0
9 files changed
+406
-16
No files matched your search
+4
-2
@@ -24,7 +24,7 @@ The install folder (`main.c` `portable_paths()` reads this layout; it stays port
|
||||
exefs/, romfs/ extracted from the player's files
|
||||
profile/ saves
|
||||
logs/ totk.log of the last run, totk.1.log to totk.4.log of the runs before it
|
||||
shader and pipeline caches prepared by the setup
|
||||
shader and pipeline caches prepared by the setup; shader-cache/shader-cache.pack holds the translations
|
||||
totk-install.json what built this folder: release version, input hashes, finished steps
|
||||
```
|
||||
|
||||
@@ -50,7 +50,9 @@ keeps `profile/`. Saves carry over both ways: 1.4.3 converts a 1.2.1 save on loa
|
||||
hashes and the game version's patch page) and link
|
||||
them with the prebuilt runtime library into `totk.exe` (`build_game.py --runtime-lib`; about 3 min on 24 cores).
|
||||
5. **Prepare** the RomFS patches and the shaders (`tools/make_romfs_patches.py`, `tools/prepare_shaders.py`,
|
||||
`totk.exe --prepare-shaders`) in the install folder.
|
||||
`totk.exe --prepare-shaders`) in the install folder, then fold the shader cache's loose files (four per shader, about
|
||||
117,000) into `shader-cache/shader-cache.pack` (`tools/pack_shader_cache.py`). totk.exe reads the pack first and
|
||||
writes the translations it makes in play as loose files; running the tool again with the game closed folds those in.
|
||||
6. **Clean up** the temp folder (generated C and objects, about 1.3 GB).
|
||||
|
||||
A cancelled or failed setup resumes from the last finished step (`totk-install.json`).
|
||||
|
||||
+2
-1
@@ -72,7 +72,8 @@ class Layout:
|
||||
def missing(self):
|
||||
"""Paths the steps need that do not exist."""
|
||||
need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py',
|
||||
self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', self.clang,
|
||||
self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py',
|
||||
self.clang,
|
||||
self.deps / 'include', self.deps / 'lib', self.runtime, self.runtime_lib, self.ninja, self.hactool,
|
||||
self.xlate, *self.dlls, *(f(v) for v in SUPPORTED for f in (self.patches, self.expected))]
|
||||
if self.shader_list:
|
||||
|
||||
+8
-1
@@ -587,7 +587,8 @@ class Prepare(Step):
|
||||
def key(self, ctx):
|
||||
layout = ctx.layout
|
||||
xlate_id = layout.xlate.with_suffix('.id')
|
||||
tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py')]
|
||||
tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py',
|
||||
'pack_shader_cache.py')]
|
||||
return fingerprint('prepare-1', ctx.keys['compile'], layout.xlate, xlate_id, shader_list_text(layout), *tools)
|
||||
|
||||
def current(self, ctx, record):
|
||||
@@ -638,6 +639,12 @@ class Prepare(Step):
|
||||
dest / 'exefs', '--list', shader_list], parse)
|
||||
if code or not (dest / 'shader-prepared.bin').is_file():
|
||||
raise InstallError('prepare.shaders_failed', code=code, log=str(log))
|
||||
|
||||
# One shader-cache.pack instead of four files per shader (~117,000): totk.exe reads the pack first.
|
||||
rep.progress('prepare', 'pack', force=True)
|
||||
code, log = run_tool(ctx, 'prepare', layout.tools / 'pack_shader_cache.py', [dest / 'shader-cache'])
|
||||
if code:
|
||||
raise InstallError('prepare.pack_failed', code=code, log=str(log))
|
||||
return {'xlate_sha256': file_sha256(xlate)}
|
||||
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
"phase.prepare.seed": "Reading the shaders from the game data",
|
||||
"phase.prepare.translate": "Translating the shaders",
|
||||
"phase.prepare.libraries": "Building the shaders for the graphics driver",
|
||||
"phase.prepare.pack": "Packing the shader cache into one file",
|
||||
"phase.cleanup.ini": "Writing totk.ini",
|
||||
"phase.cleanup.temp": "Removing temporary files",
|
||||
|
||||
@@ -76,6 +77,7 @@
|
||||
"prepare.romfs_patches_failed": "Writing the game data patches failed (exit code {code}). Details: {log}",
|
||||
"prepare.shaders_failed": "Preparing the shaders failed (exit code {code}). Details: {log}",
|
||||
"cleanup.romfs_unreadable": "The game data in {path} is missing or cannot be read after the install. Restart Windows, then run the installer again on the same folder: it extracts the game data again and keeps the rest.",
|
||||
"prepare.pack_failed": "Packing the shader cache failed (exit code {code}). Details: {log}",
|
||||
"cleanup.ini_failed": "totk.ini cannot be written: {detail}",
|
||||
|
||||
"warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts, which the installer does not extract; some text may not show.",
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
"phase.prepare.seed": "Чтение шейдеров из данных игры",
|
||||
"phase.prepare.translate": "Трансляция шейдеров",
|
||||
"phase.prepare.libraries": "Сборка шейдеров для драйвера видеокарты",
|
||||
"phase.prepare.pack": "Упаковка кэша шейдеров в один файл",
|
||||
"phase.cleanup.ini": "Запись totk.ini",
|
||||
"phase.cleanup.temp": "Удаление временных файлов",
|
||||
|
||||
@@ -76,6 +77,7 @@
|
||||
"prepare.romfs_patches_failed": "Не удалось записать патчи данных игры (код выхода {code}). Подробности: {log}",
|
||||
"prepare.shaders_failed": "Не удалось подготовить шейдеры (код выхода {code}). Подробности: {log}",
|
||||
"cleanup.romfs_unreadable": "Данные игры в {path} пропали или не читаются после установки. Перезагрузите Windows и запустите установщик снова на ту же папку: он заново распакует данные игры, остальное сохранится.",
|
||||
"prepare.pack_failed": "Не удалось упаковать кэш шейдеров (код выхода {code}). Подробности: {log}",
|
||||
"cleanup.ini_failed": "Не удаётся записать totk.ini: {detail}",
|
||||
|
||||
"warn.lang_needs_fonts": "Для языка «{lang}» нужны системные шрифты Switch, которые установка не извлекает; часть текста может не отображаться.",
|
||||
|
||||
+136
-11
@@ -18,6 +18,11 @@
|
||||
// (64 hex digits) overrides it. A translator change therefore changes every cache key without a hand-made KEY_VERSION
|
||||
// bump; KEY_VERSION is for changes on this side (key layout, how the output is used).
|
||||
//
|
||||
// Pack: tools/pack_shader_cache.py folds the .spv, .res.json, .use and in/*.in files into <cache>/shader-cache.pack
|
||||
// (one file instead of four per shader: ~117,000 files after an install) and deletes them; the installer runs it after
|
||||
// preparing the shaders. The pack is read-only here. Every read looks in the pack first and then for the loose file,
|
||||
// and new translations are written as loose files as before, until the tool runs again (section "cache pack").
|
||||
//
|
||||
// Stored inputs: every translation also writes <cache>/in/<input id>.in (magic "TKSI"; stage, the full answers, the
|
||||
// control section, the code up to its length, c1), so a pipeline key record (vk_pipeline.c), which names its shaders by
|
||||
// input id, can be translated again under a new translator without guest memory (gfx_shader_load_input,
|
||||
@@ -125,6 +130,94 @@ static INIT_ONCE paths_once = INIT_ONCE_STATIC_INIT;
|
||||
#define read_file plat_file_read
|
||||
#define write_file plat_file_write
|
||||
|
||||
// ---- cache pack (<cache>/shader-cache.pack, written by tools/pack_shader_cache.py)
|
||||
// Little-endian: "TKSPACK1", u32 count, u32 0, then count entries sorted by (kind, name), then the data. name is the
|
||||
// 32 bytes the loose file's 64 hex digits spell. The index is read once (48 bytes a file: ~5.6 MB after an install);
|
||||
// the data is read per lookup with positioned reads on one handle, safe from every thread. A pack that fails a check is
|
||||
// ignored as a whole and the loose files are used.
|
||||
enum { PACK_SPV = 1, PACK_RES = 2, PACK_USE = 3, PACK_IN = 4 };
|
||||
typedef struct { uint8_t name[32]; uint32_t kind, size; uint64_t offset; } PackEnt;
|
||||
_Static_assert(sizeof(PackEnt) == 48, "shader-cache.pack entry layout");
|
||||
static HANDLE pack_file = INVALID_HANDLE_VALUE;
|
||||
static PackEnt *pack_ix;
|
||||
static uint32_t pack_n;
|
||||
|
||||
static int pack_cmp(const PackEnt *a, uint32_t kind, const uint8_t name[32]) {
|
||||
if (a->kind != kind) return a->kind < kind ? -1 : 1;
|
||||
return memcmp(a->name, name, 32);
|
||||
}
|
||||
|
||||
static int pack_read_at(uint64_t offset, void *out, uint32_t n) {
|
||||
OVERLAPPED o = {0};
|
||||
o.Offset = (DWORD)offset, o.OffsetHigh = (DWORD)(offset >> 32);
|
||||
DWORD got = 0;
|
||||
return ReadFile(pack_file, out, n, &got, &o) && got == n;
|
||||
}
|
||||
|
||||
// Opens <cache>/shader-cache.pack and reads its index (init_paths). Nothing happens when there is none.
|
||||
static void pack_open(void) {
|
||||
char p[PATH_CAP + 32];
|
||||
wchar_t w[PATH_CAP + 32];
|
||||
snprintf(p, sizeof p, "%s\\shader-cache.pack", cache_dir);
|
||||
if (!MultiByteToWideChar(CP_UTF8, 0, p, -1, w, PATH_CAP + 32)) return;
|
||||
HANDLE h = CreateFileW(w, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, OPEN_EXISTING,
|
||||
FILE_ATTRIBUTE_NORMAL, NULL);
|
||||
if (h == INVALID_HANDLE_VALUE) return;
|
||||
LARGE_INTEGER size;
|
||||
uint8_t head[16];
|
||||
uint32_t n = 0;
|
||||
const char *why = NULL;
|
||||
PackEnt *ix = NULL;
|
||||
pack_file = h;
|
||||
if (!GetFileSizeEx(h, &size) || !pack_read_at(0, head, sizeof head) || memcmp(head, "TKSPACK1", 8)) why = "header";
|
||||
if (!why) {
|
||||
memcpy(&n, head + 8, 4);
|
||||
if (n > (1u << 24) || (uint64_t)n * sizeof *ix + sizeof head > (uint64_t)size.QuadPart ||
|
||||
!(ix = malloc((size_t)n * sizeof *ix + 1)))
|
||||
why = "index size";
|
||||
else if (!pack_read_at(sizeof head, ix, n * (uint32_t)sizeof *ix)) why = "index read";
|
||||
}
|
||||
for (uint32_t i = 0; !why && i < n; i++) {
|
||||
if (ix[i].offset + ix[i].size > (uint64_t)size.QuadPart) why = "entry out of bounds";
|
||||
else if (i && pack_cmp(&ix[i - 1], ix[i].kind, ix[i].name) >= 0) why = "index order";
|
||||
}
|
||||
if (why) {
|
||||
rt_log("gfx_shader: %s is not a usable cache pack (%s); reading the loose files", p, why);
|
||||
free(ix);
|
||||
CloseHandle(h);
|
||||
pack_file = INVALID_HANDLE_VALUE;
|
||||
return;
|
||||
}
|
||||
pack_ix = ix, pack_n = n;
|
||||
rt_log("gfx_shader: cache pack %s: %u files", p, n);
|
||||
}
|
||||
|
||||
static const PackEnt *pack_find(uint32_t kind, const uint8_t name[32]) {
|
||||
size_t lo = 0, hi = pack_n;
|
||||
while (lo < hi) {
|
||||
size_t mid = lo + (hi - lo) / 2;
|
||||
int c = pack_cmp(&pack_ix[mid], kind, name);
|
||||
if (!c) return &pack_ix[mid];
|
||||
if (c < 0) lo = mid + 1;
|
||||
else hi = mid;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// A packed file as plat_file_read returns a loose one: malloc'd, NUL after the last byte; NULL when absent or empty.
|
||||
static uint8_t *pack_read(uint32_t kind, const uint8_t name[32], uint64_t *size) {
|
||||
const PackEnt *e = pack_find(kind, name);
|
||||
uint8_t *buf = e && e->size ? malloc((size_t)e->size + 1) : NULL;
|
||||
if (!buf) return NULL;
|
||||
if (!pack_read_at(e->offset, buf, e->size)) {
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
buf[e->size] = 0;
|
||||
*size = e->size;
|
||||
return buf;
|
||||
}
|
||||
|
||||
// Appends a UTF-8 string as a JSON string body: ASCII as is, quotes and backslashes escaped, the rest as \uXXXX
|
||||
// (UTF-16 units), so the service's stdin encoding does not matter.
|
||||
static void json_esc(char **o, const char *s) {
|
||||
@@ -218,6 +311,7 @@ static BOOL CALLBACK init_paths(PINIT_ONCE once, PVOID param, PVOID *ctx) {
|
||||
if (cache_cfg && *cache_cfg && strlen(cache_cfg) < sizeof cache_dir) strcpy(cache_dir, cache_cfg);
|
||||
else if (exe_dir[0]) snprintf(cache_dir, sizeof cache_dir, "%s\\shader-cache", exe_dir);
|
||||
xlate_identity();
|
||||
if (cache_dir[0]) pack_open();
|
||||
use_preload_start();
|
||||
return TRUE;
|
||||
}
|
||||
@@ -468,15 +562,15 @@ static int spirv_complete(const uint32_t *w, uint64_t n) {
|
||||
return i == n;
|
||||
}
|
||||
|
||||
// Reads <key>.spv and <key>.res.json; NULL if either is missing or malformed (the caller translates again and
|
||||
// overwrites both). iid: the input id, NULL when the caller sets it.
|
||||
static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
|
||||
// Reads <key>.spv and <key>.res.json from the pack (packed 1) or the loose files; NULL if either is missing or
|
||||
// malformed. iid: the input id, NULL when the caller sets it.
|
||||
static GfxShader *disk_load_from(const uint8_t key[32], const uint8_t *iid, int packed) {
|
||||
static volatile LONG n_bad;
|
||||
char hex[65], p[PATH_CAP + 80];
|
||||
key_hex(key, hex);
|
||||
uint64_t spv_size = 0, res_size = 0;
|
||||
snprintf(p, PATH_CAP + 80, "%s\\%s.res.json", cache_dir, hex);
|
||||
uint8_t *res = read_file(p, &res_size);
|
||||
uint8_t *res = packed ? pack_read(PACK_RES, key, &res_size) : read_file(p, &res_size);
|
||||
if (!res) return NULL;
|
||||
if (!reflection_valid((const char *)res, res_size)) {
|
||||
LONG n = InterlockedIncrement(&n_bad);
|
||||
@@ -485,7 +579,7 @@ static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
|
||||
return NULL;
|
||||
}
|
||||
snprintf(p, PATH_CAP + 80, "%s\\%s.spv", cache_dir, hex);
|
||||
uint8_t *spv = read_file(p, &spv_size);
|
||||
uint8_t *spv = packed ? pack_read(PACK_SPV, key, &spv_size) : read_file(p, &spv_size);
|
||||
uint32_t magic = 0;
|
||||
if (spv && spv_size >= 20) memcpy(&magic, spv, 4);
|
||||
if (!spv || spv_size % 4 || magic != 0x07230203 || !spirv_complete((const uint32_t *)spv, spv_size / 4)) {
|
||||
@@ -503,6 +597,13 @@ static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// The pack, then the loose files (a newer translation of a key the pack holds a bad copy of); NULL when neither has a
|
||||
// valid pair (the caller translates again and writes loose files).
|
||||
static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
|
||||
GfxShader *s = pack_n ? disk_load_from(key, iid, 1) : NULL;
|
||||
return s ? s : disk_load_from(key, iid, 0);
|
||||
}
|
||||
|
||||
// ---- request building
|
||||
|
||||
static const char *const tex_names[] = {NULL, "1d", "1darray", "buffer", "2d", "2darray", "2dms", "2dmsarray", "3d",
|
||||
@@ -751,10 +852,25 @@ static DWORD WINAPI use_preload(void *arg) {
|
||||
(void)arg;
|
||||
uint64_t t0 = plat_now_ns();
|
||||
char pattern[PATH_CAP + 16], p[PATH_CAP + 80];
|
||||
int n = 0, bad = 0;
|
||||
for (uint32_t i = 0; i < pack_n; i++) { // the packed masks; the loose files below add the masks found since
|
||||
uint32_t u[5];
|
||||
UseFile **slot, *x;
|
||||
if (pack_ix[i].kind != PACK_USE) continue;
|
||||
if (pack_ix[i].size != sizeof u || !pack_read_at(pack_ix[i].offset, u, sizeof u) || u[0] != USE_MAGIC ||
|
||||
u[1] != USE_VERSION) {
|
||||
bad++;
|
||||
continue;
|
||||
}
|
||||
if (!*(slot = use_file_slot(pack_ix[i].name)) && (x = malloc(sizeof *x))) {
|
||||
memcpy(x->name, pack_ix[i].name, 32);
|
||||
x->next = NULL, x->m.slots = u[2], x->m.attrs = u[3], x->m.fouts = u[4];
|
||||
*slot = x, n++;
|
||||
}
|
||||
}
|
||||
snprintf(pattern, sizeof pattern, "%s\\*.use", cache_dir);
|
||||
WIN32_FIND_DATAA fd;
|
||||
HANDLE f = FindFirstFileExA(pattern, FindExInfoBasic, &fd, FindExSearchNameMatch, NULL, FIND_FIRST_EX_LARGE_FETCH);
|
||||
int n = 0, bad = 0;
|
||||
if (f != INVALID_HANDLE_VALUE) {
|
||||
do {
|
||||
UseFile e = {0};
|
||||
@@ -824,7 +940,8 @@ static int use_find(const uint8_t h[32], UseMask *m) {
|
||||
char p[PATH_CAP + 80];
|
||||
use_path_of(name, p);
|
||||
uint64_t n = 0;
|
||||
uint32_t *f = (uint32_t *)read_file(p, &n);
|
||||
uint32_t *f = (uint32_t *)pack_read(PACK_USE, name, &n);
|
||||
if (!f) f = (uint32_t *)read_file(p, &n);
|
||||
known = f && n == 20 && f[0] == USE_MAGIC && f[1] == USE_VERSION;
|
||||
if (known) m->slots = f[2], m->attrs = f[3], m->fouts = f[4];
|
||||
free(f);
|
||||
@@ -1105,7 +1222,7 @@ static void in_store(const uint8_t iid[32], int stage, const uint8_t *control, c
|
||||
const GfxShaderAnswers *a) {
|
||||
char p[PATH_CAP + 80], tmp[PATH_CAP + 120];
|
||||
in_path(iid, p);
|
||||
if (file_exists(p)) return;
|
||||
if (pack_find(PACK_IN, iid) || file_exists(p)) return;
|
||||
GfxShaderAnswers zero = {0};
|
||||
InHeader h = {IN_MAGIC, IN_VERSION, (uint32_t)stage, r->length, r->c1_len, sizeof(GfxShaderAnswers)};
|
||||
size_t n = sizeof h + sizeof zero + CONTROL_SIZE + r->length + r->c1_len;
|
||||
@@ -1127,7 +1244,8 @@ static int in_load(const uint8_t iid[32], Inputs *in) {
|
||||
char p[PATH_CAP + 80];
|
||||
in_path(iid, p);
|
||||
uint64_t n = 0;
|
||||
uint8_t *f = read_file(p, &n);
|
||||
uint8_t *f = pack_read(PACK_IN, iid, &n);
|
||||
if (!f) f = read_file(p, &n);
|
||||
InHeader h;
|
||||
if (!f || n < sizeof h) return free(f), 0;
|
||||
memcpy(&h, f, sizeof h);
|
||||
@@ -1425,7 +1543,7 @@ static DWORD WINAPI warm_worker(void *arg) {
|
||||
key_of_input(iid, key);
|
||||
key_hex(key, hex);
|
||||
snprintf(p, PATH_CAP + 80, "%s\\%s.spv", cache_dir, hex);
|
||||
if (file_exists(p)) InterlockedIncrement(&w->other_cached);
|
||||
if (pack_find(PACK_SPV, key) || file_exists(p)) InterlockedIncrement(&w->other_cached);
|
||||
else gfx_shader_load_input(iid, 1);
|
||||
}
|
||||
InterlockedIncrement64(&w->done);
|
||||
@@ -1463,7 +1581,14 @@ static size_t other_inputs(const uint8_t (*named)[32], size_t n_named, uint8_t (
|
||||
char dir[PATH_CAP + 80];
|
||||
snprintf(dir, sizeof dir, "%s\\in", cache_dir);
|
||||
InList l = {.named = named, .n_named = n_named};
|
||||
plat_file_list(dir, ".in", in_collect, &l);
|
||||
char name[68];
|
||||
for (uint32_t i = 0; i < pack_n; i++) { // the packed inputs, under the name their loose file would have
|
||||
if (pack_ix[i].kind != PACK_IN) continue;
|
||||
key_hex(pack_ix[i].name, name);
|
||||
memcpy(name + 64, ".in", 4);
|
||||
if (!in_collect(name, &l)) break;
|
||||
}
|
||||
plat_file_list(dir, ".in", in_collect, &l); // in_store writes no loose file for a packed input: no duplicates
|
||||
return *out = l.v, l.n;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""tools/pack_shader_cache.py on a fake shader cache: the pack layout gfx_shader.c reads (sorted index, bounds, data),
|
||||
loose files folded in and deleted, a second run merging new loose files, a loose file winning over its packed copy,
|
||||
malformed files skipped and --keep. No game files; stdlib only.
|
||||
|
||||
Run: python tests/pack_shader_cache_test.py
|
||||
"""
|
||||
import hashlib
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import pack_shader_cache as psc # noqa: E402
|
||||
|
||||
USE = struct.pack('<5I', 0x4d554b54, 1, 3, 0, 1) # gfx_shader.c use_store
|
||||
|
||||
|
||||
def name(seed):
|
||||
return hashlib.sha256(seed.encode()).hexdigest()
|
||||
|
||||
|
||||
def read_pack(path):
|
||||
"""{(kind, name hex): bytes}, checking what gfx_shader.c pack_open checks."""
|
||||
b = path.read_bytes()
|
||||
assert b[:8] == psc.MAGIC
|
||||
count = struct.unpack_from('<I', b, 8)[0]
|
||||
out, last = {}, None
|
||||
for i in range(count):
|
||||
nm, kind, size, offset = struct.unpack_from('<32sIIQ', b, 16 + 48 * i)
|
||||
assert offset + size <= len(b) and offset >= 16 + 48 * count
|
||||
assert last is None or (last[0], last[1]) < (kind, nm), 'index not sorted by (kind, name)'
|
||||
last = (kind, nm)
|
||||
out[(kind, nm.hex())] = b[offset:offset + size]
|
||||
return out
|
||||
|
||||
|
||||
class PackTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cache = Path(self.tmp.name) / 'shader-cache'
|
||||
(self.cache / 'in').mkdir(parents=True)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def shader(self, seed, spv=b'\x03\x02\x23\x07spirv', res=b'{"x":1}'):
|
||||
k = name(seed)
|
||||
(self.cache / f'{k}.spv').write_bytes(spv + seed.encode())
|
||||
(self.cache / f'{k}.res.json').write_bytes(res)
|
||||
(self.cache / f'{name("use" + seed)}.use').write_bytes(USE)
|
||||
(self.cache / 'in').mkdir(exist_ok=True) # in_store makes it again after a pack removed it
|
||||
(self.cache / 'in' / f'{name("in" + seed)}.in').write_bytes(b'TKSI' + seed.encode())
|
||||
return k
|
||||
|
||||
def test_pack_holds_every_file_and_deletes_the_loose_ones(self):
|
||||
keys = [self.shader(str(i)) for i in range(50)]
|
||||
(self.cache / 'notes.txt').write_text('kept')
|
||||
(self.cache / f'{keys[0]}.code').write_bytes(b'kept too')
|
||||
n, new, skipped, _ = psc.pack(self.cache)
|
||||
self.assertEqual((n, new, skipped), (200, 200, 0))
|
||||
got = read_pack(self.cache / psc.PACK)
|
||||
self.assertEqual(got[(1, keys[3])], b'\x03\x02\x23\x07spirv3')
|
||||
self.assertEqual(got[(2, keys[3])], b'{"x":1}')
|
||||
self.assertEqual(got[(3, name('use3'))], USE)
|
||||
self.assertEqual(got[(4, name('in3'))], b'TKSI3')
|
||||
left = sorted(p.name for p in self.cache.iterdir())
|
||||
self.assertEqual(left, sorted([psc.PACK, 'notes.txt', f'{keys[0]}.code'])) # in/ removed when empty
|
||||
|
||||
def test_second_run_merges_new_loose_files_and_loose_wins(self):
|
||||
a = self.shader('a')
|
||||
psc.pack(self.cache)
|
||||
b = self.shader('b')
|
||||
(self.cache / f'{a}.res.json').write_bytes(b'{"newer":1}')
|
||||
n, new, _, _ = psc.pack(self.cache)
|
||||
got = read_pack(self.cache / psc.PACK)
|
||||
self.assertEqual(n, 8)
|
||||
self.assertEqual(new, 5)
|
||||
self.assertEqual(got[(2, a)], b'{"newer":1}')
|
||||
self.assertEqual(got[(1, b)], b'\x03\x02\x23\x07spirvb')
|
||||
|
||||
def test_malformed_files_are_skipped_or_replaced_by_the_packed_copy(self):
|
||||
a = self.shader('a')
|
||||
psc.pack(self.cache)
|
||||
(self.cache / f'{name("usea")}.use').write_bytes(b'short') # bad loose copy: the packed one stays
|
||||
(self.cache / f'{name("bad")}.use').write_bytes(b'short') # bad and nowhere else: skipped
|
||||
(self.cache / f'{name("empty")}.spv').write_bytes(b'')
|
||||
n, _, skipped, _ = psc.pack(self.cache)
|
||||
got = read_pack(self.cache / psc.PACK)
|
||||
self.assertEqual(got[(3, name('usea'))], USE)
|
||||
self.assertNotIn((3, name('bad')), got)
|
||||
self.assertEqual((n, skipped), (4, 2))
|
||||
self.assertTrue((self.cache / f'{name("bad")}.use').exists()) # never deleted unless packed
|
||||
|
||||
def test_keep_leaves_the_loose_files(self):
|
||||
k = self.shader('a')
|
||||
psc.pack(self.cache, keep=True)
|
||||
self.assertTrue((self.cache / f'{k}.spv').exists())
|
||||
self.assertEqual(len(read_pack(self.cache / psc.PACK)), 4)
|
||||
|
||||
def test_a_broken_pack_is_rebuilt_from_the_loose_files(self):
|
||||
self.shader('a')
|
||||
(self.cache / psc.PACK).write_bytes(b'garbage')
|
||||
n, _, _, _ = psc.pack(self.cache)
|
||||
self.assertEqual(n, 4)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -98,7 +98,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
|
||||
|
||||
# tools/ files the installer steps run (and what they import), copied from the repo.
|
||||
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py',
|
||||
'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py']
|
||||
'pack_shader_cache.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py']
|
||||
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
|
||||
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
|
||||
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Fold the loose files of a shader cache into <cache>/shader-cache.pack and delete them.
|
||||
|
||||
The translator writes four files per shader into the cache (runtime/gfx_shader.c): <key>.spv, <key>.res.json,
|
||||
<name>.use and in/<input id>.in, about 117,000 files after an install, which are slow to copy, back up, scan and sync.
|
||||
totk.exe reads the pack first and the loose files after it, and keeps writing new translations as loose files, so
|
||||
this tool can run again at any time the game is closed: it merges the existing pack with the loose files (a loose file
|
||||
wins over a packed one of the same name) and writes the pack anew. The installer runs it after preparing the shaders.
|
||||
|
||||
Layout, little-endian (gfx_shader.c, section "cache pack"): b'TKSPACK1', u32 count, u32 0, then count entries of 48
|
||||
bytes sorted by (kind, name): name (the 32 bytes the file name's 64 hex digits spell), u32 kind (1 .spv, 2 .res.json,
|
||||
3 .use, 4 in/*.in), u32 size, u64 offset; then the data.
|
||||
|
||||
python tools\\pack_shader_cache.py <install folder>\\shader-cache [--keep]
|
||||
|
||||
--keep leaves the loose files in place. Close the game first: a running totk.exe has read the old pack's index.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAGIC = b'TKSPACK1'
|
||||
PACK = 'shader-cache.pack'
|
||||
KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4}
|
||||
NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)')
|
||||
USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks
|
||||
|
||||
|
||||
def loose_files(cache):
|
||||
"""{(kind, name): path} of the cache's loose files; *.in only in in/."""
|
||||
found = {}
|
||||
for folder, kinds in ((cache, (1, 2, 3)), (cache / 'in', (4,))):
|
||||
if not folder.is_dir():
|
||||
continue
|
||||
with os.scandir(folder) as it:
|
||||
for e in it:
|
||||
m = NAME.fullmatch(e.name)
|
||||
if not m or not e.is_file():
|
||||
continue
|
||||
kind = KINDS[m.group(2)]
|
||||
if kind in kinds:
|
||||
found[(kind, bytes.fromhex(m.group(1)))] = Path(e.path)
|
||||
return found
|
||||
|
||||
|
||||
def read_pack(path):
|
||||
"""{(kind, name): (offset, size)} of an existing pack; {} when there is none or it fails a check."""
|
||||
try:
|
||||
f = open(path, 'rb')
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
with f:
|
||||
size = os.fstat(f.fileno()).st_size
|
||||
head = f.read(16)
|
||||
if len(head) != 16 or head[:8] != MAGIC:
|
||||
print(f'{path}: not a cache pack, rebuilt from the loose files only', flush=True)
|
||||
return {}
|
||||
count = struct.unpack_from('<I', head, 8)[0]
|
||||
raw = f.read(48 * count)
|
||||
if len(raw) != 48 * count:
|
||||
print(f'{path}: truncated index, rebuilt from the loose files only', flush=True)
|
||||
return {}
|
||||
out = {}
|
||||
for i in range(count):
|
||||
name, kind, length, offset = struct.unpack_from('<32sIIQ', raw, 48 * i)
|
||||
if offset + length <= size:
|
||||
out[(kind, name)] = (offset, length)
|
||||
return out
|
||||
|
||||
|
||||
def valid(kind, data):
|
||||
return len(data) == USE_SIZE if kind == 3 else len(data) > 0
|
||||
|
||||
|
||||
def pack(cache, keep=False):
|
||||
cache = Path(cache)
|
||||
out = cache / PACK
|
||||
packed = read_pack(out)
|
||||
loose = loose_files(cache)
|
||||
keys = sorted(set(packed) | set(loose))
|
||||
tmp = out.with_name(PACK + '.tmp')
|
||||
index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys)
|
||||
old = open(out, 'rb') if packed else None
|
||||
try:
|
||||
with open(tmp, 'wb') as f:
|
||||
f.write(bytes(offset)) # header and index, written once the offsets are known
|
||||
for key in keys:
|
||||
data = None
|
||||
if key in loose: # a valid loose file, else the packed copy, else nothing
|
||||
try:
|
||||
data = loose[key].read_bytes()
|
||||
except OSError:
|
||||
pass
|
||||
if data is not None and valid(key[0], data):
|
||||
written.append(loose[key])
|
||||
else:
|
||||
data = None
|
||||
if data is None and key in packed:
|
||||
old.seek(packed[key][0])
|
||||
data = old.read(packed[key][1])
|
||||
if data is None or not valid(key[0], data):
|
||||
skipped += 1
|
||||
continue
|
||||
f.write(data)
|
||||
index.append(struct.pack('<32sIIQ', key[1], key[0], len(data), offset))
|
||||
offset += len(data)
|
||||
f.seek(0)
|
||||
f.write(MAGIC + struct.pack('<II', len(index), 0) + b''.join(index))
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
finally:
|
||||
if old:
|
||||
old.close()
|
||||
os.replace(tmp, out)
|
||||
if not keep:
|
||||
for p in written:
|
||||
p.unlink(missing_ok=True)
|
||||
try:
|
||||
(cache / 'in').rmdir() # only when empty
|
||||
except OSError:
|
||||
pass
|
||||
return len(index), len(written), skipped, offset
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description='Fold a shader cache into shader-cache.pack.')
|
||||
ap.add_argument('cache', type=Path, help='the shader-cache folder next to totk.exe')
|
||||
ap.add_argument('--keep', action='store_true', help='keep the loose files')
|
||||
a = ap.parse_args()
|
||||
if not a.cache.is_dir():
|
||||
sys.exit(f'{a.cache} is not a folder')
|
||||
n, new, skipped, size = pack(a.cache, a.keep)
|
||||
print(f'{a.cache / PACK}: {n} files, {size / 2**20:.0f} MiB ({new} loose files folded in'
|
||||
f'{", " + str(skipped) + " unreadable or empty skipped" if skipped else ""})', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user