From e4e397d28c6183d95f118d2f1539657ad1e8a964 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 07:53:12 +0300 Subject: [PATCH 01/16] Run the vendored extract scripts through a stub that adds their directory to sys.path, and read the shader list from tools/data when present The embeddable Python ignores the script directory on sys.path, so extract_romfs.py failed to import extract_exefs. prepare_shaders.py no longer needs git or refs/ when tools/data/shader_list.tsv exists. Co-Authored-By: Claude Opus 5.5 --- tools/extract.py | 9 ++++++++- tools/prepare_shaders.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tools/extract.py b/tools/extract.py index c2c0d0ed..e50d384d 100644 --- a/tools/extract.py +++ b/tools/extract.py @@ -62,9 +62,16 @@ def version_dir(expected): return 'v0' if expected['version'] == 0 else 'v' + expected['display_version'].replace('.', '') +# The embeddable Python (a ._pth file) does not put a script's directory on sys.path, so a vendored script that imports +# its siblings fails when started by path. This stub adds the directory, then runs the script as __main__. +RUN_SCRIPT = ('import os, runpy, sys; p = sys.argv[1]; sys.path.insert(0, os.path.dirname(p)); sys.argv = sys.argv[1:]; ' + "runpy.run_path(p, run_name='__main__')") + + def run_logged(command, log): + args = [str(c) for c in command] with log.open('w') as out: - code = subprocess.run([str(c) for c in command], stdout=out, stderr=subprocess.STDOUT).returncode + code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT).returncode require(code == 0, f'{Path(command[1]).name} failed; see {log.relative_to(ROOT)}') diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index ad8069a6..4596758f 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -16,8 +16,9 @@ pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "sta a 12-core machine (inventory 1, translation 3, library builds 0.6) and 1.3 GB of shader cache; the seed files can be deleted afterwards. -Game-derived output stays in the build directory (ignored). The list is read from refs/ at preparation time; nothing -of it is copied into the repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314: +Game-derived output stays in the build directory (ignored). The list is read at preparation time from tools/data/shader_list.tsv when that file exists (the installer archive +bundles it; it stays out of the repository), else from refs/ through git; nothing of it is copied into the +repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14: .venv-py314\\Scripts\\python tools\\prepare_shaders.py build\\v121 [--version v121] [--list ] [--no-run] """ @@ -50,9 +51,14 @@ CONTROL = 0x880 ANS_REC = 4 + 32 + 32 + 147 + 1 +BUNDLED_LIST = ROOT / 'tools' / 'data' / 'shader_list.tsv' # the installer archive carries a copy (no git, no refs/) + + def read_list(path): if path: return Path(path).read_text(encoding='utf-8') + if BUNDLED_LIST.is_file(): + return BUNDLED_LIST.read_text(encoding='utf-8') return subprocess.run(['git', '-C', str(ROOT / 'refs' / 'MacNativeGames'), 'show', ':'.join(LIST_REF)], check=True, capture_output=True, encoding='utf-8').stdout @@ -111,7 +117,7 @@ def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('build', type=Path, help='build directory holding totk.exe (build\\v121)') ap.add_argument('--version', default='v121', help='build/extracted/') - ap.add_argument('--list', help='a shader_list.tsv instead of the one in refs/MacNativeGames') + ap.add_argument('--list', help='a shader_list.tsv instead of tools/data/shader_list.tsv or refs/MacNativeGames') ap.add_argument('--no-run', action='store_true', help='write the seed files only') a = ap.parse_args() t0 = time.monotonic() From 278693c8bf14d7f7c32088276612d30bf6e48863 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 08:00:36 +0300 Subject: [PATCH 02/16] Let build_game.py build with an llvm-mingw toolchain and split the runtime into a static library Co-Authored-By: Claude Opus 5.5 --- tools/build_game.py | 93 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 10 deletions(-) diff --git a/tools/build_game.py b/tools/build_game.py index 4ac220cc..73f75154 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -7,6 +7,8 @@ recompiled only when it, a header it includes or its flags change. .venv\\Scripts\\python tools\\build_game.py --modules subsdk0 # one module (links if it has no calls into others) .venv\\Scripts\\python tools\\build_game.py --jobs 8 .venv\\Scripts\\python tools\\build_game.py --out build/v121-t2 # a separate build directory + .venv\\Scripts\\python tools\\build_game.py --toolchain toolchain/llvm-mingw-/llvm-mingw--ucrt-x86_64 \\ + --out build/v121-llvmmingw # llvm-mingw instead of MSYS2 UCRT64 Every runtime/*.c is compiled. build//modules.c carries the layout, the linked function tables, the expected SHA-256 of each ExeFS file (tools/data/expected-.json) and the default game paths @@ -34,6 +36,15 @@ Compile jobs share a memory budget (MEM_BUDGET, default the smaller of 50 GiB an big machine does not run out of memory on the largest chunks. Each phase reports its wall time and the peak memory of its processes (Windows job object accounting: committed private bytes). +--toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes +from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR +(default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++ +runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects. + +Split build. --make-runtime-lib compiles only runtime/ into /libtotkrt.a (what CI prebuilds). --runtime-lib PATH +then compiles only the generated C and the install-time files (modules.c and INSTALL_SIDE, which bake the player's +game hashes in) and links them with that library. The library holds no game-derived data. + Standard library only. """ import argparse @@ -67,6 +78,15 @@ RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3'] LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Wl,--Map=$builddir/totk.map'] # the map feeds tools/prof_report.py LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end of M4; libvpx static (movies) VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator +# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead. +LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lvulkan-1', '-lSDL3', '-lopus', + '-l:libvpx.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a'] +DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw' +INSTALL_SIDE = {'romfs.c'} # runtime sources that include romfs_config.h: compiled per install, not in the library +LIB_NAME = 'libtotkrt.a' +AR = UCRT / 'llvm-ar.exe' +TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain +DEPS = None # the deps directory of an llvm-mingw build # tools/data/expected-.json for each build version name. DISPLAY_VERSIONS = {'v0': '1.0.0', 'v112': '1.1.2', 'v121': '1.2.1', 'v143': '1.4.3'} @@ -182,11 +202,14 @@ def module_glue(layout, linked, hashes, paths): return '\n'.join(out) -def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False): +def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=None, make_lib=False): """build.ninja text, and the object lists per module. With prebuilt the generated objects already exist in - out/obj and are link inputs without a compile edge.""" + out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects + except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE.""" inc = f'-I {flag_path(RUNTIME)} -I {flag_path(gen)}' - rt_inc = f'{inc} -I {flag_path(out)} -I {flag_path(VMA_INCLUDE)}' # romfs_config.h, vk_mem_alloc.h + vma = f'{flag_path(DEPS / "include")} -I {flag_path(DEPS / "include" / "vma")}' if DEPS else flag_path(VMA_INCLUDE) + rt_inc = f'{inc} -I {flag_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers) + libs = [f'-L{flag_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS lines = [ '# Written by tools/build_game.py; do not edit.', 'ninja_required_version = 1.10', @@ -195,7 +218,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False): f'genflags = {" ".join(GEN_FLAGS)} {inc}', f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}', f'ldflags = {" ".join(LINK_FLAGS)}', - f'libs = {" ".join(LINK_LIBS)}', + f'libs = {" ".join(libs)}', '', 'pool gen', f' depth = {pool_depth}', @@ -229,14 +252,35 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False): lines.append(' pool = gen') objs[mod].append(o) rt = [] - for src in runtime_sources(skip) + [out / 'modules.c']: + sources = runtime_sources(skip) + if make_lib: + sources = [p for p in sources if p.name not in INSTALL_SIDE] + elif runtime_lib: + sources = [p for p in sources if p.name in INSTALL_SIDE] + [out / 'modules.c'] + else: + sources += [out / 'modules.c'] + for src in sources: o = runtime_object(out, src) lines.append(f'build {ninja_path(o)}: cc_rt {ninja_path(src)}') rt.append(o) objs['runtime'] = rt every = [o for v in objs.values() for o in v] exe = out / 'totk.exe' - lines += ['', f'build {ninja_path(exe)}: link ' + ' '.join(ninja_path(o) for o in every), + if make_lib: + exe = out / LIB_NAME + lines += ['', 'rule ar', f' command = {AR.as_posix()} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every), + 'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', ''] + return '\n'.join(lines), objs, exe + rule = 'link' + if runtime_lib: + # Whole archive: the weak probe stubs in gfx_stubs.c would otherwise satisfy references to the probes before the + # linker looks into the library, and the real probes would stay out of totk.exe. + rule = 'link_lib' + lines += ['', 'rule link_lib', + f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {flag_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out', + ' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out'] + lines += ['', f'build {ninja_path(exe)}: {rule} ' + ' '.join(ninja_path(o) for o in every) + + (f' | {ninja_path(runtime_lib)}' if runtime_lib else ''), 'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', ''] return '\n'.join(lines), objs, exe @@ -517,6 +561,14 @@ def main(): ap.add_argument('--romfs', type=Path, help="totk.exe's default RomFS directory with romfs.bin and index.json " '(default build/extracted//romfs)') ap.add_argument('--profile', type=Path, help="totk.exe's default profile directory (default build/profile)") + ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of ' + 'MSYS2 UCRT64 clang') + ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx ' + f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})') + ap.add_argument('--make-runtime-lib', action='store_true', + help=f'build only /{LIB_NAME} from runtime/ (no generated C, no totk.exe)') + ap.add_argument('--runtime-lib', type=Path, help=f'link against this {LIB_NAME} instead of compiling runtime/ ' + f'({", ".join(sorted(INSTALL_SIDE))} and modules.c still compile)') ap.add_argument('--expected', type=Path, help='expected hashes (default tools/data/expected-.json)') args = ap.parse_args() @@ -533,6 +585,14 @@ def main(): sys.exit(f'no expected hashes known for {args.version}; pass --expected') expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json' hashes = json.loads(Path(expected).read_text())['exefs_sha256'] + global CLANG, AR, TOOLCHAIN_BIN, DEPS + dlls = SDL3_DLLS + if args.toolchain: + TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve() + CLANG = TOOLCHAIN_BIN / 'clang.exe' + AR = TOOLCHAIN_BIN / 'llvm-ar.exe' + DEPS = (args.deps or DEFAULT_DEPS).resolve() + dlls = sorted((DEPS / 'bin').glob('*.dll')) for tool in (CLANG, NINJA): if not tool.is_file(): sys.exit(f'{tool} not found: install MSYS2 UCRT64 clang, lld and ninja (docs/toolchain-msys2-python.md)') @@ -566,8 +626,12 @@ def main(): skip = {s for s in args.skip_runtime.split(',') if s} main_out = (ROOT / 'build' / args.version).resolve() pairs = None - if args.full_build: + if args.make_runtime_lib: + mods, linked, why = {}, [], '' + elif args.full_build: why = '--full-build' + elif args.toolchain: + why = f'--toolchain {flag_path(args.toolchain)}' elif out == main_out: why = '' else: @@ -580,10 +644,14 @@ def main(): unlink_shared(out, mods) if why: print(f'build_game: compiling all generated C: {why}') - text, objs, exe = ninja_file(gen, out, mods, jobs, skip, pairs is not None) + runtime_lib = args.runtime_lib.resolve() if args.runtime_lib else None + if runtime_lib and not runtime_lib.is_file(): + sys.exit(f'{runtime_lib} not found: build it with --make-runtime-lib') + text, objs, exe = ninja_file(gen, out, mods, jobs, skip, pairs is not None, runtime_lib, + args.make_runtime_lib) write_if_changed(out / 'build.ninja', text) - env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', '')) + env = dict(os.environ, PATH=str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', '')) base = [str(NINJA), '-f', str(out / 'build.ninja'), '-j', str(jobs)] nfiles = sum(len(v) for v in mods.values()) print(f'build_game: {nfiles} generated files in {len(linked)} modules ({", ".join(linked)}), {jobs} jobs', flush=True) @@ -598,13 +666,18 @@ def main(): verify_objects(out, objs, pairs is not None, main_out, pairs, base, env) if args.compile_only: return + if args.make_runtime_lib: + exe.unlink(missing_ok=True) # ar rcs would keep members of deleted sources code, dt, peak_one, _ = run_measured(base + [flag_path(exe)], env) print(f'link: {dt:.1f} s, peak {mib(peak_one)}') if code: print(f'build_game: link failed, {flag_path(exe)} is the old one (a running totk.exe from this dir locks it)', file=sys.stderr) sys.exit(code) - for dll in SDL3_DLLS: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to totk.exe + if args.make_runtime_lib: + print(f'{below_root(exe)}: {mib(exe.stat().st_size)}') + return + for dll in dlls: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to totk.exe shutil.copy2(dll, exe.parent / dll.name) sections, image = pe_sections(exe) print(f'{below_root(exe)}: file {mib(exe.stat().st_size)}, ' From 227ed0f680e66a959704c8fd4a8492f79bcd7b45 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 08:06:04 +0300 Subject: [PATCH 03/16] Plan the installer that builds the game from the player's own files Co-Authored-By: Claude Opus 5.5 --- docs/installer.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 docs/installer.md diff --git a/docs/installer.md b/docs/installer.md new file mode 100644 index 00000000..441e22bc --- /dev/null +++ b/docs/installer.md @@ -0,0 +1,83 @@ +# Installer + +A public release that turns the player's own game files into a playable TotK folder. The release carries no game +data and no code derived from the game: the installer builds the game on the player's machine. The installed folder +holds only the game; the installer and its tools stay outside it. + +## What the player does + +1. Downloads `TotK-Setup--windows.zip` from GitHub Releases, unpacks it anywhere and starts `TotK-Setup.exe`. +2. Picks: + - the base game (XCI or NSP) and the 1.2.1 update (NSP); + - their `prod.keys`; + - the game language and the install folder. +3. Waits for the setup, one time. The window shows each step and its progress. +4. Plays `totk.exe` from the install folder. The installer and the source NSP or XCI can be deleted. + +The install folder (`main.c` `portable_paths()` reads this layout; it stays portable): + +``` +/ + totk.exe, SDL3.dll, libopus-0.dll, totk_xlate.exe + totk.ini language; the other keys as commented defaults (totk.ini.example) + exefs/, romfs/ extracted from the player's files + profile/ saves + shader and pipeline caches prepared by the setup + totk-install.json what built this folder: release version, input hashes, finished steps +``` + +## Updating the port + +The player runs a newer installer on the same folder. It reads `totk-install.json`, keeps `romfs/`, `exefs/`, +`profile/` and `totk.ini` after checking their hashes, rebuilds `totk.exe` and re-prepares the shaders. A newer +release that changes nothing game-derived skips the rebuild. + +## Setup steps + +1. **Check** before writing anything: the files open with the keys, the update is 1.2.1 (by hash, as + `tools/extract.py`), the keys are new enough for the update, about 22 GB is free on the target disk, the CPU has + AVX2 (`-march=x86-64-v3`), a Vulkan 1.3 GPU exists. Each failure is one clear message. +2. **Extract** ExeFS and RomFS into the install folder and verify the hashes. +3. **Lift** the ExeFS to C (`recomp/gen.py`, `recomp/patches-v121.txt`) into a temp folder next to the installer. +4. **Compile** the generated C, `modules.c` and `romfs.c` (they bake per-install layout and RomFS hashes) and link + them with the prebuilt runtime library into `totk.exe` (`build_game.py --runtime-lib`, proven on task/llvmmingw: + 3 min on 24 cores). +5. **Prepare** the RomFS patches and the shaders (`tools/make_romfs_patches.py`, `tools/prepare_shaders.py`, + `totk.exe --prepare-shaders`) in the install folder. +6. **Clean up** the temp folder (generated C and objects, about 1.3 GB). + +A cancelled or failed setup resumes from the last finished step (`totk-install.json`). + +## Release layout + +The unpacked release; `tools/make_release.py` assembles it, CI zips it: + +``` +TotK-Setup.exe starts python/pythonw.exe -m installer (a small launcher) +installer/ the installer: steps, CLI, window, en/ru text catalog +python/ Python 3.14.9 embeddable + numpy, lz4, cryptography, cffi, pycparser (+ tkinter for the window) +recomp/, tools/ the lifter and the build, extraction and shader scripts the steps run (copied from the repo) +llvm/ llvm-mingw, x86_64-only subset (Apache-2.0 with LLVM exception; mingw-w64 parts permissive) +runtime/ libtotkrt.a and the headers the generated code and modules.c/romfs.c include +bin/ totk_xlate.exe, hactool.exe, SDL3.dll, libopus-0.dll, ninja.exe +data/ shader_list.tsv (MacNativeGames, MIT) +THIRD_PARTY_NOTICES.md +``` + +- SDL3 comes from the upstream release, not MSYS2: MSYS2's DLL imports the LGPL libiconv. +- Every bundled third-party piece gets a row in `THIRD_PARTY_NOTICES.md`. +- The release contains nothing game-derived; `tools/check_content.py` runs over it in CI. + +## Constraints + +- The installer steps contain no Win32 calls: Linux follows with its own compiler and the same steps. +- The keys file path is passed to the tools; key values never appear in the UI, logs or files. +- The window is a front end over the CLI (`python -m installer --base ... --update ... --keys ... --lang ... --dest + ...`), which prints one JSON progress event per line. Tests and CI use the CLI. +- Unsigned exe: SmartScreen warns on first start. + +## Later + +- Re-warm the driver cache on start after a move, GPU or driver change (WIP on task/rewarm: stamp in + `shader-prepared.bin`). +- NSZ and XCZ input. Linux release. From 1ece85644086063cc93c1dd09b28fd078f6e1819 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 08:22:46 +0300 Subject: [PATCH 04/16] Add tools/make_release.py, which assembles the installer release folder from pinned downloads Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 24 +- tools/build_game.py | 9 +- tools/check_content.py | 8 +- tools/make_release.py | 432 +++++++++++++++++++++++++++++++ tools/prepare_shaders.py | 2 + tools/requirements-installer.txt | 7 + 6 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 tools/make_release.py create mode 100644 tools/requirements-installer.txt diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index aa56ba05..bec4fc7d 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -19,7 +19,7 @@ Code copied or adapted from another project is listed here with its source, the | deko3d block-linear image layout, as ported in MacNativeGames' `nvn_tex.c` (Apple parts dropped, layout math unchanged; licence text in `games/totk/third_party/d2r_gfx/LICENSE.deko3d`, reproduced in the file header) | https://github.com/devkitPro/deko3d (`source/dk_image.cpp`), via MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/third_party/d2r_gfx/gfx/nvn/nvn_tex.c`, `nvn_tex.h`), Copyright (C) 2018-2020 fincs | zlib | M4 task T3b: `runtime/nvn_tex.c`, `runtime/nvn_tex.h` | | MacNativeGames texture auto-layout goldens (40 original-SDK size and block-height cases) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/tests/texture_auto_layout/probe.c`), Copyright (c) 2026 jhaizhou | MIT | M4 task T3b: `runtime/nvn_tex_probe.c`; T3f: `runtime/nvn_probe.c` (four of the cases, through the procs) | | MacNativeGames block-linear deswizzler (address function; copy loop split out as `tex_bl_copy`) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/third_party/d2r_gfx/gfx/tex/nvn_deswizzle.c`), Copyright (c) 2026 jhaizhou | MIT | M4 task T5a: `runtime/tex_deswizzle.c`, `runtime/tex.h` | -| SDL 3.4.18 (MSYS2 `mingw-w64-ucrt-x86_64-SDL3`), dynamic link, `SDL3.dll` and its dependency `libiconv-2.dll` (LGPL-2.1, unmodified, not linked into the exe) copied next to `totk.exe` | https://libsdl.org, Copyright (c) 1997-2026 Sam Lantinga | zlib | `runtime/input_win.c` (gamepad input, rumble, gyro and accelerometer) | +| SDL 3.4.18, dynamic link, `SDL3.dll` copied next to `totk.exe`. Developer builds use the MSYS2 build (`mingw-w64-ucrt-x86_64-sdl3`, which imports `libiconv-2.dll`, LGPL-2.1, unmodified, copied beside it); the installer release ships the upstream release `SDL3-3.4.18-win32-x64.zip`, which imports no libiconv | https://libsdl.org, Copyright (c) 1997-2026 Sam Lantinga | zlib | `runtime/input_win.c` (gamepad input, rumble, gyro and accelerometer) | | libopus (MSYS2 `mingw-w64-ucrt-x86_64-opus`), dynamic link, `libopus-0.dll` copied next to `totk.exe` | https://opus-codec.org, Copyright (c) 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon | BSD-3-Clause | `runtime/ov_media.c` (nn::codec hardware Opus decoder) | | Vulkan Memory Allocator 3.4.0 (MSYS2 `mingw-w64-ucrt-x86_64-vulkan-memory-allocator`), header-only, compiled into `totk.exe` | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator, Copyright (c) 2017-2026 Advanced Micro Devices, Inc. | MIT | `runtime/vk_vma.cpp` (the allocator behind `runtime/vk_memory.c`) | | MacNativeGames host movie decoder (`movie::Decoder` for VP9 over libvpx: entry list, buffer and event semantics, NV12 output format, PTS pacing) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/runtime/movie_host.c`, `movie_host.h`), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/ov_movie.c`, `runtime/movie.h` | @@ -31,6 +31,26 @@ These tools run on the developer's or player's machine during the build. They ar | Tool | Source | License | Purpose | |---|---|---|---| -| hactool | https://github.com/SciresM/hactool @ `1d64a83450e025622f3468c28fc4164dad2c5ef6`, with patches from MacNativeGames | ISC | NCA and update container extraction | +| hactool (the installer release ships it, see below) | https://github.com/SciresM/hactool @ `1d64a83450e025622f3468c28fc4164dad2c5ef6`, with patches from MacNativeGames | ISC | NCA and update container extraction | | Capstone | https://www.capstone-engine.org | BSD-3-Clause | ARM64 disassembly for analysis | | Unicorn | https://www.unicorn-engine.org | GPL-2.0 (Python bindings BSD) | Differential testing of the lifter; never linked into the game | + +## Distributed in the installer release + +`tools/make_release.py` assembles these into the release folder of `docs/installer.md`. Versions and sha256 of every download are pinned in that script; licence texts are in the folder's `licenses/` and in each component's own folder. + +| Component | Source | License | Where in the release | +|---|---|---|---| +| CPython 3.14.9 embeddable package, with `tkinter` and Tcl/Tk 9.0.4 taken from the python.org 3.14.9 installer's component MSI (unpacked, never run); includes `vcruntime140*.dll` (Microsoft, redistributable with Python), OpenSSL, libffi, SQLite, zlib, libtommath | https://www.python.org/downloads/release/python-3149/ | PSF-2.0 (Tcl/Tk: Tcl/Tk licence, BSD-style) | `python/` | +| numpy 2.5.4 (with its bundled OpenBLAS), lz4 4.4.5, cryptography 50.0.2 (with its bundled OpenSSL), cffi 2.1.1, pycparser 3.11 | PyPI binary wheels, pins in `tools/requirements-installer.txt` | BSD-3-Clause; lz4 BSD-3-Clause; cryptography Apache-2.0 OR BSD-3-Clause; cffi MIT; pycparser BSD-3-Clause | `python/Lib/site-packages/` | +| llvm-mingw 20261020 (LLVM/Clang/LLD 23, mingw-w64, libc++, libunwind, compiler-rt), x86_64 subset | https://github.com/mstorsjo/llvm-mingw/releases/tag/20261020 | Apache-2.0 with LLVM exception; mingw-w64 ZPL-2.1/BSD/public domain; libc++ and libunwind Apache-2.0 with LLVM exception and MIT/UIUC; winpthreads MIT | `llvm/` | +| SDL 3.4.18 upstream Windows release (`SDL3.dll`) and mingw development package (headers, `libSDL3.dll.a`) | https://github.com/libsdl-org/SDL/releases/tag/release-3.4.18 | zlib | `bin/SDL3.dll`, `runtime/deps/` | +| libopus 1.6.1 (MSYS2 `mingw-w64-ucrt-x86_64-opus` 1.6.1-2; `libopus-0.dll` imports only KERNEL32 and the UCRT api-ms-win-crt DLLs), headers and import library | https://repo.msys2.org/mingw/ucrt64/ | BSD-3-Clause | `bin/libopus-0.dll`, `runtime/deps/` | +| libvpx 1.17.0 (MSYS2 `mingw-w64-ucrt-x86_64-libvpx` 1.17.0-1), headers and static library compiled into `totk.exe`; `libvpx-LICENSE.txt` and `libvpx-PATENTS.txt` come from the v1.17.0 source tag | https://github.com/webmproject/libvpx | BSD-3-Clause with the WebM patent grant | `runtime/deps/`, `licenses/` | +| Vulkan-Headers and Vulkan-Loader 1.4.363.0 (MSYS2 packages; headers and `libvulkan-1.dll.a` only, no loader DLL) | https://github.com/KhronosGroup | Apache-2.0 OR MIT | `runtime/deps/` | +| Vulkan Memory Allocator 3.4.0 (MSYS2 package), header | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator | MIT | `runtime/deps/` | +| Ninja 1.13.2 (`ninja-win.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja.exe` | +| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check), built with MSYS2 UCRT64 clang; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | +| `totk_xlate.exe` (Ryujinx shader translator, NativeAOT build of `tools/xlate/`, see the Ryujinx row above) | built by `tools/build_xlate.py` | MIT | `bin/totk_xlate.exe` | +| MacNativeGames shader variant list | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/assets/shader_list.tsv`), Copyright (c) 2026 MacNativeGames contributors | MIT | `data/shader_list.tsv` | + diff --git a/tools/build_game.py b/tools/build_game.py index 73f75154..baabc895 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -36,6 +36,7 @@ Compile jobs share a memory budget (MEM_BUDGET, default the smaller of 50 GiB an big machine does not run out of memory on the largest chunks. Each phase reports its wall time and the peak memory of its processes (Windows job object accounting: committed private bytes). +--ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used. --toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR (default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++ @@ -563,6 +564,8 @@ def main(): ap.add_argument('--profile', type=Path, help="totk.exe's default profile directory (default build/profile)") ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of ' 'MSYS2 UCRT64 clang') + ap.add_argument('--ninja', type=Path, help='ninja.exe to build with (default: MSYS2 UCRT64, else bin/ninja.exe of the ' + 'release folder)') ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx ' f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})') ap.add_argument('--make-runtime-lib', action='store_true', @@ -585,7 +588,11 @@ def main(): sys.exit(f'no expected hashes known for {args.version}; pass --expected') expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json' hashes = json.loads(Path(expected).read_text())['exefs_sha256'] - global CLANG, AR, TOOLCHAIN_BIN, DEPS + global CLANG, AR, NINJA, TOOLCHAIN_BIN, DEPS + if args.ninja: + NINJA = args.ninja.resolve() + elif not NINJA.is_file() and (ROOT / 'bin' / 'ninja.exe').is_file(): + NINJA = ROOT / 'bin' / 'ninja.exe' # the release folder (tools/make_release.py) dlls = SDL3_DLLS if args.toolchain: TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve() diff --git a/tools/check_content.py b/tools/check_content.py index 77baf49c..0d868bfb 100644 --- a/tools/check_content.py +++ b/tools/check_content.py @@ -47,7 +47,9 @@ def list_files(staged: bool) -> list[Path]: return [Path(p) for p in out.decode("utf-8").split("\0") if p] -def problems_for(path: Path) -> list[str]: +def problems_for(path: Path, allow_large: bool = False) -> list[str]: + """allow_large: files over MAX_BYTES (toolchain binaries in a release folder) are checked for the generated marker + only, not refused.""" found = [] if path.suffix.lower() in BLOCKED_SUFFIXES: found.append(f"blocked file type {path.suffix}") @@ -56,6 +58,10 @@ def problems_for(path: Path) -> list[str]: if not path.is_file(): return found size = path.stat().st_size + if size > MAX_BYTES and allow_large: + if GENERATED_MARKER.encode() in path.read_bytes(): + found.append("generated from game code") + return found if size > MAX_BYTES: found.append(f"{size // 1024} KiB, over the {MAX_BYTES // 1024} KiB limit") return found diff --git a/tools/make_release.py b/tools/make_release.py new file mode 100644 index 00000000..79d97a61 --- /dev/null +++ b/tools/make_release.py @@ -0,0 +1,432 @@ +"""Assemble the release folder of docs/installer.md ("Release layout") from pinned downloads. + + .venv-py314\\Scripts\\python tools\\make_release.py --out build/release/TotK-Setup [--zip build/release/TotK-Setup.zip] + .venv-py314\\Scripts\\python tools\\make_release.py --out build/release/TotK-Setup --prove + +Needs Python 3.14 (tarfile reads the MSYS2 .zst packages), uv on PATH and network access on the first run. Downloads are +cached in toolchain/release-dl/ and checked against the sha256 in PINS; a mismatch stops the build. The folder holds: + + python/ python.org embeddable 3.14.9 + the packages of tools/requirements-installer.txt + tkinter (the Tcl/Tk 9.0 + files come out of the python.org installer's own component MSI, unpacked with msiexec /a and expand.exe; + the installer is never run) + llvm/ llvm-mingw, x86_64 only, the files clang and lld need to compile and link totk.exe + runtime/ libtotkrt.a (build_game.py --make-runtime-lib with llvm/), the runtime headers, romfs.c (compiled per + install), deps/ (headers and import libraries of SDL3, Opus, libvpx, Vulkan and VMA) + bin/ totk_xlate.exe (+ .id), hactool.exe, SDL3.dll (upstream), libopus-0.dll, ninja.exe + recomp/, tools/, data/, installer/ (if the checkout has one), licenses/, THIRD_PARTY_NOTICES.md + +--prove compiles generated/v121 and links build/release-check/totk.exe with nothing but the folder's python/, llvm/, +runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there. It needs a +generated/v121 and build/extracted/v121 from the usual game build (they are read, nothing is written to them). + +The folder contains nothing derived from the game: check_content runs over it before the script finishes. +""" +import argparse +import hashlib +import os +import shutil +import struct +import subprocess +import sys +import tarfile +import time +import urllib.request +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import check_content # noqa: E402 + +ROOT = Path(__file__).resolve().parent.parent +MAIN_ROOT = (ROOT / 'build').resolve().parent # build/, generated/, toolchain/ and refs/ are junctions in a worktree +DL = MAIN_ROOT / 'toolchain' / 'release-dl' +WORK = MAIN_ROOT / 'toolchain' / 'release-work' # unpacked downloads + +MSYS2 = 'https://repo.msys2.org/mingw/ucrt64/' +SDL_VERSION = '3.4.18' +LLVM_VERSION = '20261020' +MNG_COMMIT = 'b75be8cfa704df3701e86130c8ced268e79d6b11' # MacNativeGames, the shader list (refs/MacNativeGames) +# name: (url, sha256) +PINS = { + 'python-embed': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-embed-amd64.zip', + '3c01d7cfc666f746a6f064f60fb2156ab0ec464d3d738ab796371a46a9bb24a2'), + 'python-installer': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-amd64.exe', + 'a247a0b612244a803f507b39fb7121e3602e4160d59c3b99d98e735289c53394'), + 'llvm-mingw': (f'https://github.com/mstorsjo/llvm-mingw/releases/download/{LLVM_VERSION}/' + f'llvm-mingw-{LLVM_VERSION}-ucrt-x86_64.zip', + 'fd5462b7f5d93b2ff981fcd3f9c7e427e47f3203c64ea91c27bcc444eebd74cd'), + 'sdl3-dll': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-{SDL_VERSION}-win32-x64.zip', + '75c2c0fc74e7d1206aaedc22893a35887332f4f85f5c8b74fddc10d177985f2c'), + 'sdl3-devel': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-devel-{SDL_VERSION}-mingw.tar.gz', + '049b9dd711e3fb3f8d84feb998dfec54a75e17cf539c4a80ff4d058d839771cf'), + 'ninja': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-win.zip', + '07fc8261b42b20e71d1720b39068c2e14ffcee6396b76fb7a795fb460b78dc65'), + 'opus': (MSYS2 + 'mingw-w64-ucrt-x86_64-opus-1.6.1-2-any.pkg.tar.zst', + 'bb114bcf26225f50f874497ebfd6b8224fff2a20bda3f8ee45cd07ce5f02657b'), + 'libvpx': (MSYS2 + 'mingw-w64-ucrt-x86_64-libvpx-1.17.0-1-any.pkg.tar.zst', + '8ff6919dbb88ea26ce28d22fd81d6f0d6e200cf56a4dd81ffc2d33124fe98024'), + 'libvpx-src': ('https://github.com/webmproject/libvpx/archive/refs/tags/v1.17.0.tar.gz', # LICENSE and PATENTS only + '1020f184046187baa2985dbde38e0691f49c44088bca7a1842b0236c6081dc0a'), + 'vulkan-headers': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-headers-1~1.4.363.0-1-any.pkg.tar.zst', + 'be602b2657cace531316e78b1eda4087e8f7f84ebb877f08fba198a2db06406a'), + 'vulkan-loader': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-loader-1~1.4.363.0-1-any.pkg.tar.zst', # import library only + '065076fe13d39ab97eea4740596426d5ae741b5044e48e1c9495d04835c92dbe'), + 'vma': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-memory-allocator-3.4.0-1-any.pkg.tar.zst', + '44c81162337915afecda3185f66fd98268a54454580b2717080c9dfef57fe781'), +} +# The component MSI inside the python.org installer that carries Tcl/Tk and tkinter (Burn names its payloads a0..a27). +TCLTK_PAYLOAD = ('a12', 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505') +SDL3_DLL_SHA256 = '2b4b128d8d510cc2d442693cf848c050d700d43dc667afe9e5fd82b0570d667e' # SDL3.dll inside the SDL zip + +# llvm-mingw: what clang and lld need to compile and link totk.exe for x86_64-w64-mingw32 (dynamic LLVM, libc++, compiler-rt). +LLVM_BIN = {'clang.exe', 'clang++.exe', 'clang-23.exe', 'ld.lld.exe', 'llvm-ar.exe', 'llvm-ranlib.exe', 'llvm-nm.exe', + 'llvm-objdump.exe', 'llvm-objcopy.exe', 'llvm-strip.exe', 'llvm-readobj.exe', 'llvm-dlltool.exe', + 'llvm-rc.exe', 'llvm-cvtres.exe', 'llvm-windres.exe', 'llvm-wrapper.exe', 'mingw32-common.cfg', + 'x86_64-w64-windows-gnu.cfg', 'x86_64-pc-windows-gnu.cfg', 'x86_64-w64-mingw32-clang.exe', + 'x86_64-w64-mingw32-clang++.exe', 'libLLVM-23.dll', 'libclang-cpp.dll', 'libc++.dll', 'libunwind.dll', + 'libwinpthread-1.dll', 'libffi-8.dll'} +LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32', 'busybox', 'python', 'share'} +LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} + +# tools/ files the installer steps run (and what they import), copied from the repo. +TOOLS_FILES = ['build_game.py', 'check_content.py', 'extract.py', 'make_romfs_patches.py', 'prepare_shaders.py', + 'romfs_index.py', 'shader_census.py'] +TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes +RUNTIME_SUFFIXES = {'.h', '.inc'} + +PROBES = ['--probe-kernel', '--probe-vkexec', '--probe-nvn-tex', '--probe-shader'] + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + for block in iter(lambda: f.read(1 << 20), b''): + h.update(block) + return h.hexdigest() + + +def fetch(name): + """The pinned download, from the cache or the network, checked against its sha256.""" + url, want = PINS[name] + DL.mkdir(parents=True, exist_ok=True) + path = DL / Path(url.split('?')[0]).name.replace('%7E', '~') + if name == 'libvpx-src': + path = DL / 'libvpx-1.17.0.tar.gz' + if not path.is_file(): + print(f'download {url}') + with urllib.request.urlopen(url) as r, open(path, 'wb') as f: + shutil.copyfileobj(r, f) + got = sha256_file(path) + if got != want: + sys.exit(f'{path}: sha256 {got}, pinned {want}') + return path + + +def unpack(name, member_filter=None): + """Extract a pinned archive into toolchain/release-work/ (once) and return that directory.""" + dest = WORK / name + stamp = dest / '.unpacked' + if stamp.is_file(): + return dest + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir(parents=True) + path = fetch(name) + if path.suffix == '.zip': + with zipfile.ZipFile(path) as z: + z.extractall(dest) + else: + with tarfile.open(path) as t: + t.extractall(dest, filter='data') + stamp.write_text('') + return dest + + +def copy_tree(src, dst, skip=lambda rel: False): + """Copy src into dst file by file (sorted), leaving out what skip(relative posix path) names.""" + for p in sorted(src.rglob('*')): + rel = p.relative_to(src) + if p.is_dir() or skip(rel.as_posix()): + continue + (dst / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(p, dst / rel) + + +def clean_env(*path_dirs): + """A process environment without MSYS2 and the user's tools: the Windows system directory and path_dirs only.""" + system = os.environ.get('SystemRoot', r'C:\Windows') + env = {k: v for k, v in os.environ.items() if k in ('SystemRoot', 'SYSTEMDRIVE', 'TEMP', 'TMP', 'USERPROFILE', + 'LOCALAPPDATA', 'APPDATA', 'COMSPEC', 'PATHEXT', 'NUMBER_OF_PROCESSORS')} + env['PATH'] = os.pathsep.join([*(str(Path(d)) for d in path_dirs), rf'{system}\System32', system]) + return env + + +def run(cmd, **kw): + print('+', ' '.join(str(c) for c in cmd), flush=True) + return subprocess.run([str(c) for c in cmd], check=True, **kw) + + +def mib(n): + return f'{n / (1 << 20):.1f} MiB' + + +def dir_size(p): + return sum(f.stat().st_size for f in Path(p).rglob('*') if f.is_file()) + + +# ---- python/ ------------------------------------------------------------------------------------------------------- + +def tcltk_files(): + """Directory holding the python.org installer's tcltk payload, unpacked (not installed) into toolchain/release-work.""" + dest = WORK / 'tcltk' + if (dest / 'DLLs' / '_tkinter.pyd').is_file(): + return dest + exe = fetch('python-installer') + data = exe.read_bytes() + cabs = [] + pos = data.find(b'MSCF\0\0\0\0') + while pos >= 0: + cabs.append((struct.unpack_from('/include and the x86_64 Windows builtins; no sanitizers, no other targets + if rel.startswith('lib/clang/') and '/include/' in rel: + return False + return not rel.endswith('/lib/windows/libclang_rt.builtins-x86_64.a') + return Path(rel).suffix.lower() in LLVM_SKIP_SUFFIXES or rel.startswith('include/ddk/') + + copy_tree(src, dst, skip) + return dst + + +# ---- bin/ and deps --------------------------------------------------------------------------------------------------- + +def build_bin(out): + b = out / 'bin' + b.mkdir(parents=True, exist_ok=True) + for f in ('totk_xlate.exe', 'totk_xlate.id'): + shutil.copy2(MAIN_ROOT / 'build' / 'xlate' / f, b / f) + shutil.copy2(MAIN_ROOT / 'toolchain' / 'hactool' / 'hactool.exe', b / 'hactool.exe') + dll = unpack('sdl3-dll') / 'SDL3.dll' + if sha256_file(dll) != SDL3_DLL_SHA256: + sys.exit(f'{dll}: not the pinned SDL3.dll') + shutil.copy2(dll, b / 'SDL3.dll') + shutil.copy2(unpack('opus') / 'ucrt64' / 'bin' / 'libopus-0.dll', b / 'libopus-0.dll') + shutil.copy2(unpack('ninja') / 'ninja.exe', b / 'ninja.exe') + + +def build_deps(out): + """runtime/deps: the headers and import libraries build_game.py --deps expects (include/, lib/).""" + d = out / 'runtime' / 'deps' + (d / 'include').mkdir(parents=True, exist_ok=True) + (d / 'lib').mkdir(parents=True, exist_ok=True) + sdl = unpack('sdl3-devel') / f'SDL3-{SDL_VERSION}' / 'x86_64-w64-mingw32' + copy_tree(sdl / 'include', d / 'include') + shutil.copy2(sdl / 'lib' / 'libSDL3.dll.a', d / 'lib') + for pkg, dirs, libs in (('opus', ['opus'], ['libopus.dll.a']), ('libvpx', ['vpx'], ['libvpx.a']), + ('vulkan-headers', ['vulkan', 'vk_video'], []), ('vulkan-loader', [], ['libvulkan-1.dll.a']), + ('vma', ['vma'], [])): + root = unpack(pkg) / 'ucrt64' + for inc in dirs: + copy_tree(root / 'include' / inc, d / 'include' / inc) + for lib in libs: + shutil.copy2(root / 'lib' / lib, d / 'lib') + + +def collect_licenses(out): + lic = out / 'licenses' + lic.mkdir(parents=True, exist_ok=True) + pairs = [ + (unpack('opus') / 'ucrt64' / 'share' / 'licenses' / 'opus' / 'COPYING', 'opus-COPYING.txt'), + (unpack('vulkan-headers') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-headers' / 'LICENSE', 'vulkan-headers-LICENSE.txt'), + (unpack('vulkan-loader') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-loader' / 'LICENSE', 'vulkan-loader-LICENSE.txt'), + (unpack('vma') / 'ucrt64' / 'share' / 'licenses' / 'VulkanMemoryAllocator' / 'LICENSE', 'vulkan-memory-allocator-LICENSE.txt'), + (unpack('sdl3-dll') / 'LICENSE.txt', 'SDL3-LICENSE.txt'), + (MAIN_ROOT / 'toolchain' / 'hactool' / 'LICENSE', 'hactool-LICENSE.txt'), + (MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'), + ] + for src, name in pairs: + shutil.copy2(src, lic / name) + with tarfile.open(fetch('libvpx-src')) as t: # the MSYS2 package carries no licence text; the notice needs both + for member, name in (('libvpx-1.17.0/LICENSE', 'libvpx-LICENSE.txt'), ('libvpx-1.17.0/PATENTS', 'libvpx-PATENTS.txt')): + (lic / name).write_bytes(t.extractfile(member).read()) + for name in ('LICENSE.TXT',): + shutil.copy2(out / 'llvm' / name, lic / 'llvm-mingw-LICENSE.txt') + + +# ---- runtime/, recomp/, tools/, data/ --------------------------------------------------------------------------------- + +def runtime_lib(out, llvm): + """Build libtotkrt.a with the folder's llvm/ and copy it, the headers and romfs.c into runtime/.""" + build = MAIN_ROOT / 'build' / 'release-check' / 'rtlib' + gen = MAIN_ROOT / 'generated' / 'v121' + run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, + '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', build], + env=clean_env(llvm / 'bin', out / 'bin')) + rt = out / 'runtime' + shutil.copy2(build / 'libtotkrt.a', rt / 'libtotkrt.a') + for p in sorted((ROOT / 'runtime').rglob('*')): + if p.suffix in RUNTIME_SUFFIXES and p.is_file(): + (rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime')) + shutil.copy2(ROOT / 'runtime' / 'romfs.c', rt / 'romfs.c') # build_game.py INSTALL_SIDE + + +def build_sources(out): + copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r) + for f in TOOLS_FILES: + (out / 'tools').mkdir(exist_ok=True) + shutil.copy2(ROOT / 'tools' / f, out / 'tools' / f) + for d in TOOLS_DIRS: + copy_tree(ROOT / 'tools' / d, out / 'tools' / d, lambda r: '__pycache__' in r or r.endswith('.md')) + (out / 'data').mkdir(exist_ok=True) + text = subprocess.run(['git', '-C', MAIN_ROOT / 'refs' / 'MacNativeGames', 'show', + f'{MNG_COMMIT}:games/totk/assets/shader_list.tsv'], check=True, capture_output=True).stdout + (out / 'data' / 'shader_list.tsv').write_bytes(text) + if (ROOT / 'installer').is_dir(): + copy_tree(ROOT / 'installer', out / 'installer', lambda r: '__pycache__' in r) + shutil.copy2(ROOT / 'THIRD_PARTY_NOTICES.md', out / 'THIRD_PARTY_NOTICES.md') + + +# ---- checks ---------------------------------------------------------------------------------------------------------- + +def check_folder(out): + """check_content over every file of the release folder; a hit stops the build.""" + bad = [] + for p in sorted(out.rglob('*')): + if p.is_file(): + bad += [f'{p.relative_to(out)}: {problem}' for problem in check_content.problems_for(p, allow_large=True)] + if bad: + sys.exit('game-derived content in the release folder:\n' + '\n'.join(bad)) + print(f'check_content: {sum(1 for p in out.rglob("*") if p.is_file())} files, no hit') + + +def sizes(out): + rows = [(p.name if p.is_dir() else p.name, dir_size(p) if p.is_dir() else p.stat().st_size) for p in sorted(out.iterdir())] + for name, n in rows: + print(f' {name:26} {mib(n):>12}') + print(f' {"total":26} {mib(sum(n for _, n in rows)):>12}') + + +def write_zip(out, path): + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as z: + for p in sorted(out.rglob('*')): + if p.is_file(): + info = zipfile.ZipInfo((Path(out.name) / p.relative_to(out)).as_posix(), (1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + z.writestr(info, p.read_bytes(), compresslevel=6) + print(f'{path}: {mib(path.stat().st_size)}') + + +def prove(out): + """Compile generated/v121 and link totk.exe with the folder's tools only, then run the probes next to it.""" + out = out.resolve() + gen = MAIN_ROOT / 'generated' / 'v121' + ext = MAIN_ROOT / 'build' / 'extracted' / 'v121' + chk = MAIN_ROOT / 'build' / 'release-check' + env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin') + t0 = time.time() + run([out / 'python' / 'python.exe', '-I', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', + '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime', '--runtime-lib', out / 'runtime' / 'libtotkrt.a', + '--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', + '--profile', chk / 'profile'], env=env) + print(f'prove: build {time.time() - t0:.0f} s') + for dll in ('SDL3.dll', 'libopus-0.dll'): + shutil.copy2(out / 'bin' / dll, chk / dll) + failed = [] + for probe in PROBES: + r = subprocess.run([str(chk / 'totk.exe'), probe], cwd=chk, env=clean_env(chk), capture_output=True, text=True, + timeout=900) + tail = (r.stdout.strip().splitlines() or [''])[-1] + print(f'{probe}: exit {r.returncode}: {tail}') + if r.returncode: + failed.append(probe) + if failed: + sys.exit(f'probes failed: {failed}') + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)') + ap.add_argument('--zip', type=Path, help='also write the folder as this zip') + ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder') + args = ap.parse_args() + out = args.out.resolve() + if args.prove: + prove(out) + return + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + build_python(out) + llvm = build_llvm(out) + build_bin(out) + build_deps(out) + runtime_lib(out, llvm) + build_sources(out) + collect_licenses(out) + check_folder(out) + sizes(out) + if args.zip: + write_zip(out, args.zip.resolve()) + + +if __name__ == '__main__': + main() diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index 4596758f..93d9a7c8 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -52,6 +52,8 @@ ANS_REC = 4 + 32 + 32 + 147 + 1 BUNDLED_LIST = ROOT / 'tools' / 'data' / 'shader_list.tsv' # the installer archive carries a copy (no git, no refs/) +if not BUNDLED_LIST.is_file(): + BUNDLED_LIST = ROOT / 'data' / 'shader_list.tsv' # release folder layout (tools/make_release.py) def read_list(path): diff --git a/tools/requirements-installer.txt b/tools/requirements-installer.txt new file mode 100644 index 00000000..49cb95fc --- /dev/null +++ b/tools/requirements-installer.txt @@ -0,0 +1,7 @@ +# Packages of the release's bundled Python 3.14.9 (tools/make_release.py): what the installer steps import. +# numpy: recomp/gen.py. lz4: recomp/nso.py, tools/shader_census.py. cryptography, cffi, pycparser: tools/extract.py and the vendored extract scripts. +numpy==2.5.4 +lz4==4.4.5 +cryptography==50.0.2 +cffi==2.1.1 +pycparser==3.11 From 0431b6268366bab8f7107c9f1b34ee1a9efd34ab Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 08:25:14 +0300 Subject: [PATCH 05/16] Add the installer core: the six setup steps, JSON progress events, totk-install.json resume and update, and explicit paths in the extract, build and shader tools Co-Authored-By: Claude Opus 5.5 --- installer/__init__.py | 10 + installer/__main__.py | 5 + installer/cli.py | 78 +++++ installer/events.py | 117 ++++++++ installer/hostcheck.c | 65 ++++ installer/ini.py | 69 +++++ installer/layout.py | 112 +++++++ installer/runner.py | 151 ++++++++++ installer/state.py | 54 ++++ installer/steps.py | 625 +++++++++++++++++++++++++++++++++++++++ installer/text/en.json | 80 +++++ tests/installer_test.py | 260 ++++++++++++++++ tools/build_game.py | 7 +- tools/extract.py | 68 +++-- tools/prepare_shaders.py | 5 +- 15 files changed, 1686 insertions(+), 20 deletions(-) create mode 100644 installer/__init__.py create mode 100644 installer/__main__.py create mode 100644 installer/cli.py create mode 100644 installer/events.py create mode 100644 installer/hostcheck.c create mode 100644 installer/ini.py create mode 100644 installer/layout.py create mode 100644 installer/runner.py create mode 100644 installer/state.py create mode 100644 installer/steps.py create mode 100644 installer/text/en.json create mode 100644 tests/installer_test.py diff --git a/installer/__init__.py b/installer/__init__.py new file mode 100644 index 00000000..7c523f73 --- /dev/null +++ b/installer/__init__.py @@ -0,0 +1,10 @@ +"""The TotK installer: builds a playable folder from the player's own game files (docs/installer.md). + + python -m installer --base FILE --update FILE --keys FILE --lang CODE --dest DIR [--root DIR] [--temp DIR] + +Modules: layout (where the bundled tools are, release or repository), events (JSON progress on stdout and the log +file), state (totk-install.json), runner (which steps run: resume and update), steps (the six steps), ini (totk.ini), +cli (the command line). text/en.json holds the English text of every message id the events carry. + +Standard library only, apart from what the extraction tools import (cryptography). Runs on Python 3.14. +""" diff --git a/installer/__main__.py b/installer/__main__.py new file mode 100644 index 00000000..d7e5cb2f --- /dev/null +++ b/installer/__main__.py @@ -0,0 +1,5 @@ +import sys + +from installer.cli import main + +sys.exit(main()) diff --git a/installer/cli.py b/installer/cli.py new file mode 100644 index 00000000..8be9ce34 --- /dev/null +++ b/installer/cli.py @@ -0,0 +1,78 @@ +"""Command line: python -m installer --base FILE --update FILE --keys FILE --lang CODE --dest DIR [--root DIR] + [--temp DIR] [--verify] + +A first install needs every option. On a folder with totk-install.json (resume after a cancel or failure, or update +with a newer release) only --dest is needed: --lang defaults to the recorded language, and the game files are needed +only when the extraction has to run again. Prints JSON events on stdout (events.py). Exit code: 0 done, 1 failed, +2 bad command line, 3 cancelled (Ctrl+C or Ctrl+Break). +""" +import argparse +import hashlib +import signal +import sys +import time +from pathlib import Path + +from . import layout as layouts +from . import state as statefile +from .events import Reporter +from .runner import Context, run_steps +from .steps import all_steps + +EXIT = {'ok': 0, 'failed': 1, 'cancelled': 3} + + +class Parser(argparse.ArgumentParser): + def error(self, message): + Reporter().error('cli.usage', {'detail': message}) + sys.exit(2) + + +def temp_folder(base, dest): + """One temp folder per install folder, so two installs never share generated files.""" + tag = hashlib.sha256(str(dest).lower().encode()).hexdigest()[:8] + return base / f'{dest.name or "root"}-{tag}' + + +def cancel(signum, frame): + raise KeyboardInterrupt + + +def main(argv=None): + ap = Parser(prog='python -m installer', description=__doc__.splitlines()[0]) + ap.add_argument('--base', type=Path, help='base game, XCI or NSP') + ap.add_argument('--update', type=Path, help='1.2.1 update NSP') + ap.add_argument('--keys', type=Path, help='prod.keys') + ap.add_argument('--lang', help='game language, a Switch language code (en-US, en-GB, ja, de, ru, ...)') + ap.add_argument('--dest', type=Path, required=True, help='install folder') + ap.add_argument('--root', type=Path, help='release folder or repository (default: the folder above installer/)') + ap.add_argument('--temp', type=Path, help='temp folder (default: temp/ of the release, build/install-temp in a ' + 'repository)') + ap.add_argument('--verify', action='store_true', help='hash every kept file, also those this release made') + args = ap.parse_args(argv) + + # The window cancels with Ctrl+Break to the installer's process group (CREATE_NEW_PROCESS_GROUP), which also reaches + # the tools it runs; Python's default for it ends the process without recording anything. + for name in ('SIGBREAK', 'SIGTERM'): + if hasattr(signal, name): + signal.signal(getattr(signal, name), cancel) + layout = layouts.find(args.root) + dest = args.dest.resolve() + temp = temp_folder((args.temp or layout.temp).resolve(), dest) + log = temp / 'logs' / time.strftime('install-%Y%m%d-%H%M%S.log') + rep = Reporter(log_path=log) + t0 = time.monotonic() + try: + state = statefile.load(dest) + except statefile.StateError as e: + rep.error(e.id, {'path': str(e.path)}) + rep.emit('end', result='failed', seconds=0) + return 1 + ctx = Context(layout, dest, temp, rep, args, state=state, verify=args.verify) + steps = all_steps() + rep.emit('start', release=layout.release, mode=ctx.mode, dev=layout.dev, dest=str(dest), log=str(log), + steps=[s.id for s in steps]) + result = run_steps(steps, ctx) + rep.emit('end', result=result, seconds=round(time.monotonic() - t0, 1)) + rep.close() + return EXIT[result] diff --git a/installer/events.py b/installer/events.py new file mode 100644 index 00000000..0f223ef0 --- /dev/null +++ b/installer/events.py @@ -0,0 +1,117 @@ +"""Progress events: one JSON object per line on stdout for the window and tests, the same in English to a log file. + +Every event has "type" and "t" (seconds since the start). Types: + + start release, mode (install, resume, update), dest, log, steps (the step ids in order) + step step, state: started | done (seconds) | skipped (reason: current, not_needed) + progress step, phase, and done/total/unit (bytes, chunks, objects, shaders) when the amount is known + warning id, params + error id, params, step (absent outside a step) + end result: ok | failed | cancelled, seconds + +Text is never the payload: ids name a step ("step."), a phase ("phase..") or a message (""), +and installer/text/en.json holds their English text with {param} placeholders. The log renders events with it. +""" +import json +import sys +import threading +import time +from pathlib import Path + +CATALOG = Path(__file__).resolve().parent / 'text' / 'en.json' +PROGRESS_EVERY = 0.5 # seconds between two progress events of one phase + + +def load_catalog(path=CATALOG): + return json.loads(path.read_text(encoding='utf-8')) + + +def render(catalog, id, params=None): + """The English text of id, or the id itself when the catalog lacks it.""" + text = catalog.get(id, id) + try: + return text.format(**(params or {})) + except (KeyError, IndexError, ValueError): + return f'{text} {params}' + + +class Reporter: + def __init__(self, out=None, log_path=None): + self.out = out or sys.stdout + self.t0 = time.monotonic() + self.catalog = load_catalog() + self.lock = threading.Lock() + self.log_file = None + self.last_progress = {} + if log_path: + Path(log_path).parent.mkdir(parents=True, exist_ok=True) + self.log_file = open(log_path, 'a', encoding='utf-8', buffering=1) + self.log_path = log_path + + def elapsed(self): + return time.monotonic() - self.t0 + + def log(self, text): + """A line for the log file only.""" + if self.log_file: + with self.lock: + self.log_file.write(f'[{self.elapsed():8.1f}] {text}\n') + + def emit(self, type, **fields): + event = {'type': type, 't': round(self.elapsed(), 2), **fields} + with self.lock: + self.out.write(json.dumps(event, ensure_ascii=False) + '\n') + self.out.flush() + self.log(self.describe(event)) + return event + + def describe(self, e): + c = self.catalog + kind = e['type'] + if kind == 'step': + name = render(c, 'step.' + e['step']) + extra = f' ({e["seconds"]:.1f} s)' if 'seconds' in e else f' ({e["reason"]})' if 'reason' in e else '' + return f'{name}: {e["state"]}{extra}' + if kind == 'progress': + text = render(c, f'phase.{e["step"]}.{e["phase"]}') + if e.get('total'): + text += f' {e["done"]}/{e["total"]} {e.get("unit", "")}'.rstrip() + return text + if kind in ('warning', 'error'): + return f'{kind}: {render(c, e["id"], e.get("params"))} [{e["id"]}]' + return json.dumps(e, ensure_ascii=False) + + def step(self, step, state, **fields): + return self.emit('step', step=step, state=state, **fields) + + def progress(self, step, phase, done=None, total=None, unit=None, force=False): + """A progress event, at most one per PROGRESS_EVERY per phase unless it is the first or the last.""" + now = time.monotonic() + key = (step, phase) + last = self.last_progress.get(key) + final = total is not None and done is not None and done >= total + if last is not None and not final and not force and now - last < PROGRESS_EVERY: + return None + self.last_progress[key] = now + fields = {'step': step, 'phase': phase} + if done is not None: + fields['done'] = done + if total is not None: + fields['total'] = total + if unit: + fields['unit'] = unit + return self.emit('progress', **fields) + + def warning(self, id, **params): + return self.emit('warning', id=id, params=params) + + def error(self, id, params, step=None): + fields = {'id': id, 'params': params} + if step: + fields['step'] = step + return self.emit('error', **fields) + + def close(self): + if self.log_file: + self.log_file.close() + self.log_file = None diff --git a/installer/hostcheck.c b/installer/hostcheck.c new file mode 100644 index 00000000..2f86571a --- /dev/null +++ b/installer/hostcheck.c @@ -0,0 +1,65 @@ +// Host check of the installer's first step, compiled with the bundled clang at install time: does the CPU run +// -march=x86-64-v3 code, and does a Vulkan 1.3 device exist. Prints one JSON object on stdout. +// The Vulkan loader is opened at run time, so a machine without one gets "loader": false instead of a failed start. +#include +#include +#include +#ifdef _WIN32 +#include +#define open_lib() LoadLibraryA("vulkan-1.dll") +#define lib_sym(l, n) ((void *)GetProcAddress((l), (n))) +#else +#include +#define open_lib() dlopen("libvulkan.so.1", RTLD_NOW) +#define lib_sym(l, n) dlsym((l), (n)) +#endif + +static void json_string(const char *s) { + putchar('"'); + for (; *s; s++) { + if (*s == '"' || *s == '\\') printf("\\%c", *s); + else if ((unsigned char)*s < 0x20) printf("\\u%04x", *s); + else putchar(*s); + } + putchar('"'); +} + +int main(void) { + __builtin_cpu_init(); + printf("{\"x86_64_v3\": %s, ", __builtin_cpu_supports("x86-64-v3") ? "true" : "false"); + void *lib = open_lib(); + PFN_vkGetInstanceProcAddr gipa = lib ? (PFN_vkGetInstanceProcAddr)lib_sym(lib, "vkGetInstanceProcAddr") : NULL; + if (!gipa) { + printf("\"loader\": false, \"devices\": []}\n"); + return 0; + } + printf("\"loader\": true, "); + PFN_vkCreateInstance create = (PFN_vkCreateInstance)gipa(NULL, "vkCreateInstance"); + VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, .pApplicationName = "totk-install", + .apiVersion = VK_API_VERSION_1_3}; + VkInstanceCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, .pApplicationInfo = &app}; + VkInstance inst = VK_NULL_HANDLE; + VkResult r = create ? create(&ci, NULL, &inst) : VK_ERROR_INITIALIZATION_FAILED; + if (r != VK_SUCCESS) { + printf("\"instance\": %d, \"devices\": []}\n", (int)r); + return 0; + } + PFN_vkEnumeratePhysicalDevices enumerate = (PFN_vkEnumeratePhysicalDevices)gipa(inst, "vkEnumeratePhysicalDevices"); + PFN_vkGetPhysicalDeviceProperties props = (PFN_vkGetPhysicalDeviceProperties)gipa(inst, "vkGetPhysicalDeviceProperties"); + PFN_vkDestroyInstance destroy = (PFN_vkDestroyInstance)gipa(inst, "vkDestroyInstance"); + VkPhysicalDevice devs[16]; + uint32_t n = 16; + if (enumerate(inst, &n, devs) < 0) n = 0; + printf("\"instance\": 0, \"devices\": ["); + for (uint32_t i = 0; i < n; i++) { + VkPhysicalDeviceProperties p; + props(devs[i], &p); + printf("%s{\"name\": ", i ? ", " : ""); + json_string(p.deviceName); + printf(", \"api\": [%u, %u, %u], \"type\": %d}", VK_API_VERSION_MAJOR(p.apiVersion), + VK_API_VERSION_MINOR(p.apiVersion), VK_API_VERSION_PATCH(p.apiVersion), (int)p.deviceType); + } + printf("]}\n"); + destroy(inst, NULL); + return 0; +} diff --git a/installer/ini.py b/installer/ini.py new file mode 100644 index 00000000..6bcc341f --- /dev/null +++ b/installer/ini.py @@ -0,0 +1,69 @@ +"""totk.ini: set the game language and keep everything else (keys and syntax as runtime/cfg.c reads them). + +A new file starts from totk.ini.example, whose lines are all commented defaults. The language goes into `lang` +(a Switch language code, runtime/ov_system.c): an active `lang = ...` line is rewritten in place, else the commented +`# lang = ...` line of the example is activated, else the line is added to [language] (created at the end if +missing). Other lines, comments and line order stay as they are. +""" +import os +import re +from pathlib import Path + +# runtime/ov_system.c: the codes TOTK_LANG accepts. +LANGUAGES = ('ja', 'en-US', 'en-GB', 'fr', 'fr-CA', 'de', 'it', 'es', 'es-419', 'nl', 'pt', 'pt-BR', 'ru', 'ko', + 'zh-CN', 'zh-TW', 'zh-Hans', 'zh-Hant') +# Their text falls back to the Switch shared fonts (runtime/ov_system.c), which the installer does not extract. +NEED_SHARED_FONTS = ('ko', 'zh-CN', 'zh-TW', 'zh-Hans', 'zh-Hant') + +ACTIVE = re.compile(r'^(\s*)(?:totk_)?lang(\s*=\s*)(.*?)(\s*)$', re.IGNORECASE) +COMMENTED = re.compile(r'^\s*[#;]\s*(?:totk_)?lang\s*=.*$', re.IGNORECASE) +SECTION = re.compile(r'^\s*\[(.*)\]\s*$') + + +def set_language(text, lang): + """text with `lang = `; see the module docstring.""" + lines = text.splitlines() + for i, line in enumerate(lines): + m = ACTIVE.match(line) + if m: + lines[i] = f'{m.group(1)}lang{m.group(2)}{lang}' + return '\n'.join(lines) + '\n' + for i, line in enumerate(lines): + if COMMENTED.match(line): + lines[i] = f'lang = {lang}' + return '\n'.join(lines) + '\n' + for i, line in enumerate(lines): + m = SECTION.match(line) + if m and m.group(1).strip().lower() == 'language': + lines.insert(i + 1, f'lang = {lang}') + return '\n'.join(lines) + '\n' + if lines and lines[-1].strip(): + lines.append('') + lines += ['[language]', f'lang = {lang}'] + return '\n'.join(lines) + '\n' + + +def get_language(text): + for line in text.splitlines(): + m = ACTIVE.match(line) + if m: + return m.group(3) + return None + + +def write(dest, lang, example=None): + """Write dest/totk.ini with lang; an existing file keeps its other lines. Returns the path.""" + path = Path(dest) / 'totk.ini' + if path.is_file(): + text = path.read_text(encoding='utf-8-sig') + elif example and Path(example).is_file(): + text = Path(example).read_text(encoding='utf-8') + else: + text = '' + new = set_language(text, lang) + if path.is_file() and new == text: + return path + tmp = path.with_name('totk.ini.tmp') + tmp.write_text(new, encoding='utf-8', newline='\n') + os.replace(tmp, path) + return path diff --git a/installer/layout.py b/installer/layout.py new file mode 100644 index 00000000..4d290b50 --- /dev/null +++ b/installer/layout.py @@ -0,0 +1,112 @@ +"""Where the installer finds what it runs: the one place that maps the release folder and a repository checkout. + +Release (docs/installer.md "Release layout"), root = the unpacked folder: + + python/python.exe runs every tool (numpy, lz4, cryptography, compression.zstd) + recomp/, tools/ the lifter and the scripts; recomp/patches-v121.txt, tools/data/expected-v1.2.1.json + llvm/ llvm-mingw (bin/clang.exe) + runtime/ libtotkrt.a, the headers and install-side sources (romfs.c), and include/ + lib/ of + Vulkan, SDL3, Opus, libvpx and VMA (build_game.py --deps) + bin/ totk_xlate.exe (+ totk_xlate.id), hactool.exe, ninja.exe, SDL3.dll, libopus-0.dll + data/shader_list.tsv the MacNativeGames shader list + totk.ini.example + VERSION the release version, one line + temp/ the default temp folder (created) + +Dev mode, root = a repository checkout (no python/python.exe): build/py314-embed/py/python.exe (the embeddable +Python the release ships, with the same packages), else this Python; toolchain/llvm-mingw-*/ and toolchain/deps-mingw; +runtime/ as the sources with build/install-rtlib/libtotkrt.a (tools/build_game.py --make-runtime-lib --out +build/install-rtlib); MSYS2 UCRT64 ninja.exe; toolchain/hactool; build/xlate/totk_xlate.exe; the shader list from +refs/MacNativeGames; build/install-temp. The version is dev-. +""" +import os +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +EXE = '.exe' if os.name == 'nt' else '' +UCRT = Path(r'C:\msys64\ucrt64\bin') +SHADER_LIST_REF = 'origin/HEAD:games/totk/assets/shader_list.tsv' # dev: tools/prepare_shaders.py LIST_REF + + +@dataclass(frozen=True) +class Layout: + root: Path + dev: bool + release: str + python: Path # runs the tools + tools: Path + recomp: Path + patches: Path # code patch table for recomp/gen.py + expected: Path # expected hashes of the supported update + llvm: Path # llvm-mingw tree + deps: Path # include/ and lib/ for the link + runtime: Path # runtime headers and install-side sources + runtime_lib: Path # libtotkrt.a + ninja: Path + hactool: Path + xlate: Path # totk_xlate.exe; a totk_xlate.id beside it is copied too + dlls: tuple = field(default_factory=tuple) + shader_list: Path | None = None # None: dev, read from refs/MacNativeGames through git + ini_example: Path | None = None + temp: Path | None = None + + @property + def clang(self): + return self.llvm / 'bin' / ('clang' + EXE) + + def missing(self): + """Paths the steps need that do not exist.""" + need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py', + self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', self.patches, self.expected, self.clang, + self.deps / 'include', self.deps / 'lib', self.runtime, self.runtime_lib, self.ninja, self.hactool, + self.xlate, *self.dlls] + if self.shader_list: + need.append(self.shader_list) + return [str(p) for p in need if not p.exists()] + + +def is_release(root): + return (root / 'python' / ('python' + EXE)).is_file() + + +def release_layout(root): + version = root / 'VERSION' + return Layout( + root=root, dev=False, + release=version.read_text(encoding='utf-8').strip() if version.is_file() else 'unknown', + python=root / 'python' / ('python' + EXE), + tools=root / 'tools', recomp=root / 'recomp', patches=root / 'recomp' / 'patches-v121.txt', + expected=root / 'tools' / 'data' / 'expected-v1.2.1.json', + llvm=root / 'llvm', deps=root / 'runtime', runtime=root / 'runtime', + runtime_lib=root / 'runtime' / 'libtotkrt.a', + ninja=root / 'bin' / ('ninja' + EXE), hactool=root / 'bin' / ('hactool' + EXE), + xlate=root / 'bin' / ('totk_xlate' + EXE), + dlls=(root / 'bin' / 'SDL3.dll', root / 'bin' / 'libopus-0.dll'), + shader_list=root / 'data' / 'shader_list.tsv', ini_example=root / 'totk.ini.example', temp=root / 'temp') + + +def dev_layout(root): + commit = subprocess.run(['git', '-C', str(root), 'rev-parse', '--short', 'HEAD'], capture_output=True, + text=True).stdout.strip() + embed = root / 'build' / 'py314-embed' / 'py' / ('python' + EXE) + trees = sorted(p.parent.parent for p in (root / 'toolchain').glob('llvm-mingw-*/llvm-mingw-*/bin/clang' + EXE)) + deps = root / 'toolchain' / 'deps-mingw' + return Layout( + root=root, dev=True, release=f'dev-{commit or "unknown"}', + python=embed if embed.is_file() else Path(sys.executable), + tools=root / 'tools', recomp=root / 'recomp', patches=root / 'recomp' / 'patches-v121.txt', + expected=root / 'tools' / 'data' / 'expected-v1.2.1.json', + llvm=trees[-1] if trees else root / 'toolchain' / 'llvm-mingw-missing', deps=deps, + runtime=root / 'runtime', runtime_lib=root / 'build' / 'install-rtlib' / 'libtotkrt.a', + ninja=UCRT / ('ninja' + EXE), hactool=root / 'toolchain' / 'hactool' / ('hactool' + EXE), + xlate=root / 'build' / 'xlate' / ('totk_xlate' + EXE), + dlls=tuple(sorted((deps / 'bin').glob('*.dll'))), + shader_list=None, ini_example=root / 'totk.ini.example', temp=root / 'build' / 'install-temp') + + +def find(root=None): + """The layout of root (default: the folder above this package).""" + root = Path(root or Path(__file__).resolve().parent.parent).resolve() + return release_layout(root) if is_release(root) else dev_layout(root) diff --git a/installer/runner.py b/installer/runner.py new file mode 100644 index 00000000..ebf3b52b --- /dev/null +++ b/installer/runner.py @@ -0,0 +1,151 @@ +"""Which steps run, in what order, and the bookkeeping around them: fresh install, resume and update are one rule. + +Each step has a key: a fingerprint of everything its output depends on, computed before anything runs (a step's key +includes the keys of the steps it reads from, so a change upstream reaches every step below it). A step is skipped +when totk-install.json records it with the same key and its output still matches the record (Step.current, cheap). +Otherwise it runs. A transient step (its output lives in the temp folder and only the next step reads it) runs only +when the next step runs. `always` steps (check, cleanup) run every time. + +Update mode is the same rule with a newer release: the extraction's key depends only on the expected hashes, so a +release that changes the lifter or the runtime keeps romfs/ and exefs/ and rebuilds from lift on. Before keeping the +output of a step recorded by another release (or with --verify), the check step calls Step.verify (expensive: the full +RomFS hash); a mismatch makes the step run again. + +A failed or cancelled run leaves the finished steps recorded; the next run starts at the first step that is not. +""" +import datetime +import traceback + +from . import state as statefile + + +class InstallError(Exception): + """A failure with a message id (installer/text/en.json) and its parameters.""" + + def __init__(self, id, **params): + super().__init__(id) + self.id, self.params = id, params + + +class Step: + id = '' + transient = False + always = False + + def key(self, ctx): + """Fingerprint of the step's inputs; may read ctx.keys of earlier steps.""" + return '' + + def current(self, ctx, record): + """True when the recorded output is still in place (cheap checks only).""" + return True + + def verify(self, ctx, record): + """True when the recorded output passes the full check (hashes); called before keeping another release's + output.""" + return True + + def run(self, ctx): + """Do the work. Returns fields to record with the step.""" + return {} + + +class Context: + def __init__(self, layout, dest, temp, reporter, options, state=None, verify=False): + self.layout, self.dest, self.temp, self.reporter, self.options = layout, dest, temp, reporter, options + self.state = state or statefile.empty(layout.release) + self.verify = verify + self.mode = 'install' if state is None else 'resume' if state.get('release') == layout.release else 'update' + self.keys, self.runs, self.reasons, self.steps, self.forced = {}, {}, {}, [], set() + + def record(self, step_id): + return self.state['steps'].get(step_id) + + def will_run(self, step_id): + return self.runs.get(step_id, False) + + def save(self): + self.dest.mkdir(parents=True, exist_ok=True) + statefile.save(self.dest, self.state) + + def plan(self, forced=()): + """Fill keys (once), runs and reasons. forced: step ids that must run.""" + if not self.keys: + for s in self.steps: + self.keys[s.id] = s.key(self) + for s in self.steps: + rec = self.record(s.id) + if s.always or s.id in forced: + self.runs[s.id] = True + else: + self.runs[s.id] = not (rec and rec.get('key') == self.keys[s.id] and s.current(self, rec)) + self.reasons[s.id] = '' if self.runs[s.id] else 'current' + for i in range(len(self.steps) - 2, -1, -1): + s, after = self.steps[i], self.steps[i + 1] + if s.transient and self.runs[s.id] and not self.runs[after.id]: + self.runs[s.id], self.reasons[s.id] = False, 'not_needed' + self.forced = set(forced) + + def verify_kept(self, progress_step): + """Full check of every kept step output that another release recorded (or of all with --verify). A step that + fails it is planned to run. Returns the ids that failed.""" + failed = [] + for s in self.steps: + rec = self.record(s.id) + if self.runs[s.id] or self.reasons[s.id] != 'current' or s.always: + continue + if not (self.verify or rec.get('release') != self.layout.release): + continue + if not s.verify(self, rec): + failed.append(s.id) + if failed: + for f in failed: + self.reporter.warning('state.output_changed', step=f) + self.plan(self.forced | set(failed)) + return failed + + +def now_utc(): + return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def run_steps(steps, ctx): + """Run the plan. Returns 'ok', 'failed' or 'cancelled'; events and the log say why.""" + rep = ctx.reporter + ctx.steps = steps + current = None + try: + ctx.plan() + rep.emit('plan', run=[s.id for s in steps if ctx.runs[s.id]], + skip={s.id: ctx.reasons[s.id] for s in steps if not ctx.runs[s.id]}) + for s in steps: + if not ctx.runs[s.id]: + rep.step(s.id, 'skipped', reason=ctx.reasons[s.id]) + continue + current = s.id + rep.step(s.id, 'started') + t0 = rep.elapsed() + fields = s.run(ctx) or {} + seconds = round(rep.elapsed() - t0, 1) + if not s.always: + ctx.state['steps'][s.id] = {'key': ctx.keys[s.id], 'release': ctx.layout.release, + 'finished': now_utc(), 'seconds': seconds, **fields} + ctx.save() # the first save, after the check, creates the install folder + rep.step(s.id, 'done', seconds=seconds) + current = None + ctx.state['release'] = ctx.layout.release + ctx.save() + return 'ok' + except InstallError as e: + rep.error(e.id, e.params, step=current) + return 'failed' + except statefile.StateError as e: + rep.error(e.id, {'path': str(e.path)}, step=current) + return 'failed' + except KeyboardInterrupt: + rep.log(f'cancelled during {current}') + return 'cancelled' + except Exception as e: + rep.log(traceback.format_exc()) + rep.error('internal', {'exception': type(e).__name__, 'detail': str(e)}, step=current) + return 'failed' diff --git a/installer/state.py b/installer/state.py new file mode 100644 index 00000000..be264654 --- /dev/null +++ b/installer/state.py @@ -0,0 +1,54 @@ +"""totk-install.json in the install folder: what built it and which steps finished. + + {"format": 1, "release": "...", "lang": "en-US", + "inputs": {"base": {"name", "size", "sha256"}, "update": {...}}, + "steps": {"": {"key": "", "release": "...", "finished": "", "seconds": 12.3, + ...the step's own fields}}} + +A step's key fingerprints everything its output depends on (runner.py), so a step whose recorded key equals the +current one and whose output still checks out is not run again. The file is replaced atomically after each step. +No key value or keys file content is ever stored. +""" +import json +import os +from pathlib import Path + +NAME = 'totk-install.json' +FORMAT = 1 + + +class StateError(Exception): + def __init__(self, id, path): + super().__init__(id) + self.id, self.path = id, path + + +def empty(release): + return {'format': FORMAT, 'release': release, 'inputs': {}, 'steps': {}} + + +def load(dest): + """The state of dest, or None when the folder has none. StateError for an unreadable or newer file.""" + path = Path(dest) / NAME + if not path.is_file(): + return None + try: + data = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError): + raise StateError('state.unreadable', path) from None + if not isinstance(data, dict) or not isinstance(data.get('steps'), dict): + raise StateError('state.unreadable', path) + if data.get('format', 0) > FORMAT: + raise StateError('state.newer_format', path) + return data + + +def save(dest, data): + path = Path(dest) / NAME + tmp = path.with_name(NAME + '.tmp') + with open(tmp, 'w', encoding='utf-8', newline='\n') as f: + json.dump(data, f, indent=1, ensure_ascii=False) + f.write('\n') + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) diff --git a/installer/steps.py b/installer/steps.py new file mode 100644 index 00000000..0216f750 --- /dev/null +++ b/installer/steps.py @@ -0,0 +1,625 @@ +"""The six setup steps of docs/installer.md: check, extract, lift, compile, prepare, cleanup. + +Extraction runs in this process through tools/extract.py (the package's sibling tools/, which ships with it); lift, +compile and prepare run the bundled Python on recomp/gen.py, tools/build_game.py, tools/make_romfs_patches.py and +tools/prepare_shaders.py, whose output goes to /logs/-.log and is read for progress. No Win32 call +here: the host check is a small C program built with the bundled clang (hostcheck.c). + +Install folder: exefs/, romfs/ (romfs.bin, index.json), profile/, totk.exe and its DLLs, romfs-index.bin, +romfs-patches/, totk_xlate.exe, the shader state of totk.exe --prepare-shaders, totk.ini, totk-install.json. +Temp folder (/-): check/ (ExeFS of both inputs, the host check), gen/ (C), build/ (objects), +logs/ (kept by the cleanup). +""" +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import threading +import time +from pathlib import Path + +from . import ini +from .layout import EXE +from .runner import InstallError, Step + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'tools')) +import extract # noqa: E402 +from extract_exefs import (common_tickets, container_entries, load_keys, nca_ctr_key, nca_inventory, # noqa: E402 + program_nca) + +GiB = 1 << 30 +DEST_EXTRA = 4 * GiB # beside the RomFS: shader and pipeline caches (about 2 GB), exefs, totk.exe, seed files +TEMP_NEED = 3 * GiB # generated C (about 0.9 GB) and objects +UPDATE_VERSION = '1.2.1' +HOSTCHECK = Path(__file__).resolve().parent / 'hostcheck.c' +STAMP = '.install-key' + + +# ---- helpers + +def file_sha256(path, progress=None): + """SHA-256 hex of a file; progress(done, total) after each block.""" + h = hashlib.sha256() + total = path.stat().st_size + done = 0 + with open(path, 'rb', buffering=0) as f: + buf = bytearray(8 << 20) + view = memoryview(buf) + while n := f.readinto(buf): + h.update(view[:n]) + done += n + if progress: + progress(done, total) + return h.hexdigest() + + +def fingerprint(*parts): + """SHA-256 over strings and the contents of files and directory trees (a missing path counts as missing).""" + h = hashlib.sha256() + for p in parts: + if isinstance(p, Path): + if p.is_dir(): + for f in sorted(p.rglob('*')): + if f.is_file() and '__pycache__' not in f.parts: + h.update(f.relative_to(p).as_posix().encode() + b'\0' + bytes.fromhex(file_sha256(f))) + elif p.is_file(): + h.update(p.name.encode() + b'\0' + bytes.fromhex(file_sha256(p))) + else: + h.update(b'missing\0' + p.name.encode()) + else: + h.update(str(p).encode()) + h.update(b'\1') + return h.hexdigest() + + +def remove(path): + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + elif path.exists() or path.is_symlink(): + path.unlink() + + +def replace_file(src, dst): + """Copy src over dst through a temporary name, so a cancelled copy never leaves half a file under dst's name.""" + tmp = dst.with_name(dst.name + '.tmp') + shutil.copy2(src, tmp) + os.replace(tmp, dst) + + +def kill_tree(proc): + if proc.poll() is not None: + return + if os.name == 'nt': + subprocess.run(['taskkill', '/T', '/F', '/PID', str(proc.pid)], capture_output=True) + else: + proc.kill() + + +def run_tool(ctx, step, script, args, parse=None, cwd=None): + """Run a tool script with the bundled Python. Its output goes to /logs/-