From 1ece85644086063cc93c1dd09b28fd078f6e1819 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 08:22:46 +0300 Subject: [PATCH 1/2] Add tools/make_release.py, which assembles the installer release folder from pinned downloads Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 24 +- tools/build_game.py | 9 +- tools/check_content.py | 8 +- tools/make_release.py | 432 +++++++++++++++++++++++++++++++ tools/prepare_shaders.py | 2 + tools/requirements-installer.txt | 7 + 6 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 tools/make_release.py create mode 100644 tools/requirements-installer.txt diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index aa56ba05..bec4fc7d 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -19,7 +19,7 @@ Code copied or adapted from another project is listed here with its source, the | deko3d block-linear image layout, as ported in MacNativeGames' `nvn_tex.c` (Apple parts dropped, layout math unchanged; licence text in `games/totk/third_party/d2r_gfx/LICENSE.deko3d`, reproduced in the file header) | https://github.com/devkitPro/deko3d (`source/dk_image.cpp`), via MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/third_party/d2r_gfx/gfx/nvn/nvn_tex.c`, `nvn_tex.h`), Copyright (C) 2018-2020 fincs | zlib | M4 task T3b: `runtime/nvn_tex.c`, `runtime/nvn_tex.h` | | MacNativeGames texture auto-layout goldens (40 original-SDK size and block-height cases) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/tests/texture_auto_layout/probe.c`), Copyright (c) 2026 jhaizhou | MIT | M4 task T3b: `runtime/nvn_tex_probe.c`; T3f: `runtime/nvn_probe.c` (four of the cases, through the procs) | | MacNativeGames block-linear deswizzler (address function; copy loop split out as `tex_bl_copy`) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/third_party/d2r_gfx/gfx/tex/nvn_deswizzle.c`), Copyright (c) 2026 jhaizhou | MIT | M4 task T5a: `runtime/tex_deswizzle.c`, `runtime/tex.h` | -| SDL 3.4.18 (MSYS2 `mingw-w64-ucrt-x86_64-SDL3`), dynamic link, `SDL3.dll` and its dependency `libiconv-2.dll` (LGPL-2.1, unmodified, not linked into the exe) copied next to `totk.exe` | https://libsdl.org, Copyright (c) 1997-2026 Sam Lantinga | zlib | `runtime/input_win.c` (gamepad input, rumble, gyro and accelerometer) | +| SDL 3.4.18, dynamic link, `SDL3.dll` copied next to `totk.exe`. Developer builds use the MSYS2 build (`mingw-w64-ucrt-x86_64-sdl3`, which imports `libiconv-2.dll`, LGPL-2.1, unmodified, copied beside it); the installer release ships the upstream release `SDL3-3.4.18-win32-x64.zip`, which imports no libiconv | https://libsdl.org, Copyright (c) 1997-2026 Sam Lantinga | zlib | `runtime/input_win.c` (gamepad input, rumble, gyro and accelerometer) | | libopus (MSYS2 `mingw-w64-ucrt-x86_64-opus`), dynamic link, `libopus-0.dll` copied next to `totk.exe` | https://opus-codec.org, Copyright (c) 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon | BSD-3-Clause | `runtime/ov_media.c` (nn::codec hardware Opus decoder) | | Vulkan Memory Allocator 3.4.0 (MSYS2 `mingw-w64-ucrt-x86_64-vulkan-memory-allocator`), header-only, compiled into `totk.exe` | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator, Copyright (c) 2017-2026 Advanced Micro Devices, Inc. | MIT | `runtime/vk_vma.cpp` (the allocator behind `runtime/vk_memory.c`) | | MacNativeGames host movie decoder (`movie::Decoder` for VP9 over libvpx: entry list, buffer and event semantics, NV12 output format, PTS pacing) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/runtime/movie_host.c`, `movie_host.h`), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/ov_movie.c`, `runtime/movie.h` | @@ -31,6 +31,26 @@ These tools run on the developer's or player's machine during the build. They ar | Tool | Source | License | Purpose | |---|---|---|---| -| hactool | https://github.com/SciresM/hactool @ `1d64a83450e025622f3468c28fc4164dad2c5ef6`, with patches from MacNativeGames | ISC | NCA and update container extraction | +| hactool (the installer release ships it, see below) | https://github.com/SciresM/hactool @ `1d64a83450e025622f3468c28fc4164dad2c5ef6`, with patches from MacNativeGames | ISC | NCA and update container extraction | | Capstone | https://www.capstone-engine.org | BSD-3-Clause | ARM64 disassembly for analysis | | Unicorn | https://www.unicorn-engine.org | GPL-2.0 (Python bindings BSD) | Differential testing of the lifter; never linked into the game | + +## Distributed in the installer release + +`tools/make_release.py` assembles these into the release folder of `docs/installer.md`. Versions and sha256 of every download are pinned in that script; licence texts are in the folder's `licenses/` and in each component's own folder. + +| Component | Source | License | Where in the release | +|---|---|---|---| +| CPython 3.14.9 embeddable package, with `tkinter` and Tcl/Tk 9.0.4 taken from the python.org 3.14.9 installer's component MSI (unpacked, never run); includes `vcruntime140*.dll` (Microsoft, redistributable with Python), OpenSSL, libffi, SQLite, zlib, libtommath | https://www.python.org/downloads/release/python-3149/ | PSF-2.0 (Tcl/Tk: Tcl/Tk licence, BSD-style) | `python/` | +| numpy 2.5.4 (with its bundled OpenBLAS), lz4 4.4.5, cryptography 50.0.2 (with its bundled OpenSSL), cffi 2.1.1, pycparser 3.11 | PyPI binary wheels, pins in `tools/requirements-installer.txt` | BSD-3-Clause; lz4 BSD-3-Clause; cryptography Apache-2.0 OR BSD-3-Clause; cffi MIT; pycparser BSD-3-Clause | `python/Lib/site-packages/` | +| llvm-mingw 20261020 (LLVM/Clang/LLD 23, mingw-w64, libc++, libunwind, compiler-rt), x86_64 subset | https://github.com/mstorsjo/llvm-mingw/releases/tag/20261020 | Apache-2.0 with LLVM exception; mingw-w64 ZPL-2.1/BSD/public domain; libc++ and libunwind Apache-2.0 with LLVM exception and MIT/UIUC; winpthreads MIT | `llvm/` | +| SDL 3.4.18 upstream Windows release (`SDL3.dll`) and mingw development package (headers, `libSDL3.dll.a`) | https://github.com/libsdl-org/SDL/releases/tag/release-3.4.18 | zlib | `bin/SDL3.dll`, `runtime/deps/` | +| libopus 1.6.1 (MSYS2 `mingw-w64-ucrt-x86_64-opus` 1.6.1-2; `libopus-0.dll` imports only KERNEL32 and the UCRT api-ms-win-crt DLLs), headers and import library | https://repo.msys2.org/mingw/ucrt64/ | BSD-3-Clause | `bin/libopus-0.dll`, `runtime/deps/` | +| libvpx 1.17.0 (MSYS2 `mingw-w64-ucrt-x86_64-libvpx` 1.17.0-1), headers and static library compiled into `totk.exe`; `libvpx-LICENSE.txt` and `libvpx-PATENTS.txt` come from the v1.17.0 source tag | https://github.com/webmproject/libvpx | BSD-3-Clause with the WebM patent grant | `runtime/deps/`, `licenses/` | +| Vulkan-Headers and Vulkan-Loader 1.4.363.0 (MSYS2 packages; headers and `libvulkan-1.dll.a` only, no loader DLL) | https://github.com/KhronosGroup | Apache-2.0 OR MIT | `runtime/deps/` | +| Vulkan Memory Allocator 3.4.0 (MSYS2 package), header | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator | MIT | `runtime/deps/` | +| Ninja 1.13.2 (`ninja-win.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja.exe` | +| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check), built with MSYS2 UCRT64 clang; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | +| `totk_xlate.exe` (Ryujinx shader translator, NativeAOT build of `tools/xlate/`, see the Ryujinx row above) | built by `tools/build_xlate.py` | MIT | `bin/totk_xlate.exe` | +| MacNativeGames shader variant list | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/assets/shader_list.tsv`), Copyright (c) 2026 MacNativeGames contributors | MIT | `data/shader_list.tsv` | + diff --git a/tools/build_game.py b/tools/build_game.py index 73f75154..baabc895 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -36,6 +36,7 @@ Compile jobs share a memory budget (MEM_BUDGET, default the smaller of 50 GiB an big machine does not run out of memory on the largest chunks. Each phase reports its wall time and the peak memory of its processes (Windows job object accounting: committed private bytes). +--ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used. --toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR (default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++ @@ -563,6 +564,8 @@ def main(): ap.add_argument('--profile', type=Path, help="totk.exe's default profile directory (default build/profile)") ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of ' 'MSYS2 UCRT64 clang') + ap.add_argument('--ninja', type=Path, help='ninja.exe to build with (default: MSYS2 UCRT64, else bin/ninja.exe of the ' + 'release folder)') ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx ' f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})') ap.add_argument('--make-runtime-lib', action='store_true', @@ -585,7 +588,11 @@ def main(): sys.exit(f'no expected hashes known for {args.version}; pass --expected') expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json' hashes = json.loads(Path(expected).read_text())['exefs_sha256'] - global CLANG, AR, TOOLCHAIN_BIN, DEPS + global CLANG, AR, NINJA, TOOLCHAIN_BIN, DEPS + if args.ninja: + NINJA = args.ninja.resolve() + elif not NINJA.is_file() and (ROOT / 'bin' / 'ninja.exe').is_file(): + NINJA = ROOT / 'bin' / 'ninja.exe' # the release folder (tools/make_release.py) dlls = SDL3_DLLS if args.toolchain: TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve() diff --git a/tools/check_content.py b/tools/check_content.py index 77baf49c..0d868bfb 100644 --- a/tools/check_content.py +++ b/tools/check_content.py @@ -47,7 +47,9 @@ def list_files(staged: bool) -> list[Path]: return [Path(p) for p in out.decode("utf-8").split("\0") if p] -def problems_for(path: Path) -> list[str]: +def problems_for(path: Path, allow_large: bool = False) -> list[str]: + """allow_large: files over MAX_BYTES (toolchain binaries in a release folder) are checked for the generated marker + only, not refused.""" found = [] if path.suffix.lower() in BLOCKED_SUFFIXES: found.append(f"blocked file type {path.suffix}") @@ -56,6 +58,10 @@ def problems_for(path: Path) -> list[str]: if not path.is_file(): return found size = path.stat().st_size + if size > MAX_BYTES and allow_large: + if GENERATED_MARKER.encode() in path.read_bytes(): + found.append("generated from game code") + return found if size > MAX_BYTES: found.append(f"{size // 1024} KiB, over the {MAX_BYTES // 1024} KiB limit") return found diff --git a/tools/make_release.py b/tools/make_release.py new file mode 100644 index 00000000..79d97a61 --- /dev/null +++ b/tools/make_release.py @@ -0,0 +1,432 @@ +"""Assemble the release folder of docs/installer.md ("Release layout") from pinned downloads. + + .venv-py314\\Scripts\\python tools\\make_release.py --out build/release/TotK-Setup [--zip build/release/TotK-Setup.zip] + .venv-py314\\Scripts\\python tools\\make_release.py --out build/release/TotK-Setup --prove + +Needs Python 3.14 (tarfile reads the MSYS2 .zst packages), uv on PATH and network access on the first run. Downloads are +cached in toolchain/release-dl/ and checked against the sha256 in PINS; a mismatch stops the build. The folder holds: + + python/ python.org embeddable 3.14.9 + the packages of tools/requirements-installer.txt + tkinter (the Tcl/Tk 9.0 + files come out of the python.org installer's own component MSI, unpacked with msiexec /a and expand.exe; + the installer is never run) + llvm/ llvm-mingw, x86_64 only, the files clang and lld need to compile and link totk.exe + runtime/ libtotkrt.a (build_game.py --make-runtime-lib with llvm/), the runtime headers, romfs.c (compiled per + install), deps/ (headers and import libraries of SDL3, Opus, libvpx, Vulkan and VMA) + bin/ totk_xlate.exe (+ .id), hactool.exe, SDL3.dll (upstream), libopus-0.dll, ninja.exe + recomp/, tools/, data/, installer/ (if the checkout has one), licenses/, THIRD_PARTY_NOTICES.md + +--prove compiles generated/v121 and links build/release-check/totk.exe with nothing but the folder's python/, llvm/, +runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there. It needs a +generated/v121 and build/extracted/v121 from the usual game build (they are read, nothing is written to them). + +The folder contains nothing derived from the game: check_content runs over it before the script finishes. +""" +import argparse +import hashlib +import os +import shutil +import struct +import subprocess +import sys +import tarfile +import time +import urllib.request +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import check_content # noqa: E402 + +ROOT = Path(__file__).resolve().parent.parent +MAIN_ROOT = (ROOT / 'build').resolve().parent # build/, generated/, toolchain/ and refs/ are junctions in a worktree +DL = MAIN_ROOT / 'toolchain' / 'release-dl' +WORK = MAIN_ROOT / 'toolchain' / 'release-work' # unpacked downloads + +MSYS2 = 'https://repo.msys2.org/mingw/ucrt64/' +SDL_VERSION = '3.4.18' +LLVM_VERSION = '20261020' +MNG_COMMIT = 'b75be8cfa704df3701e86130c8ced268e79d6b11' # MacNativeGames, the shader list (refs/MacNativeGames) +# name: (url, sha256) +PINS = { + 'python-embed': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-embed-amd64.zip', + '3c01d7cfc666f746a6f064f60fb2156ab0ec464d3d738ab796371a46a9bb24a2'), + 'python-installer': ('https://www.python.org/ftp/python/3.14.9/python-3.14.9-amd64.exe', + 'a247a0b612244a803f507b39fb7121e3602e4160d59c3b99d98e735289c53394'), + 'llvm-mingw': (f'https://github.com/mstorsjo/llvm-mingw/releases/download/{LLVM_VERSION}/' + f'llvm-mingw-{LLVM_VERSION}-ucrt-x86_64.zip', + 'fd5462b7f5d93b2ff981fcd3f9c7e427e47f3203c64ea91c27bcc444eebd74cd'), + 'sdl3-dll': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-{SDL_VERSION}-win32-x64.zip', + '75c2c0fc74e7d1206aaedc22893a35887332f4f85f5c8b74fddc10d177985f2c'), + 'sdl3-devel': (f'https://github.com/libsdl-org/SDL/releases/download/release-{SDL_VERSION}/SDL3-devel-{SDL_VERSION}-mingw.tar.gz', + '049b9dd711e3fb3f8d84feb998dfec54a75e17cf539c4a80ff4d058d839771cf'), + 'ninja': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-win.zip', + '07fc8261b42b20e71d1720b39068c2e14ffcee6396b76fb7a795fb460b78dc65'), + 'opus': (MSYS2 + 'mingw-w64-ucrt-x86_64-opus-1.6.1-2-any.pkg.tar.zst', + 'bb114bcf26225f50f874497ebfd6b8224fff2a20bda3f8ee45cd07ce5f02657b'), + 'libvpx': (MSYS2 + 'mingw-w64-ucrt-x86_64-libvpx-1.17.0-1-any.pkg.tar.zst', + '8ff6919dbb88ea26ce28d22fd81d6f0d6e200cf56a4dd81ffc2d33124fe98024'), + 'libvpx-src': ('https://github.com/webmproject/libvpx/archive/refs/tags/v1.17.0.tar.gz', # LICENSE and PATENTS only + '1020f184046187baa2985dbde38e0691f49c44088bca7a1842b0236c6081dc0a'), + 'vulkan-headers': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-headers-1~1.4.363.0-1-any.pkg.tar.zst', + 'be602b2657cace531316e78b1eda4087e8f7f84ebb877f08fba198a2db06406a'), + 'vulkan-loader': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-loader-1~1.4.363.0-1-any.pkg.tar.zst', # import library only + '065076fe13d39ab97eea4740596426d5ae741b5044e48e1c9495d04835c92dbe'), + 'vma': (MSYS2 + 'mingw-w64-ucrt-x86_64-vulkan-memory-allocator-3.4.0-1-any.pkg.tar.zst', + '44c81162337915afecda3185f66fd98268a54454580b2717080c9dfef57fe781'), +} +# The component MSI inside the python.org installer that carries Tcl/Tk and tkinter (Burn names its payloads a0..a27). +TCLTK_PAYLOAD = ('a12', 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505') +SDL3_DLL_SHA256 = '2b4b128d8d510cc2d442693cf848c050d700d43dc667afe9e5fd82b0570d667e' # SDL3.dll inside the SDL zip + +# llvm-mingw: what clang and lld need to compile and link totk.exe for x86_64-w64-mingw32 (dynamic LLVM, libc++, compiler-rt). +LLVM_BIN = {'clang.exe', 'clang++.exe', 'clang-23.exe', 'ld.lld.exe', 'llvm-ar.exe', 'llvm-ranlib.exe', 'llvm-nm.exe', + 'llvm-objdump.exe', 'llvm-objcopy.exe', 'llvm-strip.exe', 'llvm-readobj.exe', 'llvm-dlltool.exe', + 'llvm-rc.exe', 'llvm-cvtres.exe', 'llvm-windres.exe', 'llvm-wrapper.exe', 'mingw32-common.cfg', + 'x86_64-w64-windows-gnu.cfg', 'x86_64-pc-windows-gnu.cfg', 'x86_64-w64-mingw32-clang.exe', + 'x86_64-w64-mingw32-clang++.exe', 'libLLVM-23.dll', 'libclang-cpp.dll', 'libc++.dll', 'libunwind.dll', + 'libwinpthread-1.dll', 'libffi-8.dll'} +LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32', 'busybox', 'python', 'share'} +LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} + +# tools/ files the installer steps run (and what they import), copied from the repo. +TOOLS_FILES = ['build_game.py', 'check_content.py', 'extract.py', 'make_romfs_patches.py', 'prepare_shaders.py', + 'romfs_index.py', 'shader_census.py'] +TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes +RUNTIME_SUFFIXES = {'.h', '.inc'} + +PROBES = ['--probe-kernel', '--probe-vkexec', '--probe-nvn-tex', '--probe-shader'] + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + for block in iter(lambda: f.read(1 << 20), b''): + h.update(block) + return h.hexdigest() + + +def fetch(name): + """The pinned download, from the cache or the network, checked against its sha256.""" + url, want = PINS[name] + DL.mkdir(parents=True, exist_ok=True) + path = DL / Path(url.split('?')[0]).name.replace('%7E', '~') + if name == 'libvpx-src': + path = DL / 'libvpx-1.17.0.tar.gz' + if not path.is_file(): + print(f'download {url}') + with urllib.request.urlopen(url) as r, open(path, 'wb') as f: + shutil.copyfileobj(r, f) + got = sha256_file(path) + if got != want: + sys.exit(f'{path}: sha256 {got}, pinned {want}') + return path + + +def unpack(name, member_filter=None): + """Extract a pinned archive into toolchain/release-work/ (once) and return that directory.""" + dest = WORK / name + stamp = dest / '.unpacked' + if stamp.is_file(): + return dest + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir(parents=True) + path = fetch(name) + if path.suffix == '.zip': + with zipfile.ZipFile(path) as z: + z.extractall(dest) + else: + with tarfile.open(path) as t: + t.extractall(dest, filter='data') + stamp.write_text('') + return dest + + +def copy_tree(src, dst, skip=lambda rel: False): + """Copy src into dst file by file (sorted), leaving out what skip(relative posix path) names.""" + for p in sorted(src.rglob('*')): + rel = p.relative_to(src) + if p.is_dir() or skip(rel.as_posix()): + continue + (dst / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(p, dst / rel) + + +def clean_env(*path_dirs): + """A process environment without MSYS2 and the user's tools: the Windows system directory and path_dirs only.""" + system = os.environ.get('SystemRoot', r'C:\Windows') + env = {k: v for k, v in os.environ.items() if k in ('SystemRoot', 'SYSTEMDRIVE', 'TEMP', 'TMP', 'USERPROFILE', + 'LOCALAPPDATA', 'APPDATA', 'COMSPEC', 'PATHEXT', 'NUMBER_OF_PROCESSORS')} + env['PATH'] = os.pathsep.join([*(str(Path(d)) for d in path_dirs), rf'{system}\System32', system]) + return env + + +def run(cmd, **kw): + print('+', ' '.join(str(c) for c in cmd), flush=True) + return subprocess.run([str(c) for c in cmd], check=True, **kw) + + +def mib(n): + return f'{n / (1 << 20):.1f} MiB' + + +def dir_size(p): + return sum(f.stat().st_size for f in Path(p).rglob('*') if f.is_file()) + + +# ---- python/ ------------------------------------------------------------------------------------------------------- + +def tcltk_files(): + """Directory holding the python.org installer's tcltk payload, unpacked (not installed) into toolchain/release-work.""" + dest = WORK / 'tcltk' + if (dest / 'DLLs' / '_tkinter.pyd').is_file(): + return dest + exe = fetch('python-installer') + data = exe.read_bytes() + cabs = [] + pos = data.find(b'MSCF\0\0\0\0') + while pos >= 0: + cabs.append((struct.unpack_from('/include and the x86_64 Windows builtins; no sanitizers, no other targets + if rel.startswith('lib/clang/') and '/include/' in rel: + return False + return not rel.endswith('/lib/windows/libclang_rt.builtins-x86_64.a') + return Path(rel).suffix.lower() in LLVM_SKIP_SUFFIXES or rel.startswith('include/ddk/') + + copy_tree(src, dst, skip) + return dst + + +# ---- bin/ and deps --------------------------------------------------------------------------------------------------- + +def build_bin(out): + b = out / 'bin' + b.mkdir(parents=True, exist_ok=True) + for f in ('totk_xlate.exe', 'totk_xlate.id'): + shutil.copy2(MAIN_ROOT / 'build' / 'xlate' / f, b / f) + shutil.copy2(MAIN_ROOT / 'toolchain' / 'hactool' / 'hactool.exe', b / 'hactool.exe') + dll = unpack('sdl3-dll') / 'SDL3.dll' + if sha256_file(dll) != SDL3_DLL_SHA256: + sys.exit(f'{dll}: not the pinned SDL3.dll') + shutil.copy2(dll, b / 'SDL3.dll') + shutil.copy2(unpack('opus') / 'ucrt64' / 'bin' / 'libopus-0.dll', b / 'libopus-0.dll') + shutil.copy2(unpack('ninja') / 'ninja.exe', b / 'ninja.exe') + + +def build_deps(out): + """runtime/deps: the headers and import libraries build_game.py --deps expects (include/, lib/).""" + d = out / 'runtime' / 'deps' + (d / 'include').mkdir(parents=True, exist_ok=True) + (d / 'lib').mkdir(parents=True, exist_ok=True) + sdl = unpack('sdl3-devel') / f'SDL3-{SDL_VERSION}' / 'x86_64-w64-mingw32' + copy_tree(sdl / 'include', d / 'include') + shutil.copy2(sdl / 'lib' / 'libSDL3.dll.a', d / 'lib') + for pkg, dirs, libs in (('opus', ['opus'], ['libopus.dll.a']), ('libvpx', ['vpx'], ['libvpx.a']), + ('vulkan-headers', ['vulkan', 'vk_video'], []), ('vulkan-loader', [], ['libvulkan-1.dll.a']), + ('vma', ['vma'], [])): + root = unpack(pkg) / 'ucrt64' + for inc in dirs: + copy_tree(root / 'include' / inc, d / 'include' / inc) + for lib in libs: + shutil.copy2(root / 'lib' / lib, d / 'lib') + + +def collect_licenses(out): + lic = out / 'licenses' + lic.mkdir(parents=True, exist_ok=True) + pairs = [ + (unpack('opus') / 'ucrt64' / 'share' / 'licenses' / 'opus' / 'COPYING', 'opus-COPYING.txt'), + (unpack('vulkan-headers') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-headers' / 'LICENSE', 'vulkan-headers-LICENSE.txt'), + (unpack('vulkan-loader') / 'ucrt64' / 'share' / 'licenses' / 'vulkan-loader' / 'LICENSE', 'vulkan-loader-LICENSE.txt'), + (unpack('vma') / 'ucrt64' / 'share' / 'licenses' / 'VulkanMemoryAllocator' / 'LICENSE', 'vulkan-memory-allocator-LICENSE.txt'), + (unpack('sdl3-dll') / 'LICENSE.txt', 'SDL3-LICENSE.txt'), + (MAIN_ROOT / 'toolchain' / 'hactool' / 'LICENSE', 'hactool-LICENSE.txt'), + (MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'), + ] + for src, name in pairs: + shutil.copy2(src, lic / name) + with tarfile.open(fetch('libvpx-src')) as t: # the MSYS2 package carries no licence text; the notice needs both + for member, name in (('libvpx-1.17.0/LICENSE', 'libvpx-LICENSE.txt'), ('libvpx-1.17.0/PATENTS', 'libvpx-PATENTS.txt')): + (lic / name).write_bytes(t.extractfile(member).read()) + for name in ('LICENSE.TXT',): + shutil.copy2(out / 'llvm' / name, lic / 'llvm-mingw-LICENSE.txt') + + +# ---- runtime/, recomp/, tools/, data/ --------------------------------------------------------------------------------- + +def runtime_lib(out, llvm): + """Build libtotkrt.a with the folder's llvm/ and copy it, the headers and romfs.c into runtime/.""" + build = MAIN_ROOT / 'build' / 'release-check' / 'rtlib' + gen = MAIN_ROOT / 'generated' / 'v121' + run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, + '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', build], + env=clean_env(llvm / 'bin', out / 'bin')) + rt = out / 'runtime' + shutil.copy2(build / 'libtotkrt.a', rt / 'libtotkrt.a') + for p in sorted((ROOT / 'runtime').rglob('*')): + if p.suffix in RUNTIME_SUFFIXES and p.is_file(): + (rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime')) + shutil.copy2(ROOT / 'runtime' / 'romfs.c', rt / 'romfs.c') # build_game.py INSTALL_SIDE + + +def build_sources(out): + copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r) + for f in TOOLS_FILES: + (out / 'tools').mkdir(exist_ok=True) + shutil.copy2(ROOT / 'tools' / f, out / 'tools' / f) + for d in TOOLS_DIRS: + copy_tree(ROOT / 'tools' / d, out / 'tools' / d, lambda r: '__pycache__' in r or r.endswith('.md')) + (out / 'data').mkdir(exist_ok=True) + text = subprocess.run(['git', '-C', MAIN_ROOT / 'refs' / 'MacNativeGames', 'show', + f'{MNG_COMMIT}:games/totk/assets/shader_list.tsv'], check=True, capture_output=True).stdout + (out / 'data' / 'shader_list.tsv').write_bytes(text) + if (ROOT / 'installer').is_dir(): + copy_tree(ROOT / 'installer', out / 'installer', lambda r: '__pycache__' in r) + shutil.copy2(ROOT / 'THIRD_PARTY_NOTICES.md', out / 'THIRD_PARTY_NOTICES.md') + + +# ---- checks ---------------------------------------------------------------------------------------------------------- + +def check_folder(out): + """check_content over every file of the release folder; a hit stops the build.""" + bad = [] + for p in sorted(out.rglob('*')): + if p.is_file(): + bad += [f'{p.relative_to(out)}: {problem}' for problem in check_content.problems_for(p, allow_large=True)] + if bad: + sys.exit('game-derived content in the release folder:\n' + '\n'.join(bad)) + print(f'check_content: {sum(1 for p in out.rglob("*") if p.is_file())} files, no hit') + + +def sizes(out): + rows = [(p.name if p.is_dir() else p.name, dir_size(p) if p.is_dir() else p.stat().st_size) for p in sorted(out.iterdir())] + for name, n in rows: + print(f' {name:26} {mib(n):>12}') + print(f' {"total":26} {mib(sum(n for _, n in rows)):>12}') + + +def write_zip(out, path): + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as z: + for p in sorted(out.rglob('*')): + if p.is_file(): + info = zipfile.ZipInfo((Path(out.name) / p.relative_to(out)).as_posix(), (1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + z.writestr(info, p.read_bytes(), compresslevel=6) + print(f'{path}: {mib(path.stat().st_size)}') + + +def prove(out): + """Compile generated/v121 and link totk.exe with the folder's tools only, then run the probes next to it.""" + out = out.resolve() + gen = MAIN_ROOT / 'generated' / 'v121' + ext = MAIN_ROOT / 'build' / 'extracted' / 'v121' + chk = MAIN_ROOT / 'build' / 'release-check' + env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin') + t0 = time.time() + run([out / 'python' / 'python.exe', '-I', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', + '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime', '--runtime-lib', out / 'runtime' / 'libtotkrt.a', + '--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', + '--profile', chk / 'profile'], env=env) + print(f'prove: build {time.time() - t0:.0f} s') + for dll in ('SDL3.dll', 'libopus-0.dll'): + shutil.copy2(out / 'bin' / dll, chk / dll) + failed = [] + for probe in PROBES: + r = subprocess.run([str(chk / 'totk.exe'), probe], cwd=chk, env=clean_env(chk), capture_output=True, text=True, + timeout=900) + tail = (r.stdout.strip().splitlines() or [''])[-1] + print(f'{probe}: exit {r.returncode}: {tail}') + if r.returncode: + failed.append(probe) + if failed: + sys.exit(f'probes failed: {failed}') + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)') + ap.add_argument('--zip', type=Path, help='also write the folder as this zip') + ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder') + args = ap.parse_args() + out = args.out.resolve() + if args.prove: + prove(out) + return + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + build_python(out) + llvm = build_llvm(out) + build_bin(out) + build_deps(out) + runtime_lib(out, llvm) + build_sources(out) + collect_licenses(out) + check_folder(out) + sizes(out) + if args.zip: + write_zip(out, args.zip.resolve()) + + +if __name__ == '__main__': + main() diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index 4596758f..93d9a7c8 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -52,6 +52,8 @@ ANS_REC = 4 + 32 + 32 + 147 + 1 BUNDLED_LIST = ROOT / 'tools' / 'data' / 'shader_list.tsv' # the installer archive carries a copy (no git, no refs/) +if not BUNDLED_LIST.is_file(): + BUNDLED_LIST = ROOT / 'data' / 'shader_list.tsv' # release folder layout (tools/make_release.py) def read_list(path): diff --git a/tools/requirements-installer.txt b/tools/requirements-installer.txt new file mode 100644 index 00000000..49cb95fc --- /dev/null +++ b/tools/requirements-installer.txt @@ -0,0 +1,7 @@ +# Packages of the release's bundled Python 3.14.9 (tools/make_release.py): what the installer steps import. +# numpy: recomp/gen.py. lz4: recomp/nso.py, tools/shader_census.py. cryptography, cffi, pycparser: tools/extract.py and the vendored extract scripts. +numpy==2.5.4 +lz4==4.4.5 +cryptography==50.0.2 +cffi==2.1.1 +pycparser==3.11 From e8dab5f05310133bd16c209579a675f427fd7832 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 08:28:41 +0300 Subject: [PATCH 2/2] Put the release folder's own code on the bundled Python's search path and run build_game.py from it without -I Co-Authored-By: Claude Opus 5.5 --- tools/make_release.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/make_release.py b/tools/make_release.py index 79d97a61..7c699b7f 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -212,7 +212,9 @@ def build_python(out): py = out / 'python' with zipfile.ZipFile(fetch('python-embed')) as z: z.extractall(py) - (py / 'python314._pth').write_text('python314.zip\n.\nLib/site-packages\n', encoding='utf-8') # embeddable: no site-packages by default + (py / 'python314._pth').write_text('python314.zip\n.\nLib/site-packages\n..\n../tools\n../recomp\n', encoding='utf-8') + # The embeddable Python ignores the script directory and PYTHONPATH once a ._pth file exists, so the folder's own code + # (installer/, and tools/*.py and recomp/*.py importing their siblings) has to be on the list. env = dict(os.environ, UV_CACHE_DIR=str(MAIN_ROOT / 'toolchain' / 'uv-cache')) run(['uv', 'pip', 'install', '--target', py / 'Lib' / 'site-packages', '--python', sys.executable, '--python-platform', 'windows', '--python-version', '3.14', '--only-binary', ':all:', @@ -383,7 +385,7 @@ def prove(out): chk = MAIN_ROOT / 'build' / 'release-check' env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin') t0 = time.time() - run([out / 'python' / 'python.exe', '-I', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', + run([out / 'python' / 'python.exe', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime', '--runtime-lib', out / 'runtime' / 'libtotkrt.a', '--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', '--profile', chk / 'profile'], env=env)