Merge dev into xbox-button-icons, with the restart and the prompts on the Linux port's portable calls

The restart moves into gfx_present.c's gfx_window_closed, which the Win32 and SDL windows share, and linux.c gets
plat_restart and plat_restart_wait: posix_spawn of /proc/self/exe with no descriptor past stderr and the default
signals, and a wait on a pidfd. The prompts code takes plat_file_exists, plat_setenv, strcasecmp and strdup, and the
menu and the log say a game resolution of another shape than 16:9 (16:10 now as well as 21:9) keeps the Switch prompts.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Deon FoucheandClaude Opus 5.5 committed 2026-10-31 22:14:22 +10:00
commit 01d69bf6d1
134 files changed
+9659 -4837

No files matched your search

+145 -31
View File
@@ -3,6 +3,11 @@
Writes build/<version>/build.ninja and runs MSYS2 UCRT64 Ninja on it. Ninja keeps the build incremental: a chunk is
recompiled only when it, a header it includes or its flags change.
On Linux the same steps make build/<version>/totk with the clang, lld and ninja on PATH (or the clang and lld in --toolchain DIR/bin): the
runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and totk.rc, the libraries are
the system's (Vulkan, SDL3, Opus, libvpx, zstd; libstdc++ static), and the executable finds a libSDL3.so.0 beside it
first (rpath $ORIGIN). docs/building.md lists the packages.
.venv\\Scripts\\python tools\\build_game.py # generated/v121 -> build/v121/totk.exe
.venv\\Scripts\\python tools\\build_game.py --modules subsdk0 # one module (links if it has no calls into others)
.venv\\Scripts\\python tools\\build_game.py --jobs 8
@@ -50,7 +55,7 @@ Standard library only.
"""
import argparse
import ctypes
import ctypes.wintypes as wt
import ctypes.wintypes as wt # the job-object structures below; importing it works on every OS
import hashlib
import json
import os
@@ -58,31 +63,79 @@ import shutil
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
import romfs_index
from build_shaders import stale as stale_shaders
from check_content import GENERATED_MARKER
WIN = os.name == 'nt'
if not WIN:
import resource
ROOT = Path(__file__).resolve().parent.parent
# In a git worktree (tools/worktree.py) build/ and generated/ are junctions into the main checkout, and resolve() follows
# them. MAIN_ROOT is where they land, so resolved paths still print and compare relative to a root.
MAIN_ROOT = (ROOT / 'build').resolve().parent
UCRT = Path(r'C:\msys64\ucrt64\bin')
CLANG = UCRT / 'clang.exe'
NINJA = UCRT / 'ninja.exe'
SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll']
EXE = 'totk.exe' if WIN else 'totk'
def on_path(name):
"""name on PATH, else a path that does not exist (the tool check below names it)."""
found = shutil.which(name)
return Path(found) if found else Path('/usr/bin') / name
CLANG = UCRT / 'clang.exe' if WIN else on_path('clang')
NINJA = UCRT / 'ninja.exe' if WIN else on_path('ninja')
SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] if WIN else []
# Flags for generated code as LAS Recomp builds it (Makefile GENFLAGS, Windows), plus -march for the FP/SIMD lowering.
GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden',
'-march=x86-64-v3']
# _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain) to
# Windows 7, which hides the per-monitor DPI functions gfx_window.c and dpi_probe.c call.
RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3', '-D_WIN32_WINNT=0x0A00']
# -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c)
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3']
if WIN:
# _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain)
# to Windows 7, which hides the per-monitor DPI functions gfx_window_win.c and dpi_probe_win.c call.
RT_FLAGS.append('-D_WIN32_WINNT=0x0A00')
# -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c)
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
else:
# -no-pie: the lifted modules' function tables (hundreds of thousands of pointers) need no load-time relocation.
# $ORIGIN: a libSDL3.so.0 next to totk comes before the system's (the release's portable folder).
LINK_FLAGS = ['-fuse-ld=lld', '-no-pie', '-Wl,-z,noexecstack', "'-Wl,-rpath,$$ORIGIN'",
'"-Wl,-Map=$builddir/totk.map"']
LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-lvpx', '-lzstd', '-l:libstdc++.a', '-lm', '-lpthread', '-ldl']
VMA_INCLUDE = Path('/usr/include') # vk_mem_alloc.h; Arch (AUR) vulkan-memory-allocator, Debian libvulkan-memory-allocator-dev
# Without a system vk_mem_alloc.h, the header the Windows release pins (tools/make_release.py, VMA 3.4.0) is fetched once
# into toolchain/vma: one MIT header (THIRD_PARTY_NOTICES.md), checked against its hash.
VMA_FETCH = ('https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator/v3.4.0/include/vk_mem_alloc.h',
'8487b7995ad3b263eb73bc5b9a77d71aa69b6bef5d58a715c02d2663afd81f1a')
VMA_LOCAL = ROOT / 'toolchain' / 'vma'
def vma_include():
"""The directory holding vk_mem_alloc.h: the system's, else toolchain/vma, fetched there on first use."""
if (VMA_INCLUDE / 'vk_mem_alloc.h').is_file():
return VMA_INCLUDE
dst = VMA_LOCAL / 'vk_mem_alloc.h'
if not dst.is_file():
url, sha = VMA_FETCH
print(f'build_game: no {VMA_INCLUDE / "vk_mem_alloc.h"}; fetching VMA 3.4.0 into {VMA_LOCAL}')
try:
data = urllib.request.urlopen(url, timeout=60).read()
except OSError as e:
sys.exit(f'build_game: fetching {url} failed ({e}); install VMA or put vk_mem_alloc.h in {VMA_LOCAL}')
if hashlib.sha256(data).hexdigest() != sha:
sys.exit(f'build_game: {url} does not match the pinned hash')
VMA_LOCAL.mkdir(parents=True, exist_ok=True)
dst.write_bytes(data)
return VMA_LOCAL
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lvulkan-1', '-lSDL3', '-lopus',
'-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a']
@@ -92,7 +145,7 @@ DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw'
# functions sit on it).
INSTALL_SIDE = {'romfs.c', 'patches.c', 'ov_zstd.c'}
LIB_NAME = 'libtotkrt.a'
AR = UCRT / 'llvm-ar.exe'
AR = UCRT / 'llvm-ar.exe' if WIN else on_path('llvm-ar')
WINDRES = UCRT / 'windres.exe' # runtime/totk.rc: the manifest of totk.exe
TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain
DEPS = None # the deps directory of an llvm-mingw build
@@ -196,12 +249,21 @@ def c_string(s):
RUNTIME = ROOT / 'runtime' # --runtime: a copy of runtime/ lets a task build without touching the shared tree
def other_os(name):
"""A runtime file for another OS: Win32 ones (*_win.*, platform/win32.c, totk.rc) off Windows, the SDL window's
(*_sdl.*) and platform/linux.c on Windows."""
stem = name.rsplit('.', 1)[0]
if WIN:
return stem.endswith('_sdl') or name == 'linux.c'
return stem.endswith('_win') or name in ('win32.c', 'totk.rc')
def runtime_sources(skip=()):
"""Every runtime/*.c, *.cpp, platform/*.c, *.rc, the in-game menu (ui/*.cpp) and Dear ImGui except the names in skip,
so a task adds a file without editing this script."""
"""Every runtime/*.c, *.cpp, platform/*.c, *.rc, the in-game menu (ui/*.cpp) and Dear ImGui of this OS except the
names in skip, so a task adds a file without editing this script."""
pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc', 'ui/*.cpp', 'third_party/imgui/*.cpp',
'third_party/imgui/backends/*.cpp')
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip)
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip and not other_os(p.name))
def runtime_object(out, src):
@@ -254,10 +316,10 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects
except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE."""
inc = f'-I {cmd_path(RUNTIME)} -I {cmd_path(gen)}'
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(VMA_INCLUDE)
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(vma_include())
imgui = cmd_path(RUNTIME / 'third_party' / 'imgui')
rt_inc = f'{inc} -I {cmd_path(out)} -I {vma} -I {imgui}' # romfs_config.h, vk_mem_alloc.h (+ deps headers), imgui.h
libs = [f'-L {cmd_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS
libs = [f'-L {cmd_path(DEPS / "lib")}'] + (LLVM_LINK_LIBS if WIN else LINK_LIBS) if DEPS else LINK_LIBS
# A compiler cache (sccache) may front clang: every lift rewrites the generated chunks with the same contents, so
# the cached objects save the compile step on the next build.
cache = f'{cmd_arg(os.environ["SCCACHE"])} ' if os.environ.get('SCCACHE') else ''
@@ -324,7 +386,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
rt.append(o)
objs['runtime'] = rt
every = [o for v in objs.values() for o in v]
exe = out / 'totk.exe'
exe = out / EXE
if make_lib:
exe = out / LIB_NAME
lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
@@ -370,7 +432,7 @@ def main_objects(gen, main_out, rt_dir, mods):
pairs[o] = theirs
# Ninja runs in the main checkout: the main build.ninja is relative to it, and the worktree's runtime/ files carry newer
# mtimes that would make every object look stale. Header equality is checked byte for byte below.
env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', ''))
env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', '')) if WIN else dict(os.environ)
targets = [ninja_path(t) for t in pairs.values()]
for i in range(0, len(targets), 200):
r = subprocess.run([str(NINJA), '-f', str(main_ninja), '-n', '-d', 'explain'] + targets[i:i + 200], cwd=MAIN_ROOT,
@@ -539,7 +601,13 @@ CREATE_SUSPENDED = 0x4
def run_measured(cmd, env):
"""Run cmd in its own job object. Returns (exit code, seconds, peak bytes of one process, peak bytes of all)."""
"""Run cmd in its own job object. Returns (exit code, seconds, peak bytes of one process, peak bytes of all).
Off Windows: the largest resident set of one child process, and 0 for all of them (no job accounting)."""
if not WIN:
t0 = time.monotonic()
code = subprocess.call(cmd, cwd=ROOT, env=env)
peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss * 1024 # KiB on Linux
return code, time.monotonic() - t0, peak, 0
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
k32.CreateJobObjectW.restype = wt.HANDLE
k32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wt.LPCWSTR]
@@ -573,6 +641,9 @@ def run_measured(cmd, env):
def total_ram():
if not WIN:
return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
class MemStatus(ctypes.Structure):
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
@@ -597,6 +668,30 @@ def pe_sections(path):
return out, image
def elf_sections(path):
"""{section name: size} of the allocated sections of an ELF64 file, and the extent they span in memory."""
with open(path, 'rb') as f:
head = f.read(64)
shoff = int.from_bytes(head[0x28:0x30], 'little')
shentsize, shnum, shstrndx = (int.from_bytes(head[o:o + 2], 'little') for o in (0x3A, 0x3C, 0x3E))
f.seek(shoff)
table = f.read(shentsize * shnum)
sh = [table[i * shentsize:(i + 1) * shentsize] for i in range(shnum)]
f.seek(int.from_bytes(sh[shstrndx][0x18:0x20], 'little'))
names = f.read(int.from_bytes(sh[shstrndx][0x20:0x28], 'little'))
out, lo, hi = {}, None, 0
for s in sh:
flags = int.from_bytes(s[0x08:0x10], 'little')
addr, size = int.from_bytes(s[0x10:0x18], 'little'), int.from_bytes(s[0x20:0x28], 'little')
if not flags & 2 or not size: # SHF_ALLOC
continue
name = names[int.from_bytes(s[0:4], 'little'):].split(b'\0', 1)[0].decode(errors='replace')
out[name] = out.get(name, 0) + size
lo = addr if lo is None else min(lo, addr)
hi = max(hi, addr + size)
return out, hi - (lo or 0)
def mib(n):
return f'{n / (1 << 20):,.1f} MiB'
@@ -623,9 +718,12 @@ def main():
"build/profile-<version> otherwise, so trying another version leaves "
"the main profile alone; saves load on 1.2.1 and 1.4.3 alike)")
ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of '
'MSYS2 UCRT64 clang')
'MSYS2 UCRT64 clang; on Linux an LLVM directory (bin/clang, '
'bin/llvm-ar) instead of the clang on PATH')
ap.add_argument('--ninja', type=Path, help='ninja.exe to build with (default: MSYS2 UCRT64, else bin/ninja.exe of the '
'release folder)')
ap.add_argument('--sysroot', type=Path, help='Linux: the glibc headers and libraries to compile and link against '
'instead of the host\'s (the release\'s llvm/sysroot)')
ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx '
f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})')
ap.add_argument('--make-runtime-lib', action='store_true',
@@ -652,19 +750,33 @@ def main():
global CLANG, AR, WINDRES, NINJA, TOOLCHAIN_BIN, DEPS
if args.ninja:
NINJA = args.ninja.resolve()
elif not NINJA.is_file() and (ROOT / 'bin' / 'ninja.exe').is_file():
NINJA = ROOT / 'bin' / 'ninja.exe' # the release folder (tools/make_release.py)
elif not NINJA.is_file() and (ROOT / 'bin' / ('ninja' + ('.exe' if WIN else ''))).is_file():
NINJA = ROOT / 'bin' / ('ninja' + ('.exe' if WIN else '')) # the release folder (tools/make_release.py)
dlls = SDL3_DLLS
suffix = '.exe' if WIN else ''
if args.toolchain:
TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve()
CLANG = TOOLCHAIN_BIN / 'clang.exe'
AR = TOOLCHAIN_BIN / 'llvm-ar.exe'
CLANG = TOOLCHAIN_BIN / f'clang{suffix}'
AR = TOOLCHAIN_BIN / f'llvm-ar{suffix}'
WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe'
if args.toolchain and WIN or args.deps and (args.toolchain or not WIN): # Linux: --deps only, never deps-mingw
DEPS = (args.deps or DEFAULT_DEPS).resolve()
dlls = sorted((DEPS / 'bin').glob('*.dll'))
dlls = sorted((DEPS / 'bin').glob('*.dll')) if WIN else []
if DEPS and not WIN:
# A release (--deps): Opus, libvpx and zstd go into the executable, so it needs no more than libc, Vulkan and
# the libSDL3.so.0 beside it
for lib in ('opus', 'vpx', 'zstd'):
LINK_LIBS[LINK_LIBS.index(f'-l{lib}')] = f'-l:lib{lib}.a'
if args.sysroot and not WIN:
# Every compile and the link: the game then needs no newer glibc than the sysroot's (the Steam Runtime's)
sysroot = ['--target=x86_64-linux-gnu', f'--sysroot={cmd_path(args.sysroot.resolve())}']
GEN_FLAGS[:0] = sysroot
RT_FLAGS[:0] = sysroot
LINK_FLAGS[:0] = sysroot
for tool in (CLANG, NINJA):
if not tool.is_file():
sys.exit(f'{tool} not found: install MSYS2 UCRT64 clang, lld and ninja (docs/building.md)')
sys.exit(f'{tool} not found: install ' + ('MSYS2 UCRT64 clang, lld and ninja' if WIN else 'clang, lld and ninja')
+ ' (docs/building.md)')
if not (gen / 'layout.json').is_file():
sys.exit(f'{gen / "layout.json"} not found: run the generator first')
@@ -689,7 +801,7 @@ def main():
print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s')
else:
romfs_index.write_placeholder(out)
print(f'build_game: warning: no {paths["romfs"] / "index.json"}; totk.exe will have no RomFS', file=sys.stderr)
print(f'build_game: warning: no {paths["romfs"] / "index.json"}; {EXE} will have no RomFS', file=sys.stderr)
copy_romfs_patches(args.version, out)
global RUNTIME
RUNTIME = (args.runtime or RUNTIME).resolve()
@@ -724,13 +836,15 @@ def main():
args.make_runtime_lib)
write_if_changed(out / 'build.ninja', text)
env = dict(os.environ, PATH=str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', ''))
env = dict(os.environ)
if WIN or TOOLCHAIN_BIN:
env['PATH'] = str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', '')
base = [str(NINJA), '-f', str(out / 'build.ninja'), '-j', str(jobs)]
nfiles = sum(len(v) for v in mods.values())
print(f'build_game: {nfiles} generated files in {len(linked)} modules ({", ".join(linked)}), {jobs} jobs', flush=True)
code, dt, peak_one, peak_all = run_measured(base + ['objects'], env)
print(f'compile: {dt:.1f} s, peak {mib(peak_one)} per process, {mib(peak_all)} all processes')
print(f'compile: {dt:.1f} s, peak {mib(peak_one)} per process' + (f', {mib(peak_all)} all processes' if WIN else ''))
if code:
sys.exit(code)
for mod, files in objs.items():
@@ -752,7 +866,7 @@ def main():
return
for dll in dlls: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to totk.exe
shutil.copy2(dll, exe.parent / dll.name)
sections, image = pe_sections(exe)
sections, image = pe_sections(exe) if WIN else elf_sections(exe)
print(f'{below_root(exe)}: file {mib(exe.stat().st_size)}, '
f'image {mib(image)}; ' + ', '.join(f'{k} {mib(v)}' for k, v in sections.items() if k.startswith('.') and v))
if image > 3 * GiB // 2:
+32 -13
View File
@@ -3,7 +3,8 @@
Fetches the pinned commit into toolchain/src/hactool, resets it (local changes and build output there are discarded),
applies portable.patch (MacNativeGames' Windows-relevant changes) and unicode.patch (ours), and builds a static exe
with MSYS2 UCRT64 gcc; mbedtls comes with hactool's tree. The exe and hactool's LICENSE go to toolchain/hactool, where
tools/extract.py and tools/make_release.py take them. Run with .venv.
tools/extract.py and tools/make_release.py take them. Run with .venv. On Linux: the system's cc and make build
toolchain/hactool/hactool (unicode.patch only adds a Windows manifest, under EXEEXT .exe).
python tools/build_hactool.py [--src DIR] [--out DIR]
"""
@@ -13,15 +14,20 @@ import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
URL = 'https://github.com/SciresM/hactool.git'
COMMIT = '1d64a83450e025622f3468c28fc4164dad2c5ef6'
PATCHES = [ROOT / 'tools' / 'hactool' / n for n in ('portable.patch', 'unicode.patch')]
PATCHES = [ROOT / 'tools' / 'hactool' / n for n in ('portable.patch', 'unicode.patch', 'mbedtls-asm-clobbers.patch')]
WIN = os.name == 'nt'
MSYS_BASH = Path(r'C:\msys64\usr\bin\bash.exe')
EXE = 'hactool.exe' if WIN else 'hactool'
CONFIG_MK = ('CC = gcc\nEXEEXT = .exe\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n'
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto -static\n')
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto -static\n') if WIN else \
('CC = cc\nEXEEXT =\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n'
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto\n')
def git(src, *args):
@@ -44,31 +50,44 @@ def fetch(src):
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--src', type=Path, default=ROOT / 'toolchain' / 'src' / 'hactool', help='source checkout')
ap.add_argument('--out', type=Path, default=ROOT / 'toolchain' / 'hactool', help='where hactool.exe goes')
ap.add_argument('--out', type=Path, default=ROOT / 'toolchain' / 'hactool', help='where hactool(.exe) goes')
ap.add_argument('--cc', help='Linux: the C compiler command for hactool and its bundled mbedtls, for example '
'"clang --sysroot=DIR" (tools/make_release_linux.py); default cc')
args = ap.parse_args()
if not MSYS_BASH.is_file():
if WIN and not MSYS_BASH.is_file():
sys.exit(f'{MSYS_BASH} not found: install MSYS2 with the UCRT64 gcc and make (docs/building.md)')
if not WIN and not (shutil.which('make') and (args.cc or shutil.which('cc'))):
sys.exit('make and cc not found: install a C compiler and make (docs/building.md)')
src, out = args.src.resolve(), args.out.resolve()
fetch(src)
for p in PATCHES:
git(src, 'apply', '--whitespace=nowarn', str(p))
(src / 'config.mk').write_text(CONFIG_MK, encoding='utf-8', newline='\n')
config = CONFIG_MK.replace('CC = cc\n', f'CC = {args.cc}\n') if args.cc else CONFIG_MK
(src / 'config.mk').write_text(config, encoding='utf-8', newline='\n')
env = dict(os.environ, MSYSTEM='UCRT64', CHERE_INVOKING='1')
r = subprocess.run([str(MSYS_BASH), '-lc', f'make -j{os.cpu_count() or 4}'], cwd=src, env=env,
capture_output=True, text=True)
jobs = f'-j{os.cpu_count() or 4}'
if WIN:
env = dict(os.environ, MSYSTEM='UCRT64', CHERE_INVOKING='1')
r = subprocess.run([str(MSYS_BASH), '-lc', f'make {jobs}'], cwd=src, env=env, capture_output=True, text=True)
else:
r = subprocess.run(['make', jobs, *([f'CC={args.cc}'] if args.cc else [])], cwd=src, capture_output=True,
text=True) # CC= on the command line: the bundled mbedtls's own make reads it too
(src / 'build.log').write_text(r.stdout + r.stderr, encoding='utf-8')
if r.returncode:
sys.exit(f'make failed ({r.returncode}); log: {src / "build.log"}')
out.mkdir(parents=True, exist_ok=True)
shutil.copy2(src / 'hactool.exe', out / 'hactool.exe')
shutil.copy2(src / EXE, out / EXE)
shutil.copy2(src / 'LICENSE', out / 'LICENSE')
banner = subprocess.run([str(out / 'hactool.exe')], capture_output=True, text=True)
# hactool reads ~/.switch/prod.keys before it prints the banner and exits on a malformed key there, so the check
# runs with an empty home: it tests the binary, not the builder's keys.
with tempfile.TemporaryDirectory() as home:
env = dict(os.environ, HOME=home, USERPROFILE=home)
banner = subprocess.run([str(out / EXE)], capture_output=True, text=True, env=env)
if 'hactool (c) SciresM' not in banner.stdout + banner.stderr:
sys.exit(f'{out / "hactool.exe"} does not start')
print(f'{out / "hactool.exe"}: {COMMIT[:7]} + {", ".join(p.name for p in PATCHES)}')
sys.exit(f'{out / EXE} does not start')
print(f'{out / EXE}: {COMMIT[:7]} + {", ".join(p.name for p in PATCHES)}')
if __name__ == '__main__':
+5 -3
View File
@@ -10,13 +10,15 @@ shader. Needs glslc and spirv-val from MSYS2 UCRT64 (package mingw-w64-ucrt-x86_
import argparse
import hashlib
import os
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
BIN = Path(r'C:\msys64\ucrt64\bin')
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere glslc and spirv-val come from PATH
WIN = os.name == 'nt'
TARGET = 'vulkan1.1'
@@ -54,9 +56,9 @@ def stale(runtime):
def compile_one(src):
with tempfile.TemporaryDirectory() as tmp:
spv = Path(tmp) / 'out.spv'
subprocess.run([str(BIN / 'glslc.exe'), '-O', f'--target-env={TARGET}', '-fshader-stage=compute',
subprocess.run([str(BIN / 'glslc.exe') if WIN else 'glslc', '-O', f'--target-env={TARGET}', '-fshader-stage=compute',
'-o', str(spv), str(src)], check=True)
subprocess.run([str(BIN / 'spirv-val.exe'), f'--target-env', TARGET, str(spv)], check=True)
subprocess.run([str(BIN / 'spirv-val.exe') if WIN else 'spirv-val', f'--target-env', TARGET, str(spv)], check=True)
data = spv.read_bytes()
words = [int.from_bytes(data[i:i + 4], 'little') for i in range(0, len(data), 4)]
header_path(src).write_text(header_of(src, words), encoding='utf-8', newline='\n')
+20 -8
View File
@@ -2,7 +2,8 @@
Copies tools/vendor/ryujinx to build/xlate/ryujinx, applies tools/xlate/patches/*.patch there (tools/vendor stays
byte-identical; 0001-0004 are MNG's, 0005-0009 are ours), then runs `dotnet publish`. Needs the .NET 10 SDK, the MSVC build
tools and vswhere.exe (docs/building.md). Run with .venv.
tools and vswhere.exe (docs/building.md). Run with .venv. On Linux it builds build/xlate/totk_xlate for linux-x64, which
NativeAOT links with the system's clang (no MSVC); NativeAOT builds for the OS it runs on only.
Next to the exe it writes totk_xlate.id: the translator identity, a sha256 over everything the build reads (this script,
Program.cs, the csproj, the patches, the vendor tree). runtime/gfx_shader.c puts it into every shader cache key, so a
@@ -28,6 +29,8 @@ VENDOR = ROOT / "tools" / "vendor" / "ryujinx"
MNG_VENDOR = ROOT / "refs" / "MacNativeGames" / "vendor" / "ryujinx"
XLATE = ROOT / "tools" / "xlate"
BUILD = ROOT / "build" / "xlate"
WIN = os.name == "nt"
EXE = "totk_xlate.exe" if WIN else "totk_xlate"
VSWHERE_DIR = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio" / "Installer"
@@ -133,21 +136,25 @@ def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--id-only", action="store_true", help="write totk_xlate.id for the exe already built")
parser.add_argument("--out", type=Path, help="build directory instead of build/xlate")
parser.add_argument("--llvm", type=Path, help="Linux: an LLVM directory whose bin/clang and lld link the exe")
parser.add_argument("--sysroot", type=Path, help="Linux: link against this sysroot's glibc instead of the host's "
"(tools/make_release_linux.py)")
a = parser.parse_args()
if a.out:
global BUILD
BUILD = a.out.resolve()
if a.id_only:
write_identity(BUILD / "totk_xlate.exe")
write_identity(BUILD / EXE)
return
started = time.time()
dotnet = shutil.which("dotnet")
if not dotnet:
sys.exit("missing: dotnet (.NET 10 SDK)")
if not (VSWHERE_DIR / "vswhere.exe").is_file() and not shutil.which("vswhere"):
sys.exit(f"missing: vswhere.exe (expected in {VSWHERE_DIR}); NativeAOT cannot link without it")
env = dict(os.environ)
env["PATH"] = str(VSWHERE_DIR) + os.pathsep + env["PATH"]
if WIN:
if not (VSWHERE_DIR / "vswhere.exe").is_file() and not shutil.which("vswhere"):
sys.exit(f"missing: vswhere.exe (expected in {VSWHERE_DIR}); NativeAOT cannot link without it")
env["PATH"] = str(VSWHERE_DIR) + os.pathsep + env["PATH"]
verify_vendor()
src = prepare_sources()
@@ -159,13 +166,18 @@ def main() -> None:
out = BUILD / "publish"
shutil.rmtree(out, ignore_errors=True)
cmd = [
dotnet, "publish", str(work / "totk_xlate.csproj"), "-c", "Release", "-r", "win-x64",
dotnet, "publish", str(work / "totk_xlate.csproj"), "-c", "Release", "-r", "win-x64" if WIN else "linux-x64",
"-p:PublishAot=true", "-p:InvariantGlobalization=true", f"-p:RyuSrc={src}", "-o", str(out),
]
if a.llvm:
env["PATH"] = str(a.llvm.resolve() / "bin") + os.pathsep + env["PATH"]
cmd += ["-p:CppCompilerAndLinker=clang", "-p:LinkerFlavor=lld"]
if a.sysroot:
cmd.append(f"-p:SysRoot={a.sysroot.resolve()}")
if subprocess.run(cmd, env=env).returncode != 0:
sys.exit("dotnet publish failed")
exe = BUILD / "totk_xlate.exe"
shutil.copy2(out / "totk_xlate.exe", exe)
exe = BUILD / EXE
shutil.copy2(out / EXE, exe)
write_identity(exe)
print(f"built {exe} ({exe.stat().st_size / 1e6:.1f} MB) in {time.time() - started:.0f} s")
+16 -49
View File
@@ -1,9 +1,10 @@
#!/usr/bin/env python3
"""Fails when runtime code outside runtime/platform/ uses the Win32 API directly.
New runtime code goes through runtime/platform/platform.h. Files that still use Win32 are listed in ALLOWED with the
reason; the list shrinks as the platform layer takes over.
A file in ALLOWED that no longer matches is noted, so the entry gets removed.
Runtime code goes through runtime/platform/platform.h. The Win32 back ends of the window system are Win32 by design and
are not scanned: files named *_win.c / *_win.cpp (runtime/ and runtime/ui/), whose SDL counterparts (*_sdl.*) serve
other hosts. Any other file that still uses Win32 goes into ALLOWED with the reason; a file in ALLOWED that no longer
matches is noted, so the entry gets removed.
Usage: python tools/check_platform.py [--root DIR] (DIR holds runtime/; default: the repository root)
"""
@@ -12,51 +13,8 @@ import re
import sys
from pathlib import Path
# Per-file allowlist. "window", "input", "audio" and "probe" are permanent (OS-specific by nature); the rest wait for
# 2.7a (memory, faults, page tracking) or 2.7b (threads, time, locks, FP state).
ALLOWED = {
"gfx_window.c": "window",
"gfx_window_probe.c": "probe",
"input_win.c": "input",
"prof_win.c": "profiler",
"rdoc_win.c": "RenderDoc loader",
"audren.c": "audio",
"kernel_probe.c": "probe",
"nvn_probe.c": "probe",
"ov_probe.c": "probe",
"tex_probe.c": "probe",
"vk_exec_probe.c": "probe",
"gfx_shader_probe.c": "probe",
"gfx_shader.c": "2.7b: threads, SRW locks, interlocked counters, tick count",
"memory.c": "2.7a",
"kernel.c": "2.7a/b",
"main.c": "2.7a/b",
"dispatch.c": "2.7b",
"loader.c": "2.7a",
"svc.c": "2.7b",
"vk_memory.c": "2.7a",
"vk_texture.c": "2.7b",
"audren.h": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"boot.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"flight.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"fs_host.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"ipc.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"nvn_api_cmd.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"nvn_api_core.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"nvn_api_obj.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"nvn_procs.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"ov_gfx.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"ov_input.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"ov_media.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"ov_system.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"romfs.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"tex_astc_mt.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"vk_backend.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"vk_device.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"vk_dump.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"vk_pipeline.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
"vk_sync.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
}
# Per-file allowlist of shared files that still use Win32, with the reason. Empty: every shared file is portable.
ALLOWED = {}
INCLUDE = re.compile(r'^\s*#\s*include\s*[<"](windows|winsock2|ws2tcpip|synchapi|memoryapi|processthreadsapi|'
r'fileapi|handleapi|libloaderapi|winbase|winnt)\.h[>"]', re.M)
@@ -75,6 +33,12 @@ CALLS = [
"(Acquire|Release)SRWLock(Exclusive|Shared)", "WakeAllConditionVariable", "WakeConditionVariable",
"SleepConditionVariableSRW", "InitOnceExecuteOnce", "INIT_ONCE", "Interlocked[A-Za-z0-9]*", "TlsAlloc", "FlsAlloc",
"LoadLibrary[AW]?", "GetProcAddress", "FreeLibrary", "HANDLE", "DWORD", "LPVOID", "WINAPI",
"SuspendThread", "ResumeThread", "GetThreadContext", "SetThreadContext", "OpenThread", "CreateFiber(Ex)?",
"ConvertThreadToFiber(Ex)?", "ConvertFiberToThread", "SwitchToFiber", "DeleteFiber", "BCrypt[A-Za-z]*",
"_aligned_malloc", "_aligned_free", "_stricmp", "_strnicmp", "_strdup", "_wfopen", "_fseeki64", "_ftelli64",
"localtime_s", "strtok_s", "GetModuleHandle(Ex)?[AW]?", "ExitProcess", "SetEnvironmentVariable[AW]?", "_putenv(_s)?",
"GetLastError", "MultiByteToWideChar", "WideCharToMultiByte", "HWND", "LARGE_INTEGER", "FILETIME",
"MAX_PATH", "GetActiveProcessorCount", "timeBeginPeriod",
]
CALL_RE = re.compile(r"\b(" + "|".join(CALLS) + r")\b")
@@ -103,7 +67,10 @@ def main():
a = ap.parse_args()
rt = Path(a.runtime) if a.runtime else Path(a.root) / "runtime"
bad, remaining = [], {}
for p in sorted(list(rt.glob("*.c")) + list(rt.glob("*.h")) + list(rt.glob("*.cpp"))):
files = [p for pat in ("*.c", "*.h", "*.cpp", "ui/*.cpp", "ui/*.h") for p in rt.glob(pat)]
for p in sorted(files):
if p.stem.endswith("_win"):
continue
hits = scan(p)
if p.name in ALLOWED:
remaining[p.name] = len(hits)
+2 -1
View File
@@ -274,7 +274,8 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lam
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--config', type=Path, default=ROOT / 'config.local.toml')
parser.add_argument('--hactool', type=Path, default=ROOT / 'toolchain' / 'hactool' / 'hactool.exe')
parser.add_argument('--hactool', type=Path,
default=ROOT / 'toolchain' / 'hactool' / ('hactool.exe' if os.name == 'nt' else 'hactool'))
parser.add_argument('--base', type=Path, help='base game (default: input.base of the config)')
parser.add_argument('--update', type=Path, action='append',
help='an update NSP, repeatable (default: input.updates of the config)')
+20
View File
@@ -0,0 +1,20 @@
Applied by tools/build_hactool.py to SciresM/hactool (ISC) at 1d64a83450e025622f3468c28fc4164dad2c5ef6, ours.
The x86-64 multiply-accumulate asm of hactool's bundled mbedtls 2.6.1 (MULADDC in bn_mul.h) adds into memory through
rdi and changes the flags without saying so. gcc happens to keep that code correct; clang (the Linux release's
toolchain) keeps memory values in registers across it at -O1 and above, the RSA arithmetic goes wrong, and every
signature check fails (NCA header, NPDM, ACID), which stops the merge of the update. Later mbedtls releases name the
same two clobbers.
diff --git a/mbedtls/include/mbedtls/bn_mul.h b/mbedtls/include/mbedtls/bn_mul.h
index cac3f14..c787b0a 100644
--- a/mbedtls/include/mbedtls/bn_mul.h
+++ b/mbedtls/include/mbedtls/bn_mul.h
@@ -179,7 +179,7 @@
#define MULADDC_STOP \
: "+c" (c), "+D" (d), "+S" (s) \
: "b" (b) \
- : "rax", "rdx", "r8" \
+ : "rax", "rdx", "r8", "memory", "cc" \
);
#endif /* AMD64 */
+24 -10
View File
@@ -17,9 +17,12 @@ cached in toolchain/release-dl/ and checked against the sha256 in PINS; a mismat
VERSION the repository's VERSION plus the commit, 0.1.0+973c626; the installer's release string
TotK-Setup.exe installer/launcher/launcher.c built with llvm/ (-mwindows): starts python/pythonw.exe -m installer.gui
--prove compiles generated/v121 and links build/release-check/totk.exe with nothing but the folder's python/, llvm/,
runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there. It needs a
generated/v121 and build/extracted/v121 from the usual game build (they are read, nothing is written to them).
--prove compiles the generated C and links build/release-check/totk.exe with nothing but the folder's python/, llvm/,
runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there.
Both need one game version extracted and lifted first (tools/extract.py, recomp/gen.py): generated/v121 or, without
it, generated/v143, and for --prove its build/extracted/<version> (they are read, nothing is written to them). The
runtime library reads only what both versions' layout.h share (MOD_COUNT, MOD_MAIN_BASE), so either serves.
The folder contains nothing derived from the game: check_content runs over it before the script finishes. Then the
folder's own python runs a check-only installer pass (no game files: it must get through the release, toolchain and host
@@ -329,11 +332,21 @@ def collect_licenses(out):
# ---- runtime/, recomp/, tools/, data/ ---------------------------------------------------------------------------------
def lifted_version():
"""The game version whose generated C the runtime library and --prove build against: v121, else v143."""
for version in ('v121', 'v143'):
if (MAIN_ROOT / 'generated' / version / 'layout.json').is_file():
return version
sys.exit('neither generated/v121 nor generated/v143 exists: extract a game version and lift it first '
'(tools/extract.py, recomp/gen.py; docs/building.md)')
def runtime_lib(out, llvm):
"""Build libtotkrt.a with the folder's llvm/ and copy it, the headers and the install-side sources into runtime/."""
build = MAIN_ROOT / 'build' / 'release-check' / 'rtlib'
gen = MAIN_ROOT / 'generated' / 'v121'
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm,
version = lifted_version()
gen = MAIN_ROOT / 'generated' / version
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--version', version,
'--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', build],
env=clean_env(llvm / 'bin', out / 'bin'))
rt = out / 'runtime'
@@ -465,17 +478,18 @@ def write_zip(out, path):
def prove(out):
"""Compile generated/v121 and link totk.exe with the folder's tools only, then run the probes next to it."""
"""Compile the generated C and link totk.exe with the folder's tools only, then run the probes next to it."""
out = out.resolve()
gen = MAIN_ROOT / 'generated' / 'v121'
ext = MAIN_ROOT / 'build' / 'extracted' / 'v121'
version = lifted_version()
gen = MAIN_ROOT / 'generated' / version
ext = MAIN_ROOT / 'build' / 'extracted' / version
chk = MAIN_ROOT / 'build' / 'release-check'
env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin')
t0 = time.time()
run([out / 'python' / 'python.exe', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm',
'--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime', '--runtime-lib', out / 'runtime' / 'libtotkrt.a',
'--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs',
'--profile', chk / 'profile'], env=env)
'--ninja', out / 'bin' / 'ninja.exe', '--version', version, '--gen', gen, '--out', chk, '--exefs', ext / 'exefs',
'--romfs', ext / 'romfs', '--profile', chk / 'profile'], env=env)
print(f'prove: build {time.time() - t0:.0f} s')
for dll in ('SDL3.dll', 'libopus-0.dll'):
shutil.copy2(out / 'bin' / dll, chk / dll)
+527
View File
@@ -0,0 +1,527 @@
"""Assemble the Linux release folder of docs/installer.md ("Linux release") and pack it as an AppImage.
.venv/bin/python tools/make_release_linux.py --out build/release/TotK-Setup-linux \\
[--appimage build/release/TotK-Setup-x86_64.AppImage]
.venv/bin/python tools/make_release_linux.py --out build/release/TotK-Setup-linux --prove
The same model as the Windows release (tools/make_release.py, whose download cache, unpacking and checks this reuses): the
folder carries the setup and a toolchain, never game code; the player's machine builds the game. Everything that runs on
the player's machine is built against the Steam Runtime 3 "sniper" SDK sysroot (glibc 2.31), so the AppImage and the
game it builds run on SteamOS and on any distribution at least as new. Needs network access on the first run, make,
the .NET 10 SDK and Python 3.12+ with Pillow (.venv). The folder holds:
python/ python-build-standalone 3.14 (with tkinter) + the packages of tools/requirements-installer.txt; a .pth
file puts the folder, tools/ and recomp/ on sys.path
llvm/ LLVM 23 (clang, lld, llvm-ar, the clang resource headers) and llvm/sysroot: the files of the sniper SDK
sysroot the game's compile and link read, nothing else (recorded while building the runtime library)
runtime/ libtotkrt.a (build_game.py --make-runtime-lib --sysroot), the runtime headers, romfs.c, patches.c and
ov_zstd.c (compiled per install), totk.png (the window icon), deps/ (Vulkan headers 1.4.363, VMA, zstd
built static; SDL3, Opus and libvpx come from the sysroot)
bin/ totk_xlate (+ .id), hactool, ninja, libSDL3.so.0 (the sysroot's; the game finds it beside itself)
recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, totk.ini.example, VERSION
AppRun, totk-setup.desktop, totk-setup.png the AppImage entry: python/bin/python3 -m installer.gui
--appimage packs the folder with appimagetool and the static type-2 runtime (no libfuse2 on the host). --prove compiles
the generated C and links build/release-linux-check/totk with the folder's tools only, then runs the probes beside it.
Like tools/make_release.py, both need one game version extracted and lifted first (generated/v121, else v143).
The folder contains nothing derived from the game: check_content runs over it before the script finishes, then the
folder's own python runs a check-only installer pass and the window's --selftest.
"""
import argparse
import os
import re
import shutil
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import build_game # noqa: E402
import make_release as mr # noqa: E402
ROOT = mr.ROOT
MAIN_ROOT = mr.MAIN_ROOT
WORK = mr.WORK
CHECK = MAIN_ROOT / 'build' / 'release-linux-check'
LLVM_VERSION = '23.1.4'
LLVM_MAJOR = LLVM_VERSION.split('.')[0]
SNIPER = '3.0.20261021.266025'
SNIPER_URL = 'https://repo.steampowered.com/steamrt-images-sniper/snapshots'
PINS = {
'llvm-linux': (f'https://github.com/llvm/llvm-project/releases/download/llvmorg-{LLVM_VERSION}/'
f'LLVM-{LLVM_VERSION}-Linux-X64.tar.xz',
'25d814fda50d5fb0084b42be09605d8b1595d49d5e2e03a3345d7e112ff6f9f3'),
'llvm-license': (f'https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-{LLVM_VERSION}/llvm/LICENSE.TXT',
'8d85c1057d742e597985c7d4e6320b015a9139385cff4cbae06ffc0ebe89afee'), # the binary release carries none
'sniper-sdk': (f'{SNIPER_URL}/{SNIPER}/com.valvesoftware.SteamRuntime.Sdk-amd64%2Ci386-sniper-sysroot.tar.gz',
'8f71947035f97925a311b2ed753f22efcf2f13d6f1631945b8e537f8903ce2cf'),
'python-linux': ('https://github.com/astral-sh/python-build-standalone/releases/download/20261026/'
'cpython-3.14.10%2B20261026-x86_64-unknown-linux-gnu-install_only.tar.gz',
'1b38024924eb7f4cf0cab45d7b871d1f1991354190fccd2625ac7529a43786e2'),
'ninja-linux': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-linux.zip',
'5749cbc4e668273514150a80e387a957f933c6ed3f5f11e03fb30955e2bbead6'),
'zstd-src': ('https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz',
'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3'),
'vulkan-headers-src': ('https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.4.363.tar.gz',
'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68'),
'vma-header': build_game.VMA_FETCH,
# appimagetool and the runtime are published only under the moving "continuous" tag: pinned by hash, so a new
# upload stops the build until the pins are updated on purpose
'appimagetool': ('https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage',
'793fca0ce4bcc309dc65d3f66669f58e3755643e4dadbe03932e7b436128494f'),
'appimage-runtime': ('https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-x86_64',
'617aa77314d57c8b8e4fae9c3f0b35af7b30d5529cb69e98f64f92510ed98a46'),
}
mr.PINS.update(PINS) # one cache (toolchain/release-dl) and one fetch for both releases
# What clang, lld and the runtime build need from the LLVM release (the rest of its 8 GB stays packed).
LLVM_MEMBERS = [f'bin/clang-{LLVM_MAJOR}', 'bin/clang', 'bin/clang++', 'bin/lld', 'bin/ld.lld', 'bin/llvm-ar',
'bin/llvm-ranlib', 'bin/llvm-objcopy', 'bin/llvm-strip', f'lib/clang/{LLVM_MAJOR}/include']
# The parts of the sniper SDK a C/C++ compile and link for x86_64 can read.
SNIPER_MEMBERS = ['usr/include', 'usr/lib/x86_64-linux-gnu', 'usr/lib/gcc/x86_64-linux-gnu',
'lib/x86_64-linux-gnu', 'lib64', 'usr/share/doc/libsdl3-0', 'usr/share/doc/libopus0',
'usr/share/doc/libvpx6', 'usr/share/doc/libc6', 'usr/share/doc/gcc-10-base',
'usr/share/doc/libvulkan1']
# glibc's link libraries and their scripts (lib<name>.so, .a, _nonshared.a), and the GCC runtime pieces clang links.
GLIBC_LIBS = ['c', 'm', 'mvec', 'dl', 'pthread', 'rt', 'util']
GCC_LIBS = {'libgcc.a', 'libgcc_eh.a', 'libgcc_s.so', 'libstdc++.a', 'libsupc++.a'}
# The libraries of a release link (build_game.py LINK_LIBS with --deps: Opus, libvpx and zstd static).
RELEASE_LIBS = ['-lvulkan', '-lSDL3', '-l:libopus.a', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-lm',
'-lpthread', '-ldl']
def run(cmd, **kw):
return mr.run(cmd, **kw)
def tar_extract(name, dest, members, strip=0):
"""Extract only members (path prefixes inside the archive) of a pinned tarball with the system tar, once."""
stamp = dest / '.unpacked'
if stamp.is_file():
return dest
shutil.rmtree(dest, ignore_errors=True)
dest.mkdir(parents=True)
path = mr.fetch(name)
t0 = time.time()
run(['tar', '-xf', path, '-C', dest, f'--strip-components={strip}', '--wildcards', '--no-wildcards-match-slash',
*members])
print(f'{name}: unpacked in {time.time() - t0:.0f} s')
stamp.write_text('')
return dest
# ---- llvm/ and the sysroot -------------------------------------------------------------------------------------------
def build_llvm(out):
top = f'LLVM-{LLVM_VERSION}-Linux-X64'
src = tar_extract('llvm-linux', WORK / 'llvm-linux', [f'{top}/{m}' for m in LLVM_MEMBERS], strip=1)
dst = out / 'llvm'
shutil.copytree(src, dst, symlinks=True, ignore=shutil.ignore_patterns('.unpacked'))
strip(dst / 'bin', dst / 'bin' / 'llvm-strip')
r = subprocess.run([str(dst / 'bin' / 'clang'), '--version'], capture_output=True, text=True)
if r.returncode:
sys.exit(f'{dst / "bin" / "clang"} does not start:\n{r.stderr}')
print(r.stdout.splitlines()[0])
return dst
def strip(where, tool):
"""Drop the debug symbols of the ELF files under where (the LLVM and Python releases ship them: clang 270 MB)."""
for p in sorted(where.rglob('*')):
if p.is_file() and not p.is_symlink() and p.read_bytes()[:4] == b'\x7fELF':
subprocess.run([str(tool), '--strip-unneeded', str(p)], check=True)
def full_sysroot():
"""The sniper SDK's x86_64 compile and link files in toolchain/release-work/sniper, absolute symlinks made relative
(the SDK points some at /lib and /usr/lib, which outside its container are the host's)."""
root = WORK / 'sniper'
if (root / '.relinked').is_file():
return root
tar_extract('sniper-sdk', root, SNIPER_MEMBERS)
for p in root.rglob('*'):
if p.is_symlink() and os.readlink(p).startswith('/'):
target = root / os.readlink(p).lstrip('/')
p.unlink()
p.symlink_to(os.path.relpath(target, p.parent))
(root / '.relinked').write_text('')
return root
def sysroot_cc(llvm, sysroot):
return f'{llvm / "bin" / "clang"} --target=x86_64-linux-gnu --sysroot={sysroot} -fuse-ld=lld'
# ---- runtime/deps, bin/ --------------------------------------------------------------------------------------------
def build_deps(out, llvm, sysroot):
d = out / 'runtime' / 'deps'
(d / 'include' / 'vma').mkdir(parents=True, exist_ok=True)
(d / 'lib').mkdir(parents=True, exist_ok=True)
vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', ['Vulkan-Headers-1.4.363/include',
'Vulkan-Headers-1.4.363/LICENSE.md'], strip=1)
for sub in ('vulkan', 'vk_video'):
shutil.copytree(vk / 'include' / sub, d / 'include' / sub)
shutil.copy2(mr.fetch('vma-header'), d / 'include' / 'vma' / 'vk_mem_alloc.h')
zs = tar_extract('zstd-src', WORK / 'zstd', ['zstd-1.5.7'], strip=1)
lib = zs / 'lib'
if not (lib / 'libzstd.a').is_file():
run(['make', '-C', lib, f'-j{os.cpu_count() or 4}', 'libzstd.a', f'CC={sysroot_cc(llvm, sysroot)}',
f'AR={llvm / "bin" / "llvm-ar"}', 'ZSTD_LEGACY_SUPPORT=0', 'MOREFLAGS=-O2'], stdout=subprocess.DEVNULL)
shutil.copy2(lib / 'libzstd.a', d / 'lib' / 'libzstd.a')
for name in ('zstd.h', 'zstd_errors.h'):
shutil.copy2(lib / name, d / 'include' / name)
def build_bin(out, llvm, sysroot):
b = out / 'bin'
b.mkdir(parents=True, exist_ok=True)
zip_dir = mr.unpack('ninja-linux')
shutil.copy2(zip_dir / 'ninja', b / 'ninja')
(b / 'ninja').chmod(0o755)
hac = WORK / 'hactool-linux'
run([sys.executable, ROOT / 'tools' / 'build_hactool.py', '--src', WORK / 'hactool-src', '--out', hac,
'--cc', sysroot_cc(llvm, sysroot)])
shutil.copy2(hac / 'hactool', b / 'hactool')
xl = WORK / 'xlate-linux'
run([sys.executable, ROOT / 'tools' / 'build_xlate.py', '--out', xl, '--llvm', llvm, '--sysroot', sysroot])
shutil.copy2(xl / 'totk_xlate', b / 'totk_xlate')
shutil.copy2(xl / 'totk_xlate.id', b / 'totk_xlate.id')
sdl = (sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu' / 'libSDL3.so.0').resolve()
shutil.copy2(sdl, b / 'libSDL3.so.0')
# ---- runtime/ ----------------------------------------------------------------------------------------------------------
def runtime_lib(out, llvm, sysroot):
build = CHECK / 'rtlib'
shutil.rmtree(build, ignore_errors=True)
version = mr.lifted_version()
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--sysroot', sysroot,
'--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja', '--version', version,
'--gen', MAIN_ROOT / 'generated' / version, '--out', build])
rt = out / 'runtime'
shutil.copy2(build / 'libtotkrt.a', rt / 'libtotkrt.a')
for p in sorted((ROOT / 'runtime').rglob('*')):
if p.suffix in mr.RUNTIME_SUFFIXES and p.is_file():
(rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime'))
for name in build_game.INSTALL_SIDE:
shutil.copy2(ROOT / 'runtime' / name, rt / name)
from PIL import Image # Tk loads no .ico off Windows: the window and the AppImage take a PNG
with Image.open(ROOT / 'runtime' / 'totk.ico') as ico:
ico.size = max(ico.ico.sizes())
ico.convert('RGBA').save(rt / 'totk.png')
return build
def used_sysroot_files(out, llvm, sysroot, rtlib_build):
"""The sysroot files a game build reads: the headers of the runtime's compile (its .d files), of the install-side
sources and of hostcheck, and every library and object of a link (lld --trace) of the runtime library with the
flags and libraries build_game.py uses, its references to the generated code left unresolved."""
used = set()
for dep in rtlib_build.rglob('*.d'):
used.update(re.findall(r'\S+', dep.read_text(encoding='utf-8', errors='replace').replace('\\\n', ' ')))
probe = CHECK / 'sysroot-probe'
shutil.rmtree(probe, ignore_errors=True)
probe.mkdir(parents=True)
cc = [llvm / 'bin' / 'clang', '--target=x86_64-linux-gnu', f'--sysroot={sysroot}', '-I', out / 'runtime',
'-I', out / 'runtime' / 'deps' / 'include', '-I', CHECK / 'rtlib']
for i, src in enumerate([*(out / 'runtime' / n for n in sorted(build_game.INSTALL_SIDE)),
ROOT / 'installer' / 'hostcheck.c']):
# romfs.c needs the per-install romfs_config.h: whatever it reads up to there still lands in the .d file
subprocess.run([str(c) for c in [*cc, '-fsyntax-only', '-MD', '-MF', probe / f'{i}.d', src]],
capture_output=True)
if (probe / f'{i}.d').is_file():
used.update(re.findall(r'\S+', (probe / f'{i}.d').read_text().replace('\\\n', ' ')))
trace = subprocess.run([str(c) for c in [
llvm / 'bin' / 'clang', '--target=x86_64-linux-gnu', f'--sysroot={sysroot}', *build_game.LINK_FLAGS[:3],
'-Wl,--trace', '-Wl,--unresolved-symbols=ignore-all', '-Wl,--whole-archive', out / 'runtime' / 'libtotkrt.a',
'-Wl,--no-whole-archive', '-L', out / 'runtime' / 'deps' / 'lib',
*RELEASE_LIBS, '-o', probe / 'trace.out']],
capture_output=True, text=True)
if trace.returncode:
sys.exit(f'sysroot trace link failed:\n{trace.stderr[-3000:]}')
used.update(line.split('(')[0] for line in trace.stdout.splitlines())
root = str(sysroot.resolve())
files = set()
for p in used:
path = Path(p)
if not path.is_absolute():
continue
for q in (path, path.resolve()): # the name read and, through symlinks, the file
if str(q).startswith(root + '/') and q.exists():
files.add(q)
# The C library and kernel headers whole: which of them a compile reads depends on its flags (at -O1 glibc's inline
# variants pull in bits/stdio.h and others), and the setup compiles the generated code, hostcheck and the
# install-side sources with flags of its own
inc = sysroot / 'usr' / 'include'
arch = inc / 'x86_64-linux-gnu'
files.update(p for p in [*inc.glob('*.h'), *arch.glob('*.h')] if p.is_file())
for d in (arch / 'bits', arch / 'sys', arch / 'gnu', arch / 'asm', inc / 'linux', inc / 'asm-generic'):
files.update(p for p in d.rglob('*') if p.is_file() or p.is_symlink())
# The C runtime whole, for PIE links (hostcheck) as well as the game's: lld's trace names the libraries a link
# script loads but not the script (libc.so, libm.so, libgcc_s.so), and clang's GCC detection needs the crt objects
lib = sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu'
for name in GLIBC_LIBS:
files.update(p for p in lib.glob(f'lib{name}[._]*') if p.suffix in ('.so', '.a') or '.so' in p.name)
files.update(p for p in (sysroot / 'lib' / 'x86_64-linux-gnu').glob(f'lib{name}.so*'))
files.update(lib.glob('*crt*.o'))
files.update((sysroot / 'lib64').glob('ld-linux*'))
files.update((sysroot / 'lib' / 'x86_64-linux-gnu').glob('ld-linux*'))
files.update((sysroot / 'lib' / 'x86_64-linux-gnu').glob('libgcc_s.so*'))
gcc = next((sysroot / 'usr' / 'lib' / 'gcc' / 'x86_64-linux-gnu').iterdir())
files.update(p for p in gcc.iterdir() if p.name.startswith('crt') or p.name in GCC_LIBS)
return files
def link_chain(path):
"""path and every link on the way to its file: libvulkan.so -> libvulkan.so.1 -> libvulkan.so.1.4.309."""
hops = [path]
while hops[-1].is_symlink() and len(hops) < 16:
hops.append(hops[-1].parent / os.readlink(hops[-1]))
return hops
def build_sysroot(out, sysroot, files):
dst = out / 'llvm' / 'sysroot'
files = {Path(os.path.normpath(h)) for f in files for h in link_chain(f)}
for f in sorted(files):
rel = f.relative_to(sysroot)
(dst / rel).parent.mkdir(parents=True, exist_ok=True)
if f.is_symlink():
if not (dst / rel).is_symlink():
(dst / rel).symlink_to(os.readlink(f))
elif not (dst / rel).exists():
shutil.copy2(f, dst / rel)
# a symlink copied above may name a file the build reads only through it
for link in [p for p in dst.rglob('*') if p.is_symlink()]:
target = (sysroot / link.relative_to(dst)).resolve()
mirrored = dst / target.relative_to(sysroot.resolve())
if not mirrored.exists():
mirrored.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(target, mirrored)
print(f'sysroot: {sum(1 for p in dst.rglob("*") if p.is_file())} files, {mr.mib(mr.dir_size(dst))}')
# ---- python/ ---------------------------------------------------------------------------------------------------------
def build_python(out):
src = mr.unpack('python-linux') / 'python'
py = out / 'python'
shutil.copytree(src, py, symlinks=True,
ignore=shutil.ignore_patterns('test', 'idlelib', 'turtledemo', '__pycache__', '*.a'))
exe = py / 'bin' / 'python3'
env = dict(os.environ, PIP_DISABLE_PIP_VERSION_CHECK='1')
run([exe, '-m', 'pip', 'install', '--no-compile', '--only-binary', ':all:', '-r',
ROOT / 'tools' / 'requirements-installer.txt'], env=env, stdout=subprocess.DEVNULL)
run([exe, '-m', 'pip', 'uninstall', '-y', 'pip'], stdout=subprocess.DEVNULL) # the setup installs nothing
strip(py, out / 'llvm' / 'bin' / 'llvm-strip')
site = next(py.glob('lib/python3.*/site-packages'))
# The release folder, tools/ and recomp/ on sys.path, as the Windows python's ._pth does: `python -m installer` and
# the tools importing their siblings work from any directory
(site / 'totk-release.pth').write_text('../../../..\n../../../../tools\n../../../../recomp\n', encoding='utf-8')
check = subprocess.run([str(exe), '-c', 'import tkinter, numpy, lz4.block, cryptography, cffi, pycparser; '
'print("python ok: Tcl", tkinter.Tcl().eval("info patchlevel"), "numpy", numpy.__version__)'],
cwd='/', capture_output=True, text=True)
if check.returncode:
sys.exit('bundled python check failed:\n' + check.stderr)
print(check.stdout.strip())
# ---- licences, AppImage entry ------------------------------------------------------------------------------------------
def collect_licenses(out, sysroot):
lic = out / 'licenses'
lic.mkdir(parents=True, exist_ok=True)
doc = sysroot / 'usr' / 'share' / 'doc'
pairs = [
(mr.fetch('llvm-license'), 'llvm-LICENSE.txt'),
(doc / 'libsdl3-0' / 'copyright', 'SDL3-copyright.txt'),
(doc / 'libopus0' / 'copyright', 'opus-copyright.txt'),
(doc / 'libvpx6' / 'copyright', 'libvpx-copyright.txt'),
(doc / 'libc6' / 'copyright', 'glibc-copyright.txt'),
(doc / 'gcc-10-base' / 'copyright', 'libstdc++-copyright.txt'),
(doc / 'libvulkan1' / 'copyright', 'vulkan-loader-copyright.txt'),
(WORK / 'vulkan-headers' / 'LICENSE.md', 'vulkan-headers-LICENSE.txt'),
(WORK / 'zstd' / 'LICENSE', 'zstd-LICENSE.txt'),
(out / 'python' / 'lib' / f'python{sys_version(out)}' / 'LICENSE.txt', 'python-LICENSE.txt'),
(ROOT / 'runtime' / 'shaders' / 'ffx' / 'LICENSE.txt', 'FidelityFX-FSR-LICENSE.txt'),
(WORK / 'hactool-linux' / 'LICENSE', 'hactool-LICENSE.txt'),
(MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'),
]
for src, name in pairs:
shutil.copy2(src, lic / name)
import tarfile # Debian's libvpx copyright carries no patent grant: the WebM PATENTS text from the pinned source tag
with tarfile.open(mr.fetch('libvpx-src')) as t:
(lic / 'libvpx-PATENTS.txt').write_bytes(t.extractfile('libvpx-1.17.0/PATENTS').read())
head = (out / 'runtime' / 'deps' / 'include' / 'vma' / 'vk_mem_alloc.h').read_text(encoding='utf-8').split('*/')[0]
(lic / 'vulkan-memory-allocator-LICENSE.txt').write_text(head.lstrip('/*') + '\n', encoding='utf-8')
def sys_version(out):
return next((out / 'python' / 'lib').glob('python3.*')).name.removeprefix('python')
APPRUN = '''#!/bin/sh
# The AppImage entry: the setup window, run by the bundled Python from the read-only mount (no bytecode written there).
HERE="$(dirname "$(readlink -f "$0")")"
export PYTHONDONTWRITEBYTECODE=1
exec "$HERE/python/bin/python3" -m installer.gui "$@"
'''
DESKTOP = '''[Desktop Entry]
Type=Application
Name=TotK Recomp Setup
Comment=Builds Tears of the Kingdom for this PC from your own game files
Exec=AppRun
Icon=totk-setup
Categories=Game;
Terminal=false
'''
def build_entry(out):
(out / 'AppRun').write_text(APPRUN, encoding='utf-8')
(out / 'AppRun').chmod(0o755)
(out / 'totk-setup.desktop').write_text(DESKTOP, encoding='utf-8')
shutil.copy2(out / 'runtime' / 'totk.png', out / 'totk-setup.png')
(out / '.DirIcon').symlink_to('totk-setup.png')
def write_appimage(out, path):
tool = mr.fetch('appimagetool')
tool.chmod(0o755)
runtime = mr.fetch('appimage-runtime')
path.parent.mkdir(parents=True, exist_ok=True)
env = dict(os.environ, ARCH='x86_64', APPIMAGE_EXTRACT_AND_RUN='1')
run([tool, '--no-appstream', '--runtime-file', runtime, out, path], env=env)
print(f'{path}: {mr.mib(path.stat().st_size)}')
# ---- checks -------------------------------------------------------------------------------------------------------------
# The newest glibc any program or library in the folder may need: the official LLVM binaries and the cryptography
# wheel are built against 2.34 (Ubuntu 22.04, Debian 12, SteamOS 3 have it); the game itself links the sysroot's 2.31.
GLIBC_FLOOR = (2, 34)
def check_glibc(out):
"""Every ELF file in the folder (llvm/sysroot aside: glibc itself, read by the link) needs glibc GLIBC_FLOOR or
older; prints which files set the floor."""
newest, by = (0, 0), []
for p in sorted(out.rglob('*')):
if not p.is_file() or p.is_symlink() or out / 'llvm' / 'sysroot' in p.parents:
continue
with open(p, 'rb') as f:
if f.read(4) != b'\x7fELF':
continue
text = subprocess.run(['readelf', '-V', str(p)], capture_output=True, text=True).stdout
need = max((tuple(map(int, v.split('.'))) for v in re.findall(r'GLIBC_(\d+\.\d+(?:\.\d+)?)', text)),
default=(0, 0))
if need > newest:
newest, by = need, []
if need == newest:
by.append(str(p.relative_to(out)))
floor = '.'.join(map(str, GLIBC_FLOOR))
if newest > GLIBC_FLOOR:
sys.exit(f'glibc check: {", ".join(by[:8])} need glibc {".".join(map(str, newest))}, above {floor}')
print(f'glibc check: the folder needs glibc {".".join(map(str, newest))} (floor {floor}), set by '
f'{", ".join(by[:6])}{" and others" if len(by) > 6 else ""}')
def check_bundle(out):
"""The folder's own python runs the installer without game files and the window's selftest."""
import json
chk = CHECK / 'bundle'
shutil.rmtree(chk, ignore_errors=True)
py = out / 'python' / 'bin' / 'python3'
env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1')
r = subprocess.run([str(py), '-m', 'installer', '--lang', 'en-US', '--dest', str(chk / 'install'), '--temp',
str(chk / 'temp'), '--root', str(out)], cwd='/', env=env, capture_output=True, text=True,
timeout=600)
events = [json.loads(line) for line in r.stdout.splitlines() if line.startswith('{')]
errors = [e['id'] for e in events if e['type'] == 'error']
started = any(e['type'] == 'start' and e.get('release') == (out / 'VERSION').read_text().strip() for e in events)
steps = [e['phase'] for e in events if e['type'] == 'progress']
if errors != ['check.inputs_needed'] or not started or 'inputs' not in steps:
sys.exit(f'bundle check: exit {r.returncode}, errors {errors}, release read {started}\n{r.stdout[-2000:]}\n'
f'{r.stderr[-2000:]}')
print(f'bundle check: the installer ran the host check from the folder and stopped at the game files '
f'(exit {r.returncode})')
r = subprocess.run([str(py), '-m', 'installer.gui', '--selftest', '--root', str(out)], cwd='/', env=env,
capture_output=True, text=True, timeout=300)
if r.returncode:
sys.exit(f'bundle check: window selftest exit {r.returncode}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
print('bundle check: the window built every screen in both languages and closed')
def prove(out):
"""Compile the generated C and link totk with the folder's tools only, then run the probes beside it."""
out = out.resolve()
version = mr.lifted_version()
gen = MAIN_ROOT / 'generated' / version
ext = MAIN_ROOT / 'build' / 'extracted' / version
chk = CHECK / 'game'
env = dict(os.environ, PATH=os.pathsep.join([str(out / 'llvm' / 'bin'), str(out / 'bin'), '/usr/bin', '/bin']))
t0 = time.time()
run([out / 'python' / 'bin' / 'python3', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm',
'--sysroot', out / 'llvm' / 'sysroot', '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime',
'--runtime-lib', out / 'runtime' / 'libtotkrt.a', '--ninja', out / 'bin' / 'ninja', '--version', version,
'--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', '--profile', chk / 'profile'],
env=env)
print(f'prove: build {time.time() - t0:.0f} s')
shutil.copy2(out / 'bin' / 'libSDL3.so.0', chk / 'libSDL3.so.0')
needed = subprocess.run(['readelf', '-d', str(chk / 'totk')], capture_output=True, text=True).stdout
print('prove: needs', ', '.join(re.findall(r'\[(lib[^\]]+)\]', needed)))
glibc = sorted(set(re.findall(r'GLIBC_(\d+\.\d+)', subprocess.run(['readelf', '-V', str(chk / 'totk')],
capture_output=True, text=True).stdout)),
key=lambda v: tuple(map(int, v.split('.'))))
print(f'prove: newest glibc symbol version {glibc[-1] if glibc else "none"}')
failed = []
for probe in ['--selftest', *mr.PROBES]:
r = subprocess.run([str(chk / 'totk'), probe], cwd=chk, capture_output=True, text=True, timeout=900)
tail = (r.stdout.strip().splitlines() or [''])[-1]
print(f'{probe}: exit {r.returncode}: {tail}')
if r.returncode:
failed.append(probe)
if failed:
sys.exit(f'probes failed: {failed}')
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
ap.add_argument('--appimage', type=Path, help='also pack the folder as this AppImage')
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
args = ap.parse_args()
out = args.out.resolve()
if args.prove:
prove(out)
return
if out.exists():
shutil.rmtree(out)
out.mkdir(parents=True)
llvm = build_llvm(out)
sysroot = full_sysroot()
build_deps(out, llvm, sysroot)
build_bin(out, llvm, sysroot)
rtlib = runtime_lib(out, llvm, sysroot)
build_sysroot(out, sysroot, used_sysroot_files(out, llvm, sysroot, rtlib))
build_python(out)
mr.build_sources(out)
collect_licenses(out, sysroot)
build_entry(out)
mr.check_folder(out)
check_glibc(out)
check_bundle(out)
mr.sizes(out)
if args.appimage:
write_appimage(out, args.appimage.resolve())
if __name__ == '__main__':
main()
+6 -5
View File
@@ -11,10 +11,11 @@ captured at once (upstream checked weapon, shield, material and ability wheel at
The game-over flow fix of the same upstream script is 1.0.0 only (1.2.1 and 1.4.3 run DmF_SY_GameOver.110) and is
left out.
--aspect <w>x<h> (a game resolution wider than 16:9, such as 3440x1440) also lays the UI out for that shape
(tools/ultrawide_layout.py): menus stay 16:9 in the middle, full-screen backgrounds cover the width and the HUD moves
to the screen's corners. totk.exe loads that overlay from romfs-patches-<w>x<h> when the game resolution is that size
(runtime/romfs.c overlay_init); the camera's side of it is the patch set aspect (recomp/patches-v143.txt).
--aspect <w>x<h> (a game resolution of another shape than 16:9, such as 3440x1440 or 1280x800) also lays the UI out
for that shape (tools/ultrawide_layout.py): menus keep their 16:9 proportions in the middle, full-screen backgrounds
cover the screen and the HUD moves to the screen's corners (wider) or its top and bottom edges (narrower). totk.exe
loads that overlay from romfs-patches-<w>x<h> when the game resolution is that size (runtime/romfs.c overlay_init);
the camera's side of it is the patch set aspect (recomp/patches-v143.txt).
--prompts xbox | xbox-label: the same overlay with Xbox button icons in the game's prompts, for prompts = xbox
(runtime/romfs.c overlay_init loads romfs-patches-xbox or romfs-patches-xbox-label in place of romfs-patches). Drawn
@@ -56,7 +57,7 @@ ap.add_argument('--version', choices=('v121', 'v143'), default='v121')
ap.add_argument('--romfs', type=Path, help='default build/extracted/<version>/romfs')
ap.add_argument('--out', type=Path,
help='default build/<version>/romfs-patches (romfs-patches-<style> with --prompts)')
ap.add_argument('--aspect', help='<w>x<h> of a game resolution wider than 16:9: also lay the UI out for it')
ap.add_argument('--aspect', help='<w>x<h> of a game resolution of another shape than 16:9: also lay the UI out for it')
ap.add_argument('--prompts', choices=('xbox', 'xbox-label'),
help='Xbox button icons in the prompts: xbox for pad_layout = nintendo (A shown as B, X as Y), '
'xbox-label for pad_layout = xbox or auto (the face buttons keep their letters)')
+8 -6
View File
@@ -20,7 +20,9 @@ from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
DLLS = ('SDL3.dll', 'libopus-0.dll', 'libiconv-2.dll')
EXE = '.exe' if os.name == 'nt' else ''
# A Linux source build links the system's SDL3, Opus and libvpx: the other machine needs them installed
DLLS = ('SDL3.dll', 'libopus-0.dll', 'libiconv-2.dll') if os.name == 'nt' else ()
# Prepared state next to the exe: translator answers, programs, the prepared list, pipeline cache and its keys, and the
# variant lists `totk.exe --prepare-shaders` reads (the driver keys its library cache by exe path, so each copy runs it).
STATE = ('romfs-index.bin', 'pipeline-cache.bin', 'pipeline-keys.bin', 'shader-answers.bin', 'shader-programs.bin',
@@ -51,7 +53,7 @@ def main():
ap.add_argument('--version', default='v121', help='game version (v121 or v143): the defaults below follow it')
ap.add_argument('--build', type=Path, help='build dir with the prepared state (default build/<version>)')
ap.add_argument('--exe', type=Path, help='totk.exe to package (default: <build>/totk.exe); its DLLs are taken from --build')
ap.add_argument('--xlate', type=Path, default=ROOT / 'build' / 'xlate' / 'totk_xlate.exe')
ap.add_argument('--xlate', type=Path, default=ROOT / 'build' / 'xlate' / f'totk_xlate{EXE}')
ap.add_argument('--extracted', type=Path, help='default build/extracted/<version>')
ap.add_argument('--out', type=Path, help='default build/<version>-portable')
ap.add_argument('--with-romfs', action='store_true', help='also copy romfs.bin (about 16 GB)')
@@ -62,7 +64,7 @@ def main():
a.out = a.out or ROOT / 'build' / f'{a.version}-portable'
build, out = a.build.resolve(), a.out.resolve()
exe = (a.exe or build / 'totk.exe').resolve()
exe = (a.exe or build / f'totk{EXE}').resolve()
fonts = a.extracted.resolve().parent / 'firmware' / 'shared-fonts'
exefs = a.extracted.resolve() / 'exefs'
romfs = a.extracted.resolve() / 'romfs' / 'romfs.bin'
@@ -78,10 +80,10 @@ def main():
print('package_build: warning: the folder holds game code; keep it out of the repository')
out.mkdir(parents=True, exist_ok=True)
copy(exe, out / 'totk.exe')
copy(exe, out / f'totk{EXE}')
for d in DLLS:
copy(build / d, out / d)
copy(a.xlate, out / 'totk_xlate.exe')
copy(a.xlate, out / f'totk_xlate{EXE}')
idf = a.xlate.with_suffix('.id')
if idf.exists():
copy(idf, out / 'totk_xlate.id')
@@ -104,7 +106,7 @@ def main():
print('package_build: copying romfs.bin (16 GB) ...')
shutil.copyfile(romfs, out / 'romfs' / 'romfs.bin')
print(f'package_build: {out}: {n} files in folders, {size_of(out) / 2**30:.2f} GiB'
+ ('' if a.with_romfs else '; no RomFS: put romfs.bin into romfs\\ or set TOTK_ROMFS'))
+ ('' if a.with_romfs else f'; no RomFS: put romfs.bin into romfs{os.sep} or set TOTK_ROMFS'))
if __name__ == '__main__':
+2 -1
View File
@@ -28,6 +28,7 @@ bundles it; it stays out of the repository), else from refs/ through git with th
import argparse
import hashlib
import json
import os
import struct
import subprocess
import sys
@@ -183,7 +184,7 @@ def main():
f'seed written in {time.monotonic() - t0:.1f} s', flush=True)
if a.no_run:
return 0
exe = out / 'totk.exe'
exe = out / ('totk.exe' if os.name == 'nt' else 'totk')
r = subprocess.run([str(exe), '--prepare-shaders'], cwd=str(out))
print(f'totk.exe --prepare-shaders: rc={r.returncode}; total {time.monotonic() - t0:.1f} s')
return r.returncode
+4 -2
View File
@@ -21,7 +21,9 @@ import compression.zstd as zstd
import concurrent.futures
import hashlib
import json
import os
import re
import shutil
import struct
import subprocess
import sys
@@ -32,8 +34,8 @@ from pathlib import Path
import lz4.block
ROOT = Path(__file__).resolve().parent.parent
XLATE = ROOT / 'build' / 'xlate' / 'totk_xlate.exe'
SPIRV_VAL = Path(r'C:\msys64\ucrt64\bin\spirv-val.exe')
XLATE = ROOT / 'build' / 'xlate' / ('totk_xlate.exe' if os.name == 'nt' else 'totk_xlate')
SPIRV_VAL = Path(r'C:\msys64\ucrt64\bin\spirv-val.exe') if os.name == 'nt' else Path(shutil.which('spirv-val') or '/usr/bin/spirv-val')
ZSTD = b'\x28\xb5\x2f\xfd'
# NVN program control section magic, u32 0x98761234 (d2r-mobile dm/gfx/tools/shader_map.py)
CONTROL_MAGIC = b'\x34\x12\x76\x98'
+127 -14
View File
@@ -1,15 +1,16 @@
# Adapted from MacNativeGames games/totk/scripts/ultrawide_layout.py @ 933b6a0 (MIT, Copyright (c) 2026
# MacNativeGames contributors). Used by tools/make_romfs_patches.py --aspect.
"""The game's UI layouts for a canvas wider than 16:9 (output_res = 3440x1440 in totk.ini), edited in place in the
decompressed UI/LayoutArchive/Common.Product.*.blarc (tools/make_romfs_patches.py --aspect).
"""The game's UI layouts for a canvas of another shape than 16:9 (output_res = 3440x1440 or 1280x800 in totk.ini),
edited in place in the decompressed UI/LayoutArchive/Common.Product.*.blarc (tools/make_romfs_patches.py --aspect).
The game lays its UI out on a 1920x1080 grid and maps that grid onto the canvas, so on a 21:9 canvas every layout comes
out 34% too wide. Here each layout's root pane is narrowed to 16:9 (scale x = 16/9 over the aspect), backgrounds that
must cover the whole screen are widened back, and the HUD's parts move out to the screen's corners. The pane list and
the distances are the ones the community TOTK-Ultrawide mod for emulators found for 1.0.0 .. 1.2.1 (GPL; only those
values are taken, no code; THIRD_PARTY_NOTICES.md). This is an independent implementation, checked against 1.4.3's
Common.Product.140: every pane is found by name and every pane field and animation key holds the value the edit
expects before it is written.
out 34% too wide and on a 16:10 one 11% too tall. Here each layout's root pane is scaled back to 16:9 on the stretched
axis (scale x = 16/9 over the aspect when wider, scale y = the aspect over 16/9 when narrower), backgrounds that must
cover the whole screen are stretched back, and the HUD's parts move out to the screen's corners (wider) or its top and
bottom edges (narrower). The pane list and the distances are the ones the community TOTK-Ultrawide mod for emulators
found for 1.0.0 .. 1.2.1 (GPL; only those values are taken, no code; THIRD_PARTY_NOTICES.md). This is an independent
implementation, checked against 1.4.3's Common.Product.140: every pane is found by name and every pane field and
animation key holds the value the edit expects before it is written.
"""
import struct
@@ -82,6 +83,110 @@ def edits(k, hud_corner):
('PlayerStatus_00', 'Pa_CompanionLife_00', 'x', -(840 + w))]
return out
SYSTEM_MENUS = ('SystemLoadList_00', 'SystemOption_00', 'SystemSaveLoad_00')
def tall_edits(k):
"""(layout, pane, field, value) for a canvas narrower than 16:9: k = the aspect over 16/9 (the root's scale y),
h = how far a 1080-high layout's edge moves out in its own units (540 * (1 / k - 1))."""
h, tall = 540 * (1 / k - 1), 1 / k
out = [('AppAlbum_00', 'N_FadeInOut_00', 'scale_y', k), ('AppAlbum_00', 'N_Open_01', 'scale_y', tall),
*[('AppMenuOverlay_00', p, 'scale_y', k) for p in ('N_FadeInOut_00', 'N_FadeInOut_01', 'N_Icon_00',
'N_TitleOne_01')],
('CameraPointer_00', 'N_Type_01', 'scale_y', k), ('CameraSystemWindow_00', 'N_FirstOut_00', 'scale_y', k),
*[('LoadingFade_00', p, 'scale_y', k) for p in ('Pa_IconPlayer_00', 'Pa_LargeDungeonMap_00',
'Pa_LargeDungeonMap_01', 'Pa_Map_00', 'Pa_Map_01')],
*[('AppMap_00', f'N_BlurInOut_0{i}', 'scale_y', tall) for i in range(4)],
*[(layout, 'P_BG_00', 'scale_y', tall) for layout in ('AppOpenDemoWindow_00', 'GameOver_00', 'LoadingFade_00',
'PaLargeDungeonMap_00', 'PaRecipeCardList_00')],
('LoadingFade_00', 'N_Base_00', 'scale_y', tall),
*[(layout, 'N_BG_00', 'scale_y', tall) for layout in ('AutoBuilder_00', 'OwnedHorseList_00', 'ShopMenu_00',
'StaffRoll_00')],
*[('CameraPointer_00', p, 'scale_y', tall) for p in ('N_InOutScope_00', 'Pa_CameraFocus_00', 'S_Scissor_00',
'W_Shadow_01')],
('CameraSystemWindow_00', 'N_InOut_00', 'scale_y', tall), ('MessageLog_01', 'P_Capture_00', 'scale_y', tall),
*[(layout, 'N_SystemBG_00', 'scale_y', tall) for layout in ('SystemActionGuide_00',) + SYSTEM_MENUS]]
# The HUD at the screen's top and bottom edges instead of the middle 16:9
out += [(name, 'RootPane', 'y', 540 * (1 - k)) for name in (
'AppAlbum_00', 'AppPictureBook_00', 'ChallengeLog_00', 'Challenge_00', 'InstantTipsWithPict_00', 'InstantTips_00',
'MiniGameTimer_00', 'NumDisplaySecond_00', 'NumDisplay_00', 'PickUpWin_00', 'PlayerStatus_00', 'Pouch_00',
'ShopMenu_00', 'SystemActionGuide_00') + SYSTEM_MENUS]
out += [(name, 'RootPane', 'y', -540 * (1 - k)) for name in (
'CtrlGuide_00', 'DoCommand_00', 'EnvironmentInfo_00', 'HorseGauge_00', 'MessageDialog_00', 'MessageLog_01',
'PauseMenuOverlay_00', 'SaveLoadIcon_00', 'TreasureFullShortCut_00')]
out += [('AppAlbum_00', 'N_PicLoad_01', 'y', -540 * (1 - k)), ('AppMap_00', 'N_AllMapTime_00', 'y', -h),
('AppMenuOverlay_00', 'A_Align_00', 'y', -(505 + h)), ('AppMenuOverlay_00', 'Pa_GuideLeft_00', 'y', -(505 + h)),
('AppMenuOverlay_00', 'N_TitleTab_00', 'y', 491 + h),
('LoadingFade_00', 'N_Logo_00', 'y', -h), ('LoadingFade_00', 'N_Sage_00', 'y', -477 - 2 * h),
('LoadingFade_00', 'N_Status_00', 'y', h), ('LoadingFade_00', 'N_Tips_00', 'y', -(360 + h)),
('Location_00', 'N_Large_00', 'y', 270 + h), ('Location_00', 'N_Small_00', 'y', -(395 + h)),
('MessageLog_01', 'P_Capture_00', 'y', h),
('PaExtraStaminaGauge_00', 'N_BasePattern_00', 'y', h), ('PaExtraStaminaGauge_00', 'N_GaugeMainPattern_00', 'y', h),
('Pouch_00', 'N_Info_00', 'y', -h), ('Pouch_00', 'Pa_ExtraStamina_00', 'y', -h),
('ShopMenu_00', 'A_AlignGuide_00', 'y', -(477 + h)),
*[(layout, 'N_SystemBG_00', 'y', -h) for layout in ('SystemActionGuide_00',) + SYSTEM_MENUS],
('SystemLoadList_00', 'Pa_Guide_00', 'y', -(474 + h)), ('SystemOption_00', 'A_Guide_00', 'y', -(474 + h))]
return out
# BFLAN animation targets: FLPA (pane SRT) translate y, FLTS (texture SRT) scale y
FLPA_Y, FLTS_SY = 1, 4
def anim_keys(blarc, lo, pane, tag, target):
"""The absolute offsets of each key (frame, value, slope: 12 bytes) of the hermite curves of one pane, animation
tag (b'FLPA', b'FLTS') and target in the BFLAN at blarc[lo:], one list per curve. Found by walking the pai1 section,
so they hold in any version whose animation has that curve."""
if blarc[lo:lo + 4] != b'FLAN': raise SystemExit(f'{lo:#x}: not a BFLAN')
pos, out = lo + struct.unpack_from('<H', blarc, lo + 6)[0], []
for _ in range(struct.unpack_from('<H', blarc, lo + 0x10)[0]):
magic, size = bytes(blarc[pos:pos + 4]), struct.unpack_from('<I', blarc, pos + 4)[0]
if magic == b'pai1':
table = pos + struct.unpack_from('<I', blarc, pos + 16)[0]
for e in range(struct.unpack_from('<H', blarc, pos + 14)[0]):
eo = pos + struct.unpack_from('<I', blarc, table + 4 * e)[0]
if bytes(blarc[eo:eo + 28]).split(b'\0')[0].decode() != pane: continue
for t in range(blarc[eo + 28]):
to = eo + struct.unpack_from('<I', blarc, eo + 32 + 4 * t)[0]
if bytes(blarc[to:to + 4]) != tag: continue
for g in range(blarc[to + 4]):
go = to + struct.unpack_from('<I', blarc, to + 8 + 4 * g)[0]
if blarc[go + 1] != target: continue
if blarc[go + 2] != 2: raise SystemExit(f'{pane}: a {tag.decode()} curve that is not hermite')
keys = go + struct.unpack_from('<I', blarc, go + 8)[0]
out.append([keys + 12 * i for i in range(struct.unpack_from('<H', blarc, go + 4)[0])])
pos += size
return out
def tall_anim(blarc, members, k, changed):
"""The animation keys that hold positions or zooms on the screen's height, for a canvas narrower than 16:9."""
h = 540 * (1 / k - 1)
get = lambda at: struct.unpack_from('<f', blarc, at)[0]
put = lambda at, v: struct.pack_into('<f', blarc, at, v)
def anim(name):
member = f'anim/{name}.bflan'
if member not in members: raise SystemExit(f'{name}: no animation {member} in the archive')
changed.add(member)
return members[member][0]
# The minimap's four cut panes slide in on straight lines from 0 at frame 0 to y -360, the bottom of a 16:9 layout
# (hermite keys: the first one's slope is the end point); the end point moves with the screen's edge.
lo = anim('AppMap_00_MiniMap')
for pane in ('N_Cut_00', 'N_Cut_01', 'N_Cut_02', 'N_Cut_03'):
curves = anim_keys(blarc, lo, pane, b'FLPA', FLPA_Y)
if len(curves) != 1 or len(curves[0]) != 2: raise SystemExit(f'AppMap_00_MiniMap {pane}: {len(curves)} y curves')
k0, k1 = curves[0]
if not (get(k0) == 0 and get(k0 + 4) == 0 and abs(get(k0 + 8) + 360) < 1e-3 and get(k1) == 1 and
abs(get(k1 + 4) + 360) < 1e-3 and get(k1 + 8) == 0):
raise SystemExit(f'AppMap_00_MiniMap {pane}: not a line from 0 to y -360')
put(k0 + 8, -(360 + h)), put(k1 + 4, -(360 + h))
# The large place name of a discovery holds its height (285, then 260) in flat keys; they move to the top edge.
curves = anim_keys(blarc, anim('Location_00_IsDiscovery'), 'N_Large_00', b'FLPA', FLPA_Y)
values = [get(key + 4) for keys in curves for key in keys]
if len(curves) != 1 or values != [285, 285, 260]: raise SystemExit(f'Location_00_IsDiscovery N_Large_00: keys {values}')
for key in curves[0]: put(key + 4, get(key + 4) + h)
# The dragon's tears map icon zooms its picture to 1.08 and 1.28 in flat keys; on the stretched axis by 1 / k more.
curves = anim_keys(blarc, anim('PaMapIconDragonTears_00_Zoom'), 'P_Geoglyph_00', b'FLTS', FLTS_SY)
values = [round(get(key + 4), 4) for keys in curves for key in keys]
if len(curves) != 1 or values != [1.08, 1.28]: raise SystemExit(f'PaMapIconDragonTears_00_Zoom P_Geoglyph_00: keys {values}')
for key in curves[0]: put(key + 4, get(key + 4) / k)
def anim_edits(k, hud_corner):
"""(animation, byte offset, value the game ships, new value): floats inside BFLAN key tables."""
w = 960 * (1 / k - 1)
@@ -109,9 +214,13 @@ def find_pane(found, layout, name):
return found[name]
def apply(blarc, members, width, height, hud_corner=True):
"""Edits blarc (bytearray) in place for a width x height canvas. Returns the members it changed."""
k = (16 / 9) / (width / height)
assert k < 1, 'only for canvases wider than 16:9'
"""Edits blarc (bytearray) in place for a width x height canvas of another shape than 16:9. Returns the members it
changed. hud_corner (wider canvases): the HUD goes to the screen's corners; a narrower one always has it at the
top and bottom edges."""
wide = width / height > 16 / 9
k = (16 / 9) / (width / height) if wide else (width / height) / (16 / 9)
assert k < 1, 'only for canvases of another shape than 16:9'
root_scale = 'scale_x' if wide else 'scale_y'
changed, layouts = set(), {}
unknown = sorted(n for n in KEEP_ROOT if f'blyt/{n}.bflyt' not in members)
if unknown: raise SystemExit(f'KEEP_ROOT names no layout: {unknown}')
@@ -123,17 +232,21 @@ def apply(blarc, members, width, height, hud_corner=True):
return find_pane(layouts[member], layout, pane)
for member in sorted(m for m in members if m.startswith('blyt/') and m.endswith('.bflyt')):
if member[5:-6] in KEEP_ROOT: continue
at = pane_offset(member[5:-6], 'RootPane') + FIELD['scale_x']
at = pane_offset(member[5:-6], 'RootPane') + FIELD[root_scale]
have = struct.unpack_from('<f', blarc, at)[0]
if abs(have - 1.0) > 1e-3: raise SystemExit(f'{member[5:-6]} RootPane scale_x: {have}, expected 1.0')
if abs(have - 1.0) > 1e-3: raise SystemExit(f'{member[5:-6]} RootPane {root_scale}: {have}, expected 1.0')
struct.pack_into('<f', blarc, at, k)
# Every edit's value at k = 1 is the value the game ships (a scale of 1, the 16:9 position): checked before writing,
# so a version with other numbers stops here instead of getting wrong positions. OFF moves a pane away: not checked.
for (layout, pane, field, value), (_, _, _, shipped) in zip(edits(k, hud_corner), edits(1.0, hud_corner)):
table = zip(edits(k, hud_corner), edits(1.0, hud_corner)) if wide else zip(tall_edits(k), tall_edits(1.0))
for (layout, pane, field, value), (_, _, _, shipped) in table:
at = pane_offset(layout, pane) + FIELD[field]
have = struct.unpack_from('<f', blarc, at)[0]
if value != 10000.0 and abs(have - shipped) > 1e-3: raise SystemExit(f'{layout} {pane} {field}: {have}, expected {shipped}')
struct.pack_into('<f', blarc, at, value)
if not wide:
tall_anim(blarc, members, k, changed)
return sorted(changed)
for anim, offset, shipped, value in anim_edits(k, hud_corner):
member = f'anim/{anim}.bflan'
if member not in members: raise SystemExit(f'{anim}: no animation {member} in the archive')
+3
View File
@@ -104,6 +104,9 @@ static class SwshXlate
// "softfloat64": true lowers FP64 operations to FP32 (Ryujinx's DoubleToFloat pass), for a host without shaderFloat64.
public bool QueryHostSupportsShaderFloat64() => _job["softfloat64"] is not JsonValue v || !(v.TryGetValue<bool>(out bool b) && b);
public bool QueryHostSupportsTextureShadowLod() => true;
// Sub-texel precision bits of a host whose gathers round differently from Maxwell (AMD, Intel): Ryujinx then
// biases gather coordinates so the same texels are picked (TexturePass.InsertCoordGatherBias). 0: no bias.
public int QueryHostGatherBiasPrecision() => _job["gather_bias"] is JsonNode n ? (int)n : 0;
// 作业 reduced = true:放宽精度——翻译器只给强制精确的运算打 NoContraction(缺省全部打:保持 Maxwell 分开舍入的逐位结果,代价是 MSL 里大量禁优化的 spvFMul 等)
// Job reduced = true: relaxed precision - the translator marks NoContraction only on operations that must be exact (the default marks all: keeps the bit-exact results of Maxwell's separate rounding, at the cost of many optimization-disabled spvFMul etc. in the MSL)
public bool QueryHostReducedPrecision() => _job["reduced"] is JsonNode n && (bool)n;
+6 -3
View File
@@ -1,4 +1,4 @@
"""Smoke test for build/xlate/totk_xlate.exe: no game files needed.
"""Smoke test for build/xlate/totk_xlate(.exe): no game files needed.
Sends one hand-assembled Maxwell fragment shader (writes the constant colour 1.0, 0.5, 0.25, 1.0 to render target 0)
through --serve, checks the reply, validates the SPIR-V with spirv-val, then sends a malformed line and checks the
@@ -8,6 +8,8 @@ service survives and still answers.
"""
import json
import os
import shutil
import struct
import subprocess
import sys
@@ -15,8 +17,9 @@ import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
EXE = ROOT / "build" / "xlate" / "totk_xlate.exe"
SPIRV_VAL = Path(r"C:\msys64\ucrt64\bin\spirv-val.exe")
EXE = ROOT / "build" / "xlate" / ("totk_xlate.exe" if os.name == "nt" else "totk_xlate")
SPIRV_VAL = (Path(r"C:\msys64\ucrt64\bin\spirv-val.exe") if os.name == "nt"
else Path(shutil.which("spirv-val") or "/usr/bin/spirv-val"))
PT = 7 << 16 # predicate "always"