mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-22 13:05:57 +00:00
Merge dev into xbox-button-icons, with the restart and the prompts on the Linux port's portable calls
The restart moves into gfx_present.c's gfx_window_closed, which the Win32 and SDL windows share, and linux.c gets plat_restart and plat_restart_wait: posix_spawn of /proc/self/exe with no descriptor past stderr and the default signals, and a wait on a pidfd. The prompts code takes plat_file_exists, plat_setenv, strcasecmp and strdup, and the menu and the log say a game resolution of another shape than 16:9 (16:10 now as well as 21:9) keeps the Switch prompts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
01d69bf6d1
134 files changed
+9659
-4837
No files matched your search
+145
-31
@@ -3,6 +3,11 @@
|
||||
Writes build/<version>/build.ninja and runs MSYS2 UCRT64 Ninja on it. Ninja keeps the build incremental: a chunk is
|
||||
recompiled only when it, a header it includes or its flags change.
|
||||
|
||||
On Linux the same steps make build/<version>/totk with the clang, lld and ninja on PATH (or the clang and lld in --toolchain DIR/bin): the
|
||||
runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and totk.rc, the libraries are
|
||||
the system's (Vulkan, SDL3, Opus, libvpx, zstd; libstdc++ static), and the executable finds a libSDL3.so.0 beside it
|
||||
first (rpath $ORIGIN). docs/building.md lists the packages.
|
||||
|
||||
.venv\\Scripts\\python tools\\build_game.py # generated/v121 -> build/v121/totk.exe
|
||||
.venv\\Scripts\\python tools\\build_game.py --modules subsdk0 # one module (links if it has no calls into others)
|
||||
.venv\\Scripts\\python tools\\build_game.py --jobs 8
|
||||
@@ -50,7 +55,7 @@ Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import ctypes.wintypes as wt
|
||||
import ctypes.wintypes as wt # the job-object structures below; importing it works on every OS
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
@@ -58,31 +63,79 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import romfs_index
|
||||
from build_shaders import stale as stale_shaders
|
||||
from check_content import GENERATED_MARKER
|
||||
|
||||
WIN = os.name == 'nt'
|
||||
if not WIN:
|
||||
import resource
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
# In a git worktree (tools/worktree.py) build/ and generated/ are junctions into the main checkout, and resolve() follows
|
||||
# them. MAIN_ROOT is where they land, so resolved paths still print and compare relative to a root.
|
||||
MAIN_ROOT = (ROOT / 'build').resolve().parent
|
||||
UCRT = Path(r'C:\msys64\ucrt64\bin')
|
||||
CLANG = UCRT / 'clang.exe'
|
||||
NINJA = UCRT / 'ninja.exe'
|
||||
SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll']
|
||||
EXE = 'totk.exe' if WIN else 'totk'
|
||||
|
||||
|
||||
def on_path(name):
|
||||
"""name on PATH, else a path that does not exist (the tool check below names it)."""
|
||||
found = shutil.which(name)
|
||||
return Path(found) if found else Path('/usr/bin') / name
|
||||
|
||||
|
||||
CLANG = UCRT / 'clang.exe' if WIN else on_path('clang')
|
||||
NINJA = UCRT / 'ninja.exe' if WIN else on_path('ninja')
|
||||
SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] if WIN else []
|
||||
|
||||
# Flags for generated code as LAS Recomp builds it (Makefile GENFLAGS, Windows), plus -march for the FP/SIMD lowering.
|
||||
GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden',
|
||||
'-march=x86-64-v3']
|
||||
# _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain) to
|
||||
# Windows 7, which hides the per-monitor DPI functions gfx_window.c and dpi_probe.c call.
|
||||
RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3', '-D_WIN32_WINNT=0x0A00']
|
||||
# -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path
|
||||
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py
|
||||
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c)
|
||||
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
|
||||
RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3']
|
||||
if WIN:
|
||||
# _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain)
|
||||
# to Windows 7, which hides the per-monitor DPI functions gfx_window_win.c and dpi_probe_win.c call.
|
||||
RT_FLAGS.append('-D_WIN32_WINNT=0x0A00')
|
||||
# -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path
|
||||
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py
|
||||
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c)
|
||||
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
|
||||
else:
|
||||
# -no-pie: the lifted modules' function tables (hundreds of thousands of pointers) need no load-time relocation.
|
||||
# $ORIGIN: a libSDL3.so.0 next to totk comes before the system's (the release's portable folder).
|
||||
LINK_FLAGS = ['-fuse-ld=lld', '-no-pie', '-Wl,-z,noexecstack', "'-Wl,-rpath,$$ORIGIN'",
|
||||
'"-Wl,-Map=$builddir/totk.map"']
|
||||
LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-lvpx', '-lzstd', '-l:libstdc++.a', '-lm', '-lpthread', '-ldl']
|
||||
VMA_INCLUDE = Path('/usr/include') # vk_mem_alloc.h; Arch (AUR) vulkan-memory-allocator, Debian libvulkan-memory-allocator-dev
|
||||
# Without a system vk_mem_alloc.h, the header the Windows release pins (tools/make_release.py, VMA 3.4.0) is fetched once
|
||||
# into toolchain/vma: one MIT header (THIRD_PARTY_NOTICES.md), checked against its hash.
|
||||
VMA_FETCH = ('https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator/v3.4.0/include/vk_mem_alloc.h',
|
||||
'8487b7995ad3b263eb73bc5b9a77d71aa69b6bef5d58a715c02d2663afd81f1a')
|
||||
VMA_LOCAL = ROOT / 'toolchain' / 'vma'
|
||||
|
||||
|
||||
def vma_include():
|
||||
"""The directory holding vk_mem_alloc.h: the system's, else toolchain/vma, fetched there on first use."""
|
||||
if (VMA_INCLUDE / 'vk_mem_alloc.h').is_file():
|
||||
return VMA_INCLUDE
|
||||
dst = VMA_LOCAL / 'vk_mem_alloc.h'
|
||||
if not dst.is_file():
|
||||
url, sha = VMA_FETCH
|
||||
print(f'build_game: no {VMA_INCLUDE / "vk_mem_alloc.h"}; fetching VMA 3.4.0 into {VMA_LOCAL}')
|
||||
try:
|
||||
data = urllib.request.urlopen(url, timeout=60).read()
|
||||
except OSError as e:
|
||||
sys.exit(f'build_game: fetching {url} failed ({e}); install VMA or put vk_mem_alloc.h in {VMA_LOCAL}')
|
||||
if hashlib.sha256(data).hexdigest() != sha:
|
||||
sys.exit(f'build_game: {url} does not match the pinned hash')
|
||||
VMA_LOCAL.mkdir(parents=True, exist_ok=True)
|
||||
dst.write_bytes(data)
|
||||
return VMA_LOCAL
|
||||
|
||||
|
||||
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
|
||||
LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lvulkan-1', '-lSDL3', '-lopus',
|
||||
'-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a']
|
||||
@@ -92,7 +145,7 @@ DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw'
|
||||
# functions sit on it).
|
||||
INSTALL_SIDE = {'romfs.c', 'patches.c', 'ov_zstd.c'}
|
||||
LIB_NAME = 'libtotkrt.a'
|
||||
AR = UCRT / 'llvm-ar.exe'
|
||||
AR = UCRT / 'llvm-ar.exe' if WIN else on_path('llvm-ar')
|
||||
WINDRES = UCRT / 'windres.exe' # runtime/totk.rc: the manifest of totk.exe
|
||||
TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain
|
||||
DEPS = None # the deps directory of an llvm-mingw build
|
||||
@@ -196,12 +249,21 @@ def c_string(s):
|
||||
RUNTIME = ROOT / 'runtime' # --runtime: a copy of runtime/ lets a task build without touching the shared tree
|
||||
|
||||
|
||||
def other_os(name):
|
||||
"""A runtime file for another OS: Win32 ones (*_win.*, platform/win32.c, totk.rc) off Windows, the SDL window's
|
||||
(*_sdl.*) and platform/linux.c on Windows."""
|
||||
stem = name.rsplit('.', 1)[0]
|
||||
if WIN:
|
||||
return stem.endswith('_sdl') or name == 'linux.c'
|
||||
return stem.endswith('_win') or name in ('win32.c', 'totk.rc')
|
||||
|
||||
|
||||
def runtime_sources(skip=()):
|
||||
"""Every runtime/*.c, *.cpp, platform/*.c, *.rc, the in-game menu (ui/*.cpp) and Dear ImGui except the names in skip,
|
||||
so a task adds a file without editing this script."""
|
||||
"""Every runtime/*.c, *.cpp, platform/*.c, *.rc, the in-game menu (ui/*.cpp) and Dear ImGui of this OS except the
|
||||
names in skip, so a task adds a file without editing this script."""
|
||||
pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc', 'ui/*.cpp', 'third_party/imgui/*.cpp',
|
||||
'third_party/imgui/backends/*.cpp')
|
||||
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip)
|
||||
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip and not other_os(p.name))
|
||||
|
||||
|
||||
def runtime_object(out, src):
|
||||
@@ -254,10 +316,10 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
|
||||
out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects
|
||||
except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE."""
|
||||
inc = f'-I {cmd_path(RUNTIME)} -I {cmd_path(gen)}'
|
||||
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(VMA_INCLUDE)
|
||||
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(vma_include())
|
||||
imgui = cmd_path(RUNTIME / 'third_party' / 'imgui')
|
||||
rt_inc = f'{inc} -I {cmd_path(out)} -I {vma} -I {imgui}' # romfs_config.h, vk_mem_alloc.h (+ deps headers), imgui.h
|
||||
libs = [f'-L {cmd_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS
|
||||
libs = [f'-L {cmd_path(DEPS / "lib")}'] + (LLVM_LINK_LIBS if WIN else LINK_LIBS) if DEPS else LINK_LIBS
|
||||
# A compiler cache (sccache) may front clang: every lift rewrites the generated chunks with the same contents, so
|
||||
# the cached objects save the compile step on the next build.
|
||||
cache = f'{cmd_arg(os.environ["SCCACHE"])} ' if os.environ.get('SCCACHE') else ''
|
||||
@@ -324,7 +386,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
|
||||
rt.append(o)
|
||||
objs['runtime'] = rt
|
||||
every = [o for v in objs.values() for o in v]
|
||||
exe = out / 'totk.exe'
|
||||
exe = out / EXE
|
||||
if make_lib:
|
||||
exe = out / LIB_NAME
|
||||
lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
|
||||
@@ -370,7 +432,7 @@ def main_objects(gen, main_out, rt_dir, mods):
|
||||
pairs[o] = theirs
|
||||
# Ninja runs in the main checkout: the main build.ninja is relative to it, and the worktree's runtime/ files carry newer
|
||||
# mtimes that would make every object look stale. Header equality is checked byte for byte below.
|
||||
env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', ''))
|
||||
env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', '')) if WIN else dict(os.environ)
|
||||
targets = [ninja_path(t) for t in pairs.values()]
|
||||
for i in range(0, len(targets), 200):
|
||||
r = subprocess.run([str(NINJA), '-f', str(main_ninja), '-n', '-d', 'explain'] + targets[i:i + 200], cwd=MAIN_ROOT,
|
||||
@@ -539,7 +601,13 @@ CREATE_SUSPENDED = 0x4
|
||||
|
||||
|
||||
def run_measured(cmd, env):
|
||||
"""Run cmd in its own job object. Returns (exit code, seconds, peak bytes of one process, peak bytes of all)."""
|
||||
"""Run cmd in its own job object. Returns (exit code, seconds, peak bytes of one process, peak bytes of all).
|
||||
Off Windows: the largest resident set of one child process, and 0 for all of them (no job accounting)."""
|
||||
if not WIN:
|
||||
t0 = time.monotonic()
|
||||
code = subprocess.call(cmd, cwd=ROOT, env=env)
|
||||
peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss * 1024 # KiB on Linux
|
||||
return code, time.monotonic() - t0, peak, 0
|
||||
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
|
||||
k32.CreateJobObjectW.restype = wt.HANDLE
|
||||
k32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wt.LPCWSTR]
|
||||
@@ -573,6 +641,9 @@ def run_measured(cmd, env):
|
||||
|
||||
|
||||
def total_ram():
|
||||
if not WIN:
|
||||
return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
|
||||
|
||||
class MemStatus(ctypes.Structure):
|
||||
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
|
||||
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
|
||||
@@ -597,6 +668,30 @@ def pe_sections(path):
|
||||
return out, image
|
||||
|
||||
|
||||
def elf_sections(path):
|
||||
"""{section name: size} of the allocated sections of an ELF64 file, and the extent they span in memory."""
|
||||
with open(path, 'rb') as f:
|
||||
head = f.read(64)
|
||||
shoff = int.from_bytes(head[0x28:0x30], 'little')
|
||||
shentsize, shnum, shstrndx = (int.from_bytes(head[o:o + 2], 'little') for o in (0x3A, 0x3C, 0x3E))
|
||||
f.seek(shoff)
|
||||
table = f.read(shentsize * shnum)
|
||||
sh = [table[i * shentsize:(i + 1) * shentsize] for i in range(shnum)]
|
||||
f.seek(int.from_bytes(sh[shstrndx][0x18:0x20], 'little'))
|
||||
names = f.read(int.from_bytes(sh[shstrndx][0x20:0x28], 'little'))
|
||||
out, lo, hi = {}, None, 0
|
||||
for s in sh:
|
||||
flags = int.from_bytes(s[0x08:0x10], 'little')
|
||||
addr, size = int.from_bytes(s[0x10:0x18], 'little'), int.from_bytes(s[0x20:0x28], 'little')
|
||||
if not flags & 2 or not size: # SHF_ALLOC
|
||||
continue
|
||||
name = names[int.from_bytes(s[0:4], 'little'):].split(b'\0', 1)[0].decode(errors='replace')
|
||||
out[name] = out.get(name, 0) + size
|
||||
lo = addr if lo is None else min(lo, addr)
|
||||
hi = max(hi, addr + size)
|
||||
return out, hi - (lo or 0)
|
||||
|
||||
|
||||
def mib(n):
|
||||
return f'{n / (1 << 20):,.1f} MiB'
|
||||
|
||||
@@ -623,9 +718,12 @@ def main():
|
||||
"build/profile-<version> otherwise, so trying another version leaves "
|
||||
"the main profile alone; saves load on 1.2.1 and 1.4.3 alike)")
|
||||
ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of '
|
||||
'MSYS2 UCRT64 clang')
|
||||
'MSYS2 UCRT64 clang; on Linux an LLVM directory (bin/clang, '
|
||||
'bin/llvm-ar) instead of the clang on PATH')
|
||||
ap.add_argument('--ninja', type=Path, help='ninja.exe to build with (default: MSYS2 UCRT64, else bin/ninja.exe of the '
|
||||
'release folder)')
|
||||
ap.add_argument('--sysroot', type=Path, help='Linux: the glibc headers and libraries to compile and link against '
|
||||
'instead of the host\'s (the release\'s llvm/sysroot)')
|
||||
ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx '
|
||||
f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})')
|
||||
ap.add_argument('--make-runtime-lib', action='store_true',
|
||||
@@ -652,19 +750,33 @@ def main():
|
||||
global CLANG, AR, WINDRES, NINJA, TOOLCHAIN_BIN, DEPS
|
||||
if args.ninja:
|
||||
NINJA = args.ninja.resolve()
|
||||
elif not NINJA.is_file() and (ROOT / 'bin' / 'ninja.exe').is_file():
|
||||
NINJA = ROOT / 'bin' / 'ninja.exe' # the release folder (tools/make_release.py)
|
||||
elif not NINJA.is_file() and (ROOT / 'bin' / ('ninja' + ('.exe' if WIN else ''))).is_file():
|
||||
NINJA = ROOT / 'bin' / ('ninja' + ('.exe' if WIN else '')) # the release folder (tools/make_release.py)
|
||||
dlls = SDL3_DLLS
|
||||
suffix = '.exe' if WIN else ''
|
||||
if args.toolchain:
|
||||
TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve()
|
||||
CLANG = TOOLCHAIN_BIN / 'clang.exe'
|
||||
AR = TOOLCHAIN_BIN / 'llvm-ar.exe'
|
||||
CLANG = TOOLCHAIN_BIN / f'clang{suffix}'
|
||||
AR = TOOLCHAIN_BIN / f'llvm-ar{suffix}'
|
||||
WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe'
|
||||
if args.toolchain and WIN or args.deps and (args.toolchain or not WIN): # Linux: --deps only, never deps-mingw
|
||||
DEPS = (args.deps or DEFAULT_DEPS).resolve()
|
||||
dlls = sorted((DEPS / 'bin').glob('*.dll'))
|
||||
dlls = sorted((DEPS / 'bin').glob('*.dll')) if WIN else []
|
||||
if DEPS and not WIN:
|
||||
# A release (--deps): Opus, libvpx and zstd go into the executable, so it needs no more than libc, Vulkan and
|
||||
# the libSDL3.so.0 beside it
|
||||
for lib in ('opus', 'vpx', 'zstd'):
|
||||
LINK_LIBS[LINK_LIBS.index(f'-l{lib}')] = f'-l:lib{lib}.a'
|
||||
if args.sysroot and not WIN:
|
||||
# Every compile and the link: the game then needs no newer glibc than the sysroot's (the Steam Runtime's)
|
||||
sysroot = ['--target=x86_64-linux-gnu', f'--sysroot={cmd_path(args.sysroot.resolve())}']
|
||||
GEN_FLAGS[:0] = sysroot
|
||||
RT_FLAGS[:0] = sysroot
|
||||
LINK_FLAGS[:0] = sysroot
|
||||
for tool in (CLANG, NINJA):
|
||||
if not tool.is_file():
|
||||
sys.exit(f'{tool} not found: install MSYS2 UCRT64 clang, lld and ninja (docs/building.md)')
|
||||
sys.exit(f'{tool} not found: install ' + ('MSYS2 UCRT64 clang, lld and ninja' if WIN else 'clang, lld and ninja')
|
||||
+ ' (docs/building.md)')
|
||||
if not (gen / 'layout.json').is_file():
|
||||
sys.exit(f'{gen / "layout.json"} not found: run the generator first')
|
||||
|
||||
@@ -689,7 +801,7 @@ def main():
|
||||
print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s')
|
||||
else:
|
||||
romfs_index.write_placeholder(out)
|
||||
print(f'build_game: warning: no {paths["romfs"] / "index.json"}; totk.exe will have no RomFS', file=sys.stderr)
|
||||
print(f'build_game: warning: no {paths["romfs"] / "index.json"}; {EXE} will have no RomFS', file=sys.stderr)
|
||||
copy_romfs_patches(args.version, out)
|
||||
global RUNTIME
|
||||
RUNTIME = (args.runtime or RUNTIME).resolve()
|
||||
@@ -724,13 +836,15 @@ def main():
|
||||
args.make_runtime_lib)
|
||||
write_if_changed(out / 'build.ninja', text)
|
||||
|
||||
env = dict(os.environ, PATH=str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', ''))
|
||||
env = dict(os.environ)
|
||||
if WIN or TOOLCHAIN_BIN:
|
||||
env['PATH'] = str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', '')
|
||||
base = [str(NINJA), '-f', str(out / 'build.ninja'), '-j', str(jobs)]
|
||||
nfiles = sum(len(v) for v in mods.values())
|
||||
print(f'build_game: {nfiles} generated files in {len(linked)} modules ({", ".join(linked)}), {jobs} jobs', flush=True)
|
||||
|
||||
code, dt, peak_one, peak_all = run_measured(base + ['objects'], env)
|
||||
print(f'compile: {dt:.1f} s, peak {mib(peak_one)} per process, {mib(peak_all)} all processes')
|
||||
print(f'compile: {dt:.1f} s, peak {mib(peak_one)} per process' + (f', {mib(peak_all)} all processes' if WIN else ''))
|
||||
if code:
|
||||
sys.exit(code)
|
||||
for mod, files in objs.items():
|
||||
@@ -752,7 +866,7 @@ def main():
|
||||
return
|
||||
for dll in dlls: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to totk.exe
|
||||
shutil.copy2(dll, exe.parent / dll.name)
|
||||
sections, image = pe_sections(exe)
|
||||
sections, image = pe_sections(exe) if WIN else elf_sections(exe)
|
||||
print(f'{below_root(exe)}: file {mib(exe.stat().st_size)}, '
|
||||
f'image {mib(image)}; ' + ', '.join(f'{k} {mib(v)}' for k, v in sections.items() if k.startswith('.') and v))
|
||||
if image > 3 * GiB // 2:
|
||||
|
||||
+32
-13
@@ -3,7 +3,8 @@
|
||||
Fetches the pinned commit into toolchain/src/hactool, resets it (local changes and build output there are discarded),
|
||||
applies portable.patch (MacNativeGames' Windows-relevant changes) and unicode.patch (ours), and builds a static exe
|
||||
with MSYS2 UCRT64 gcc; mbedtls comes with hactool's tree. The exe and hactool's LICENSE go to toolchain/hactool, where
|
||||
tools/extract.py and tools/make_release.py take them. Run with .venv.
|
||||
tools/extract.py and tools/make_release.py take them. Run with .venv. On Linux: the system's cc and make build
|
||||
toolchain/hactool/hactool (unicode.patch only adds a Windows manifest, under EXEEXT .exe).
|
||||
|
||||
python tools/build_hactool.py [--src DIR] [--out DIR]
|
||||
"""
|
||||
@@ -13,15 +14,20 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
URL = 'https://github.com/SciresM/hactool.git'
|
||||
COMMIT = '1d64a83450e025622f3468c28fc4164dad2c5ef6'
|
||||
PATCHES = [ROOT / 'tools' / 'hactool' / n for n in ('portable.patch', 'unicode.patch')]
|
||||
PATCHES = [ROOT / 'tools' / 'hactool' / n for n in ('portable.patch', 'unicode.patch', 'mbedtls-asm-clobbers.patch')]
|
||||
WIN = os.name == 'nt'
|
||||
MSYS_BASH = Path(r'C:\msys64\usr\bin\bash.exe')
|
||||
EXE = 'hactool.exe' if WIN else 'hactool'
|
||||
CONFIG_MK = ('CC = gcc\nEXEEXT = .exe\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n'
|
||||
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto -static\n')
|
||||
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto -static\n') if WIN else \
|
||||
('CC = cc\nEXEEXT =\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n'
|
||||
'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto\n')
|
||||
|
||||
|
||||
def git(src, *args):
|
||||
@@ -44,31 +50,44 @@ def fetch(src):
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--src', type=Path, default=ROOT / 'toolchain' / 'src' / 'hactool', help='source checkout')
|
||||
ap.add_argument('--out', type=Path, default=ROOT / 'toolchain' / 'hactool', help='where hactool.exe goes')
|
||||
ap.add_argument('--out', type=Path, default=ROOT / 'toolchain' / 'hactool', help='where hactool(.exe) goes')
|
||||
ap.add_argument('--cc', help='Linux: the C compiler command for hactool and its bundled mbedtls, for example '
|
||||
'"clang --sysroot=DIR" (tools/make_release_linux.py); default cc')
|
||||
args = ap.parse_args()
|
||||
if not MSYS_BASH.is_file():
|
||||
if WIN and not MSYS_BASH.is_file():
|
||||
sys.exit(f'{MSYS_BASH} not found: install MSYS2 with the UCRT64 gcc and make (docs/building.md)')
|
||||
if not WIN and not (shutil.which('make') and (args.cc or shutil.which('cc'))):
|
||||
sys.exit('make and cc not found: install a C compiler and make (docs/building.md)')
|
||||
src, out = args.src.resolve(), args.out.resolve()
|
||||
|
||||
fetch(src)
|
||||
for p in PATCHES:
|
||||
git(src, 'apply', '--whitespace=nowarn', str(p))
|
||||
(src / 'config.mk').write_text(CONFIG_MK, encoding='utf-8', newline='\n')
|
||||
config = CONFIG_MK.replace('CC = cc\n', f'CC = {args.cc}\n') if args.cc else CONFIG_MK
|
||||
(src / 'config.mk').write_text(config, encoding='utf-8', newline='\n')
|
||||
|
||||
env = dict(os.environ, MSYSTEM='UCRT64', CHERE_INVOKING='1')
|
||||
r = subprocess.run([str(MSYS_BASH), '-lc', f'make -j{os.cpu_count() or 4}'], cwd=src, env=env,
|
||||
capture_output=True, text=True)
|
||||
jobs = f'-j{os.cpu_count() or 4}'
|
||||
if WIN:
|
||||
env = dict(os.environ, MSYSTEM='UCRT64', CHERE_INVOKING='1')
|
||||
r = subprocess.run([str(MSYS_BASH), '-lc', f'make {jobs}'], cwd=src, env=env, capture_output=True, text=True)
|
||||
else:
|
||||
r = subprocess.run(['make', jobs, *([f'CC={args.cc}'] if args.cc else [])], cwd=src, capture_output=True,
|
||||
text=True) # CC= on the command line: the bundled mbedtls's own make reads it too
|
||||
(src / 'build.log').write_text(r.stdout + r.stderr, encoding='utf-8')
|
||||
if r.returncode:
|
||||
sys.exit(f'make failed ({r.returncode}); log: {src / "build.log"}')
|
||||
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src / 'hactool.exe', out / 'hactool.exe')
|
||||
shutil.copy2(src / EXE, out / EXE)
|
||||
shutil.copy2(src / 'LICENSE', out / 'LICENSE')
|
||||
banner = subprocess.run([str(out / 'hactool.exe')], capture_output=True, text=True)
|
||||
# hactool reads ~/.switch/prod.keys before it prints the banner and exits on a malformed key there, so the check
|
||||
# runs with an empty home: it tests the binary, not the builder's keys.
|
||||
with tempfile.TemporaryDirectory() as home:
|
||||
env = dict(os.environ, HOME=home, USERPROFILE=home)
|
||||
banner = subprocess.run([str(out / EXE)], capture_output=True, text=True, env=env)
|
||||
if 'hactool (c) SciresM' not in banner.stdout + banner.stderr:
|
||||
sys.exit(f'{out / "hactool.exe"} does not start')
|
||||
print(f'{out / "hactool.exe"}: {COMMIT[:7]} + {", ".join(p.name for p in PATCHES)}')
|
||||
sys.exit(f'{out / EXE} does not start')
|
||||
print(f'{out / EXE}: {COMMIT[:7]} + {", ".join(p.name for p in PATCHES)}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -10,13 +10,15 @@ shader. Needs glslc and spirv-val from MSYS2 UCRT64 (package mingw-w64-ucrt-x86_
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||
BIN = Path(r'C:\msys64\ucrt64\bin') # Windows; elsewhere glslc and spirv-val come from PATH
|
||||
WIN = os.name == 'nt'
|
||||
TARGET = 'vulkan1.1'
|
||||
|
||||
|
||||
@@ -54,9 +56,9 @@ def stale(runtime):
|
||||
def compile_one(src):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
spv = Path(tmp) / 'out.spv'
|
||||
subprocess.run([str(BIN / 'glslc.exe'), '-O', f'--target-env={TARGET}', '-fshader-stage=compute',
|
||||
subprocess.run([str(BIN / 'glslc.exe') if WIN else 'glslc', '-O', f'--target-env={TARGET}', '-fshader-stage=compute',
|
||||
'-o', str(spv), str(src)], check=True)
|
||||
subprocess.run([str(BIN / 'spirv-val.exe'), f'--target-env', TARGET, str(spv)], check=True)
|
||||
subprocess.run([str(BIN / 'spirv-val.exe') if WIN else 'spirv-val', f'--target-env', TARGET, str(spv)], check=True)
|
||||
data = spv.read_bytes()
|
||||
words = [int.from_bytes(data[i:i + 4], 'little') for i in range(0, len(data), 4)]
|
||||
header_path(src).write_text(header_of(src, words), encoding='utf-8', newline='\n')
|
||||
|
||||
+20
-8
@@ -2,7 +2,8 @@
|
||||
|
||||
Copies tools/vendor/ryujinx to build/xlate/ryujinx, applies tools/xlate/patches/*.patch there (tools/vendor stays
|
||||
byte-identical; 0001-0004 are MNG's, 0005-0009 are ours), then runs `dotnet publish`. Needs the .NET 10 SDK, the MSVC build
|
||||
tools and vswhere.exe (docs/building.md). Run with .venv.
|
||||
tools and vswhere.exe (docs/building.md). Run with .venv. On Linux it builds build/xlate/totk_xlate for linux-x64, which
|
||||
NativeAOT links with the system's clang (no MSVC); NativeAOT builds for the OS it runs on only.
|
||||
|
||||
Next to the exe it writes totk_xlate.id: the translator identity, a sha256 over everything the build reads (this script,
|
||||
Program.cs, the csproj, the patches, the vendor tree). runtime/gfx_shader.c puts it into every shader cache key, so a
|
||||
@@ -28,6 +29,8 @@ VENDOR = ROOT / "tools" / "vendor" / "ryujinx"
|
||||
MNG_VENDOR = ROOT / "refs" / "MacNativeGames" / "vendor" / "ryujinx"
|
||||
XLATE = ROOT / "tools" / "xlate"
|
||||
BUILD = ROOT / "build" / "xlate"
|
||||
WIN = os.name == "nt"
|
||||
EXE = "totk_xlate.exe" if WIN else "totk_xlate"
|
||||
VSWHERE_DIR = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio" / "Installer"
|
||||
|
||||
|
||||
@@ -133,21 +136,25 @@ def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--id-only", action="store_true", help="write totk_xlate.id for the exe already built")
|
||||
parser.add_argument("--out", type=Path, help="build directory instead of build/xlate")
|
||||
parser.add_argument("--llvm", type=Path, help="Linux: an LLVM directory whose bin/clang and lld link the exe")
|
||||
parser.add_argument("--sysroot", type=Path, help="Linux: link against this sysroot's glibc instead of the host's "
|
||||
"(tools/make_release_linux.py)")
|
||||
a = parser.parse_args()
|
||||
if a.out:
|
||||
global BUILD
|
||||
BUILD = a.out.resolve()
|
||||
if a.id_only:
|
||||
write_identity(BUILD / "totk_xlate.exe")
|
||||
write_identity(BUILD / EXE)
|
||||
return
|
||||
started = time.time()
|
||||
dotnet = shutil.which("dotnet")
|
||||
if not dotnet:
|
||||
sys.exit("missing: dotnet (.NET 10 SDK)")
|
||||
if not (VSWHERE_DIR / "vswhere.exe").is_file() and not shutil.which("vswhere"):
|
||||
sys.exit(f"missing: vswhere.exe (expected in {VSWHERE_DIR}); NativeAOT cannot link without it")
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = str(VSWHERE_DIR) + os.pathsep + env["PATH"]
|
||||
if WIN:
|
||||
if not (VSWHERE_DIR / "vswhere.exe").is_file() and not shutil.which("vswhere"):
|
||||
sys.exit(f"missing: vswhere.exe (expected in {VSWHERE_DIR}); NativeAOT cannot link without it")
|
||||
env["PATH"] = str(VSWHERE_DIR) + os.pathsep + env["PATH"]
|
||||
|
||||
verify_vendor()
|
||||
src = prepare_sources()
|
||||
@@ -159,13 +166,18 @@ def main() -> None:
|
||||
out = BUILD / "publish"
|
||||
shutil.rmtree(out, ignore_errors=True)
|
||||
cmd = [
|
||||
dotnet, "publish", str(work / "totk_xlate.csproj"), "-c", "Release", "-r", "win-x64",
|
||||
dotnet, "publish", str(work / "totk_xlate.csproj"), "-c", "Release", "-r", "win-x64" if WIN else "linux-x64",
|
||||
"-p:PublishAot=true", "-p:InvariantGlobalization=true", f"-p:RyuSrc={src}", "-o", str(out),
|
||||
]
|
||||
if a.llvm:
|
||||
env["PATH"] = str(a.llvm.resolve() / "bin") + os.pathsep + env["PATH"]
|
||||
cmd += ["-p:CppCompilerAndLinker=clang", "-p:LinkerFlavor=lld"]
|
||||
if a.sysroot:
|
||||
cmd.append(f"-p:SysRoot={a.sysroot.resolve()}")
|
||||
if subprocess.run(cmd, env=env).returncode != 0:
|
||||
sys.exit("dotnet publish failed")
|
||||
exe = BUILD / "totk_xlate.exe"
|
||||
shutil.copy2(out / "totk_xlate.exe", exe)
|
||||
exe = BUILD / EXE
|
||||
shutil.copy2(out / EXE, exe)
|
||||
write_identity(exe)
|
||||
print(f"built {exe} ({exe.stat().st_size / 1e6:.1f} MB) in {time.time() - started:.0f} s")
|
||||
|
||||
|
||||
+16
-49
@@ -1,9 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fails when runtime code outside runtime/platform/ uses the Win32 API directly.
|
||||
|
||||
New runtime code goes through runtime/platform/platform.h. Files that still use Win32 are listed in ALLOWED with the
|
||||
reason; the list shrinks as the platform layer takes over.
|
||||
A file in ALLOWED that no longer matches is noted, so the entry gets removed.
|
||||
Runtime code goes through runtime/platform/platform.h. The Win32 back ends of the window system are Win32 by design and
|
||||
are not scanned: files named *_win.c / *_win.cpp (runtime/ and runtime/ui/), whose SDL counterparts (*_sdl.*) serve
|
||||
other hosts. Any other file that still uses Win32 goes into ALLOWED with the reason; a file in ALLOWED that no longer
|
||||
matches is noted, so the entry gets removed.
|
||||
|
||||
Usage: python tools/check_platform.py [--root DIR] (DIR holds runtime/; default: the repository root)
|
||||
"""
|
||||
@@ -12,51 +13,8 @@ import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Per-file allowlist. "window", "input", "audio" and "probe" are permanent (OS-specific by nature); the rest wait for
|
||||
# 2.7a (memory, faults, page tracking) or 2.7b (threads, time, locks, FP state).
|
||||
ALLOWED = {
|
||||
"gfx_window.c": "window",
|
||||
"gfx_window_probe.c": "probe",
|
||||
"input_win.c": "input",
|
||||
"prof_win.c": "profiler",
|
||||
"rdoc_win.c": "RenderDoc loader",
|
||||
"audren.c": "audio",
|
||||
"kernel_probe.c": "probe",
|
||||
"nvn_probe.c": "probe",
|
||||
"ov_probe.c": "probe",
|
||||
"tex_probe.c": "probe",
|
||||
"vk_exec_probe.c": "probe",
|
||||
"gfx_shader_probe.c": "probe",
|
||||
"gfx_shader.c": "2.7b: threads, SRW locks, interlocked counters, tick count",
|
||||
"memory.c": "2.7a",
|
||||
"kernel.c": "2.7a/b",
|
||||
"main.c": "2.7a/b",
|
||||
"dispatch.c": "2.7b",
|
||||
"loader.c": "2.7a",
|
||||
"svc.c": "2.7b",
|
||||
"vk_memory.c": "2.7a",
|
||||
"vk_texture.c": "2.7b",
|
||||
"audren.h": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"boot.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"flight.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"fs_host.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"ipc.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"nvn_api_cmd.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"nvn_api_core.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"nvn_api_obj.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"nvn_procs.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"ov_gfx.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"ov_input.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"ov_media.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"ov_system.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"romfs.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"tex_astc_mt.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"vk_backend.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"vk_device.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"vk_dump.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"vk_pipeline.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
"vk_sync.c": "pending: SRW locks, threads, files; not in 2.7a/b/c scope yet",
|
||||
}
|
||||
# Per-file allowlist of shared files that still use Win32, with the reason. Empty: every shared file is portable.
|
||||
ALLOWED = {}
|
||||
|
||||
INCLUDE = re.compile(r'^\s*#\s*include\s*[<"](windows|winsock2|ws2tcpip|synchapi|memoryapi|processthreadsapi|'
|
||||
r'fileapi|handleapi|libloaderapi|winbase|winnt)\.h[>"]', re.M)
|
||||
@@ -75,6 +33,12 @@ CALLS = [
|
||||
"(Acquire|Release)SRWLock(Exclusive|Shared)", "WakeAllConditionVariable", "WakeConditionVariable",
|
||||
"SleepConditionVariableSRW", "InitOnceExecuteOnce", "INIT_ONCE", "Interlocked[A-Za-z0-9]*", "TlsAlloc", "FlsAlloc",
|
||||
"LoadLibrary[AW]?", "GetProcAddress", "FreeLibrary", "HANDLE", "DWORD", "LPVOID", "WINAPI",
|
||||
"SuspendThread", "ResumeThread", "GetThreadContext", "SetThreadContext", "OpenThread", "CreateFiber(Ex)?",
|
||||
"ConvertThreadToFiber(Ex)?", "ConvertFiberToThread", "SwitchToFiber", "DeleteFiber", "BCrypt[A-Za-z]*",
|
||||
"_aligned_malloc", "_aligned_free", "_stricmp", "_strnicmp", "_strdup", "_wfopen", "_fseeki64", "_ftelli64",
|
||||
"localtime_s", "strtok_s", "GetModuleHandle(Ex)?[AW]?", "ExitProcess", "SetEnvironmentVariable[AW]?", "_putenv(_s)?",
|
||||
"GetLastError", "MultiByteToWideChar", "WideCharToMultiByte", "HWND", "LARGE_INTEGER", "FILETIME",
|
||||
"MAX_PATH", "GetActiveProcessorCount", "timeBeginPeriod",
|
||||
]
|
||||
CALL_RE = re.compile(r"\b(" + "|".join(CALLS) + r")\b")
|
||||
|
||||
@@ -103,7 +67,10 @@ def main():
|
||||
a = ap.parse_args()
|
||||
rt = Path(a.runtime) if a.runtime else Path(a.root) / "runtime"
|
||||
bad, remaining = [], {}
|
||||
for p in sorted(list(rt.glob("*.c")) + list(rt.glob("*.h")) + list(rt.glob("*.cpp"))):
|
||||
files = [p for pat in ("*.c", "*.h", "*.cpp", "ui/*.cpp", "ui/*.h") for p in rt.glob(pat)]
|
||||
for p in sorted(files):
|
||||
if p.stem.endswith("_win"):
|
||||
continue
|
||||
hits = scan(p)
|
||||
if p.name in ALLOWED:
|
||||
remaining[p.name] = len(hits)
|
||||
|
||||
+2
-1
@@ -274,7 +274,8 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lam
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--config', type=Path, default=ROOT / 'config.local.toml')
|
||||
parser.add_argument('--hactool', type=Path, default=ROOT / 'toolchain' / 'hactool' / 'hactool.exe')
|
||||
parser.add_argument('--hactool', type=Path,
|
||||
default=ROOT / 'toolchain' / 'hactool' / ('hactool.exe' if os.name == 'nt' else 'hactool'))
|
||||
parser.add_argument('--base', type=Path, help='base game (default: input.base of the config)')
|
||||
parser.add_argument('--update', type=Path, action='append',
|
||||
help='an update NSP, repeatable (default: input.updates of the config)')
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
Applied by tools/build_hactool.py to SciresM/hactool (ISC) at 1d64a83450e025622f3468c28fc4164dad2c5ef6, ours.
|
||||
The x86-64 multiply-accumulate asm of hactool's bundled mbedtls 2.6.1 (MULADDC in bn_mul.h) adds into memory through
|
||||
rdi and changes the flags without saying so. gcc happens to keep that code correct; clang (the Linux release's
|
||||
toolchain) keeps memory values in registers across it at -O1 and above, the RSA arithmetic goes wrong, and every
|
||||
signature check fails (NCA header, NPDM, ACID), which stops the merge of the update. Later mbedtls releases name the
|
||||
same two clobbers.
|
||||
|
||||
diff --git a/mbedtls/include/mbedtls/bn_mul.h b/mbedtls/include/mbedtls/bn_mul.h
|
||||
index cac3f14..c787b0a 100644
|
||||
--- a/mbedtls/include/mbedtls/bn_mul.h
|
||||
+++ b/mbedtls/include/mbedtls/bn_mul.h
|
||||
@@ -179,7 +179,7 @@
|
||||
#define MULADDC_STOP \
|
||||
: "+c" (c), "+D" (d), "+S" (s) \
|
||||
: "b" (b) \
|
||||
- : "rax", "rdx", "r8" \
|
||||
+ : "rax", "rdx", "r8", "memory", "cc" \
|
||||
);
|
||||
|
||||
#endif /* AMD64 */
|
||||
+24
-10
@@ -17,9 +17,12 @@ cached in toolchain/release-dl/ and checked against the sha256 in PINS; a mismat
|
||||
VERSION the repository's VERSION plus the commit, 0.1.0+973c626; the installer's release string
|
||||
TotK-Setup.exe installer/launcher/launcher.c built with llvm/ (-mwindows): starts python/pythonw.exe -m installer.gui
|
||||
|
||||
--prove compiles generated/v121 and links build/release-check/totk.exe with nothing but the folder's python/, llvm/,
|
||||
runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there. It needs a
|
||||
generated/v121 and build/extracted/v121 from the usual game build (they are read, nothing is written to them).
|
||||
--prove compiles the generated C and links build/release-check/totk.exe with nothing but the folder's python/, llvm/,
|
||||
runtime/ and bin/ (PATH holds the Windows system directory and those), then runs the four probes there.
|
||||
|
||||
Both need one game version extracted and lifted first (tools/extract.py, recomp/gen.py): generated/v121 or, without
|
||||
it, generated/v143, and for --prove its build/extracted/<version> (they are read, nothing is written to them). The
|
||||
runtime library reads only what both versions' layout.h share (MOD_COUNT, MOD_MAIN_BASE), so either serves.
|
||||
|
||||
The folder contains nothing derived from the game: check_content runs over it before the script finishes. Then the
|
||||
folder's own python runs a check-only installer pass (no game files: it must get through the release, toolchain and host
|
||||
@@ -329,11 +332,21 @@ def collect_licenses(out):
|
||||
|
||||
# ---- runtime/, recomp/, tools/, data/ ---------------------------------------------------------------------------------
|
||||
|
||||
def lifted_version():
|
||||
"""The game version whose generated C the runtime library and --prove build against: v121, else v143."""
|
||||
for version in ('v121', 'v143'):
|
||||
if (MAIN_ROOT / 'generated' / version / 'layout.json').is_file():
|
||||
return version
|
||||
sys.exit('neither generated/v121 nor generated/v143 exists: extract a game version and lift it first '
|
||||
'(tools/extract.py, recomp/gen.py; docs/building.md)')
|
||||
|
||||
|
||||
def runtime_lib(out, llvm):
|
||||
"""Build libtotkrt.a with the folder's llvm/ and copy it, the headers and the install-side sources into runtime/."""
|
||||
build = MAIN_ROOT / 'build' / 'release-check' / 'rtlib'
|
||||
gen = MAIN_ROOT / 'generated' / 'v121'
|
||||
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm,
|
||||
version = lifted_version()
|
||||
gen = MAIN_ROOT / 'generated' / version
|
||||
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--version', version,
|
||||
'--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', build],
|
||||
env=clean_env(llvm / 'bin', out / 'bin'))
|
||||
rt = out / 'runtime'
|
||||
@@ -465,17 +478,18 @@ def write_zip(out, path):
|
||||
|
||||
|
||||
def prove(out):
|
||||
"""Compile generated/v121 and link totk.exe with the folder's tools only, then run the probes next to it."""
|
||||
"""Compile the generated C and link totk.exe with the folder's tools only, then run the probes next to it."""
|
||||
out = out.resolve()
|
||||
gen = MAIN_ROOT / 'generated' / 'v121'
|
||||
ext = MAIN_ROOT / 'build' / 'extracted' / 'v121'
|
||||
version = lifted_version()
|
||||
gen = MAIN_ROOT / 'generated' / version
|
||||
ext = MAIN_ROOT / 'build' / 'extracted' / version
|
||||
chk = MAIN_ROOT / 'build' / 'release-check'
|
||||
env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin')
|
||||
t0 = time.time()
|
||||
run([out / 'python' / 'python.exe', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm',
|
||||
'--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime', '--runtime-lib', out / 'runtime' / 'libtotkrt.a',
|
||||
'--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs',
|
||||
'--profile', chk / 'profile'], env=env)
|
||||
'--ninja', out / 'bin' / 'ninja.exe', '--version', version, '--gen', gen, '--out', chk, '--exefs', ext / 'exefs',
|
||||
'--romfs', ext / 'romfs', '--profile', chk / 'profile'], env=env)
|
||||
print(f'prove: build {time.time() - t0:.0f} s')
|
||||
for dll in ('SDL3.dll', 'libopus-0.dll'):
|
||||
shutil.copy2(out / 'bin' / dll, chk / dll)
|
||||
|
||||
@@ -0,0 +1,527 @@
|
||||
"""Assemble the Linux release folder of docs/installer.md ("Linux release") and pack it as an AppImage.
|
||||
|
||||
.venv/bin/python tools/make_release_linux.py --out build/release/TotK-Setup-linux \\
|
||||
[--appimage build/release/TotK-Setup-x86_64.AppImage]
|
||||
.venv/bin/python tools/make_release_linux.py --out build/release/TotK-Setup-linux --prove
|
||||
|
||||
The same model as the Windows release (tools/make_release.py, whose download cache, unpacking and checks this reuses): the
|
||||
folder carries the setup and a toolchain, never game code; the player's machine builds the game. Everything that runs on
|
||||
the player's machine is built against the Steam Runtime 3 "sniper" SDK sysroot (glibc 2.31), so the AppImage and the
|
||||
game it builds run on SteamOS and on any distribution at least as new. Needs network access on the first run, make,
|
||||
the .NET 10 SDK and Python 3.12+ with Pillow (.venv). The folder holds:
|
||||
|
||||
python/ python-build-standalone 3.14 (with tkinter) + the packages of tools/requirements-installer.txt; a .pth
|
||||
file puts the folder, tools/ and recomp/ on sys.path
|
||||
llvm/ LLVM 23 (clang, lld, llvm-ar, the clang resource headers) and llvm/sysroot: the files of the sniper SDK
|
||||
sysroot the game's compile and link read, nothing else (recorded while building the runtime library)
|
||||
runtime/ libtotkrt.a (build_game.py --make-runtime-lib --sysroot), the runtime headers, romfs.c, patches.c and
|
||||
ov_zstd.c (compiled per install), totk.png (the window icon), deps/ (Vulkan headers 1.4.363, VMA, zstd
|
||||
built static; SDL3, Opus and libvpx come from the sysroot)
|
||||
bin/ totk_xlate (+ .id), hactool, ninja, libSDL3.so.0 (the sysroot's; the game finds it beside itself)
|
||||
recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, totk.ini.example, VERSION
|
||||
AppRun, totk-setup.desktop, totk-setup.png the AppImage entry: python/bin/python3 -m installer.gui
|
||||
|
||||
--appimage packs the folder with appimagetool and the static type-2 runtime (no libfuse2 on the host). --prove compiles
|
||||
the generated C and links build/release-linux-check/totk with the folder's tools only, then runs the probes beside it.
|
||||
Like tools/make_release.py, both need one game version extracted and lifted first (generated/v121, else v143).
|
||||
|
||||
The folder contains nothing derived from the game: check_content runs over it before the script finishes, then the
|
||||
folder's own python runs a check-only installer pass and the window's --selftest.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import build_game # noqa: E402
|
||||
import make_release as mr # noqa: E402
|
||||
|
||||
ROOT = mr.ROOT
|
||||
MAIN_ROOT = mr.MAIN_ROOT
|
||||
WORK = mr.WORK
|
||||
CHECK = MAIN_ROOT / 'build' / 'release-linux-check'
|
||||
|
||||
LLVM_VERSION = '23.1.4'
|
||||
LLVM_MAJOR = LLVM_VERSION.split('.')[0]
|
||||
SNIPER = '3.0.20261021.266025'
|
||||
SNIPER_URL = 'https://repo.steampowered.com/steamrt-images-sniper/snapshots'
|
||||
PINS = {
|
||||
'llvm-linux': (f'https://github.com/llvm/llvm-project/releases/download/llvmorg-{LLVM_VERSION}/'
|
||||
f'LLVM-{LLVM_VERSION}-Linux-X64.tar.xz',
|
||||
'25d814fda50d5fb0084b42be09605d8b1595d49d5e2e03a3345d7e112ff6f9f3'),
|
||||
'llvm-license': (f'https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-{LLVM_VERSION}/llvm/LICENSE.TXT',
|
||||
'8d85c1057d742e597985c7d4e6320b015a9139385cff4cbae06ffc0ebe89afee'), # the binary release carries none
|
||||
'sniper-sdk': (f'{SNIPER_URL}/{SNIPER}/com.valvesoftware.SteamRuntime.Sdk-amd64%2Ci386-sniper-sysroot.tar.gz',
|
||||
'8f71947035f97925a311b2ed753f22efcf2f13d6f1631945b8e537f8903ce2cf'),
|
||||
'python-linux': ('https://github.com/astral-sh/python-build-standalone/releases/download/20261026/'
|
||||
'cpython-3.14.10%2B20261026-x86_64-unknown-linux-gnu-install_only.tar.gz',
|
||||
'1b38024924eb7f4cf0cab45d7b871d1f1991354190fccd2625ac7529a43786e2'),
|
||||
'ninja-linux': ('https://github.com/ninja-build/ninja/releases/download/v1.13.2/ninja-linux.zip',
|
||||
'5749cbc4e668273514150a80e387a957f933c6ed3f5f11e03fb30955e2bbead6'),
|
||||
'zstd-src': ('https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz',
|
||||
'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3'),
|
||||
'vulkan-headers-src': ('https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.4.363.tar.gz',
|
||||
'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68'),
|
||||
'vma-header': build_game.VMA_FETCH,
|
||||
# appimagetool and the runtime are published only under the moving "continuous" tag: pinned by hash, so a new
|
||||
# upload stops the build until the pins are updated on purpose
|
||||
'appimagetool': ('https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage',
|
||||
'793fca0ce4bcc309dc65d3f66669f58e3755643e4dadbe03932e7b436128494f'),
|
||||
'appimage-runtime': ('https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-x86_64',
|
||||
'617aa77314d57c8b8e4fae9c3f0b35af7b30d5529cb69e98f64f92510ed98a46'),
|
||||
}
|
||||
mr.PINS.update(PINS) # one cache (toolchain/release-dl) and one fetch for both releases
|
||||
|
||||
# What clang, lld and the runtime build need from the LLVM release (the rest of its 8 GB stays packed).
|
||||
LLVM_MEMBERS = [f'bin/clang-{LLVM_MAJOR}', 'bin/clang', 'bin/clang++', 'bin/lld', 'bin/ld.lld', 'bin/llvm-ar',
|
||||
'bin/llvm-ranlib', 'bin/llvm-objcopy', 'bin/llvm-strip', f'lib/clang/{LLVM_MAJOR}/include']
|
||||
# The parts of the sniper SDK a C/C++ compile and link for x86_64 can read.
|
||||
SNIPER_MEMBERS = ['usr/include', 'usr/lib/x86_64-linux-gnu', 'usr/lib/gcc/x86_64-linux-gnu',
|
||||
'lib/x86_64-linux-gnu', 'lib64', 'usr/share/doc/libsdl3-0', 'usr/share/doc/libopus0',
|
||||
'usr/share/doc/libvpx6', 'usr/share/doc/libc6', 'usr/share/doc/gcc-10-base',
|
||||
'usr/share/doc/libvulkan1']
|
||||
# glibc's link libraries and their scripts (lib<name>.so, .a, _nonshared.a), and the GCC runtime pieces clang links.
|
||||
GLIBC_LIBS = ['c', 'm', 'mvec', 'dl', 'pthread', 'rt', 'util']
|
||||
GCC_LIBS = {'libgcc.a', 'libgcc_eh.a', 'libgcc_s.so', 'libstdc++.a', 'libsupc++.a'}
|
||||
# The libraries of a release link (build_game.py LINK_LIBS with --deps: Opus, libvpx and zstd static).
|
||||
RELEASE_LIBS = ['-lvulkan', '-lSDL3', '-l:libopus.a', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-lm',
|
||||
'-lpthread', '-ldl']
|
||||
|
||||
|
||||
def run(cmd, **kw):
|
||||
return mr.run(cmd, **kw)
|
||||
|
||||
|
||||
def tar_extract(name, dest, members, strip=0):
|
||||
"""Extract only members (path prefixes inside the archive) of a pinned tarball with the system tar, once."""
|
||||
stamp = dest / '.unpacked'
|
||||
if stamp.is_file():
|
||||
return dest
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
dest.mkdir(parents=True)
|
||||
path = mr.fetch(name)
|
||||
t0 = time.time()
|
||||
run(['tar', '-xf', path, '-C', dest, f'--strip-components={strip}', '--wildcards', '--no-wildcards-match-slash',
|
||||
*members])
|
||||
print(f'{name}: unpacked in {time.time() - t0:.0f} s')
|
||||
stamp.write_text('')
|
||||
return dest
|
||||
|
||||
|
||||
# ---- llvm/ and the sysroot -------------------------------------------------------------------------------------------
|
||||
|
||||
def build_llvm(out):
|
||||
top = f'LLVM-{LLVM_VERSION}-Linux-X64'
|
||||
src = tar_extract('llvm-linux', WORK / 'llvm-linux', [f'{top}/{m}' for m in LLVM_MEMBERS], strip=1)
|
||||
dst = out / 'llvm'
|
||||
shutil.copytree(src, dst, symlinks=True, ignore=shutil.ignore_patterns('.unpacked'))
|
||||
strip(dst / 'bin', dst / 'bin' / 'llvm-strip')
|
||||
r = subprocess.run([str(dst / 'bin' / 'clang'), '--version'], capture_output=True, text=True)
|
||||
if r.returncode:
|
||||
sys.exit(f'{dst / "bin" / "clang"} does not start:\n{r.stderr}')
|
||||
print(r.stdout.splitlines()[0])
|
||||
return dst
|
||||
|
||||
|
||||
def strip(where, tool):
|
||||
"""Drop the debug symbols of the ELF files under where (the LLVM and Python releases ship them: clang 270 MB)."""
|
||||
for p in sorted(where.rglob('*')):
|
||||
if p.is_file() and not p.is_symlink() and p.read_bytes()[:4] == b'\x7fELF':
|
||||
subprocess.run([str(tool), '--strip-unneeded', str(p)], check=True)
|
||||
|
||||
|
||||
def full_sysroot():
|
||||
"""The sniper SDK's x86_64 compile and link files in toolchain/release-work/sniper, absolute symlinks made relative
|
||||
(the SDK points some at /lib and /usr/lib, which outside its container are the host's)."""
|
||||
root = WORK / 'sniper'
|
||||
if (root / '.relinked').is_file():
|
||||
return root
|
||||
tar_extract('sniper-sdk', root, SNIPER_MEMBERS)
|
||||
for p in root.rglob('*'):
|
||||
if p.is_symlink() and os.readlink(p).startswith('/'):
|
||||
target = root / os.readlink(p).lstrip('/')
|
||||
p.unlink()
|
||||
p.symlink_to(os.path.relpath(target, p.parent))
|
||||
(root / '.relinked').write_text('')
|
||||
return root
|
||||
|
||||
|
||||
def sysroot_cc(llvm, sysroot):
|
||||
return f'{llvm / "bin" / "clang"} --target=x86_64-linux-gnu --sysroot={sysroot} -fuse-ld=lld'
|
||||
|
||||
|
||||
# ---- runtime/deps, bin/ --------------------------------------------------------------------------------------------
|
||||
|
||||
def build_deps(out, llvm, sysroot):
|
||||
d = out / 'runtime' / 'deps'
|
||||
(d / 'include' / 'vma').mkdir(parents=True, exist_ok=True)
|
||||
(d / 'lib').mkdir(parents=True, exist_ok=True)
|
||||
vk = tar_extract('vulkan-headers-src', WORK / 'vulkan-headers', ['Vulkan-Headers-1.4.363/include',
|
||||
'Vulkan-Headers-1.4.363/LICENSE.md'], strip=1)
|
||||
for sub in ('vulkan', 'vk_video'):
|
||||
shutil.copytree(vk / 'include' / sub, d / 'include' / sub)
|
||||
shutil.copy2(mr.fetch('vma-header'), d / 'include' / 'vma' / 'vk_mem_alloc.h')
|
||||
zs = tar_extract('zstd-src', WORK / 'zstd', ['zstd-1.5.7'], strip=1)
|
||||
lib = zs / 'lib'
|
||||
if not (lib / 'libzstd.a').is_file():
|
||||
run(['make', '-C', lib, f'-j{os.cpu_count() or 4}', 'libzstd.a', f'CC={sysroot_cc(llvm, sysroot)}',
|
||||
f'AR={llvm / "bin" / "llvm-ar"}', 'ZSTD_LEGACY_SUPPORT=0', 'MOREFLAGS=-O2'], stdout=subprocess.DEVNULL)
|
||||
shutil.copy2(lib / 'libzstd.a', d / 'lib' / 'libzstd.a')
|
||||
for name in ('zstd.h', 'zstd_errors.h'):
|
||||
shutil.copy2(lib / name, d / 'include' / name)
|
||||
|
||||
|
||||
def build_bin(out, llvm, sysroot):
|
||||
b = out / 'bin'
|
||||
b.mkdir(parents=True, exist_ok=True)
|
||||
zip_dir = mr.unpack('ninja-linux')
|
||||
shutil.copy2(zip_dir / 'ninja', b / 'ninja')
|
||||
(b / 'ninja').chmod(0o755)
|
||||
hac = WORK / 'hactool-linux'
|
||||
run([sys.executable, ROOT / 'tools' / 'build_hactool.py', '--src', WORK / 'hactool-src', '--out', hac,
|
||||
'--cc', sysroot_cc(llvm, sysroot)])
|
||||
shutil.copy2(hac / 'hactool', b / 'hactool')
|
||||
xl = WORK / 'xlate-linux'
|
||||
run([sys.executable, ROOT / 'tools' / 'build_xlate.py', '--out', xl, '--llvm', llvm, '--sysroot', sysroot])
|
||||
shutil.copy2(xl / 'totk_xlate', b / 'totk_xlate')
|
||||
shutil.copy2(xl / 'totk_xlate.id', b / 'totk_xlate.id')
|
||||
sdl = (sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu' / 'libSDL3.so.0').resolve()
|
||||
shutil.copy2(sdl, b / 'libSDL3.so.0')
|
||||
|
||||
|
||||
# ---- runtime/ ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
def runtime_lib(out, llvm, sysroot):
|
||||
build = CHECK / 'rtlib'
|
||||
shutil.rmtree(build, ignore_errors=True)
|
||||
version = mr.lifted_version()
|
||||
run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--sysroot', sysroot,
|
||||
'--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja', '--version', version,
|
||||
'--gen', MAIN_ROOT / 'generated' / version, '--out', build])
|
||||
rt = out / 'runtime'
|
||||
shutil.copy2(build / 'libtotkrt.a', rt / 'libtotkrt.a')
|
||||
for p in sorted((ROOT / 'runtime').rglob('*')):
|
||||
if p.suffix in mr.RUNTIME_SUFFIXES and p.is_file():
|
||||
(rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, rt / p.relative_to(ROOT / 'runtime'))
|
||||
for name in build_game.INSTALL_SIDE:
|
||||
shutil.copy2(ROOT / 'runtime' / name, rt / name)
|
||||
from PIL import Image # Tk loads no .ico off Windows: the window and the AppImage take a PNG
|
||||
with Image.open(ROOT / 'runtime' / 'totk.ico') as ico:
|
||||
ico.size = max(ico.ico.sizes())
|
||||
ico.convert('RGBA').save(rt / 'totk.png')
|
||||
return build
|
||||
|
||||
|
||||
def used_sysroot_files(out, llvm, sysroot, rtlib_build):
|
||||
"""The sysroot files a game build reads: the headers of the runtime's compile (its .d files), of the install-side
|
||||
sources and of hostcheck, and every library and object of a link (lld --trace) of the runtime library with the
|
||||
flags and libraries build_game.py uses, its references to the generated code left unresolved."""
|
||||
used = set()
|
||||
for dep in rtlib_build.rglob('*.d'):
|
||||
used.update(re.findall(r'\S+', dep.read_text(encoding='utf-8', errors='replace').replace('\\\n', ' ')))
|
||||
probe = CHECK / 'sysroot-probe'
|
||||
shutil.rmtree(probe, ignore_errors=True)
|
||||
probe.mkdir(parents=True)
|
||||
cc = [llvm / 'bin' / 'clang', '--target=x86_64-linux-gnu', f'--sysroot={sysroot}', '-I', out / 'runtime',
|
||||
'-I', out / 'runtime' / 'deps' / 'include', '-I', CHECK / 'rtlib']
|
||||
for i, src in enumerate([*(out / 'runtime' / n for n in sorted(build_game.INSTALL_SIDE)),
|
||||
ROOT / 'installer' / 'hostcheck.c']):
|
||||
# romfs.c needs the per-install romfs_config.h: whatever it reads up to there still lands in the .d file
|
||||
subprocess.run([str(c) for c in [*cc, '-fsyntax-only', '-MD', '-MF', probe / f'{i}.d', src]],
|
||||
capture_output=True)
|
||||
if (probe / f'{i}.d').is_file():
|
||||
used.update(re.findall(r'\S+', (probe / f'{i}.d').read_text().replace('\\\n', ' ')))
|
||||
trace = subprocess.run([str(c) for c in [
|
||||
llvm / 'bin' / 'clang', '--target=x86_64-linux-gnu', f'--sysroot={sysroot}', *build_game.LINK_FLAGS[:3],
|
||||
'-Wl,--trace', '-Wl,--unresolved-symbols=ignore-all', '-Wl,--whole-archive', out / 'runtime' / 'libtotkrt.a',
|
||||
'-Wl,--no-whole-archive', '-L', out / 'runtime' / 'deps' / 'lib',
|
||||
*RELEASE_LIBS, '-o', probe / 'trace.out']],
|
||||
capture_output=True, text=True)
|
||||
if trace.returncode:
|
||||
sys.exit(f'sysroot trace link failed:\n{trace.stderr[-3000:]}')
|
||||
used.update(line.split('(')[0] for line in trace.stdout.splitlines())
|
||||
root = str(sysroot.resolve())
|
||||
files = set()
|
||||
for p in used:
|
||||
path = Path(p)
|
||||
if not path.is_absolute():
|
||||
continue
|
||||
for q in (path, path.resolve()): # the name read and, through symlinks, the file
|
||||
if str(q).startswith(root + '/') and q.exists():
|
||||
files.add(q)
|
||||
# The C library and kernel headers whole: which of them a compile reads depends on its flags (at -O1 glibc's inline
|
||||
# variants pull in bits/stdio.h and others), and the setup compiles the generated code, hostcheck and the
|
||||
# install-side sources with flags of its own
|
||||
inc = sysroot / 'usr' / 'include'
|
||||
arch = inc / 'x86_64-linux-gnu'
|
||||
files.update(p for p in [*inc.glob('*.h'), *arch.glob('*.h')] if p.is_file())
|
||||
for d in (arch / 'bits', arch / 'sys', arch / 'gnu', arch / 'asm', inc / 'linux', inc / 'asm-generic'):
|
||||
files.update(p for p in d.rglob('*') if p.is_file() or p.is_symlink())
|
||||
# The C runtime whole, for PIE links (hostcheck) as well as the game's: lld's trace names the libraries a link
|
||||
# script loads but not the script (libc.so, libm.so, libgcc_s.so), and clang's GCC detection needs the crt objects
|
||||
lib = sysroot / 'usr' / 'lib' / 'x86_64-linux-gnu'
|
||||
for name in GLIBC_LIBS:
|
||||
files.update(p for p in lib.glob(f'lib{name}[._]*') if p.suffix in ('.so', '.a') or '.so' in p.name)
|
||||
files.update(p for p in (sysroot / 'lib' / 'x86_64-linux-gnu').glob(f'lib{name}.so*'))
|
||||
files.update(lib.glob('*crt*.o'))
|
||||
files.update((sysroot / 'lib64').glob('ld-linux*'))
|
||||
files.update((sysroot / 'lib' / 'x86_64-linux-gnu').glob('ld-linux*'))
|
||||
files.update((sysroot / 'lib' / 'x86_64-linux-gnu').glob('libgcc_s.so*'))
|
||||
gcc = next((sysroot / 'usr' / 'lib' / 'gcc' / 'x86_64-linux-gnu').iterdir())
|
||||
files.update(p for p in gcc.iterdir() if p.name.startswith('crt') or p.name in GCC_LIBS)
|
||||
return files
|
||||
|
||||
|
||||
def link_chain(path):
|
||||
"""path and every link on the way to its file: libvulkan.so -> libvulkan.so.1 -> libvulkan.so.1.4.309."""
|
||||
hops = [path]
|
||||
while hops[-1].is_symlink() and len(hops) < 16:
|
||||
hops.append(hops[-1].parent / os.readlink(hops[-1]))
|
||||
return hops
|
||||
|
||||
|
||||
def build_sysroot(out, sysroot, files):
|
||||
dst = out / 'llvm' / 'sysroot'
|
||||
files = {Path(os.path.normpath(h)) for f in files for h in link_chain(f)}
|
||||
for f in sorted(files):
|
||||
rel = f.relative_to(sysroot)
|
||||
(dst / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||
if f.is_symlink():
|
||||
if not (dst / rel).is_symlink():
|
||||
(dst / rel).symlink_to(os.readlink(f))
|
||||
elif not (dst / rel).exists():
|
||||
shutil.copy2(f, dst / rel)
|
||||
# a symlink copied above may name a file the build reads only through it
|
||||
for link in [p for p in dst.rglob('*') if p.is_symlink()]:
|
||||
target = (sysroot / link.relative_to(dst)).resolve()
|
||||
mirrored = dst / target.relative_to(sysroot.resolve())
|
||||
if not mirrored.exists():
|
||||
mirrored.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(target, mirrored)
|
||||
print(f'sysroot: {sum(1 for p in dst.rglob("*") if p.is_file())} files, {mr.mib(mr.dir_size(dst))}')
|
||||
|
||||
|
||||
# ---- python/ ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
def build_python(out):
|
||||
src = mr.unpack('python-linux') / 'python'
|
||||
py = out / 'python'
|
||||
shutil.copytree(src, py, symlinks=True,
|
||||
ignore=shutil.ignore_patterns('test', 'idlelib', 'turtledemo', '__pycache__', '*.a'))
|
||||
exe = py / 'bin' / 'python3'
|
||||
env = dict(os.environ, PIP_DISABLE_PIP_VERSION_CHECK='1')
|
||||
run([exe, '-m', 'pip', 'install', '--no-compile', '--only-binary', ':all:', '-r',
|
||||
ROOT / 'tools' / 'requirements-installer.txt'], env=env, stdout=subprocess.DEVNULL)
|
||||
run([exe, '-m', 'pip', 'uninstall', '-y', 'pip'], stdout=subprocess.DEVNULL) # the setup installs nothing
|
||||
strip(py, out / 'llvm' / 'bin' / 'llvm-strip')
|
||||
site = next(py.glob('lib/python3.*/site-packages'))
|
||||
# The release folder, tools/ and recomp/ on sys.path, as the Windows python's ._pth does: `python -m installer` and
|
||||
# the tools importing their siblings work from any directory
|
||||
(site / 'totk-release.pth').write_text('../../../..\n../../../../tools\n../../../../recomp\n', encoding='utf-8')
|
||||
check = subprocess.run([str(exe), '-c', 'import tkinter, numpy, lz4.block, cryptography, cffi, pycparser; '
|
||||
'print("python ok: Tcl", tkinter.Tcl().eval("info patchlevel"), "numpy", numpy.__version__)'],
|
||||
cwd='/', capture_output=True, text=True)
|
||||
if check.returncode:
|
||||
sys.exit('bundled python check failed:\n' + check.stderr)
|
||||
print(check.stdout.strip())
|
||||
|
||||
|
||||
# ---- licences, AppImage entry ------------------------------------------------------------------------------------------
|
||||
|
||||
def collect_licenses(out, sysroot):
|
||||
lic = out / 'licenses'
|
||||
lic.mkdir(parents=True, exist_ok=True)
|
||||
doc = sysroot / 'usr' / 'share' / 'doc'
|
||||
pairs = [
|
||||
(mr.fetch('llvm-license'), 'llvm-LICENSE.txt'),
|
||||
(doc / 'libsdl3-0' / 'copyright', 'SDL3-copyright.txt'),
|
||||
(doc / 'libopus0' / 'copyright', 'opus-copyright.txt'),
|
||||
(doc / 'libvpx6' / 'copyright', 'libvpx-copyright.txt'),
|
||||
(doc / 'libc6' / 'copyright', 'glibc-copyright.txt'),
|
||||
(doc / 'gcc-10-base' / 'copyright', 'libstdc++-copyright.txt'),
|
||||
(doc / 'libvulkan1' / 'copyright', 'vulkan-loader-copyright.txt'),
|
||||
(WORK / 'vulkan-headers' / 'LICENSE.md', 'vulkan-headers-LICENSE.txt'),
|
||||
(WORK / 'zstd' / 'LICENSE', 'zstd-LICENSE.txt'),
|
||||
(out / 'python' / 'lib' / f'python{sys_version(out)}' / 'LICENSE.txt', 'python-LICENSE.txt'),
|
||||
(ROOT / 'runtime' / 'shaders' / 'ffx' / 'LICENSE.txt', 'FidelityFX-FSR-LICENSE.txt'),
|
||||
(WORK / 'hactool-linux' / 'LICENSE', 'hactool-LICENSE.txt'),
|
||||
(MAIN_ROOT / 'refs' / 'MacNativeGames' / 'LICENSE', 'MacNativeGames-LICENSE.txt'),
|
||||
]
|
||||
for src, name in pairs:
|
||||
shutil.copy2(src, lic / name)
|
||||
import tarfile # Debian's libvpx copyright carries no patent grant: the WebM PATENTS text from the pinned source tag
|
||||
with tarfile.open(mr.fetch('libvpx-src')) as t:
|
||||
(lic / 'libvpx-PATENTS.txt').write_bytes(t.extractfile('libvpx-1.17.0/PATENTS').read())
|
||||
head = (out / 'runtime' / 'deps' / 'include' / 'vma' / 'vk_mem_alloc.h').read_text(encoding='utf-8').split('*/')[0]
|
||||
(lic / 'vulkan-memory-allocator-LICENSE.txt').write_text(head.lstrip('/*') + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def sys_version(out):
|
||||
return next((out / 'python' / 'lib').glob('python3.*')).name.removeprefix('python')
|
||||
|
||||
|
||||
APPRUN = '''#!/bin/sh
|
||||
# The AppImage entry: the setup window, run by the bundled Python from the read-only mount (no bytecode written there).
|
||||
HERE="$(dirname "$(readlink -f "$0")")"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec "$HERE/python/bin/python3" -m installer.gui "$@"
|
||||
'''
|
||||
|
||||
DESKTOP = '''[Desktop Entry]
|
||||
Type=Application
|
||||
Name=TotK Recomp Setup
|
||||
Comment=Builds Tears of the Kingdom for this PC from your own game files
|
||||
Exec=AppRun
|
||||
Icon=totk-setup
|
||||
Categories=Game;
|
||||
Terminal=false
|
||||
'''
|
||||
|
||||
|
||||
def build_entry(out):
|
||||
(out / 'AppRun').write_text(APPRUN, encoding='utf-8')
|
||||
(out / 'AppRun').chmod(0o755)
|
||||
(out / 'totk-setup.desktop').write_text(DESKTOP, encoding='utf-8')
|
||||
shutil.copy2(out / 'runtime' / 'totk.png', out / 'totk-setup.png')
|
||||
(out / '.DirIcon').symlink_to('totk-setup.png')
|
||||
|
||||
|
||||
def write_appimage(out, path):
|
||||
tool = mr.fetch('appimagetool')
|
||||
tool.chmod(0o755)
|
||||
runtime = mr.fetch('appimage-runtime')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
env = dict(os.environ, ARCH='x86_64', APPIMAGE_EXTRACT_AND_RUN='1')
|
||||
run([tool, '--no-appstream', '--runtime-file', runtime, out, path], env=env)
|
||||
print(f'{path}: {mr.mib(path.stat().st_size)}')
|
||||
|
||||
|
||||
# ---- checks -------------------------------------------------------------------------------------------------------------
|
||||
|
||||
# The newest glibc any program or library in the folder may need: the official LLVM binaries and the cryptography
|
||||
# wheel are built against 2.34 (Ubuntu 22.04, Debian 12, SteamOS 3 have it); the game itself links the sysroot's 2.31.
|
||||
GLIBC_FLOOR = (2, 34)
|
||||
|
||||
|
||||
def check_glibc(out):
|
||||
"""Every ELF file in the folder (llvm/sysroot aside: glibc itself, read by the link) needs glibc GLIBC_FLOOR or
|
||||
older; prints which files set the floor."""
|
||||
newest, by = (0, 0), []
|
||||
for p in sorted(out.rglob('*')):
|
||||
if not p.is_file() or p.is_symlink() or out / 'llvm' / 'sysroot' in p.parents:
|
||||
continue
|
||||
with open(p, 'rb') as f:
|
||||
if f.read(4) != b'\x7fELF':
|
||||
continue
|
||||
text = subprocess.run(['readelf', '-V', str(p)], capture_output=True, text=True).stdout
|
||||
need = max((tuple(map(int, v.split('.'))) for v in re.findall(r'GLIBC_(\d+\.\d+(?:\.\d+)?)', text)),
|
||||
default=(0, 0))
|
||||
if need > newest:
|
||||
newest, by = need, []
|
||||
if need == newest:
|
||||
by.append(str(p.relative_to(out)))
|
||||
floor = '.'.join(map(str, GLIBC_FLOOR))
|
||||
if newest > GLIBC_FLOOR:
|
||||
sys.exit(f'glibc check: {", ".join(by[:8])} need glibc {".".join(map(str, newest))}, above {floor}')
|
||||
print(f'glibc check: the folder needs glibc {".".join(map(str, newest))} (floor {floor}), set by '
|
||||
f'{", ".join(by[:6])}{" and others" if len(by) > 6 else ""}')
|
||||
|
||||
|
||||
def check_bundle(out):
|
||||
"""The folder's own python runs the installer without game files and the window's selftest."""
|
||||
import json
|
||||
chk = CHECK / 'bundle'
|
||||
shutil.rmtree(chk, ignore_errors=True)
|
||||
py = out / 'python' / 'bin' / 'python3'
|
||||
env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1')
|
||||
r = subprocess.run([str(py), '-m', 'installer', '--lang', 'en-US', '--dest', str(chk / 'install'), '--temp',
|
||||
str(chk / 'temp'), '--root', str(out)], cwd='/', env=env, capture_output=True, text=True,
|
||||
timeout=600)
|
||||
events = [json.loads(line) for line in r.stdout.splitlines() if line.startswith('{')]
|
||||
errors = [e['id'] for e in events if e['type'] == 'error']
|
||||
started = any(e['type'] == 'start' and e.get('release') == (out / 'VERSION').read_text().strip() for e in events)
|
||||
steps = [e['phase'] for e in events if e['type'] == 'progress']
|
||||
if errors != ['check.inputs_needed'] or not started or 'inputs' not in steps:
|
||||
sys.exit(f'bundle check: exit {r.returncode}, errors {errors}, release read {started}\n{r.stdout[-2000:]}\n'
|
||||
f'{r.stderr[-2000:]}')
|
||||
print(f'bundle check: the installer ran the host check from the folder and stopped at the game files '
|
||||
f'(exit {r.returncode})')
|
||||
r = subprocess.run([str(py), '-m', 'installer.gui', '--selftest', '--root', str(out)], cwd='/', env=env,
|
||||
capture_output=True, text=True, timeout=300)
|
||||
if r.returncode:
|
||||
sys.exit(f'bundle check: window selftest exit {r.returncode}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}')
|
||||
print('bundle check: the window built every screen in both languages and closed')
|
||||
|
||||
|
||||
def prove(out):
|
||||
"""Compile the generated C and link totk with the folder's tools only, then run the probes beside it."""
|
||||
out = out.resolve()
|
||||
version = mr.lifted_version()
|
||||
gen = MAIN_ROOT / 'generated' / version
|
||||
ext = MAIN_ROOT / 'build' / 'extracted' / version
|
||||
chk = CHECK / 'game'
|
||||
env = dict(os.environ, PATH=os.pathsep.join([str(out / 'llvm' / 'bin'), str(out / 'bin'), '/usr/bin', '/bin']))
|
||||
t0 = time.time()
|
||||
run([out / 'python' / 'bin' / 'python3', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm',
|
||||
'--sysroot', out / 'llvm' / 'sysroot', '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime',
|
||||
'--runtime-lib', out / 'runtime' / 'libtotkrt.a', '--ninja', out / 'bin' / 'ninja', '--version', version,
|
||||
'--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', '--profile', chk / 'profile'],
|
||||
env=env)
|
||||
print(f'prove: build {time.time() - t0:.0f} s')
|
||||
shutil.copy2(out / 'bin' / 'libSDL3.so.0', chk / 'libSDL3.so.0')
|
||||
needed = subprocess.run(['readelf', '-d', str(chk / 'totk')], capture_output=True, text=True).stdout
|
||||
print('prove: needs', ', '.join(re.findall(r'\[(lib[^\]]+)\]', needed)))
|
||||
glibc = sorted(set(re.findall(r'GLIBC_(\d+\.\d+)', subprocess.run(['readelf', '-V', str(chk / 'totk')],
|
||||
capture_output=True, text=True).stdout)),
|
||||
key=lambda v: tuple(map(int, v.split('.'))))
|
||||
print(f'prove: newest glibc symbol version {glibc[-1] if glibc else "none"}')
|
||||
failed = []
|
||||
for probe in ['--selftest', *mr.PROBES]:
|
||||
r = subprocess.run([str(chk / 'totk'), probe], cwd=chk, capture_output=True, text=True, timeout=900)
|
||||
tail = (r.stdout.strip().splitlines() or [''])[-1]
|
||||
print(f'{probe}: exit {r.returncode}: {tail}')
|
||||
if r.returncode:
|
||||
failed.append(probe)
|
||||
if failed:
|
||||
sys.exit(f'probes failed: {failed}')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
|
||||
ap.add_argument('--appimage', type=Path, help='also pack the folder as this AppImage')
|
||||
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
|
||||
args = ap.parse_args()
|
||||
out = args.out.resolve()
|
||||
if args.prove:
|
||||
prove(out)
|
||||
return
|
||||
if out.exists():
|
||||
shutil.rmtree(out)
|
||||
out.mkdir(parents=True)
|
||||
llvm = build_llvm(out)
|
||||
sysroot = full_sysroot()
|
||||
build_deps(out, llvm, sysroot)
|
||||
build_bin(out, llvm, sysroot)
|
||||
rtlib = runtime_lib(out, llvm, sysroot)
|
||||
build_sysroot(out, sysroot, used_sysroot_files(out, llvm, sysroot, rtlib))
|
||||
build_python(out)
|
||||
mr.build_sources(out)
|
||||
collect_licenses(out, sysroot)
|
||||
build_entry(out)
|
||||
mr.check_folder(out)
|
||||
check_glibc(out)
|
||||
check_bundle(out)
|
||||
mr.sizes(out)
|
||||
if args.appimage:
|
||||
write_appimage(out, args.appimage.resolve())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -11,10 +11,11 @@ captured at once (upstream checked weapon, shield, material and ability wheel at
|
||||
The game-over flow fix of the same upstream script is 1.0.0 only (1.2.1 and 1.4.3 run DmF_SY_GameOver.110) and is
|
||||
left out.
|
||||
|
||||
--aspect <w>x<h> (a game resolution wider than 16:9, such as 3440x1440) also lays the UI out for that shape
|
||||
(tools/ultrawide_layout.py): menus stay 16:9 in the middle, full-screen backgrounds cover the width and the HUD moves
|
||||
to the screen's corners. totk.exe loads that overlay from romfs-patches-<w>x<h> when the game resolution is that size
|
||||
(runtime/romfs.c overlay_init); the camera's side of it is the patch set aspect (recomp/patches-v143.txt).
|
||||
--aspect <w>x<h> (a game resolution of another shape than 16:9, such as 3440x1440 or 1280x800) also lays the UI out
|
||||
for that shape (tools/ultrawide_layout.py): menus keep their 16:9 proportions in the middle, full-screen backgrounds
|
||||
cover the screen and the HUD moves to the screen's corners (wider) or its top and bottom edges (narrower). totk.exe
|
||||
loads that overlay from romfs-patches-<w>x<h> when the game resolution is that size (runtime/romfs.c overlay_init);
|
||||
the camera's side of it is the patch set aspect (recomp/patches-v143.txt).
|
||||
|
||||
--prompts xbox | xbox-label: the same overlay with Xbox button icons in the game's prompts, for prompts = xbox
|
||||
(runtime/romfs.c overlay_init loads romfs-patches-xbox or romfs-patches-xbox-label in place of romfs-patches). Drawn
|
||||
@@ -56,7 +57,7 @@ ap.add_argument('--version', choices=('v121', 'v143'), default='v121')
|
||||
ap.add_argument('--romfs', type=Path, help='default build/extracted/<version>/romfs')
|
||||
ap.add_argument('--out', type=Path,
|
||||
help='default build/<version>/romfs-patches (romfs-patches-<style> with --prompts)')
|
||||
ap.add_argument('--aspect', help='<w>x<h> of a game resolution wider than 16:9: also lay the UI out for it')
|
||||
ap.add_argument('--aspect', help='<w>x<h> of a game resolution of another shape than 16:9: also lay the UI out for it')
|
||||
ap.add_argument('--prompts', choices=('xbox', 'xbox-label'),
|
||||
help='Xbox button icons in the prompts: xbox for pad_layout = nintendo (A shown as B, X as Y), '
|
||||
'xbox-label for pad_layout = xbox or auto (the face buttons keep their letters)')
|
||||
|
||||
@@ -20,7 +20,9 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DLLS = ('SDL3.dll', 'libopus-0.dll', 'libiconv-2.dll')
|
||||
EXE = '.exe' if os.name == 'nt' else ''
|
||||
# A Linux source build links the system's SDL3, Opus and libvpx: the other machine needs them installed
|
||||
DLLS = ('SDL3.dll', 'libopus-0.dll', 'libiconv-2.dll') if os.name == 'nt' else ()
|
||||
# Prepared state next to the exe: translator answers, programs, the prepared list, pipeline cache and its keys, and the
|
||||
# variant lists `totk.exe --prepare-shaders` reads (the driver keys its library cache by exe path, so each copy runs it).
|
||||
STATE = ('romfs-index.bin', 'pipeline-cache.bin', 'pipeline-keys.bin', 'shader-answers.bin', 'shader-programs.bin',
|
||||
@@ -51,7 +53,7 @@ def main():
|
||||
ap.add_argument('--version', default='v121', help='game version (v121 or v143): the defaults below follow it')
|
||||
ap.add_argument('--build', type=Path, help='build dir with the prepared state (default build/<version>)')
|
||||
ap.add_argument('--exe', type=Path, help='totk.exe to package (default: <build>/totk.exe); its DLLs are taken from --build')
|
||||
ap.add_argument('--xlate', type=Path, default=ROOT / 'build' / 'xlate' / 'totk_xlate.exe')
|
||||
ap.add_argument('--xlate', type=Path, default=ROOT / 'build' / 'xlate' / f'totk_xlate{EXE}')
|
||||
ap.add_argument('--extracted', type=Path, help='default build/extracted/<version>')
|
||||
ap.add_argument('--out', type=Path, help='default build/<version>-portable')
|
||||
ap.add_argument('--with-romfs', action='store_true', help='also copy romfs.bin (about 16 GB)')
|
||||
@@ -62,7 +64,7 @@ def main():
|
||||
a.out = a.out or ROOT / 'build' / f'{a.version}-portable'
|
||||
|
||||
build, out = a.build.resolve(), a.out.resolve()
|
||||
exe = (a.exe or build / 'totk.exe').resolve()
|
||||
exe = (a.exe or build / f'totk{EXE}').resolve()
|
||||
fonts = a.extracted.resolve().parent / 'firmware' / 'shared-fonts'
|
||||
exefs = a.extracted.resolve() / 'exefs'
|
||||
romfs = a.extracted.resolve() / 'romfs' / 'romfs.bin'
|
||||
@@ -78,10 +80,10 @@ def main():
|
||||
print('package_build: warning: the folder holds game code; keep it out of the repository')
|
||||
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
copy(exe, out / 'totk.exe')
|
||||
copy(exe, out / f'totk{EXE}')
|
||||
for d in DLLS:
|
||||
copy(build / d, out / d)
|
||||
copy(a.xlate, out / 'totk_xlate.exe')
|
||||
copy(a.xlate, out / f'totk_xlate{EXE}')
|
||||
idf = a.xlate.with_suffix('.id')
|
||||
if idf.exists():
|
||||
copy(idf, out / 'totk_xlate.id')
|
||||
@@ -104,7 +106,7 @@ def main():
|
||||
print('package_build: copying romfs.bin (16 GB) ...')
|
||||
shutil.copyfile(romfs, out / 'romfs' / 'romfs.bin')
|
||||
print(f'package_build: {out}: {n} files in folders, {size_of(out) / 2**30:.2f} GiB'
|
||||
+ ('' if a.with_romfs else '; no RomFS: put romfs.bin into romfs\\ or set TOTK_ROMFS'))
|
||||
+ ('' if a.with_romfs else f'; no RomFS: put romfs.bin into romfs{os.sep} or set TOTK_ROMFS'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -28,6 +28,7 @@ bundles it; it stays out of the repository), else from refs/ through git with th
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -183,7 +184,7 @@ def main():
|
||||
f'seed written in {time.monotonic() - t0:.1f} s', flush=True)
|
||||
if a.no_run:
|
||||
return 0
|
||||
exe = out / 'totk.exe'
|
||||
exe = out / ('totk.exe' if os.name == 'nt' else 'totk')
|
||||
r = subprocess.run([str(exe), '--prepare-shaders'], cwd=str(out))
|
||||
print(f'totk.exe --prepare-shaders: rc={r.returncode}; total {time.monotonic() - t0:.1f} s')
|
||||
return r.returncode
|
||||
|
||||
@@ -21,7 +21,9 @@ import compression.zstd as zstd
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -32,8 +34,8 @@ from pathlib import Path
|
||||
import lz4.block
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
XLATE = ROOT / 'build' / 'xlate' / 'totk_xlate.exe'
|
||||
SPIRV_VAL = Path(r'C:\msys64\ucrt64\bin\spirv-val.exe')
|
||||
XLATE = ROOT / 'build' / 'xlate' / ('totk_xlate.exe' if os.name == 'nt' else 'totk_xlate')
|
||||
SPIRV_VAL = Path(r'C:\msys64\ucrt64\bin\spirv-val.exe') if os.name == 'nt' else Path(shutil.which('spirv-val') or '/usr/bin/spirv-val')
|
||||
ZSTD = b'\x28\xb5\x2f\xfd'
|
||||
# NVN program control section magic, u32 0x98761234 (d2r-mobile dm/gfx/tools/shader_map.py)
|
||||
CONTROL_MAGIC = b'\x34\x12\x76\x98'
|
||||
|
||||
+127
-14
@@ -1,15 +1,16 @@
|
||||
# Adapted from MacNativeGames games/totk/scripts/ultrawide_layout.py @ 933b6a0 (MIT, Copyright (c) 2026
|
||||
# MacNativeGames contributors). Used by tools/make_romfs_patches.py --aspect.
|
||||
"""The game's UI layouts for a canvas wider than 16:9 (output_res = 3440x1440 in totk.ini), edited in place in the
|
||||
decompressed UI/LayoutArchive/Common.Product.*.blarc (tools/make_romfs_patches.py --aspect).
|
||||
"""The game's UI layouts for a canvas of another shape than 16:9 (output_res = 3440x1440 or 1280x800 in totk.ini),
|
||||
edited in place in the decompressed UI/LayoutArchive/Common.Product.*.blarc (tools/make_romfs_patches.py --aspect).
|
||||
|
||||
The game lays its UI out on a 1920x1080 grid and maps that grid onto the canvas, so on a 21:9 canvas every layout comes
|
||||
out 34% too wide. Here each layout's root pane is narrowed to 16:9 (scale x = 16/9 over the aspect), backgrounds that
|
||||
must cover the whole screen are widened back, and the HUD's parts move out to the screen's corners. The pane list and
|
||||
the distances are the ones the community TOTK-Ultrawide mod for emulators found for 1.0.0 .. 1.2.1 (GPL; only those
|
||||
values are taken, no code; THIRD_PARTY_NOTICES.md). This is an independent implementation, checked against 1.4.3's
|
||||
Common.Product.140: every pane is found by name and every pane field and animation key holds the value the edit
|
||||
expects before it is written.
|
||||
out 34% too wide and on a 16:10 one 11% too tall. Here each layout's root pane is scaled back to 16:9 on the stretched
|
||||
axis (scale x = 16/9 over the aspect when wider, scale y = the aspect over 16/9 when narrower), backgrounds that must
|
||||
cover the whole screen are stretched back, and the HUD's parts move out to the screen's corners (wider) or its top and
|
||||
bottom edges (narrower). The pane list and the distances are the ones the community TOTK-Ultrawide mod for emulators
|
||||
found for 1.0.0 .. 1.2.1 (GPL; only those values are taken, no code; THIRD_PARTY_NOTICES.md). This is an independent
|
||||
implementation, checked against 1.4.3's Common.Product.140: every pane is found by name and every pane field and
|
||||
animation key holds the value the edit expects before it is written.
|
||||
"""
|
||||
import struct
|
||||
|
||||
@@ -82,6 +83,110 @@ def edits(k, hud_corner):
|
||||
('PlayerStatus_00', 'Pa_CompanionLife_00', 'x', -(840 + w))]
|
||||
return out
|
||||
|
||||
SYSTEM_MENUS = ('SystemLoadList_00', 'SystemOption_00', 'SystemSaveLoad_00')
|
||||
|
||||
def tall_edits(k):
|
||||
"""(layout, pane, field, value) for a canvas narrower than 16:9: k = the aspect over 16/9 (the root's scale y),
|
||||
h = how far a 1080-high layout's edge moves out in its own units (540 * (1 / k - 1))."""
|
||||
h, tall = 540 * (1 / k - 1), 1 / k
|
||||
out = [('AppAlbum_00', 'N_FadeInOut_00', 'scale_y', k), ('AppAlbum_00', 'N_Open_01', 'scale_y', tall),
|
||||
*[('AppMenuOverlay_00', p, 'scale_y', k) for p in ('N_FadeInOut_00', 'N_FadeInOut_01', 'N_Icon_00',
|
||||
'N_TitleOne_01')],
|
||||
('CameraPointer_00', 'N_Type_01', 'scale_y', k), ('CameraSystemWindow_00', 'N_FirstOut_00', 'scale_y', k),
|
||||
*[('LoadingFade_00', p, 'scale_y', k) for p in ('Pa_IconPlayer_00', 'Pa_LargeDungeonMap_00',
|
||||
'Pa_LargeDungeonMap_01', 'Pa_Map_00', 'Pa_Map_01')],
|
||||
*[('AppMap_00', f'N_BlurInOut_0{i}', 'scale_y', tall) for i in range(4)],
|
||||
*[(layout, 'P_BG_00', 'scale_y', tall) for layout in ('AppOpenDemoWindow_00', 'GameOver_00', 'LoadingFade_00',
|
||||
'PaLargeDungeonMap_00', 'PaRecipeCardList_00')],
|
||||
('LoadingFade_00', 'N_Base_00', 'scale_y', tall),
|
||||
*[(layout, 'N_BG_00', 'scale_y', tall) for layout in ('AutoBuilder_00', 'OwnedHorseList_00', 'ShopMenu_00',
|
||||
'StaffRoll_00')],
|
||||
*[('CameraPointer_00', p, 'scale_y', tall) for p in ('N_InOutScope_00', 'Pa_CameraFocus_00', 'S_Scissor_00',
|
||||
'W_Shadow_01')],
|
||||
('CameraSystemWindow_00', 'N_InOut_00', 'scale_y', tall), ('MessageLog_01', 'P_Capture_00', 'scale_y', tall),
|
||||
*[(layout, 'N_SystemBG_00', 'scale_y', tall) for layout in ('SystemActionGuide_00',) + SYSTEM_MENUS]]
|
||||
# The HUD at the screen's top and bottom edges instead of the middle 16:9
|
||||
out += [(name, 'RootPane', 'y', 540 * (1 - k)) for name in (
|
||||
'AppAlbum_00', 'AppPictureBook_00', 'ChallengeLog_00', 'Challenge_00', 'InstantTipsWithPict_00', 'InstantTips_00',
|
||||
'MiniGameTimer_00', 'NumDisplaySecond_00', 'NumDisplay_00', 'PickUpWin_00', 'PlayerStatus_00', 'Pouch_00',
|
||||
'ShopMenu_00', 'SystemActionGuide_00') + SYSTEM_MENUS]
|
||||
out += [(name, 'RootPane', 'y', -540 * (1 - k)) for name in (
|
||||
'CtrlGuide_00', 'DoCommand_00', 'EnvironmentInfo_00', 'HorseGauge_00', 'MessageDialog_00', 'MessageLog_01',
|
||||
'PauseMenuOverlay_00', 'SaveLoadIcon_00', 'TreasureFullShortCut_00')]
|
||||
out += [('AppAlbum_00', 'N_PicLoad_01', 'y', -540 * (1 - k)), ('AppMap_00', 'N_AllMapTime_00', 'y', -h),
|
||||
('AppMenuOverlay_00', 'A_Align_00', 'y', -(505 + h)), ('AppMenuOverlay_00', 'Pa_GuideLeft_00', 'y', -(505 + h)),
|
||||
('AppMenuOverlay_00', 'N_TitleTab_00', 'y', 491 + h),
|
||||
('LoadingFade_00', 'N_Logo_00', 'y', -h), ('LoadingFade_00', 'N_Sage_00', 'y', -477 - 2 * h),
|
||||
('LoadingFade_00', 'N_Status_00', 'y', h), ('LoadingFade_00', 'N_Tips_00', 'y', -(360 + h)),
|
||||
('Location_00', 'N_Large_00', 'y', 270 + h), ('Location_00', 'N_Small_00', 'y', -(395 + h)),
|
||||
('MessageLog_01', 'P_Capture_00', 'y', h),
|
||||
('PaExtraStaminaGauge_00', 'N_BasePattern_00', 'y', h), ('PaExtraStaminaGauge_00', 'N_GaugeMainPattern_00', 'y', h),
|
||||
('Pouch_00', 'N_Info_00', 'y', -h), ('Pouch_00', 'Pa_ExtraStamina_00', 'y', -h),
|
||||
('ShopMenu_00', 'A_AlignGuide_00', 'y', -(477 + h)),
|
||||
*[(layout, 'N_SystemBG_00', 'y', -h) for layout in ('SystemActionGuide_00',) + SYSTEM_MENUS],
|
||||
('SystemLoadList_00', 'Pa_Guide_00', 'y', -(474 + h)), ('SystemOption_00', 'A_Guide_00', 'y', -(474 + h))]
|
||||
return out
|
||||
|
||||
# BFLAN animation targets: FLPA (pane SRT) translate y, FLTS (texture SRT) scale y
|
||||
FLPA_Y, FLTS_SY = 1, 4
|
||||
|
||||
def anim_keys(blarc, lo, pane, tag, target):
|
||||
"""The absolute offsets of each key (frame, value, slope: 12 bytes) of the hermite curves of one pane, animation
|
||||
tag (b'FLPA', b'FLTS') and target in the BFLAN at blarc[lo:], one list per curve. Found by walking the pai1 section,
|
||||
so they hold in any version whose animation has that curve."""
|
||||
if blarc[lo:lo + 4] != b'FLAN': raise SystemExit(f'{lo:#x}: not a BFLAN')
|
||||
pos, out = lo + struct.unpack_from('<H', blarc, lo + 6)[0], []
|
||||
for _ in range(struct.unpack_from('<H', blarc, lo + 0x10)[0]):
|
||||
magic, size = bytes(blarc[pos:pos + 4]), struct.unpack_from('<I', blarc, pos + 4)[0]
|
||||
if magic == b'pai1':
|
||||
table = pos + struct.unpack_from('<I', blarc, pos + 16)[0]
|
||||
for e in range(struct.unpack_from('<H', blarc, pos + 14)[0]):
|
||||
eo = pos + struct.unpack_from('<I', blarc, table + 4 * e)[0]
|
||||
if bytes(blarc[eo:eo + 28]).split(b'\0')[0].decode() != pane: continue
|
||||
for t in range(blarc[eo + 28]):
|
||||
to = eo + struct.unpack_from('<I', blarc, eo + 32 + 4 * t)[0]
|
||||
if bytes(blarc[to:to + 4]) != tag: continue
|
||||
for g in range(blarc[to + 4]):
|
||||
go = to + struct.unpack_from('<I', blarc, to + 8 + 4 * g)[0]
|
||||
if blarc[go + 1] != target: continue
|
||||
if blarc[go + 2] != 2: raise SystemExit(f'{pane}: a {tag.decode()} curve that is not hermite')
|
||||
keys = go + struct.unpack_from('<I', blarc, go + 8)[0]
|
||||
out.append([keys + 12 * i for i in range(struct.unpack_from('<H', blarc, go + 4)[0])])
|
||||
pos += size
|
||||
return out
|
||||
|
||||
def tall_anim(blarc, members, k, changed):
|
||||
"""The animation keys that hold positions or zooms on the screen's height, for a canvas narrower than 16:9."""
|
||||
h = 540 * (1 / k - 1)
|
||||
get = lambda at: struct.unpack_from('<f', blarc, at)[0]
|
||||
put = lambda at, v: struct.pack_into('<f', blarc, at, v)
|
||||
def anim(name):
|
||||
member = f'anim/{name}.bflan'
|
||||
if member not in members: raise SystemExit(f'{name}: no animation {member} in the archive')
|
||||
changed.add(member)
|
||||
return members[member][0]
|
||||
# The minimap's four cut panes slide in on straight lines from 0 at frame 0 to y -360, the bottom of a 16:9 layout
|
||||
# (hermite keys: the first one's slope is the end point); the end point moves with the screen's edge.
|
||||
lo = anim('AppMap_00_MiniMap')
|
||||
for pane in ('N_Cut_00', 'N_Cut_01', 'N_Cut_02', 'N_Cut_03'):
|
||||
curves = anim_keys(blarc, lo, pane, b'FLPA', FLPA_Y)
|
||||
if len(curves) != 1 or len(curves[0]) != 2: raise SystemExit(f'AppMap_00_MiniMap {pane}: {len(curves)} y curves')
|
||||
k0, k1 = curves[0]
|
||||
if not (get(k0) == 0 and get(k0 + 4) == 0 and abs(get(k0 + 8) + 360) < 1e-3 and get(k1) == 1 and
|
||||
abs(get(k1 + 4) + 360) < 1e-3 and get(k1 + 8) == 0):
|
||||
raise SystemExit(f'AppMap_00_MiniMap {pane}: not a line from 0 to y -360')
|
||||
put(k0 + 8, -(360 + h)), put(k1 + 4, -(360 + h))
|
||||
# The large place name of a discovery holds its height (285, then 260) in flat keys; they move to the top edge.
|
||||
curves = anim_keys(blarc, anim('Location_00_IsDiscovery'), 'N_Large_00', b'FLPA', FLPA_Y)
|
||||
values = [get(key + 4) for keys in curves for key in keys]
|
||||
if len(curves) != 1 or values != [285, 285, 260]: raise SystemExit(f'Location_00_IsDiscovery N_Large_00: keys {values}')
|
||||
for key in curves[0]: put(key + 4, get(key + 4) + h)
|
||||
# The dragon's tears map icon zooms its picture to 1.08 and 1.28 in flat keys; on the stretched axis by 1 / k more.
|
||||
curves = anim_keys(blarc, anim('PaMapIconDragonTears_00_Zoom'), 'P_Geoglyph_00', b'FLTS', FLTS_SY)
|
||||
values = [round(get(key + 4), 4) for keys in curves for key in keys]
|
||||
if len(curves) != 1 or values != [1.08, 1.28]: raise SystemExit(f'PaMapIconDragonTears_00_Zoom P_Geoglyph_00: keys {values}')
|
||||
for key in curves[0]: put(key + 4, get(key + 4) / k)
|
||||
|
||||
def anim_edits(k, hud_corner):
|
||||
"""(animation, byte offset, value the game ships, new value): floats inside BFLAN key tables."""
|
||||
w = 960 * (1 / k - 1)
|
||||
@@ -109,9 +214,13 @@ def find_pane(found, layout, name):
|
||||
return found[name]
|
||||
|
||||
def apply(blarc, members, width, height, hud_corner=True):
|
||||
"""Edits blarc (bytearray) in place for a width x height canvas. Returns the members it changed."""
|
||||
k = (16 / 9) / (width / height)
|
||||
assert k < 1, 'only for canvases wider than 16:9'
|
||||
"""Edits blarc (bytearray) in place for a width x height canvas of another shape than 16:9. Returns the members it
|
||||
changed. hud_corner (wider canvases): the HUD goes to the screen's corners; a narrower one always has it at the
|
||||
top and bottom edges."""
|
||||
wide = width / height > 16 / 9
|
||||
k = (16 / 9) / (width / height) if wide else (width / height) / (16 / 9)
|
||||
assert k < 1, 'only for canvases of another shape than 16:9'
|
||||
root_scale = 'scale_x' if wide else 'scale_y'
|
||||
changed, layouts = set(), {}
|
||||
unknown = sorted(n for n in KEEP_ROOT if f'blyt/{n}.bflyt' not in members)
|
||||
if unknown: raise SystemExit(f'KEEP_ROOT names no layout: {unknown}')
|
||||
@@ -123,17 +232,21 @@ def apply(blarc, members, width, height, hud_corner=True):
|
||||
return find_pane(layouts[member], layout, pane)
|
||||
for member in sorted(m for m in members if m.startswith('blyt/') and m.endswith('.bflyt')):
|
||||
if member[5:-6] in KEEP_ROOT: continue
|
||||
at = pane_offset(member[5:-6], 'RootPane') + FIELD['scale_x']
|
||||
at = pane_offset(member[5:-6], 'RootPane') + FIELD[root_scale]
|
||||
have = struct.unpack_from('<f', blarc, at)[0]
|
||||
if abs(have - 1.0) > 1e-3: raise SystemExit(f'{member[5:-6]} RootPane scale_x: {have}, expected 1.0')
|
||||
if abs(have - 1.0) > 1e-3: raise SystemExit(f'{member[5:-6]} RootPane {root_scale}: {have}, expected 1.0')
|
||||
struct.pack_into('<f', blarc, at, k)
|
||||
# Every edit's value at k = 1 is the value the game ships (a scale of 1, the 16:9 position): checked before writing,
|
||||
# so a version with other numbers stops here instead of getting wrong positions. OFF moves a pane away: not checked.
|
||||
for (layout, pane, field, value), (_, _, _, shipped) in zip(edits(k, hud_corner), edits(1.0, hud_corner)):
|
||||
table = zip(edits(k, hud_corner), edits(1.0, hud_corner)) if wide else zip(tall_edits(k), tall_edits(1.0))
|
||||
for (layout, pane, field, value), (_, _, _, shipped) in table:
|
||||
at = pane_offset(layout, pane) + FIELD[field]
|
||||
have = struct.unpack_from('<f', blarc, at)[0]
|
||||
if value != 10000.0 and abs(have - shipped) > 1e-3: raise SystemExit(f'{layout} {pane} {field}: {have}, expected {shipped}')
|
||||
struct.pack_into('<f', blarc, at, value)
|
||||
if not wide:
|
||||
tall_anim(blarc, members, k, changed)
|
||||
return sorted(changed)
|
||||
for anim, offset, shipped, value in anim_edits(k, hud_corner):
|
||||
member = f'anim/{anim}.bflan'
|
||||
if member not in members: raise SystemExit(f'{anim}: no animation {member} in the archive')
|
||||
|
||||
@@ -104,6 +104,9 @@ static class SwshXlate
|
||||
// "softfloat64": true lowers FP64 operations to FP32 (Ryujinx's DoubleToFloat pass), for a host without shaderFloat64.
|
||||
public bool QueryHostSupportsShaderFloat64() => _job["softfloat64"] is not JsonValue v || !(v.TryGetValue<bool>(out bool b) && b);
|
||||
public bool QueryHostSupportsTextureShadowLod() => true;
|
||||
// Sub-texel precision bits of a host whose gathers round differently from Maxwell (AMD, Intel): Ryujinx then
|
||||
// biases gather coordinates so the same texels are picked (TexturePass.InsertCoordGatherBias). 0: no bias.
|
||||
public int QueryHostGatherBiasPrecision() => _job["gather_bias"] is JsonNode n ? (int)n : 0;
|
||||
// 作业 reduced = true:放宽精度——翻译器只给强制精确的运算打 NoContraction(缺省全部打:保持 Maxwell 分开舍入的逐位结果,代价是 MSL 里大量禁优化的 spvFMul 等)
|
||||
// Job reduced = true: relaxed precision - the translator marks NoContraction only on operations that must be exact (the default marks all: keeps the bit-exact results of Maxwell's separate rounding, at the cost of many optimization-disabled spvFMul etc. in the MSL)
|
||||
public bool QueryHostReducedPrecision() => _job["reduced"] is JsonNode n && (bool)n;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Smoke test for build/xlate/totk_xlate.exe: no game files needed.
|
||||
"""Smoke test for build/xlate/totk_xlate(.exe): no game files needed.
|
||||
|
||||
Sends one hand-assembled Maxwell fragment shader (writes the constant colour 1.0, 0.5, 0.25, 1.0 to render target 0)
|
||||
through --serve, checks the reply, validates the SPIR-V with spirv-val, then sends a malformed line and checks the
|
||||
@@ -8,6 +8,8 @@ service survives and still answers.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -15,8 +17,9 @@ import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EXE = ROOT / "build" / "xlate" / "totk_xlate.exe"
|
||||
SPIRV_VAL = Path(r"C:\msys64\ucrt64\bin\spirv-val.exe")
|
||||
EXE = ROOT / "build" / "xlate" / ("totk_xlate.exe" if os.name == "nt" else "totk_xlate")
|
||||
SPIRV_VAL = (Path(r"C:\msys64\ucrt64\bin\spirv-val.exe") if os.name == "nt"
|
||||
else Path(shutil.which("spirv-val") or "/usr/bin/spirv-val"))
|
||||
|
||||
PT = 7 << 16 # predicate "always"
|
||||
|
||||
|
||||
Reference in new issue
Block a user