mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 12:25:56 +00:00
The rebuild for ExeFS patches, code cheats and mod plugins ran the setup window from the setup folder, whose absolute path build-cache/rebuild.txt held, with the generated C and objects in that folder's temp/. A moved or deleted setup folder left code mods without a rebuild. - A new Kit step after the check copies the release folder without TotK-Setup.exe into <game>/build-kit (installer/layout.py KIT_ITEMS), and every later step runs its tools from there. A newer setup replaces the kit, the kit's own rebuild keeps it. - The release's temp folder is <game>/build-cache/work, so the lift cache and the objects stay with the game. - rebuild.txt names the kit relative to the game folder; runtime/cheats.c and runtime/romfs.c resolve relative entries against the exe's folder. - runtime_key drops the lines of clang --version that name its folder, so a rebuild from the kit or a moved game folder keeps the compile and prepare steps the setup recorded. - An update over an install of an older setup moves that setup's temp folder into build-cache/work when both are on one disk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
234 lines
12 KiB
Python
234 lines
12 KiB
Python
"""Where the installer finds what it runs: the one place that maps the release folder and a repository checkout.
|
|
|
|
Release (docs/installer.md "Release layout"), root = the unpacked folder:
|
|
|
|
python/python.exe runs every tool (numpy, lz4, cryptography, compression.zstd); its ._pth puts .., ../tools
|
|
and ../recomp on sys.path, so `python -m installer` imports from the release folder
|
|
recomp/, tools/ the lifter and the scripts; recomp/patches-<v>.txt, tools/data/expected-v<version>.json for
|
|
each supported update (SUPPORTED)
|
|
llvm/ llvm-mingw (bin/clang.exe)
|
|
runtime/ libtotkrt.a and libtotkrt-v2.a (x86-64-v3 / v2), the headers, install-side sources (romfs.c,
|
|
patches.c, ov_zstd.c) and totk.ico; runtime/deps/ holds
|
|
include/ + lib/ of Vulkan, SDL3, Opus, libvpx and VMA (build_game.py --deps)
|
|
bin/ totk_xlate.exe (+ totk_xlate.id), hactool.exe, ninja.exe, SDL3.dll, libopus-0.dll
|
|
data/shader_list.tsv the MacNativeGames shader list
|
|
data/pgo-<v>.profdata the execution profile the update's code is compiled with (tools/pgo.py; Windows only);
|
|
without one the game is built at -O1
|
|
totk.ini.example
|
|
VERSION the release version, one line
|
|
|
|
Build kit (Windows release): the setup copies the release folder without TotK-Setup.exe into <install>/build-kit
|
|
(KIT_ITEMS, steps.py Kit) and runs every later step from it, so the game's rebuild for its mods
|
|
(build-cache/rebuild.txt, relative to the install folder) needs neither the setup folder nor its path: the setup can be
|
|
deleted and the install folder moved. The kit is a release folder itself (release_layout). The temp folder with the
|
|
rebuild's working state (gen/, build/, the lift cache) is <install>/build-cache/work (TEMP_IN_DEST), so it moves
|
|
with the install and survives a newer setup's kit.
|
|
|
|
Linux release (the AppImage's folder, docs/installer.md "Linux release"): python/bin/python3 (python-build-standalone,
|
|
with tkinter), llvm/ with clang, lld and llvm-ar plus llvm/sysroot (the glibc headers and libraries the game links
|
|
against), runtime/deps/ with include/ and the static Opus, libvpx and zstd, bin/ without .exe and with libSDL3.so.0.
|
|
|
|
Dev mode, root = a repository checkout (no python/python.exe): build/py314-embed/py/python.exe (the embeddable
|
|
Python the release ships, with the same packages), else this Python; toolchain/llvm-mingw-*/ and toolchain/deps-mingw;
|
|
runtime/ as the sources with build/install-rtlib/libtotkrt.a and libtotkrt-v2.a (tools/build_game.py --make-runtime-lib
|
|
--out build/install-rtlib, once per --cpu); MSYS2 UCRT64 ninja.exe; toolchain/hactool; build/xlate/totk_xlate.exe; the shader list from
|
|
refs/MacNativeGames; build/pgo/pgo-<v>.profdata when there is one; build/install-temp/<install name>-<hash>. Dev mode and
|
|
the Linux release install no build kit: the repository, or the AppImage file, is what the rebuild runs. The version is VERSION with -dev and the commit: 0.1.0-dev+973c626. On Linux,
|
|
dev mode takes clang, lld and ninja from PATH and the system's libraries (no deps folder, no SDL3 copy).
|
|
"""
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
WIN = os.name == 'nt'
|
|
EXE = '.exe' if WIN else ''
|
|
UCRT = Path(r'C:\msys64\ucrt64\bin')
|
|
SUPPORTED = ('1.2.1', '1.4.3', '1.1.0') # updates the installer builds; the check identifies the player's by hash
|
|
KIT = 'build-kit' # the build kit's folder in the install folder
|
|
# What the build kit holds of a release folder (tools/make_release.py writes them): all but TotK-Setup.exe. licenses/
|
|
# and THIRD_PARTY_NOTICES.md go along with the binaries they cover.
|
|
KIT_ITEMS = ('python', 'llvm', 'runtime', 'bin', 'recomp', 'tools', 'data', 'installer', 'licenses',
|
|
'THIRD_PARTY_NOTICES.md', 'totk.ini.example', 'VERSION')
|
|
# The temp folder inside the install folder, for a layout that names none (every release); the check step does not
|
|
# count it when it requires an empty install folder. Linux keeps the name its installs already use.
|
|
TEMP_IN_DEST = Path('build-cache', 'work') if WIN else Path('.setup-temp')
|
|
|
|
|
|
def version_tag(version):
|
|
"""'1.4.3' -> 'v143': the version name of build_game.py, make_romfs_patches.py and the patch tables."""
|
|
return 'v' + version.replace('.', '')
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Layout:
|
|
root: Path
|
|
dev: bool
|
|
release: str
|
|
python: Path # runs the tools
|
|
tools: Path
|
|
recomp: Path
|
|
llvm: Path # llvm-mingw tree; on Linux the tree with bin/clang (a release's, or the system's /usr)
|
|
deps: Path | None # include/ and lib/ for the link; None: the system's (Linux dev mode)
|
|
runtime: Path # runtime headers and install-side sources
|
|
runtime_lib: Path # libtotkrt.a
|
|
ninja: Path
|
|
hactool: Path
|
|
xlate: Path # totk_xlate.exe; a totk_xlate.id beside it is copied too
|
|
dlls: tuple = field(default_factory=tuple)
|
|
shader_list: Path | None = None # None: dev, read from refs/MacNativeGames through git
|
|
ini_example: Path | None = None
|
|
temp: Path | None = None
|
|
sysroot: Path | None = None # Linux release: what the game links against, instead of the host's
|
|
runtime_lib_v2: Path | None = None # libtotkrt-v2.a: the runtime for CPUs without AVX2 (build_game.py --cpu v2)
|
|
pgo_dir: Path | None = None # pgo-<v>.profdata (tools/pgo.py); None: no profiles (Linux)
|
|
|
|
@property
|
|
def clang(self):
|
|
return self.llvm / 'bin' / ('clang' + EXE)
|
|
|
|
def runtime_lib_for(self, cpu):
|
|
"""The prebuilt runtime library for cpu (v2 or v3). None in dev mode when it was not built: the runtime is
|
|
then compiled from the sources in runtime/ instead."""
|
|
lib = self.runtime_lib_v2 if cpu == 'v2' else self.runtime_lib
|
|
if lib and lib.is_file():
|
|
return lib
|
|
return None if self.dev else lib
|
|
|
|
def pgo(self, version):
|
|
"""The execution profile build_game.py --pgo compiles the update's code with, None without one (the game is
|
|
then built at -O1)."""
|
|
p = self.pgo_dir / f'pgo-{version_tag(version)}.profdata' if self.pgo_dir else None
|
|
return p if p and p.is_file() else None
|
|
|
|
def patches(self, version):
|
|
"""The code patch table for recomp/gen.py."""
|
|
return self.recomp / f'patches-{version_tag(version)}.txt'
|
|
|
|
def expected(self, version):
|
|
"""The expected hashes of the update."""
|
|
return self.tools / 'data' / f'expected-v{version}.json'
|
|
|
|
def missing(self, cpu='v3'):
|
|
"""Paths the steps need that do not exist. cpu picks the runtime library the host will use."""
|
|
lib = self.runtime_lib_for(cpu)
|
|
need = [self.python, self.tools / 'extract.py', self.tools / 'ncz.py', self.tools / 'build_game.py',
|
|
self.tools / 'prepare_shaders.py',
|
|
self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.tools / 'ultrawide_layout.py',
|
|
self.tools / 'button_icons.py',
|
|
self.tools / 'vendor' / 'macnativegames' / 'extract_shared_fonts.py',
|
|
self.recomp / 'gen.py',
|
|
self.clang,
|
|
*((self.deps / 'include', self.deps / 'lib') if self.deps else ()), self.runtime, *([lib] if lib else ()),
|
|
self.ninja, self.hactool,
|
|
self.xlate, *self.dlls, *(f(v) for v in SUPPORTED for f in (self.patches, self.expected))]
|
|
if self.shader_list:
|
|
need.append(self.shader_list)
|
|
if self.sysroot:
|
|
need.append(self.sysroot)
|
|
return [str(p) for p in need if not p.exists()]
|
|
|
|
|
|
def release_python(root):
|
|
return root / 'python' / ('python.exe' if WIN else 'bin/python3')
|
|
|
|
|
|
def is_release(root):
|
|
return release_python(root).is_file()
|
|
|
|
|
|
def release_layout(root):
|
|
version = root / 'VERSION'
|
|
return Layout(
|
|
root=root, dev=False,
|
|
release=version.read_text(encoding='utf-8').strip() if version.is_file() else 'unknown',
|
|
python=release_python(root),
|
|
tools=root / 'tools', recomp=root / 'recomp',
|
|
llvm=root / 'llvm', deps=root / 'runtime' / 'deps', runtime=root / 'runtime',
|
|
runtime_lib=root / 'runtime' / 'libtotkrt.a',
|
|
runtime_lib_v2=root / 'runtime' / 'libtotkrt-v2.a',
|
|
ninja=root / 'bin' / ('ninja' + EXE), hactool=root / 'bin' / ('hactool' + EXE),
|
|
xlate=root / 'bin' / ('totk_xlate' + EXE),
|
|
dlls=(root / 'bin' / 'SDL3.dll', root / 'bin' / 'libopus-0.dll') if WIN else (root / 'bin' / 'libSDL3.so.0',),
|
|
shader_list=root / 'data' / 'shader_list.tsv', ini_example=root / 'totk.ini.example',
|
|
pgo_dir=root / 'data' if WIN else None,
|
|
# None: the install folder's TEMP_IN_DEST (temp_for). Windows: the rebuild's working state stays with the install
|
|
# and its build kit. Linux: the AppImage is mounted read-only, and on a Steam Deck /home is often too full for
|
|
# the generated C while the install folder is on the SD card
|
|
temp=None,
|
|
sysroot=None if WIN else root / 'llvm' / 'sysroot')
|
|
|
|
|
|
def dev_layout(root):
|
|
if not WIN:
|
|
return dev_layout_posix(root)
|
|
commit = subprocess.run(['git', '-C', str(root), 'rev-parse', '--short', 'HEAD'], capture_output=True,
|
|
text=True).stdout.strip()
|
|
embed = root / 'build' / 'py314-embed' / 'py' / ('python' + EXE)
|
|
trees = sorted(p.parent.parent for p in (root / 'toolchain').glob('llvm-mingw-*/llvm-mingw-*/bin/clang' + EXE))
|
|
deps = root / 'toolchain' / 'deps-mingw'
|
|
semver = (root / 'VERSION').read_text(encoding='utf-8').strip()
|
|
return Layout(
|
|
root=root, dev=True, release=f'{semver}-dev+{commit or "unknown"}',
|
|
python=embed if embed.is_file() else Path(sys.executable),
|
|
tools=root / 'tools', recomp=root / 'recomp',
|
|
llvm=trees[-1] if trees else root / 'toolchain' / 'llvm-mingw-missing', deps=deps,
|
|
runtime=root / 'runtime', runtime_lib=root / 'build' / 'install-rtlib' / 'libtotkrt.a',
|
|
runtime_lib_v2=root / 'build' / 'install-rtlib' / 'libtotkrt-v2.a',
|
|
ninja=UCRT / ('ninja' + EXE), hactool=root / 'toolchain' / 'hactool' / ('hactool' + EXE),
|
|
xlate=root / 'build' / 'xlate' / ('totk_xlate' + EXE),
|
|
dlls=tuple(sorted((deps / 'bin').glob('*.dll'))),
|
|
shader_list=None, ini_example=root / 'totk.ini.example', temp=root / 'build' / 'install-temp',
|
|
pgo_dir=root / 'build' / 'pgo')
|
|
|
|
|
|
def dev_commit(root):
|
|
return subprocess.run(['git', '-C', str(root), 'rev-parse', '--short', 'HEAD'], capture_output=True,
|
|
text=True).stdout.strip()
|
|
|
|
|
|
def dev_layout_posix(root):
|
|
clang, ninja = shutil.which('clang'), shutil.which('ninja')
|
|
semver = (root / 'VERSION').read_text(encoding='utf-8').strip()
|
|
return Layout(
|
|
root=root, dev=True, release=f'{semver}-dev+{dev_commit(root) or "unknown"}',
|
|
python=Path(sys.executable), tools=root / 'tools', recomp=root / 'recomp',
|
|
llvm=Path(clang).parent.parent if clang else root / 'toolchain' / 'clang-missing', deps=None,
|
|
runtime=root / 'runtime', runtime_lib=root / 'build' / 'install-rtlib' / 'libtotkrt.a',
|
|
runtime_lib_v2=root / 'build' / 'install-rtlib' / 'libtotkrt-v2.a',
|
|
ninja=Path(ninja) if ninja else root / 'toolchain' / 'ninja-missing',
|
|
hactool=root / 'toolchain' / 'hactool' / 'hactool', xlate=root / 'build' / 'xlate' / 'totk_xlate',
|
|
shader_list=None, ini_example=root / 'totk.ini.example', temp=root / 'build' / 'install-temp')
|
|
|
|
|
|
def find(root=None):
|
|
"""The layout of root (default: the folder above this package)."""
|
|
root = Path(root or Path(__file__).resolve().parent.parent).resolve()
|
|
return release_layout(root) if is_release(root) else dev_layout(root)
|
|
|
|
|
|
def uses_kit(layout):
|
|
"""True when the setup installs a build kit (Windows release); dev mode and the Linux AppImage run from where they
|
|
are."""
|
|
return WIN and not layout.dev
|
|
|
|
|
|
def kit_folder(dest):
|
|
return Path(dest) / KIT
|
|
|
|
|
|
def temp_folder(base, dest):
|
|
"""One temp folder per install folder below a shared base (dev mode, --temp), so two installs never share
|
|
generated files."""
|
|
tag = hashlib.sha256(str(dest).lower().encode()).hexdigest()[:8]
|
|
return base / f'{dest.name or "root"}-{tag}'
|
|
|
|
|
|
def temp_for(layout, dest, override=None):
|
|
"""The temp folder of a run on dest: below --temp or the layout's, else TEMP_IN_DEST in the install folder."""
|
|
base = override or layout.temp
|
|
return temp_folder(Path(base).resolve(), dest) if base else dest / TEMP_IN_DEST
|