diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index aa56ba05..bec4fc7d 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -19,7 +19,7 @@ Code copied or adapted from another project is listed here with its source, the | deko3d block-linear image layout, as ported in MacNativeGames' `nvn_tex.c` (Apple parts dropped, layout math unchanged; licence text in `games/totk/third_party/d2r_gfx/LICENSE.deko3d`, reproduced in the file header) | https://github.com/devkitPro/deko3d (`source/dk_image.cpp`), via MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/third_party/d2r_gfx/gfx/nvn/nvn_tex.c`, `nvn_tex.h`), Copyright (C) 2018-2020 fincs | zlib | M4 task T3b: `runtime/nvn_tex.c`, `runtime/nvn_tex.h` | | MacNativeGames texture auto-layout goldens (40 original-SDK size and block-height cases) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/tests/texture_auto_layout/probe.c`), Copyright (c) 2026 jhaizhou | MIT | M4 task T3b: `runtime/nvn_tex_probe.c`; T3f: `runtime/nvn_probe.c` (four of the cases, through the procs) | | MacNativeGames block-linear deswizzler (address function; copy loop split out as `tex_bl_copy`) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/third_party/d2r_gfx/gfx/tex/nvn_deswizzle.c`), Copyright (c) 2026 jhaizhou | MIT | M4 task T5a: `runtime/tex_deswizzle.c`, `runtime/tex.h` | -| SDL 3.4.18 (MSYS2 `mingw-w64-ucrt-x86_64-SDL3`), dynamic link, `SDL3.dll` and its dependency `libiconv-2.dll` (LGPL-2.1, unmodified, not linked into the exe) copied next to `totk.exe` | https://libsdl.org, Copyright (c) 1997-2026 Sam Lantinga | zlib | `runtime/input_win.c` (gamepad input, rumble, gyro and accelerometer) | +| SDL 3.4.18, dynamic link, `SDL3.dll` copied next to `totk.exe`. Developer builds use the MSYS2 build (`mingw-w64-ucrt-x86_64-sdl3`, which imports `libiconv-2.dll`, LGPL-2.1, unmodified, copied beside it); the installer release ships the upstream release `SDL3-3.4.18-win32-x64.zip`, which imports no libiconv | https://libsdl.org, Copyright (c) 1997-2026 Sam Lantinga | zlib | `runtime/input_win.c` (gamepad input, rumble, gyro and accelerometer) | | libopus (MSYS2 `mingw-w64-ucrt-x86_64-opus`), dynamic link, `libopus-0.dll` copied next to `totk.exe` | https://opus-codec.org, Copyright (c) 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon | BSD-3-Clause | `runtime/ov_media.c` (nn::codec hardware Opus decoder) | | Vulkan Memory Allocator 3.4.0 (MSYS2 `mingw-w64-ucrt-x86_64-vulkan-memory-allocator`), header-only, compiled into `totk.exe` | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator, Copyright (c) 2017-2026 Advanced Micro Devices, Inc. | MIT | `runtime/vk_vma.cpp` (the allocator behind `runtime/vk_memory.c`) | | MacNativeGames host movie decoder (`movie::Decoder` for VP9 over libvpx: entry list, buffer and event semantics, NV12 output format, PTS pacing) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/runtime/movie_host.c`, `movie_host.h`), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/ov_movie.c`, `runtime/movie.h` | @@ -31,6 +31,26 @@ These tools run on the developer's or player's machine during the build. They ar | Tool | Source | License | Purpose | |---|---|---|---| -| hactool | https://github.com/SciresM/hactool @ `1d64a83450e025622f3468c28fc4164dad2c5ef6`, with patches from MacNativeGames | ISC | NCA and update container extraction | +| hactool (the installer release ships it, see below) | https://github.com/SciresM/hactool @ `1d64a83450e025622f3468c28fc4164dad2c5ef6`, with patches from MacNativeGames | ISC | NCA and update container extraction | | Capstone | https://www.capstone-engine.org | BSD-3-Clause | ARM64 disassembly for analysis | | Unicorn | https://www.unicorn-engine.org | GPL-2.0 (Python bindings BSD) | Differential testing of the lifter; never linked into the game | + +## Distributed in the installer release + +`tools/make_release.py` assembles these into the release folder of `docs/installer.md`. Versions and sha256 of every download are pinned in that script; licence texts are in the folder's `licenses/` and in each component's own folder. + +| Component | Source | License | Where in the release | +|---|---|---|---| +| CPython 3.14.9 embeddable package, with `tkinter` and Tcl/Tk 9.0.4 taken from the python.org 3.14.9 installer's component MSI (unpacked, never run); includes `vcruntime140*.dll` (Microsoft, redistributable with Python), OpenSSL, libffi, SQLite, zlib, libtommath | https://www.python.org/downloads/release/python-3149/ | PSF-2.0 (Tcl/Tk: Tcl/Tk licence, BSD-style) | `python/` | +| numpy 2.5.4 (with its bundled OpenBLAS), lz4 4.4.5, cryptography 50.0.2 (with its bundled OpenSSL), cffi 2.1.1, pycparser 3.11 | PyPI binary wheels, pins in `tools/requirements-installer.txt` | BSD-3-Clause; lz4 BSD-3-Clause; cryptography Apache-2.0 OR BSD-3-Clause; cffi MIT; pycparser BSD-3-Clause | `python/Lib/site-packages/` | +| llvm-mingw 20261020 (LLVM/Clang/LLD 23, mingw-w64, libc++, libunwind, compiler-rt), x86_64 subset | https://github.com/mstorsjo/llvm-mingw/releases/tag/20261020 | Apache-2.0 with LLVM exception; mingw-w64 ZPL-2.1/BSD/public domain; libc++ and libunwind Apache-2.0 with LLVM exception and MIT/UIUC; winpthreads MIT | `llvm/` | +| SDL 3.4.18 upstream Windows release (`SDL3.dll`) and mingw development package (headers, `libSDL3.dll.a`) | https://github.com/libsdl-org/SDL/releases/tag/release-3.4.18 | zlib | `bin/SDL3.dll`, `runtime/deps/` | +| libopus 1.6.1 (MSYS2 `mingw-w64-ucrt-x86_64-opus` 1.6.1-2; `libopus-0.dll` imports only KERNEL32 and the UCRT api-ms-win-crt DLLs), headers and import library | https://repo.msys2.org/mingw/ucrt64/ | BSD-3-Clause | `bin/libopus-0.dll`, `runtime/deps/` | +| libvpx 1.17.0 (MSYS2 `mingw-w64-ucrt-x86_64-libvpx` 1.17.0-1), headers and static library compiled into `totk.exe`; `libvpx-LICENSE.txt` and `libvpx-PATENTS.txt` come from the v1.17.0 source tag | https://github.com/webmproject/libvpx | BSD-3-Clause with the WebM patent grant | `runtime/deps/`, `licenses/` | +| Vulkan-Headers and Vulkan-Loader 1.4.363.0 (MSYS2 packages; headers and `libvulkan-1.dll.a` only, no loader DLL) | https://github.com/KhronosGroup | Apache-2.0 OR MIT | `runtime/deps/` | +| Vulkan Memory Allocator 3.4.0 (MSYS2 package), header | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator | MIT | `runtime/deps/` | +| Ninja 1.13.2 (`ninja-win.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja.exe` | +| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check), built with MSYS2 UCRT64 clang; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | +| `totk_xlate.exe` (Ryujinx shader translator, NativeAOT build of `tools/xlate/`, see the Ryujinx row above) | built by `tools/build_xlate.py` | MIT | `bin/totk_xlate.exe` | +| MacNativeGames shader variant list | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/assets/shader_list.tsv`), Copyright (c) 2026 MacNativeGames contributors | MIT | `data/shader_list.tsv` | + diff --git a/docs/installer.md b/docs/installer.md new file mode 100644 index 00000000..c0738dea --- /dev/null +++ b/docs/installer.md @@ -0,0 +1,113 @@ +# Installer + +A public release that turns the player's own game files into a playable TotK folder. The release carries no game +data and no code derived from the game: the installer builds the game on the player's machine. The installed folder +holds only the game; the installer and its tools stay outside it. + +## What the player does + +1. Downloads `TotK-Setup--windows.zip` from GitHub Releases, unpacks it anywhere and starts `TotK-Setup.exe`. +2. Picks: + - the base game (XCI or NSP) and the 1.2.1 update (NSP); + - their `prod.keys`; + - the game language and the install folder. +3. Waits for the setup, one time. The window shows each step and its progress. +4. Plays `totk.exe` from the install folder. The installer and the source NSP or XCI can be deleted. + +The install folder (`main.c` `portable_paths()` reads this layout; it stays portable): + +``` +/ + totk.exe, SDL3.dll, libopus-0.dll, totk_xlate.exe + totk.ini language; the other keys as commented defaults (totk.ini.example) + exefs/, romfs/ extracted from the player's files + profile/ saves + shader and pipeline caches prepared by the setup + totk-install.json what built this folder: release version, input hashes, finished steps +``` + +## Updating the port + +The player runs a newer installer on the same folder. It reads `totk-install.json`, keeps `romfs/`, `exefs/`, +`profile/` and `totk.ini` after checking their hashes, rebuilds `totk.exe` and re-prepares the shaders. A newer +release that changes nothing game-derived skips the rebuild. + +## Setup steps + +1. **Check** before writing anything: the files open with the keys, the update is 1.2.1 (by hash, as + `tools/extract.py`), the keys are new enough for the update, enough space is free on the target disk (about 36 GiB at the peak of a + first install, 20 GiB after it), the CPU has + AVX2 (`-march=x86-64-v3`), a Vulkan 1.3 GPU exists. Each failure is one clear message. +2. **Extract** ExeFS and RomFS into the install folder and verify the hashes. +3. **Lift** the ExeFS to C (`recomp/gen.py`, `recomp/patches-v121.txt`) into a temp folder next to the installer. +4. **Compile** the generated C, `modules.c` and `romfs.c` (they bake per-install layout and RomFS hashes) and link + them with the prebuilt runtime library into `totk.exe` (`build_game.py --runtime-lib`, proven on task/llvmmingw: + 3 min on 24 cores). +5. **Prepare** the RomFS patches and the shaders (`tools/make_romfs_patches.py`, `tools/prepare_shaders.py`, + `totk.exe --prepare-shaders`) in the install folder. +6. **Clean up** the temp folder (generated C and objects, about 1.3 GB). + +A cancelled or failed setup resumes from the last finished step (`totk-install.json`). + +## Command line + +`installer/` (stdlib only; `installer/layout.py` maps the release folder and, in a repository, the dev toolchain): + +``` +python -m installer --base FILE --update FILE --keys FILE --lang CODE --dest DIR [--root DIR] [--temp DIR] [--verify] +``` + +- On a folder with `totk-install.json` only `--dest` is needed; the game files only when the extraction must run again. +- stdout: one JSON event per line, types `start`, `plan`, `step` (started, done, skipped), `progress` (step, phase, + done/total/unit), `space` (bytes needed and free on the install disk and the temp disk), `warning`, `error`, `end` (`installer/events.py`). Messages are ids with parameters; + `installer/text/en.json` holds their English text. The human-readable log goes to `/logs/`. +- Exit codes: 0 done, 1 failed, 2 bad command line, 3 cancelled. The window cancels with Ctrl+Break to the + installer's process group. +- Each step records a key over its inputs (runner.py). A step with the recorded key and intact output is skipped, so + resume and update are one rule; output kept from another release is re-hashed first. +- Peak space on the install disk is about 36 GiB, not 22: the extraction copies both Program NCAs (16.3 GiB) next to + the merged RomFS (15.9 GiB) before it deletes them. The check computes the exact figure. +- Window (`python -m installer.gui`, `installer/gui.py` draws, `installer/guilogic.py` decides): form, step list with + progress, space line, collapsible log, Cancel (Ctrl+Break to the CLI's process group, then it waits for the exit), + finish screen with Open folder. English or Russian, chosen from the Windows display language, switchable in the + window; every sentence is an id in `installer/text/.json`. A folder with `totk-install.json` needs only the + folder. `--selftest [--shots DIR]` builds every screen in both languages and closes itself. +- `tests/installer_gui_test.py`: event model, form, catalogs, cancel, the self-closing window and one end-to-end run + of the CLI in `--verify` mode (needs `build/install-test` and `build/extracted/v121`; see its docstring). +- `tests/installer_test.py`: bookkeeping with fake steps. `tests/installer_e2e.py`: a dev-mode run with the files of + `config.local.toml`. + +## Release layout + +The unpacked release; `tools/make_release.py` assembles it, CI zips it: + +``` +TotK-Setup.exe starts python/pythonw.exe -m installer.gui (installer/launcher/launcher.c, no console) +installer/ the installer: steps, CLI, window, en/ru text catalog +python/ Python 3.14.9 embeddable + numpy, lz4, cryptography, cffi, pycparser (+ tkinter for the window) +recomp/, tools/ the lifter and the build, extraction and shader scripts the steps run (copied from the repo) +llvm/ llvm-mingw, x86_64-only subset (Apache-2.0 with LLVM exception; mingw-w64 parts permissive) +runtime/ libtotkrt.a and the headers the generated code and modules.c/romfs.c include +bin/ totk_xlate.exe, hactool.exe, SDL3.dll, libopus-0.dll, ninja.exe +data/ shader_list.tsv (MacNativeGames, MIT) +THIRD_PARTY_NOTICES.md +totk.ini.example, VERSION the default ini; the release string (git describe of the build) +``` + +- SDL3 comes from the upstream release, not MSYS2: MSYS2's DLL imports the LGPL libiconv. +- Every bundled third-party piece gets a row in `THIRD_PARTY_NOTICES.md`. +- The release contains nothing game-derived; `tools/check_content.py` runs over it in CI. + +## Constraints + +- The installer steps contain no Win32 calls: Linux follows with its own compiler and the same steps. +- The keys file path is passed to the tools; key values never appear in the UI, logs or files. +- The window is a front end over the CLI (`python -m installer --base ... --update ... --keys ... --lang ... --dest + ...`), which prints one JSON progress event per line. Tests and CI use the CLI. +- Unsigned exe: SmartScreen warns on first start. + +## Later + +- Re-warm the driver cache on start after a move, GPU or driver change (WIP on task/rewarm: stamp in + `shader-prepared.bin`). +- NSZ and XCZ input. Linux release. diff --git a/installer/__init__.py b/installer/__init__.py new file mode 100644 index 00000000..7c523f73 --- /dev/null +++ b/installer/__init__.py @@ -0,0 +1,10 @@ +"""The TotK installer: builds a playable folder from the player's own game files (docs/installer.md). + + python -m installer --base FILE --update FILE --keys FILE --lang CODE --dest DIR [--root DIR] [--temp DIR] + +Modules: layout (where the bundled tools are, release or repository), events (JSON progress on stdout and the log +file), state (totk-install.json), runner (which steps run: resume and update), steps (the six steps), ini (totk.ini), +cli (the command line). text/en.json holds the English text of every message id the events carry. + +Standard library only, apart from what the extraction tools import (cryptography). Runs on Python 3.14. +""" diff --git a/installer/__main__.py b/installer/__main__.py new file mode 100644 index 00000000..d7e5cb2f --- /dev/null +++ b/installer/__main__.py @@ -0,0 +1,5 @@ +import sys + +from installer.cli import main + +sys.exit(main()) diff --git a/installer/cli.py b/installer/cli.py new file mode 100644 index 00000000..8be9ce34 --- /dev/null +++ b/installer/cli.py @@ -0,0 +1,78 @@ +"""Command line: python -m installer --base FILE --update FILE --keys FILE --lang CODE --dest DIR [--root DIR] + [--temp DIR] [--verify] + +A first install needs every option. On a folder with totk-install.json (resume after a cancel or failure, or update +with a newer release) only --dest is needed: --lang defaults to the recorded language, and the game files are needed +only when the extraction has to run again. Prints JSON events on stdout (events.py). Exit code: 0 done, 1 failed, +2 bad command line, 3 cancelled (Ctrl+C or Ctrl+Break). +""" +import argparse +import hashlib +import signal +import sys +import time +from pathlib import Path + +from . import layout as layouts +from . import state as statefile +from .events import Reporter +from .runner import Context, run_steps +from .steps import all_steps + +EXIT = {'ok': 0, 'failed': 1, 'cancelled': 3} + + +class Parser(argparse.ArgumentParser): + def error(self, message): + Reporter().error('cli.usage', {'detail': message}) + sys.exit(2) + + +def temp_folder(base, dest): + """One temp folder per install folder, so two installs never share generated files.""" + tag = hashlib.sha256(str(dest).lower().encode()).hexdigest()[:8] + return base / f'{dest.name or "root"}-{tag}' + + +def cancel(signum, frame): + raise KeyboardInterrupt + + +def main(argv=None): + ap = Parser(prog='python -m installer', description=__doc__.splitlines()[0]) + ap.add_argument('--base', type=Path, help='base game, XCI or NSP') + ap.add_argument('--update', type=Path, help='1.2.1 update NSP') + ap.add_argument('--keys', type=Path, help='prod.keys') + ap.add_argument('--lang', help='game language, a Switch language code (en-US, en-GB, ja, de, ru, ...)') + ap.add_argument('--dest', type=Path, required=True, help='install folder') + ap.add_argument('--root', type=Path, help='release folder or repository (default: the folder above installer/)') + ap.add_argument('--temp', type=Path, help='temp folder (default: temp/ of the release, build/install-temp in a ' + 'repository)') + ap.add_argument('--verify', action='store_true', help='hash every kept file, also those this release made') + args = ap.parse_args(argv) + + # The window cancels with Ctrl+Break to the installer's process group (CREATE_NEW_PROCESS_GROUP), which also reaches + # the tools it runs; Python's default for it ends the process without recording anything. + for name in ('SIGBREAK', 'SIGTERM'): + if hasattr(signal, name): + signal.signal(getattr(signal, name), cancel) + layout = layouts.find(args.root) + dest = args.dest.resolve() + temp = temp_folder((args.temp or layout.temp).resolve(), dest) + log = temp / 'logs' / time.strftime('install-%Y%m%d-%H%M%S.log') + rep = Reporter(log_path=log) + t0 = time.monotonic() + try: + state = statefile.load(dest) + except statefile.StateError as e: + rep.error(e.id, {'path': str(e.path)}) + rep.emit('end', result='failed', seconds=0) + return 1 + ctx = Context(layout, dest, temp, rep, args, state=state, verify=args.verify) + steps = all_steps() + rep.emit('start', release=layout.release, mode=ctx.mode, dev=layout.dev, dest=str(dest), log=str(log), + steps=[s.id for s in steps]) + result = run_steps(steps, ctx) + rep.emit('end', result=result, seconds=round(time.monotonic() - t0, 1)) + rep.close() + return EXIT[result] diff --git a/installer/events.py b/installer/events.py new file mode 100644 index 00000000..121ccfe9 --- /dev/null +++ b/installer/events.py @@ -0,0 +1,119 @@ +"""Progress events: one JSON object per line on stdout for the window and tests, the same in English to a log file. + +Every event has "type" and "t" (seconds since the start). Types: + + start release, mode (install, resume, update), dest, log, steps (the step ids in order) + step step, state: started | done (seconds) | skipped (reason: current, not_needed) + progress step, phase, and done/total/unit (bytes, chunks, objects, shaders) when the amount is known + space dest, need, free (bytes on the install disk; with the temp folder on the same disk need includes it), + temp, temp_need, temp_free, same_disk + warning id, params + error id, params, step (absent outside a step) + end result: ok | failed | cancelled, seconds + +Text is never the payload: ids name a step ("step."), a phase ("phase..") or a message (""), +and installer/text/en.json holds their English text with {param} placeholders. The log renders events with it. +""" +import json +import sys +import threading +import time +from pathlib import Path + +CATALOG = Path(__file__).resolve().parent / 'text' / 'en.json' +PROGRESS_EVERY = 0.5 # seconds between two progress events of one phase + + +def load_catalog(path=CATALOG): + return json.loads(path.read_text(encoding='utf-8')) + + +def render(catalog, id, params=None): + """The English text of id, or the id itself when the catalog lacks it.""" + text = catalog.get(id, id) + try: + return text.format(**(params or {})) + except (KeyError, IndexError, ValueError): + return f'{text} {params}' + + +class Reporter: + def __init__(self, out=None, log_path=None): + self.out = out or sys.stdout + self.t0 = time.monotonic() + self.catalog = load_catalog() + self.lock = threading.Lock() + self.log_file = None + self.last_progress = {} + if log_path: + Path(log_path).parent.mkdir(parents=True, exist_ok=True) + self.log_file = open(log_path, 'a', encoding='utf-8', buffering=1) + self.log_path = log_path + + def elapsed(self): + return time.monotonic() - self.t0 + + def log(self, text): + """A line for the log file only.""" + if self.log_file: + with self.lock: + self.log_file.write(f'[{self.elapsed():8.1f}] {text}\n') + + def emit(self, type, **fields): + event = {'type': type, 't': round(self.elapsed(), 2), **fields} + with self.lock: + self.out.write(json.dumps(event, ensure_ascii=False) + '\n') + self.out.flush() + self.log(self.describe(event)) + return event + + def describe(self, e): + c = self.catalog + kind = e['type'] + if kind == 'step': + name = render(c, 'step.' + e['step']) + extra = f' ({e["seconds"]:.1f} s)' if 'seconds' in e else f' ({e["reason"]})' if 'reason' in e else '' + return f'{name}: {e["state"]}{extra}' + if kind == 'progress': + text = render(c, f'phase.{e["step"]}.{e["phase"]}') + if e.get('total'): + text += f' {e["done"]}/{e["total"]} {e.get("unit", "")}'.rstrip() + return text + if kind in ('warning', 'error'): + return f'{kind}: {render(c, e["id"], e.get("params"))} [{e["id"]}]' + return json.dumps(e, ensure_ascii=False) + + def step(self, step, state, **fields): + return self.emit('step', step=step, state=state, **fields) + + def progress(self, step, phase, done=None, total=None, unit=None, force=False): + """A progress event, at most one per PROGRESS_EVERY per phase unless it is the first or the last.""" + now = time.monotonic() + key = (step, phase) + last = self.last_progress.get(key) + final = total is not None and done is not None and done >= total + if last is not None and not final and not force and now - last < PROGRESS_EVERY: + return None + self.last_progress[key] = now + fields = {'step': step, 'phase': phase} + if done is not None: + fields['done'] = done + if total is not None: + fields['total'] = total + if unit: + fields['unit'] = unit + return self.emit('progress', **fields) + + def warning(self, id, **params): + return self.emit('warning', id=id, params=params) + + def error(self, id, params, step=None): + fields = {'id': id, 'params': params} + if step: + fields['step'] = step + return self.emit('error', **fields) + + def close(self): + if self.log_file: + self.log_file.close() + self.log_file = None diff --git a/installer/gui.py b/installer/gui.py new file mode 100644 index 00000000..ac031400 --- /dev/null +++ b/installer/gui.py @@ -0,0 +1,587 @@ +"""The installer window: python -m installer.gui + +A front end over the CLI (docs/installer.md): it collects the game files, the keys file path, the game language and the +install folder, runs `python -m installer ...` as a subprocess and shows its JSON events. Decisions live in guilogic.py; +this file only draws. The window text is in installer/text/.json under "ui.*" and "launcher.*" ids. + + --selftest [--shots DIR] build every screen in both languages and close by itself (PNG per screen with --shots) + --lang en|ru window language (default: the Windows display language) +""" +import argparse +import json +import os +import sys +import tkinter as tk +from pathlib import Path +from tkinter import filedialog, messagebox, ttk + +try: + from PIL import ImageGrab # only the screenshots of --selftest --shots need it; the release has no PIL +except ImportError: + ImageGrab = None + +from . import ini +from . import layout as layouts +from .guilogic import (DONE, FAILED, PENDING, RUNNING, SKIPPED, UI_LANGUAGES, Catalog, Form, Model, Process, + amount, default_game_language, describe_event, parse_line, windows_ui_language) + +POLL_MS = 100 +GLYPH = {PENDING: '\u25cb', RUNNING: '\u25b6', DONE: '\u2714', SKIPPED: '\u2013', FAILED: '\u2716'} +COLOR = {PENDING: '#808080', RUNNING: '#1a5fb4', DONE: '#26a269', SKIPPED: '#808080', FAILED: '#c01c28'} +PAD = 14 + + +def fix_tcl_paths(): + """The embeddable Python keeps Tcl/Tk next to python.exe (python/tcl); point Tcl at it when it is not found.""" + tcl = Path(sys.executable).resolve().parent / 'tcl' + if tcl.is_dir() and 'TCL_LIBRARY' not in os.environ: + for lib, var in (('tcl9.0', 'TCL_LIBRARY'), ('tk9.0', 'TK_LIBRARY')): + if (tcl / lib).is_dir(): + os.environ[var] = str(tcl / lib) + + +class App: + def __init__(self, root, ui_lang, layout, run_args=None): + self.root, self.layout = root, layout + self.cat = Catalog(ui_lang) + self.model = Model() + self.process = None + self.closing = False + self.run_args = run_args or {} # extra CLI options for tests: temp, verify + self.log_entries = [] # events and raw lines of the run, rendered by render_log + self.bound = [] # (widget, option, id, params) retranslated on a language switch + self.vars = {n: tk.StringVar() for n in ('base', 'update', 'keys', 'dest')} + self.vars['lang'] = tk.StringVar() + self.lang_code = default_game_language(ui_lang) + self.screen = None + + root.title(self.cat('ui.title')) + root.minsize(680, 560) + root.protocol('WM_DELETE_WINDOW', self.on_close) + style = ttk.Style(root) + style.configure('Title.TLabel', font=('Segoe UI', 15, 'bold')) + style.configure('Notice.TLabel', background='#fff3cd', foreground='#664d03', padding=8) + style.configure('Err.TLabel', foreground='#c01c28') + + top = ttk.Frame(root, padding=(PAD, PAD, PAD, 0)) + top.pack(fill='x') + self.title = self.label(top, 'ui.title', style='Title.TLabel') + self.title.pack(side='left') + sw = ttk.Frame(top) + sw.pack(side='right') + self.ui_var = tk.StringVar(value=self.cat.lang) + for code in UI_LANGUAGES: + ttk.Radiobutton(sw, text=code.upper(), value=code, variable=self.ui_var, style='Toolbutton', + command=self.switch_language).pack(side='left', padx=1) + self.body = ttk.Frame(root, padding=PAD) + self.body.pack(fill='both', expand=True) + self.frames = {'form': self.build_form(), 'run': self.build_run(), 'finish': self.build_finish()} + self.vars['lang'].set(self.lang_label(self.lang_code)) + self.vars['dest'].trace_add('write', lambda *_: self.refresh_form()) + self.show('form') + + # ---- text ----------------------------------------------------------------------------------------------------- + + def bind_text(self, widget, option, id, **params): + self.bound.append((widget, option, id, params)) + widget.configure(**{option: self.cat(id, **params)}) + + def label(self, parent, id, **kw): + w = ttk.Label(parent, **kw) + self.bind_text(w, 'text', id) + return w + + def button(self, parent, id, command, **kw): + w = ttk.Button(parent, command=command, **kw) + self.bind_text(w, 'text', id) + return w + + def retext(self): + self.root.title(self.cat('ui.title')) + for widget, option, id, params in self.bound: + widget.configure(**{option: self.cat(id, **params)}) + self.fill_languages() + self.refresh_form() + self.refresh_run() + self.refresh_finish() + self.render_log() + + def switch_language(self): + self.cat.set_language(self.ui_var.get()) + code = self.selected_lang() + self.retext() + self.vars['lang'].set(self.lang_label(code)) + + def lang_label(self, code): + return self.cat('lang.' + code) + + def fill_languages(self): + code = self.selected_lang() + self.lang_box.configure(values=[self.lang_label(c) for c in ini.LANGUAGES]) + self.vars['lang'].set(self.lang_label(code)) + + def selected_lang(self): + """The game language code behind the combobox text (kept when it is not one of the labels).""" + shown = self.vars['lang'].get() + for code in ini.LANGUAGES: + for lang in UI_LANGUAGES: + if shown == Catalog(lang)('lang.' + code): + return code + return self.lang_code + + # ---- screens -------------------------------------------------------------------------------------------------- + + def show(self, name): + for f in self.frames.values(): + f.pack_forget() + self.frames[name].pack(fill='both', expand=True) + self.screen = name + + def notice(self, parent): + w = ttk.Label(parent, style='Notice.TLabel', wraplength=620, justify='left') + self.bind_text(w, 'text', 'ui.notice') + return w + + def build_form(self): + f = ttk.Frame(self.body) + self.form_intro = self.label(f, 'ui.intro', wraplength=620, justify='left') + self.form_intro.pack(anchor='w', pady=(0, 8)) + self.notice(f).pack(fill='x', pady=(0, 10)) + grid = ttk.Frame(f) + grid.pack(fill='x') + grid.columnconfigure(1, weight=1) + rows = (('base', 'ui.base', 'ui.pick_game'), ('update', 'ui.update', 'ui.pick_update'), + ('keys', 'ui.keys', 'ui.pick_keys')) + self.form_rows = {} + for i, (name, label_id, pick_id) in enumerate(rows): + lab = self.label(grid, label_id) + lab.grid(row=i, column=0, sticky='w', pady=4, padx=(0, 10)) + ttk.Entry(grid, textvariable=self.vars[name]).grid(row=i, column=1, sticky='ew', pady=4) + b = self.button(grid, 'ui.browse', lambda n=name, p=pick_id: self.pick_file(n, p)) + b.grid(row=i, column=2, padx=(8, 0)) + self.form_rows[name] = lab + i = len(rows) + self.label(grid, 'ui.game_language').grid(row=i, column=0, sticky='w', pady=4) + self.lang_box = ttk.Combobox(grid, textvariable=self.vars['lang'], state='readonly', width=30) + self.lang_box.grid(row=i, column=1, sticky='w', pady=4) + self.lang_touched = False + self.lang_box.bind('<>', lambda _e: setattr(self, 'lang_touched', True)) + self.fill_languages() + self.label(grid, 'ui.dest').grid(row=i + 1, column=0, sticky='w', pady=4) + ttk.Entry(grid, textvariable=self.vars['dest']).grid(row=i + 1, column=1, sticky='ew', pady=4) + self.button(grid, 'ui.browse', self.pick_dest).grid(row=i + 1, column=2, padx=(8, 0)) + self.update_hint = ttk.Label(f, wraplength=620, justify='left') + self.update_hint.pack(anchor='w', pady=(8, 0)) + self.form_error = ttk.Label(f, style='Err.TLabel', wraplength=620, justify='left') + self.form_error.pack(anchor='w', pady=(8, 0)) + self.label(f, 'ui.keys_private', wraplength=620, justify='left').pack(anchor='w', pady=(8, 0)) + bar = ttk.Frame(f) + bar.pack(side='bottom', fill='x', pady=(12, 0)) + self.start_button = self.button(bar, 'ui.start', self.start) + self.start_button.pack(side='right') + return f + + def build_run(self): + f = ttk.Frame(self.body) + self.run_header = ttk.Label(f, font=('Segoe UI', 11, 'bold')) + self.run_header.pack(anchor='w') + self.step_rows = {} + self.steps_box = ttk.Frame(f) + self.steps_box.pack(fill='x', pady=(8, 8)) + self.phase_label = ttk.Label(f, wraplength=620, justify='left') + self.phase_label.pack(anchor='w') + self.bar = ttk.Progressbar(f, mode='determinate', maximum=1000) + self.bar.pack(fill='x', pady=(4, 6)) + self.space_label = ttk.Label(f, wraplength=620, justify='left') + self.space_label.pack(anchor='w') + self.warn_label = ttk.Label(f, wraplength=620, justify='left', foreground='#9a6700') + self.warn_label.pack(anchor='w') + bar = ttk.Frame(f) + bar.pack(side='bottom', fill='x', pady=(8, 0)) + self.cancel_button = self.button(bar, 'ui.cancel', self.cancel) + self.cancel_button.pack(side='right') + self.log_toggle = self.button(bar, 'ui.show_log', self.toggle_log) + self.log_toggle.pack(side='left') + self.log_frame = ttk.Frame(f) + self.log = tk.Text(self.log_frame, height=10, wrap='word', state='disabled', font=('Consolas', 9)) + scroll = ttk.Scrollbar(self.log_frame, command=self.log.yview) + self.log.configure(yscrollcommand=scroll.set) + scroll.pack(side='right', fill='y') + self.log.pack(side='left', fill='both', expand=True) + self.log_open = False + return f + + def build_finish(self): + f = ttk.Frame(self.body) + self.finish_title = ttk.Label(f, style='Title.TLabel', wraplength=620, justify='left') + self.finish_title.pack(anchor='w') + self.finish_text = ttk.Label(f, wraplength=620, justify='left') + self.finish_text.pack(anchor='w', pady=(8, 8)) + self.finish_steps = ttk.Frame(f) + self.finish_steps.pack(fill='x', pady=(0, 8)) + self.finish_notice = self.notice(f) + self.finish_notice.pack(fill='x', pady=(8, 0)) + bar = ttk.Frame(f) + bar.pack(side='bottom', fill='x', pady=(12, 0)) + self.button(bar, 'ui.close', self.on_close).pack(side='right') + self.open_button = self.button(bar, 'ui.open_folder', self.open_folder) + self.open_button.pack(side='right', padx=(0, 8)) + self.back_button = self.button(bar, 'ui.back', lambda: self.show('form')) + self.back_button.pack(side='left') + return f + + # ---- form ----------------------------------------------------------------------------------------------------- + + def form(self): + return Form(base=self.vars['base'].get(), update=self.vars['update'].get(), keys=self.vars['keys'].get(), + lang=self.selected_lang(), dest=self.vars['dest'].get()) + + def refresh_form(self): + form = self.form() + state = form.installed_state() + if state is not None: + self.update_hint.configure(text=self.cat('ui.update_mode')) + if state.get('lang') in ini.LANGUAGES and not self.lang_touched: + self.vars['lang'].set(self.lang_label(state['lang'])) + else: + self.update_hint.configure(text='') + + def pick_file(self, name, pick_id): + types = [(self.cat('ui.type_game'), '*.nsp *.xci'), (self.cat('ui.type_all'), '*.*')] + if name == 'update': + types = [(self.cat('ui.type_nsp'), '*.nsp'), (self.cat('ui.type_all'), '*.*')] + if name == 'keys': + types = [(self.cat('ui.type_keys'), '*.keys'), (self.cat('ui.type_all'), '*.*')] + path = filedialog.askopenfilename(title=self.cat(pick_id), filetypes=types) + if path: + self.vars[name].set(os.path.normpath(path)) + + def pick_dest(self): + path = filedialog.askdirectory(title=self.cat('ui.pick_dest'), mustexist=False) + if path: + self.vars['dest'].set(os.path.normpath(path)) + + # ---- run ------------------------------------------------------------------------------------------------------ + + def start(self): + form = self.form() + problems = form.problems() + if problems: + self.form_error.configure(text=' '.join(self.cat(p) for p in problems)) + return + self.form_error.configure(text='') + command = form.command(self.layout.python, root=self.layout.root if self.layout.dev else None, + temp=self.run_args.get('temp'), verify=self.run_args.get('verify', False)) + self.begin(command, self.layout.root) + + def begin(self, command, cwd): + self.model = Model() + self.model.dest = self.vars['dest'].get().strip() + self.clear_log() + self.build_step_rows(self.model.steps) + self.cancel_button.configure(state='normal') + self.bound = [b for b in self.bound if b[0] is not self.cancel_button] + self.bind_text(self.cancel_button, 'text', 'ui.cancel') + self.show('run') + self.refresh_run() + self.process = Process(command, cwd=cwd, helper_python=self.layout.python) + self.root.after(POLL_MS, self.poll) + + def build_step_rows(self, steps): + for w in self.steps_box.winfo_children(): + w.destroy() + self.step_rows = {} + for s in steps: + row = ttk.Frame(self.steps_box) + row.pack(fill='x', pady=1) + glyph = ttk.Label(row, width=3) + glyph.pack(side='left') + name = ttk.Label(row) + name.pack(side='left') + self.step_rows[s] = (glyph, name) + + def poll(self): + if self.process is None: + return + try: + while True: + line = self.process.lines.get_nowait() + if line is not None: + self.feed(line) + except Exception: + pass + code = self.process.poll() + self.refresh_run() + if code is None: + self.root.after(POLL_MS, self.poll) + return + self.model.finish(code) + self.process = None + self.refresh_run() + self.refresh_finish() + self.show('finish') + if self.closing: + self.root.destroy() + + def feed(self, line): + """One stdout line of the CLI: an event updates the model and the log, anything else goes to the log as is.""" + event = parse_line(line) + if event is None: + if line.strip(): + self.log_entries.append(line.rstrip()) + self.append_log(line.rstrip()) + return + before = list(self.model.steps) + self.model.apply(event) + if self.model.steps != before: + self.build_step_rows(self.model.steps) + self.log_entries.append(event) + self.append_log(describe_event(self.cat, event)) + + def refresh_run(self): + m = self.model + for s, (glyph, name) in self.step_rows.items(): + state = m.states.get(s, PENDING) + glyph.configure(text=GLYPH[state], foreground=COLOR[state]) + name.configure(text=self.cat('step.' + s) + (f' ({self.cat("ui.state_" + state)})' + if state in (SKIPPED, FAILED) else ''), + font=('Segoe UI', 10, 'bold') if state == RUNNING else ('Segoe UI', 10)) + mode = m.mode or 'install' + self.run_header.configure(text=self.cat('ui.running_' + mode if mode in ('install', 'resume', 'update') + else 'ui.running_install')) + if m.current and m.phase: + text = self.cat(f'phase.{m.current}.{m.phase}') + if m.total: + text += ' ' + amount(self.cat, m.done, m.total, m.unit) + elif m.current: + text = self.cat('step.' + m.current) + else: + text = '' + self.phase_label.configure(text=text) + frac = m.fraction() + if frac is None: + if str(self.bar.cget('mode')) != 'indeterminate': + self.bar.configure(mode='indeterminate') + self.bar.start(15) + else: + if str(self.bar.cget('mode')) == 'indeterminate': + self.bar.stop() + self.bar.configure(mode='determinate') + self.bar.configure(value=frac * 1000) + self.space_label.configure(text=m.need_text(self.cat)) + self.warn_label.configure(text='\n'.join(self.cat(i, **p) for i, p in m.warnings)) + + def toggle_log(self): + self.log_open = not self.log_open + if self.log_open: + self.log_frame.pack(fill='both', expand=True, pady=(8, 0)) + else: + self.log_frame.pack_forget() + self.bound = [b for b in self.bound if b[0] is not self.log_toggle] + self.bind_text(self.log_toggle, 'text', 'ui.hide_log' if self.log_open else 'ui.show_log') + + def clear_log(self): + self.log_entries = [] + self.render_log() + + def render_log(self): + """The whole log from the kept events in the current language (raw CLI lines stay as they are).""" + self.log.configure(state='normal') + self.log.delete('1.0', 'end') + for entry in self.log_entries: + text = entry if isinstance(entry, str) else describe_event(self.cat, entry) + if text: + self.log.insert('end', text + '\n') + self.log.see('end') + self.log.configure(state='disabled') + + def append_log(self, text): + if text: + self.log.configure(state='normal') + self.log.insert('end', text + '\n') + self.log.see('end') + self.log.configure(state='disabled') + + def cancel(self): + if self.process: + self.cancel_button.configure(state='disabled') + self.bound = [b for b in self.bound if b[0] is not self.cancel_button] + self.bind_text(self.cancel_button, 'text', 'ui.cancelling') + self.process.cancel() + + def on_close(self): + if self.process: + if not messagebox.askyesno(self.cat('ui.title'), self.cat('ui.confirm_cancel')): + return + self.closing = True + self.cancel() + return + self.root.destroy() + + # ---- finish --------------------------------------------------------------------------------------------------- + + def refresh_finish(self): + m = self.model + result = m.result or 'ok' + self.finish_title.configure(text=self.cat('ui.result_' + result)) + if result == 'ok': + text = self.cat('ui.done_text', dest=m.dest) + elif result == 'cancelled': + text = self.cat('ui.cancelled_text') + else: + err = m.error or ('ui.process_died', {'code': m.exit_code}, None) + text = self.cat(err[0], **err[1]) + if m.log_path: + text += '\n\n' + self.cat('ui.log_file', path=m.log_path) + self.finish_text.configure(text=text) + for w in self.finish_steps.winfo_children(): + w.destroy() + for s in m.steps: + state = m.states.get(s, PENDING) + ttk.Label(self.finish_steps, text=f'{GLYPH[state]} {self.cat("step." + s)}', + foreground=COLOR[state]).pack(anchor='w') + if result == 'ok': + self.open_button.state(['!disabled']) + else: + self.open_button.state(['disabled' if not Path(m.dest or '.').is_dir() else '!disabled']) + + def open_folder(self): + dest = self.model.dest + if dest and Path(dest).is_dir(): + os.startfile(dest) + + +# ---- self test --------------------------------------------------------------------------------------------------- + +SAMPLE_START = {'type': 'start', 'release': 'selftest', 'mode': 'install', 'dest': 'Z:/selftest/TotK', 'log': 'D:\\log.txt', + 'steps': ['check', 'extract', 'lift', 'compile', 'prepare', 'cleanup']} +SAMPLE_RUN = [ + SAMPLE_START, + {'type': 'plan', 'run': ['check', 'extract'], 'skip': {}}, + {'type': 'step', 'step': 'check', 'state': 'started'}, + {'type': 'progress', 'step': 'check', 'phase': 'inputs'}, + {'type': 'space', 'dest': 'Z:/selftest/TotK', 'need': 36_400_000_000, 'free': 210_000_000_000, 'temp': 'C:\\t', + 'temp_need': 0, 'temp_free': 0, 'same_disk': True}, + {'type': 'warning', 'id': 'warn.lang_needs_fonts', 'params': {'lang': 'ko'}}, + {'type': 'step', 'step': 'check', 'state': 'done', 'seconds': 12.0}, + {'type': 'step', 'step': 'extract', 'state': 'started'}, + {'type': 'progress', 'step': 'extract', 'phase': 'copy', 'done': 6_000_000_000, 'total': 16_000_000_000, + 'unit': 'bytes'}, +] + + +SAMPLE_OK = [SAMPLE_START, {'type': 'plan', 'run': ['check', 'extract'], 'skip': {'lift': 'current', 'compile': 'current', + 'prepare': 'current', 'cleanup': 'not_needed'}}, + {'type': 'step', 'step': 'check', 'state': 'started'}, + {'type': 'step', 'step': 'check', 'state': 'done', 'seconds': 1.0}, + {'type': 'step', 'step': 'extract', 'state': 'started'}, + {'type': 'step', 'step': 'extract', 'state': 'done', 'seconds': 1.0}, + {'type': 'step', 'step': 'lift', 'state': 'skipped', 'reason': 'current'}, + {'type': 'step', 'step': 'compile', 'state': 'skipped', 'reason': 'current'}, + {'type': 'step', 'step': 'prepare', 'state': 'skipped', 'reason': 'current'}, + {'type': 'step', 'step': 'cleanup', 'state': 'done', 'seconds': 0.1}, + {'type': 'end', 'result': 'ok', 'seconds': 3}] + + +def grab(root, path): + if ImageGrab is None: + raise SystemExit('--shots needs Pillow') + root.attributes('-topmost', True) + root.update() + x, y, w, h = root.winfo_rootx(), root.winfo_rooty(), root.winfo_width(), root.winfo_height() + ImageGrab.grab(bbox=(x, y, x + w, y + h), all_screens=True).save(path) + + +def selftest(layout, shots=None): + """Every screen in both languages; the window closes itself. Returns 0 when nothing raised.""" + fix_tcl_paths() + root = tk.Tk() + root.geometry('720x640+60+40') + app = App(root, 'en', layout) + steps, mixed = [], [] + + def shot(name): + root.update() + if shots: + Path(shots).mkdir(parents=True, exist_ok=True) + grab(root, Path(shots) / f'{name}.png') + + def run_all(): + for lang in UI_LANGUAGES: + app.ui_var.set(lang) + app.switch_language() + tag = lang + app.vars['dest'].set('Z:/selftest/TotK') + app.vars['base'].set('Z:/selftest/totk.xci') + app.vars['update'].set('Z:/selftest/totk-1.2.1.nsp') + app.vars['keys'].set('Z:/selftest/prod.keys') + app.show('form') + app.start() # missing files: shows the problems + steps.append(app.form_error.cget('text')) + shot(f'form-{tag}') + app.model = Model() + app.clear_log() + app.build_step_rows(app.model.steps) + app.show('run') + for e in SAMPLE_RUN: + app.feed(json.dumps(e)) + app.log_open = False + app.toggle_log() + app.refresh_run() + shot(f'run-{tag}') + if lang == 'en': # the log follows the window language: no English line may remain after the switch + app.ui_var.set('ru') + app.switch_language() + text = app.log.get('1.0', 'end') + mixed.append('Checking' in text or 'Extracting' not in text and 'Распаковка' not in text) + app.ui_var.set('en') + app.switch_language() + app.model = Model() + for e in SAMPLE_OK: + app.model.apply(e) + app.model.dest = 'Z:/selftest/TotK' + app.model.finish(0) + app.refresh_finish() + app.show('finish') + shot(f'finish-ok-{tag}') + app.model = Model() + for e in SAMPLE_RUN[:3] + [{'type': 'error', 'id': 'check.disk_space', 'step': 'check', + 'params': {'path': 'Z:/selftest/TotK', 'need_gb': 36.4, 'free_gb': 20.1}}, + {'type': 'end', 'result': 'failed', 'seconds': 3}]: + app.model.apply(e) + app.model.finish(1) + app.refresh_finish() + shot(f'finish-failed-{tag}') + app.model = Model() + app.model.finish(3) + app.refresh_finish() + shot(f'finish-cancelled-{tag}') + app.toggle_log() + root.destroy() + + root.after(200, run_all) + root.after(60000, root.destroy) # never leaves a window open + root.mainloop() + return 0 if len(steps) == len(UI_LANGUAGES) and all(steps) and mixed == [False] else 1 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--selftest', action='store_true') + ap.add_argument('--shots', type=Path) + ap.add_argument('--lang', choices=UI_LANGUAGES) + ap.add_argument('--root', type=Path) + args = ap.parse_args(argv) + layout = layouts.find(args.root) + if args.selftest: + return selftest(layout, args.shots) + fix_tcl_paths() + root = tk.Tk() + root.geometry('720x640') + App(root, args.lang or windows_ui_language(), layout) + root.mainloop() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/installer/guilogic.py b/installer/guilogic.py new file mode 100644 index 00000000..7d517024 --- /dev/null +++ b/installer/guilogic.py @@ -0,0 +1,361 @@ +"""The installer window without Tk: text catalogs, the form, the event model and the CLI subprocess. + +gui.py draws; everything it decides lives here so tests/installer_gui_test.py runs it headless. + + Catalog text of an id in the window language (installer/text/.json), English when that lacks it + Form what the player entered; command() and problems() build and check the CLI call + Model folds the CLI's JSON events (events.py) into step states, progress, space, errors and the result + Process runs `python -m installer ...`, cancels it with Ctrl+Break and feeds lines to a queue +""" +import ctypes +import json +import os +import queue +import subprocess +import sys +import threading +import time +from dataclasses import dataclass +from pathlib import Path + +from . import ini +from . import state as statefile +from .events import render + +TEXT = Path(__file__).resolve().parent / 'text' +UI_LANGUAGES = ('en', 'ru') +STEP_ORDER = ('check', 'extract', 'lift', 'compile', 'prepare', 'cleanup') +# message parameter -> catalog id prefix of the names that replace its raw values +PARAM_NAMES = {'lang': 'lang.', 'known': 'lang.', 'step': 'step.', 'input': 'param.input.'} +LANG_RU = 0x19 # primary language id of Russian in a Windows LANGID + + +def load_texts(lang): + path = TEXT / f'{lang}.json' + return json.loads(path.read_text(encoding='utf-8')) if path.is_file() else {} + + +class Catalog: + """Window text: ids from text/.json, then text/en.json, then the id itself.""" + + def __init__(self, lang='en'): + self.english = load_texts('en') + self.set_language(lang) + + def set_language(self, lang): + self.lang = lang if lang in UI_LANGUAGES else 'en' + self.texts = load_texts(self.lang) + + def has(self, id): + return id in self.texts or id in self.english + + def __call__(self, id, **params): + text = self.texts.get(id) or self.english.get(id) + if text is None: + return id + return render({id: text}, id, {k: self.param(k, v) for k, v in params.items()}) + + def param(self, name, value): + """A message parameter as the player should read it: language codes, step ids and input names become the + catalog's names (the ones the pickers and the step list show); the rest stays as it is.""" + prefix = PARAM_NAMES.get(name) + if prefix: + if isinstance(value, (list, tuple)): + return ', '.join(self.param(name, v) for v in value) + ident = prefix + str(value) + if self.has(ident): + return self(ident) + if name == 'known': + return ', '.join(self(prefix + c) if self.has(prefix + c) else c for c in str(value).split(', ')) + return flat(value) + + +def flat(value): + return ', '.join(str(v) for v in value) if isinstance(value, (list, tuple)) else value + + +def windows_ui_language(): + """'ru' when the Windows display language is Russian, else 'en'.""" + try: + langid = ctypes.windll.kernel32.GetUserDefaultUILanguage() + except (AttributeError, OSError): + return 'en' + return 'ru' if langid & 0x3ff == LANG_RU else 'en' + + +def default_game_language(ui): + return 'ru' if ui == 'ru' else 'en-US' + + +def gib(n): + """Bytes as decimal gigabytes with one digit, the unit the check.disk_space message uses.""" + return f'{n / 1e9:.1f}' + + +def amount(cat, done, total, unit): + """'6.0 / 16.0 GB', '120 / 4000 files'..., the progress of a phase whose size is known.""" + if unit == 'bytes': + return cat('ui.amount_bytes', done=gib(done or 0), total=gib(total)) + id = f'ui.amount_{unit}' + return cat(id if cat.has(id) else 'ui.amount_other', done=done or 0, total=total, unit=unit or '') + + +# ---- the form ---------------------------------------------------------------------------------------------------- + +@dataclass +class Form: + base: str = '' + update: str = '' + keys: str = '' + lang: str = '' + dest: str = '' + + def installed_state(self): + """The recorded state of the install folder (totk-install.json), or None. Never raises.""" + if not self.dest.strip(): + return None + try: + return statefile.load(Path(self.dest.strip())) + except statefile.StateError: + return None + + def update_mode(self): + return self.installed_state() is not None + + def problems(self): + """Ids of the entries that are missing or wrong, in screen order. A folder with totk-install.json needs only + itself; the CLI asks for the game files when it must extract again.""" + found = [] + update = self.update_mode() + if not self.dest.strip(): + found.append('ui.need_dest') + if not update: + for name in ('base', 'update', 'keys'): + value = getattr(self, name).strip() + if not value: + found.append(f'ui.need_{name}') + elif not Path(value).is_file(): + found.append(f'ui.missing_{name}') + if self.lang not in ini.LANGUAGES: + found.append('ui.need_lang') + else: + for name in ('base', 'update', 'keys'): + value = getattr(self, name).strip() + if value and not Path(value).is_file(): + found.append(f'ui.missing_{name}') + return found + + def command(self, python, root=None, temp=None, verify=False): + cmd = [str(python), '-m', 'installer'] + for flag, value in (('--base', self.base), ('--update', self.update), ('--keys', self.keys), + ('--lang', self.lang), ('--dest', self.dest)): + if value.strip(): + cmd += [flag, value.strip()] + if root: + cmd += ['--root', str(root)] + if temp: + cmd += ['--temp', str(temp)] + if verify: + cmd.append('--verify') + return cmd + + +# ---- events -> state --------------------------------------------------------------------------------------------- + +PENDING, RUNNING, DONE, SKIPPED, FAILED = 'pending', 'running', 'done', 'skipped', 'failed' + + +def parse_line(line): + """The event dict of a stdout line, or None when it is not a JSON event (a traceback line, noise).""" + line = line.strip() + if not line.startswith('{'): + return None + try: + event = json.loads(line) + except ValueError: + return None + return event if isinstance(event, dict) and isinstance(event.get('type'), str) else None + + +class Model: + """What the window shows, folded from events. apply() takes one event; nothing else mutates it.""" + + def __init__(self): + self.steps = list(STEP_ORDER) + self.states = {s: PENDING for s in self.steps} + self.mode = '' + self.release = '' + self.log_path = '' + self.dest = '' + self.current = None # step id running + self.phase = None + self.done = self.total = None + self.unit = None + self.space = None # the space event + self.warnings = [] # (id, params) + self.error = None # (id, params, step) + self.result = None # ok | failed | cancelled once the end event arrived + self.seconds = None + self.exit_code = None + + def apply(self, e): + kind = e['type'] + if kind == 'start': + self.__init__() # a new run: nothing of an earlier one stays + self.steps = list(e.get('steps') or STEP_ORDER) + self.states = {s: PENDING for s in self.steps} + self.mode, self.release = e.get('mode', ''), e.get('release', '') + self.log_path, self.dest = e.get('log', ''), e.get('dest', '') + elif kind == 'plan': + for s, reason in (e.get('skip') or {}).items(): + if s in self.states and self.states[s] == PENDING: + self.states[s] = SKIPPED + elif kind == 'step': + s, state = e.get('step'), e.get('state') + if s not in self.states: + self.states[s] = PENDING + self.steps.append(s) + if state == 'started': + self.states[s], self.current = RUNNING, s + self.phase = self.done = self.total = self.unit = None + elif state == 'done': + self.states[s] = DONE + self.current = None if self.current == s else self.current + elif state == 'skipped': + self.states[s] = SKIPPED + elif kind == 'progress': + if e.get('step') in self.states and self.states[e['step']] == PENDING: + self.states[e['step']] = RUNNING + self.current = e.get('step', self.current) + self.phase = e.get('phase') + self.done, self.total, self.unit = e.get('done'), e.get('total'), e.get('unit') + elif kind == 'space': + self.space = e + elif kind == 'warning': + self.warnings.append((e.get('id', ''), e.get('params') or {})) + elif kind == 'error': + self.error = (e.get('id', ''), e.get('params') or {}, e.get('step')) + if e.get('step') in self.states: + self.states[e['step']] = FAILED + elif kind == 'end': + self.result = e.get('result') + self.seconds = e.get('seconds') + self.current = None + + def fraction(self): + """0..1 of the current phase, or None when its size is unknown.""" + if self.total: + return max(0.0, min(1.0, (self.done or 0) / self.total)) + return None + + def finish(self, exit_code): + """The process ended. A missing end event (the process died) counts as failed, or cancelled on exit code 3.""" + self.exit_code = exit_code + if self.result is None: + self.result = 'cancelled' if exit_code == 3 else 'failed' + if self.result == 'failed' and self.error is None: + self.error = ('ui.process_died', {'code': exit_code}, self.current) + for s, state in self.states.items(): + if state == RUNNING: + self.states[s] = FAILED if self.result == 'failed' else PENDING + + def need_text(self, cat): + """The space line, or '' before the check step reported.""" + sp = self.space + if not sp: + return '' + key = 'ui.space_one' if sp.get('same_disk') or not sp.get('temp_need') else 'ui.space_two' + return cat(key, path=sp.get('dest', ''), need=gib(sp.get('need', 0)), free=gib(sp.get('free', 0)), + temp=sp.get('temp', ''), temp_need=gib(sp.get('temp_need', 0)), + temp_free=gib(sp.get('temp_free', 0))) + + +def describe_event(cat, e): + """One log line for an event, in the window language.""" + kind = e['type'] + if kind == 'step': + text = f'{cat("step." + e["step"])}: {cat("ui.state_" + e["state"])}' + if 'seconds' in e: + text += f' ({cat("ui.seconds", n=format(e["seconds"], ".1f"))})' + elif e.get('reason'): + text += f' ({e["reason"]})' + return text + if kind == 'progress': + text = cat(f'phase.{e["step"]}.{e["phase"]}') + if e.get('total'): + text += ' ' + amount(cat, e.get('done'), e['total'], e.get('unit')) + return text + if kind in ('warning', 'error'): + return f'{cat("ui.log_" + kind)}: {cat(e["id"], **(e.get("params") or {}))}' + if kind == 'space': + return cat('ui.log_space', need=gib(e.get('need', 0)), free=gib(e.get('free', 0))) + if kind == 'end': + return f'{cat("ui.log_end")}: {cat("ui.result_" + str(e.get("result")))}' + return '' + + +# ---- the CLI subprocess ------------------------------------------------------------------------------------------ + +CREATE_NEW_PROCESS_GROUP = 0x200 +CREATE_NO_WINDOW = 0x08000000 + +_BREAK = """ +import ctypes, sys, time +k = ctypes.windll.kernel32 +pid = int(sys.argv[1]) +k.FreeConsole() +if k.AttachConsole(pid): + k.SetConsoleCtrlHandler(None, True) + k.GenerateConsoleCtrlEvent(1, pid) + time.sleep(1) + k.FreeConsole() +else: + sys.exit(1) +""" + + +def send_break(proc, python=None): + """Ctrl+Break to the process group of proc (started with CREATE_NEW_PROCESS_GROUP). The window itself may have no + console (pythonw), so a short helper process attaches to the child's console and sends it.""" + helper = [str(python or sys.executable), '-c', _BREAK, str(proc.pid)] + flags = CREATE_NO_WINDOW if os.name == 'nt' else 0 + r = subprocess.run(helper, creationflags=flags, capture_output=True) + return r.returncode == 0 + + +class Process: + """`python -m installer ...` with its stdout lines going to self.lines (a queue). + + poll() returns the exit code once the process ended and its output is drained, else None.""" + + def __init__(self, command, cwd=None, helper_python=None, creationflags=None): + self.command = command + self.helper_python = helper_python + self.lines = queue.Queue() + flags = (CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW) if creationflags is None else creationflags + env = dict(os.environ, PYTHONIOENCODING='utf-8', PYTHONUTF8='1', PYTHONUNBUFFERED='1') + self.proc = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, creationflags=flags) + self.cancelled_at = None + self.reader = threading.Thread(target=self.read, daemon=True) + self.reader.start() + + def read(self): + for raw in iter(self.proc.stdout.readline, b''): + self.lines.put(raw.decode('utf-8', errors='replace')) + self.proc.stdout.close() + + def cancel(self): + if self.cancelled_at is None and self.proc.poll() is None: + self.cancelled_at = time.monotonic() + send_break(self.proc, self.helper_python) + + def poll(self): + code = self.proc.poll() + if code is not None and not self.reader.is_alive() and self.lines.empty(): + return code + return None + + def kill(self): + if self.proc.poll() is None: + self.proc.kill() diff --git a/installer/hostcheck.c b/installer/hostcheck.c new file mode 100644 index 00000000..2f86571a --- /dev/null +++ b/installer/hostcheck.c @@ -0,0 +1,65 @@ +// Host check of the installer's first step, compiled with the bundled clang at install time: does the CPU run +// -march=x86-64-v3 code, and does a Vulkan 1.3 device exist. Prints one JSON object on stdout. +// The Vulkan loader is opened at run time, so a machine without one gets "loader": false instead of a failed start. +#include +#include +#include +#ifdef _WIN32 +#include +#define open_lib() LoadLibraryA("vulkan-1.dll") +#define lib_sym(l, n) ((void *)GetProcAddress((l), (n))) +#else +#include +#define open_lib() dlopen("libvulkan.so.1", RTLD_NOW) +#define lib_sym(l, n) dlsym((l), (n)) +#endif + +static void json_string(const char *s) { + putchar('"'); + for (; *s; s++) { + if (*s == '"' || *s == '\\') printf("\\%c", *s); + else if ((unsigned char)*s < 0x20) printf("\\u%04x", *s); + else putchar(*s); + } + putchar('"'); +} + +int main(void) { + __builtin_cpu_init(); + printf("{\"x86_64_v3\": %s, ", __builtin_cpu_supports("x86-64-v3") ? "true" : "false"); + void *lib = open_lib(); + PFN_vkGetInstanceProcAddr gipa = lib ? (PFN_vkGetInstanceProcAddr)lib_sym(lib, "vkGetInstanceProcAddr") : NULL; + if (!gipa) { + printf("\"loader\": false, \"devices\": []}\n"); + return 0; + } + printf("\"loader\": true, "); + PFN_vkCreateInstance create = (PFN_vkCreateInstance)gipa(NULL, "vkCreateInstance"); + VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, .pApplicationName = "totk-install", + .apiVersion = VK_API_VERSION_1_3}; + VkInstanceCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, .pApplicationInfo = &app}; + VkInstance inst = VK_NULL_HANDLE; + VkResult r = create ? create(&ci, NULL, &inst) : VK_ERROR_INITIALIZATION_FAILED; + if (r != VK_SUCCESS) { + printf("\"instance\": %d, \"devices\": []}\n", (int)r); + return 0; + } + PFN_vkEnumeratePhysicalDevices enumerate = (PFN_vkEnumeratePhysicalDevices)gipa(inst, "vkEnumeratePhysicalDevices"); + PFN_vkGetPhysicalDeviceProperties props = (PFN_vkGetPhysicalDeviceProperties)gipa(inst, "vkGetPhysicalDeviceProperties"); + PFN_vkDestroyInstance destroy = (PFN_vkDestroyInstance)gipa(inst, "vkDestroyInstance"); + VkPhysicalDevice devs[16]; + uint32_t n = 16; + if (enumerate(inst, &n, devs) < 0) n = 0; + printf("\"instance\": 0, \"devices\": ["); + for (uint32_t i = 0; i < n; i++) { + VkPhysicalDeviceProperties p; + props(devs[i], &p); + printf("%s{\"name\": ", i ? ", " : ""); + json_string(p.deviceName); + printf(", \"api\": [%u, %u, %u], \"type\": %d}", VK_API_VERSION_MAJOR(p.apiVersion), + VK_API_VERSION_MINOR(p.apiVersion), VK_API_VERSION_PATCH(p.apiVersion), (int)p.deviceType); + } + printf("]}\n"); + destroy(inst, NULL); + return 0; +} diff --git a/installer/ini.py b/installer/ini.py new file mode 100644 index 00000000..6bcc341f --- /dev/null +++ b/installer/ini.py @@ -0,0 +1,69 @@ +"""totk.ini: set the game language and keep everything else (keys and syntax as runtime/cfg.c reads them). + +A new file starts from totk.ini.example, whose lines are all commented defaults. The language goes into `lang` +(a Switch language code, runtime/ov_system.c): an active `lang = ...` line is rewritten in place, else the commented +`# lang = ...` line of the example is activated, else the line is added to [language] (created at the end if +missing). Other lines, comments and line order stay as they are. +""" +import os +import re +from pathlib import Path + +# runtime/ov_system.c: the codes TOTK_LANG accepts. +LANGUAGES = ('ja', 'en-US', 'en-GB', 'fr', 'fr-CA', 'de', 'it', 'es', 'es-419', 'nl', 'pt', 'pt-BR', 'ru', 'ko', + 'zh-CN', 'zh-TW', 'zh-Hans', 'zh-Hant') +# Their text falls back to the Switch shared fonts (runtime/ov_system.c), which the installer does not extract. +NEED_SHARED_FONTS = ('ko', 'zh-CN', 'zh-TW', 'zh-Hans', 'zh-Hant') + +ACTIVE = re.compile(r'^(\s*)(?:totk_)?lang(\s*=\s*)(.*?)(\s*)$', re.IGNORECASE) +COMMENTED = re.compile(r'^\s*[#;]\s*(?:totk_)?lang\s*=.*$', re.IGNORECASE) +SECTION = re.compile(r'^\s*\[(.*)\]\s*$') + + +def set_language(text, lang): + """text with `lang = `; see the module docstring.""" + lines = text.splitlines() + for i, line in enumerate(lines): + m = ACTIVE.match(line) + if m: + lines[i] = f'{m.group(1)}lang{m.group(2)}{lang}' + return '\n'.join(lines) + '\n' + for i, line in enumerate(lines): + if COMMENTED.match(line): + lines[i] = f'lang = {lang}' + return '\n'.join(lines) + '\n' + for i, line in enumerate(lines): + m = SECTION.match(line) + if m and m.group(1).strip().lower() == 'language': + lines.insert(i + 1, f'lang = {lang}') + return '\n'.join(lines) + '\n' + if lines and lines[-1].strip(): + lines.append('') + lines += ['[language]', f'lang = {lang}'] + return '\n'.join(lines) + '\n' + + +def get_language(text): + for line in text.splitlines(): + m = ACTIVE.match(line) + if m: + return m.group(3) + return None + + +def write(dest, lang, example=None): + """Write dest/totk.ini with lang; an existing file keeps its other lines. Returns the path.""" + path = Path(dest) / 'totk.ini' + if path.is_file(): + text = path.read_text(encoding='utf-8-sig') + elif example and Path(example).is_file(): + text = Path(example).read_text(encoding='utf-8') + else: + text = '' + new = set_language(text, lang) + if path.is_file() and new == text: + return path + tmp = path.with_name('totk.ini.tmp') + tmp.write_text(new, encoding='utf-8', newline='\n') + os.replace(tmp, path) + return path diff --git a/installer/launcher/launcher.c b/installer/launcher/launcher.c new file mode 100644 index 00000000..0644234b --- /dev/null +++ b/installer/launcher/launcher.c @@ -0,0 +1,38 @@ +/* TotK-Setup.exe: starts python\pythonw.exe -m installer.gui with this exe's folder as the working directory. + + No console window. The two message strings come from the catalog (installer/text/.json, ids launcher.title and + launcher.python_missing); tools/make_release.py writes them into launcher_text.h before it compiles this file. + Build: llvm/bin/clang -O2 -mwindows launcher.c -o TotK-Setup.exe */ +#include +#include + +#include "launcher_text.h" + +#define LANGID_RUSSIAN 0x19 +#define PATH_CHARS (MAX_PATH * 2) + +static void fail(void) +{ + int russian = (GetUserDefaultUILanguage() & 0x3ff) == LANGID_RUSSIAN; + MessageBoxW(NULL, russian ? LAUNCHER_PYTHON_MISSING_RU : LAUNCHER_PYTHON_MISSING_EN, + russian ? LAUNCHER_TITLE_RU : LAUNCHER_TITLE_EN, MB_OK | MB_ICONERROR); +} + +int WINAPI WinMain(HINSTANCE instance, HINSTANCE previous, LPSTR command_line, int show) +{ + wchar_t folder[PATH_CHARS], python[PATH_CHARS], command[PATH_CHARS + 64]; + DWORD n = GetModuleFileNameW(NULL, folder, PATH_CHARS); + STARTUPINFOW si = { sizeof si }; + PROCESS_INFORMATION pi; + (void)instance; (void)previous; (void)command_line; (void)show; + if (n == 0 || n >= PATH_CHARS) { fail(); return 1; } + while (n > 0 && folder[n - 1] != L'\\') n--; + folder[n > 0 ? n - 1 : 0] = 0; + if (swprintf(python, PATH_CHARS, L"%ls\\python\\pythonw.exe", folder) < 0 || + GetFileAttributesW(python) == INVALID_FILE_ATTRIBUTES) { fail(); return 1; } + if (swprintf(command, PATH_CHARS + 64, L"\"%ls\" -m installer.gui", python) < 0) { fail(); return 1; } + if (!CreateProcessW(python, command, NULL, NULL, FALSE, 0, NULL, folder, &si, &pi)) { fail(); return 1; } + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + return 0; +} diff --git a/installer/layout.py b/installer/layout.py new file mode 100644 index 00000000..42bb2544 --- /dev/null +++ b/installer/layout.py @@ -0,0 +1,113 @@ +"""Where the installer finds what it runs: the one place that maps the release folder and a repository checkout. + +Release (docs/installer.md "Release layout"), root = the unpacked folder: + + python/python.exe runs every tool (numpy, lz4, cryptography, compression.zstd); its ._pth puts .., ../tools + and ../recomp on sys.path, so `python -m installer` imports from the release folder + recomp/, tools/ the lifter and the scripts; recomp/patches-v121.txt, tools/data/expected-v1.2.1.json + llvm/ llvm-mingw (bin/clang.exe) + runtime/ libtotkrt.a, the headers and install-side sources (romfs.c); runtime/deps/ holds + include/ + lib/ of Vulkan, SDL3, Opus, libvpx and VMA (build_game.py --deps) + bin/ totk_xlate.exe (+ totk_xlate.id), hactool.exe, ninja.exe, SDL3.dll, libopus-0.dll + data/shader_list.tsv the MacNativeGames shader list + totk.ini.example + VERSION the release version, one line + temp/ the default temp folder (created) + +Dev mode, root = a repository checkout (no python/python.exe): build/py314-embed/py/python.exe (the embeddable +Python the release ships, with the same packages), else this Python; toolchain/llvm-mingw-*/ and toolchain/deps-mingw; +runtime/ as the sources with build/install-rtlib/libtotkrt.a (tools/build_game.py --make-runtime-lib --out +build/install-rtlib); MSYS2 UCRT64 ninja.exe; toolchain/hactool; build/xlate/totk_xlate.exe; the shader list from +refs/MacNativeGames; build/install-temp. The version is dev-. +""" +import os +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +EXE = '.exe' if os.name == 'nt' else '' +UCRT = Path(r'C:\msys64\ucrt64\bin') +SHADER_LIST_REF = 'origin/HEAD:games/totk/assets/shader_list.tsv' # dev: tools/prepare_shaders.py LIST_REF + + +@dataclass(frozen=True) +class Layout: + root: Path + dev: bool + release: str + python: Path # runs the tools + tools: Path + recomp: Path + patches: Path # code patch table for recomp/gen.py + expected: Path # expected hashes of the supported update + llvm: Path # llvm-mingw tree + deps: Path # include/ and lib/ for the link + runtime: Path # runtime headers and install-side sources + runtime_lib: Path # libtotkrt.a + ninja: Path + hactool: Path + xlate: Path # totk_xlate.exe; a totk_xlate.id beside it is copied too + dlls: tuple = field(default_factory=tuple) + shader_list: Path | None = None # None: dev, read from refs/MacNativeGames through git + ini_example: Path | None = None + temp: Path | None = None + + @property + def clang(self): + return self.llvm / 'bin' / ('clang' + EXE) + + def missing(self): + """Paths the steps need that do not exist.""" + need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py', + self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', self.patches, self.expected, self.clang, + self.deps / 'include', self.deps / 'lib', self.runtime, self.runtime_lib, self.ninja, self.hactool, + self.xlate, *self.dlls] + if self.shader_list: + need.append(self.shader_list) + return [str(p) for p in need if not p.exists()] + + +def is_release(root): + return (root / 'python' / ('python' + EXE)).is_file() + + +def release_layout(root): + version = root / 'VERSION' + return Layout( + root=root, dev=False, + release=version.read_text(encoding='utf-8').strip() if version.is_file() else 'unknown', + python=root / 'python' / ('python' + EXE), + tools=root / 'tools', recomp=root / 'recomp', patches=root / 'recomp' / 'patches-v121.txt', + expected=root / 'tools' / 'data' / 'expected-v1.2.1.json', + llvm=root / 'llvm', deps=root / 'runtime' / 'deps', runtime=root / 'runtime', + runtime_lib=root / 'runtime' / 'libtotkrt.a', + ninja=root / 'bin' / ('ninja' + EXE), hactool=root / 'bin' / ('hactool' + EXE), + xlate=root / 'bin' / ('totk_xlate' + EXE), + dlls=(root / 'bin' / 'SDL3.dll', root / 'bin' / 'libopus-0.dll'), + shader_list=root / 'data' / 'shader_list.tsv', ini_example=root / 'totk.ini.example', temp=root / 'temp') + + +def dev_layout(root): + commit = subprocess.run(['git', '-C', str(root), 'rev-parse', '--short', 'HEAD'], capture_output=True, + text=True).stdout.strip() + embed = root / 'build' / 'py314-embed' / 'py' / ('python' + EXE) + trees = sorted(p.parent.parent for p in (root / 'toolchain').glob('llvm-mingw-*/llvm-mingw-*/bin/clang' + EXE)) + deps = root / 'toolchain' / 'deps-mingw' + return Layout( + root=root, dev=True, release=f'dev-{commit or "unknown"}', + python=embed if embed.is_file() else Path(sys.executable), + tools=root / 'tools', recomp=root / 'recomp', patches=root / 'recomp' / 'patches-v121.txt', + expected=root / 'tools' / 'data' / 'expected-v1.2.1.json', + llvm=trees[-1] if trees else root / 'toolchain' / 'llvm-mingw-missing', deps=deps, + runtime=root / 'runtime', runtime_lib=root / 'build' / 'install-rtlib' / 'libtotkrt.a', + ninja=UCRT / ('ninja' + EXE), hactool=root / 'toolchain' / 'hactool' / ('hactool' + EXE), + xlate=root / 'build' / 'xlate' / ('totk_xlate' + EXE), + dlls=tuple(sorted((deps / 'bin').glob('*.dll'))), + shader_list=None, ini_example=root / 'totk.ini.example', temp=root / 'build' / 'install-temp') + + +def find(root=None): + """The layout of root (default: the folder above this package).""" + root = Path(root or Path(__file__).resolve().parent.parent).resolve() + return release_layout(root) if is_release(root) else dev_layout(root) diff --git a/installer/runner.py b/installer/runner.py new file mode 100644 index 00000000..ebf3b52b --- /dev/null +++ b/installer/runner.py @@ -0,0 +1,151 @@ +"""Which steps run, in what order, and the bookkeeping around them: fresh install, resume and update are one rule. + +Each step has a key: a fingerprint of everything its output depends on, computed before anything runs (a step's key +includes the keys of the steps it reads from, so a change upstream reaches every step below it). A step is skipped +when totk-install.json records it with the same key and its output still matches the record (Step.current, cheap). +Otherwise it runs. A transient step (its output lives in the temp folder and only the next step reads it) runs only +when the next step runs. `always` steps (check, cleanup) run every time. + +Update mode is the same rule with a newer release: the extraction's key depends only on the expected hashes, so a +release that changes the lifter or the runtime keeps romfs/ and exefs/ and rebuilds from lift on. Before keeping the +output of a step recorded by another release (or with --verify), the check step calls Step.verify (expensive: the full +RomFS hash); a mismatch makes the step run again. + +A failed or cancelled run leaves the finished steps recorded; the next run starts at the first step that is not. +""" +import datetime +import traceback + +from . import state as statefile + + +class InstallError(Exception): + """A failure with a message id (installer/text/en.json) and its parameters.""" + + def __init__(self, id, **params): + super().__init__(id) + self.id, self.params = id, params + + +class Step: + id = '' + transient = False + always = False + + def key(self, ctx): + """Fingerprint of the step's inputs; may read ctx.keys of earlier steps.""" + return '' + + def current(self, ctx, record): + """True when the recorded output is still in place (cheap checks only).""" + return True + + def verify(self, ctx, record): + """True when the recorded output passes the full check (hashes); called before keeping another release's + output.""" + return True + + def run(self, ctx): + """Do the work. Returns fields to record with the step.""" + return {} + + +class Context: + def __init__(self, layout, dest, temp, reporter, options, state=None, verify=False): + self.layout, self.dest, self.temp, self.reporter, self.options = layout, dest, temp, reporter, options + self.state = state or statefile.empty(layout.release) + self.verify = verify + self.mode = 'install' if state is None else 'resume' if state.get('release') == layout.release else 'update' + self.keys, self.runs, self.reasons, self.steps, self.forced = {}, {}, {}, [], set() + + def record(self, step_id): + return self.state['steps'].get(step_id) + + def will_run(self, step_id): + return self.runs.get(step_id, False) + + def save(self): + self.dest.mkdir(parents=True, exist_ok=True) + statefile.save(self.dest, self.state) + + def plan(self, forced=()): + """Fill keys (once), runs and reasons. forced: step ids that must run.""" + if not self.keys: + for s in self.steps: + self.keys[s.id] = s.key(self) + for s in self.steps: + rec = self.record(s.id) + if s.always or s.id in forced: + self.runs[s.id] = True + else: + self.runs[s.id] = not (rec and rec.get('key') == self.keys[s.id] and s.current(self, rec)) + self.reasons[s.id] = '' if self.runs[s.id] else 'current' + for i in range(len(self.steps) - 2, -1, -1): + s, after = self.steps[i], self.steps[i + 1] + if s.transient and self.runs[s.id] and not self.runs[after.id]: + self.runs[s.id], self.reasons[s.id] = False, 'not_needed' + self.forced = set(forced) + + def verify_kept(self, progress_step): + """Full check of every kept step output that another release recorded (or of all with --verify). A step that + fails it is planned to run. Returns the ids that failed.""" + failed = [] + for s in self.steps: + rec = self.record(s.id) + if self.runs[s.id] or self.reasons[s.id] != 'current' or s.always: + continue + if not (self.verify or rec.get('release') != self.layout.release): + continue + if not s.verify(self, rec): + failed.append(s.id) + if failed: + for f in failed: + self.reporter.warning('state.output_changed', step=f) + self.plan(self.forced | set(failed)) + return failed + + +def now_utc(): + return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def run_steps(steps, ctx): + """Run the plan. Returns 'ok', 'failed' or 'cancelled'; events and the log say why.""" + rep = ctx.reporter + ctx.steps = steps + current = None + try: + ctx.plan() + rep.emit('plan', run=[s.id for s in steps if ctx.runs[s.id]], + skip={s.id: ctx.reasons[s.id] for s in steps if not ctx.runs[s.id]}) + for s in steps: + if not ctx.runs[s.id]: + rep.step(s.id, 'skipped', reason=ctx.reasons[s.id]) + continue + current = s.id + rep.step(s.id, 'started') + t0 = rep.elapsed() + fields = s.run(ctx) or {} + seconds = round(rep.elapsed() - t0, 1) + if not s.always: + ctx.state['steps'][s.id] = {'key': ctx.keys[s.id], 'release': ctx.layout.release, + 'finished': now_utc(), 'seconds': seconds, **fields} + ctx.save() # the first save, after the check, creates the install folder + rep.step(s.id, 'done', seconds=seconds) + current = None + ctx.state['release'] = ctx.layout.release + ctx.save() + return 'ok' + except InstallError as e: + rep.error(e.id, e.params, step=current) + return 'failed' + except statefile.StateError as e: + rep.error(e.id, {'path': str(e.path)}, step=current) + return 'failed' + except KeyboardInterrupt: + rep.log(f'cancelled during {current}') + return 'cancelled' + except Exception as e: + rep.log(traceback.format_exc()) + rep.error('internal', {'exception': type(e).__name__, 'detail': str(e)}, step=current) + return 'failed' diff --git a/installer/state.py b/installer/state.py new file mode 100644 index 00000000..be264654 --- /dev/null +++ b/installer/state.py @@ -0,0 +1,54 @@ +"""totk-install.json in the install folder: what built it and which steps finished. + + {"format": 1, "release": "...", "lang": "en-US", + "inputs": {"base": {"name", "size", "sha256"}, "update": {...}}, + "steps": {"": {"key": "", "release": "...", "finished": "", "seconds": 12.3, + ...the step's own fields}}} + +A step's key fingerprints everything its output depends on (runner.py), so a step whose recorded key equals the +current one and whose output still checks out is not run again. The file is replaced atomically after each step. +No key value or keys file content is ever stored. +""" +import json +import os +from pathlib import Path + +NAME = 'totk-install.json' +FORMAT = 1 + + +class StateError(Exception): + def __init__(self, id, path): + super().__init__(id) + self.id, self.path = id, path + + +def empty(release): + return {'format': FORMAT, 'release': release, 'inputs': {}, 'steps': {}} + + +def load(dest): + """The state of dest, or None when the folder has none. StateError for an unreadable or newer file.""" + path = Path(dest) / NAME + if not path.is_file(): + return None + try: + data = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError): + raise StateError('state.unreadable', path) from None + if not isinstance(data, dict) or not isinstance(data.get('steps'), dict): + raise StateError('state.unreadable', path) + if data.get('format', 0) > FORMAT: + raise StateError('state.newer_format', path) + return data + + +def save(dest, data): + path = Path(dest) / NAME + tmp = path.with_name(NAME + '.tmp') + with open(tmp, 'w', encoding='utf-8', newline='\n') as f: + json.dump(data, f, indent=1, ensure_ascii=False) + f.write('\n') + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) diff --git a/installer/steps.py b/installer/steps.py new file mode 100644 index 00000000..adfa871a --- /dev/null +++ b/installer/steps.py @@ -0,0 +1,629 @@ +"""The six setup steps of docs/installer.md: check, extract, lift, compile, prepare, cleanup. + +Extraction runs in this process through tools/extract.py (the package's sibling tools/, which ships with it); lift, +compile and prepare run the bundled Python on recomp/gen.py, tools/build_game.py, tools/make_romfs_patches.py and +tools/prepare_shaders.py, whose output goes to /logs/-.log and is read for progress. No Win32 call +here: the host check is a small C program built with the bundled clang (hostcheck.c). + +Install folder: exefs/, romfs/ (romfs.bin, index.json), profile/, totk.exe and its DLLs, romfs-index.bin, +romfs-patches/, totk_xlate.exe, the shader state of totk.exe --prepare-shaders, totk.ini, totk-install.json. +Temp folder (/-): check/ (ExeFS of both inputs, the host check), gen/ (C), build/ (objects), +logs/ (kept by the cleanup). +""" +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import threading +import time +from pathlib import Path + +from . import ini +from .layout import EXE +from .runner import InstallError, Step + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'tools')) +import extract # noqa: E402 +from extract_exefs import (common_tickets, container_entries, load_keys, nca_ctr_key, nca_inventory, # noqa: E402 + program_nca) + +GiB = 1 << 30 +DEST_EXTRA = 4 * GiB # beside the RomFS: shader and pipeline caches (about 2 GB), exefs, totk.exe, seed files +TEMP_NEED = 3 * GiB # generated C (about 0.9 GB) and objects +UPDATE_VERSION = '1.2.1' +HOSTCHECK = Path(__file__).resolve().parent / 'hostcheck.c' +STAMP = '.install-key' + + +# ---- helpers + +def file_sha256(path, progress=None): + """SHA-256 hex of a file; progress(done, total) after each block.""" + h = hashlib.sha256() + total = path.stat().st_size + done = 0 + with open(path, 'rb', buffering=0) as f: + buf = bytearray(8 << 20) + view = memoryview(buf) + while n := f.readinto(buf): + h.update(view[:n]) + done += n + if progress: + progress(done, total) + return h.hexdigest() + + +def fingerprint(*parts): + """SHA-256 over strings and the contents of files and directory trees (a missing path counts as missing).""" + h = hashlib.sha256() + for p in parts: + if isinstance(p, Path): + if p.is_dir(): + for f in sorted(p.rglob('*')): + if f.is_file() and '__pycache__' not in f.parts: + h.update(f.relative_to(p).as_posix().encode() + b'\0' + bytes.fromhex(file_sha256(f))) + elif p.is_file(): + h.update(p.name.encode() + b'\0' + bytes.fromhex(file_sha256(p))) + else: + h.update(b'missing\0' + p.name.encode()) + else: + h.update(str(p).encode()) + h.update(b'\1') + return h.hexdigest() + + +def remove(path): + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + elif path.exists() or path.is_symlink(): + path.unlink() + + +def replace_file(src, dst): + """Copy src over dst through a temporary name, so a cancelled copy never leaves half a file under dst's name.""" + tmp = dst.with_name(dst.name + '.tmp') + shutil.copy2(src, tmp) + os.replace(tmp, dst) + + +def kill_tree(proc): + if proc.poll() is not None: + return + if os.name == 'nt': + subprocess.run(['taskkill', '/T', '/F', '/PID', str(proc.pid)], capture_output=True) + else: + proc.kill() + + +def run_tool(ctx, step, script, args, parse=None, cwd=None): + """Run a tool script with the bundled Python. Its output goes to /logs/-