Add the installer core: the six setup steps, JSON progress events, totk-install.json resume and update, and explicit paths in the extract, build and shader tools

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

installer/__init__.py

@@ -0,0 +1,10 @@
+"""The TotK installer: builds a playable folder from the player's own game files (docs/installer.md).
+
+    python -m installer --base FILE --update FILE --keys FILE --lang CODE --dest DIR [--root DIR] [--temp DIR]
+
+Modules: layout (where the bundled tools are, release or repository), events (JSON progress on stdout and the log
+file), state (totk-install.json), runner (which steps run: resume and update), steps (the six steps), ini (totk.ini),
+cli (the command line). text/en.json holds the English text of every message id the events carry.
+
+Standard library only, apart from what the extraction tools import (cryptography). Runs on Python 3.14.
+"""

installer/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from installer.cli import main
+
+sys.exit(main())

installer/cli.py

@@ -0,0 +1,78 @@
+"""Command line: python -m installer --base FILE --update FILE --keys FILE --lang CODE --dest DIR [--root DIR]
+                                     [--temp DIR] [--verify]
+
+A first install needs every option. On a folder with totk-install.json (resume after a cancel or failure, or update
+with a newer release) only --dest is needed: --lang defaults to the recorded language, and the game files are needed
+only when the extraction has to run again. Prints JSON events on stdout (events.py). Exit code: 0 done, 1 failed,
+2 bad command line, 3 cancelled (Ctrl+C or Ctrl+Break).
+"""
+import argparse
+import hashlib
+import signal
+import sys
+import time
+from pathlib import Path
+
+from . import layout as layouts
+from . import state as statefile
+from .events import Reporter
+from .runner import Context, run_steps
+from .steps import all_steps
+
+EXIT = {'ok': 0, 'failed': 1, 'cancelled': 3}
+
+
+class Parser(argparse.ArgumentParser):
+    def error(self, message):
+        Reporter().error('cli.usage', {'detail': message})
+        sys.exit(2)
+
+
+def temp_folder(base, dest):
+    """One temp folder per install folder, so two installs never share generated files."""
+    tag = hashlib.sha256(str(dest).lower().encode()).hexdigest()[:8]
+    return base / f'{dest.name or "root"}-{tag}'
+
+
+def cancel(signum, frame):
+    raise KeyboardInterrupt
+
+
+def main(argv=None):
+    ap = Parser(prog='python -m installer', description=__doc__.splitlines()[0])
+    ap.add_argument('--base', type=Path, help='base game, XCI or NSP')
+    ap.add_argument('--update', type=Path, help='1.2.1 update NSP')
+    ap.add_argument('--keys', type=Path, help='prod.keys')
+    ap.add_argument('--lang', help='game language, a Switch language code (en-US, en-GB, ja, de, ru, ...)')
+    ap.add_argument('--dest', type=Path, required=True, help='install folder')
+    ap.add_argument('--root', type=Path, help='release folder or repository (default: the folder above installer/)')
+    ap.add_argument('--temp', type=Path, help='temp folder (default: temp/ of the release, build/install-temp in a '
+                                              'repository)')
+    ap.add_argument('--verify', action='store_true', help='hash every kept file, also those this release made')
+    args = ap.parse_args(argv)
+
+    # The window cancels with Ctrl+Break to the installer's process group (CREATE_NEW_PROCESS_GROUP), which also reaches
+    # the tools it runs; Python's default for it ends the process without recording anything.
+    for name in ('SIGBREAK', 'SIGTERM'):
+        if hasattr(signal, name):
+            signal.signal(getattr(signal, name), cancel)
+    layout = layouts.find(args.root)
+    dest = args.dest.resolve()
+    temp = temp_folder((args.temp or layout.temp).resolve(), dest)
+    log = temp / 'logs' / time.strftime('install-%Y%m%d-%H%M%S.log')
+    rep = Reporter(log_path=log)
+    t0 = time.monotonic()
+    try:
+        state = statefile.load(dest)
+    except statefile.StateError as e:
+        rep.error(e.id, {'path': str(e.path)})
+        rep.emit('end', result='failed', seconds=0)
+        return 1
+    ctx = Context(layout, dest, temp, rep, args, state=state, verify=args.verify)
+    steps = all_steps()
+    rep.emit('start', release=layout.release, mode=ctx.mode, dev=layout.dev, dest=str(dest), log=str(log),
+             steps=[s.id for s in steps])
+    result = run_steps(steps, ctx)
+    rep.emit('end', result=result, seconds=round(time.monotonic() - t0, 1))
+    rep.close()
+    return EXIT[result]

installer/events.py

@@ -0,0 +1,117 @@
+"""Progress events: one JSON object per line on stdout for the window and tests, the same in English to a log file.
+
+Every event has "type" and "t" (seconds since the start). Types:
+
+    start     release, mode (install, resume, update), dest, log, steps (the step ids in order)
+    step      step, state: started | done (seconds) | skipped (reason: current, not_needed)
+    progress  step, phase, and done/total/unit (bytes, chunks, objects, shaders) when the amount is known
+    warning   id, params
+    error     id, params, step (absent outside a step)
+    end       result: ok | failed | cancelled, seconds
+
+Text is never the payload: ids name a step ("step.<id>"), a phase ("phase.<step>.<phase>") or a message ("<id>"),
+and installer/text/en.json holds their English text with {param} placeholders. The log renders events with it.
+"""
+import json
+import sys
+import threading
+import time
+from pathlib import Path
+
+CATALOG = Path(__file__).resolve().parent / 'text' / 'en.json'
+PROGRESS_EVERY = 0.5   # seconds between two progress events of one phase
+
+
+def load_catalog(path=CATALOG):
+    return json.loads(path.read_text(encoding='utf-8'))
+
+
+def render(catalog, id, params=None):
+    """The English text of id, or the id itself when the catalog lacks it."""
+    text = catalog.get(id, id)
+    try:
+        return text.format(**(params or {}))
+    except (KeyError, IndexError, ValueError):
+        return f'{text} {params}'
+
+
+class Reporter:
+    def __init__(self, out=None, log_path=None):
+        self.out = out or sys.stdout
+        self.t0 = time.monotonic()
+        self.catalog = load_catalog()
+        self.lock = threading.Lock()
+        self.log_file = None
+        self.last_progress = {}
+        if log_path:
+            Path(log_path).parent.mkdir(parents=True, exist_ok=True)
+            self.log_file = open(log_path, 'a', encoding='utf-8', buffering=1)
+        self.log_path = log_path
+
+    def elapsed(self):
+        return time.monotonic() - self.t0
+
+    def log(self, text):
+        """A line for the log file only."""
+        if self.log_file:
+            with self.lock:
+                self.log_file.write(f'[{self.elapsed():8.1f}] {text}\n')
+
+    def emit(self, type, **fields):
+        event = {'type': type, 't': round(self.elapsed(), 2), **fields}
+        with self.lock:
+            self.out.write(json.dumps(event, ensure_ascii=False) + '\n')
+            self.out.flush()
+        self.log(self.describe(event))
+        return event
+
+    def describe(self, e):
+        c = self.catalog
+        kind = e['type']
+        if kind == 'step':
+            name = render(c, 'step.' + e['step'])
+            extra = f' ({e["seconds"]:.1f} s)' if 'seconds' in e else f' ({e["reason"]})' if 'reason' in e else ''
+            return f'{name}: {e["state"]}{extra}'
+        if kind == 'progress':
+            text = render(c, f'phase.{e["step"]}.{e["phase"]}')
+            if e.get('total'):
+                text += f' {e["done"]}/{e["total"]} {e.get("unit", "")}'.rstrip()
+            return text
+        if kind in ('warning', 'error'):
+            return f'{kind}: {render(c, e["id"], e.get("params"))} [{e["id"]}]'
+        return json.dumps(e, ensure_ascii=False)
+
+    def step(self, step, state, **fields):
+        return self.emit('step', step=step, state=state, **fields)
+
+    def progress(self, step, phase, done=None, total=None, unit=None, force=False):
+        """A progress event, at most one per PROGRESS_EVERY per phase unless it is the first or the last."""
+        now = time.monotonic()
+        key = (step, phase)
+        last = self.last_progress.get(key)
+        final = total is not None and done is not None and done >= total
+        if last is not None and not final and not force and now - last < PROGRESS_EVERY:
+            return None
+        self.last_progress[key] = now
+        fields = {'step': step, 'phase': phase}
+        if done is not None:
+            fields['done'] = done
+        if total is not None:
+            fields['total'] = total
+        if unit:
+            fields['unit'] = unit
+        return self.emit('progress', **fields)
+
+    def warning(self, id, **params):
+        return self.emit('warning', id=id, params=params)
+
+    def error(self, id, params, step=None):
+        fields = {'id': id, 'params': params}
+        if step:
+            fields['step'] = step
+        return self.emit('error', **fields)
+
+    def close(self):
+        if self.log_file:
+            self.log_file.close()
+            self.log_file = None

installer/hostcheck.c

@@ -0,0 +1,65 @@
+// Host check of the installer's first step, compiled with the bundled clang at install time: does the CPU run
+// -march=x86-64-v3 code, and does a Vulkan 1.3 device exist. Prints one JSON object on stdout.
+// The Vulkan loader is opened at run time, so a machine without one gets "loader": false instead of a failed start.
+#include <stdio.h>
+#include <string.h>
+#include <vulkan/vulkan.h>
+#ifdef _WIN32
+#include <windows.h>
+#define open_lib() LoadLibraryA("vulkan-1.dll")
+#define lib_sym(l, n) ((void *)GetProcAddress((l), (n)))
+#else
+#include <dlfcn.h>
+#define open_lib() dlopen("libvulkan.so.1", RTLD_NOW)
+#define lib_sym(l, n) dlsym((l), (n))
+#endif
+
+static void json_string(const char *s) {
+    putchar('"');
+    for (; *s; s++) {
+        if (*s == '"' || *s == '\\') printf("\\%c", *s);
+        else if ((unsigned char)*s < 0x20) printf("\\u%04x", *s);
+        else putchar(*s);
+    }
+    putchar('"');
+}
+
+int main(void) {
+    __builtin_cpu_init();
+    printf("{\"x86_64_v3\": %s, ", __builtin_cpu_supports("x86-64-v3") ? "true" : "false");
+    void *lib = open_lib();
+    PFN_vkGetInstanceProcAddr gipa = lib ? (PFN_vkGetInstanceProcAddr)lib_sym(lib, "vkGetInstanceProcAddr") : NULL;
+    if (!gipa) {
+        printf("\"loader\": false, \"devices\": []}\n");
+        return 0;
+    }
+    printf("\"loader\": true, ");
+    PFN_vkCreateInstance create = (PFN_vkCreateInstance)gipa(NULL, "vkCreateInstance");
+    VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, .pApplicationName = "totk-install",
+                             .apiVersion = VK_API_VERSION_1_3};
+    VkInstanceCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, .pApplicationInfo = &app};
+    VkInstance inst = VK_NULL_HANDLE;
+    VkResult r = create ? create(&ci, NULL, &inst) : VK_ERROR_INITIALIZATION_FAILED;
+    if (r != VK_SUCCESS) {
+        printf("\"instance\": %d, \"devices\": []}\n", (int)r);
+        return 0;
+    }
+    PFN_vkEnumeratePhysicalDevices enumerate = (PFN_vkEnumeratePhysicalDevices)gipa(inst, "vkEnumeratePhysicalDevices");
+    PFN_vkGetPhysicalDeviceProperties props = (PFN_vkGetPhysicalDeviceProperties)gipa(inst, "vkGetPhysicalDeviceProperties");
+    PFN_vkDestroyInstance destroy = (PFN_vkDestroyInstance)gipa(inst, "vkDestroyInstance");
+    VkPhysicalDevice devs[16];
+    uint32_t n = 16;
+    if (enumerate(inst, &n, devs) < 0) n = 0;
+    printf("\"instance\": 0, \"devices\": [");
+    for (uint32_t i = 0; i < n; i++) {
+        VkPhysicalDeviceProperties p;
+        props(devs[i], &p);
+        printf("%s{\"name\": ", i ? ", " : "");
+        json_string(p.deviceName);
+        printf(", \"api\": [%u, %u, %u], \"type\": %d}", VK_API_VERSION_MAJOR(p.apiVersion),
+               VK_API_VERSION_MINOR(p.apiVersion), VK_API_VERSION_PATCH(p.apiVersion), (int)p.deviceType);
+    }
+    printf("]}\n");
+    destroy(inst, NULL);
+    return 0;
+}

installer/ini.py

@@ -0,0 +1,69 @@
+"""totk.ini: set the game language and keep everything else (keys and syntax as runtime/cfg.c reads them).
+
+A new file starts from totk.ini.example, whose lines are all commented defaults. The language goes into `lang`
+(a Switch language code, runtime/ov_system.c): an active `lang = ...` line is rewritten in place, else the commented
+`# lang = ...` line of the example is activated, else the line is added to [language] (created at the end if
+missing). Other lines, comments and line order stay as they are.
+"""
+import os
+import re
+from pathlib import Path
+
+# runtime/ov_system.c: the codes TOTK_LANG accepts.
+LANGUAGES = ('ja', 'en-US', 'en-GB', 'fr', 'fr-CA', 'de', 'it', 'es', 'es-419', 'nl', 'pt', 'pt-BR', 'ru', 'ko',
+             'zh-CN', 'zh-TW', 'zh-Hans', 'zh-Hant')
+# Their text falls back to the Switch shared fonts (runtime/ov_system.c), which the installer does not extract.
+NEED_SHARED_FONTS = ('ko', 'zh-CN', 'zh-TW', 'zh-Hans', 'zh-Hant')
+
+ACTIVE = re.compile(r'^(\s*)(?:totk_)?lang(\s*=\s*)(.*?)(\s*)$', re.IGNORECASE)
+COMMENTED = re.compile(r'^\s*[#;]\s*(?:totk_)?lang\s*=.*$', re.IGNORECASE)
+SECTION = re.compile(r'^\s*\[(.*)\]\s*$')
+
+
+def set_language(text, lang):
+    """text with `lang = <lang>`; see the module docstring."""
+    lines = text.splitlines()
+    for i, line in enumerate(lines):
+        m = ACTIVE.match(line)
+        if m:
+            lines[i] = f'{m.group(1)}lang{m.group(2)}{lang}'
+            return '\n'.join(lines) + '\n'
+    for i, line in enumerate(lines):
+        if COMMENTED.match(line):
+            lines[i] = f'lang = {lang}'
+            return '\n'.join(lines) + '\n'
+    for i, line in enumerate(lines):
+        m = SECTION.match(line)
+        if m and m.group(1).strip().lower() == 'language':
+            lines.insert(i + 1, f'lang = {lang}')
+            return '\n'.join(lines) + '\n'
+    if lines and lines[-1].strip():
+        lines.append('')
+    lines += ['[language]', f'lang = {lang}']
+    return '\n'.join(lines) + '\n'
+
+
+def get_language(text):
+    for line in text.splitlines():
+        m = ACTIVE.match(line)
+        if m:
+            return m.group(3)
+    return None
+
+
+def write(dest, lang, example=None):
+    """Write dest/totk.ini with lang; an existing file keeps its other lines. Returns the path."""
+    path = Path(dest) / 'totk.ini'
+    if path.is_file():
+        text = path.read_text(encoding='utf-8-sig')
+    elif example and Path(example).is_file():
+        text = Path(example).read_text(encoding='utf-8')
+    else:
+        text = ''
+    new = set_language(text, lang)
+    if path.is_file() and new == text:
+        return path
+    tmp = path.with_name('totk.ini.tmp')
+    tmp.write_text(new, encoding='utf-8', newline='\n')
+    os.replace(tmp, path)
+    return path

installer/layout.py

@@ -0,0 +1,112 @@
+"""Where the installer finds what it runs: the one place that maps the release folder and a repository checkout.
+
+Release (docs/installer.md "Release layout"), root = the unpacked folder:
+
+    python/python.exe       runs every tool (numpy, lz4, cryptography, compression.zstd)
+    recomp/, tools/         the lifter and the scripts; recomp/patches-v121.txt, tools/data/expected-v1.2.1.json
+    llvm/                   llvm-mingw (bin/clang.exe)
+    runtime/                libtotkrt.a, the headers and install-side sources (romfs.c), and include/ + lib/ of
+                            Vulkan, SDL3, Opus, libvpx and VMA (build_game.py --deps)
+    bin/                    totk_xlate.exe (+ totk_xlate.id), hactool.exe, ninja.exe, SDL3.dll, libopus-0.dll
+    data/shader_list.tsv    the MacNativeGames shader list
+    totk.ini.example
+    VERSION                 the release version, one line
+    temp/                   the default temp folder (created)
+
+Dev mode, root = a repository checkout (no python/python.exe): build/py314-embed/py/python.exe (the embeddable
+Python the release ships, with the same packages), else this Python; toolchain/llvm-mingw-*/ and toolchain/deps-mingw;
+runtime/ as the sources with build/install-rtlib/libtotkrt.a (tools/build_game.py --make-runtime-lib --out
+build/install-rtlib); MSYS2 UCRT64 ninja.exe; toolchain/hactool; build/xlate/totk_xlate.exe; the shader list from
+refs/MacNativeGames; build/install-temp. The version is dev-<commit>.
+"""
+import os
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+EXE = '.exe' if os.name == 'nt' else ''
+UCRT = Path(r'C:\msys64\ucrt64\bin')
+SHADER_LIST_REF = 'origin/HEAD:games/totk/assets/shader_list.tsv'   # dev: tools/prepare_shaders.py LIST_REF
+
+
+@dataclass(frozen=True)
+class Layout:
+    root: Path
+    dev: bool
+    release: str
+    python: Path          # runs the tools
+    tools: Path
+    recomp: Path
+    patches: Path         # code patch table for recomp/gen.py
+    expected: Path        # expected hashes of the supported update
+    llvm: Path            # llvm-mingw tree
+    deps: Path            # include/ and lib/ for the link
+    runtime: Path         # runtime headers and install-side sources
+    runtime_lib: Path     # libtotkrt.a
+    ninja: Path
+    hactool: Path
+    xlate: Path           # totk_xlate.exe; a totk_xlate.id beside it is copied too
+    dlls: tuple = field(default_factory=tuple)
+    shader_list: Path | None = None   # None: dev, read from refs/MacNativeGames through git
+    ini_example: Path | None = None
+    temp: Path | None = None
+
+    @property
+    def clang(self):
+        return self.llvm / 'bin' / ('clang' + EXE)
+
+    def missing(self):
+        """Paths the steps need that do not exist."""
+        need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py',
+                self.tools / 'make_romfs_patches.py', self.recomp / 'gen.py', self.patches, self.expected, self.clang,
+                self.deps / 'include', self.deps / 'lib', self.runtime, self.runtime_lib, self.ninja, self.hactool,
+                self.xlate, *self.dlls]
+        if self.shader_list:
+            need.append(self.shader_list)
+        return [str(p) for p in need if not p.exists()]
+
+
+def is_release(root):
+    return (root / 'python' / ('python' + EXE)).is_file()
+
+
+def release_layout(root):
+    version = root / 'VERSION'
+    return Layout(
+        root=root, dev=False,
+        release=version.read_text(encoding='utf-8').strip() if version.is_file() else 'unknown',
+        python=root / 'python' / ('python' + EXE),
+        tools=root / 'tools', recomp=root / 'recomp', patches=root / 'recomp' / 'patches-v121.txt',
+        expected=root / 'tools' / 'data' / 'expected-v1.2.1.json',
+        llvm=root / 'llvm', deps=root / 'runtime', runtime=root / 'runtime',
+        runtime_lib=root / 'runtime' / 'libtotkrt.a',
+        ninja=root / 'bin' / ('ninja' + EXE), hactool=root / 'bin' / ('hactool' + EXE),
+        xlate=root / 'bin' / ('totk_xlate' + EXE),
+        dlls=(root / 'bin' / 'SDL3.dll', root / 'bin' / 'libopus-0.dll'),
+        shader_list=root / 'data' / 'shader_list.tsv', ini_example=root / 'totk.ini.example', temp=root / 'temp')
+
+
+def dev_layout(root):
+    commit = subprocess.run(['git', '-C', str(root), 'rev-parse', '--short', 'HEAD'], capture_output=True,
+                            text=True).stdout.strip()
+    embed = root / 'build' / 'py314-embed' / 'py' / ('python' + EXE)
+    trees = sorted(p.parent.parent for p in (root / 'toolchain').glob('llvm-mingw-*/llvm-mingw-*/bin/clang' + EXE))
+    deps = root / 'toolchain' / 'deps-mingw'
+    return Layout(
+        root=root, dev=True, release=f'dev-{commit or "unknown"}',
+        python=embed if embed.is_file() else Path(sys.executable),
+        tools=root / 'tools', recomp=root / 'recomp', patches=root / 'recomp' / 'patches-v121.txt',
+        expected=root / 'tools' / 'data' / 'expected-v1.2.1.json',
+        llvm=trees[-1] if trees else root / 'toolchain' / 'llvm-mingw-missing', deps=deps,
+        runtime=root / 'runtime', runtime_lib=root / 'build' / 'install-rtlib' / 'libtotkrt.a',
+        ninja=UCRT / ('ninja' + EXE), hactool=root / 'toolchain' / 'hactool' / ('hactool' + EXE),
+        xlate=root / 'build' / 'xlate' / ('totk_xlate' + EXE),
+        dlls=tuple(sorted((deps / 'bin').glob('*.dll'))),
+        shader_list=None, ini_example=root / 'totk.ini.example', temp=root / 'build' / 'install-temp')
+
+
+def find(root=None):
+    """The layout of root (default: the folder above this package)."""
+    root = Path(root or Path(__file__).resolve().parent.parent).resolve()
+    return release_layout(root) if is_release(root) else dev_layout(root)

installer/runner.py

@@ -0,0 +1,151 @@
+"""Which steps run, in what order, and the bookkeeping around them: fresh install, resume and update are one rule.
+
+Each step has a key: a fingerprint of everything its output depends on, computed before anything runs (a step's key
+includes the keys of the steps it reads from, so a change upstream reaches every step below it). A step is skipped
+when totk-install.json records it with the same key and its output still matches the record (Step.current, cheap).
+Otherwise it runs. A transient step (its output lives in the temp folder and only the next step reads it) runs only
+when the next step runs. `always` steps (check, cleanup) run every time.
+
+Update mode is the same rule with a newer release: the extraction's key depends only on the expected hashes, so a
+release that changes the lifter or the runtime keeps romfs/ and exefs/ and rebuilds from lift on. Before keeping the
+output of a step recorded by another release (or with --verify), the check step calls Step.verify (expensive: the full
+RomFS hash); a mismatch makes the step run again.
+
+A failed or cancelled run leaves the finished steps recorded; the next run starts at the first step that is not.
+"""
+import datetime
+import traceback
+
+from . import state as statefile
+
+
+class InstallError(Exception):
+    """A failure with a message id (installer/text/en.json) and its parameters."""
+
+    def __init__(self, id, **params):
+        super().__init__(id)
+        self.id, self.params = id, params
+
+
+class Step:
+    id = ''
+    transient = False
+    always = False
+
+    def key(self, ctx):
+        """Fingerprint of the step's inputs; may read ctx.keys of earlier steps."""
+        return ''
+
+    def current(self, ctx, record):
+        """True when the recorded output is still in place (cheap checks only)."""
+        return True
+
+    def verify(self, ctx, record):
+        """True when the recorded output passes the full check (hashes); called before keeping another release's
+        output."""
+        return True
+
+    def run(self, ctx):
+        """Do the work. Returns fields to record with the step."""
+        return {}
+
+
+class Context:
+    def __init__(self, layout, dest, temp, reporter, options, state=None, verify=False):
+        self.layout, self.dest, self.temp, self.reporter, self.options = layout, dest, temp, reporter, options
+        self.state = state or statefile.empty(layout.release)
+        self.verify = verify
+        self.mode = 'install' if state is None else 'resume' if state.get('release') == layout.release else 'update'
+        self.keys, self.runs, self.reasons, self.steps, self.forced = {}, {}, {}, [], set()
+
+    def record(self, step_id):
+        return self.state['steps'].get(step_id)
+
+    def will_run(self, step_id):
+        return self.runs.get(step_id, False)
+
+    def save(self):
+        self.dest.mkdir(parents=True, exist_ok=True)
+        statefile.save(self.dest, self.state)
+
+    def plan(self, forced=()):
+        """Fill keys (once), runs and reasons. forced: step ids that must run."""
+        if not self.keys:
+            for s in self.steps:
+                self.keys[s.id] = s.key(self)
+        for s in self.steps:
+            rec = self.record(s.id)
+            if s.always or s.id in forced:
+                self.runs[s.id] = True
+            else:
+                self.runs[s.id] = not (rec and rec.get('key') == self.keys[s.id] and s.current(self, rec))
+            self.reasons[s.id] = '' if self.runs[s.id] else 'current'
+        for i in range(len(self.steps) - 2, -1, -1):
+            s, after = self.steps[i], self.steps[i + 1]
+            if s.transient and self.runs[s.id] and not self.runs[after.id]:
+                self.runs[s.id], self.reasons[s.id] = False, 'not_needed'
+        self.forced = set(forced)
+
+    def verify_kept(self, progress_step):
+        """Full check of every kept step output that another release recorded (or of all with --verify). A step that
+        fails it is planned to run. Returns the ids that failed."""
+        failed = []
+        for s in self.steps:
+            rec = self.record(s.id)
+            if self.runs[s.id] or self.reasons[s.id] != 'current' or s.always:
+                continue
+            if not (self.verify or rec.get('release') != self.layout.release):
+                continue
+            if not s.verify(self, rec):
+                failed.append(s.id)
+        if failed:
+            for f in failed:
+                self.reporter.warning('state.output_changed', step=f)
+            self.plan(self.forced | set(failed))
+        return failed
+
+
+def now_utc():
+    return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
+def run_steps(steps, ctx):
+    """Run the plan. Returns 'ok', 'failed' or 'cancelled'; events and the log say why."""
+    rep = ctx.reporter
+    ctx.steps = steps
+    current = None
+    try:
+        ctx.plan()
+        rep.emit('plan', run=[s.id for s in steps if ctx.runs[s.id]],
+                 skip={s.id: ctx.reasons[s.id] for s in steps if not ctx.runs[s.id]})
+        for s in steps:
+            if not ctx.runs[s.id]:
+                rep.step(s.id, 'skipped', reason=ctx.reasons[s.id])
+                continue
+            current = s.id
+            rep.step(s.id, 'started')
+            t0 = rep.elapsed()
+            fields = s.run(ctx) or {}
+            seconds = round(rep.elapsed() - t0, 1)
+            if not s.always:
+                ctx.state['steps'][s.id] = {'key': ctx.keys[s.id], 'release': ctx.layout.release,
+                                            'finished': now_utc(), 'seconds': seconds, **fields}
+            ctx.save()   # the first save, after the check, creates the install folder
+            rep.step(s.id, 'done', seconds=seconds)
+            current = None
+        ctx.state['release'] = ctx.layout.release
+        ctx.save()
+        return 'ok'
+    except InstallError as e:
+        rep.error(e.id, e.params, step=current)
+        return 'failed'
+    except statefile.StateError as e:
+        rep.error(e.id, {'path': str(e.path)}, step=current)
+        return 'failed'
+    except KeyboardInterrupt:
+        rep.log(f'cancelled during {current}')
+        return 'cancelled'
+    except Exception as e:
+        rep.log(traceback.format_exc())
+        rep.error('internal', {'exception': type(e).__name__, 'detail': str(e)}, step=current)
+        return 'failed'

installer/state.py

@@ -0,0 +1,54 @@
+"""totk-install.json in the install folder: what built it and which steps finished.
+
+    {"format": 1, "release": "...", "lang": "en-US",
+     "inputs": {"base": {"name", "size", "sha256"}, "update": {...}},
+     "steps": {"<step>": {"key": "<input fingerprint>", "release": "...", "finished": "<UTC time>", "seconds": 12.3,
+                          ...the step's own fields}}}
+
+A step's key fingerprints everything its output depends on (runner.py), so a step whose recorded key equals the
+current one and whose output still checks out is not run again. The file is replaced atomically after each step.
+No key value or keys file content is ever stored.
+"""
+import json
+import os
+from pathlib import Path
+
+NAME = 'totk-install.json'
+FORMAT = 1
+
+
+class StateError(Exception):
+    def __init__(self, id, path):
+        super().__init__(id)
+        self.id, self.path = id, path
+
+
+def empty(release):
+    return {'format': FORMAT, 'release': release, 'inputs': {}, 'steps': {}}
+
+
+def load(dest):
+    """The state of dest, or None when the folder has none. StateError for an unreadable or newer file."""
+    path = Path(dest) / NAME
+    if not path.is_file():
+        return None
+    try:
+        data = json.loads(path.read_text(encoding='utf-8'))
+    except (OSError, ValueError):
+        raise StateError('state.unreadable', path) from None
+    if not isinstance(data, dict) or not isinstance(data.get('steps'), dict):
+        raise StateError('state.unreadable', path)
+    if data.get('format', 0) > FORMAT:
+        raise StateError('state.newer_format', path)
+    return data
+
+
+def save(dest, data):
+    path = Path(dest) / NAME
+    tmp = path.with_name(NAME + '.tmp')
+    with open(tmp, 'w', encoding='utf-8', newline='\n') as f:
+        json.dump(data, f, indent=1, ensure_ascii=False)
+        f.write('\n')
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, path)

installer/steps.py

@@ -0,0 +1,625 @@
+"""The six setup steps of docs/installer.md: check, extract, lift, compile, prepare, cleanup.
+
+Extraction runs in this process through tools/extract.py (the package's sibling tools/, which ships with it); lift,
+compile and prepare run the bundled Python on recomp/gen.py, tools/build_game.py, tools/make_romfs_patches.py and
+tools/prepare_shaders.py, whose output goes to <temp>/logs/<step>-<tool>.log and is read for progress. No Win32 call
+here: the host check is a small C program built with the bundled clang (hostcheck.c).
+
+Install folder: exefs/, romfs/ (romfs.bin, index.json), profile/, totk.exe and its DLLs, romfs-index.bin,
+romfs-patches/, totk_xlate.exe, the shader state of totk.exe --prepare-shaders, totk.ini, totk-install.json.
+Temp folder (<temp>/<dest name>-<hash>): check/ (ExeFS of both inputs, the host check), gen/ (C), build/ (objects),
+logs/ (kept by the cleanup).
+"""
+import hashlib
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+from . import ini
+from .layout import EXE
+from .runner import InstallError, Step
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'tools'))
+import extract  # noqa: E402
+from extract_exefs import (common_tickets, container_entries, load_keys, nca_ctr_key, nca_inventory,  # noqa: E402
+                           program_nca)
+
+GiB = 1 << 30
+DEST_EXTRA = 4 * GiB      # beside the RomFS: shader and pipeline caches (about 2 GB), exefs, totk.exe, seed files
+TEMP_NEED = 3 * GiB       # generated C (about 0.9 GB) and objects
+UPDATE_VERSION = '1.2.1'
+HOSTCHECK = Path(__file__).resolve().parent / 'hostcheck.c'
+STAMP = '.install-key'
+
+
+# ---- helpers
+
+def file_sha256(path, progress=None):
+    """SHA-256 hex of a file; progress(done, total) after each block."""
+    h = hashlib.sha256()
+    total = path.stat().st_size
+    done = 0
+    with open(path, 'rb', buffering=0) as f:
+        buf = bytearray(8 << 20)
+        view = memoryview(buf)
+        while n := f.readinto(buf):
+            h.update(view[:n])
+            done += n
+            if progress:
+                progress(done, total)
+    return h.hexdigest()
+
+
+def fingerprint(*parts):
+    """SHA-256 over strings and the contents of files and directory trees (a missing path counts as missing)."""
+    h = hashlib.sha256()
+    for p in parts:
+        if isinstance(p, Path):
+            if p.is_dir():
+                for f in sorted(p.rglob('*')):
+                    if f.is_file() and '__pycache__' not in f.parts:
+                        h.update(f.relative_to(p).as_posix().encode() + b'\0' + bytes.fromhex(file_sha256(f)))
+            elif p.is_file():
+                h.update(p.name.encode() + b'\0' + bytes.fromhex(file_sha256(p)))
+            else:
+                h.update(b'missing\0' + p.name.encode())
+        else:
+            h.update(str(p).encode())
+        h.update(b'\1')
+    return h.hexdigest()
+
+
+def remove(path):
+    if path.is_dir() and not path.is_symlink():
+        shutil.rmtree(path)
+    elif path.exists() or path.is_symlink():
+        path.unlink()
+
+
+def replace_file(src, dst):
+    """Copy src over dst through a temporary name, so a cancelled copy never leaves half a file under dst's name."""
+    tmp = dst.with_name(dst.name + '.tmp')
+    shutil.copy2(src, tmp)
+    os.replace(tmp, dst)
+
+
+def kill_tree(proc):
+    if proc.poll() is not None:
+        return
+    if os.name == 'nt':
+        subprocess.run(['taskkill', '/T', '/F', '/PID', str(proc.pid)], capture_output=True)
+    else:
+        proc.kill()
+
+
+def run_tool(ctx, step, script, args, parse=None, cwd=None):
+    """Run a tool script with the bundled Python. Its output goes to <temp>/logs/<step>-<script>.log and, line by line,
+    to parse(line). Returns (exit code, log path)."""
+    logs = ctx.temp / 'logs'
+    logs.mkdir(parents=True, exist_ok=True)
+    log = logs / f'{step}-{Path(script).stem}.log'
+    command = [str(ctx.layout.python), '-c', extract.RUN_SCRIPT, str(script), *map(str, args)]
+    ctx.reporter.log('run ' + ' '.join(command[3:]))
+    env = dict(os.environ, PYTHONUNBUFFERED='1', PYTHONIOENCODING='utf-8')
+    with open(log, 'w', encoding='utf-8', newline='\n') as out:
+        proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd or ctx.layout.root,
+                                env=env, text=True, encoding='utf-8', errors='replace', bufsize=1)
+        try:
+            for line in proc.stdout:
+                out.write(line)
+                if parse:
+                    parse(line.rstrip())
+            code = proc.wait()
+        except BaseException:
+            kill_tree(proc)
+            raise
+    ctx.reporter.log(f'{Path(script).name}: exit code {code}; output in {log}')
+    return code, log
+
+
+def free_bytes(path):
+    """Free bytes on the disk of path (its nearest existing parent) and that parent's device."""
+    p = Path(path)
+    while not p.exists():
+        p = p.parent
+    return shutil.disk_usage(p).free, os.stat(p).st_dev
+
+
+def expected_update():
+    return extract.load_expected()[UPDATE_VERSION]
+
+
+def shader_list_text(layout):
+    if layout.shader_list:
+        return layout.shader_list.read_text(encoding='utf-8')
+    refs = layout.root / 'refs' / 'MacNativeGames'
+    r = subprocess.run(['git', '-C', str(refs), 'show', 'origin/HEAD:games/totk/assets/shader_list.tsv'],
+                       capture_output=True, encoding='utf-8')
+    if r.returncode:
+        raise InstallError('prepare.shader_list_missing')
+    return r.stdout
+
+
+class Watch(threading.Thread):
+    """Polls a measure() of bytes written while a long call runs and reports it as progress."""
+
+    def __init__(self, reporter, step):
+        super().__init__(daemon=True)
+        self.reporter, self.step = reporter, step
+        self.phase, self.measure, self.total = None, None, None
+        self.stopped = threading.Event()
+
+    def set(self, phase, measure=None, total=None):
+        self.phase, self.measure, self.total = phase, measure, total
+        self.reporter.progress(self.step, phase, 0 if total else None, total, 'bytes' if total else None, force=True)
+
+    def run(self):
+        while not self.stopped.wait(1.0):
+            phase, measure, total = self.phase, self.measure, self.total
+            if phase and measure:
+                try:
+                    done = min(measure(), total)
+                except OSError:
+                    continue
+                self.reporter.progress(self.step, phase, done, total, 'bytes')
+
+    def stop(self):
+        self.stopped.set()
+
+
+def size_of(paths):
+    return sum(p.stat().st_size for p in paths if p.is_file())
+
+
+# ---- 1. check
+
+class Check(Step):
+    id = 'check'
+    always = True
+
+    def run(self, ctx):
+        layout, rep, dest = ctx.layout, ctx.reporter, ctx.dest
+        missing = layout.missing()
+        if missing:
+            raise InstallError('check.release_incomplete', paths=missing[:5])
+        if dest.exists() and not dest.is_dir():
+            raise InstallError('check.dest_not_folder', path=str(dest))
+        if ctx.mode == 'install' and dest.is_dir() and any(dest.iterdir()):
+            raise InstallError('check.dest_not_empty', path=str(dest))
+        lang = ctx.options.lang or ctx.state.get('lang')
+        if not lang:
+            raise InstallError('check.lang_missing')
+        if lang not in ini.LANGUAGES:
+            raise InstallError('check.lang_unknown', lang=lang, known=', '.join(ini.LANGUAGES))
+        if lang in ini.NEED_SHARED_FONTS:
+            rep.warning('warn.lang_needs_fonts', lang=lang)
+        ctx.lang = lang
+        ctx.temp.mkdir(parents=True, exist_ok=True)
+
+        rep.progress('check', 'host', force=True)
+        self.host(ctx)
+        rep.progress('check', 'verify', force=True)
+        ctx.verify_kept('check')
+        ncas = 0
+        if ctx.will_run('extract'):
+            rep.progress('check', 'inputs', force=True)
+            ncas = self.inputs(ctx)
+        self.space(ctx, ncas)
+        if ctx.will_run('extract'):
+            rep.progress('check', 'versions', force=True)
+            self.versions(ctx)
+        return {}
+
+    def host(self, ctx):
+        layout, work = ctx.layout, ctx.temp / 'check'
+        work.mkdir(parents=True, exist_ok=True)
+        exe = work / ('hostcheck' + EXE)
+        command = [str(layout.clang), '-O1', str(HOSTCHECK), '-I', str(layout.deps / 'include'), '-o', str(exe)]
+        if os.name != 'nt':
+            command.append('-ldl')
+        r = subprocess.run(command, capture_output=True, text=True, errors='replace')
+        if r.returncode:
+            ctx.reporter.log(r.stdout + r.stderr)
+            raise InstallError('check.toolchain_broken', detail=(r.stderr or r.stdout).strip()[-400:])
+        r = subprocess.run([str(exe)], capture_output=True, text=True, errors='replace')
+        try:
+            host = json.loads(r.stdout)
+        except ValueError:
+            raise InstallError('check.host_probe_failed', code=r.returncode) from None
+        ctx.reporter.log(f'host: {r.stdout.strip()}')
+        if not host['x86_64_v3']:
+            raise InstallError('check.cpu_no_avx2')
+        if not host['loader']:
+            raise InstallError('check.vulkan_missing')
+        devices = host['devices']
+        if not devices:
+            raise InstallError('check.vulkan_no_device')
+        if not any(tuple(d['api'][:2]) >= (1, 3) for d in devices):
+            raise InstallError('check.vulkan_too_old',
+                               devices=', '.join(f'{d["name"]} ({d["api"][0]}.{d["api"][1]})' for d in devices))
+        ctx.host = host
+
+    def inputs(self, ctx):
+        """Open both files with the keys as the extraction will. Returns the bytes of both Program NCAs."""
+        o = ctx.options
+        for name in ('base', 'update', 'keys'):
+            path = getattr(o, name)
+            if not path:
+                raise InstallError('check.inputs_needed')
+            if not Path(path).is_file():
+                raise InstallError('check.input_missing', input=name, path=str(path))
+        try:
+            keys = load_keys(Path(o.keys))
+        except (ValueError, OSError, UnicodeDecodeError):
+            raise InstallError('check.keys_unreadable') from None
+        if 'header_key' not in keys:
+            raise InstallError('check.keys_no_header_key')
+        total = 0
+        for name, is_update in (('base', False), ('update', True)):
+            path = Path(getattr(o, name))
+            with path.open('rb') as f:
+                total += self.open_input(f, path.stat().st_size, keys, name, is_update)
+        del keys
+        return total
+
+    @staticmethod
+    def open_input(f, size, keys, name, is_update):
+        try:
+            kind, entries = container_entries(f, size)
+        except ValueError:
+            raise InstallError('check.not_game_file', input=name) from None
+        if is_update and kind != 'nsp':
+            raise InstallError('check.update_not_nsp')
+        try:
+            tickets = common_tickets(f, entries)
+        except ValueError as e:
+            if 'Personalized' in str(e):
+                raise InstallError('check.ticket_personalized', input=name) from None
+            raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
+        try:
+            records = nca_inventory(f, entries, keys)
+        except ValueError as e:
+            if 'NCA3' in str(e):
+                raise InstallError('check.keys_wrong', input=name) from None
+            raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
+        try:
+            record, header = program_nca(records, update=is_update)
+        except ValueError as e:
+            if is_update and 'BKTR' in str(e):
+                raise InstallError('check.update_is_base') from None
+            raise InstallError('check.update_not_totk' if is_update else 'check.base_not_totk', detail=str(e)) from None
+        try:
+            nca_ctr_key(header, keys, tickets)
+        except ValueError as e:
+            if 'ticket' in str(e):
+                raise InstallError('check.ticket_missing', input=name) from None
+            raise InstallError('check.keys_too_old', input=name,
+                               generation=max(header[0x206], header[0x220])) from None
+        return record['size']
+
+    def space(self, ctx, ncas):
+        dest_need = temp_need = 0
+        if ctx.will_run('extract'):
+            dest_need += expected_update()['romfs']['size'] + ncas
+            romfs = ctx.dest / 'romfs' / 'romfs.bin'
+            if romfs.is_file():   # a resumed extraction replaces it
+                dest_need -= romfs.stat().st_size
+        if ctx.will_run('prepare') or ctx.will_run('extract'):
+            dest_need += DEST_EXTRA
+        if ctx.will_run('lift') or ctx.will_run('compile'):
+            temp_need += TEMP_NEED
+        dest_free, dest_dev = free_bytes(ctx.dest)
+        temp_free, temp_dev = free_bytes(ctx.temp)
+        ctx.reporter.log(f'space: install folder needs {dest_need / GiB:.1f} GiB of {dest_free / GiB:.1f} free, '
+                         f'temp folder {temp_need / GiB:.1f} GiB of {temp_free / GiB:.1f}')
+        if dest_dev == temp_dev:
+            dest_need += temp_need
+            temp_need = 0
+        for path, need, free in ((ctx.dest, dest_need, dest_free), (ctx.temp, temp_need, temp_free)):
+            if need > free:
+                raise InstallError('check.disk_space', path=str(path), need_gb=round(need / 1e9, 1),
+                                   free_gb=round(free / 1e9, 1))
+
+    def versions(self, ctx):
+        """Extract both ExeFS into the temp folder (verifying every block) and identify the versions by hash."""
+        o, work = ctx.options, ctx.temp / 'check'
+        logs = ctx.temp / 'logs'
+        logs.mkdir(parents=True, exist_ok=True)
+        versions = extract.load_expected()
+        for name, is_update in (('base', False), ('update', True)):
+            out = work / name
+            remove(out)
+            try:
+                extract.extract_exefs(Path(getattr(o, name)), Path(o.keys), out, logs, update=is_update)
+            except ValueError:
+                raise InstallError('check.exefs_failed', input=name, log=str(logs / f'extract_exefs-{name}.log')) \
+                    from None
+        problems = extract.exefs_problems(work / 'base' / 'exefs', versions['1.0.0'])
+        if problems:
+            raise InstallError('check.base_wrong_version', modules=', '.join(problems))
+        try:
+            found = extract.identify_update(work / 'update' / 'exefs', versions)['display_version']
+        except ValueError:
+            found = 'unknown'
+        if found != UPDATE_VERSION:
+            raise InstallError('check.update_wrong_version', found=found, wanted=UPDATE_VERSION)
+
+
+# ---- 2. extract
+
+class Extract(Step):
+    id = 'extract'
+
+    def key(self, ctx):
+        # The output is whatever matches the expected hashes, whichever files and tools produced it.
+        return fingerprint('extract-1', ctx.layout.expected)
+
+    def current(self, ctx, record):
+        expected = expected_update()
+        romfs = ctx.dest / 'romfs'
+        if extract.exefs_problems(ctx.dest / 'exefs', expected) or not extract.romfs_verified(romfs, expected):
+            return False
+        st = (romfs / 'romfs.bin').stat()
+        return st.st_size == record.get('romfs_size') and st.st_mtime_ns == record.get('romfs_mtime_ns')
+
+    def verify(self, ctx, record):
+        image = ctx.dest / 'romfs' / 'romfs.bin'
+        rep = ctx.reporter
+        digest = file_sha256(image, lambda d, t: rep.progress('check', 'verify_romfs', d, t, 'bytes'))
+        return digest == expected_update()['romfs']['sha256']
+
+    def run(self, ctx):
+        o, dest, rep = ctx.options, ctx.dest, ctx.reporter
+        expected = expected_update()
+        base, update, keys = Path(o.base), Path(o.update), Path(o.keys)
+        inputs = {}
+        for name, path in (('base', base), ('update', update)):
+            known = ctx.state['inputs'].get(name, {})
+            st = path.stat()
+            if known.get('size') == st.st_size and known.get('mtime_ns') == st.st_mtime_ns and known.get('sha256'):
+                inputs[name] = known
+                continue
+            digest = file_sha256(path, lambda d, t, n=name: rep.progress('extract', 'hash_' + n, d, t, 'bytes'))
+            inputs[name] = {'name': path.name, 'size': st.st_size, 'mtime_ns': st.st_mtime_ns, 'sha256': digest}
+        ctx.state['inputs'] = inputs
+        ctx.save()
+
+        dest.mkdir(parents=True, exist_ok=True)
+        for stale in dest.glob('update-merge-*'):   # a killed merge leaves its NCA copies (and key copies) behind
+            remove(stale)
+        rep.progress('extract', 'exefs', force=True)
+        staged = ctx.temp / 'check' / 'update' / 'exefs'
+        if not staged.is_dir():
+            remove(ctx.temp / 'check' / 'update')
+            try:
+                extract.extract_exefs(update, keys, ctx.temp / 'check' / 'update', ctx.temp / 'logs', update=True)
+            except ValueError:
+                raise InstallError('check.exefs_failed', input='update',
+                                   log=str(ctx.temp / 'logs' / 'extract_exefs-update.log')) from None
+        remove(dest / 'exefs')
+        shutil.move(staged, dest / 'exefs')
+        problems = extract.exefs_problems(dest / 'exefs', expected)
+        if problems:
+            raise InstallError('extract.exefs_mismatch', modules=', '.join(problems))
+        if extract.sha256(dest / 'exefs' / 'main.npdm') != expected['exefs_sha256']['main.npdm']:
+            rep.warning('warn.npdm_differs')
+
+        romfs = dest / 'romfs'
+        if not extract.romfs_verified(romfs, expected):
+            self.merge(ctx, base, update, keys, romfs, expected)
+        report = json.loads((romfs / 'index.json').read_text())
+        if report['romfs_size'] != expected['romfs']['size'] or report['romfs_sha256'] != expected['romfs']['sha256']:
+            raise InstallError('extract.romfs_mismatch', size=report['romfs_size'], sha256=report['romfs_sha256'][:16])
+        (dest / 'profile').mkdir(exist_ok=True)
+        st = (romfs / 'romfs.bin').stat()
+        return {'romfs_size': st.st_size, 'romfs_mtime_ns': st.st_mtime_ns}
+
+    def merge(self, ctx, base, update, keys, romfs, expected):
+        dest, logs = ctx.dest, ctx.temp / 'logs'
+        ncas = extract.merge_space(base, update, keys)
+        watch = Watch(ctx.reporter, 'extract')
+
+        def phase(name):
+            if name == 'copy':
+                watch.set('copy', lambda: size_of(dest.glob('update-merge-*/*.nca')), ncas)
+            elif name == 'merge':
+                watch.set('merge', lambda: size_of(dest.glob('update-merge-*/result/romfs.bin')),
+                          expected['romfs']['size'])
+            else:
+                watch.set(name)
+
+        watch.start()
+        try:
+            extract.merge_update_romfs(base, update, keys, ctx.layout.hactool, romfs, logs, phase=phase)
+        except ValueError as e:
+            raise InstallError('extract.romfs_merge_failed', detail=str(e), log=str(logs / 'merge_romfs-'
+                                                                                     f'{dest.name}.log')) from None
+        finally:
+            watch.stop()
+
+
+# ---- 3. lift
+
+EMITTED = re.compile(r'^emitted (\d+)/(\d+) chunks')
+DISCOVERED = re.compile(r'^(\w+): \d+ functions \(.*discovery')
+
+
+class Lift(Step):
+    id = 'lift'
+    transient = True
+
+    def key(self, ctx):
+        return fingerprint('lift-1', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches)
+
+    def current(self, ctx, record):
+        gen = ctx.temp / 'gen'
+        stamp = gen / STAMP
+        return (gen / 'layout.json').is_file() and stamp.is_file() and stamp.read_text().strip() == ctx.keys['lift']
+
+    def run(self, ctx):
+        layout, rep, gen = ctx.layout, ctx.reporter, ctx.temp / 'gen'
+        remove(gen)
+        rep.progress('lift', 'discover', force=True)
+
+        def parse(line):
+            m = EMITTED.match(line)
+            if m:
+                rep.progress('lift', 'emit', int(m.group(1)), int(m.group(2)), 'chunks')
+
+        code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py',
+                             ['--exefs', ctx.dest / 'exefs', '--out', gen, '--patches', layout.patches], parse)
+        if code:
+            raise InstallError('lift.failed', code=code, log=str(log))
+        (gen / STAMP).write_text(ctx.keys['lift'] + '\n')
+        return {}
+
+
+# ---- 4. compile
+
+NINJA_LINE = re.compile(r'^\[(\d+)/(\d+)\] ')
+
+
+class Compile(Step):
+    id = 'compile'
+
+    def key(self, ctx):
+        layout = ctx.layout
+        clang = subprocess.run([str(layout.clang), '--version'], capture_output=True, text=True).stdout
+        tools = [layout.tools / n for n in ('build_game.py', 'romfs_index.py', 'check_content.py')]
+        return fingerprint('compile-1', ctx.keys['lift'], layout.runtime, layout.runtime_lib, layout.deps, *tools,
+                           clang, *[d.name for d in layout.dlls])
+
+    def current(self, ctx, record):
+        exe = ctx.dest / ('totk' + EXE)
+        if not exe.is_file() or not (ctx.dest / 'romfs-index.bin').is_file():
+            return False
+        if not all((ctx.dest / d.name).is_file() for d in ctx.layout.dlls):
+            return False
+        return file_sha256(exe) == record.get('exe_sha256')
+
+    def run(self, ctx):
+        layout, rep, dest = ctx.layout, ctx.reporter, ctx.dest
+        build = ctx.temp / 'build'
+        phase = ['compile']
+
+        def parse(line):
+            m = NINJA_LINE.match(line)
+            if m:
+                rep.progress('compile', phase[0], int(m.group(1)), int(m.group(2)), 'objects')
+            elif line.startswith('compile: '):
+                phase[0] = 'link'
+                rep.progress('compile', 'link', force=True)
+
+        rep.progress('compile', 'compile', force=True)
+        code, log = run_tool(ctx, 'compile', layout.tools / 'build_game.py', [
+            '--gen', ctx.temp / 'gen', '--out', build, '--toolchain', layout.llvm, '--deps', layout.deps,
+            '--runtime', layout.runtime, '--runtime-lib', layout.runtime_lib, '--ninja', layout.ninja,
+            '--expected', layout.expected, '--exefs', dest / 'exefs', '--romfs', dest / 'romfs',
+            '--profile', dest / 'profile'], parse)
+        exe = build / ('totk' + EXE)
+        if code or not exe.is_file():
+            raise InstallError('compile.failed', code=code, log=str(log))
+        try:
+            replace_file(exe, dest / exe.name)
+        except PermissionError:
+            raise InstallError('compile.exe_in_use', path=str(dest / exe.name)) from None
+        replace_file(build / 'romfs-index.bin', dest / 'romfs-index.bin')
+        for dll in layout.dlls:
+            replace_file(dll, dest / dll.name)
+        return {'exe_sha256': file_sha256(dest / exe.name)}
+
+
+# ---- 5. prepare
+
+TRANSLATED = re.compile(r'gfx_shader: prepare: (\d+) of (\d+) pairs')
+
+
+class Prepare(Step):
+    id = 'prepare'
+
+    def key(self, ctx):
+        layout = ctx.layout
+        xlate_id = layout.xlate.with_suffix('.id')
+        tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py')]
+        return fingerprint('prepare-1', ctx.keys['compile'], layout.xlate, xlate_id, shader_list_text(layout), *tools)
+
+    def current(self, ctx, record):
+        dest = ctx.dest
+        xlate = dest / ctx.layout.xlate.name
+        return ((dest / 'shader-prepared.bin').is_file() and (dest / 'romfs-patches' / 'manifest.tsv').is_file()
+                and xlate.is_file() and file_sha256(xlate) == record.get('xlate_sha256'))
+
+    def run(self, ctx):
+        layout, rep, dest = ctx.layout, ctx.reporter, ctx.dest
+        xlate = dest / layout.xlate.name
+        if xlate.is_file() and file_sha256(xlate) != file_sha256(layout.xlate):
+            # Another translator: its cache keys differ, so the old translations would only take space.
+            for name in ('shader-cache', 'shader-prepared.bin'):
+                remove(dest / name)
+        replace_file(layout.xlate, xlate)
+        xlate_id = layout.xlate.with_suffix('.id')
+        if xlate_id.is_file():
+            replace_file(xlate_id, xlate.with_suffix('.id'))
+        else:
+            remove(xlate.with_suffix('.id'))
+
+        rep.progress('prepare', 'romfs_patches', force=True)
+        remove(dest / 'romfs-patches')
+        code, log = run_tool(ctx, 'prepare', layout.tools / 'make_romfs_patches.py',
+                             ['--romfs', dest / 'romfs', '--out', dest / 'romfs-patches'])
+        if code:
+            raise InstallError('prepare.romfs_patches_failed', code=code, log=str(log))
+
+        shader_list = layout.shader_list
+        if not shader_list:
+            shader_list = ctx.temp / 'shader_list.tsv'
+            shader_list.write_text(shader_list_text(layout), encoding='utf-8', newline='\n')
+
+        def parse(line):
+            m = TRANSLATED.search(line)
+            if m:
+                rep.progress('prepare', 'translate', int(m.group(1)), int(m.group(2)), 'shaders')
+            elif 'listed variants' in line:
+                rep.progress('prepare', 'translate', force=True)
+            elif 'gfx_shader: prepare:' in line and 'translated' in line and 'programs' in line:
+                rep.progress('prepare', 'libraries', force=True)
+
+        rep.progress('prepare', 'seed', force=True)
+        code, log = run_tool(ctx, 'prepare', layout.tools / 'prepare_shaders.py',
+                             [dest, '--romfs', dest / 'romfs', '--exefs', dest / 'exefs', '--list', shader_list], parse)
+        if code or not (dest / 'shader-prepared.bin').is_file():
+            raise InstallError('prepare.shaders_failed', code=code, log=str(log))
+        return {'xlate_sha256': file_sha256(xlate)}
+
+
+# ---- 6. clean up
+
+class Cleanup(Step):
+    id = 'cleanup'
+    always = True
+
+    def run(self, ctx):
+        rep, dest = ctx.reporter, ctx.dest
+        rep.progress('cleanup', 'ini', force=True)
+        try:
+            ini.write(dest, ctx.lang, ctx.layout.ini_example)
+        except OSError as e:
+            raise InstallError('cleanup.ini_failed', detail=str(e)) from None
+        ctx.state['lang'] = ctx.lang
+        (dest / 'profile').mkdir(exist_ok=True)
+        rep.progress('cleanup', 'temp', force=True)
+        for stale in dest.glob('update-merge-*'):
+            remove(stale)
+        for name in ('gen', 'build', 'check', 'shader_list.tsv'):
+            remove(ctx.temp / name)
+        return {}
+
+
+def all_steps():
+    return [Check(), Extract(), Lift(), Compile(), Prepare(), Cleanup()]

installer/text/en.json

@@ -0,0 +1,80 @@
+{
+ "step.check": "Checking the computer and the game files",
+ "step.extract": "Extracting the game",
+ "step.lift": "Translating the game code to C",
+ "step.compile": "Compiling totk.exe",
+ "step.prepare": "Preparing the shaders",
+ "step.cleanup": "Cleaning up",
+
+ "phase.check.host": "Checking the processor and the graphics card",
+ "phase.check.verify": "Checking the files already installed",
+ "phase.check.verify_romfs": "Checking the installed game data",
+ "phase.check.inputs": "Opening the game files with the keys",
+ "phase.check.versions": "Checking the game versions",
+ "phase.extract.hash_base": "Computing the checksum of the base game",
+ "phase.extract.hash_update": "Computing the checksum of the update",
+ "phase.extract.exefs": "Extracting the game code",
+ "phase.extract.copy": "Copying the game data",
+ "phase.extract.merge": "Applying the update to the game data",
+ "phase.extract.index": "Verifying the game data",
+ "phase.lift.discover": "Finding the functions",
+ "phase.lift.emit": "Writing C code",
+ "phase.compile.compile": "Compiling",
+ "phase.compile.link": "Linking totk.exe",
+ "phase.prepare.romfs_patches": "Writing the 60 fps game data patches",
+ "phase.prepare.seed": "Reading the shaders from the game data",
+ "phase.prepare.translate": "Translating the shaders",
+ "phase.prepare.libraries": "Building the shaders for the graphics driver",
+ "phase.cleanup.ini": "Writing totk.ini",
+ "phase.cleanup.temp": "Removing temporary files",
+
+ "cli.usage": "Invalid command line: {detail}",
+ "internal": "Unexpected error ({exception}): {detail}",
+ "state.unreadable": "{path} cannot be read. Choose another folder or delete that file.",
+ "state.newer_format": "{path} was written by a newer installer. Use that installer or another folder.",
+ "state.output_changed": "Installed files of step '{step}' changed since they were made; that step runs again.",
+
+ "check.release_incomplete": "The installer folder is incomplete; missing: {paths}. Unpack the release again.",
+ "check.dest_not_folder": "The install location {path} is a file, not a folder.",
+ "check.dest_not_empty": "The install folder {path} is not empty and holds no earlier installation. Choose an empty folder.",
+ "check.lang_missing": "Choose the game language.",
+ "check.lang_unknown": "Unknown language '{lang}'. Known: {known}.",
+ "check.toolchain_broken": "The bundled compiler does not work: {detail}",
+ "check.host_probe_failed": "The computer check did not run (exit code {code}).",
+ "check.cpu_no_avx2": "This processor lacks AVX2 (x86-64-v3), which the game needs. Intel Haswell (2013) or AMD Excavator / Zen and newer have it.",
+ "check.vulkan_missing": "No Vulkan driver is installed. Install the latest driver for your graphics card.",
+ "check.vulkan_no_device": "Vulkan finds no graphics card. Install the latest driver for your graphics card.",
+ "check.vulkan_too_old": "The game needs a graphics card with Vulkan 1.3; found: {devices}. Update the graphics driver.",
+ "check.inputs_needed": "The game files are needed: choose the base game, the 1.2.1 update and prod.keys.",
+ "check.input_missing": "File not found ({input}): {path}",
+ "check.keys_unreadable": "The keys file cannot be read. Choose a prod.keys file.",
+ "check.keys_no_header_key": "The keys file has no header_key. Dump prod.keys from your console again.",
+ "check.keys_wrong": "The keys cannot open the {input} file. Use the prod.keys of your console.",
+ "check.keys_too_old": "The keys file is too old for the {input} file (needs key generation {generation}). Dump prod.keys from a console with newer firmware.",
+ "check.not_game_file": "The {input} file is not an XCI or NSP game file.",
+ "check.file_unreadable": "The {input} file cannot be read: {detail}",
+ "check.ticket_personalized": "The {input} file carries a personalized ticket; a common ticket is needed.",
+ "check.ticket_missing": "The {input} file lacks the ticket its content needs.",
+ "check.base_not_totk": "The base game file is not The Legend of Zelda: Tears of the Kingdom ({detail}).",
+ "check.update_not_nsp": "The update must be an NSP file.",
+ "check.update_is_base": "The update file holds a base game. Choose the 1.2.1 update NSP.",
+ "check.update_not_totk": "The update file is not a Tears of the Kingdom update ({detail}).",
+ "check.exefs_failed": "The game code of the {input} file failed its checks; see {log}.",
+ "check.base_wrong_version": "The base game is not version 1.0.0 ({modules} differ).",
+ "check.update_wrong_version": "The update is version {found}; this release needs {wanted}.",
+ "check.disk_space": "Not enough free space for {path}: {need_gb} GB needed, {free_gb} GB free.",
+
+ "extract.exefs_mismatch": "The extracted game code does not match version 1.2.1 ({modules}).",
+ "extract.romfs_merge_failed": "Applying the update to the game data failed: {detail}. Details: {log}",
+ "extract.romfs_mismatch": "The extracted game data does not match version 1.2.1 (size {size}, checksum {sha256}...).",
+ "lift.failed": "Translating the game code failed (exit code {code}). Details: {log}",
+ "compile.failed": "Compiling totk.exe failed (exit code {code}). Details: {log}",
+ "compile.exe_in_use": "{path} is in use. Close the game and run the installer again.",
+ "prepare.shader_list_missing": "The shader list is missing.",
+ "prepare.romfs_patches_failed": "Writing the game data patches failed (exit code {code}). Details: {log}",
+ "prepare.shaders_failed": "Preparing the shaders failed (exit code {code}). Details: {log}",
+ "cleanup.ini_failed": "totk.ini cannot be written: {detail}",
+
+ "warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts, which the installer does not extract; some text may not show.",
+ "warn.npdm_differs": "main.npdm differs from the reference 1.2.1 dump; only the code modules are used."
+}

tests/installer_test.py

@@ -0,0 +1,260 @@
+"""installer/ bookkeeping with fake steps: fresh install, resume after a failure or cancel, transient steps, update
+with a newer release, totk.ini and the event stream. No game files, no tools; stdlib only.
+
+Run: python tests/installer_test.py
+"""
+import io
+import json
+import re
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+from installer import ini, runner, state  # noqa: E402
+from installer.events import Reporter  # noqa: E402
+from installer.layout import Layout  # noqa: E402
+
+
+class Fake(runner.Step):
+    """A step whose output is <dir>/<id>.out holding its key; dir is the temp folder for a transient step."""
+
+    def __init__(self, id, transient=False, always=False, previous=None):
+        self.id, self.transient, self.always, self.previous = id, transient, always, previous
+
+    def out(self, ctx):
+        return (ctx.temp if self.transient else ctx.dest) / f'{self.id}.out'
+
+    def key(self, ctx):
+        upstream = ctx.keys[self.previous] if self.previous else ''
+        return f'{self.id}:{ctx.options.inputs.get(self.id, "v1")}:{upstream}'
+
+    def current(self, ctx, record):
+        p = self.out(ctx)
+        return p.is_file() and p.read_text() == ctx.keys[self.id]
+
+    def verify(self, ctx, record):
+        ctx.options.verified.append(self.id)
+        return ctx.options.verify_ok.get(self.id, True)
+
+    def run(self, ctx):
+        o = ctx.options
+        o.ran.append(self.id)
+        if self.id == 'check':
+            ctx.verify_kept(self.id)
+        failure = o.fail.pop(self.id, None)
+        if failure == 'error':
+            raise runner.InstallError('fake.failed', step=self.id)
+        if failure == 'cancel':
+            raise KeyboardInterrupt
+        if self.id == 'cleanup':
+            for p in ctx.temp.glob('*.out'):
+                p.unlink()
+            return {}
+        p = self.out(ctx)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(ctx.keys[self.id])
+        return {'note': self.id}
+
+
+def fake_steps():
+    return [Fake('check', always=True), Fake('extract'), Fake('lift', transient=True, previous='extract'),
+            Fake('compile', previous='lift'), Fake('prepare', previous='compile'), Fake('cleanup', always=True)]
+
+
+class Bed:
+    """A dest and temp folder plus one run of the fake steps per call."""
+
+    def __init__(self, base):
+        self.dest, self.temp = base / 'dest', base / 'temp'
+        self.temp.mkdir()
+
+    def run(self, release='1.0', inputs=None, fail=None, verify_ok=None, verify=False):
+        layout = Layout(root=Path('.'), dev=True, release=release, python=Path('python'), tools=Path('tools'),
+                        recomp=Path('recomp'), patches=Path('p'), expected=Path('e'), llvm=Path('llvm'),
+                        deps=Path('deps'), runtime=Path('rt'), runtime_lib=Path('lib'), ninja=Path('ninja'),
+                        hactool=Path('hactool'), xlate=Path('xlate'))
+        options = SimpleNamespace(inputs=inputs or {}, fail=dict(fail or {}), verify_ok=verify_ok or {}, ran=[],
+                                  verified=[])
+        out = io.StringIO()
+        ctx = runner.Context(layout, self.dest, self.temp, Reporter(out=out), options, state=state.load(self.dest),
+                             verify=verify)
+        mode = ctx.mode
+        result = runner.run_steps(fake_steps(), ctx)
+        events = [json.loads(line) for line in out.getvalue().splitlines()]
+        return SimpleNamespace(result=result, ran=options.ran, verified=options.verified, events=events, mode=mode,
+                               state=state.load(self.dest))
+
+    @staticmethod
+    def skipped(r):
+        return {e['step']: e['reason'] for e in r.events if e['type'] == 'step' and e['state'] == 'skipped'}
+
+
+ALL = ['check', 'extract', 'lift', 'compile', 'prepare', 'cleanup']
+
+
+class RunnerTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.bed = Bed(Path(self.tmp.name))
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_fresh_install_runs_everything_and_records_it(self):
+        r = self.bed.run()
+        self.assertEqual((r.result, r.mode, r.ran), ('ok', 'install', ALL))
+        self.assertEqual(set(r.state['steps']), {'extract', 'lift', 'compile', 'prepare'})
+        self.assertEqual(r.state['steps']['compile']['note'], 'compile')
+        self.assertEqual(r.state['release'], '1.0')
+        started = [e['step'] for e in r.events if e['type'] == 'step' and e['state'] == 'started']
+        done = [e['step'] for e in r.events if e['type'] == 'step' and e['state'] == 'done']
+        self.assertEqual((started, done), (ALL, ALL))
+
+    def test_rerun_skips_finished_steps_and_unneeded_transient(self):
+        self.bed.run()
+        r = self.bed.run()
+        self.assertEqual((r.result, r.mode, r.ran), ('ok', 'resume', ['check', 'cleanup']))
+        self.assertEqual(self.bed.skipped(r), {'extract': 'current', 'lift': 'not_needed', 'compile': 'current',
+                                               'prepare': 'current'})
+        self.assertEqual(r.verified, [])   # same release: no full checks
+
+    def test_resume_after_failure(self):
+        r = self.bed.run(fail={'compile': 'error'})
+        self.assertEqual((r.result, r.ran), ('failed', ['check', 'extract', 'lift', 'compile']))
+        error = [e for e in r.events if e['type'] == 'error'][0]
+        self.assertEqual((error['id'], error['step'], error['params']), ('fake.failed', 'compile', {'step': 'compile'}))
+        self.assertEqual(set(r.state['steps']), {'extract', 'lift'})
+        r = self.bed.run()
+        # lift's output is still in the temp folder (no cleanup ran), so only compile and later run.
+        self.assertEqual((r.result, r.mode, r.ran), ('ok', 'resume', ['check', 'compile', 'prepare', 'cleanup']))
+
+    def test_resume_after_cancel_with_transient_output_gone(self):
+        r = self.bed.run(fail={'prepare': 'cancel'})
+        self.assertEqual(r.result, 'cancelled')
+        self.assertNotIn('prepare', r.state['steps'])
+        for p in self.bed.temp.glob('*.out'):
+            p.unlink()
+        r = self.bed.run()
+        self.assertEqual(r.ran, ['check', 'prepare', 'cleanup'])
+        self.assertEqual(self.bed.skipped(r)['lift'], 'not_needed')
+
+    def test_failed_check_writes_nothing(self):
+        r = self.bed.run(fail={'check': 'error'})
+        self.assertEqual(r.result, 'failed')
+        self.assertFalse(self.bed.dest.exists())
+
+    def test_removed_output_runs_again(self):
+        self.bed.run()
+        (self.bed.dest / 'compile.out').unlink()
+        r = self.bed.run()
+        self.assertEqual(r.ran, ['check', 'lift', 'compile', 'cleanup'])
+
+    def test_update_with_new_lifter_keeps_extraction(self):
+        self.bed.run(release='1.0')
+        r = self.bed.run(release='1.1', inputs={'lift': 'v2'})
+        self.assertEqual((r.result, r.mode), ('ok', 'update'))
+        self.assertEqual(r.ran, ['check', 'lift', 'compile', 'prepare', 'cleanup'])
+        self.assertEqual(r.verified, ['extract'])   # kept from another release: full check
+        self.assertEqual(r.state['steps']['extract']['release'], '1.0')
+        self.assertEqual(r.state['steps']['compile']['release'], '1.1')
+        self.assertEqual(r.state['release'], '1.1')
+
+    def test_update_that_changes_nothing_skips_the_rebuild(self):
+        self.bed.run(release='1.0')
+        r = self.bed.run(release='1.1')
+        self.assertEqual(r.ran, ['check', 'cleanup'])
+        self.assertEqual(r.verified, ['extract', 'compile', 'prepare'])
+
+    def test_update_with_changed_kept_files_extracts_again(self):
+        self.bed.run(release='1.0')
+        r = self.bed.run(release='1.1', verify_ok={'extract': False})
+        self.assertEqual(r.ran, ['check', 'extract', 'cleanup'])
+        warnings = [e for e in r.events if e['type'] == 'warning']
+        self.assertEqual([(w['id'], w['params']) for w in warnings], [('state.output_changed', {'step': 'extract'})])
+
+    def test_verify_option_checks_same_release(self):
+        self.bed.run()
+        r = self.bed.run(verify=True)
+        self.assertEqual(r.verified, ['extract', 'compile', 'prepare'])
+
+    def test_state_errors(self):
+        self.bed.dest.mkdir()
+        (self.bed.dest / state.NAME).write_text('{not json')
+        with self.assertRaises(state.StateError) as e:
+            state.load(self.bed.dest)
+        self.assertEqual(e.exception.id, 'state.unreadable')
+        (self.bed.dest / state.NAME).write_text(json.dumps({'format': 99, 'steps': {}}))
+        with self.assertRaises(state.StateError) as e:
+            state.load(self.bed.dest)
+        self.assertEqual(e.exception.id, 'state.newer_format')
+
+
+class IniTest(unittest.TestCase):
+    EXAMPLE = '[display]\n# vk_present = mailbox\n\n[language]\n# locale = en-US\n# lang = en-US\n# x = 1\n'
+
+    def test_example_line_is_activated(self):
+        out = ini.set_language(self.EXAMPLE, 'ru')
+        self.assertIn('\nlang = ru\n', out)
+        self.assertNotIn('# lang', out)
+        self.assertIn('# locale = en-US', out)
+
+    def test_active_line_is_rewritten_and_the_rest_kept(self):
+        text = '[display]\nvk_present = fifo\n[language]\n  LANG =  ja  \nfps60 = 0\n'
+        out = ini.set_language(text, 'de')
+        self.assertEqual(out, '[display]\nvk_present = fifo\n[language]\n  lang =  de\nfps60 = 0\n')
+        self.assertEqual(ini.get_language(out), 'de')
+
+    def test_missing_section_is_added(self):
+        self.assertEqual(ini.set_language('fps60 = 0\n', 'fr'), 'fps60 = 0\n\n[language]\nlang = fr\n')
+        self.assertEqual(ini.set_language('[language]\nlocale = de-DE\n', 'fr'),
+                         '[language]\nlang = fr\nlocale = de-DE\n')
+
+    def test_write_keeps_user_lines(self):
+        with tempfile.TemporaryDirectory() as d:
+            d = Path(d)
+            example = d / 'totk.ini.example'
+            example.write_text(self.EXAMPLE)
+            path = ini.write(d / '', 'en-GB', example)
+            self.assertEqual(ini.get_language(path.read_text()), 'en-GB')
+            path.write_text(path.read_text().replace('# vk_present = mailbox', 'vk_present = fifo'))
+            ini.write(d, 'ja', example)
+            text = path.read_text()
+            self.assertIn('vk_present = fifo', text)
+            self.assertEqual(ini.get_language(text), 'ja')
+
+
+class EventsTest(unittest.TestCase):
+    def test_lines_are_json_and_the_log_is_english(self):
+        with tempfile.TemporaryDirectory() as d:
+            out = io.StringIO()
+            rep = Reporter(out=out, log_path=Path(d) / 'x.log')
+            rep.step('check', 'started')
+            rep.progress('extract', 'copy', 0, 10, 'bytes')
+            rep.progress('extract', 'copy', 5, 10, 'bytes')    # throttled
+            rep.progress('extract', 'copy', 10, 10, 'bytes')   # last one always goes out
+            rep.error('check.disk_space', {'path': 'D:\\x', 'need_gb': 40, 'free_gb': 3}, step='check')
+            rep.close()
+            events = [json.loads(line) for line in out.getvalue().splitlines()]
+            self.assertEqual([e['type'] for e in events], ['step', 'progress', 'progress', 'error'])
+            self.assertEqual([e.get('done') for e in events[1:3]], [0, 10])
+            self.assertTrue(all('t' in e for e in events))
+            log = (Path(d) / 'x.log').read_text(encoding='utf-8')
+            self.assertIn('40 GB needed, 3 GB free', log)
+            self.assertIn('[check.disk_space]', log)
+
+    def test_catalog_has_every_id_the_code_uses(self):
+        catalog = json.loads((ROOT / 'installer' / 'text' / 'en.json').read_text(encoding='utf-8'))
+        used = set()
+        for path in (ROOT / 'installer').glob('*.py'):
+            used |= set(re.findall(r"InstallError\('([a-z_.]+)'", path.read_text()))
+            used |= set(re.findall(r"warning\('([a-z_.]+)'", path.read_text()))
+            used |= {f'phase.{s}.{p}' for s, p in re.findall(r"progress\('(\w+)', '(\w+)'[,)]", path.read_text())}
+        self.assertEqual(sorted(i for i in used if i not in catalog), [])
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/build_game.py

@@ -39,7 +39,8 @@ of its processes (Windows job object accounting: committed private bytes).
 --toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes
 from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR
 (default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++
-runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects.
+runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects. --ninja PATH names
+another Ninja (the installer's bundled one).
 
 Split build. --make-runtime-lib compiles only runtime/ into <out>/libtotkrt.a (what CI prebuilds). --runtime-lib PATH
 then compiles only the generated C and the install-time files (modules.c and INSTALL_SIDE, which bake the player's
@@ -543,6 +544,7 @@ def mib(n):
 
 
 def main():
+    global CLANG, AR, TOOLCHAIN_BIN, DEPS, NINJA
     ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     ap.add_argument('--version', default='v121', help='game version (default v121)')
     ap.add_argument('--gen', type=Path, help='generated C directory (default generated/<version>)')
@@ -571,6 +573,7 @@ def main():
                                                     f'({", ".join(sorted(INSTALL_SIDE))} and modules.c still compile)')
     ap.add_argument('--expected', type=Path,
                     help='expected hashes (default tools/data/expected-<display version>.json)')
+    ap.add_argument('--ninja', type=Path, help=f'ninja.exe to run (default {NINJA})')
     args = ap.parse_args()
 
     gen = (args.gen or ROOT / 'generated' / args.version).resolve()
@@ -585,7 +588,7 @@ def main():
             sys.exit(f'no expected hashes known for {args.version}; pass --expected')
         expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json'
     hashes = json.loads(Path(expected).read_text())['exefs_sha256']
-    global CLANG, AR, TOOLCHAIN_BIN, DEPS
+    NINJA = (args.ninja or NINJA).resolve()
     dlls = SDL3_DLLS
     if args.toolchain:
         TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve()

tools/extract.py

@@ -9,7 +9,9 @@ Steps, each skipped when its verified output already exists:
 Steps 1 and 3 run for every file in `input.updates`. Each update's version is identified by the SHA-256 of its `main`,
 not by its file name. Output: build/extracted/v0/{exefs,romfs} and build/extracted/v<version>/{exefs,romfs}.
 
-Runs under .venv-py314. Key values are read by the tools from the keys file and never printed.
+--base, --update, --keys and --work name the files and the output directory instead of the config (the installer
+imports the functions instead). Runs under .venv-py314. Key values are read by the tools from the keys file and never
+printed.
 """
 import argparse
 import hashlib
@@ -45,6 +47,11 @@ def say(message):
     print(message, flush=True)
 
 
+def shown(path):
+    """path relative to the repository when it is below it (log lines), else as given."""
+    return path.relative_to(ROOT) if path.is_relative_to(ROOT) else path
+
+
 def sha256(path):
     with path.open('rb') as f:
         return hashlib.file_digest(f, 'sha256').hexdigest()
@@ -72,7 +79,7 @@ def run_logged(command, log):
     args = [str(c) for c in command]
     with log.open('w') as out:
         code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT).returncode
-    require(code == 0, f'{Path(command[1]).name} failed; see {log.relative_to(ROOT)}')
+    require(code == 0, f'{Path(command[1]).name} failed; see {shown(log)}')
 
 
 def exefs_problems(exefs, expected):
@@ -132,13 +139,9 @@ def extract_base_romfs(out, keys, logs):
                 '--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log')
 
 
-def merge_update_romfs(base, update, keys_path, hactool, output, logs):
-    """extract_update_romfs.merge() without its Darwin-only bounded view of the base: the base Program NCA is copied out.
-
-    Needs room for both Program NCAs (about 17 GB together) and the merged image (about 17 GB).
-    """
-    if output.exists():
-        shutil.rmtree(output)
+def program_ncas(base, update, keys_path):
+    """(tickets, [(source, record, header)] of the base and the update Program NCA). Raises ValueError for a file
+    that is no TotK base game or update, or keys that cannot open it."""
     keys = load_keys(keys_path)
     tickets = {}
     inputs = []
@@ -153,6 +156,28 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs):
             rights = header[0x230:0x240]
             require(rights == bytes(16) or rights in tickets, 'Missing common ticket')
             inputs.append((source, record, header))
+    return tickets, inputs
+
+
+def merge_space(base, update, keys_path):
+    """Bytes merge_update_romfs needs free next to its output, beyond the merged image itself: the two Program NCAs."""
+    _, inputs = program_ncas(base, update, keys_path)
+    return inputs[0][1]['size'] + inputs[1][1]['size']
+
+
+MERGE_PHASES = {'copy': '  copying the base and update Program NCAs', 'merge': '  merging with hactool',
+                'index': '  hashing and indexing the merged image'}
+
+
+def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lambda name: say(MERGE_PHASES[name])):
+    """extract_update_romfs.merge() without its Darwin-only bounded view of the base: the base Program NCA is copied out.
+
+    Needs room for both Program NCAs (about 17 GB together) and the merged image (about 17 GB). phase(name) is called
+    as each part of MERGE_PHASES starts.
+    """
+    if output.exists():
+        shutil.rmtree(output)
+    tickets, inputs = program_ncas(base, update, keys_path)
     sections = [i for i in range(4) if inputs[1][2][0x400 + i * 512 + 4] == 4]
     require(len(sections) == 1, 'Expected one BKTR section')
     need = inputs[0][1]['size'] + inputs[1][1]['size'] + 18 * (1 << 30)
@@ -167,13 +192,13 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs):
                 f.write(f'{rights.hex()} = {key.hex()}\n')
         hactool_keys = tmp / 'hactool.keys'
         hactool_prod_keys(keys_path, hactool_keys)
-        say('  copying the base and update Program NCAs')
+        phase('copy')
         base_nca, update_nca = tmp / 'base.nca', tmp / 'update.nca'
         for (source, record, _), target in zip(inputs, (base_nca, update_nca)):
             copy_range(source, record_offset(record), record['size'], target)
         result = tmp / 'result'
         result.mkdir()
-        say('  merging with hactool')
+        phase('merge')
         command = [str(hactool), '-x', '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
                    f'--basenca={base_nca}', f'--section{sections[0]}={result / "romfs.bin"}', str(update_nca)]
         environment = dict(os.environ, MNG_TITLE_KEYS=str(title_keys))
@@ -182,10 +207,10 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs):
         log_path.write_text(log)
         checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
         require(completed.returncode == 0 and not HACTOOL_ERROR.search(checked),
-                f'hactool merge failed; see {log_path.relative_to(ROOT)}')
+                f'hactool merge failed; see {shown(log_path)}')
         require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'hactool did not verify all six IVFC levels')
         image = result / 'romfs.bin'
-        say('  hashing and indexing the merged image')
+        phase('index')
         files = index_romfs(image)
         report = dict(status='pass', romfs_size=image.stat().st_size, romfs_sha256=sha256(image), files=files,
                       integrity='BKTR merge; all six IVFC levels verified')
@@ -197,12 +222,21 @@ def main():
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument('--config', type=Path, default=ROOT / 'config.local.toml')
     parser.add_argument('--hactool', type=Path, default=ROOT / 'toolchain' / 'hactool' / 'hactool.exe')
+    parser.add_argument('--base', type=Path, help='base game (default: input.base of the config)')
+    parser.add_argument('--update', type=Path, action='append',
+                        help='an update NSP, repeatable (default: input.updates of the config)')
+    parser.add_argument('--keys', type=Path, help='prod.keys (default: input.keys of the config)')
+    parser.add_argument('--work', type=Path, help='output directory with extracted/ and logs/ (default: work.dir of '
+                                                  'the config, else build)')
     args = parser.parse_args()
 
-    config = tomllib.loads(args.config.read_text(encoding='utf-8'))
-    base, keys = Path(config['input']['base']), Path(config['input']['keys'])
-    updates = config['input'].get('updates', [])
-    work = ROOT / config.get('work', {}).get('dir', 'build')
+    config = {}
+    if not (args.base and args.keys and args.update and args.work):
+        config = tomllib.loads(args.config.read_text(encoding='utf-8'))
+    given = config.get('input', {})
+    base, keys = args.base or Path(given['base']), args.keys or Path(given['keys'])
+    updates = args.update or given.get('updates', [])
+    work = args.work or ROOT / config.get('work', {}).get('dir', 'build')
     extracted, logs = work / 'extracted', work / 'logs'
     logs.mkdir(parents=True, exist_ok=True)
     versions = load_expected()

tools/prepare_shaders.py

@@ -21,6 +21,7 @@ bundles it; it stays out of the repository), else from refs/ through git; nothin
 repository. Needs Python 3.14 (compression.zstd), so run it in .venv-py314 or the embeddable 3.14:
 
     .venv-py314\\Scripts\\python tools\\prepare_shaders.py build\\v121 [--version v121] [--list <tsv>] [--no-run]
+                                                     [--romfs DIR --exefs DIR]   # instead of build\\extracted\\<version>
 """
 import argparse
 import hashlib
@@ -117,13 +118,15 @@ def main():
     ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     ap.add_argument('build', type=Path, help='build directory holding totk.exe (build\\v121)')
     ap.add_argument('--version', default='v121', help='build/extracted/<version>')
+    ap.add_argument('--romfs', type=Path, help='RomFS directory (default build/extracted/<version>/romfs)')
+    ap.add_argument('--exefs', type=Path, help='ExeFS directory (default build/extracted/<version>/exefs)')
     ap.add_argument('--list', help='a shader_list.tsv instead of tools/data/shader_list.tsv or refs/MacNativeGames')
     ap.add_argument('--no-run', action='store_true', help='write the seed files only')
     a = ap.parse_args()
     t0 = time.monotonic()
     variants, prio = parse_list(read_list(a.list))
     extracted = ROOT / 'build' / 'extracted' / a.version
-    shaders, _ = inventory(extracted / 'romfs', extracted / 'exefs')
+    shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs')
     t1 = time.monotonic()
     progs, recs, skipped = {}, [], {'not in this version': 0, 'stage': 0}
     for key, deps in variants: