totk-pc-recomp/installer/steps.py

"""The eight setup steps of docs/installer.md: check, fonts, extract, lift, compile, prepare, dlss, cleanup.

Extraction runs in this process through tools/extract.py (the package's sibling tools/, which ships with it); fonts
runs the vendored extract_shared_fonts.py on the firmware folder; lift, compile and prepare run the bundled Python on
recomp/gen.py, tools/build_game.py, tools/make_romfs_patches.py and tools/prepare_shaders.py, whose output goes to
<temp>/logs/<step>-<tool>.log and is read for progress. No Win32 call here: the host check is a small C program built
with the bundled clang (hostcheck.c).

Install folder: exefs/, romfs/ (romfs.bin, index.json), profile/, shared-fonts/ (Chinese and Korean text), totk.exe
and its DLLs, nvngx_dlss.dll (with --dlss), romfs-index.bin, romfs-patches/, romfs-patches-xbox/,
romfs-patches-xbox-label/, romfs-patches-pc/, romfs-patches-<style>-<w>x<h>/ (each of the three for each shaped size),
totk_xlate.exe, the
shader state of totk.exe --prepare-shaders, totk.ini, totk-install.json.
Temp folder (<temp>/<dest name>-<hash>): check/ (ExeFS of both inputs, the host check), unpacked/ (the NSP of an NSZ
or XCZ input until the extraction is done), gen/ (C), build/ (objects), logs/. The cleanup keeps gen/, build/ and
logs/: the next run lifts into the same gen/, where recomp/gen.py rewrites only the chunks whose C changed, and Ninja
in build/ recompiles only those. A change of the mods' code cheats then costs the lift (about a minute) and the chunks
it touches, not a full compile.

Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status and exefs.data (tools/exefs_patches.py) and
rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild),
one argument per line; totk.exe runs it at start when an enabled mod has code cheats the build lacks
(runtime/cheats.c).
"""
import hashlib
import http.client
import json
import os
import re
import shutil
import signal
import subprocess
import sys
import threading
import time
import urllib.request
from pathlib import Path

from . import ini
from .layout import EXE, SUPPORTED, version_tag
from .runner import InstallError, Step

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'tools'))
import extract  # noqa: E402
import ncz  # noqa: E402
import shader_list  # noqa: E402
from extract_exefs import (common_tickets, container_entries, load_keys, nca_ctr_key, nca_inventory,  # noqa: E402
                           program_nca)

GiB = 1 << 30
DEST_EXTRA = 4 * GiB      # beside the RomFS: shader and pipeline caches (about 2 GB), exefs, totk.exe, seed files
TEMP_NEED = 3 * GiB       # generated C (about 0.9 GB) and objects
HOSTCHECK = Path(__file__).resolve().parent / 'hostcheck.c'
STAMP = '.install-key'


# ---- helpers

def file_sha256(path, progress=None):
    """SHA-256 hex of a file; progress(done, total) after each block."""
    h = hashlib.sha256()
    total = path.stat().st_size
    done = 0
    with open(path, 'rb', buffering=0) as f:
        buf = bytearray(8 << 20)
        view = memoryview(buf)
        while n := f.readinto(buf):
            h.update(view[:n])
            done += n
            if progress:
                progress(done, total)
    return h.hexdigest()


def fingerprint(*parts):
    """SHA-256 over strings and the contents of files and directory trees (a missing path counts as missing)."""
    h = hashlib.sha256()
    for p in parts:
        if isinstance(p, Path):
            if p.is_dir():
                for f in sorted(p.rglob('*')):
                    if f.is_file() and '__pycache__' not in f.parts:
                        h.update(f.relative_to(p).as_posix().encode() + b'\0' + bytes.fromhex(file_sha256(f)))
            elif p.is_file():
                h.update(p.name.encode() + b'\0' + bytes.fromhex(file_sha256(p)))
            else:
                h.update(b'missing\0' + p.name.encode())
        else:
            h.update(str(p).encode())
        h.update(b'\1')
    return h.hexdigest()


def remove(path):
    if path.is_dir() and not path.is_symlink():
        shutil.rmtree(path)
    elif path.exists() or path.is_symlink():
        path.unlink()


def replace_file(src, dst):
    """Copy src over dst through a temporary name, so a cancelled copy never leaves half a file under dst's name."""
    tmp = dst.with_name(dst.name + '.tmp')
    shutil.copy2(src, tmp)
    os.replace(tmp, dst)


def kill_tree(proc):
    if proc.poll() is not None:
        return
    if os.name == 'nt':
        subprocess.run(['taskkill', '/T', '/F', '/PID', str(proc.pid)], capture_output=True)
        return
    try:   # run_tool starts each tool in a session of its own: the group holds ninja, clang and the game
        os.killpg(proc.pid, signal.SIGKILL)
    except OSError:
        proc.kill()


def linux_glibc():
    """(major, minor) of the C library this Linux process runs on, None elsewhere or when it is not glibc."""
    try:
        name = os.confstr('CS_GNU_LIBC_VERSION') if sys.platform.startswith('linux') else None
    except (ValueError, OSError):
        return None
    m = re.match(r'glibc (\d+)\.(\d+)', name or '')
    return (int(m.group(1)), int(m.group(2))) if m else None


# The bundled clang, lld and the cryptography wheel are built against glibc 2.34 (tools/make_release_linux.py
# GLIBC_FLOOR); on an older one they cannot start
GLIBC_NEEDED = (2, 34)


def linux_kernel():
    """(major, minor) of the running Linux kernel, None elsewhere."""
    if not sys.platform.startswith('linux'):
        return None
    m = re.match(r'(\d+)\.(\d+)', os.uname().release)
    return (int(m.group(1)), int(m.group(2))) if m else None


def run_tool(ctx, step, script, args, parse=None, cwd=None):
    """Run a tool script with the bundled Python. Its output goes to <temp>/logs/<step>-<script>.log and, line by line,
    to parse(line). Returns (exit code, log path)."""
    logs = ctx.temp / 'logs'
    logs.mkdir(parents=True, exist_ok=True)
    log = logs / f'{step}-{Path(script).stem}.log'
    command = [str(ctx.layout.python), '-c', extract.RUN_SCRIPT, str(script), *map(str, args)]
    ctx.reporter.log('run ' + ' '.join(command[3:]))
    env = dict(os.environ, PYTHONUNBUFFERED='1', PYTHONIOENCODING='utf-8')
    with open(log, 'w', encoding='utf-8', newline='\n') as out:
        proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd or ctx.layout.root,
                                env=env, text=True, encoding='utf-8', errors='replace', bufsize=1,
                                start_new_session=os.name != 'nt')
        try:
            for line in proc.stdout:
                out.write(line)
                if parse:
                    parse(line.rstrip())
            code = proc.wait()
        except BaseException:
            kill_tree(proc)
            raise
    ctx.reporter.log(f'{Path(script).name}: exit code {code}; output in {log}')
    return code, log


def free_bytes(path):
    """Free bytes on the disk of path (its nearest existing parent) and that parent's device."""
    p = Path(path)
    # An unavailable drive or share has no existing ancestor: let disk_usage report its OS error at the root.
    while not p.exists() and p.parent != p:
        p = p.parent
    return shutil.disk_usage(p).free, os.stat(p).st_dev


def raise_if_damaged(error, name, path):
    """InstallError check.input_damaged when a reader's ValueError says the file's data fails its hashes or ends early
    (extract.DAMAGE_TEXT)."""
    if extract.DAMAGE_TEXT.search(str(error)):
        raise InstallError('check.input_damaged', input=name, path=str(path), detail=str(error)) from None


def game_file(ctx, name):
    """The file the extraction reads for the base or the update: the NSP that Check.unpack made from an NSZ or XCZ,
    else the chosen file itself."""
    return getattr(ctx, 'unpacked', {}).get(name) or Path(getattr(ctx.options, name))


def game_version(ctx):
    """The update this install is built from: what the check identified, else the recorded one. An install made before
    1.4.3 support records no version, and that install is 1.2.1: name it here rather than leaning on SUPPORTED's order."""
    return ctx.state.get('version', '1.2.1')


def expected_update(ctx):
    return extract.load_expected()[game_version(ctx)]


# Releases before 0.3.1 recorded the 0x20000 update as 1.1.1 (pull request #220); it is 1.1.0 (#371), with the same
# files, hashes and patch sites under new names.
RENAMED_VERSIONS = {'1.1.1': '1.1.0'}


def migrate_state(ctx):
    """Read a recorded update by its current name. The extraction it recorded stays: its record gets the key of the
    renamed expected file, and Extract.current and the full check of output another release kept
    (runner.Context.verify_kept) still test the files against the expected hashes."""
    old = ctx.state.get('version')
    if old not in RENAMED_VERSIONS:
        return
    ctx.state['version'] = RENAMED_VERSIONS[old]
    record = ctx.state['steps'].get('extract')
    if record:
        record['key'] = Extract().key(ctx)
    ctx.reporter.log(f'state: the update recorded as {old} is {ctx.state["version"]}')


def shader_list_text(layout):
    if layout.shader_list:
        return layout.shader_list.read_text(encoding='utf-8')
    try:
        return shader_list.history(layout.root / 'refs' / 'MacNativeGames')
    except subprocess.CalledProcessError:
        raise InstallError('prepare.shader_list_missing')


class Watch(threading.Thread):
    """Polls a measure() of bytes written while a long call runs and reports it as progress."""

    def __init__(self, reporter, step):
        super().__init__(daemon=True)
        self.reporter, self.step = reporter, step
        self.phase, self.measure, self.total = None, None, None
        self.stopped = threading.Event()

    def set(self, phase, measure=None, total=None):
        self.phase, self.measure, self.total = phase, measure, total
        self.reporter.progress(self.step, phase, 0 if total else None, total, 'bytes' if total else None, force=True)

    def run(self):
        while not self.stopped.wait(1.0):
            phase, measure, total = self.phase, self.measure, self.total
            if phase and measure:
                try:
                    done = min(measure(), total)
                except OSError:
                    continue
                self.reporter.progress(self.step, phase, done, total, 'bytes')

    def stop(self):
        self.stopped.set()


def size_of(paths):
    return sum(p.stat().st_size for p in paths if p.is_file())


# The Switch system shared fonts (runtime/ov_system.c): TotK falls back to them for Chinese, Korean and Traditional
# Chinese text. tools/vendor/macnativegames/extract_shared_fonts.py writes these names.
SHARED_FONTS = 'shared-fonts'
SHARED_FONT_FILES = ('0-standard.bfttf', '1-chinese-simplified.bfttf', '2-chinese-simplified-extension.bfttf',
                     '3-chinese-traditional.bfttf', '4-korean.bfttf', '5-nintendo-extension.bfttf')
FONTS_NEED = 32 << 20     # the six BFTTF files and their manifest


def requested_lang(ctx):
    """The language this run uses: --lang, else the one recorded for the install (Check canonicalizes it)."""
    return ini.canonical_language(getattr(ctx.options, 'lang', None) or ctx.state.get('lang'))


def firmware_dir(value):
    """The firmware NCA folder behind a --firmware path: the folder itself, or its registered/ child (the emulators'
    layout, Contents/registered)."""
    if not value:
        return None
    path = Path(value)
    return path / 'registered' if (path / 'registered').is_dir() else path


def firmware_has_ncas(folder):
    """Whether folder holds firmware NCAs: .nca files, or the hashed directories with a 00 file (registered/)."""
    try:
        entries = list(Path(folder).iterdir())
    except OSError:
        return False
    return any((e.is_file() and e.suffix.lower() == '.nca') or (e.is_dir() and (e / '00').is_file()) for e in entries)


def firmware_stamp(folder):
    """A cheap identity of the firmware NCAs: names, sizes and mtimes, not their contents (the folder is ~326 MB)."""
    try:
        entries = sorted(Path(folder).iterdir())
    except OSError:
        return 'missing'
    rows = []
    for entry in entries:
        if entry.is_dir():
            file = entry / '00'
            if not file.is_file():
                continue
        elif not entry.is_file() or entry.suffix.lower() != '.nca':
            continue
        else:
            file = entry
        st = file.stat()
        rows.append((entry.name, st.st_size, st.st_mtime_ns))
    return rows


def shared_fonts_ready(dest):
    """Whether dest/shared-fonts holds a complete set that the game can load."""
    folder = Path(dest) / SHARED_FONTS
    return (folder / 'manifest.json').is_file() and all((folder / name).is_file() for name in SHARED_FONT_FILES)


# ---- 1. check

UPDATE_TITLE_ID = '0100F2C0115B6800'


def base_misuse(records):
    """Name why a file chosen as the base game holds an update or a combined package, or return None.

    The Meta NCA (content type 1) carries the package's own title id: the update's id (...800) for an update NSP, and
    several Meta NCAs mean base, update and DLC repacked into one file.
    """
    metadata = [r['title_id'] for r, _ in records if r['content_type'] == 1]
    if len(metadata) > 1:
        return 'check.base_combined'
    if UPDATE_TITLE_ID in metadata:
        return 'check.base_is_update'
    return None


class Check(Step):
    id = 'check'
    always = True

    def key(self, ctx):
        # The compile step fingerprints the CPU level, and every key is computed before a step runs: probe the host
        # here, once, so both steps see the same answer (the check step itself is always run). A failed probe is
        # reported by run(), where the error names this step.
        try:
            self.host(ctx)
        except InstallError:
            pass
        return ''

    def run(self, ctx):
        layout, rep, dest = ctx.layout, ctx.reporter, ctx.dest
        missing = layout.missing(getattr(ctx, 'cpu', 'v3'))
        if missing:
            raise InstallError('check.release_incomplete', paths=missing[:5])
        if dest.exists() and not dest.is_dir():
            raise InstallError('check.dest_not_folder', path=str(dest))
        own = (ctx.temp, *ctx.temp.parents)   # a temp folder inside the install folder (installer/cli.py) is ours
        if ctx.mode == 'install' and dest.is_dir() and any(p not in own for p in dest.iterdir()):
            raise InstallError('check.dest_not_empty', path=str(dest))
        lang = requested_lang(ctx)
        if not lang:
            raise InstallError('check.lang_missing')
        if lang not in ini.LANGUAGES:
            raise InstallError('check.lang_unknown', lang=lang, known=', '.join(ini.LANGUAGES))
        firmware = firmware_dir(getattr(ctx.options, 'firmware', None))
        if firmware is not None:
            if not firmware.is_dir():
                raise InstallError('check.firmware_missing', path=str(firmware))
            if not firmware_has_ncas(firmware):
                raise InstallError('check.firmware_empty', path=str(firmware))
        if lang in ini.NEED_SHARED_FONTS:
            if firmware is None:
                rep.warning('warn.lang_needs_fonts', lang=lang)
            elif not shared_fonts_ready(dest):
                keys = getattr(ctx.options, 'keys', None)
                if not keys or not Path(keys).is_file():
                    raise InstallError('check.fonts_keys_needed')
                try:
                    key_values = load_keys(Path(keys))
                except (ValueError, OSError, UnicodeDecodeError):
                    raise InstallError('check.keys_unreadable') from None
                if 'header_key' not in key_values:
                    raise InstallError('check.keys_no_header_key')
        if (sys.platform == 'linux' and ctx.will_run('prepare')
                and not (os.environ.get('DISPLAY') or os.environ.get('WAYLAND_DISPLAY'))):
            # totk --prepare-shaders opens a window and needs a present-capable queue family; without a display it
            # fails after lift and compile (~15 min), so stop here
            raise InstallError('check.no_display')
        ctx.lang = lang
        ctx.temp.mkdir(parents=True, exist_ok=True)

        rep.progress('check', 'host', force=True)
        self.host(ctx)
        rep.progress('check', 'verify', force=True)
        ctx.verify_kept('check')
        ncas = 0
        if ctx.will_run('extract'):
            rep.progress('check', 'inputs', force=True)
            ncas = self.inputs(ctx)
        self.space(ctx, ncas)
        if ctx.will_run('extract'):
            rep.progress('check', 'versions', force=True)
            self.versions(ctx)
        return {}

    def host(self, ctx):
        if getattr(ctx, 'host', None) is not None:
            return
        layout, work = ctx.layout, ctx.temp / 'check'
        glibc = linux_glibc()
        if glibc and glibc < GLIBC_NEEDED and not layout.dev:
            raise InstallError('check.glibc_old', version='.'.join(map(str, glibc)),
                               need='.'.join(map(str, GLIBC_NEEDED)))
        work.mkdir(parents=True, exist_ok=True)
        exe = work / ('hostcheck' + EXE)
        command = [str(layout.clang), '-O1', str(HOSTCHECK), '-o', str(exe)]
        if layout.deps:
            command += ['-I', str(layout.deps / 'include')]
        if layout.sysroot:
            command += [f'--sysroot={layout.sysroot}', '-fuse-ld=lld']
        if os.name != 'nt':
            command.append('-ldl')
        r = subprocess.run(command, capture_output=True, text=True, errors='replace')
        if r.returncode:
            ctx.reporter.log(r.stdout + r.stderr)
            raise InstallError('check.toolchain_broken', detail=(r.stderr or r.stdout).strip()[-400:])
        r = subprocess.run([str(exe)], capture_output=True, text=True, errors='replace')
        try:
            host = json.loads(r.stdout)
        except ValueError:
            raise InstallError('check.host_probe_failed', code=r.returncode) from None
        ctx.reporter.log(f'host: {r.stdout.strip()}')
        if not host['x86_64_v2']:
            raise InstallError('check.cpu_too_old')
        # v3 is the faster build; v2 runs on CPUs without AVX2/FMA (runtime/fma.c supplies the fused multiply-add).
        ctx.cpu = 'v3' if host['x86_64_v3'] else 'v2'
        if not host['loader']:
            raise InstallError('check.vulkan_missing')
        devices = host['devices']
        if not devices:
            raise InstallError('check.vulkan_no_device')
        if not any(tuple(d['api'][:2]) >= (1, 3) for d in devices):
            raise InstallError('check.vulkan_too_old',
                               devices=', '.join(f'{d["name"]} ({d["api"][0]}.{d["api"][1]})' for d in devices))
        kernel = linux_kernel()
        if kernel and kernel < (6, 7):
            # Without userfaultfd's asynchronous write protection nothing tracks the CPU's writes: the game copies whole
            # pools to the GPU at each submit and its texture checks see every page as changed (docs/building.md, Linux)
            ctx.reporter.warning('warn.kernel_old', version='.'.join(map(str, kernel)))
        ctx.host = host

    def inputs(self, ctx):
        """Open both files with the keys as the extraction will. Returns the bytes of both Program NCAs."""
        o = ctx.options
        for name in ('base', 'update', 'keys'):
            path = getattr(o, name)
            if not path:
                raise InstallError('check.inputs_needed')
            if not Path(path).is_file():
                raise InstallError('check.input_missing', input=name, path=str(path))
        try:
            keys = load_keys(Path(o.keys))
        except (ValueError, OSError, UnicodeDecodeError):
            raise InstallError('check.keys_unreadable') from None
        if 'header_key' not in keys:
            raise InstallError('check.keys_no_header_key')
        self.unpack(ctx)
        total = 0
        for name, is_update in (('base', False), ('update', True)):
            path = game_file(ctx, name)
            with path.open('rb') as f:
                total += self.open_input(f, path, keys, name, is_update)
        del keys
        return total

    def unpack(self, ctx):
        """Unpack an NSZ or XCZ (a game file compressed by the nsz tool) into <temp>/unpacked/<base|update>.nsp with
        tools/ncz.py, which checks every NCA against its hash; the extraction then reads that NSP. An earlier run's NSP
        of the same file is reused. The extract step removes the folder once it is done."""
        o, rep, folder = ctx.options, ctx.reporter, ctx.temp / 'unpacked'
        ctx.unpacked = {}
        jobs = []
        for name in ('base', 'update'):
            path = Path(getattr(o, name))
            try:
                found = ncz.compressed(path)
                if found is None:
                    continue
                if name == 'update' and found[0] != 'nsz':
                    raise InstallError('check.update_not_nsp')
                size = ncz.unpacked_size(path)
            except ValueError as e:
                raise_if_damaged(e, name, path)
                raise InstallError('check.not_game_file', input=name) from None
            st = path.stat()
            jobs.append((name, path, size, f'{path.resolve()}\n{st.st_size}\n{st.st_mtime_ns}\n{size}\n'))
        if not jobs:
            remove(folder)
            return
        done = []
        for name, path, size, tag in jobs:
            target, stamp = folder / f'{name}.nsp', folder / f'{name}.source'
            try:
                if target.stat().st_size == size and stamp.read_text(encoding='utf-8') == tag:
                    done.append(name)
                    ctx.unpacked[name] = target
            except OSError:
                pass
        need = sum(size for name, _, size, _ in jobs if name not in done)
        free, _ = free_bytes(folder)
        rep.log(f'unpack: {", ".join(name for name, *_ in jobs)}: {need / GiB:.1f} GiB to write, '
                f'{free / GiB:.1f} free')
        if need > free:
            raise InstallError('check.disk_space', path=str(folder), need_gb=round(need / 1e9, 1),
                               free_gb=round(free / 1e9, 1))
        for name, path, size, tag in jobs:
            if name in done:
                continue
            target, stamp = folder / f'{name}.nsp', folder / f'{name}.source'
            remove(stamp)
            try:
                folder.mkdir(parents=True, exist_ok=True)
                ncz.unpack(path, target, lambda d, t, n=name: rep.progress('check', 'unpack_' + n, d, t, 'bytes'))
                stamp.write_text(tag, encoding='utf-8')
            except ncz.DamagedFile as e:
                raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None
            except ValueError as e:
                raise_if_damaged(e, name, path)
                raise InstallError('check.unpack_failed', input=name, detail=str(e)) from None
            except OSError as e:
                raise InstallError('check.unpack_failed', input=name, detail=str(e)) from None
            ctx.unpacked[name] = target

    @staticmethod
    def open_input(f, path, keys, name, is_update):
        try:
            kind, entries = container_entries(f, path.stat().st_size)
        except ValueError as e:
            raise_if_damaged(e, name, path)
            raise InstallError('check.not_game_file', input=name) from None
        if is_update and kind != 'nsp':
            raise InstallError('check.update_not_nsp')
        try:
            tickets = common_tickets(f, entries)
        except ValueError as e:
            if 'Personalized' in str(e):
                raise InstallError('check.ticket_personalized', input=name) from None
            raise_if_damaged(e, name, path)
            raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
        try:
            records = nca_inventory(f, entries, keys)
        except ValueError as e:
            if 'NCA3' in str(e):
                raise InstallError('check.keys_wrong', input=name) from None
            raise_if_damaged(e, name, path)
            raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
        try:
            record, header = program_nca(records, update=is_update)
        except ValueError as e:
            if is_update and 'BKTR' in str(e):
                raise InstallError('check.update_is_base') from None
            if not is_update:
                reason = base_misuse(records)
                if reason:
                    raise InstallError(reason) from None
            raise InstallError('check.update_not_totk' if is_update else 'check.base_not_totk', detail=str(e)) from None
        try:
            nca_ctr_key(header, keys, tickets)
        except ValueError as e:
            if 'ticket' in str(e):
                raise InstallError('check.ticket_missing', input=name) from None
            raise InstallError('check.keys_too_old', input=name,
                               generation=max(header[0x206], header[0x220])) from None
        return record['size']

    def space(self, ctx, ncas):
        dest_need = temp_need = 0
        if ctx.will_run('extract'):
            # the update is identified after this estimate: the larger RomFS of the supported ones
            dest_need += max(extract.load_expected()[v]['romfs']['size'] for v in SUPPORTED) + ncas
            romfs = ctx.dest / 'romfs' / 'romfs.bin'
            if romfs.is_file():   # a resumed extraction replaces it
                dest_need -= romfs.stat().st_size
        if ctx.will_run('fonts'):
            dest_need += FONTS_NEED
        if ctx.will_run('prepare') or ctx.will_run('extract'):
            dest_need += DEST_EXTRA
        if ctx.will_run('lift') or ctx.will_run('compile'):
            temp_need += TEMP_NEED
        dest_free, dest_dev = free_bytes(ctx.dest)
        temp_free, temp_dev = free_bytes(ctx.temp)
        ctx.reporter.log(f'space: install folder needs {dest_need / GiB:.1f} GiB of {dest_free / GiB:.1f} free, '
                         f'temp folder {temp_need / GiB:.1f} GiB of {temp_free / GiB:.1f}')
        if dest_dev == temp_dev:
            dest_need += temp_need
            temp_need = 0
        ctx.reporter.emit('space', dest=str(ctx.dest), need=dest_need, free=dest_free, temp=str(ctx.temp),
                          temp_need=temp_need, temp_free=temp_free, same_disk=dest_dev == temp_dev)
        for path, need, free in ((ctx.dest, dest_need, dest_free), (ctx.temp, temp_need, temp_free)):
            if need > free:
                raise InstallError('check.disk_space', path=str(path), need_gb=round(need / 1e9, 1),
                                   free_gb=round(free / 1e9, 1))

    def versions(self, ctx):
        """Extract both ExeFS into the temp folder (verifying every block) and identify the versions by hash."""
        o, work = ctx.options, ctx.temp / 'check'
        logs = ctx.temp / 'logs'
        logs.mkdir(parents=True, exist_ok=True)
        versions = extract.load_expected()
        for name, is_update in (('base', False), ('update', True)):
            out = work / name
            remove(out)
            try:
                extract.extract_exefs(game_file(ctx, name), Path(o.keys), out, logs, update=is_update)
            except extract.DamagedFile as e:
                raise InstallError('check.input_damaged', input=name, path=str(getattr(o, name)),
                                   detail=e.detail) from None
            except ValueError:
                raise InstallError('check.exefs_failed', input=name, log=str(logs / f'extract_exefs-{name}.log')) \
                    from None
        problems = extract.exefs_problems(work / 'base' / 'exefs', versions['1.0.0'])
        if problems:
            raise InstallError('check.base_wrong_version', modules=', '.join(problems))
        try:
            found = extract.identify_update(work / 'update' / 'exefs', versions)['display_version']
        except ValueError:
            found = 'unknown'
        if found not in SUPPORTED:
            raise InstallError('check.update_wrong_version', found=found, wanted=' / '.join(SUPPORTED))
        if found != game_version(ctx):   # another update than the recorded one: every step builds again from it
            ctx.reporter.log(f'check: update {found} (recorded {game_version(ctx)})')
            ctx.state['version'] = found
            ctx.keys = {}
            ctx.plan(ctx.forced | {'extract', 'lift', 'compile', 'prepare'})
        ctx.state['version'] = found


# ---- 2. fonts

class Fonts(Step):
    """The Switch system fonts for Chinese, Korean and Traditional Chinese text (tools/extract_fonts.py, which runs the
    vendored extract_shared_fonts.py). The firmware folder is optional: without it the setup warns and the game runs
    with those glyphs missing. The extraction verifies the firmware NCAs and writes the six BFTTF files the console
    puts in shared memory."""

    id = 'fonts'

    def key(self, ctx):
        lang = requested_lang(ctx)
        if lang not in ini.NEED_SHARED_FONTS:
            return fingerprint('fonts-1', 'not-needed')
        firmware = firmware_dir(getattr(ctx.options, 'firmware', None))
        if firmware is None:
            return fingerprint('fonts-1', 'no-firmware')
        return fingerprint('fonts-1', lang, str(firmware), firmware_stamp(firmware))

    def current(self, ctx, record):
        if requested_lang(ctx) not in ini.NEED_SHARED_FONTS or not getattr(ctx.options, 'firmware', None):
            return True
        return shared_fonts_ready(ctx.dest)

    def verify(self, ctx, record):
        # An update run may omit --firmware; the installed set still has to be there for these languages.
        if requested_lang(ctx) in ini.NEED_SHARED_FONTS:
            return shared_fonts_ready(ctx.dest)
        return True

    def run(self, ctx):
        lang = getattr(ctx, 'lang', None) or requested_lang(ctx)
        if lang not in ini.NEED_SHARED_FONTS:
            return {}
        firmware = firmware_dir(getattr(ctx.options, 'firmware', None))
        if firmware is None:
            return {}   # Check warned: the game runs, some text may not show
        keys = getattr(ctx.options, 'keys', None)
        if not keys:
            raise InstallError('check.fonts_keys_needed')
        out = ctx.temp / 'shared-fonts'
        remove(out)
        ctx.reporter.progress('fonts', 'extract', force=True)
        tool = ctx.layout.tools / 'vendor' / 'macnativegames' / 'extract_shared_fonts.py'
        code, log = run_tool(ctx, 'fonts', tool, ['--keys', keys, '--firmware', firmware, '--output', out])
        if code:
            detail = ''
            try:
                detail = log.read_text(encoding='utf-8', errors='replace').strip().splitlines()[-1][:300]
            except (OSError, IndexError):
                pass
            raise InstallError('fonts.failed', code=code, detail=detail, log=str(log))
        missing = [name for name in SHARED_FONT_FILES if not (out / name).is_file()]
        if missing or not (out / 'manifest.json').is_file():
            raise InstallError('fonts.incomplete', files=', '.join(missing) or 'manifest.json')
        ctx.reporter.progress('fonts', 'copy', force=True)
        try:
            ctx.dest.mkdir(parents=True, exist_ok=True)
            remove(ctx.dest / SHARED_FONTS)
            shutil.move(str(out), str(ctx.dest / SHARED_FONTS))
        except OSError as e:
            raise InstallError('fonts.copy_failed', path=str(ctx.dest / SHARED_FONTS), detail=str(e)) from None
        return {'files': len(SHARED_FONT_FILES)}


# ---- 3. extract

class Extract(Step):
    id = 'extract'

    def key(self, ctx):
        # The output is whatever matches the expected hashes, whichever files and tools produced it.
        return fingerprint('extract-1', ctx.layout.expected(game_version(ctx)))

    def current(self, ctx, record):
        # Another update file than the recorded one (1.2.1 -> 1.4.3) extracts again; the check then identifies it.
        update, known = getattr(ctx.options, 'update', None), ctx.state['inputs'].get('update')
        if update and known and Path(update).is_file():
            st = Path(update).stat()
            if (st.st_size, st.st_mtime_ns) != (known.get('size'), known.get('mtime_ns')) \
                    and file_sha256(Path(update)) != known.get('sha256'):
                return False
        expected = expected_update(ctx)
        romfs = ctx.dest / 'romfs'
        if extract.exefs_problems(ctx.dest / 'exefs', expected) or not extract.romfs_verified(romfs, expected):
            return False
        st = (romfs / 'romfs.bin').stat()
        return st.st_size == record.get('romfs_size') and st.st_mtime_ns == record.get('romfs_mtime_ns')

    def verify(self, ctx, record):
        image = ctx.dest / 'romfs' / 'romfs.bin'
        rep = ctx.reporter
        digest = file_sha256(image, lambda d, t: rep.progress('check', 'verify_romfs', d, t, 'bytes'))
        return digest == expected_update(ctx)['romfs']['sha256']

    def run(self, ctx):
        o, dest, rep = ctx.options, ctx.dest, ctx.reporter
        expected = expected_update(ctx)
        keys = Path(o.keys)
        inputs = {}
        for name, path in (('base', Path(o.base)), ('update', Path(o.update))):
            known = ctx.state['inputs'].get(name, {})
            st = path.stat()
            if known.get('size') == st.st_size and known.get('mtime_ns') == st.st_mtime_ns and known.get('sha256'):
                inputs[name] = known
                continue
            digest = file_sha256(path, lambda d, t, n=name: rep.progress('extract', 'hash_' + n, d, t, 'bytes'))
            inputs[name] = {'name': path.name, 'size': st.st_size, 'mtime_ns': st.st_mtime_ns, 'sha256': digest}
        ctx.state['inputs'] = inputs
        ctx.save()
        base, update = game_file(ctx, 'base'), game_file(ctx, 'update')

        dest.mkdir(parents=True, exist_ok=True)
        for stale in dest.glob('update-merge-*'):   # a killed merge leaves its NCA copies (and key copies) behind
            remove(stale)
        rep.progress('extract', 'exefs', force=True)
        staged = ctx.temp / 'check' / 'update' / 'exefs'
        if not staged.is_dir():
            remove(ctx.temp / 'check' / 'update')
            try:
                extract.extract_exefs(update, keys, ctx.temp / 'check' / 'update', ctx.temp / 'logs', update=True)
            except extract.DamagedFile as e:
                raise InstallError('check.input_damaged', input='update', path=str(o.update), detail=e.detail) from None
            except ValueError:
                raise InstallError('check.exefs_failed', input='update',
                                   log=str(ctx.temp / 'logs' / 'extract_exefs-update.log')) from None
        remove(dest / 'exefs')
        shutil.move(staged, dest / 'exefs')
        problems = extract.exefs_problems(dest / 'exefs', expected)
        if problems:
            raise InstallError('extract.exefs_mismatch', version=game_version(ctx), modules=', '.join(problems))
        if extract.sha256(dest / 'exefs' / 'main.npdm') != expected['exefs_sha256']['main.npdm']:
            rep.warning('warn.npdm_differs', version=game_version(ctx))

        romfs = dest / 'romfs'
        if self.id in ctx.forced:
            remove(romfs)   # the full hash failed: index.json still claims the extraction's hash
        if not extract.romfs_verified(romfs, expected):
            self.merge(ctx, base, update, keys, romfs, expected)
        report = json.loads((romfs / 'index.json').read_text())
        if report['romfs_size'] != expected['romfs']['size'] or report['romfs_sha256'] != expected['romfs']['sha256']:
            raise InstallError('extract.romfs_mismatch', version=game_version(ctx), size=report['romfs_size'],
                               sha256=report['romfs_sha256'][:16])
        (dest / 'profile').mkdir(exist_ok=True)
        remove(ctx.temp / 'unpacked')   # the NSPs unpacked from an NSZ or XCZ (Check.unpack)
        st = (romfs / 'romfs.bin').stat()
        return {'romfs_size': st.st_size, 'romfs_mtime_ns': st.st_mtime_ns}

    def merge(self, ctx, base, update, keys, romfs, expected):
        dest, logs = ctx.dest, ctx.temp / 'logs'
        ncas = extract.merge_space(base, update, keys)
        watch = Watch(ctx.reporter, 'extract')

        def phase(name):
            if name == 'copy':
                watch.set('copy', lambda: size_of(dest.glob('update-merge-*/*.nca')), ncas)
            elif name == 'merge':
                watch.set('merge', lambda: size_of(dest.glob('update-merge-*/result/romfs.bin')),
                          expected['romfs']['size'])
            else:
                watch.set(name)

        watch.start()
        try:
            extract.merge_update_romfs(base, update, keys, ctx.layout.hactool, romfs, logs, phase=phase)
        except extract.DamagedFile as e:
            name = 'base' if e.path == base else 'update'
            raise InstallError('check.input_damaged', input=name, path=str(getattr(ctx.options, name)),
                               detail=e.detail) from None
        except ValueError as e:
            raise InstallError('extract.romfs_merge_failed', detail=str(e), log=str(logs / 'merge_romfs-'
                                                                                     f'{dest.name}.log')) from None
        finally:
            watch.stop()


# ---- 4. lift

EMITTED = re.compile(r'^emitted (\d+)/(\d+) chunks')
DISCOVERED = re.compile(r'^(\w+): \d+ functions \(.*discovery')


CHEAT_PLACES = ('cheats', 'atmosphere/contents/0100F2C0115B6000/cheats', 'contents/0100F2C0115B6000/cheats',
                '0100F2C0115B6000/cheats')


def cheat_files(ctx):
    """The Atmosphere cheat files of the mods folder (<dest>/mods/<mod>/cheats/*.txt and the Atmosphere places, as
    tools/cheats.py finds them), for the lift key. All of them: the ones the build reads go by the build id of the
    installed main, and the keys are computed before Extract runs (runner plan), when a fresh install has no main and an
    update may still have another version's."""
    mods = ctx.dest / 'mods'
    if not mods.is_dir():
        return []
    return sorted(f for mod in mods.iterdir() if mod.is_dir() for where in CHEAT_PLACES
                  for f in ((mod / where).glob('*.txt') if (mod / where).is_dir() else ()))


EXEFS_PLACES = ('exefs', 'atmosphere/contents/0100F2C0115B6000/exefs', 'contents/0100F2C0115B6000/exefs',
                '0100F2C0115B6000/exefs')


def exefs_files(ctx):
    """The ExeFS patch files of the mods folder (.ips and .pchtxt in each mod's exefs places and its
    atmosphere/exefs_patches/<name>/, as tools/exefs_patches.py finds them), for the lift key."""
    mods = ctx.dest / 'mods'
    if not mods.is_dir():
        return []
    out = []
    for mod in (m for m in mods.iterdir() if m.is_dir()):
        folders = [mod / w for w in EXEFS_PLACES if (mod / w).is_dir()]
        patches = mod / 'atmosphere' / 'exefs_patches'
        folders += [d for d in patches.iterdir() if d.is_dir()] if patches.is_dir() else []
        out += [f for d in folders for f in d.iterdir() if f.is_file() and f.suffix.lower() in ('.ips', '.pchtxt')]
    return sorted(out)


def exefs_status(ctx):
    """The build's ExeFS patch status file (tools/exefs_patches.py --status), which runtime/mods.c reads."""
    return ctx.dest / 'build-cache' / 'exefs.status'


def exefs_data(ctx):
    """The data writes of the mods' ExeFS patch files (tools/exefs_patches.py --data), which runtime/mods.c stores into
    main before the game runs."""
    return ctx.dest / 'build-cache' / 'exefs.data'


def cheats_status(ctx):
    """The build's cheat status file (tools/cheats.py --status), which runtime/cheats.c reads: in build-cache, not in a
    mod's own cheats folder, which the player (or TKMM's merge) may empty."""
    return ctx.dest / 'build-cache' / 'cheats.status'


class Lift(Step):
    id = 'lift'
    transient = True

    def key(self, ctx):
        # fingerprint takes a file's name and content only; the paths under mods/ go in too, because tools/cheats.py
        # and tools/exefs_patches.py name each patch set after its mod folder, so a renamed folder needs a lift
        files = cheat_files(ctx) + exefs_files(ctx)
        return fingerprint('lift-5', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)),
                           ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *files,
                           *(f.relative_to(ctx.dest / 'mods').as_posix() for f in files))

    def current(self, ctx, record):
        gen = ctx.temp / 'gen'
        stamp = gen / STAMP
        return (gen / 'layout.json').is_file() and stamp.is_file() and stamp.read_text().strip() == ctx.keys['lift']

    def run(self, ctx):
        # gen/ of the last run stays: gen.py rewrites only the chunks whose C changed (and removes the ones a smaller
        # partition no longer has), so Ninja recompiles only those
        layout, rep, gen = ctx.layout, ctx.reporter, ctx.temp / 'gen'
        (gen / STAMP).unlink(missing_ok=True)
        rep.progress('lift', 'discover', force=True)

        def parse(line):
            m = EMITTED.match(line)
            if m:
                rep.progress('lift', 'emit', int(m.group(1)), int(m.group(2)), 'chunks')

        # The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets and data writes
        # (tools/cheats.py, tools/exefs_patches.py); their status files tell the runtime what the build has. Both are
        # the player's downloads, so files the tools refuse cost those cheats or patches, not the install: the game is
        # built without them and the setup warns.
        plain = layout.patches(game_version(ctx))
        # (tool, patch file it writes, its outputs for the runtime: the status first, then the data file, the files it
        # reads, the warning when it fails)
        stages = [('cheats.py', 'patches-cheats.txt', [cheats_status(ctx)], cheat_files(ctx), 'warn.cheats_skipped'),
                  ('exefs_patches.py', 'patches-exefs.txt', [exefs_status(ctx), exefs_data(ctx)], exefs_files(ctx),
                   'warn.exefs_skipped')]
        patches, skipped = plain, {}

        def drop(outputs):   # no status: the runtime treats every such file as not built
            for f in outputs:
                f.unlink(missing_ok=True)

        for tool, out, outputs, inputs, warn in stages:
            drop(outputs)
            if not inputs:
                continue
            code, log = run_tool(ctx, 'lift', layout.tools / tool,
                                 ['--exefs', ctx.dest / 'exefs', '--mods', ctx.dest / 'mods', '--patches', patches,
                                  '--out', ctx.temp / out, '--status', outputs[0],
                                  *(['--data', outputs[1]] if len(outputs) > 1 else [])])
            if code:
                drop(outputs)
                skipped[warn] = log
            else:
                patches = ctx.temp / out
        args = ['--exefs', ctx.dest / 'exefs', '--out', gen]
        code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches], parse)
        if code and patches != plain:
            for tool, out, outputs, inputs, warn in stages:
                if inputs and warn not in skipped:
                    drop(outputs)
                    skipped[warn] = log
            code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', plain], parse)
        if code:
            raise InstallError('lift.failed', code=code, log=str(log))
        # The mods' data writes must stay off the bytes the lifted code reads as constants (its jump tables), which only
        # the lift knows: gen.py lists them in folded.txt, and a file that writes them becomes unsupported
        if exefs_data(ctx).is_file():
            code, log = run_tool(ctx, 'lift', layout.tools / 'exefs_patches.py',
                                 ['--exefs', ctx.dest / 'exefs', '--status', exefs_status(ctx), '--data',
                                  exefs_data(ctx), '--folded', gen / 'main' / 'folded.txt'])
            if code:
                drop([exefs_status(ctx), exefs_data(ctx)])
                skipped.setdefault('warn.exefs_skipped', log)
        for warn, log in skipped.items():
            rep.warning(warn, log=str(log))
        (gen / STAMP).write_text(ctx.keys['lift'] + '\n')
        return {}


# ---- 5. compile

NINJA_LINE = re.compile(r'^\[(\d+)/(\d+)\] ')


def runtime_key(ctx):
    """Fingerprint of what totk.exe is built from besides the lifted code: the runtime, the toolchain and the build
    tools. Shader preparation depends on these, not on the code cheats a lift adds."""
    layout, cpu = ctx.layout, getattr(ctx, 'cpu', 'v3')
    clang = subprocess.run([str(layout.clang), '--version'], capture_output=True, text=True).stdout
    tools = [layout.tools / n for n in ('build_game.py', 'romfs_index.py', 'check_content.py', 'check_isa.py')]
    return fingerprint('runtime-1', cpu, layout.runtime_lib_for(cpu), layout.runtime, layout.deps or 'system',
                       layout.sysroot or 'host', *tools, clang, *[d.name for d in layout.dlls])


class Compile(Step):
    id = 'compile'

    def key(self, ctx):
        return fingerprint('compile-3', ctx.keys['lift'], runtime_key(ctx),
                           ctx.layout.pgo(game_version(ctx)) or 'no profile')

    def current(self, ctx, record):
        exe = ctx.dest / ('totk' + EXE)
        if not exe.is_file() or not (ctx.dest / 'romfs-index.bin').is_file():
            return False
        if not all((ctx.dest / d.name).is_file() for d in ctx.layout.dlls):
            return False
        return file_sha256(exe) == record.get('exe_sha256')

    def run(self, ctx):
        layout, rep, dest = ctx.layout, ctx.reporter, ctx.dest
        build = ctx.temp / 'build'
        phase = ['compile']

        def parse(line):
            m = NINJA_LINE.match(line)
            if m:
                rep.progress('compile', phase[0], int(m.group(1)), int(m.group(2)), 'objects')
            elif line.startswith('compile: '):
                phase[0] = 'link'
                rep.progress('compile', 'link', force=True)

        rep.progress('compile', 'compile', force=True)
        cpu = getattr(ctx, 'cpu', 'v3')
        lib = layout.runtime_lib_for(cpu)
        pgo = layout.pgo(game_version(ctx))
        code, log = run_tool(ctx, 'compile', layout.tools / 'build_game.py', [
            '--gen', ctx.temp / 'gen', '--out', build, '--toolchain', layout.llvm, '--cpu', cpu,
            *(['--deps', layout.deps] if layout.deps else []), *(['--sysroot', layout.sysroot] if layout.sysroot else []),
            '--runtime', layout.runtime, *(['--runtime-lib', lib] if lib else ()), *(['--pgo', pgo] if pgo else ()),
            '--ninja', layout.ninja,
            '--version', version_tag(game_version(ctx)), '--expected', layout.expected(game_version(ctx)),
            '--exefs', dest / 'exefs', '--romfs', dest / 'romfs',
            '--profile', dest / 'profile'], parse)
        exe = build / ('totk' + EXE)
        if code or not exe.is_file():
            raise InstallError('compile.failed', code=code, log=str(log))
        try:
            replace_file(exe, dest / exe.name)
        except PermissionError:
            raise InstallError('compile.exe_in_use', path=str(dest / exe.name)) from None
        replace_file(build / 'romfs-index.bin', dest / 'romfs-index.bin')
        for dll in layout.dlls:
            replace_file(dll, dest / dll.name)
        return {'exe_sha256': file_sha256(dest / exe.name), 'cpu': cpu}


# ---- 6. prepare

TRANSLATED = re.compile(r'gfx_shader: prepare: (\d+) of (\d+) pairs')
# runtime/vk_pipeline.c lb_warm_prepared: variants the driver crashed on in an earlier run (tools/prepare_shaders.py
# runs the driver step again after such a crash)
SKIPPED = re.compile(r'prepare-shaders: (\d+) variants skipped: the graphics driver crashed')


# The game resolutions of another shape than 16:9 that runtime/ui/menu.cpp row_output_res lists: 21:9 and 32:9 monitors,
# 16:10 screens (the Steam Deck's 1280x800, handhelds and laptops) and 3:2 screens (Surface and Framework laptops and
# tablets)
ASPECT_SIZES = ('2560x1080', '3440x1440', '3840x1600', '5120x2160', '3840x1080', '5120x1440',
                '1280x800', '1680x1050', '1920x1200', '2560x1600',
                '1920x1280', '2160x1440', '2256x1504', '2880x1920', '3000x2000')


class Prepare(Step):
    id = 'prepare'

    def key(self, ctx):
        layout = ctx.layout
        xlate_id = layout.xlate.with_suffix('.id')
        tools = [layout.tools / n for n in ('prepare_shaders.py', 'shader_census.py', 'make_romfs_patches.py',
                                            'pack_shader_cache.py', 'ultrawide_layout.py', 'button_icons.py')]
        # not the compile key: a rebuild for the mods' code cheats keeps the prepared shaders
        return fingerprint('prepare-2', ctx.keys['extract'], runtime_key(ctx), layout.xlate, xlate_id,
                           shader_list_text(layout), *tools)

    def current(self, ctx, record):
        dest = ctx.dest
        xlate = dest / ctx.layout.xlate.name
        return ((dest / 'shader-prepared.bin').is_file() and (dest / 'romfs-patches' / 'manifest.tsv').is_file()
                and xlate.is_file() and file_sha256(xlate) == record.get('xlate_sha256'))

    def run(self, ctx):
        layout, rep, dest = ctx.layout, ctx.reporter, ctx.dest
        xlate = dest / layout.xlate.name
        if xlate.is_file() and file_sha256(xlate) != file_sha256(layout.xlate):
            # Another translator: its cache keys differ, so the old translations would only take space.
            for name in ('shader-cache', 'shader-prepared.bin'):
                remove(dest / name)
        # totk.exe starts totk_xlate.exe as a child while it prepares shaders; a preparation that was stopped from
        # outside leaves that child holding this file, and Windows then refuses to replace it (issue #174).
        try:
            replace_file(layout.xlate, xlate)
        except PermissionError:
            raise InstallError('prepare.xlate_in_use', path=str(xlate)) from None
        xlate_id = layout.xlate.with_suffix('.id')
        if xlate_id.is_file():
            try:
                replace_file(xlate_id, xlate.with_suffix('.id'))
            except PermissionError:
                raise InstallError('prepare.xlate_in_use', path=str(xlate.with_suffix('.id'))) from None
        else:
            remove(xlate.with_suffix('.id'))

        rep.progress('prepare', 'romfs_patches', force=True)
        remove(dest / 'romfs-patches')
        code, log = run_tool(ctx, 'prepare', layout.tools / 'make_romfs_patches.py',
                             ['--version', version_tag(game_version(ctx)), '--romfs', dest / 'romfs', '--out',
                              dest / 'romfs-patches'])
        if code:
            raise InstallError('prepare.romfs_patches_failed', code=code, log=str(log))
        # The same overlay with the UI laid out for each game resolution of another shape the settings menu offers
        # (runtime/romfs.c overlay_init picks the one that matches; the camera's side is the patch set aspect,
        # recomp/patches-v<ver>.txt).
        for size in ASPECT_SIZES:
            remove(dest / f'romfs-patches-{size}')
            code, log = run_tool(ctx, 'prepare', layout.tools / 'make_romfs_patches.py',
                                 ['--version', version_tag(game_version(ctx)), '--romfs', dest / 'romfs', '--out',
                                  dest / f'romfs-patches-{size}', '--aspect', size])
            if code:
                raise InstallError('prepare.romfs_patches_failed', code=code, log=str(log))
        # The same with Xbox button icons in the prompts, for prompts = xbox (make_romfs_patches.py --prompts):
        # romfs-patches-xbox for the Nintendo pad layout, romfs-patches-xbox-label for xbox and auto (runtime/input_map.c
        # input_prompts_folder). Both are written on every install, so the menu choices work without another run of
        # the setup. Optional: when one fails, the game keeps the Switch prompts with that layout and the menu says the
        # files are missing, so the setup warns once and goes on. romfs-patches-pc likewise, with keyboard and mouse
        # prompts for prompts = pc.
        failed = None
        for style in ('xbox', 'xbox-label', 'pc'):
            out = dest / f'romfs-patches-{style}'
            remove(out)
            code, log = run_tool(ctx, f'prepare-{style}', layout.tools / 'make_romfs_patches.py',
                                 ['--version', version_tag(game_version(ctx)), '--romfs', dest / 'romfs', '--out', out,
                                  '--prompts', style])
            if code:
                remove(out)
                failed = failed or (code, log)
        # Each prompts' folder once more for each shape of ASPECT_SIZES, romfs-patches-<style>-<w>x<h> with that
        # shape's UI layout (--prompts <style> --aspect), which runtime/romfs.c overlay_init loads in place of
        # romfs-patches-<w>x<h>: without it a player with Xbox or PC prompts on a screen that is not 16:9 chose between
        # the prompts and the screen's shape. Only the Common archive differs from romfs-patches-<style>; --reuse links
        # the fonts, messages and size table from there instead of compressing them again, so each folder takes
        # seconds and a few MiB. Optional like romfs-patches-<style>, and skipped without it.
        for style in ('xbox', 'xbox-label', 'pc'):
            plain = dest / f'romfs-patches-{style}'
            for size in ASPECT_SIZES:
                out = dest / f'romfs-patches-{style}-{size}'
                remove(out)
                if not (plain / 'manifest.tsv').is_file():
                    continue
                code, log = run_tool(ctx, f'prepare-{style}-{size}', layout.tools / 'make_romfs_patches.py',
                                     ['--version', version_tag(game_version(ctx)), '--romfs', dest / 'romfs', '--out',
                                      out, '--prompts', style, '--aspect', size, '--reuse', plain])
                if code:
                    remove(out)
                    failed = failed or (code, log)
        if failed:
            rep.warning('warn.xbox_prompts_failed', code=failed[0], log=str(failed[1]))

        shader_list = layout.shader_list
        if not shader_list:
            shader_list = ctx.temp / 'shader_list.tsv'
            shader_list.write_text(shader_list_text(layout), encoding='utf-8', newline='\n')

        skipped = [0]

        def parse(line):
            m = TRANSLATED.search(line)
            if m:
                rep.progress('prepare', 'translate', int(m.group(1)), int(m.group(2)), 'shaders')
            elif SKIPPED.search(line):
                skipped[0] = int(SKIPPED.search(line).group(1))
            elif 'listed variants' in line:
                rep.progress('prepare', 'translate', force=True)
            elif 'gfx_shader: prepare:' in line and 'translated' in line and 'programs' in line:
                rep.progress('prepare', 'libraries', force=True)

        rep.progress('prepare', 'seed', force=True)
        code, log = run_tool(ctx, 'prepare', layout.tools / 'prepare_shaders.py',
                             [dest, '--version', version_tag(game_version(ctx)), '--romfs', dest / 'romfs', '--exefs',
                              dest / 'exefs', '--list', shader_list], parse)
        if code or not (dest / 'shader-prepared.bin').is_file():
            raise InstallError('prepare.shaders_failed', code=code, log=str(log))
        if skipped[0]:
            rep.warning('warn.shaders_skipped', count=skipped[0], log=str(log))

        # One shader-cache.pack instead of four files per shader (~117,000): totk.exe reads the pack first.
        rep.progress('prepare', 'pack', force=True)
        code, log = run_tool(ctx, 'prepare', layout.tools / 'pack_shader_cache.py', [dest / 'shader-cache'])
        if code:
            raise InstallError('prepare.pack_failed', code=code, log=str(log))
        return {'xlate_sha256': file_sha256(xlate)}


# ---- 7. NVIDIA DLSS

# NVIDIA's DLSS library, which NGX loads from the folder of totk.exe (runtime/vk_dlaa.c). The setup does not ship it:
# with --dlss it downloads NVIDIA's own copy from the DLSS SDK repository (NVIDIA RTX SDK license) at a pinned tag and
# checks its hash. The game runs without it; dlaa and the DLSS modes then do nothing.
DLSS_DLL = 'nvngx_dlss.dll'
DLSS_VERSION = '310.9.1'
DLSS_URL = f'https://raw.githubusercontent.com/NVIDIA/DLSS/v{DLSS_VERSION}/lib/Windows_x86_64/rel/{DLSS_DLL}'
DLSS_SIZE = 58956912
DLSS_SHA256 = '3975567b8943c53acce397f2b72380092f84f162d00b0d2c7d08a1025c563983'
DLSS_LICENSE = 'https://github.com/NVIDIA/DLSS/blob/main/LICENSE.txt'


def nvidia_rtx(host):
    """An NVIDIA RTX GPU among the host check's Vulkan devices: the GPUs DLSS runs on."""
    return any('NVIDIA' in d['name'].upper() and 'RTX' in d['name'].upper() for d in (host or {}).get('devices', []))


class Dlss(Step):
    """With --dlss on Windows and an NVIDIA RTX GPU: nvngx_dlss.dll next to totk.exe. A DLSS library the player put
    there (another version) is kept. A failed download warns and is tried again on the next run; the game runs
    without it."""

    id = 'dlss'

    def key(self, ctx):
        if not getattr(ctx.options, 'dlss', False):
            return fingerprint('dlss-1', 'off')
        return fingerprint('dlss-1', DLSS_VERSION, DLSS_SHA256)

    def current(self, ctx, record):
        if record.get('failed'):
            return False
        return not record.get('installed') or (ctx.dest / DLSS_DLL).is_file()

    def run(self, ctx):
        if not getattr(ctx.options, 'dlss', False):
            return {}
        if os.name != 'nt':
            return {'skipped': 'not_windows'}
        if not nvidia_rtx(getattr(ctx, 'host', None)):
            # The box is ticked by default, so most players without an RTX GPU never chose it: a log line, no warning
            ctx.reporter.log('dlss: no NVIDIA RTX GPU among the Vulkan devices, nothing downloaded')
            return {'skipped': 'no_rtx'}
        rep, target = ctx.reporter, ctx.dest / DLSS_DLL
        if target.is_file():
            if file_sha256(target) == DLSS_SHA256:
                return {'installed': DLSS_VERSION}
            rep.warning('warn.dlss_kept', path=str(target))
            return {'kept': True}
        part = ctx.temp / (DLSS_DLL + '.part')
        rep.progress('dlss', 'download', 0, DLSS_SIZE, 'bytes', force=True)
        rep.log(f'dlss: {DLSS_URL} (license {DLSS_LICENSE})')
        try:
            part.parent.mkdir(parents=True, exist_ok=True)
            h, done = hashlib.sha256(), 0
            with urllib.request.urlopen(DLSS_URL, timeout=60) as r, open(part, 'wb') as f:
                while block := r.read(1 << 20):
                    f.write(block)
                    h.update(block)
                    done += len(block)
                    rep.progress('dlss', 'download', min(done, DLSS_SIZE), DLSS_SIZE, 'bytes')
            if done != DLSS_SIZE or h.hexdigest() != DLSS_SHA256:
                raise OSError(f'{done} bytes, SHA-256 {h.hexdigest()[:16]}, not the expected file')
            replace_file(part, target)
            remove(part)
        # urllib's errors, timeouts and a bad file; a connection cut mid-file raises http.client.IncompleteRead,
        # which is not an OSError
        except (OSError, http.client.HTTPException) as e:
            remove(part)
            rep.warning('warn.dlss_download_failed', url=DLSS_URL, detail=str(e)[:300])
            return {'failed': True}
        return {'installed': DLSS_VERSION}


# ---- 8. clean up

class Cleanup(Step):
    id = 'cleanup'
    always = True

    def run(self, ctx):
        rep, dest = ctx.reporter, ctx.dest
        rep.progress('cleanup', 'ini', force=True)
        try:
            ini.write(dest, ctx.lang, ctx.layout.ini_example)
        except OSError as e:
            raise InstallError('cleanup.ini_failed', detail=str(e)) from None
        ctx.state['lang'] = ctx.lang
        (dest / 'profile').mkdir(exist_ok=True)
        rep.progress('cleanup', 'temp', force=True)
        for stale in dest.glob('update-merge-*'):
            remove(stale)
        for name in ('check', 'unpacked', 'shader_list.tsv', 'shared-fonts'):   # gen/ and build/ stay for the next rebuild
            remove(ctx.temp / name)
        romfs_problem(dest / 'romfs', expected_update(ctx))
        write_rebuild_command(ctx)
        return {}


# Puts the release folder (the argument after -c) on sys.path and runs installer.gui with the rest. `-m installer.gui`
# finds the package only through the release Python's ._pth: the Linux release's Python and the dev-mode embeddable one
# lack that entry, and totk.exe starts the command in the install folder.
REBUILD_BOOT = ('import runpy, sys; sys.path.insert(0, sys.argv.pop(1)); '
                "runpy.run_module('installer.gui', run_name='__main__', alter_sys=True)")


def rebuild_command(ctx):
    """The command that starts the setup window in its rebuild mode on this install folder (installer/gui.py
    --rebuild): pythonw beside the setup's Python on Windows, so no console opens. In a repository that is the Python
    running this setup: the tools' embeddable Python there has no tkinter."""
    layout = ctx.layout
    python = Path(sys.executable) if layout.dev else layout.python
    if os.name == 'nt' and python.with_name('pythonw.exe').is_file():
        python = python.with_name('pythonw.exe')
    return [str(python), '-c', REBUILD_BOOT, str(layout.root), '--root', str(layout.root), '--rebuild', str(ctx.dest)]


def write_rebuild_command(ctx):
    """build-cache/rebuild.txt in the install folder: rebuild_command, one argument per line, for totk.exe."""
    cache = ctx.dest / 'build-cache'
    cache.mkdir(exist_ok=True)
    tmp = cache / 'rebuild.txt.tmp'
    tmp.write_text('\n'.join(rebuild_command(ctx)) + '\n', encoding='utf-8', newline='\n')
    os.replace(tmp, cache / 'rebuild.txt')


def romfs_problem(romfs, expected):
    """Raise cleanup.romfs_unreadable unless romfs/ still holds the verified image. The game opens romfs.bin at start;
    an install whose romfs/ went empty and unreadable after the extraction verified it (a folder Windows was still
    deleting, #107) reported success and the game stopped at its first file read."""
    try:
        ok = extract.romfs_verified(romfs, expected) and (romfs / 'romfs.bin').stat().st_size == expected['romfs']['size']
        if ok:
            with open(romfs / 'romfs.bin', 'rb') as f:
                f.read(1)
    except (OSError, ValueError):
        ok = False
    if not ok:
        raise InstallError('cleanup.romfs_unreadable', path=str(romfs))


def all_steps():
    return [Check(), Fonts(), Extract(), Lift(), Compile(), Prepare(), Dlss(), Cleanup()]