diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index bec4fc7d..b5fc4fcf 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -50,7 +50,7 @@ These tools run on the developer's or player's machine during the build. They ar
| Vulkan-Headers and Vulkan-Loader 1.4.363.0 (MSYS2 packages; headers and `libvulkan-1.dll.a` only, no loader DLL) | https://github.com/KhronosGroup | Apache-2.0 OR MIT | `runtime/deps/` |
| Vulkan Memory Allocator 3.4.0 (MSYS2 package), header | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator | MIT | `runtime/deps/` |
| Ninja 1.13.2 (`ninja-win.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja.exe` |
-| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check), built with MSYS2 UCRT64 clang; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` |
+| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check; our `unicode-patches.diff`: an embedded manifest that makes UTF-8 the process code page), built with MSYS2 UCRT64 gcc; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` |
| `totk_xlate.exe` (Ryujinx shader translator, NativeAOT build of `tools/xlate/`, see the Ryujinx row above) | built by `tools/build_xlate.py` | MIT | `bin/totk_xlate.exe` |
| MacNativeGames shader variant list | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/assets/shader_list.tsv`), Copyright (c) 2026 MacNativeGames contributors | MIT | `data/shader_list.tsv` |
diff --git a/docs/installer.md b/docs/installer.md
index c0738dea..8f895bb5 100644
--- a/docs/installer.md
+++ b/docs/installer.md
@@ -104,6 +104,13 @@ totk.ini.example, VERSION the default ini; the release string (git describe of
- The keys file path is passed to the tools; key values never appear in the UI, logs or files.
- The window is a front end over the CLI (`python -m installer --base ... --update ... --keys ... --lang ... --dest
...`), which prints one JSON progress event per line. Tests and CI use the CLI.
+- Paths may hold any Unicode characters (a Cyrillic user name puts the temp and install folders under it). Every exe
+ the setup runs or builds reads its arguments and paths as UTF-8 through an `activeCodePage` manifest: hactool
+ (`toolchain/hactool/unicode-patches.diff`), totk.exe (`runtime/totk.rc`), ninja (upstream). The Python steps write
+ their files and logs as UTF-8, and `build_game.py` quotes paths with spaces on its Ninja command lines.
+- A game file whose data fails its hash checks or ends early stops the setup with `check.input_damaged`, which names
+ the file and asks to copy it again (`extract.DamagedFile`). A failed merge checks the base Program NCA alone to tell
+ which file is damaged.
- Unsigned exe: SmartScreen warns on first start.
## Later
diff --git a/docs/toolchain-msys2-python.md b/docs/toolchain-msys2-python.md
index fd9b5cc7..3b0b0a82 100644
--- a/docs/toolchain-msys2-python.md
+++ b/docs/toolchain-msys2-python.md
@@ -95,6 +95,15 @@ Applied the portable part of MacNativeGames `build_app.py:build_hactool()` (diff
2. BKTR: allow 0x200-aligned padding after the subsection table (`nca.c`).
3. BKTR: accept the subsection total-size layout of newer cumulative updates (`nca.c`).
+Ours (`toolchain\hactool\unicode-patches.diff`): `hactool.manifest` with `UTF-8`,
+embedded through `hactool.rc` (Makefile rule `hactool.res.o`, `windres`). hactool converts argv, `HOME` and
+`USERPROFILE` from UTF-8 to UTF-16 (`filepath.c`); without the manifest they arrive in the ANSI code page, and a
+non-ASCII user profile or path stops it with `Failed to convert ... to UTF-16!`. Needs Windows 10 1903 or later.
+
+Rebuild: in an MSYS2 UCRT64 shell (`MSYSTEM=UCRT64 C:\msys64\usr\bin\bash.exe -lc ...`) in `toolchain\src\hactool`,
+`make clean && make -j8`, then copy `hactool.exe` to `toolchain\hactool\`. gcc writes temporary files to `TMP`; where
+it cannot (sandboxed shells), point `TMP` and `TEMP` at a writable folder.
+
Not applied: the `MNG_BASE_RANGE_OFFSET/SIZE` base-file hook. It uses `funopen` (BSD/macOS only); on Windows the
base NCA must be extracted to a file first.
diff --git a/installer/steps.py b/installer/steps.py
index adfa871a..6ffedce0 100644
--- a/installer/steps.py
+++ b/installer/steps.py
@@ -131,6 +131,13 @@ def free_bytes(path):
return shutil.disk_usage(p).free, os.stat(p).st_dev
+def raise_if_damaged(error, name, path):
+ """InstallError check.input_damaged when a reader's ValueError says the file's data fails its hashes or ends early
+ (extract.DAMAGE_TEXT)."""
+ if extract.DAMAGE_TEXT.search(str(error)):
+ raise InstallError('check.input_damaged', input=name, path=str(path), detail=str(error)) from None
+
+
def expected_update():
return extract.load_expected()[UPDATE_VERSION]
@@ -264,15 +271,16 @@ class Check(Step):
for name, is_update in (('base', False), ('update', True)):
path = Path(getattr(o, name))
with path.open('rb') as f:
- total += self.open_input(f, path.stat().st_size, keys, name, is_update)
+ total += self.open_input(f, path, keys, name, is_update)
del keys
return total
@staticmethod
- def open_input(f, size, keys, name, is_update):
+ def open_input(f, path, keys, name, is_update):
try:
- kind, entries = container_entries(f, size)
- except ValueError:
+ kind, entries = container_entries(f, path.stat().st_size)
+ except ValueError as e:
+ raise_if_damaged(e, name, path)
raise InstallError('check.not_game_file', input=name) from None
if is_update and kind != 'nsp':
raise InstallError('check.update_not_nsp')
@@ -281,12 +289,14 @@ class Check(Step):
except ValueError as e:
if 'Personalized' in str(e):
raise InstallError('check.ticket_personalized', input=name) from None
+ raise_if_damaged(e, name, path)
raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
try:
records = nca_inventory(f, entries, keys)
except ValueError as e:
if 'NCA3' in str(e):
raise InstallError('check.keys_wrong', input=name) from None
+ raise_if_damaged(e, name, path)
raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
try:
record, header = program_nca(records, update=is_update)
@@ -339,6 +349,8 @@ class Check(Step):
remove(out)
try:
extract.extract_exefs(Path(getattr(o, name)), Path(o.keys), out, logs, update=is_update)
+ except extract.DamagedFile as e:
+ raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None
except ValueError:
raise InstallError('check.exefs_failed', input=name, log=str(logs / f'extract_exefs-{name}.log')) \
from None
@@ -401,6 +413,8 @@ class Extract(Step):
remove(ctx.temp / 'check' / 'update')
try:
extract.extract_exefs(update, keys, ctx.temp / 'check' / 'update', ctx.temp / 'logs', update=True)
+ except extract.DamagedFile as e:
+ raise InstallError('check.input_damaged', input='update', path=str(e.path), detail=e.detail) from None
except ValueError:
raise InstallError('check.exefs_failed', input='update',
log=str(ctx.temp / 'logs' / 'extract_exefs-update.log')) from None
@@ -441,6 +455,9 @@ class Extract(Step):
watch.start()
try:
extract.merge_update_romfs(base, update, keys, ctx.layout.hactool, romfs, logs, phase=phase)
+ except extract.DamagedFile as e:
+ raise InstallError('check.input_damaged', input='base' if e.path == base else 'update', path=str(e.path),
+ detail=e.detail) from None
except ValueError as e:
raise InstallError('extract.romfs_merge_failed', detail=str(e), log=str(logs / 'merge_romfs-'
f'{dest.name}.log')) from None
diff --git a/installer/text/en.json b/installer/text/en.json
index bdf5b0fd..7b9abf49 100644
--- a/installer/text/en.json
+++ b/installer/text/en.json
@@ -16,6 +16,7 @@
"phase.extract.exefs": "Extracting the game code",
"phase.extract.copy": "Copying the game data",
"phase.extract.merge": "Applying the update to the game data",
+ "phase.extract.diagnose": "Finding which game file is damaged",
"phase.extract.index": "Verifying the game data",
"phase.lift.discover": "Finding the functions",
"phase.lift.emit": "Writing C code",
@@ -60,6 +61,7 @@
"check.update_is_base": "The update file holds a base game. Choose the 1.2.1 update NSP.",
"check.update_not_totk": "The update file is not a Tears of the Kingdom update ({detail}).",
"check.exefs_failed": "The game code of the {input} file failed its checks; see {log}.",
+ "check.input_damaged": "The {input} file is damaged or incomplete: {path} ({detail}). Copy that file again from its source, or dump it again, then run the setup again.",
"check.base_wrong_version": "The base game is not version 1.0.0 ({modules} differ).",
"check.update_wrong_version": "The update is version {found}; this release needs {wanted}.",
"check.disk_space": "Not enough free space for {path}: {need_gb} GB needed, {free_gb} GB free.",
diff --git a/installer/text/ru.json b/installer/text/ru.json
index f045ea6d..d3c3afb6 100644
--- a/installer/text/ru.json
+++ b/installer/text/ru.json
@@ -16,6 +16,7 @@
"phase.extract.exefs": "Распаковка кода игры",
"phase.extract.copy": "Копирование данных игры",
"phase.extract.merge": "Применение обновления к данным игры",
+ "phase.extract.diagnose": "Поиск повреждённого файла игры",
"phase.extract.index": "Проверка данных игры",
"phase.lift.discover": "Поиск функций",
"phase.lift.emit": "Запись кода на C",
@@ -60,6 +61,7 @@
"check.update_is_base": "Файл обновления содержит базовую игру. Выберите NSP с обновлением 1.2.1.",
"check.update_not_totk": "Файл обновления — не обновление Tears of the Kingdom ({detail}).",
"check.exefs_failed": "Код игры из файла «{input}» не прошёл проверку; подробности в {log}.",
+ "check.input_damaged": "Файл «{input}» повреждён или скопирован не полностью: {path} ({detail}). Скопируйте этот файл заново из источника или сделайте дамп ещё раз, затем снова запустите установку.",
"check.base_wrong_version": "Базовая игра не версии 1.0.0 (отличаются модули: {modules}).",
"check.update_wrong_version": "Версия обновления {found}; этому выпуску нужна {wanted}.",
"check.disk_space": "Не хватает свободного места для {path}: нужно {need_gb} ГБ, свободно {free_gb} ГБ.",
diff --git a/runtime/totk.manifest b/runtime/totk.manifest
new file mode 100644
index 00000000..8fe37fa1
--- /dev/null
+++ b/runtime/totk.manifest
@@ -0,0 +1,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ UTF-8
+
+
+
diff --git a/runtime/totk.rc b/runtime/totk.rc
new file mode 100644
index 00000000..b26fe565
--- /dev/null
+++ b/runtime/totk.rc
@@ -0,0 +1,3 @@
+/* The application manifest of totk.exe (totk.manifest). tools/build_game.py compiles it with the runtime objects, so it
+ is also a member of libtotkrt.a, which the release links with --whole-archive. */
+1 24 "totk.manifest"
diff --git a/tools/build_game.py b/tools/build_game.py
index baabc895..3f3aebd4 100644
--- a/tools/build_game.py
+++ b/tools/build_game.py
@@ -76,7 +76,7 @@ SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll']
GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden',
'-march=x86-64-v3']
RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3']
-LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Wl,--Map=$builddir/totk.map'] # the map feeds tools/prof_report.py
+LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '"-Wl,--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end of M4; libvpx static (movies)
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
@@ -86,6 +86,7 @@ DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw'
INSTALL_SIDE = {'romfs.c'} # runtime sources that include romfs_config.h: compiled per install, not in the library
LIB_NAME = 'libtotkrt.a'
AR = UCRT / 'llvm-ar.exe'
+WINDRES = UCRT / 'windres.exe' # runtime/totk.rc: the manifest of totk.exe
TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain
DEPS = None # the deps directory of an llvm-mingw build
# tools/data/expected-.json for each build version name.
@@ -117,6 +118,17 @@ def flag_path(p):
return below_root(p).as_posix()
+def cmd_arg(s):
+ """s for a Ninja command line: Ninja-escaped, and quoted when it holds a space (a user or folder name), as clang and
+ lld split their Windows command line at unquoted spaces."""
+ s = s.replace('$', '$$')
+ return f'"{s}"' if ' ' in s else s
+
+
+def cmd_path(p):
+ return cmd_arg(flag_path(p))
+
+
def read_layout(gen):
"""[(name, base, size)] from layout.json (same format as recomp/layout.py read(), without its NSO dependency)."""
data = json.loads((gen / 'layout.json').read_text())
@@ -146,10 +158,10 @@ def chunk_files(gen, mod):
def write_if_changed(path, text):
"""Keep the mtime of an unchanged file so Ninja does not rebuild from it."""
- if path.exists() and path.read_text() == text:
+ if path.exists() and path.read_text(encoding='utf-8', errors='replace') == text:
return
path.parent.mkdir(parents=True, exist_ok=True)
- path.write_text(text, newline='\n')
+ path.write_text(text, encoding='utf-8', newline='\n') # Ninja and clang read UTF-8; paths may be non-ASCII
def c_string(s):
@@ -160,9 +172,9 @@ RUNTIME = ROOT / 'runtime' # --runtime: a copy of runtime/ lets a task build w
def runtime_sources(skip=()):
- """Every runtime/*.c and runtime/platform/*.c except the names in skip, so a task adds a file without editing this
- script."""
- pats = ('*.c', '*.cpp', 'platform/*.c')
+ """Every runtime/*.c, runtime/platform/*.c and runtime/*.rc except the names in skip, so a task adds a file without
+ editing this script."""
+ pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc')
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip)
@@ -207,15 +219,15 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
"""build.ninja text, and the object lists per module. With prebuilt the generated objects already exist in
out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects
except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE."""
- inc = f'-I {flag_path(RUNTIME)} -I {flag_path(gen)}'
- vma = f'{flag_path(DEPS / "include")} -I {flag_path(DEPS / "include" / "vma")}' if DEPS else flag_path(VMA_INCLUDE)
- rt_inc = f'{inc} -I {flag_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers)
- libs = [f'-L{flag_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS
+ inc = f'-I {cmd_path(RUNTIME)} -I {cmd_path(gen)}'
+ vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(VMA_INCLUDE)
+ rt_inc = f'{inc} -I {cmd_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers)
+ libs = [f'-L {cmd_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS
lines = [
'# Written by tools/build_game.py; do not edit.',
'ninja_required_version = 1.10',
f'builddir = {ninja_path(out)}',
- f'cc = {CLANG.as_posix()}',
+ f'cc = {cmd_arg(CLANG.as_posix())}',
f'genflags = {" ".join(GEN_FLAGS)} {inc}',
f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}',
f'ldflags = {" ".join(LINK_FLAGS)}',
@@ -236,6 +248,10 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
' depfile = $out.d',
' description = CC $in',
'',
+ 'rule rc',
+ f' command = {cmd_arg(WINDRES.as_posix())} -I {cmd_path(RUNTIME)} -O coff $in -o $out',
+ ' description = RC $in',
+ '',
'rule link',
' command = $cc $ldflags @$out.rsp $libs -o $out',
' rspfile = $out.rsp',
@@ -262,14 +278,14 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
sources += [out / 'modules.c']
for src in sources:
o = runtime_object(out, src)
- lines.append(f'build {ninja_path(o)}: cc_rt {ninja_path(src)}')
+ lines.append(f'build {ninja_path(o)}: {"rc" if src.suffix == ".rc" else "cc_rt"} {ninja_path(src)}')
rt.append(o)
objs['runtime'] = rt
every = [o for v in objs.values() for o in v]
exe = out / 'totk.exe'
if make_lib:
exe = out / LIB_NAME
- lines += ['', 'rule ar', f' command = {AR.as_posix()} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
+ lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
return '\n'.join(lines), objs, exe
rule = 'link'
@@ -278,7 +294,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
# linker looks into the library, and the real probes would stay out of totk.exe.
rule = 'link_lib'
lines += ['', 'rule link_lib',
- f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {flag_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out',
+ f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {cmd_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out',
' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out']
lines += ['', f'build {ninja_path(exe)}: {rule} ' + ' '.join(ninja_path(o) for o in every) +
(f' | {ninja_path(runtime_lib)}' if runtime_lib else ''),
@@ -295,7 +311,7 @@ def main_objects(gen, main_out, rt_dir, mods):
return None, f'{flag_path(main_ninja)} not found'
if gen != (ROOT / 'generated' / main_out.name).resolve():
return None, 'generated C directory is not the main one'
- lines = main_ninja.read_text().splitlines()
+ lines = main_ninja.read_text(encoding='utf-8').splitlines()
sources = dict(l[6:].split(': cc_gen ') for l in lines if l.startswith('build ') and ': cc_gen ' in l)
flags = next((l for l in lines if l.startswith('genflags = ')), '')
if not flags.startswith('genflags = ' + ' '.join(GEN_FLAGS) + ' -I '):
@@ -383,7 +399,7 @@ def read_manifest(path):
"""{object path below obj/: (sha256, size, mtime_ns)}; empty when the file is missing."""
man = {}
if path.is_file():
- for line in path.read_text().splitlines():
+ for line in path.read_text(encoding='utf-8').splitlines():
sha, size, mtime, rel = line.split(' ', 3)
man[rel] = (sha, int(size), int(mtime))
return man
@@ -588,7 +604,7 @@ def main():
sys.exit(f'no expected hashes known for {args.version}; pass --expected')
expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json'
hashes = json.loads(Path(expected).read_text())['exefs_sha256']
- global CLANG, AR, NINJA, TOOLCHAIN_BIN, DEPS
+ global CLANG, AR, WINDRES, NINJA, TOOLCHAIN_BIN, DEPS
if args.ninja:
NINJA = args.ninja.resolve()
elif not NINJA.is_file() and (ROOT / 'bin' / 'ninja.exe').is_file():
@@ -598,6 +614,7 @@ def main():
TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve()
CLANG = TOOLCHAIN_BIN / 'clang.exe'
AR = TOOLCHAIN_BIN / 'llvm-ar.exe'
+ WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe'
DEPS = (args.deps or DEFAULT_DEPS).resolve()
dlls = sorted((DEPS / 'bin').glob('*.dll'))
for tool in (CLANG, NINJA):
diff --git a/tools/extract.py b/tools/extract.py
index 318f2ec9..cdc78a08 100644
--- a/tools/extract.py
+++ b/tools/extract.py
@@ -41,6 +41,22 @@ LONG_HEX = re.compile(r'[0-9A-Fa-f]{32,}')
# say nothing about the data. The IVFC levels and the image hash decide.
UNSIGNED = re.compile(r'\s*(?:Fixed-Key Signature|NPDM Signature) \(FAIL\)')
HACTOOL_ERROR = re.compile(r'\(FAIL\)|\(INVALID\)|Failed to|Error:')
+# hactool lines of a hash check that failed: a RomFS level, a section's superblock or its hash table.
+HACTOOL_HASH_FAIL = re.compile(r'^\s*(?:Level \d|Superblock Hash|Hash Table) \(FAIL\)', re.M)
+# Errors of the vendored extractors (tools/vendor/macnativegames) that mean a game file's data does not match its own
+# hashes or ends early: the file is damaged or was not copied completely. DAMAGE finds them in a script's log.
+DAMAGE_TEXT = re.compile(r'hash mismatch|hash-table mismatch|FS header hash|IVFC level \d+ block \d+ failed|Truncated|'
+ r'out of bounds|Section bounds')
+DAMAGE = re.compile(rf'^\w*Error: (.*(?:{DAMAGE_TEXT.pattern}).*)$', re.M)
+
+
+class DamagedFile(ValueError):
+ """A game file whose data fails its hash checks or ends early. path: that file; detail: the check that failed."""
+
+ def __init__(self, path, detail):
+ self.path, self.detail = Path(path), detail
+ super().__init__(f'{self.path} is damaged or incomplete ({detail}). Copy that file again from its source '
+ f'(or dump it again) and retry.')
def say(message):
@@ -75,10 +91,18 @@ RUN_SCRIPT = ('import os, runpy, sys; p = sys.argv[1]; sys.path.insert(0, os.pat
"runpy.run_path(p, run_name='__main__')")
-def run_logged(command, log):
+def run_logged(command, log, source=None):
+ """Run a vendored script with its output in log. source: the game file it reads; a failed hash check or a short
+ read of that file raises DamagedFile."""
args = [str(c) for c in command]
- with log.open('w') as out:
- code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT).returncode
+ env = dict(os.environ, PYTHONIOENCODING='utf-8') # its messages may name non-ASCII paths
+ with log.open('wb') as out:
+ code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT,
+ env=env).returncode
+ if code and source:
+ damage = DAMAGE.search(log.read_text(encoding='utf-8', errors='replace'))
+ if damage:
+ raise DamagedFile(source, damage.group(1).strip())
require(code == 0, f'{Path(command[1]).name} failed; see {shown(log)}')
@@ -98,7 +122,7 @@ def extract_exefs(game, keys, out, logs, *, update):
shutil.rmtree(exefs)
out.mkdir(parents=True, exist_ok=True)
command = [sys.executable, VENDOR / 'extract_exefs.py', game, '--keys', keys, '--output', exefs]
- run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log')
+ run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log', source=game)
return exefs
@@ -131,12 +155,13 @@ def check_romfs(romfs, expected):
say(f' RomFS {expected["display_version"]}: {report["romfs_size"]:,} bytes, {len(report["files"]):,} files, hash matches')
-def extract_base_romfs(out, keys, logs):
+def extract_base_romfs(out, keys, logs, base=None):
romfs = out / 'romfs'
if romfs.exists():
shutil.rmtree(romfs)
run_logged([sys.executable, VENDOR / 'extract_romfs.py', '--keys', keys,
- '--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log')
+ '--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log',
+ source=base)
def program_ncas(base, update, keys_path):
@@ -166,9 +191,33 @@ def merge_space(base, update, keys_path):
MERGE_PHASES = {'copy': ' copying the base and update Program NCAs', 'merge': ' merging with hactool',
+ 'diagnose': ' a hash check failed: checking the base Program NCA alone',
'index': ' hashing and indexing the merged image'}
+def run_hactool(command, environment):
+ """(exit code, stdout and stderr with long hex strings removed). hactool prints UTF-8 (its manifest)."""
+ completed = subprocess.run(command, env=environment, capture_output=True, encoding='utf-8', errors='replace')
+ return completed.returncode, LONG_HEX.sub('', completed.stdout + completed.stderr)
+
+
+def damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path):
+ """The DamagedFile for a merge whose hash checks failed. The merged RomFS mixes data of both files, so hactool
+ verifies the base Program NCA alone: if it fails, the base game file is damaged, else the update file is. The
+ output is appended to log_path."""
+ code, log = run_hactool([str(hactool), '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
+ str(base_nca)], environment)
+ with log_path.open('a', encoding='utf-8') as f:
+ f.write(f'\n---- base Program NCA alone (exit code {code}) ----\n{log}')
+ checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
+ if HACTOOL_HASH_FAIL.search(checked):
+ return DamagedFile(base, f'its game data fails the hash check; see {shown(log_path)}')
+ if all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)):
+ return DamagedFile(update, f'its game data fails the hash check; see {shown(log_path)}')
+ return ValueError(f'the game data fails the hash check: the base game file {base} or the update file {update} is '
+ f'damaged or incomplete. Copy them again from their source and retry; see {shown(log_path)}')
+
+
def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lambda name: say(MERGE_PHASES[name])):
"""extract_update_romfs.merge() without its Darwin-only bounded view of the base: the base Program NCA is copied out.
@@ -195,19 +244,23 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lam
phase('copy')
base_nca, update_nca = tmp / 'base.nca', tmp / 'update.nca'
for (source, record, _), target in zip(inputs, (base_nca, update_nca)):
- copy_range(source, record_offset(record), record['size'], target)
+ try:
+ copy_range(source, record_offset(record), record['size'], target)
+ except ValueError as e: # the file ends inside its Program NCA
+ raise DamagedFile(source, str(e)) from None
result = tmp / 'result'
result.mkdir()
phase('merge')
command = [str(hactool), '-x', '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
f'--basenca={base_nca}', f'--section{sections[0]}={result / "romfs.bin"}', str(update_nca)]
environment = dict(os.environ, MNG_TITLE_KEYS=str(title_keys))
- completed = subprocess.run(command, env=environment, capture_output=True, text=True, errors='replace')
- log = LONG_HEX.sub('', completed.stdout + completed.stderr)
- log_path.write_text(log)
+ code, log = run_hactool(command, environment)
+ log_path.write_text(log, encoding='utf-8')
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
- require(completed.returncode == 0 and not HACTOOL_ERROR.search(checked),
- f'hactool merge failed; see {shown(log_path)}')
+ if HACTOOL_HASH_FAIL.search(checked):
+ phase('diagnose')
+ raise damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path)
+ require(code == 0 and not HACTOOL_ERROR.search(checked), f'hactool merge failed; see {shown(log_path)}')
require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'hactool did not verify all six IVFC levels')
image = result / 'romfs.bin'
phase('index')
@@ -253,7 +306,7 @@ def main():
report_npdm(base_out / 'exefs', v0)
if not romfs_verified(base_out / 'romfs', v0):
say(' extracting RomFS (17 GB)')
- extract_base_romfs(base_out, keys, logs)
+ extract_base_romfs(base_out, keys, logs, base)
check_romfs(base_out / 'romfs', v0)
for update in updates: