Merge the UTF-8 code page manifests, quoted paths and damaged file messages that let the installer and the game run from non-ASCII and spaced paths

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-24 20:36:37 +03:00
commit 0eb0ba99b3
10 files changed
+163 -35

No files matched your search

+1 -1
View File
@@ -50,7 +50,7 @@ These tools run on the developer's or player's machine during the build. They ar
| Vulkan-Headers and Vulkan-Loader 1.4.363.0 (MSYS2 packages; headers and `libvulkan-1.dll.a` only, no loader DLL) | https://github.com/KhronosGroup | Apache-2.0 OR MIT | `runtime/deps/` |
| Vulkan Memory Allocator 3.4.0 (MSYS2 package), header | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator | MIT | `runtime/deps/` |
| Ninja 1.13.2 (`ninja-win.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja.exe` |
| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check), built with MSYS2 UCRT64 clang; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` |
| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check; our `unicode-patches.diff`: an embedded manifest that makes UTF-8 the process code page), built with MSYS2 UCRT64 gcc; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` |
| `totk_xlate.exe` (Ryujinx shader translator, NativeAOT build of `tools/xlate/`, see the Ryujinx row above) | built by `tools/build_xlate.py` | MIT | `bin/totk_xlate.exe` |
| MacNativeGames shader variant list | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/assets/shader_list.tsv`), Copyright (c) 2026 MacNativeGames contributors | MIT | `data/shader_list.tsv` |
+7
View File
@@ -104,6 +104,13 @@ totk.ini.example, VERSION the default ini; the release string (git describe of
- The keys file path is passed to the tools; key values never appear in the UI, logs or files.
- The window is a front end over the CLI (`python -m installer --base ... --update ... --keys ... --lang ... --dest
...`), which prints one JSON progress event per line. Tests and CI use the CLI.
- Paths may hold any Unicode characters (a Cyrillic user name puts the temp and install folders under it). Every exe
the setup runs or builds reads its arguments and paths as UTF-8 through an `activeCodePage` manifest: hactool
(`toolchain/hactool/unicode-patches.diff`), totk.exe (`runtime/totk.rc`), ninja (upstream). The Python steps write
their files and logs as UTF-8, and `build_game.py` quotes paths with spaces on its Ninja command lines.
- A game file whose data fails its hash checks or ends early stops the setup with `check.input_damaged`, which names
the file and asks to copy it again (`extract.DamagedFile`). A failed merge checks the base Program NCA alone to tell
which file is damaged.
- Unsigned exe: SmartScreen warns on first start.
## Later
+9
View File
@@ -95,6 +95,15 @@ Applied the portable part of MacNativeGames `build_app.py:build_hactool()` (diff
2. BKTR: allow 0x200-aligned padding after the subsection table (`nca.c`).
3. BKTR: accept the subsection total-size layout of newer cumulative updates (`nca.c`).
Ours (`toolchain\hactool\unicode-patches.diff`): `hactool.manifest` with `<activeCodePage>UTF-8</activeCodePage>`,
embedded through `hactool.rc` (Makefile rule `hactool.res.o`, `windres`). hactool converts argv, `HOME` and
`USERPROFILE` from UTF-8 to UTF-16 (`filepath.c`); without the manifest they arrive in the ANSI code page, and a
non-ASCII user profile or path stops it with `Failed to convert ... to UTF-16!`. Needs Windows 10 1903 or later.
Rebuild: in an MSYS2 UCRT64 shell (`MSYSTEM=UCRT64 C:\msys64\usr\bin\bash.exe -lc ...`) in `toolchain\src\hactool`,
`make clean && make -j8`, then copy `hactool.exe` to `toolchain\hactool\`. gcc writes temporary files to `TMP`; where
it cannot (sandboxed shells), point `TMP` and `TEMP` at a writable folder.
Not applied: the `MNG_BASE_RANGE_OFFSET/SIZE` base-file hook. It uses `funopen` (BSD/macOS only); on Windows the
base NCA must be extracted to a file first.
+21 -4
View File
@@ -131,6 +131,13 @@ def free_bytes(path):
return shutil.disk_usage(p).free, os.stat(p).st_dev
def raise_if_damaged(error, name, path):
"""InstallError check.input_damaged when a reader's ValueError says the file's data fails its hashes or ends early
(extract.DAMAGE_TEXT)."""
if extract.DAMAGE_TEXT.search(str(error)):
raise InstallError('check.input_damaged', input=name, path=str(path), detail=str(error)) from None
def expected_update():
return extract.load_expected()[UPDATE_VERSION]
@@ -264,15 +271,16 @@ class Check(Step):
for name, is_update in (('base', False), ('update', True)):
path = Path(getattr(o, name))
with path.open('rb') as f:
total += self.open_input(f, path.stat().st_size, keys, name, is_update)
total += self.open_input(f, path, keys, name, is_update)
del keys
return total
@staticmethod
def open_input(f, size, keys, name, is_update):
def open_input(f, path, keys, name, is_update):
try:
kind, entries = container_entries(f, size)
except ValueError:
kind, entries = container_entries(f, path.stat().st_size)
except ValueError as e:
raise_if_damaged(e, name, path)
raise InstallError('check.not_game_file', input=name) from None
if is_update and kind != 'nsp':
raise InstallError('check.update_not_nsp')
@@ -281,12 +289,14 @@ class Check(Step):
except ValueError as e:
if 'Personalized' in str(e):
raise InstallError('check.ticket_personalized', input=name) from None
raise_if_damaged(e, name, path)
raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
try:
records = nca_inventory(f, entries, keys)
except ValueError as e:
if 'NCA3' in str(e):
raise InstallError('check.keys_wrong', input=name) from None
raise_if_damaged(e, name, path)
raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
try:
record, header = program_nca(records, update=is_update)
@@ -339,6 +349,8 @@ class Check(Step):
remove(out)
try:
extract.extract_exefs(Path(getattr(o, name)), Path(o.keys), out, logs, update=is_update)
except extract.DamagedFile as e:
raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None
except ValueError:
raise InstallError('check.exefs_failed', input=name, log=str(logs / f'extract_exefs-{name}.log')) \
from None
@@ -401,6 +413,8 @@ class Extract(Step):
remove(ctx.temp / 'check' / 'update')
try:
extract.extract_exefs(update, keys, ctx.temp / 'check' / 'update', ctx.temp / 'logs', update=True)
except extract.DamagedFile as e:
raise InstallError('check.input_damaged', input='update', path=str(e.path), detail=e.detail) from None
except ValueError:
raise InstallError('check.exefs_failed', input='update',
log=str(ctx.temp / 'logs' / 'extract_exefs-update.log')) from None
@@ -441,6 +455,9 @@ class Extract(Step):
watch.start()
try:
extract.merge_update_romfs(base, update, keys, ctx.layout.hactool, romfs, logs, phase=phase)
except extract.DamagedFile as e:
raise InstallError('check.input_damaged', input='base' if e.path == base else 'update', path=str(e.path),
detail=e.detail) from None
except ValueError as e:
raise InstallError('extract.romfs_merge_failed', detail=str(e), log=str(logs / 'merge_romfs-'
f'{dest.name}.log')) from None
+2
View File
@@ -16,6 +16,7 @@
"phase.extract.exefs": "Extracting the game code",
"phase.extract.copy": "Copying the game data",
"phase.extract.merge": "Applying the update to the game data",
"phase.extract.diagnose": "Finding which game file is damaged",
"phase.extract.index": "Verifying the game data",
"phase.lift.discover": "Finding the functions",
"phase.lift.emit": "Writing C code",
@@ -60,6 +61,7 @@
"check.update_is_base": "The update file holds a base game. Choose the 1.2.1 update NSP.",
"check.update_not_totk": "The update file is not a Tears of the Kingdom update ({detail}).",
"check.exefs_failed": "The game code of the {input} file failed its checks; see {log}.",
"check.input_damaged": "The {input} file is damaged or incomplete: {path} ({detail}). Copy that file again from its source, or dump it again, then run the setup again.",
"check.base_wrong_version": "The base game is not version 1.0.0 ({modules} differ).",
"check.update_wrong_version": "The update is version {found}; this release needs {wanted}.",
"check.disk_space": "Not enough free space for {path}: {need_gb} GB needed, {free_gb} GB free.",
+2
View File
@@ -16,6 +16,7 @@
"phase.extract.exefs": "Распаковка кода игры",
"phase.extract.copy": "Копирование данных игры",
"phase.extract.merge": "Применение обновления к данным игры",
"phase.extract.diagnose": "Поиск повреждённого файла игры",
"phase.extract.index": "Проверка данных игры",
"phase.lift.discover": "Поиск функций",
"phase.lift.emit": "Запись кода на C",
@@ -60,6 +61,7 @@
"check.update_is_base": "Файл обновления содержит базовую игру. Выберите NSP с обновлением 1.2.1.",
"check.update_not_totk": "Файл обновления — не обновление Tears of the Kingdom ({detail}).",
"check.exefs_failed": "Код игры из файла «{input}» не прошёл проверку; подробности в {log}.",
"check.input_damaged": "Файл «{input}» повреждён или скопирован не полностью: {path} ({detail}). Скопируйте этот файл заново из источника или сделайте дамп ещё раз, затем снова запустите установку.",
"check.base_wrong_version": "Базовая игра не версии 1.0.0 (отличаются модули: {modules}).",
"check.update_wrong_version": "Версия обновления {found}; этому выпуску нужна {wanted}.",
"check.disk_space": "Не хватает свободного места для {path}: нужно {need_gb} ГБ, свободно {free_gb} ГБ.",
+18
View File
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!-- Embedded by totk.rc. activeCodePage makes UTF-8 the process code page (Windows 10 1903 and later): the narrow file
functions (fopen, CreateFileA, GetModuleFileNameA) and getenv then carry UTF-8, the same encoding platform/win32.c
and fs_host.c convert with, so an install folder or user profile with a non-ASCII name works. -->
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
<security>
<requestedPrivileges>
<requestedExecutionLevel level="asInvoker"/>
</requestedPrivileges>
</security>
</trustInfo>
<application xmlns="urn:schemas-microsoft-com:asm.v3">
<windowsSettings>
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
</windowsSettings>
</application>
</assembly>
+3
View File
@@ -0,0 +1,3 @@
/* The application manifest of totk.exe (totk.manifest). tools/build_game.py compiles it with the runtime objects, so it
is also a member of libtotkrt.a, which the release links with --whole-archive. */
1 24 "totk.manifest"
+34 -17
View File
@@ -76,7 +76,7 @@ SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll']
GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden',
'-march=x86-64-v3']
RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3']
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Wl,--Map=$builddir/totk.map'] # the map feeds tools/prof_report.py
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '"-Wl,--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end of M4; libvpx static (movies)
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
@@ -86,6 +86,7 @@ DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw'
INSTALL_SIDE = {'romfs.c'} # runtime sources that include romfs_config.h: compiled per install, not in the library
LIB_NAME = 'libtotkrt.a'
AR = UCRT / 'llvm-ar.exe'
WINDRES = UCRT / 'windres.exe' # runtime/totk.rc: the manifest of totk.exe
TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain
DEPS = None # the deps directory of an llvm-mingw build
# tools/data/expected-<display version>.json for each build version name.
@@ -117,6 +118,17 @@ def flag_path(p):
return below_root(p).as_posix()
def cmd_arg(s):
"""s for a Ninja command line: Ninja-escaped, and quoted when it holds a space (a user or folder name), as clang and
lld split their Windows command line at unquoted spaces."""
s = s.replace('$', '$$')
return f'"{s}"' if ' ' in s else s
def cmd_path(p):
return cmd_arg(flag_path(p))
def read_layout(gen):
"""[(name, base, size)] from layout.json (same format as recomp/layout.py read(), without its NSO dependency)."""
data = json.loads((gen / 'layout.json').read_text())
@@ -146,10 +158,10 @@ def chunk_files(gen, mod):
def write_if_changed(path, text):
"""Keep the mtime of an unchanged file so Ninja does not rebuild from it."""
if path.exists() and path.read_text() == text:
if path.exists() and path.read_text(encoding='utf-8', errors='replace') == text:
return
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, newline='\n')
path.write_text(text, encoding='utf-8', newline='\n') # Ninja and clang read UTF-8; paths may be non-ASCII
def c_string(s):
@@ -160,9 +172,9 @@ RUNTIME = ROOT / 'runtime' # --runtime: a copy of runtime/ lets a task build w
def runtime_sources(skip=()):
"""Every runtime/*.c and runtime/platform/*.c except the names in skip, so a task adds a file without editing this
script."""
pats = ('*.c', '*.cpp', 'platform/*.c')
"""Every runtime/*.c, runtime/platform/*.c and runtime/*.rc except the names in skip, so a task adds a file without
editing this script."""
pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc')
return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip)
@@ -207,15 +219,15 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
"""build.ninja text, and the object lists per module. With prebuilt the generated objects already exist in
out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects
except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE."""
inc = f'-I {flag_path(RUNTIME)} -I {flag_path(gen)}'
vma = f'{flag_path(DEPS / "include")} -I {flag_path(DEPS / "include" / "vma")}' if DEPS else flag_path(VMA_INCLUDE)
rt_inc = f'{inc} -I {flag_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers)
libs = [f'-L{flag_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS
inc = f'-I {cmd_path(RUNTIME)} -I {cmd_path(gen)}'
vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(VMA_INCLUDE)
rt_inc = f'{inc} -I {cmd_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers)
libs = [f'-L {cmd_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS
lines = [
'# Written by tools/build_game.py; do not edit.',
'ninja_required_version = 1.10',
f'builddir = {ninja_path(out)}',
f'cc = {CLANG.as_posix()}',
f'cc = {cmd_arg(CLANG.as_posix())}',
f'genflags = {" ".join(GEN_FLAGS)} {inc}',
f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}',
f'ldflags = {" ".join(LINK_FLAGS)}',
@@ -236,6 +248,10 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
' depfile = $out.d',
' description = CC $in',
'',
'rule rc',
f' command = {cmd_arg(WINDRES.as_posix())} -I {cmd_path(RUNTIME)} -O coff $in -o $out',
' description = RC $in',
'',
'rule link',
' command = $cc $ldflags @$out.rsp $libs -o $out',
' rspfile = $out.rsp',
@@ -262,14 +278,14 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
sources += [out / 'modules.c']
for src in sources:
o = runtime_object(out, src)
lines.append(f'build {ninja_path(o)}: cc_rt {ninja_path(src)}')
lines.append(f'build {ninja_path(o)}: {"rc" if src.suffix == ".rc" else "cc_rt"} {ninja_path(src)}')
rt.append(o)
objs['runtime'] = rt
every = [o for v in objs.values() for o in v]
exe = out / 'totk.exe'
if make_lib:
exe = out / LIB_NAME
lines += ['', 'rule ar', f' command = {AR.as_posix()} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
return '\n'.join(lines), objs, exe
rule = 'link'
@@ -278,7 +294,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
# linker looks into the library, and the real probes would stay out of totk.exe.
rule = 'link_lib'
lines += ['', 'rule link_lib',
f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {flag_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out',
f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {cmd_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out',
' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out']
lines += ['', f'build {ninja_path(exe)}: {rule} ' + ' '.join(ninja_path(o) for o in every) +
(f' | {ninja_path(runtime_lib)}' if runtime_lib else ''),
@@ -295,7 +311,7 @@ def main_objects(gen, main_out, rt_dir, mods):
return None, f'{flag_path(main_ninja)} not found'
if gen != (ROOT / 'generated' / main_out.name).resolve():
return None, 'generated C directory is not the main one'
lines = main_ninja.read_text().splitlines()
lines = main_ninja.read_text(encoding='utf-8').splitlines()
sources = dict(l[6:].split(': cc_gen ') for l in lines if l.startswith('build ') and ': cc_gen ' in l)
flags = next((l for l in lines if l.startswith('genflags = ')), '')
if not flags.startswith('genflags = ' + ' '.join(GEN_FLAGS) + ' -I '):
@@ -383,7 +399,7 @@ def read_manifest(path):
"""{object path below obj/: (sha256, size, mtime_ns)}; empty when the file is missing."""
man = {}
if path.is_file():
for line in path.read_text().splitlines():
for line in path.read_text(encoding='utf-8').splitlines():
sha, size, mtime, rel = line.split(' ', 3)
man[rel] = (sha, int(size), int(mtime))
return man
@@ -588,7 +604,7 @@ def main():
sys.exit(f'no expected hashes known for {args.version}; pass --expected')
expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json'
hashes = json.loads(Path(expected).read_text())['exefs_sha256']
global CLANG, AR, NINJA, TOOLCHAIN_BIN, DEPS
global CLANG, AR, WINDRES, NINJA, TOOLCHAIN_BIN, DEPS
if args.ninja:
NINJA = args.ninja.resolve()
elif not NINJA.is_file() and (ROOT / 'bin' / 'ninja.exe').is_file():
@@ -598,6 +614,7 @@ def main():
TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve()
CLANG = TOOLCHAIN_BIN / 'clang.exe'
AR = TOOLCHAIN_BIN / 'llvm-ar.exe'
WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe'
DEPS = (args.deps or DEFAULT_DEPS).resolve()
dlls = sorted((DEPS / 'bin').glob('*.dll'))
for tool in (CLANG, NINJA):
+66 -13
View File
@@ -41,6 +41,22 @@ LONG_HEX = re.compile(r'[0-9A-Fa-f]{32,}')
# say nothing about the data. The IVFC levels and the image hash decide.
UNSIGNED = re.compile(r'\s*(?:Fixed-Key Signature|NPDM Signature) \(FAIL\)')
HACTOOL_ERROR = re.compile(r'\(FAIL\)|\(INVALID\)|Failed to|Error:')
# hactool lines of a hash check that failed: a RomFS level, a section's superblock or its hash table.
HACTOOL_HASH_FAIL = re.compile(r'^\s*(?:Level \d|Superblock Hash|Hash Table) \(FAIL\)', re.M)
# Errors of the vendored extractors (tools/vendor/macnativegames) that mean a game file's data does not match its own
# hashes or ends early: the file is damaged or was not copied completely. DAMAGE finds them in a script's log.
DAMAGE_TEXT = re.compile(r'hash mismatch|hash-table mismatch|FS header hash|IVFC level \d+ block \d+ failed|Truncated|'
r'out of bounds|Section bounds')
DAMAGE = re.compile(rf'^\w*Error: (.*(?:{DAMAGE_TEXT.pattern}).*)$', re.M)
class DamagedFile(ValueError):
"""A game file whose data fails its hash checks or ends early. path: that file; detail: the check that failed."""
def __init__(self, path, detail):
self.path, self.detail = Path(path), detail
super().__init__(f'{self.path} is damaged or incomplete ({detail}). Copy that file again from its source '
f'(or dump it again) and retry.')
def say(message):
@@ -75,10 +91,18 @@ RUN_SCRIPT = ('import os, runpy, sys; p = sys.argv[1]; sys.path.insert(0, os.pat
"runpy.run_path(p, run_name='__main__')")
def run_logged(command, log):
def run_logged(command, log, source=None):
"""Run a vendored script with its output in log. source: the game file it reads; a failed hash check or a short
read of that file raises DamagedFile."""
args = [str(c) for c in command]
with log.open('w') as out:
code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT).returncode
env = dict(os.environ, PYTHONIOENCODING='utf-8') # its messages may name non-ASCII paths
with log.open('wb') as out:
code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT,
env=env).returncode
if code and source:
damage = DAMAGE.search(log.read_text(encoding='utf-8', errors='replace'))
if damage:
raise DamagedFile(source, damage.group(1).strip())
require(code == 0, f'{Path(command[1]).name} failed; see {shown(log)}')
@@ -98,7 +122,7 @@ def extract_exefs(game, keys, out, logs, *, update):
shutil.rmtree(exefs)
out.mkdir(parents=True, exist_ok=True)
command = [sys.executable, VENDOR / 'extract_exefs.py', game, '--keys', keys, '--output', exefs]
run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log')
run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log', source=game)
return exefs
@@ -131,12 +155,13 @@ def check_romfs(romfs, expected):
say(f' RomFS {expected["display_version"]}: {report["romfs_size"]:,} bytes, {len(report["files"]):,} files, hash matches')
def extract_base_romfs(out, keys, logs):
def extract_base_romfs(out, keys, logs, base=None):
romfs = out / 'romfs'
if romfs.exists():
shutil.rmtree(romfs)
run_logged([sys.executable, VENDOR / 'extract_romfs.py', '--keys', keys,
'--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log')
'--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log',
source=base)
def program_ncas(base, update, keys_path):
@@ -166,9 +191,33 @@ def merge_space(base, update, keys_path):
MERGE_PHASES = {'copy': ' copying the base and update Program NCAs', 'merge': ' merging with hactool',
'diagnose': ' a hash check failed: checking the base Program NCA alone',
'index': ' hashing and indexing the merged image'}
def run_hactool(command, environment):
"""(exit code, stdout and stderr with long hex strings removed). hactool prints UTF-8 (its manifest)."""
completed = subprocess.run(command, env=environment, capture_output=True, encoding='utf-8', errors='replace')
return completed.returncode, LONG_HEX.sub('<hex removed>', completed.stdout + completed.stderr)
def damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path):
"""The DamagedFile for a merge whose hash checks failed. The merged RomFS mixes data of both files, so hactool
verifies the base Program NCA alone: if it fails, the base game file is damaged, else the update file is. The
output is appended to log_path."""
code, log = run_hactool([str(hactool), '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
str(base_nca)], environment)
with log_path.open('a', encoding='utf-8') as f:
f.write(f'\n---- base Program NCA alone (exit code {code}) ----\n{log}')
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
if HACTOOL_HASH_FAIL.search(checked):
return DamagedFile(base, f'its game data fails the hash check; see {shown(log_path)}')
if all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)):
return DamagedFile(update, f'its game data fails the hash check; see {shown(log_path)}')
return ValueError(f'the game data fails the hash check: the base game file {base} or the update file {update} is '
f'damaged or incomplete. Copy them again from their source and retry; see {shown(log_path)}')
def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lambda name: say(MERGE_PHASES[name])):
"""extract_update_romfs.merge() without its Darwin-only bounded view of the base: the base Program NCA is copied out.
@@ -195,19 +244,23 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lam
phase('copy')
base_nca, update_nca = tmp / 'base.nca', tmp / 'update.nca'
for (source, record, _), target in zip(inputs, (base_nca, update_nca)):
copy_range(source, record_offset(record), record['size'], target)
try:
copy_range(source, record_offset(record), record['size'], target)
except ValueError as e: # the file ends inside its Program NCA
raise DamagedFile(source, str(e)) from None
result = tmp / 'result'
result.mkdir()
phase('merge')
command = [str(hactool), '-x', '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
f'--basenca={base_nca}', f'--section{sections[0]}={result / "romfs.bin"}', str(update_nca)]
environment = dict(os.environ, MNG_TITLE_KEYS=str(title_keys))
completed = subprocess.run(command, env=environment, capture_output=True, text=True, errors='replace')
log = LONG_HEX.sub('<hex removed>', completed.stdout + completed.stderr)
log_path.write_text(log)
code, log = run_hactool(command, environment)
log_path.write_text(log, encoding='utf-8')
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
require(completed.returncode == 0 and not HACTOOL_ERROR.search(checked),
f'hactool merge failed; see {shown(log_path)}')
if HACTOOL_HASH_FAIL.search(checked):
phase('diagnose')
raise damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path)
require(code == 0 and not HACTOOL_ERROR.search(checked), f'hactool merge failed; see {shown(log_path)}')
require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'hactool did not verify all six IVFC levels')
image = result / 'romfs.bin'
phase('index')
@@ -253,7 +306,7 @@ def main():
report_npdm(base_out / 'exefs', v0)
if not romfs_verified(base_out / 'romfs', v0):
say(' extracting RomFS (17 GB)')
extract_base_romfs(base_out, keys, logs)
extract_base_romfs(base_out, keys, logs, base)
check_romfs(base_out / 'romfs', v0)
for update in updates: