Say which game file is damaged when its hashes fail

The ExeFS and RomFS extractors report a failed block hash or a short read as an opaque script failure, and a failed
hactool merge as a generic error. extract.DamagedFile now names the file; a merge whose hash checks fail runs hactool
on the base Program NCA alone to tell the base game from the update. The installer shows check.input_damaged: the file
is damaged or incomplete and should be copied again. hactool output is decoded and its log written as UTF-8.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-24 19:35:40 +03:00
1 parent c50f31164d
commit 1820a83f30
4 files changed
+91 -17

No files matched your search

+21 -4
View File
@@ -131,6 +131,13 @@ def free_bytes(path):
return shutil.disk_usage(p).free, os.stat(p).st_dev
def raise_if_damaged(error, name, path):
"""InstallError check.input_damaged when a reader's ValueError says the file's data fails its hashes or ends early
(extract.DAMAGE_TEXT)."""
if extract.DAMAGE_TEXT.search(str(error)):
raise InstallError('check.input_damaged', input=name, path=str(path), detail=str(error)) from None
def expected_update():
return extract.load_expected()[UPDATE_VERSION]
@@ -264,15 +271,16 @@ class Check(Step):
for name, is_update in (('base', False), ('update', True)):
path = Path(getattr(o, name))
with path.open('rb') as f:
total += self.open_input(f, path.stat().st_size, keys, name, is_update)
total += self.open_input(f, path, keys, name, is_update)
del keys
return total
@staticmethod
def open_input(f, size, keys, name, is_update):
def open_input(f, path, keys, name, is_update):
try:
kind, entries = container_entries(f, size)
except ValueError:
kind, entries = container_entries(f, path.stat().st_size)
except ValueError as e:
raise_if_damaged(e, name, path)
raise InstallError('check.not_game_file', input=name) from None
if is_update and kind != 'nsp':
raise InstallError('check.update_not_nsp')
@@ -281,12 +289,14 @@ class Check(Step):
except ValueError as e:
if 'Personalized' in str(e):
raise InstallError('check.ticket_personalized', input=name) from None
raise_if_damaged(e, name, path)
raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
try:
records = nca_inventory(f, entries, keys)
except ValueError as e:
if 'NCA3' in str(e):
raise InstallError('check.keys_wrong', input=name) from None
raise_if_damaged(e, name, path)
raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None
try:
record, header = program_nca(records, update=is_update)
@@ -339,6 +349,8 @@ class Check(Step):
remove(out)
try:
extract.extract_exefs(Path(getattr(o, name)), Path(o.keys), out, logs, update=is_update)
except extract.DamagedFile as e:
raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None
except ValueError:
raise InstallError('check.exefs_failed', input=name, log=str(logs / f'extract_exefs-{name}.log')) \
from None
@@ -401,6 +413,8 @@ class Extract(Step):
remove(ctx.temp / 'check' / 'update')
try:
extract.extract_exefs(update, keys, ctx.temp / 'check' / 'update', ctx.temp / 'logs', update=True)
except extract.DamagedFile as e:
raise InstallError('check.input_damaged', input='update', path=str(e.path), detail=e.detail) from None
except ValueError:
raise InstallError('check.exefs_failed', input='update',
log=str(ctx.temp / 'logs' / 'extract_exefs-update.log')) from None
@@ -441,6 +455,9 @@ class Extract(Step):
watch.start()
try:
extract.merge_update_romfs(base, update, keys, ctx.layout.hactool, romfs, logs, phase=phase)
except extract.DamagedFile as e:
raise InstallError('check.input_damaged', input='base' if e.path == base else 'update', path=str(e.path),
detail=e.detail) from None
except ValueError as e:
raise InstallError('extract.romfs_merge_failed', detail=str(e), log=str(logs / 'merge_romfs-'
f'{dest.name}.log')) from None
+2
View File
@@ -16,6 +16,7 @@
"phase.extract.exefs": "Extracting the game code",
"phase.extract.copy": "Copying the game data",
"phase.extract.merge": "Applying the update to the game data",
"phase.extract.diagnose": "Finding which game file is damaged",
"phase.extract.index": "Verifying the game data",
"phase.lift.discover": "Finding the functions",
"phase.lift.emit": "Writing C code",
@@ -60,6 +61,7 @@
"check.update_is_base": "The update file holds a base game. Choose the 1.2.1 update NSP.",
"check.update_not_totk": "The update file is not a Tears of the Kingdom update ({detail}).",
"check.exefs_failed": "The game code of the {input} file failed its checks; see {log}.",
"check.input_damaged": "The {input} file is damaged or incomplete: {path} ({detail}). Copy that file again from its source, or dump it again, then run the setup again.",
"check.base_wrong_version": "The base game is not version 1.0.0 ({modules} differ).",
"check.update_wrong_version": "The update is version {found}; this release needs {wanted}.",
"check.disk_space": "Not enough free space for {path}: {need_gb} GB needed, {free_gb} GB free.",
+2
View File
@@ -16,6 +16,7 @@
"phase.extract.exefs": "Распаковка кода игры",
"phase.extract.copy": "Копирование данных игры",
"phase.extract.merge": "Применение обновления к данным игры",
"phase.extract.diagnose": "Поиск повреждённого файла игры",
"phase.extract.index": "Проверка данных игры",
"phase.lift.discover": "Поиск функций",
"phase.lift.emit": "Запись кода на C",
@@ -60,6 +61,7 @@
"check.update_is_base": "Файл обновления содержит базовую игру. Выберите NSP с обновлением 1.2.1.",
"check.update_not_totk": "Файл обновления — не обновление Tears of the Kingdom ({detail}).",
"check.exefs_failed": "Код игры из файла «{input}» не прошёл проверку; подробности в {log}.",
"check.input_damaged": "Файл «{input}» повреждён или скопирован не полностью: {path} ({detail}). Скопируйте этот файл заново из источника или сделайте дамп ещё раз, затем снова запустите установку.",
"check.base_wrong_version": "Базовая игра не версии 1.0.0 (отличаются модули: {modules}).",
"check.update_wrong_version": "Версия обновления {found}; этому выпуску нужна {wanted}.",
"check.disk_space": "Не хватает свободного места для {path}: нужно {need_gb} ГБ, свободно {free_gb} ГБ.",
+66 -13
View File
@@ -41,6 +41,22 @@ LONG_HEX = re.compile(r'[0-9A-Fa-f]{32,}')
# say nothing about the data. The IVFC levels and the image hash decide.
UNSIGNED = re.compile(r'\s*(?:Fixed-Key Signature|NPDM Signature) \(FAIL\)')
HACTOOL_ERROR = re.compile(r'\(FAIL\)|\(INVALID\)|Failed to|Error:')
# hactool lines of a hash check that failed: a RomFS level, a section's superblock or its hash table.
HACTOOL_HASH_FAIL = re.compile(r'^\s*(?:Level \d|Superblock Hash|Hash Table) \(FAIL\)', re.M)
# Errors of the vendored extractors (tools/vendor/macnativegames) that mean a game file's data does not match its own
# hashes or ends early: the file is damaged or was not copied completely. DAMAGE finds them in a script's log.
DAMAGE_TEXT = re.compile(r'hash mismatch|hash-table mismatch|FS header hash|IVFC level \d+ block \d+ failed|Truncated|'
r'out of bounds|Section bounds')
DAMAGE = re.compile(rf'^\w*Error: (.*(?:{DAMAGE_TEXT.pattern}).*)$', re.M)
class DamagedFile(ValueError):
"""A game file whose data fails its hash checks or ends early. path: that file; detail: the check that failed."""
def __init__(self, path, detail):
self.path, self.detail = Path(path), detail
super().__init__(f'{self.path} is damaged or incomplete ({detail}). Copy that file again from its source '
f'(or dump it again) and retry.')
def say(message):
@@ -75,10 +91,18 @@ RUN_SCRIPT = ('import os, runpy, sys; p = sys.argv[1]; sys.path.insert(0, os.pat
"runpy.run_path(p, run_name='__main__')")
def run_logged(command, log):
def run_logged(command, log, source=None):
"""Run a vendored script with its output in log. source: the game file it reads; a failed hash check or a short
read of that file raises DamagedFile."""
args = [str(c) for c in command]
with log.open('w') as out:
code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT).returncode
env = dict(os.environ, PYTHONIOENCODING='utf-8') # its messages may name non-ASCII paths
with log.open('wb') as out:
code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT,
env=env).returncode
if code and source:
damage = DAMAGE.search(log.read_text(encoding='utf-8', errors='replace'))
if damage:
raise DamagedFile(source, damage.group(1).strip())
require(code == 0, f'{Path(command[1]).name} failed; see {shown(log)}')
@@ -98,7 +122,7 @@ def extract_exefs(game, keys, out, logs, *, update):
shutil.rmtree(exefs)
out.mkdir(parents=True, exist_ok=True)
command = [sys.executable, VENDOR / 'extract_exefs.py', game, '--keys', keys, '--output', exefs]
run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log')
run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log', source=game)
return exefs
@@ -131,12 +155,13 @@ def check_romfs(romfs, expected):
say(f' RomFS {expected["display_version"]}: {report["romfs_size"]:,} bytes, {len(report["files"]):,} files, hash matches')
def extract_base_romfs(out, keys, logs):
def extract_base_romfs(out, keys, logs, base=None):
romfs = out / 'romfs'
if romfs.exists():
shutil.rmtree(romfs)
run_logged([sys.executable, VENDOR / 'extract_romfs.py', '--keys', keys,
'--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log')
'--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log',
source=base)
def program_ncas(base, update, keys_path):
@@ -166,9 +191,33 @@ def merge_space(base, update, keys_path):
MERGE_PHASES = {'copy': ' copying the base and update Program NCAs', 'merge': ' merging with hactool',
'diagnose': ' a hash check failed: checking the base Program NCA alone',
'index': ' hashing and indexing the merged image'}
def run_hactool(command, environment):
"""(exit code, stdout and stderr with long hex strings removed). hactool prints UTF-8 (its manifest)."""
completed = subprocess.run(command, env=environment, capture_output=True, encoding='utf-8', errors='replace')
return completed.returncode, LONG_HEX.sub('<hex removed>', completed.stdout + completed.stderr)
def damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path):
"""The DamagedFile for a merge whose hash checks failed. The merged RomFS mixes data of both files, so hactool
verifies the base Program NCA alone: if it fails, the base game file is damaged, else the update file is. The
output is appended to log_path."""
code, log = run_hactool([str(hactool), '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
str(base_nca)], environment)
with log_path.open('a', encoding='utf-8') as f:
f.write(f'\n---- base Program NCA alone (exit code {code}) ----\n{log}')
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
if HACTOOL_HASH_FAIL.search(checked):
return DamagedFile(base, f'its game data fails the hash check; see {shown(log_path)}')
if all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)):
return DamagedFile(update, f'its game data fails the hash check; see {shown(log_path)}')
return ValueError(f'the game data fails the hash check: the base game file {base} or the update file {update} is '
f'damaged or incomplete. Copy them again from their source and retry; see {shown(log_path)}')
def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lambda name: say(MERGE_PHASES[name])):
"""extract_update_romfs.merge() without its Darwin-only bounded view of the base: the base Program NCA is copied out.
@@ -195,19 +244,23 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lam
phase('copy')
base_nca, update_nca = tmp / 'base.nca', tmp / 'update.nca'
for (source, record, _), target in zip(inputs, (base_nca, update_nca)):
copy_range(source, record_offset(record), record['size'], target)
try:
copy_range(source, record_offset(record), record['size'], target)
except ValueError as e: # the file ends inside its Program NCA
raise DamagedFile(source, str(e)) from None
result = tmp / 'result'
result.mkdir()
phase('merge')
command = [str(hactool), '-x', '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys),
f'--basenca={base_nca}', f'--section{sections[0]}={result / "romfs.bin"}', str(update_nca)]
environment = dict(os.environ, MNG_TITLE_KEYS=str(title_keys))
completed = subprocess.run(command, env=environment, capture_output=True, text=True, errors='replace')
log = LONG_HEX.sub('<hex removed>', completed.stdout + completed.stderr)
log_path.write_text(log)
code, log = run_hactool(command, environment)
log_path.write_text(log, encoding='utf-8')
checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line))
require(completed.returncode == 0 and not HACTOOL_ERROR.search(checked),
f'hactool merge failed; see {shown(log_path)}')
if HACTOOL_HASH_FAIL.search(checked):
phase('diagnose')
raise damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path)
require(code == 0 and not HACTOOL_ERROR.search(checked), f'hactool merge failed; see {shown(log_path)}')
require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'hactool did not verify all six IVFC levels')
image = result / 'romfs.bin'
phase('index')
@@ -253,7 +306,7 @@ def main():
report_npdm(base_out / 'exefs', v0)
if not romfs_verified(base_out / 'romfs', v0):
say(' extracting RomFS (17 GB)')
extract_base_romfs(base_out, keys, logs)
extract_base_romfs(base_out, keys, logs, base)
check_romfs(base_out / 'romfs', v0)
for update in updates: