From c50f31164d5e4433f17e663365be1697ddc4b2e7 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 19:35:33 +0300 Subject: [PATCH 1/4] Embed a manifest in totk.exe that makes UTF-8 its code page GetModuleFileNameA, fopen, CreateFileA and getenv returned and took ANSI code page bytes while platform/win32.c, fs_host.c and romfs.c convert paths as UTF-8, so an install folder under a Cyrillic user name lost its RomFS, profile, RomFS patches and caches. runtime/totk.rc is compiled with the runtime objects (windres, or llvm-windres with --toolchain), so it is also a member of libtotkrt.a. build_game.py writes build.ninja and modules.c as UTF-8. Co-Authored-By: Claude Opus 5.5 --- runtime/totk.manifest | 18 ++++++++++++++++++ runtime/totk.rc | 3 +++ tools/build_game.py | 24 +++++++++++++++--------- 3 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 runtime/totk.manifest create mode 100644 runtime/totk.rc diff --git a/runtime/totk.manifest b/runtime/totk.manifest new file mode 100644 index 00000000..8fe37fa1 --- /dev/null +++ b/runtime/totk.manifest @@ -0,0 +1,18 @@ + + + + + + + + + + + + + UTF-8 + + + diff --git a/runtime/totk.rc b/runtime/totk.rc new file mode 100644 index 00000000..b26fe565 --- /dev/null +++ b/runtime/totk.rc @@ -0,0 +1,3 @@ +/* The application manifest of totk.exe (totk.manifest). tools/build_game.py compiles it with the runtime objects, so it + is also a member of libtotkrt.a, which the release links with --whole-archive. */ +1 24 "totk.manifest" diff --git a/tools/build_game.py b/tools/build_game.py index baabc895..65493976 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -86,6 +86,7 @@ DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw' INSTALL_SIDE = {'romfs.c'} # runtime sources that include romfs_config.h: compiled per install, not in the library LIB_NAME = 'libtotkrt.a' AR = UCRT / 'llvm-ar.exe' +WINDRES = UCRT / 'windres.exe' # runtime/totk.rc: the manifest of totk.exe TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain DEPS = None # the deps directory of an llvm-mingw build # tools/data/expected-.json for each build version name. @@ -146,10 +147,10 @@ def chunk_files(gen, mod): def write_if_changed(path, text): """Keep the mtime of an unchanged file so Ninja does not rebuild from it.""" - if path.exists() and path.read_text() == text: + if path.exists() and path.read_text(encoding='utf-8', errors='replace') == text: return path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text, newline='\n') + path.write_text(text, encoding='utf-8', newline='\n') # Ninja and clang read UTF-8; paths may be non-ASCII def c_string(s): @@ -160,9 +161,9 @@ RUNTIME = ROOT / 'runtime' # --runtime: a copy of runtime/ lets a task build w def runtime_sources(skip=()): - """Every runtime/*.c and runtime/platform/*.c except the names in skip, so a task adds a file without editing this - script.""" - pats = ('*.c', '*.cpp', 'platform/*.c') + """Every runtime/*.c, runtime/platform/*.c and runtime/*.rc except the names in skip, so a task adds a file without + editing this script.""" + pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc') return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip) @@ -236,6 +237,10 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= ' depfile = $out.d', ' description = CC $in', '', + 'rule rc', + f' command = {WINDRES.as_posix()} -I {flag_path(RUNTIME)} -O coff $in -o $out', + ' description = RC $in', + '', 'rule link', ' command = $cc $ldflags @$out.rsp $libs -o $out', ' rspfile = $out.rsp', @@ -262,7 +267,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= sources += [out / 'modules.c'] for src in sources: o = runtime_object(out, src) - lines.append(f'build {ninja_path(o)}: cc_rt {ninja_path(src)}') + lines.append(f'build {ninja_path(o)}: {"rc" if src.suffix == ".rc" else "cc_rt"} {ninja_path(src)}') rt.append(o) objs['runtime'] = rt every = [o for v in objs.values() for o in v] @@ -295,7 +300,7 @@ def main_objects(gen, main_out, rt_dir, mods): return None, f'{flag_path(main_ninja)} not found' if gen != (ROOT / 'generated' / main_out.name).resolve(): return None, 'generated C directory is not the main one' - lines = main_ninja.read_text().splitlines() + lines = main_ninja.read_text(encoding='utf-8').splitlines() sources = dict(l[6:].split(': cc_gen ') for l in lines if l.startswith('build ') and ': cc_gen ' in l) flags = next((l for l in lines if l.startswith('genflags = ')), '') if not flags.startswith('genflags = ' + ' '.join(GEN_FLAGS) + ' -I '): @@ -383,7 +388,7 @@ def read_manifest(path): """{object path below obj/: (sha256, size, mtime_ns)}; empty when the file is missing.""" man = {} if path.is_file(): - for line in path.read_text().splitlines(): + for line in path.read_text(encoding='utf-8').splitlines(): sha, size, mtime, rel = line.split(' ', 3) man[rel] = (sha, int(size), int(mtime)) return man @@ -588,7 +593,7 @@ def main(): sys.exit(f'no expected hashes known for {args.version}; pass --expected') expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json' hashes = json.loads(Path(expected).read_text())['exefs_sha256'] - global CLANG, AR, NINJA, TOOLCHAIN_BIN, DEPS + global CLANG, AR, WINDRES, NINJA, TOOLCHAIN_BIN, DEPS if args.ninja: NINJA = args.ninja.resolve() elif not NINJA.is_file() and (ROOT / 'bin' / 'ninja.exe').is_file(): @@ -598,6 +603,7 @@ def main(): TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve() CLANG = TOOLCHAIN_BIN / 'clang.exe' AR = TOOLCHAIN_BIN / 'llvm-ar.exe' + WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe' DEPS = (args.deps or DEFAULT_DEPS).resolve() dlls = sorted((DEPS / 'bin').glob('*.dll')) for tool in (CLANG, NINJA): From 1820a83f30e1eef7618966a51cfa10dbf5476826 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 19:35:40 +0300 Subject: [PATCH 2/4] Say which game file is damaged when its hashes fail The ExeFS and RomFS extractors report a failed block hash or a short read as an opaque script failure, and a failed hactool merge as a generic error. extract.DamagedFile now names the file; a merge whose hash checks fail runs hactool on the base Program NCA alone to tell the base game from the update. The installer shows check.input_damaged: the file is damaged or incomplete and should be copied again. hactool output is decoded and its log written as UTF-8. Co-Authored-By: Claude Opus 5.5 --- installer/steps.py | 25 ++++++++++--- installer/text/en.json | 2 ++ installer/text/ru.json | 2 ++ tools/extract.py | 79 +++++++++++++++++++++++++++++++++++------- 4 files changed, 91 insertions(+), 17 deletions(-) diff --git a/installer/steps.py b/installer/steps.py index adfa871a..6ffedce0 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -131,6 +131,13 @@ def free_bytes(path): return shutil.disk_usage(p).free, os.stat(p).st_dev +def raise_if_damaged(error, name, path): + """InstallError check.input_damaged when a reader's ValueError says the file's data fails its hashes or ends early + (extract.DAMAGE_TEXT).""" + if extract.DAMAGE_TEXT.search(str(error)): + raise InstallError('check.input_damaged', input=name, path=str(path), detail=str(error)) from None + + def expected_update(): return extract.load_expected()[UPDATE_VERSION] @@ -264,15 +271,16 @@ class Check(Step): for name, is_update in (('base', False), ('update', True)): path = Path(getattr(o, name)) with path.open('rb') as f: - total += self.open_input(f, path.stat().st_size, keys, name, is_update) + total += self.open_input(f, path, keys, name, is_update) del keys return total @staticmethod - def open_input(f, size, keys, name, is_update): + def open_input(f, path, keys, name, is_update): try: - kind, entries = container_entries(f, size) - except ValueError: + kind, entries = container_entries(f, path.stat().st_size) + except ValueError as e: + raise_if_damaged(e, name, path) raise InstallError('check.not_game_file', input=name) from None if is_update and kind != 'nsp': raise InstallError('check.update_not_nsp') @@ -281,12 +289,14 @@ class Check(Step): except ValueError as e: if 'Personalized' in str(e): raise InstallError('check.ticket_personalized', input=name) from None + raise_if_damaged(e, name, path) raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None try: records = nca_inventory(f, entries, keys) except ValueError as e: if 'NCA3' in str(e): raise InstallError('check.keys_wrong', input=name) from None + raise_if_damaged(e, name, path) raise InstallError('check.file_unreadable', input=name, detail=str(e)) from None try: record, header = program_nca(records, update=is_update) @@ -339,6 +349,8 @@ class Check(Step): remove(out) try: extract.extract_exefs(Path(getattr(o, name)), Path(o.keys), out, logs, update=is_update) + except extract.DamagedFile as e: + raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None except ValueError: raise InstallError('check.exefs_failed', input=name, log=str(logs / f'extract_exefs-{name}.log')) \ from None @@ -401,6 +413,8 @@ class Extract(Step): remove(ctx.temp / 'check' / 'update') try: extract.extract_exefs(update, keys, ctx.temp / 'check' / 'update', ctx.temp / 'logs', update=True) + except extract.DamagedFile as e: + raise InstallError('check.input_damaged', input='update', path=str(e.path), detail=e.detail) from None except ValueError: raise InstallError('check.exefs_failed', input='update', log=str(ctx.temp / 'logs' / 'extract_exefs-update.log')) from None @@ -441,6 +455,9 @@ class Extract(Step): watch.start() try: extract.merge_update_romfs(base, update, keys, ctx.layout.hactool, romfs, logs, phase=phase) + except extract.DamagedFile as e: + raise InstallError('check.input_damaged', input='base' if e.path == base else 'update', path=str(e.path), + detail=e.detail) from None except ValueError as e: raise InstallError('extract.romfs_merge_failed', detail=str(e), log=str(logs / 'merge_romfs-' f'{dest.name}.log')) from None diff --git a/installer/text/en.json b/installer/text/en.json index bdf5b0fd..7b9abf49 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -16,6 +16,7 @@ "phase.extract.exefs": "Extracting the game code", "phase.extract.copy": "Copying the game data", "phase.extract.merge": "Applying the update to the game data", + "phase.extract.diagnose": "Finding which game file is damaged", "phase.extract.index": "Verifying the game data", "phase.lift.discover": "Finding the functions", "phase.lift.emit": "Writing C code", @@ -60,6 +61,7 @@ "check.update_is_base": "The update file holds a base game. Choose the 1.2.1 update NSP.", "check.update_not_totk": "The update file is not a Tears of the Kingdom update ({detail}).", "check.exefs_failed": "The game code of the {input} file failed its checks; see {log}.", + "check.input_damaged": "The {input} file is damaged or incomplete: {path} ({detail}). Copy that file again from its source, or dump it again, then run the setup again.", "check.base_wrong_version": "The base game is not version 1.0.0 ({modules} differ).", "check.update_wrong_version": "The update is version {found}; this release needs {wanted}.", "check.disk_space": "Not enough free space for {path}: {need_gb} GB needed, {free_gb} GB free.", diff --git a/installer/text/ru.json b/installer/text/ru.json index f045ea6d..d3c3afb6 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -16,6 +16,7 @@ "phase.extract.exefs": "Распаковка кода игры", "phase.extract.copy": "Копирование данных игры", "phase.extract.merge": "Применение обновления к данным игры", + "phase.extract.diagnose": "Поиск повреждённого файла игры", "phase.extract.index": "Проверка данных игры", "phase.lift.discover": "Поиск функций", "phase.lift.emit": "Запись кода на C", @@ -60,6 +61,7 @@ "check.update_is_base": "Файл обновления содержит базовую игру. Выберите NSP с обновлением 1.2.1.", "check.update_not_totk": "Файл обновления — не обновление Tears of the Kingdom ({detail}).", "check.exefs_failed": "Код игры из файла «{input}» не прошёл проверку; подробности в {log}.", + "check.input_damaged": "Файл «{input}» повреждён или скопирован не полностью: {path} ({detail}). Скопируйте этот файл заново из источника или сделайте дамп ещё раз, затем снова запустите установку.", "check.base_wrong_version": "Базовая игра не версии 1.0.0 (отличаются модули: {modules}).", "check.update_wrong_version": "Версия обновления {found}; этому выпуску нужна {wanted}.", "check.disk_space": "Не хватает свободного места для {path}: нужно {need_gb} ГБ, свободно {free_gb} ГБ.", diff --git a/tools/extract.py b/tools/extract.py index 318f2ec9..cdc78a08 100644 --- a/tools/extract.py +++ b/tools/extract.py @@ -41,6 +41,22 @@ LONG_HEX = re.compile(r'[0-9A-Fa-f]{32,}') # say nothing about the data. The IVFC levels and the image hash decide. UNSIGNED = re.compile(r'\s*(?:Fixed-Key Signature|NPDM Signature) \(FAIL\)') HACTOOL_ERROR = re.compile(r'\(FAIL\)|\(INVALID\)|Failed to|Error:') +# hactool lines of a hash check that failed: a RomFS level, a section's superblock or its hash table. +HACTOOL_HASH_FAIL = re.compile(r'^\s*(?:Level \d|Superblock Hash|Hash Table) \(FAIL\)', re.M) +# Errors of the vendored extractors (tools/vendor/macnativegames) that mean a game file's data does not match its own +# hashes or ends early: the file is damaged or was not copied completely. DAMAGE finds them in a script's log. +DAMAGE_TEXT = re.compile(r'hash mismatch|hash-table mismatch|FS header hash|IVFC level \d+ block \d+ failed|Truncated|' + r'out of bounds|Section bounds') +DAMAGE = re.compile(rf'^\w*Error: (.*(?:{DAMAGE_TEXT.pattern}).*)$', re.M) + + +class DamagedFile(ValueError): + """A game file whose data fails its hash checks or ends early. path: that file; detail: the check that failed.""" + + def __init__(self, path, detail): + self.path, self.detail = Path(path), detail + super().__init__(f'{self.path} is damaged or incomplete ({detail}). Copy that file again from its source ' + f'(or dump it again) and retry.') def say(message): @@ -75,10 +91,18 @@ RUN_SCRIPT = ('import os, runpy, sys; p = sys.argv[1]; sys.path.insert(0, os.pat "runpy.run_path(p, run_name='__main__')") -def run_logged(command, log): +def run_logged(command, log, source=None): + """Run a vendored script with its output in log. source: the game file it reads; a failed hash check or a short + read of that file raises DamagedFile.""" args = [str(c) for c in command] - with log.open('w') as out: - code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT).returncode + env = dict(os.environ, PYTHONIOENCODING='utf-8') # its messages may name non-ASCII paths + with log.open('wb') as out: + code = subprocess.run([args[0], '-c', RUN_SCRIPT, *args[1:]], stdout=out, stderr=subprocess.STDOUT, + env=env).returncode + if code and source: + damage = DAMAGE.search(log.read_text(encoding='utf-8', errors='replace')) + if damage: + raise DamagedFile(source, damage.group(1).strip()) require(code == 0, f'{Path(command[1]).name} failed; see {shown(log)}') @@ -98,7 +122,7 @@ def extract_exefs(game, keys, out, logs, *, update): shutil.rmtree(exefs) out.mkdir(parents=True, exist_ok=True) command = [sys.executable, VENDOR / 'extract_exefs.py', game, '--keys', keys, '--output', exefs] - run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log') + run_logged(command + (['--update'] if update else []), logs / f'extract_exefs-{out.name}.log', source=game) return exefs @@ -131,12 +155,13 @@ def check_romfs(romfs, expected): say(f' RomFS {expected["display_version"]}: {report["romfs_size"]:,} bytes, {len(report["files"]):,} files, hash matches') -def extract_base_romfs(out, keys, logs): +def extract_base_romfs(out, keys, logs, base=None): romfs = out / 'romfs' if romfs.exists(): shutil.rmtree(romfs) run_logged([sys.executable, VENDOR / 'extract_romfs.py', '--keys', keys, - '--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log') + '--manifest', out / 'extraction-manifest.json', '--output', romfs], logs / 'extract_romfs-v0.log', + source=base) def program_ncas(base, update, keys_path): @@ -166,9 +191,33 @@ def merge_space(base, update, keys_path): MERGE_PHASES = {'copy': ' copying the base and update Program NCAs', 'merge': ' merging with hactool', + 'diagnose': ' a hash check failed: checking the base Program NCA alone', 'index': ' hashing and indexing the merged image'} +def run_hactool(command, environment): + """(exit code, stdout and stderr with long hex strings removed). hactool prints UTF-8 (its manifest).""" + completed = subprocess.run(command, env=environment, capture_output=True, encoding='utf-8', errors='replace') + return completed.returncode, LONG_HEX.sub('', completed.stdout + completed.stderr) + + +def damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path): + """The DamagedFile for a merge whose hash checks failed. The merged RomFS mixes data of both files, so hactool + verifies the base Program NCA alone: if it fails, the base game file is damaged, else the update file is. The + output is appended to log_path.""" + code, log = run_hactool([str(hactool), '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys), + str(base_nca)], environment) + with log_path.open('a', encoding='utf-8') as f: + f.write(f'\n---- base Program NCA alone (exit code {code}) ----\n{log}') + checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line)) + if HACTOOL_HASH_FAIL.search(checked): + return DamagedFile(base, f'its game data fails the hash check; see {shown(log_path)}') + if all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)): + return DamagedFile(update, f'its game data fails the hash check; see {shown(log_path)}') + return ValueError(f'the game data fails the hash check: the base game file {base} or the update file {update} is ' + f'damaged or incomplete. Copy them again from their source and retry; see {shown(log_path)}') + + def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lambda name: say(MERGE_PHASES[name])): """extract_update_romfs.merge() without its Darwin-only bounded view of the base: the base Program NCA is copied out. @@ -195,19 +244,23 @@ def merge_update_romfs(base, update, keys_path, hactool, output, logs, phase=lam phase('copy') base_nca, update_nca = tmp / 'base.nca', tmp / 'update.nca' for (source, record, _), target in zip(inputs, (base_nca, update_nca)): - copy_range(source, record_offset(record), record['size'], target) + try: + copy_range(source, record_offset(record), record['size'], target) + except ValueError as e: # the file ends inside its Program NCA + raise DamagedFile(source, str(e)) from None result = tmp / 'result' result.mkdir() phase('merge') command = [str(hactool), '-x', '-y', '--disablekeywarns', '--suppresskeys', '-k', str(hactool_keys), f'--basenca={base_nca}', f'--section{sections[0]}={result / "romfs.bin"}', str(update_nca)] environment = dict(os.environ, MNG_TITLE_KEYS=str(title_keys)) - completed = subprocess.run(command, env=environment, capture_output=True, text=True, errors='replace') - log = LONG_HEX.sub('', completed.stdout + completed.stderr) - log_path.write_text(log) + code, log = run_hactool(command, environment) + log_path.write_text(log, encoding='utf-8') checked = '\n'.join(line for line in log.splitlines() if not UNSIGNED.match(line)) - require(completed.returncode == 0 and not HACTOOL_ERROR.search(checked), - f'hactool merge failed; see {shown(log_path)}') + if HACTOOL_HASH_FAIL.search(checked): + phase('diagnose') + raise damaged_merge_input(hactool, hactool_keys, environment, base, update, base_nca, log_path) + require(code == 0 and not HACTOOL_ERROR.search(checked), f'hactool merge failed; see {shown(log_path)}') require(all(re.search(rf'Level {i} \(GOOD\)', log) for i in range(6)), 'hactool did not verify all six IVFC levels') image = result / 'romfs.bin' phase('index') @@ -253,7 +306,7 @@ def main(): report_npdm(base_out / 'exefs', v0) if not romfs_verified(base_out / 'romfs', v0): say(' extracting RomFS (17 GB)') - extract_base_romfs(base_out, keys, logs) + extract_base_romfs(base_out, keys, logs, base) check_romfs(base_out / 'romfs', v0) for update in updates: From a4bb2d535577896d20ceb73afec96084cde7c402 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 19:35:41 +0300 Subject: [PATCH 3/4] Document the UTF-8 manifests of hactool and totk.exe and the damaged file message Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 2 +- docs/installer.md | 7 +++++++ docs/toolchain-msys2-python.md | 9 +++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index bec4fc7d..b5fc4fcf 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -50,7 +50,7 @@ These tools run on the developer's or player's machine during the build. They ar | Vulkan-Headers and Vulkan-Loader 1.4.363.0 (MSYS2 packages; headers and `libvulkan-1.dll.a` only, no loader DLL) | https://github.com/KhronosGroup | Apache-2.0 OR MIT | `runtime/deps/` | | Vulkan Memory Allocator 3.4.0 (MSYS2 package), header | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator | MIT | `runtime/deps/` | | Ninja 1.13.2 (`ninja-win.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja.exe` | -| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check), built with MSYS2 UCRT64 clang; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | +| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check; our `unicode-patches.diff`: an embedded manifest that makes UTF-8 the process code page), built with MSYS2 UCRT64 gcc; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | | `totk_xlate.exe` (Ryujinx shader translator, NativeAOT build of `tools/xlate/`, see the Ryujinx row above) | built by `tools/build_xlate.py` | MIT | `bin/totk_xlate.exe` | | MacNativeGames shader variant list | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/assets/shader_list.tsv`), Copyright (c) 2026 MacNativeGames contributors | MIT | `data/shader_list.tsv` | diff --git a/docs/installer.md b/docs/installer.md index c0738dea..8f240a76 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -104,6 +104,13 @@ totk.ini.example, VERSION the default ini; the release string (git describe of - The keys file path is passed to the tools; key values never appear in the UI, logs or files. - The window is a front end over the CLI (`python -m installer --base ... --update ... --keys ... --lang ... --dest ...`), which prints one JSON progress event per line. Tests and CI use the CLI. +- Paths may hold any Unicode characters (a Cyrillic user name puts the temp and install folders under it). Every exe + the setup runs or builds reads its arguments and paths as UTF-8 through an `activeCodePage` manifest: hactool + (`toolchain/hactool/unicode-patches.diff`), totk.exe (`runtime/totk.rc`), ninja (upstream). The Python steps write + their files and logs as UTF-8. +- A game file whose data fails its hash checks or ends early stops the setup with `check.input_damaged`, which names + the file and asks to copy it again (`extract.DamagedFile`). A failed merge checks the base Program NCA alone to tell + which file is damaged. - Unsigned exe: SmartScreen warns on first start. ## Later diff --git a/docs/toolchain-msys2-python.md b/docs/toolchain-msys2-python.md index fd9b5cc7..3b0b0a82 100644 --- a/docs/toolchain-msys2-python.md +++ b/docs/toolchain-msys2-python.md @@ -95,6 +95,15 @@ Applied the portable part of MacNativeGames `build_app.py:build_hactool()` (diff 2. BKTR: allow 0x200-aligned padding after the subsection table (`nca.c`). 3. BKTR: accept the subsection total-size layout of newer cumulative updates (`nca.c`). +Ours (`toolchain\hactool\unicode-patches.diff`): `hactool.manifest` with `UTF-8`, +embedded through `hactool.rc` (Makefile rule `hactool.res.o`, `windres`). hactool converts argv, `HOME` and +`USERPROFILE` from UTF-8 to UTF-16 (`filepath.c`); without the manifest they arrive in the ANSI code page, and a +non-ASCII user profile or path stops it with `Failed to convert ... to UTF-16!`. Needs Windows 10 1903 or later. + +Rebuild: in an MSYS2 UCRT64 shell (`MSYSTEM=UCRT64 C:\msys64\usr\bin\bash.exe -lc ...`) in `toolchain\src\hactool`, +`make clean && make -j8`, then copy `hactool.exe` to `toolchain\hactool\`. gcc writes temporary files to `TMP`; where +it cannot (sandboxed shells), point `TMP` and `TEMP` at a writable folder. + Not applied: the `MNG_BASE_RANGE_OFFSET/SIZE` base-file hook. It uses `funopen` (BSD/macOS only); on Windows the base NCA must be extracted to a file first. From 02158f3f464c3f487d8b4b3486f18053a882b0c6 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sat, 24 Oct 2026 19:59:02 +0300 Subject: [PATCH 4/4] Quote paths with spaces on the Ninja command lines of build_game.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An install folder with a space in its name (or a user name with one) made the compile step fail: clang split -I temp/Игра TotK-/gen at the space. Include, library and tool paths and the link map path are now quoted when they hold a space. Co-Authored-By: Claude Opus 5.5 --- docs/installer.md | 2 +- tools/build_game.py | 29 ++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/docs/installer.md b/docs/installer.md index 8f240a76..8f895bb5 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -107,7 +107,7 @@ totk.ini.example, VERSION the default ini; the release string (git describe of - Paths may hold any Unicode characters (a Cyrillic user name puts the temp and install folders under it). Every exe the setup runs or builds reads its arguments and paths as UTF-8 through an `activeCodePage` manifest: hactool (`toolchain/hactool/unicode-patches.diff`), totk.exe (`runtime/totk.rc`), ninja (upstream). The Python steps write - their files and logs as UTF-8. + their files and logs as UTF-8, and `build_game.py` quotes paths with spaces on its Ninja command lines. - A game file whose data fails its hash checks or ends early stops the setup with `check.input_damaged`, which names the file and asks to copy it again (`extract.DamagedFile`). A failed merge checks the base Program NCA alone to tell which file is damaged. diff --git a/tools/build_game.py b/tools/build_game.py index 65493976..3f3aebd4 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -76,7 +76,7 @@ SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden', '-march=x86-64-v3'] RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3'] -LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Wl,--Map=$builddir/totk.map'] # the map feeds tools/prof_report.py +LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '"-Wl,--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end of M4; libvpx static (movies) VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator # --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead. @@ -118,6 +118,17 @@ def flag_path(p): return below_root(p).as_posix() +def cmd_arg(s): + """s for a Ninja command line: Ninja-escaped, and quoted when it holds a space (a user or folder name), as clang and + lld split their Windows command line at unquoted spaces.""" + s = s.replace('$', '$$') + return f'"{s}"' if ' ' in s else s + + +def cmd_path(p): + return cmd_arg(flag_path(p)) + + def read_layout(gen): """[(name, base, size)] from layout.json (same format as recomp/layout.py read(), without its NSO dependency).""" data = json.loads((gen / 'layout.json').read_text()) @@ -208,15 +219,15 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= """build.ninja text, and the object lists per module. With prebuilt the generated objects already exist in out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE.""" - inc = f'-I {flag_path(RUNTIME)} -I {flag_path(gen)}' - vma = f'{flag_path(DEPS / "include")} -I {flag_path(DEPS / "include" / "vma")}' if DEPS else flag_path(VMA_INCLUDE) - rt_inc = f'{inc} -I {flag_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers) - libs = [f'-L{flag_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS + inc = f'-I {cmd_path(RUNTIME)} -I {cmd_path(gen)}' + vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(VMA_INCLUDE) + rt_inc = f'{inc} -I {cmd_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers) + libs = [f'-L {cmd_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS lines = [ '# Written by tools/build_game.py; do not edit.', 'ninja_required_version = 1.10', f'builddir = {ninja_path(out)}', - f'cc = {CLANG.as_posix()}', + f'cc = {cmd_arg(CLANG.as_posix())}', f'genflags = {" ".join(GEN_FLAGS)} {inc}', f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}', f'ldflags = {" ".join(LINK_FLAGS)}', @@ -238,7 +249,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= ' description = CC $in', '', 'rule rc', - f' command = {WINDRES.as_posix()} -I {flag_path(RUNTIME)} -O coff $in -o $out', + f' command = {cmd_arg(WINDRES.as_posix())} -I {cmd_path(RUNTIME)} -O coff $in -o $out', ' description = RC $in', '', 'rule link', @@ -274,7 +285,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= exe = out / 'totk.exe' if make_lib: exe = out / LIB_NAME - lines += ['', 'rule ar', f' command = {AR.as_posix()} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every), + lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every), 'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', ''] return '\n'.join(lines), objs, exe rule = 'link' @@ -283,7 +294,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= # linker looks into the library, and the real probes would stay out of totk.exe. rule = 'link_lib' lines += ['', 'rule link_lib', - f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {flag_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out', + f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {cmd_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out', ' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out'] lines += ['', f'build {ninja_path(exe)}: {rule} ' + ' '.join(ninja_path(o) for o in every) + (f' | {ninja_path(runtime_lib)}' if runtime_lib else ''),