From afe3404434f7b850ced45601a090eca9a82e79a3 Mon Sep 17 00:00:00 2001 From: itaskaev2 <165683571+itaskaev2@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:02:25 +0100 Subject: [PATCH] Accept NSZ and XCZ game files in the setup The check step tells a compressed file by its content (an NSP or XCI holding NCZ files), whatever its name, and unpacks it into the temp folder with tools/ncz.py before the extraction reads it. Every NCA comes back byte for byte and is checked against the hash in its name. The file dialog has one filter for all game files, and the form says that compressed files are unpacked automatically, in every setup language. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + docs/installer.md | 16 ++- installer/cli.py | 4 +- installer/gui.py | 6 +- installer/layout.py | 3 +- installer/steps.py | 95 +++++++++++++-- installer/text/en.json | 15 ++- installer/text/pt-BR.json | 15 ++- installer/text/ru.json | 15 ++- installer/text/zh-Hans.json | 15 ++- tests/ncz_test.py | 173 +++++++++++++++++++++++++++ tools/make_release.py | 7 +- tools/ncz.py | 231 ++++++++++++++++++++++++++++++++++++ 13 files changed, 547 insertions(+), 49 deletions(-) create mode 100644 tests/ncz_test.py create mode 100644 tools/ncz.py diff --git a/README.md b/README.md index a6eee2b9..26abca65 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). - Native code, not emulation - Game updates 1.1.0, 1.2.1 and 1.4.3; 1.2.1 and 1.4.3 saves carry over between them +- Takes NSP and XCI game files, and compressed NSZ and XCZ as they are: the setup unpacks them itself - 60 fps by default; up to 144 fps or the monitor's refresh rate (experimental) - Custom resolutions (1080p, 2k...), supersampling and AMD FSR 1 upscaling - HDR10 output with peak, game and HUD brightness settings diff --git a/docs/installer.md b/docs/installer.md index 8a8e6b78..09f3f0d6 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -9,13 +9,14 @@ holds only the game; the installer and its tools stay outside it. 1. Downloads `TotK-Setup--windows.zip` from GitHub Releases (none is published yet), unpacks it anywhere and starts `TotK-Setup.exe`. 2. Picks: - - the base game (XCI or NSP) and the 1.1.0, 1.2.1 or 1.4.3 update (NSP); + - the base game (XCI or NSP) and the 1.1.0, 1.2.1 or 1.4.3 update (NSP); an NSZ or XCZ (compressed by the nsz + tool) works as it is, the setup unpacks it itself; - their `prod.keys`; - optionally the firmware folder (`system/Contents/registered` of an unpacked firmware or an emulator's install), needed for Chinese, Korean and Traditional Chinese text; - the game language and the install folder. 3. Waits for the setup, one time. The window shows each step and its progress. -4. Plays `totk.exe` from the install folder. The installer and the source NSP or XCI can be deleted. +4. Plays `totk.exe` from the install folder. The installer and the source NSP or XCI (or NSZ, XCZ) can be deleted. The install folder (`main.c` `portable_paths()` reads this layout; it stays portable): @@ -44,7 +45,7 @@ loads the saves 1.4.3 wrote. ## Setup steps -1. **Check** before writing anything: the files open with the keys, the update is 1.1.0, 1.2.1 or 1.4.3 (by hash, as +1. **Check** before writing anything: an NSZ or XCZ input is unpacked first (below), the files open with the keys, the update is 1.1.0, 1.2.1 or 1.4.3 (by hash, as `tools/extract.py`), the keys are new enough for the update, enough space is free on the target disk (about 36 GiB at the peak of a first install, 20 GiB after it), the CPU is at least x86-64-v2 (SSE4.2; with AVX2/FMA the faster `v3` build is used, otherwise the `v2` one), a Vulkan 1.3 GPU exists. Each failure is one clear message. @@ -213,8 +214,15 @@ bin/ totk_xlate, hactool, ninja, libSDL3.so.0 the file and asks to copy it again (`extract.DamagedFile`). A failed merge checks the base Program NCA alone to tell which file is damaged. - Unsigned exe: SmartScreen warns on first start. +- NSZ and XCZ input (game files compressed by the nsz tool: NCAs stored as NCZ, their sections decrypted and + compressed with zstd): the check step tells them by their content, not their name (an NSP or XCI holding NCZ files), + so the file dialog has one filter for all four. It unpacks each into `/unpacked/.nsp` + (`tools/ncz.py`), an NSP of the NSZ's files or of the XCZ's secure partition, and the extraction reads that. + Unpacking encrypts the sections again, so every NCA comes back byte for byte, and checks each against the hash its + name carries; a mismatch is `check.input_damaged`. It needs the unpacked size free in the temp folder (checked + first), runs at about 500 MB/s, and an interrupted setup reuses a finished NSP of the same file. The extract step + deletes the folder when it is done. An XCZ update is refused like an XCI one. ## Later - Re-warm the driver cache on start after a move, GPU or driver change. -- NSZ and XCZ input. diff --git a/installer/cli.py b/installer/cli.py index f95a0084..88acc4f3 100644 --- a/installer/cli.py +++ b/installer/cli.py @@ -47,8 +47,8 @@ def cancel(signum, frame): def main(argv=None): ap = Parser(prog='python -m installer', description=__doc__.splitlines()[0]) - ap.add_argument('--base', type=Path, help='base game, XCI or NSP') - ap.add_argument('--update', type=Path, help='the update NSP: 1.1.0, 1.2.1 or 1.4.3') + ap.add_argument('--base', type=Path, help='base game, XCI or NSP (or NSZ, XCZ: unpacked first)') + ap.add_argument('--update', type=Path, help='the update NSP or NSZ: 1.1.0, 1.2.1 or 1.4.3') ap.add_argument('--keys', type=Path, help='prod.keys') ap.add_argument('--firmware', type=Path, help='Switch firmware folder (Contents/registered) for the shared fonts ' 'of Chinese, Korean and Traditional Chinese text') diff --git a/installer/gui.py b/installer/gui.py index 93b3a0f4..ed2e48ed 100644 --- a/installer/gui.py +++ b/installer/gui.py @@ -225,6 +225,7 @@ class App: self.label(grid, 'ui.dest').grid(row=i + 1, column=0, sticky='w', pady=4) ttk.Entry(grid, textvariable=self.vars['dest']).grid(row=i + 1, column=1, sticky='ew', pady=4) self.button(grid, 'ui.browse', self.pick_dest).grid(row=i + 1, column=2, padx=(8, 0)) + self.label(f, 'ui.compressed_help', wraplength=620, justify='left').pack(anchor='w', pady=(8, 0)) self.label(f, 'ui.firmware_help', wraplength=620, justify='left').pack(anchor='w', pady=(8, 0)) self.update_hint = ttk.Label(f, wraplength=620, justify='left') self.update_hint.pack(anchor='w', pady=(8, 0)) @@ -303,9 +304,8 @@ class App: self.fit() def pick_file(self, name, pick_id): - types = [(self.cat('ui.type_game'), '*.nsp *.xci'), (self.cat('ui.type_all'), '*.*')] - if name == 'update': - types = [(self.cat('ui.type_nsp'), '*.nsp'), (self.cat('ui.type_all'), '*.*')] + # One filter for every game file: the setup tells an NSZ or XCZ from a plain file by its content (Check.unpack) + types = [(self.cat('ui.type_game'), '*.nsp *.xci *.nsz *.xcz')] if name == 'keys': types = [(self.cat('ui.type_keys'), '*.keys'), (self.cat('ui.type_all'), '*.*')] path = filedialog.askopenfilename(title=self.cat(pick_id), filetypes=types) diff --git a/installer/layout.py b/installer/layout.py index 5b5c498a..0c6995d8 100644 --- a/installer/layout.py +++ b/installer/layout.py @@ -90,7 +90,8 @@ class Layout: def missing(self, cpu='v3'): """Paths the steps need that do not exist. cpu picks the runtime library the host will use.""" lib = self.runtime_lib_for(cpu) - need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py', + need = [self.python, self.tools / 'extract.py', self.tools / 'ncz.py', self.tools / 'build_game.py', + self.tools / 'prepare_shaders.py', self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.tools / 'ultrawide_layout.py', self.tools / 'button_icons.py', self.tools / 'vendor' / 'macnativegames' / 'extract_shared_fonts.py', diff --git a/installer/steps.py b/installer/steps.py index 4c9f2ea5..57aef6bf 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -9,10 +9,11 @@ with the bundled clang (hostcheck.c). Install folder: exefs/, romfs/ (romfs.bin, index.json), profile/, shared-fonts/ (Chinese and Korean text), totk.exe and its DLLs, romfs-index.bin, romfs-patches/, romfs-patches-xbox/, romfs-patches-xbox-label/, totk_xlate.exe, the shader state of totk.exe --prepare-shaders, totk.ini, totk-install.json. -Temp folder (/-): check/ (ExeFS of both inputs, the host check), gen/ (C), build/ (objects), -logs/. The cleanup keeps gen/, build/ and logs/: the next run lifts into the same gen/, where recomp/gen.py rewrites -only the chunks whose C changed, and Ninja in build/ recompiles only those. A change of the mods' code cheats then -costs the lift (about a minute) and the chunks it touches, not a full compile. +Temp folder (/-): check/ (ExeFS of both inputs, the host check), unpacked/ (the NSP of an NSZ +or XCZ input until the extraction is done), gen/ (C), build/ (objects), logs/. The cleanup keeps gen/, build/ and +logs/: the next run lifts into the same gen/, where recomp/gen.py rewrites only the chunks whose C changed, and Ninja +in build/ recompiles only those. A change of the mods' code cheats then costs the lift (about a minute) and the chunks +it touches, not a full compile. Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status (tools/exefs_patches.py) and rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild), one argument per line; totk.exe runs it at start @@ -36,6 +37,7 @@ from .runner import InstallError, Step sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'tools')) import extract # noqa: E402 +import ncz # noqa: E402 import shader_list # noqa: E402 from extract_exefs import (common_tickets, container_entries, load_keys, nca_ctr_key, nca_inventory, # noqa: E402 program_nca) @@ -174,6 +176,12 @@ def raise_if_damaged(error, name, path): raise InstallError('check.input_damaged', input=name, path=str(path), detail=str(error)) from None +def game_file(ctx, name): + """The file the extraction reads for the base or the update: the NSP that Check.unpack made from an NSZ or XCZ, + else the chosen file itself.""" + return getattr(ctx, 'unpacked', {}).get(name) or Path(getattr(ctx.options, name)) + + def game_version(ctx): """The update this install is built from: what the check identified, else the recorded one. An install made before 1.4.3 support records no version, and that install is 1.2.1: name it here rather than leaning on SUPPORTED's order.""" @@ -432,14 +440,73 @@ class Check(Step): raise InstallError('check.keys_unreadable') from None if 'header_key' not in keys: raise InstallError('check.keys_no_header_key') + self.unpack(ctx) total = 0 for name, is_update in (('base', False), ('update', True)): - path = Path(getattr(o, name)) + path = game_file(ctx, name) with path.open('rb') as f: total += self.open_input(f, path, keys, name, is_update) del keys return total + def unpack(self, ctx): + """Unpack an NSZ or XCZ (a game file compressed by the nsz tool) into /unpacked/.nsp with + tools/ncz.py, which checks every NCA against its hash; the extraction then reads that NSP. An earlier run's NSP + of the same file is reused. The extract step removes the folder once it is done.""" + o, rep, folder = ctx.options, ctx.reporter, ctx.temp / 'unpacked' + ctx.unpacked = {} + jobs = [] + for name in ('base', 'update'): + path = Path(getattr(o, name)) + try: + found = ncz.compressed(path) + if found is None: + continue + if name == 'update' and found[0] != 'nsz': + raise InstallError('check.update_not_nsp') + size = ncz.unpacked_size(path) + except ValueError as e: + raise_if_damaged(e, name, path) + raise InstallError('check.not_game_file', input=name) from None + st = path.stat() + jobs.append((name, path, size, f'{path.resolve()}\n{st.st_size}\n{st.st_mtime_ns}\n{size}\n')) + if not jobs: + remove(folder) + return + done = [] + for name, path, size, tag in jobs: + target, stamp = folder / f'{name}.nsp', folder / f'{name}.source' + try: + if target.stat().st_size == size and stamp.read_text(encoding='utf-8') == tag: + done.append(name) + ctx.unpacked[name] = target + except OSError: + pass + need = sum(size for name, _, size, _ in jobs if name not in done) + free, _ = free_bytes(folder) + rep.log(f'unpack: {", ".join(name for name, *_ in jobs)}: {need / GiB:.1f} GiB to write, ' + f'{free / GiB:.1f} free') + if need > free: + raise InstallError('check.disk_space', path=str(folder), need_gb=round(need / 1e9, 1), + free_gb=round(free / 1e9, 1)) + for name, path, size, tag in jobs: + if name in done: + continue + target, stamp = folder / f'{name}.nsp', folder / f'{name}.source' + remove(stamp) + try: + folder.mkdir(parents=True, exist_ok=True) + ncz.unpack(path, target, lambda d, t, n=name: rep.progress('check', 'unpack_' + n, d, t, 'bytes')) + stamp.write_text(tag, encoding='utf-8') + except ncz.DamagedFile as e: + raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None + except ValueError as e: + raise_if_damaged(e, name, path) + raise InstallError('check.unpack_failed', input=name, detail=str(e)) from None + except OSError as e: + raise InstallError('check.unpack_failed', input=name, detail=str(e)) from None + ctx.unpacked[name] = target + @staticmethod def open_input(f, path, keys, name, is_update): try: @@ -520,9 +587,10 @@ class Check(Step): out = work / name remove(out) try: - extract.extract_exefs(Path(getattr(o, name)), Path(o.keys), out, logs, update=is_update) + extract.extract_exefs(game_file(ctx, name), Path(o.keys), out, logs, update=is_update) except extract.DamagedFile as e: - raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None + raise InstallError('check.input_damaged', input=name, path=str(getattr(o, name)), + detail=e.detail) from None except ValueError: raise InstallError('check.exefs_failed', input=name, log=str(logs / f'extract_exefs-{name}.log')) \ from None @@ -641,9 +709,9 @@ class Extract(Step): def run(self, ctx): o, dest, rep = ctx.options, ctx.dest, ctx.reporter expected = expected_update(ctx) - base, update, keys = Path(o.base), Path(o.update), Path(o.keys) + keys = Path(o.keys) inputs = {} - for name, path in (('base', base), ('update', update)): + for name, path in (('base', Path(o.base)), ('update', Path(o.update))): known = ctx.state['inputs'].get(name, {}) st = path.stat() if known.get('size') == st.st_size and known.get('mtime_ns') == st.st_mtime_ns and known.get('sha256'): @@ -653,6 +721,7 @@ class Extract(Step): inputs[name] = {'name': path.name, 'size': st.st_size, 'mtime_ns': st.st_mtime_ns, 'sha256': digest} ctx.state['inputs'] = inputs ctx.save() + base, update = game_file(ctx, 'base'), game_file(ctx, 'update') dest.mkdir(parents=True, exist_ok=True) for stale in dest.glob('update-merge-*'): # a killed merge leaves its NCA copies (and key copies) behind @@ -664,7 +733,7 @@ class Extract(Step): try: extract.extract_exefs(update, keys, ctx.temp / 'check' / 'update', ctx.temp / 'logs', update=True) except extract.DamagedFile as e: - raise InstallError('check.input_damaged', input='update', path=str(e.path), detail=e.detail) from None + raise InstallError('check.input_damaged', input='update', path=str(o.update), detail=e.detail) from None except ValueError: raise InstallError('check.exefs_failed', input='update', log=str(ctx.temp / 'logs' / 'extract_exefs-update.log')) from None @@ -686,6 +755,7 @@ class Extract(Step): raise InstallError('extract.romfs_mismatch', version=game_version(ctx), size=report['romfs_size'], sha256=report['romfs_sha256'][:16]) (dest / 'profile').mkdir(exist_ok=True) + remove(ctx.temp / 'unpacked') # the NSPs unpacked from an NSZ or XCZ (Check.unpack) st = (romfs / 'romfs.bin').stat() return {'romfs_size': st.st_size, 'romfs_mtime_ns': st.st_mtime_ns} @@ -707,7 +777,8 @@ class Extract(Step): try: extract.merge_update_romfs(base, update, keys, ctx.layout.hactool, romfs, logs, phase=phase) except extract.DamagedFile as e: - raise InstallError('check.input_damaged', input='base' if e.path == base else 'update', path=str(e.path), + name = 'base' if e.path == base else 'update' + raise InstallError('check.input_damaged', input=name, path=str(getattr(ctx.options, name)), detail=e.detail) from None except ValueError as e: raise InstallError('extract.romfs_merge_failed', detail=str(e), log=str(logs / 'merge_romfs-' @@ -1040,7 +1111,7 @@ class Cleanup(Step): rep.progress('cleanup', 'temp', force=True) for stale in dest.glob('update-merge-*'): remove(stale) - for name in ('check', 'shader_list.tsv', 'shared-fonts'): # gen/ and build/ stay for the next rebuild + for name in ('check', 'unpacked', 'shader_list.tsv', 'shared-fonts'): # gen/ and build/ stay for the next rebuild remove(ctx.temp / name) romfs_problem(dest / 'romfs', expected_update(ctx)) write_rebuild_command(ctx) diff --git a/installer/text/en.json b/installer/text/en.json index b054e806..bd2aa67a 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -11,6 +11,8 @@ "phase.check.verify": "Checking the files already installed", "phase.check.verify_romfs": "Checking the installed game data", "phase.check.inputs": "Opening the game files with the keys", + "phase.check.unpack_base": "Unpacking the compressed base game", + "phase.check.unpack_update": "Unpacking the compressed update", "phase.check.versions": "Checking the game versions", "phase.fonts.extract": "Extracting the shared fonts from the firmware", "phase.fonts.copy": "Copying the shared fonts into the install folder", @@ -58,14 +60,15 @@ "check.keys_no_header_key": "The keys file has no header_key. Dump prod.keys from your console again.", "check.keys_wrong": "The keys cannot open the {input} file. Use the prod.keys of your console.", "check.keys_too_old": "The keys file is too old for the {input} file (needs key generation {generation}). Dump prod.keys from a console with newer firmware.", - "check.not_game_file": "The {input} file is not an XCI or NSP game file.", + "check.not_game_file": "The {input} file is not an XCI, NSP, XCZ or NSZ game file.", "check.file_unreadable": "The {input} file cannot be read: {detail}", + "check.unpack_failed": "The compressed {input} file could not be unpacked: {detail}", "check.ticket_personalized": "The {input} file carries a personalized ticket; a common ticket is needed.", "check.ticket_missing": "The {input} file lacks the ticket its content needs.", "check.base_not_totk": "The base game file is not The Legend of Zelda: Tears of the Kingdom ({detail}).", "check.base_is_update": "The base game file is an update. Choose the base game NSP or XCI (version 1.0.0) here and this file as the update.", "check.base_combined": "The base game file holds several packages (base game, update or DLC in one file). Choose a base game dump that contains only the game.", - "check.update_not_nsp": "The update must be an NSP file.", + "check.update_not_nsp": "The update must be an NSP or NSZ file.", "check.update_is_base": "The update file holds a base game. Choose the 1.1.0 or 1.2.1 or 1.4.3 update NSP.", "check.update_not_totk": "The update file is not a Tears of the Kingdom update ({detail}).", "check.exefs_failed": "The game code of the {input} file failed its checks; see {log}.", @@ -109,8 +112,8 @@ "ui.intro": "Builds a playable Tears of the Kingdom folder from your own game files. This takes a while and runs once.", "ui.notice": "The folder this setup builds contains the game. It is for your own use only; do not share or upload it.", "ui.keys_private": "The keys file is read in place. Its contents are never shown, copied or stored.", - "ui.base": "Base game (NSP or XCI)", - "ui.update": "Update 1.1.0 or 1.2.1 or 1.4.3 (NSP)", + "ui.base": "Base game (NSP, XCI, NSZ or XCZ)", + "ui.update": "Update 1.1.0 or 1.2.1 or 1.4.3 (NSP or NSZ)", "ui.keys": "prod.keys", "ui.firmware": "Firmware folder (optional)", "ui.game_language": "Game language", @@ -122,8 +125,8 @@ "ui.pick_firmware": "Choose the Switch firmware folder (Contents/registered)", "ui.pick_dest": "Choose the install folder", "ui.firmware_help": "For Chinese, Korean and Traditional Chinese text: the firmware's Contents/registered folder, for example from Ryujinx or Yuzu. The setup copies the Switch system fonts from it into the install folder.", - "ui.type_game": "Game files (NSP, XCI)", - "ui.type_nsp": "NSP files", + "ui.compressed_help": "Compressed NSZ and XCZ files work as they are: the setup unpacks them by itself (until the game data is extracted, that needs as much free space as the unpacked files take).", + "ui.type_game": "Game files (NSP, XCI, NSZ, XCZ)", "ui.type_keys": "Key files (*.keys)", "ui.type_all": "All files", "ui.start": "Start", diff --git a/installer/text/pt-BR.json b/installer/text/pt-BR.json index 0cdcbc0a..083bb282 100644 --- a/installer/text/pt-BR.json +++ b/installer/text/pt-BR.json @@ -11,6 +11,8 @@ "phase.check.verify": "Verificando os arquivos já instalados", "phase.check.verify_romfs": "Verificando os dados do jogo instalados", "phase.check.inputs": "Abrindo os arquivos do jogo com as chaves", + "phase.check.unpack_base": "Descompactando o jogo base compactado", + "phase.check.unpack_update": "Descompactando a atualização compactada", "phase.check.versions": "Verificando as versões do jogo", "phase.fonts.extract": "Extraindo as fontes compartilhadas do firmware", "phase.fonts.copy": "Copiando as fontes compartilhadas para a pasta de instalação", @@ -58,14 +60,15 @@ "check.keys_no_header_key": "O arquivo de chaves não tem header_key. Extraia o prod.keys do seu console de novo.", "check.keys_wrong": "As chaves não abrem o arquivo de {input}. Use o prod.keys do seu console.", "check.keys_too_old": "O arquivo de chaves é antigo demais para o arquivo de {input} (precisa da geração de chaves {generation}). Extraia o prod.keys de um console com firmware mais novo.", - "check.not_game_file": "O arquivo de {input} não é um arquivo de jogo XCI ou NSP.", + "check.not_game_file": "O arquivo de {input} não é um arquivo de jogo XCI, NSP, XCZ ou NSZ.", "check.file_unreadable": "Não foi possível ler o arquivo de {input}: {detail}", + "check.unpack_failed": "Não foi possível descompactar o arquivo compactado de {input}: {detail}", "check.ticket_personalized": "O arquivo de {input} tem um ticket personalizado; é preciso um ticket comum.", "check.ticket_missing": "O arquivo de {input} não tem o ticket de que o conteúdo precisa.", "check.base_not_totk": "O arquivo do jogo base não é The Legend of Zelda: Tears of the Kingdom ({detail}).", "check.base_is_update": "O arquivo do jogo base é uma atualização. Escolha aqui o NSP ou XCI do jogo base (versão 1.0.0) e este arquivo como a atualização.", "check.base_combined": "O arquivo do jogo base contém vários pacotes (jogo base, atualização ou DLC em um só arquivo). Escolha um dump do jogo base que contenha só o jogo.", - "check.update_not_nsp": "A atualização precisa ser um arquivo NSP.", + "check.update_not_nsp": "A atualização precisa ser um arquivo NSP ou NSZ.", "check.update_is_base": "O arquivo de atualização contém um jogo base. Escolha o NSP da atualização 1.1.0, 1.2.1 ou 1.4.3.", "check.update_not_totk": "O arquivo de atualização não é uma atualização de Tears of the Kingdom ({detail}).", "check.exefs_failed": "O código do jogo no arquivo de {input} falhou nas verificações; veja {log}.", @@ -109,8 +112,8 @@ "ui.intro": "Monta uma pasta jogável de Tears of the Kingdom a partir dos seus próprios arquivos do jogo. Isso demora um pouco e só precisa ser feito uma vez.", "ui.notice": "A pasta que esta instalação monta contém o jogo. Ela é só para seu uso pessoal; não compartilhe nem envie para a internet.", "ui.keys_private": "O arquivo de chaves é lido no lugar. O conteúdo dele nunca é mostrado, copiado nem guardado.", - "ui.base": "Jogo base (NSP ou XCI)", - "ui.update": "Atualização 1.1.0, 1.2.1 ou 1.4.3 (NSP)", + "ui.base": "Jogo base (NSP, XCI, NSZ ou XCZ)", + "ui.update": "Atualização 1.1.0, 1.2.1 ou 1.4.3 (NSP ou NSZ)", "ui.keys": "prod.keys", "ui.firmware": "Pasta de firmware (opcional)", "ui.game_language": "Idioma do jogo", @@ -122,8 +125,8 @@ "ui.pick_firmware": "Escolha a pasta de firmware do Switch (Contents/registered)", "ui.pick_dest": "Escolha a pasta de instalação", "ui.firmware_help": "Para textos em chinês, coreano e chinês tradicional: a pasta Contents/registered do firmware, por exemplo do Ryujinx ou do Yuzu. A instalação copia dela as fontes do sistema do Switch para a pasta de instalação.", - "ui.type_game": "Arquivos de jogo (NSP, XCI)", - "ui.type_nsp": "Arquivos NSP", + "ui.compressed_help": "Arquivos compactados NSZ e XCZ funcionam do jeito que estão: a instalação os descompacta sozinha (para isso, até os dados do jogo serem extraídos, ela precisa de tanto espaço livre quanto ocupam os arquivos descompactados).", + "ui.type_game": "Arquivos de jogo (NSP, XCI, NSZ, XCZ)", "ui.type_keys": "Arquivos de chaves (*.keys)", "ui.type_all": "Todos os arquivos", "ui.start": "Iniciar", diff --git a/installer/text/ru.json b/installer/text/ru.json index e583dd71..4102627b 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -11,6 +11,8 @@ "phase.check.verify": "Проверка уже установленных файлов", "phase.check.verify_romfs": "Проверка установленных данных игры", "phase.check.inputs": "Открытие игровых файлов с помощью ключей", + "phase.check.unpack_base": "Распаковка сжатой базовой игры", + "phase.check.unpack_update": "Распаковка сжатого обновления", "phase.check.versions": "Проверка версий игры", "phase.fonts.extract": "Извлечение общих шрифтов из прошивки", "phase.fonts.copy": "Копирование общих шрифтов в папку установки", @@ -58,14 +60,15 @@ "check.keys_no_header_key": "В файле ключей нет header_key. Снова получите prod.keys со своей консоли.", "check.keys_wrong": "Эти ключи не открывают файл «{input}». Используйте prod.keys со своей консоли.", "check.keys_too_old": "Файл ключей слишком старый для файла «{input}» (нужно поколение ключей {generation}). Получите prod.keys с консоли с более новой прошивкой.", - "check.not_game_file": "Файл «{input}» не является игровым файлом XCI или NSP.", + "check.not_game_file": "Файл «{input}» не является игровым файлом XCI, NSP, XCZ или NSZ.", "check.file_unreadable": "Не удаётся прочитать файл «{input}»: {detail}", + "check.unpack_failed": "Не удалось распаковать сжатый файл «{input}»: {detail}", "check.ticket_personalized": "В файле «{input}» личный тикет; нужен общий тикет.", "check.ticket_missing": "В файле «{input}» нет тикета, который нужен его содержимому.", "check.base_not_totk": "Файл базовой игры — не The Legend of Zelda: Tears of the Kingdom ({detail}).", "check.base_is_update": "Файл базовой игры — это обновление. Выберите здесь NSP или XCI базовой игры (версия 1.0.0), а этот файл укажите как обновление.", "check.base_combined": "Файл базовой игры содержит несколько пакетов (игра, обновление или DLC в одном файле). Выберите дамп, в котором только базовая игра.", - "check.update_not_nsp": "Обновление должно быть файлом NSP.", + "check.update_not_nsp": "Обновление должно быть файлом NSP или NSZ.", "check.update_is_base": "Файл обновления содержит базовую игру. Выберите NSP с обновлением 1.1.0 или 1.2.1 или 1.4.3.", "check.update_not_totk": "Файл обновления — не обновление Tears of the Kingdom ({detail}).", "check.exefs_failed": "Код игры из файла «{input}» не прошёл проверку; подробности в {log}.", @@ -109,8 +112,8 @@ "ui.intro": "Собирает готовую к запуску папку с Tears of the Kingdom из ваших игровых файлов. Занимает много времени и выполняется один раз.", "ui.notice": "Папка, которую создаёт установка, содержит игру. Она предназначена только для личного использования: не передавайте и не выкладывайте её.", "ui.keys_private": "Файл ключей читается на месте. Его содержимое не показывается, не копируется и не сохраняется.", - "ui.base": "Базовая игра (NSP или XCI)", - "ui.update": "Обновление 1.1.0 или 1.2.1 или 1.4.3 (NSP)", + "ui.base": "Базовая игра (NSP, XCI, NSZ или XCZ)", + "ui.update": "Обновление 1.1.0 или 1.2.1 или 1.4.3 (NSP или NSZ)", "ui.keys": "prod.keys", "ui.firmware": "Папка прошивки (необязательно)", "ui.game_language": "Язык игры", @@ -122,8 +125,8 @@ "ui.pick_firmware": "Выберите папку прошивки Switch (Contents/registered)", "ui.pick_dest": "Выберите папку установки", "ui.firmware_help": "Для текста на китайском и корейском: папка Contents/registered прошивки, например из Ryujinx или Yuzu. Установка скопирует из неё системные шрифты Switch в папку установки.", - "ui.type_game": "Файлы игры (NSP, XCI)", - "ui.type_nsp": "Файлы NSP", + "ui.compressed_help": "Сжатые файлы NSZ и XCZ подходят как есть: установка сама их распакует (для этого, пока извлекаются данные игры, нужно столько свободного места, сколько занимают распакованные файлы).", + "ui.type_game": "Файлы игры (NSP, XCI, NSZ, XCZ)", "ui.type_keys": "Файлы ключей (*.keys)", "ui.type_all": "Все файлы", "ui.start": "Начать", diff --git a/installer/text/zh-Hans.json b/installer/text/zh-Hans.json index d858c38e..968fc260 100644 --- a/installer/text/zh-Hans.json +++ b/installer/text/zh-Hans.json @@ -11,6 +11,8 @@ "phase.check.verify": "检查已安装的文件", "phase.check.verify_romfs": "检查已安装的游戏数据", "phase.check.inputs": "用密钥打开游戏文件", + "phase.check.unpack_base": "正在解压压缩的游戏本体", + "phase.check.unpack_update": "正在解压压缩的更新", "phase.check.versions": "检查游戏版本", "phase.fonts.extract": "正在从固件中提取共享字体", "phase.fonts.copy": "正在将共享字体复制到安装文件夹", @@ -58,14 +60,15 @@ "check.keys_no_header_key": "密钥文件中没有 header_key。请从你的主机重新导出 prod.keys。", "check.keys_wrong": "这些密钥无法打开{input}文件。请使用你自己主机的 prod.keys。", "check.keys_too_old": "密钥文件对{input}文件来说太旧(需要密钥代数 {generation})。请从固件更新的主机导出 prod.keys。", - "check.not_game_file": "{input}文件不是 XCI 或 NSP 游戏文件。", + "check.not_game_file": "{input}文件不是 XCI、NSP、XCZ 或 NSZ 游戏文件。", "check.file_unreadable": "无法读取{input}文件:{detail}", + "check.unpack_failed": "无法解压压缩的{input}文件:{detail}", "check.ticket_personalized": "{input}文件带有个人化票据(personalized ticket);需要通用票据(common ticket)。", "check.ticket_missing": "{input}文件缺少其内容所需的票据(ticket)。", "check.base_not_totk": "游戏本体文件不是《塞尔达传说 王国之泪》({detail})。", "check.base_is_update": "基础游戏文件是一个更新。请在这里选择基础游戏的 NSP 或 XCI(版本 1.0.0),并把此文件选为更新。", "check.base_combined": "基础游戏文件包含多个内容包(基础游戏、更新或 DLC 合在一个文件中)。请选择只包含游戏本体的基础游戏转储。", - "check.update_not_nsp": "更新必须是 NSP 文件。", + "check.update_not_nsp": "更新必须是 NSP 或 NSZ 文件。", "check.update_is_base": "更新文件里是游戏本体。请选择 1.1.0、1.2.1 或 1.4.3 的更新 NSP。", "check.update_not_totk": "更新文件不是《王国之泪》的更新({detail})。", "check.exefs_failed": "{input}文件的游戏代码未通过检查;详见 {log}。", @@ -109,8 +112,8 @@ "ui.intro": "用你自己的游戏文件构建一个可以游玩的《王国之泪》文件夹。耗时较长,只需运行一次。", "ui.notice": "此安装程序构建的文件夹包含游戏本身,仅供你个人使用;请勿分享或上传。", "ui.keys_private": "密钥文件只在原位置读取,其内容绝不会被显示、复制或保存。", - "ui.base": "游戏本体(NSP 或 XCI)", - "ui.update": "1.1.0、1.2.1 或 1.4.3 更新(NSP)", + "ui.base": "游戏本体(NSP、XCI、NSZ 或 XCZ)", + "ui.update": "1.1.0、1.2.1 或 1.4.3 更新(NSP 或 NSZ)", "ui.keys": "prod.keys", "ui.firmware": "固件文件夹(可选)", "ui.game_language": "游戏语言", @@ -122,8 +125,8 @@ "ui.pick_firmware": "选择 Switch 固件文件夹(Contents/registered)", "ui.pick_dest": "选择安装文件夹", "ui.firmware_help": "用于中文、韩文和繁体中文文本:固件的 Contents/registered 文件夹,例如来自 Ryujinx 或 Yuzu。安装程序会从中把 Switch 系统字体复制到安装文件夹。", - "ui.type_game": "游戏文件(NSP、XCI)", - "ui.type_nsp": "NSP 文件", + "ui.compressed_help": "压缩的 NSZ 和 XCZ 文件可以直接使用:安装程序会自行解压(在游戏数据提取完成之前,需要与解压后文件大小相同的可用空间)。", + "ui.type_game": "游戏文件(NSP、XCI、NSZ、XCZ)", "ui.type_keys": "密钥文件(*.keys)", "ui.type_all": "所有文件", "ui.start": "开始", diff --git a/tests/ncz_test.py b/tests/ncz_test.py new file mode 100644 index 00000000..106f1a0d --- /dev/null +++ b/tests/ncz_test.py @@ -0,0 +1,173 @@ +"""tools/ncz.py and the installer's unpacking of NSZ input (installer/steps.py, Check.unpack) on small made-up files: +an NCA with an AES-CTR section and a plain one, stored as an NCZ with one zstd stream or as NCZBLOCK blocks. + +Run: .venv-py314\\Scripts\\python tests/ncz_test.py +""" +import hashlib +import io +import os +import struct +import sys +import tempfile +import unittest +from compression import zstd +from pathlib import Path +from types import SimpleNamespace + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / 'tools')) +import ncz # noqa: E402 +from installer import steps # noqa: E402 +from installer.events import Reporter # noqa: E402 + +KEY, COUNTER = bytes(range(16)), bytes(range(16, 32)) +# (offset, size, crypto type): an AES-CTR section, then a plain one +SECTIONS = ((0x4000, 0x9000, 3), (0xD000, 0x3200, 1)) + + +def pfs0(files): + """An NSP of (name, data) pairs.""" + strings = b''.join(name.encode() + b'\0' for name, _ in files) + strings += bytes(-(16 + 0x18 * len(files) + len(strings)) % 0x20) + head = b'PFS0' + struct.pack('Q', 0x4000 >> 4) + secret = Cipher(algorithms.AES(KEY), modes.CTR(counter)).encryptor().update(plain) + header = rand(0x4000) + return header + secret + rest, plain + rest + + +def ncz_file(nca, body, blocks=False): + out = nca[:0x4000] + b'NCZSECTN' + struct.pack(' len(p) // 2 else p for p in pieces] # random blocks stay as they are + out += b'NCZBLOCK' + bytes([2, 1, 0, exponent]) + struct.pack('/unpacked/base.nsp, a plain NSP update is read in place, a second run + reuses the NSP.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + base = Path(self.tmp.name) + nca, body = made_up_nca() + name = hashlib.sha256(nca).hexdigest()[:32] + self.base, self.update = base / 'base.nsz', base / 'update.nsp' + self.base.write_bytes(pfs0([(name + '.ncz', ncz_file(nca, body))])) + self.update.write_bytes(pfs0([('a.tik', b'x' * 64)])) + self.expected = pfs0([(name + '.nca', nca)]) + self.out = io.StringIO() + self.ctx = SimpleNamespace(options=SimpleNamespace(base=str(self.base), update=str(self.update)), + temp=base / 'temp', reporter=Reporter(out=self.out, log_path=base / 'x.log')) + + def tearDown(self): + self.ctx.reporter.close() + self.tmp.cleanup() + + def test_unpacks_the_base_and_reuses_it(self): + steps.Check().unpack(self.ctx) + target = self.ctx.temp / 'unpacked' / 'base.nsp' + self.assertEqual(target.read_bytes(), self.expected) + self.assertEqual(steps.game_file(self.ctx, 'base'), target) + self.assertEqual(steps.game_file(self.ctx, 'update'), self.update) + self.assertIn('"phase": "unpack_base"', self.out.getvalue()) + mtime = target.stat().st_mtime_ns + steps.Check().unpack(self.ctx) + self.assertEqual(target.stat().st_mtime_ns, mtime) + + def test_content_decides_not_the_name(self): + renamed = self.base.with_name('base.nsp') + self.base.rename(renamed) + self.ctx.options.base = str(renamed) + steps.Check().unpack(self.ctx) + self.assertEqual((self.ctx.temp / 'unpacked' / 'base.nsp').read_bytes(), self.expected) + + def test_damaged_base_names_the_chosen_file(self): + data = bytearray(self.base.read_bytes()) + data[-100] ^= 0xFF + self.base.write_bytes(bytes(data)) + with self.assertRaises(steps.InstallError) as e: + steps.Check().unpack(self.ctx) + self.assertIn(e.exception.id, ('check.input_damaged', 'check.unpack_failed')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/make_release.py b/tools/make_release.py index 814eff63..c5fa64a0 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -105,9 +105,10 @@ LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32' LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. -TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py', 'hostmem.py', - 'make_romfs_patches.py', 'pack_shader_cache.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', - 'shader_list.py', 'ultrawide_layout.py', 'cheats.py', 'exefs_patches.py'] +TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py', + 'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py', + 'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py', + 'exefs_patches.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon diff --git a/tools/ncz.py b/tools/ncz.py new file mode 100644 index 00000000..d41dfe4d --- /dev/null +++ b/tools/ncz.py @@ -0,0 +1,231 @@ +"""Unpack an NSZ or XCZ (a game file compressed by the nsz tool) into a plain NSP. + +An NSZ is an NSP whose NCAs are stored as NCZ files: the first 0x4000 bytes of the NCA (its encrypted header) as they +are, then an NCZSECTN table of the NCA's sections with the AES-CTR key and counter of each, then the rest of the NCA +decrypted and compressed with zstd, either as one stream or as an NCZBLOCK table of independently compressed blocks. +An XCZ is an XCI whose secure partition holds NCZ files. Unpacking encrypts the sections again, so every NCA comes back +byte for byte; its file name is the first half of its SHA-256, which is checked. The output is an NSP of the secure +partition (an XCZ) or of the NSZ's files, with .ncz renamed to .nca. Format: nicoboss/nsz, nsz/NszDecompressor.py +(__decompressNcz) and nsz/BlockDecompressorReader.py; this is our own code. + +usage: ncz.py . Runs under .venv-py314 (compression.zstd). +""" +import hashlib +import io +import struct +import sys +from compression import zstd +from pathlib import Path + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +sys.path.insert(0, str(Path(__file__).resolve().parent / 'vendor' / 'macnativegames')) +from extract_exefs import container_entries, read_at, require # noqa: E402 + +HEADER = 0x4000 # the NCA header, kept encrypted and uncompressed +CHUNK = 1 << 20 +CRYPTO_CTR = (3, 4) # AES-CTR and BKTR sections; the other types are stored as they are + + +class DamagedFile(ValueError): + """An NCZ whose unpacked NCA does not match its name's hash, or whose data ends early.""" + + def __init__(self, path, detail): + self.path, self.detail = Path(path), detail + super().__init__(f'{self.path} is damaged or incomplete ({detail})') + + +def compressed(path): + """('nsz' or 'xcz', entries) for a game file holding NCZ files, else None. Raises ValueError for a file that is + neither an NSP nor an XCI.""" + with Path(path).open('rb') as f: + kind, entries = container_entries(f, Path(path).stat().st_size) + if not any(name.lower().endswith('.ncz') for name, _, _ in entries): + return None + return ('nsz' if kind == 'nsp' else 'xcz'), entries + + +def sections(f, start): + """[(offset, size, crypto type, key, counter)] of the NCZ at start, and the offset of its data after the table.""" + head = read_at(f, start + HEADER, 16) + require(head[:8] == b'NCZSECTN', 'NCZ without its NCZSECTN table') + count = struct.unpack_from('= HEADER, 'NCZ section inside the NCA header') + for (a, size, *_), (b, *_) in zip(out, out[1:]): + require(a + size <= b, 'Overlapping NCZ sections') + return out, start + HEADER + 16 + count * 0x40 + + +def nca_size(f, start): + table, _ = sections(f, start) + return table[-1][0] + table[-1][1] + + +def layout(source): + """(entries, NSP header, NSP size) of the NSP that unpack writes for the NSZ or XCZ source: its files in order, + .ncz renamed to .nca, the PFS0 table padded to 0x20 bytes.""" + _, entries = compressed(source) + with Path(source).open('rb') as f: + sizes = [nca_size(f, start) if name.lower().endswith('.ncz') else length for name, start, length in entries] + names = [name[:-4] + '.nca' if name.lower().endswith('.ncz') else name for name, _, _ in entries] + strings = b''.join(n.encode('utf-8') + b'\0' for n in names) + strings += bytes(-(16 + 0x18 * len(names) + len(strings)) % 0x20) + head = b'PFS0' + struct.pack(' at), total) + n = min(len(pending), stop - at) + crypto = section and section[2] in CRYPTO_CTR + if crypto and at + n < stop: + n -= n % 16 # the counter of the next piece starts at a whole AES block + if not n: + break + data, pending = pending[:n], pending[n:] + if crypto: + require(at % 16 == 0, 'Unaligned NCZ section') + counter = section[4][:8] + struct.pack('>Q', at >> 4) + data = Cipher(algorithms.AES(section[3]), modes.CTR(counter)).encryptor().update(data) + out.write(data) + digest.update(data) + at += n + progress(n) + if at == total: + break + except (zstd.ZstdError, EOFError) as e: # EOFError: the stream ends before its end-of-frame marker + raise DamagedFile(source, f'{name}: {e}') from None + if at != total or pending: + raise DamagedFile(source, f'{name}: {at - HEADER:,} of {total - HEADER:,} bytes') + stem = name[:-4] + if len(stem) == 32 and digest.hexdigest()[:32] != stem.lower(): + raise DamagedFile(source, f'{name}: hash mismatch') + + +def unpack(source, target, progress=lambda done, total: None): + """Write the NSP of the NSZ or XCZ source to target (through target.part). progress(done, total) in bytes.""" + source, target = Path(source), Path(target) + kind = compressed(source)[0] + entries, head, total = layout(source) + part = target.with_name(target.name + '.part') + done = 0 + + def step(n): + nonlocal done + done += n + progress(done, total) + + with source.open('rb') as f, part.open('wb') as out: + out.write(head) + step(len(head)) + for name, start, length in entries: + if name.lower().endswith('.ncz'): + unpack_ncz(f, start, start + length, out, name, source, step) + continue + for at in range(start, start + length, CHUNK): + data = read_at(f, at, min(CHUNK, start + length - at)) + out.write(data) + step(len(data)) + part.replace(target) + return kind + + +def main(): + source, target = sys.argv[1:3] + shown = [-1] + + def progress(done, total): + percent = done * 100 // total + if percent != shown[0]: + shown[0] = percent + print(f'\r{percent}%', end='', flush=True) + + kind = unpack(source, target, progress) + print(f'\n{kind}: wrote {target}') + + +if __name__ == '__main__': + main()