diff --git a/README.md b/README.md index a6eee2b9..26abca65 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). - Native code, not emulation - Game updates 1.1.0, 1.2.1 and 1.4.3; 1.2.1 and 1.4.3 saves carry over between them +- Takes NSP and XCI game files, and compressed NSZ and XCZ as they are: the setup unpacks them itself - 60 fps by default; up to 144 fps or the monitor's refresh rate (experimental) - Custom resolutions (1080p, 2k...), supersampling and AMD FSR 1 upscaling - HDR10 output with peak, game and HUD brightness settings diff --git a/docs/installer.md b/docs/installer.md index 8a8e6b78..09f3f0d6 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -9,13 +9,14 @@ holds only the game; the installer and its tools stay outside it. 1. Downloads `TotK-Setup--windows.zip` from GitHub Releases (none is published yet), unpacks it anywhere and starts `TotK-Setup.exe`. 2. Picks: - - the base game (XCI or NSP) and the 1.1.0, 1.2.1 or 1.4.3 update (NSP); + - the base game (XCI or NSP) and the 1.1.0, 1.2.1 or 1.4.3 update (NSP); an NSZ or XCZ (compressed by the nsz + tool) works as it is, the setup unpacks it itself; - their `prod.keys`; - optionally the firmware folder (`system/Contents/registered` of an unpacked firmware or an emulator's install), needed for Chinese, Korean and Traditional Chinese text; - the game language and the install folder. 3. Waits for the setup, one time. The window shows each step and its progress. -4. Plays `totk.exe` from the install folder. The installer and the source NSP or XCI can be deleted. +4. Plays `totk.exe` from the install folder. The installer and the source NSP or XCI (or NSZ, XCZ) can be deleted. The install folder (`main.c` `portable_paths()` reads this layout; it stays portable): @@ -44,7 +45,7 @@ loads the saves 1.4.3 wrote. ## Setup steps -1. **Check** before writing anything: the files open with the keys, the update is 1.1.0, 1.2.1 or 1.4.3 (by hash, as +1. **Check** before writing anything: an NSZ or XCZ input is unpacked first (below), the files open with the keys, the update is 1.1.0, 1.2.1 or 1.4.3 (by hash, as `tools/extract.py`), the keys are new enough for the update, enough space is free on the target disk (about 36 GiB at the peak of a first install, 20 GiB after it), the CPU is at least x86-64-v2 (SSE4.2; with AVX2/FMA the faster `v3` build is used, otherwise the `v2` one), a Vulkan 1.3 GPU exists. Each failure is one clear message. @@ -213,8 +214,15 @@ bin/ totk_xlate, hactool, ninja, libSDL3.so.0 the file and asks to copy it again (`extract.DamagedFile`). A failed merge checks the base Program NCA alone to tell which file is damaged. - Unsigned exe: SmartScreen warns on first start. +- NSZ and XCZ input (game files compressed by the nsz tool: NCAs stored as NCZ, their sections decrypted and + compressed with zstd): the check step tells them by their content, not their name (an NSP or XCI holding NCZ files), + so the file dialog has one filter for all four. It unpacks each into `/unpacked/.nsp` + (`tools/ncz.py`), an NSP of the NSZ's files or of the XCZ's secure partition, and the extraction reads that. + Unpacking encrypts the sections again, so every NCA comes back byte for byte, and checks each against the hash its + name carries; a mismatch is `check.input_damaged`. It needs the unpacked size free in the temp folder (checked + first), runs at about 500 MB/s, and an interrupted setup reuses a finished NSP of the same file. The extract step + deletes the folder when it is done. An XCZ update is refused like an XCI one. ## Later - Re-warm the driver cache on start after a move, GPU or driver change. -- NSZ and XCZ input. diff --git a/installer/cli.py b/installer/cli.py index f95a0084..88acc4f3 100644 --- a/installer/cli.py +++ b/installer/cli.py @@ -47,8 +47,8 @@ def cancel(signum, frame): def main(argv=None): ap = Parser(prog='python -m installer', description=__doc__.splitlines()[0]) - ap.add_argument('--base', type=Path, help='base game, XCI or NSP') - ap.add_argument('--update', type=Path, help='the update NSP: 1.1.0, 1.2.1 or 1.4.3') + ap.add_argument('--base', type=Path, help='base game, XCI or NSP (or NSZ, XCZ: unpacked first)') + ap.add_argument('--update', type=Path, help='the update NSP or NSZ: 1.1.0, 1.2.1 or 1.4.3') ap.add_argument('--keys', type=Path, help='prod.keys') ap.add_argument('--firmware', type=Path, help='Switch firmware folder (Contents/registered) for the shared fonts ' 'of Chinese, Korean and Traditional Chinese text') diff --git a/installer/gui.py b/installer/gui.py index 93b3a0f4..ed2e48ed 100644 --- a/installer/gui.py +++ b/installer/gui.py @@ -225,6 +225,7 @@ class App: self.label(grid, 'ui.dest').grid(row=i + 1, column=0, sticky='w', pady=4) ttk.Entry(grid, textvariable=self.vars['dest']).grid(row=i + 1, column=1, sticky='ew', pady=4) self.button(grid, 'ui.browse', self.pick_dest).grid(row=i + 1, column=2, padx=(8, 0)) + self.label(f, 'ui.compressed_help', wraplength=620, justify='left').pack(anchor='w', pady=(8, 0)) self.label(f, 'ui.firmware_help', wraplength=620, justify='left').pack(anchor='w', pady=(8, 0)) self.update_hint = ttk.Label(f, wraplength=620, justify='left') self.update_hint.pack(anchor='w', pady=(8, 0)) @@ -303,9 +304,8 @@ class App: self.fit() def pick_file(self, name, pick_id): - types = [(self.cat('ui.type_game'), '*.nsp *.xci'), (self.cat('ui.type_all'), '*.*')] - if name == 'update': - types = [(self.cat('ui.type_nsp'), '*.nsp'), (self.cat('ui.type_all'), '*.*')] + # One filter for every game file: the setup tells an NSZ or XCZ from a plain file by its content (Check.unpack) + types = [(self.cat('ui.type_game'), '*.nsp *.xci *.nsz *.xcz')] if name == 'keys': types = [(self.cat('ui.type_keys'), '*.keys'), (self.cat('ui.type_all'), '*.*')] path = filedialog.askopenfilename(title=self.cat(pick_id), filetypes=types) diff --git a/installer/layout.py b/installer/layout.py index 5b5c498a..0c6995d8 100644 --- a/installer/layout.py +++ b/installer/layout.py @@ -90,7 +90,8 @@ class Layout: def missing(self, cpu='v3'): """Paths the steps need that do not exist. cpu picks the runtime library the host will use.""" lib = self.runtime_lib_for(cpu) - need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py', + need = [self.python, self.tools / 'extract.py', self.tools / 'ncz.py', self.tools / 'build_game.py', + self.tools / 'prepare_shaders.py', self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.tools / 'ultrawide_layout.py', self.tools / 'button_icons.py', self.tools / 'vendor' / 'macnativegames' / 'extract_shared_fonts.py', diff --git a/installer/steps.py b/installer/steps.py index 4c9f2ea5..57aef6bf 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -9,10 +9,11 @@ with the bundled clang (hostcheck.c). Install folder: exefs/, romfs/ (romfs.bin, index.json), profile/, shared-fonts/ (Chinese and Korean text), totk.exe and its DLLs, romfs-index.bin, romfs-patches/, romfs-patches-xbox/, romfs-patches-xbox-label/, totk_xlate.exe, the shader state of totk.exe --prepare-shaders, totk.ini, totk-install.json. -Temp folder (/-): check/ (ExeFS of both inputs, the host check), gen/ (C), build/ (objects), -logs/. The cleanup keeps gen/, build/ and logs/: the next run lifts into the same gen/, where recomp/gen.py rewrites -only the chunks whose C changed, and Ninja in build/ recompiles only those. A change of the mods' code cheats then -costs the lift (about a minute) and the chunks it touches, not a full compile. +Temp folder (/-): check/ (ExeFS of both inputs, the host check), unpacked/ (the NSP of an NSZ +or XCZ input until the extraction is done), gen/ (C), build/ (objects), logs/. The cleanup keeps gen/, build/ and +logs/: the next run lifts into the same gen/, where recomp/gen.py rewrites only the chunks whose C changed, and Ninja +in build/ recompiles only those. A change of the mods' code cheats then costs the lift (about a minute) and the chunks +it touches, not a full compile. Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status (tools/exefs_patches.py) and rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild), one argument per line; totk.exe runs it at start @@ -36,6 +37,7 @@ from .runner import InstallError, Step sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'tools')) import extract # noqa: E402 +import ncz # noqa: E402 import shader_list # noqa: E402 from extract_exefs import (common_tickets, container_entries, load_keys, nca_ctr_key, nca_inventory, # noqa: E402 program_nca) @@ -174,6 +176,12 @@ def raise_if_damaged(error, name, path): raise InstallError('check.input_damaged', input=name, path=str(path), detail=str(error)) from None +def game_file(ctx, name): + """The file the extraction reads for the base or the update: the NSP that Check.unpack made from an NSZ or XCZ, + else the chosen file itself.""" + return getattr(ctx, 'unpacked', {}).get(name) or Path(getattr(ctx.options, name)) + + def game_version(ctx): """The update this install is built from: what the check identified, else the recorded one. An install made before 1.4.3 support records no version, and that install is 1.2.1: name it here rather than leaning on SUPPORTED's order.""" @@ -432,14 +440,73 @@ class Check(Step): raise InstallError('check.keys_unreadable') from None if 'header_key' not in keys: raise InstallError('check.keys_no_header_key') + self.unpack(ctx) total = 0 for name, is_update in (('base', False), ('update', True)): - path = Path(getattr(o, name)) + path = game_file(ctx, name) with path.open('rb') as f: total += self.open_input(f, path, keys, name, is_update) del keys return total + def unpack(self, ctx): + """Unpack an NSZ or XCZ (a game file compressed by the nsz tool) into /unpacked/.nsp with + tools/ncz.py, which checks every NCA against its hash; the extraction then reads that NSP. An earlier run's NSP + of the same file is reused. The extract step removes the folder once it is done.""" + o, rep, folder = ctx.options, ctx.reporter, ctx.temp / 'unpacked' + ctx.unpacked = {} + jobs = [] + for name in ('base', 'update'): + path = Path(getattr(o, name)) + try: + found = ncz.compressed(path) + if found is None: + continue + if name == 'update' and found[0] != 'nsz': + raise InstallError('check.update_not_nsp') + size = ncz.unpacked_size(path) + except ValueError as e: + raise_if_damaged(e, name, path) + raise InstallError('check.not_game_file', input=name) from None + st = path.stat() + jobs.append((name, path, size, f'{path.resolve()}\n{st.st_size}\n{st.st_mtime_ns}\n{size}\n')) + if not jobs: + remove(folder) + return + done = [] + for name, path, size, tag in jobs: + target, stamp = folder / f'{name}.nsp', folder / f'{name}.source' + try: + if target.stat().st_size == size and stamp.read_text(encoding='utf-8') == tag: + done.append(name) + ctx.unpacked[name] = target + except OSError: + pass + need = sum(size for name, _, size, _ in jobs if name not in done) + free, _ = free_bytes(folder) + rep.log(f'unpack: {", ".join(name for name, *_ in jobs)}: {need / GiB:.1f} GiB to write, ' + f'{free / GiB:.1f} free') + if need > free: + raise InstallError('check.disk_space', path=str(folder), need_gb=round(need / 1e9, 1), + free_gb=round(free / 1e9, 1)) + for name, path, size, tag in jobs: + if name in done: + continue + target, stamp = folder / f'{name}.nsp', folder / f'{name}.source' + remove(stamp) + try: + folder.mkdir(parents=True, exist_ok=True) + ncz.unpack(path, target, lambda d, t, n=name: rep.progress('check', 'unpack_' + n, d, t, 'bytes')) + stamp.write_text(tag, encoding='utf-8') + except ncz.DamagedFile as e: + raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None + except ValueError as e: + raise_if_damaged(e, name, path) + raise InstallError('check.unpack_failed', input=name, detail=str(e)) from None + except OSError as e: + raise InstallError('check.unpack_failed', input=name, detail=str(e)) from None + ctx.unpacked[name] = target + @staticmethod def open_input(f, path, keys, name, is_update): try: @@ -520,9 +587,10 @@ class Check(Step): out = work / name remove(out) try: - extract.extract_exefs(Path(getattr(o, name)), Path(o.keys), out, logs, update=is_update) + extract.extract_exefs(game_file(ctx, name), Path(o.keys), out, logs, update=is_update) except extract.DamagedFile as e: - raise InstallError('check.input_damaged', input=name, path=str(e.path), detail=e.detail) from None + raise InstallError('check.input_damaged', input=name, path=str(getattr(o, name)), + detail=e.detail) from None except ValueError: raise InstallError('check.exefs_failed', input=name, log=str(logs / f'extract_exefs-{name}.log')) \ from None @@ -641,9 +709,9 @@ class Extract(Step): def run(self, ctx): o, dest, rep = ctx.options, ctx.dest, ctx.reporter expected = expected_update(ctx) - base, update, keys = Path(o.base), Path(o.update), Path(o.keys) + keys = Path(o.keys) inputs = {} - for name, path in (('base', base), ('update', update)): + for name, path in (('base', Path(o.base)), ('update', Path(o.update))): known = ctx.state['inputs'].get(name, {}) st = path.stat() if known.get('size') == st.st_size and known.get('mtime_ns') == st.st_mtime_ns and known.get('sha256'): @@ -653,6 +721,7 @@ class Extract(Step): inputs[name] = {'name': path.name, 'size': st.st_size, 'mtime_ns': st.st_mtime_ns, 'sha256': digest} ctx.state['inputs'] = inputs ctx.save() + base, update = game_file(ctx, 'base'), game_file(ctx, 'update') dest.mkdir(parents=True, exist_ok=True) for stale in dest.glob('update-merge-*'): # a killed merge leaves its NCA copies (and key copies) behind @@ -664,7 +733,7 @@ class Extract(Step): try: extract.extract_exefs(update, keys, ctx.temp / 'check' / 'update', ctx.temp / 'logs', update=True) except extract.DamagedFile as e: - raise InstallError('check.input_damaged', input='update', path=str(e.path), detail=e.detail) from None + raise InstallError('check.input_damaged', input='update', path=str(o.update), detail=e.detail) from None except ValueError: raise InstallError('check.exefs_failed', input='update', log=str(ctx.temp / 'logs' / 'extract_exefs-update.log')) from None @@ -686,6 +755,7 @@ class Extract(Step): raise InstallError('extract.romfs_mismatch', version=game_version(ctx), size=report['romfs_size'], sha256=report['romfs_sha256'][:16]) (dest / 'profile').mkdir(exist_ok=True) + remove(ctx.temp / 'unpacked') # the NSPs unpacked from an NSZ or XCZ (Check.unpack) st = (romfs / 'romfs.bin').stat() return {'romfs_size': st.st_size, 'romfs_mtime_ns': st.st_mtime_ns} @@ -707,7 +777,8 @@ class Extract(Step): try: extract.merge_update_romfs(base, update, keys, ctx.layout.hactool, romfs, logs, phase=phase) except extract.DamagedFile as e: - raise InstallError('check.input_damaged', input='base' if e.path == base else 'update', path=str(e.path), + name = 'base' if e.path == base else 'update' + raise InstallError('check.input_damaged', input=name, path=str(getattr(ctx.options, name)), detail=e.detail) from None except ValueError as e: raise InstallError('extract.romfs_merge_failed', detail=str(e), log=str(logs / 'merge_romfs-' @@ -1040,7 +1111,7 @@ class Cleanup(Step): rep.progress('cleanup', 'temp', force=True) for stale in dest.glob('update-merge-*'): remove(stale) - for name in ('check', 'shader_list.tsv', 'shared-fonts'): # gen/ and build/ stay for the next rebuild + for name in ('check', 'unpacked', 'shader_list.tsv', 'shared-fonts'): # gen/ and build/ stay for the next rebuild remove(ctx.temp / name) romfs_problem(dest / 'romfs', expected_update(ctx)) write_rebuild_command(ctx) diff --git a/installer/text/en.json b/installer/text/en.json index b054e806..bd2aa67a 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -11,6 +11,8 @@ "phase.check.verify": "Checking the files already installed", "phase.check.verify_romfs": "Checking the installed game data", "phase.check.inputs": "Opening the game files with the keys", + "phase.check.unpack_base": "Unpacking the compressed base game", + "phase.check.unpack_update": "Unpacking the compressed update", "phase.check.versions": "Checking the game versions", "phase.fonts.extract": "Extracting the shared fonts from the firmware", "phase.fonts.copy": "Copying the shared fonts into the install folder", @@ -58,14 +60,15 @@ "check.keys_no_header_key": "The keys file has no header_key. Dump prod.keys from your console again.", "check.keys_wrong": "The keys cannot open the {input} file. Use the prod.keys of your console.", "check.keys_too_old": "The keys file is too old for the {input} file (needs key generation {generation}). Dump prod.keys from a console with newer firmware.", - "check.not_game_file": "The {input} file is not an XCI or NSP game file.", + "check.not_game_file": "The {input} file is not an XCI, NSP, XCZ or NSZ game file.", "check.file_unreadable": "The {input} file cannot be read: {detail}", + "check.unpack_failed": "The compressed {input} file could not be unpacked: {detail}", "check.ticket_personalized": "The {input} file carries a personalized ticket; a common ticket is needed.", "check.ticket_missing": "The {input} file lacks the ticket its content needs.", "check.base_not_totk": "The base game file is not The Legend of Zelda: Tears of the Kingdom ({detail}).", "check.base_is_update": "The base game file is an update. Choose the base game NSP or XCI (version 1.0.0) here and this file as the update.", "check.base_combined": "The base game file holds several packages (base game, update or DLC in one file). Choose a base game dump that contains only the game.", - "check.update_not_nsp": "The update must be an NSP file.", + "check.update_not_nsp": "The update must be an NSP or NSZ file.", "check.update_is_base": "The update file holds a base game. Choose the 1.1.0 or 1.2.1 or 1.4.3 update NSP.", "check.update_not_totk": "The update file is not a Tears of the Kingdom update ({detail}).", "check.exefs_failed": "The game code of the {input} file failed its checks; see {log}.", @@ -109,8 +112,8 @@ "ui.intro": "Builds a playable Tears of the Kingdom folder from your own game files. This takes a while and runs once.", "ui.notice": "The folder this setup builds contains the game. It is for your own use only; do not share or upload it.", "ui.keys_private": "The keys file is read in place. Its contents are never shown, copied or stored.", - "ui.base": "Base game (NSP or XCI)", - "ui.update": "Update 1.1.0 or 1.2.1 or 1.4.3 (NSP)", + "ui.base": "Base game (NSP, XCI, NSZ or XCZ)", + "ui.update": "Update 1.1.0 or 1.2.1 or 1.4.3 (NSP or NSZ)", "ui.keys": "prod.keys", "ui.firmware": "Firmware folder (optional)", "ui.game_language": "Game language", @@ -122,8 +125,8 @@ "ui.pick_firmware": "Choose the Switch firmware folder (Contents/registered)", "ui.pick_dest": "Choose the install folder", "ui.firmware_help": "For Chinese, Korean and Traditional Chinese text: the firmware's Contents/registered folder, for example from Ryujinx or Yuzu. The setup copies the Switch system fonts from it into the install folder.", - "ui.type_game": "Game files (NSP, XCI)", - "ui.type_nsp": "NSP files", + "ui.compressed_help": "Compressed NSZ and XCZ files work as they are: the setup unpacks them by itself (until the game data is extracted, that needs as much free space as the unpacked files take).", + "ui.type_game": "Game files (NSP, XCI, NSZ, XCZ)", "ui.type_keys": "Key files (*.keys)", "ui.type_all": "All files", "ui.start": "Start", diff --git a/installer/text/pt-BR.json b/installer/text/pt-BR.json index 0cdcbc0a..083bb282 100644 --- a/installer/text/pt-BR.json +++ b/installer/text/pt-BR.json @@ -11,6 +11,8 @@ "phase.check.verify": "Verificando os arquivos já instalados", "phase.check.verify_romfs": "Verificando os dados do jogo instalados", "phase.check.inputs": "Abrindo os arquivos do jogo com as chaves", + "phase.check.unpack_base": "Descompactando o jogo base compactado", + "phase.check.unpack_update": "Descompactando a atualização compactada", "phase.check.versions": "Verificando as versões do jogo", "phase.fonts.extract": "Extraindo as fontes compartilhadas do firmware", "phase.fonts.copy": "Copiando as fontes compartilhadas para a pasta de instalação", @@ -58,14 +60,15 @@ "check.keys_no_header_key": "O arquivo de chaves não tem header_key. Extraia o prod.keys do seu console de novo.", "check.keys_wrong": "As chaves não abrem o arquivo de {input}. Use o prod.keys do seu console.", "check.keys_too_old": "O arquivo de chaves é antigo demais para o arquivo de {input} (precisa da geração de chaves {generation}). Extraia o prod.keys de um console com firmware mais novo.", - "check.not_game_file": "O arquivo de {input} não é um arquivo de jogo XCI ou NSP.", + "check.not_game_file": "O arquivo de {input} não é um arquivo de jogo XCI, NSP, XCZ ou NSZ.", "check.file_unreadable": "Não foi possível ler o arquivo de {input}: {detail}", + "check.unpack_failed": "Não foi possível descompactar o arquivo compactado de {input}: {detail}", "check.ticket_personalized": "O arquivo de {input} tem um ticket personalizado; é preciso um ticket comum.", "check.ticket_missing": "O arquivo de {input} não tem o ticket de que o conteúdo precisa.", "check.base_not_totk": "O arquivo do jogo base não é The Legend of Zelda: Tears of the Kingdom ({detail}).", "check.base_is_update": "O arquivo do jogo base é uma atualização. Escolha aqui o NSP ou XCI do jogo base (versão 1.0.0) e este arquivo como a atualização.", "check.base_combined": "O arquivo do jogo base contém vários pacotes (jogo base, atualização ou DLC em um só arquivo). Escolha um dump do jogo base que contenha só o jogo.", - "check.update_not_nsp": "A atualização precisa ser um arquivo NSP.", + "check.update_not_nsp": "A atualização precisa ser um arquivo NSP ou NSZ.", "check.update_is_base": "O arquivo de atualização contém um jogo base. Escolha o NSP da atualização 1.1.0, 1.2.1 ou 1.4.3.", "check.update_not_totk": "O arquivo de atualização não é uma atualização de Tears of the Kingdom ({detail}).", "check.exefs_failed": "O código do jogo no arquivo de {input} falhou nas verificações; veja {log}.", @@ -109,8 +112,8 @@ "ui.intro": "Monta uma pasta jogável de Tears of the Kingdom a partir dos seus próprios arquivos do jogo. Isso demora um pouco e só precisa ser feito uma vez.", "ui.notice": "A pasta que esta instalação monta contém o jogo. Ela é só para seu uso pessoal; não compartilhe nem envie para a internet.", "ui.keys_private": "O arquivo de chaves é lido no lugar. O conteúdo dele nunca é mostrado, copiado nem guardado.", - "ui.base": "Jogo base (NSP ou XCI)", - "ui.update": "Atualização 1.1.0, 1.2.1 ou 1.4.3 (NSP)", + "ui.base": "Jogo base (NSP, XCI, NSZ ou XCZ)", + "ui.update": "Atualização 1.1.0, 1.2.1 ou 1.4.3 (NSP ou NSZ)", "ui.keys": "prod.keys", "ui.firmware": "Pasta de firmware (opcional)", "ui.game_language": "Idioma do jogo", @@ -122,8 +125,8 @@ "ui.pick_firmware": "Escolha a pasta de firmware do Switch (Contents/registered)", "ui.pick_dest": "Escolha a pasta de instalação", "ui.firmware_help": "Para textos em chinês, coreano e chinês tradicional: a pasta Contents/registered do firmware, por exemplo do Ryujinx ou do Yuzu. A instalação copia dela as fontes do sistema do Switch para a pasta de instalação.", - "ui.type_game": "Arquivos de jogo (NSP, XCI)", - "ui.type_nsp": "Arquivos NSP", + "ui.compressed_help": "Arquivos compactados NSZ e XCZ funcionam do jeito que estão: a instalação os descompacta sozinha (para isso, até os dados do jogo serem extraídos, ela precisa de tanto espaço livre quanto ocupam os arquivos descompactados).", + "ui.type_game": "Arquivos de jogo (NSP, XCI, NSZ, XCZ)", "ui.type_keys": "Arquivos de chaves (*.keys)", "ui.type_all": "Todos os arquivos", "ui.start": "Iniciar", diff --git a/installer/text/ru.json b/installer/text/ru.json index e583dd71..4102627b 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -11,6 +11,8 @@ "phase.check.verify": "Проверка уже установленных файлов", "phase.check.verify_romfs": "Проверка установленных данных игры", "phase.check.inputs": "Открытие игровых файлов с помощью ключей", + "phase.check.unpack_base": "Распаковка сжатой базовой игры", + "phase.check.unpack_update": "Распаковка сжатого обновления", "phase.check.versions": "Проверка версий игры", "phase.fonts.extract": "Извлечение общих шрифтов из прошивки", "phase.fonts.copy": "Копирование общих шрифтов в папку установки", @@ -58,14 +60,15 @@ "check.keys_no_header_key": "В файле ключей нет header_key. Снова получите prod.keys со своей консоли.", "check.keys_wrong": "Эти ключи не открывают файл «{input}». Используйте prod.keys со своей консоли.", "check.keys_too_old": "Файл ключей слишком старый для файла «{input}» (нужно поколение ключей {generation}). Получите prod.keys с консоли с более новой прошивкой.", - "check.not_game_file": "Файл «{input}» не является игровым файлом XCI или NSP.", + "check.not_game_file": "Файл «{input}» не является игровым файлом XCI, NSP, XCZ или NSZ.", "check.file_unreadable": "Не удаётся прочитать файл «{input}»: {detail}", + "check.unpack_failed": "Не удалось распаковать сжатый файл «{input}»: {detail}", "check.ticket_personalized": "В файле «{input}» личный тикет; нужен общий тикет.", "check.ticket_missing": "В файле «{input}» нет тикета, который нужен его содержимому.", "check.base_not_totk": "Файл базовой игры — не The Legend of Zelda: Tears of the Kingdom ({detail}).", "check.base_is_update": "Файл базовой игры — это обновление. Выберите здесь NSP или XCI базовой игры (версия 1.0.0), а этот файл укажите как обновление.", "check.base_combined": "Файл базовой игры содержит несколько пакетов (игра, обновление или DLC в одном файле). Выберите дамп, в котором только базовая игра.", - "check.update_not_nsp": "Обновление должно быть файлом NSP.", + "check.update_not_nsp": "Обновление должно быть файлом NSP или NSZ.", "check.update_is_base": "Файл обновления содержит базовую игру. Выберите NSP с обновлением 1.1.0 или 1.2.1 или 1.4.3.", "check.update_not_totk": "Файл обновления — не обновление Tears of the Kingdom ({detail}).", "check.exefs_failed": "Код игры из файла «{input}» не прошёл проверку; подробности в {log}.", @@ -109,8 +112,8 @@ "ui.intro": "Собирает готовую к запуску папку с Tears of the Kingdom из ваших игровых файлов. Занимает много времени и выполняется один раз.", "ui.notice": "Папка, которую создаёт установка, содержит игру. Она предназначена только для личного использования: не передавайте и не выкладывайте её.", "ui.keys_private": "Файл ключей читается на месте. Его содержимое не показывается, не копируется и не сохраняется.", - "ui.base": "Базовая игра (NSP или XCI)", - "ui.update": "Обновление 1.1.0 или 1.2.1 или 1.4.3 (NSP)", + "ui.base": "Базовая игра (NSP, XCI, NSZ или XCZ)", + "ui.update": "Обновление 1.1.0 или 1.2.1 или 1.4.3 (NSP или NSZ)", "ui.keys": "prod.keys", "ui.firmware": "Папка прошивки (необязательно)", "ui.game_language": "Язык игры", @@ -122,8 +125,8 @@ "ui.pick_firmware": "Выберите папку прошивки Switch (Contents/registered)", "ui.pick_dest": "Выберите папку установки", "ui.firmware_help": "Для текста на китайском и корейском: папка Contents/registered прошивки, например из Ryujinx или Yuzu. Установка скопирует из неё системные шрифты Switch в папку установки.", - "ui.type_game": "Файлы игры (NSP, XCI)", - "ui.type_nsp": "Файлы NSP", + "ui.compressed_help": "Сжатые файлы NSZ и XCZ подходят как есть: установка сама их распакует (для этого, пока извлекаются данные игры, нужно столько свободного места, сколько занимают распакованные файлы).", + "ui.type_game": "Файлы игры (NSP, XCI, NSZ, XCZ)", "ui.type_keys": "Файлы ключей (*.keys)", "ui.type_all": "Все файлы", "ui.start": "Начать", diff --git a/installer/text/zh-Hans.json b/installer/text/zh-Hans.json index d858c38e..968fc260 100644 --- a/installer/text/zh-Hans.json +++ b/installer/text/zh-Hans.json @@ -11,6 +11,8 @@ "phase.check.verify": "检查已安装的文件", "phase.check.verify_romfs": "检查已安装的游戏数据", "phase.check.inputs": "用密钥打开游戏文件", + "phase.check.unpack_base": "正在解压压缩的游戏本体", + "phase.check.unpack_update": "正在解压压缩的更新", "phase.check.versions": "检查游戏版本", "phase.fonts.extract": "正在从固件中提取共享字体", "phase.fonts.copy": "正在将共享字体复制到安装文件夹", @@ -58,14 +60,15 @@ "check.keys_no_header_key": "密钥文件中没有 header_key。请从你的主机重新导出 prod.keys。", "check.keys_wrong": "这些密钥无法打开{input}文件。请使用你自己主机的 prod.keys。", "check.keys_too_old": "密钥文件对{input}文件来说太旧(需要密钥代数 {generation})。请从固件更新的主机导出 prod.keys。", - "check.not_game_file": "{input}文件不是 XCI 或 NSP 游戏文件。", + "check.not_game_file": "{input}文件不是 XCI、NSP、XCZ 或 NSZ 游戏文件。", "check.file_unreadable": "无法读取{input}文件:{detail}", + "check.unpack_failed": "无法解压压缩的{input}文件:{detail}", "check.ticket_personalized": "{input}文件带有个人化票据(personalized ticket);需要通用票据(common ticket)。", "check.ticket_missing": "{input}文件缺少其内容所需的票据(ticket)。", "check.base_not_totk": "游戏本体文件不是《塞尔达传说 王国之泪》({detail})。", "check.base_is_update": "基础游戏文件是一个更新。请在这里选择基础游戏的 NSP 或 XCI(版本 1.0.0),并把此文件选为更新。", "check.base_combined": "基础游戏文件包含多个内容包(基础游戏、更新或 DLC 合在一个文件中)。请选择只包含游戏本体的基础游戏转储。", - "check.update_not_nsp": "更新必须是 NSP 文件。", + "check.update_not_nsp": "更新必须是 NSP 或 NSZ 文件。", "check.update_is_base": "更新文件里是游戏本体。请选择 1.1.0、1.2.1 或 1.4.3 的更新 NSP。", "check.update_not_totk": "更新文件不是《王国之泪》的更新({detail})。", "check.exefs_failed": "{input}文件的游戏代码未通过检查;详见 {log}。", @@ -109,8 +112,8 @@ "ui.intro": "用你自己的游戏文件构建一个可以游玩的《王国之泪》文件夹。耗时较长,只需运行一次。", "ui.notice": "此安装程序构建的文件夹包含游戏本身,仅供你个人使用;请勿分享或上传。", "ui.keys_private": "密钥文件只在原位置读取,其内容绝不会被显示、复制或保存。", - "ui.base": "游戏本体(NSP 或 XCI)", - "ui.update": "1.1.0、1.2.1 或 1.4.3 更新(NSP)", + "ui.base": "游戏本体(NSP、XCI、NSZ 或 XCZ)", + "ui.update": "1.1.0、1.2.1 或 1.4.3 更新(NSP 或 NSZ)", "ui.keys": "prod.keys", "ui.firmware": "固件文件夹(可选)", "ui.game_language": "游戏语言", @@ -122,8 +125,8 @@ "ui.pick_firmware": "选择 Switch 固件文件夹(Contents/registered)", "ui.pick_dest": "选择安装文件夹", "ui.firmware_help": "用于中文、韩文和繁体中文文本:固件的 Contents/registered 文件夹,例如来自 Ryujinx 或 Yuzu。安装程序会从中把 Switch 系统字体复制到安装文件夹。", - "ui.type_game": "游戏文件(NSP、XCI)", - "ui.type_nsp": "NSP 文件", + "ui.compressed_help": "压缩的 NSZ 和 XCZ 文件可以直接使用:安装程序会自行解压(在游戏数据提取完成之前,需要与解压后文件大小相同的可用空间)。", + "ui.type_game": "游戏文件(NSP、XCI、NSZ、XCZ)", "ui.type_keys": "密钥文件(*.keys)", "ui.type_all": "所有文件", "ui.start": "开始", diff --git a/tests/ncz_test.py b/tests/ncz_test.py new file mode 100644 index 00000000..106f1a0d --- /dev/null +++ b/tests/ncz_test.py @@ -0,0 +1,173 @@ +"""tools/ncz.py and the installer's unpacking of NSZ input (installer/steps.py, Check.unpack) on small made-up files: +an NCA with an AES-CTR section and a plain one, stored as an NCZ with one zstd stream or as NCZBLOCK blocks. + +Run: .venv-py314\\Scripts\\python tests/ncz_test.py +""" +import hashlib +import io +import os +import struct +import sys +import tempfile +import unittest +from compression import zstd +from pathlib import Path +from types import SimpleNamespace + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / 'tools')) +import ncz # noqa: E402 +from installer import steps # noqa: E402 +from installer.events import Reporter # noqa: E402 + +KEY, COUNTER = bytes(range(16)), bytes(range(16, 32)) +# (offset, size, crypto type): an AES-CTR section, then a plain one +SECTIONS = ((0x4000, 0x9000, 3), (0xD000, 0x3200, 1)) + + +def pfs0(files): + """An NSP of (name, data) pairs.""" + strings = b''.join(name.encode() + b'\0' for name, _ in files) + strings += bytes(-(16 + 0x18 * len(files) + len(strings)) % 0x20) + head = b'PFS0' + struct.pack('Q', 0x4000 >> 4) + secret = Cipher(algorithms.AES(KEY), modes.CTR(counter)).encryptor().update(plain) + header = rand(0x4000) + return header + secret + rest, plain + rest + + +def ncz_file(nca, body, blocks=False): + out = nca[:0x4000] + b'NCZSECTN' + struct.pack(' len(p) // 2 else p for p in pieces] # random blocks stay as they are + out += b'NCZBLOCK' + bytes([2, 1, 0, exponent]) + struct.pack('/unpacked/base.nsp, a plain NSP update is read in place, a second run + reuses the NSP.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + base = Path(self.tmp.name) + nca, body = made_up_nca() + name = hashlib.sha256(nca).hexdigest()[:32] + self.base, self.update = base / 'base.nsz', base / 'update.nsp' + self.base.write_bytes(pfs0([(name + '.ncz', ncz_file(nca, body))])) + self.update.write_bytes(pfs0([('a.tik', b'x' * 64)])) + self.expected = pfs0([(name + '.nca', nca)]) + self.out = io.StringIO() + self.ctx = SimpleNamespace(options=SimpleNamespace(base=str(self.base), update=str(self.update)), + temp=base / 'temp', reporter=Reporter(out=self.out, log_path=base / 'x.log')) + + def tearDown(self): + self.ctx.reporter.close() + self.tmp.cleanup() + + def test_unpacks_the_base_and_reuses_it(self): + steps.Check().unpack(self.ctx) + target = self.ctx.temp / 'unpacked' / 'base.nsp' + self.assertEqual(target.read_bytes(), self.expected) + self.assertEqual(steps.game_file(self.ctx, 'base'), target) + self.assertEqual(steps.game_file(self.ctx, 'update'), self.update) + self.assertIn('"phase": "unpack_base"', self.out.getvalue()) + mtime = target.stat().st_mtime_ns + steps.Check().unpack(self.ctx) + self.assertEqual(target.stat().st_mtime_ns, mtime) + + def test_content_decides_not_the_name(self): + renamed = self.base.with_name('base.nsp') + self.base.rename(renamed) + self.ctx.options.base = str(renamed) + steps.Check().unpack(self.ctx) + self.assertEqual((self.ctx.temp / 'unpacked' / 'base.nsp').read_bytes(), self.expected) + + def test_damaged_base_names_the_chosen_file(self): + data = bytearray(self.base.read_bytes()) + data[-100] ^= 0xFF + self.base.write_bytes(bytes(data)) + with self.assertRaises(steps.InstallError) as e: + steps.Check().unpack(self.ctx) + self.assertIn(e.exception.id, ('check.input_damaged', 'check.unpack_failed')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/make_release.py b/tools/make_release.py index 814eff63..c5fa64a0 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -105,9 +105,10 @@ LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32' LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. -TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py', 'hostmem.py', - 'make_romfs_patches.py', 'pack_shader_cache.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', - 'shader_list.py', 'ultrawide_layout.py', 'cheats.py', 'exefs_patches.py'] +TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py', + 'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py', + 'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py', + 'exefs_patches.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon diff --git a/tools/ncz.py b/tools/ncz.py new file mode 100644 index 00000000..d41dfe4d --- /dev/null +++ b/tools/ncz.py @@ -0,0 +1,231 @@ +"""Unpack an NSZ or XCZ (a game file compressed by the nsz tool) into a plain NSP. + +An NSZ is an NSP whose NCAs are stored as NCZ files: the first 0x4000 bytes of the NCA (its encrypted header) as they +are, then an NCZSECTN table of the NCA's sections with the AES-CTR key and counter of each, then the rest of the NCA +decrypted and compressed with zstd, either as one stream or as an NCZBLOCK table of independently compressed blocks. +An XCZ is an XCI whose secure partition holds NCZ files. Unpacking encrypts the sections again, so every NCA comes back +byte for byte; its file name is the first half of its SHA-256, which is checked. The output is an NSP of the secure +partition (an XCZ) or of the NSZ's files, with .ncz renamed to .nca. Format: nicoboss/nsz, nsz/NszDecompressor.py +(__decompressNcz) and nsz/BlockDecompressorReader.py; this is our own code. + +usage: ncz.py . Runs under .venv-py314 (compression.zstd). +""" +import hashlib +import io +import struct +import sys +from compression import zstd +from pathlib import Path + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +sys.path.insert(0, str(Path(__file__).resolve().parent / 'vendor' / 'macnativegames')) +from extract_exefs import container_entries, read_at, require # noqa: E402 + +HEADER = 0x4000 # the NCA header, kept encrypted and uncompressed +CHUNK = 1 << 20 +CRYPTO_CTR = (3, 4) # AES-CTR and BKTR sections; the other types are stored as they are + + +class DamagedFile(ValueError): + """An NCZ whose unpacked NCA does not match its name's hash, or whose data ends early.""" + + def __init__(self, path, detail): + self.path, self.detail = Path(path), detail + super().__init__(f'{self.path} is damaged or incomplete ({detail})') + + +def compressed(path): + """('nsz' or 'xcz', entries) for a game file holding NCZ files, else None. Raises ValueError for a file that is + neither an NSP nor an XCI.""" + with Path(path).open('rb') as f: + kind, entries = container_entries(f, Path(path).stat().st_size) + if not any(name.lower().endswith('.ncz') for name, _, _ in entries): + return None + return ('nsz' if kind == 'nsp' else 'xcz'), entries + + +def sections(f, start): + """[(offset, size, crypto type, key, counter)] of the NCZ at start, and the offset of its data after the table.""" + head = read_at(f, start + HEADER, 16) + require(head[:8] == b'NCZSECTN', 'NCZ without its NCZSECTN table') + count = struct.unpack_from('= HEADER, 'NCZ section inside the NCA header') + for (a, size, *_), (b, *_) in zip(out, out[1:]): + require(a + size <= b, 'Overlapping NCZ sections') + return out, start + HEADER + 16 + count * 0x40 + + +def nca_size(f, start): + table, _ = sections(f, start) + return table[-1][0] + table[-1][1] + + +def layout(source): + """(entries, NSP header, NSP size) of the NSP that unpack writes for the NSZ or XCZ source: its files in order, + .ncz renamed to .nca, the PFS0 table padded to 0x20 bytes.""" + _, entries = compressed(source) + with Path(source).open('rb') as f: + sizes = [nca_size(f, start) if name.lower().endswith('.ncz') else length for name, start, length in entries] + names = [name[:-4] + '.nca' if name.lower().endswith('.ncz') else name for name, _, _ in entries] + strings = b''.join(n.encode('utf-8') + b'\0' for n in names) + strings += bytes(-(16 + 0x18 * len(names) + len(strings)) % 0x20) + head = b'PFS0' + struct.pack(' at), total) + n = min(len(pending), stop - at) + crypto = section and section[2] in CRYPTO_CTR + if crypto and at + n < stop: + n -= n % 16 # the counter of the next piece starts at a whole AES block + if not n: + break + data, pending = pending[:n], pending[n:] + if crypto: + require(at % 16 == 0, 'Unaligned NCZ section') + counter = section[4][:8] + struct.pack('>Q', at >> 4) + data = Cipher(algorithms.AES(section[3]), modes.CTR(counter)).encryptor().update(data) + out.write(data) + digest.update(data) + at += n + progress(n) + if at == total: + break + except (zstd.ZstdError, EOFError) as e: # EOFError: the stream ends before its end-of-frame marker + raise DamagedFile(source, f'{name}: {e}') from None + if at != total or pending: + raise DamagedFile(source, f'{name}: {at - HEADER:,} of {total - HEADER:,} bytes') + stem = name[:-4] + if len(stem) == 32 and digest.hexdigest()[:32] != stem.lower(): + raise DamagedFile(source, f'{name}: hash mismatch') + + +def unpack(source, target, progress=lambda done, total: None): + """Write the NSP of the NSZ or XCZ source to target (through target.part). progress(done, total) in bytes.""" + source, target = Path(source), Path(target) + kind = compressed(source)[0] + entries, head, total = layout(source) + part = target.with_name(target.name + '.part') + done = 0 + + def step(n): + nonlocal done + done += n + progress(done, total) + + with source.open('rb') as f, part.open('wb') as out: + out.write(head) + step(len(head)) + for name, start, length in entries: + if name.lower().endswith('.ncz'): + unpack_ncz(f, start, start + length, out, name, source, step) + continue + for at in range(start, start + length, CHUNK): + data = read_at(f, at, min(CHUNK, start + length - at)) + out.write(data) + step(len(data)) + part.replace(target) + return kind + + +def main(): + source, target = sys.argv[1:3] + shown = [-1] + + def progress(done, total): + percent = done * 100 // total + if percent != shown[0]: + shown[0] = percent + print(f'\r{percent}%', end='', flush=True) + + kind = unpack(source, target, progress) + print(f'\n{kind}: wrote {target}') + + +if __name__ == '__main__': + main()