diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 406efdae..44db3260 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -55,7 +55,7 @@ These tools run on the developer's or player's machine during the build. They ar | Vulkan-Headers and Vulkan-Loader 1.4.363.0 (MSYS2 packages; headers and `libvulkan-1.dll.a` only, no loader DLL) | https://github.com/KhronosGroup | Apache-2.0 OR MIT | `runtime/deps/` | | Vulkan Memory Allocator 3.4.0 (MSYS2 package), header | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator | MIT | `runtime/deps/` | | Ninja 1.13.2 (`ninja-win.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja.exe` | -| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus MacNativeGames' `portable-patches.diff`: `MNG_TITLE_KEYS` and a relaxed BKTR size check; our `unicode-patches.diff`: an embedded manifest that makes UTF-8 the process code page), built with MSYS2 UCRT64 gcc; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | +| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus `tools/hactool/portable.patch`, from MacNativeGames' `build_hactool()`: `MNG_TITLE_KEYS` and relaxed BKTR layout checks; our `tools/hactool/unicode.patch`: an embedded manifest that makes UTF-8 the process code page), built by `tools/build_hactool.py` with MSYS2 UCRT64 gcc; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | | `totk_xlate.exe` (Ryujinx shader translator, NativeAOT build of `tools/xlate/`, see the Ryujinx row above) | built by `tools/build_xlate.py` | MIT | `bin/totk_xlate.exe` | | MacNativeGames shader variant list | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/assets/shader_list.tsv`), Copyright (c) 2026 MacNativeGames contributors | MIT | `data/shader_list.tsv` | diff --git a/docs/building.md b/docs/building.md index 1f2feeb5..4229ee24 100644 --- a/docs/building.md +++ b/docs/building.md @@ -17,7 +17,7 @@ Versions are the ones the build was verified with; newer patch releases are fine | .NET SDK | 10.0 | winget `Microsoft.DotNet.SDK.10` | the shader translator (`totk_xlate.exe`) | | Vulkan SDK | 1.4.363 | winget `KhronosGroup.VulkanSDK` | validation layers, `vulkaninfo` | | uv | 0.13 | standalone installer from astral.sh | the two Python environments | -| hactool | 1.4.0 @ `1d64a83` | built from source, see below | NCA and update extraction | +| hactool | 1.4.0 @ `1d64a83` | `.venv\Scripts\python tools\build_hactool.py`, see below | NCA and update extraction | MSYS2 packages, after `pacman -Syuu` (run it twice; the first pass updates the core and exits): @@ -37,13 +37,13 @@ PATH: ### hactool -`tools/extract.py` runs `toolchain\hactool\hactool.exe`. Build it from SciresM/hactool at `1d64a83` in an MSYS2 UCRT64 -shell (`config.mk`: `CC=gcc EXEEXT=.exe`, static link, `make -j8`) with two changes: +`tools/build_hactool.py` fetches SciresM/hactool at `1d64a83` into `toolchain\src\hactool`, applies the two patches in +`tools/hactool/` and builds a static `toolchain\hactool\hactool.exe` with MSYS2 UCRT64 gcc (a few seconds): -1. The portable part of MacNativeGames' `build_app.py` `build_hactool()`: the `MNG_TITLE_KEYS` variable, and the two - BKTR fixes in `nca.c` for the subsection table padding and the subsection layout of newer cumulative updates. -2. A manifest with `UTF-8` embedded through a `.rc` file, so paths under a non-ASCII - user profile reach it intact. +- `portable.patch`: the parts of MacNativeGames' `build_hactool()` that run on Windows, the `MNG_TITLE_KEYS` variable and + the two BKTR fixes in `nca.c` for the subsection table padding and the subsection layout of newer cumulative updates; +- `unicode.patch`: a manifest with `UTF-8`, so paths under a non-ASCII user profile + reach it intact. hactool reads `-k `; MSYS2 shells set `HOME`, which hides its default key location, so pass `-k` there. diff --git a/tools/build_hactool.py b/tools/build_hactool.py new file mode 100644 index 00000000..3be021e2 --- /dev/null +++ b/tools/build_hactool.py @@ -0,0 +1,75 @@ +"""Build hactool: toolchain/hactool/hactool.exe (SciresM/hactool at a pinned commit + tools/hactool/*.patch). + +Fetches the pinned commit into toolchain/src/hactool, resets it (local changes and build output there are discarded), +applies portable.patch (MacNativeGames' Windows-relevant changes) and unicode.patch (ours), and builds a static exe +with MSYS2 UCRT64 gcc; mbedtls comes with hactool's tree. The exe and hactool's LICENSE go to toolchain/hactool, where +tools/extract.py and tools/make_release.py take them. Run with .venv. + + python tools/build_hactool.py [--src DIR] [--out DIR] +""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +URL = 'https://github.com/SciresM/hactool.git' +COMMIT = '1d64a83450e025622f3468c28fc4164dad2c5ef6' +PATCHES = [ROOT / 'tools' / 'hactool' / n for n in ('portable.patch', 'unicode.patch')] +MSYS_BASH = Path(r'C:\msys64\usr\bin\bash.exe') +CONFIG_MK = ('CC = gcc\nEXEEXT = .exe\nCFLAGS = -O2 -Wall -Wextra -pedantic -std=gnu11\n' + 'LDFLAGS = -lmbedtls -lmbedx509 -lmbedcrypto -static\n') + + +def git(src, *args): + subprocess.run(['git', '-C', str(src), *args], check=True) + + +def fetch(src): + """src holds COMMIT with no local changes or untracked files.""" + if not (src / '.git').is_dir(): + src.mkdir(parents=True, exist_ok=True) + git(src, 'init', '-q') + git(src, 'remote', 'add', 'origin', URL) + head = subprocess.run(['git', '-C', str(src), 'rev-parse', 'HEAD'], capture_output=True, text=True).stdout.strip() + if head != COMMIT: + git(src, 'fetch', '-q', '--depth', '1', 'origin', COMMIT) + git(src, 'checkout', '-q', '-f', COMMIT) + git(src, 'clean', '-q', '-fdx') + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--src', type=Path, default=ROOT / 'toolchain' / 'src' / 'hactool', help='source checkout') + ap.add_argument('--out', type=Path, default=ROOT / 'toolchain' / 'hactool', help='where hactool.exe goes') + args = ap.parse_args() + if not MSYS_BASH.is_file(): + sys.exit(f'{MSYS_BASH} not found: install MSYS2 with the UCRT64 gcc and make (docs/building.md)') + src, out = args.src.resolve(), args.out.resolve() + + fetch(src) + for p in PATCHES: + git(src, 'apply', '--whitespace=nowarn', str(p)) + (src / 'config.mk').write_text(CONFIG_MK, encoding='utf-8', newline='\n') + + env = dict(os.environ, MSYSTEM='UCRT64', CHERE_INVOKING='1') + r = subprocess.run([str(MSYS_BASH), '-lc', f'make -j{os.cpu_count() or 4}'], cwd=src, env=env, + capture_output=True, text=True) + (src / 'build.log').write_text(r.stdout + r.stderr, encoding='utf-8') + if r.returncode: + sys.exit(f'make failed ({r.returncode}); log: {src / "build.log"}') + + out.mkdir(parents=True, exist_ok=True) + shutil.copy2(src / 'hactool.exe', out / 'hactool.exe') + shutil.copy2(src / 'LICENSE', out / 'LICENSE') + banner = subprocess.run([str(out / 'hactool.exe')], capture_output=True, text=True) + if 'hactool (c) SciresM' not in banner.stdout + banner.stderr: + sys.exit(f'{out / "hactool.exe"} does not start') + print(f'{out / "hactool.exe"}: {COMMIT[:7]} + {", ".join(p.name for p in PATCHES)}') + + +if __name__ == '__main__': + main() diff --git a/tools/hactool/portable.patch b/tools/hactool/portable.patch new file mode 100644 index 00000000..517f9bda --- /dev/null +++ b/tools/hactool/portable.patch @@ -0,0 +1,43 @@ +Applied by tools/build_hactool.py to SciresM/hactool (ISC) at 1d64a83450e025622f3468c28fc4164dad2c5ef6. +From MacNativeGames build_app.py build_hactool() (MIT, Copyright (c) 2026 MacNativeGames contributors), the parts +that run on Windows: MNG_TITLE_KEYS names the title-key file, and the BKTR checks accept the subsection table +padding and the subsection layout of newer cumulative updates. Its funopen base-range hook is macOS-only and left out. + +diff --git a/main.c b/main.c +index 07f53cb..7921cc4 100644 +--- a/main.c ++++ b/main.c +@@ -472,7 +472,7 @@ int main(int argc, char **argv) { + } + + /* Try to load titlekeys. */ +- FILE *titlekeyfile = open_key_file("title"); ++ FILE *titlekeyfile = getenv("MNG_TITLE_KEYS") ? fopen(getenv("MNG_TITLE_KEYS"), "rb") : open_key_file("title"); + if (titlekeyfile != NULL) { + extkeys_parse_titlekeys(&tool_ctx.settings, titlekeyfile); + } +diff --git a/nca.c b/nca.c +index 8a84366..2fe877f 100644 +--- a/nca.c ++++ b/nca.c +@@ -1149,7 +1149,8 @@ void nca_process_bktr_section(nca_section_ctx_t *ctx) { + /* Validate magics. */ + if (sb->relocation_header.magic == MAGIC_BKTR && sb->subsection_header.magic == MAGIC_BKTR) { + if (sb->relocation_header.offset + sb->relocation_header.size != sb->subsection_header.offset || +- sb->subsection_header.offset + sb->subsection_header.size != ctx->size) { ++ sb->subsection_header.offset + sb->subsection_header.size > ctx->size || ++ (ctx->size - (sb->subsection_header.offset + sb->subsection_header.size)) % 0x200 != 0) { + fprintf(stderr, "Invalid BKTR layout!\n"); + exit(EXIT_FAILURE); + } +@@ -1180,7 +1181,9 @@ void nca_process_bktr_section(nca_section_ctx_t *ctx) { + ctx->bktr_ctx.relocation_block = relocs; + ctx->bktr_ctx.subsection_block = subs; + +- if (ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset) { ++ if (ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset && ++ (sb->relocation_header.size > sb->subsection_header.offset || ++ ctx->bktr_ctx.subsection_block->total_size != sb->subsection_header.offset - sb->relocation_header.size)) { + free(relocs); + free(subs); + ctx->bktr_ctx.relocation_block = NULL; diff --git a/tools/hactool/unicode.patch b/tools/hactool/unicode.patch new file mode 100644 index 00000000..eda188f6 --- /dev/null +++ b/tools/hactool/unicode.patch @@ -0,0 +1,62 @@ +Applied by tools/build_hactool.py after portable.patch. Ours: a manifest embedded through hactool.rc makes UTF-8 the +process code page (Windows 10 1903 and later), so paths and user profiles with non-ASCII names reach filepath.c intact. + +diff --git a/Makefile b/Makefile +index e1913b8..73b7e3f 100644 +--- a/Makefile ++++ b/Makefile +@@ -13,9 +13,19 @@ all: + .c.o: + $(CC) $(INCLUDE) -c $(CFLAGS) -o $@ $< + +-hactool$(EXEEXT): save.o sha.o aes.o extkeys.o rsa.o npdm.o bktr.o kip.o packages.o pki.o pfs0.o hfs0.o nca0_romfs.o romfs.o utils.o nax0.o nso.o lz4.o nca.o xci.o main.o filepath.o ConvertUTF.o cJSON.o ++# Windows: a manifest makes UTF-8 the process code page (Windows 10 1903 and later), so argv, getenv and fopen carry ++# UTF-8 as filepath.c expects: paths and user profiles with non-ASCII names work. ++ifeq ($(EXEEXT),.exe) ++WINDRES ?= windres ++RESOURCES = hactool.res.o ++endif ++ ++hactool$(EXEEXT): save.o sha.o aes.o extkeys.o rsa.o npdm.o bktr.o kip.o packages.o pki.o pfs0.o hfs0.o nca0_romfs.o romfs.o utils.o nax0.o nso.o lz4.o nca.o xci.o main.o filepath.o ConvertUTF.o cJSON.o $(RESOURCES) + $(CC) -o $@ $^ -L $(LIBDIR) $(LDFLAGS) + ++hactool.res.o: hactool.rc hactool.manifest ++ $(WINDRES) -O coff hactool.rc -o $@ ++ + aes.o: aes.h types.h + + bktr.o: bktr.h types.h +diff --git a/hactool.manifest b/hactool.manifest +new file mode 100644 +index 0000000..4073219 +--- /dev/null ++++ b/hactool.manifest +@@ -0,0 +1,20 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ UTF-8 ++ ++ ++ +diff --git a/hactool.rc b/hactool.rc +new file mode 100644 +index 0000000..543aa10 +--- /dev/null ++++ b/hactool.rc +@@ -0,0 +1,2 @@ ++/* Windows: argv, getenv and the narrow file functions use UTF-8 (Windows 10 1903 and later), as filepath.c expects. */ ++1 24 "hactool.manifest"