From c3dae879cbfe7432049096c4010483e509715f43 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 1 Nov 2026 14:29:40 +0300 Subject: [PATCH] Build the mods' ExeFS patches into the game and switch them with their mod A mod that patched the game code (.ips, .pchtxt) stayed off as a whole. tools/exefs_patches.py now reads IPS, IPS32 and IPSwitch files from each mod's exefs folder and from atmosphere/exefs_patches//. It keeps the ones whose build id is main's and turns their writes into main's code into one patch set per file. The writes go through the same rules as the code cheats (tools/cheats.py convert). A file that writes main's data, or that the rules cannot express, is unsupported as a whole. The setup's lift runs it after cheats.py and writes build-cache/exefs.status. runtime/mods.c checks each patch file against that status: - a built file's set is switched on with its mod; - a file the build lacks keeps the mod off and starts the rebuild at the next start, like a code cheat does; - an unsupported file keeps the mod off, and the Mods tab shows the reason; - a file for another version is counted and left out. A mod patch on a site of one of the port's own sets turns that set off while the mod is on, and the Mods tab names it. The Mods tab also gets the "Rebuild the game now" row. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- docs/installer.md | 17 +-- docs/tester-guide.md | 8 +- installer/steps.py | 71 ++++++++--- installer/text/en.json | 1 + installer/text/pt-BR.json | 1 + installer/text/ru.json | 1 + installer/text/zh-Hans.json | 1 + runtime/boot.c | 2 + runtime/cheats.c | 25 ++-- runtime/mods.c | 228 +++++++++++++++++++++++++++++++++-- runtime/mods.h | 27 ++++- runtime/mods_probe.c | 21 +++- runtime/ui/menu.cpp | 35 ++++-- runtime/ui_text.c | 37 ++++-- runtime/ui_text.h | 4 + tests/installer_test.py | 11 ++ tools/exefs_patches.py | 230 ++++++++++++++++++++++++++++++++++++ tools/make_release.py | 2 +- 19 files changed, 651 insertions(+), 75 deletions(-) create mode 100644 tools/exefs_patches.py diff --git a/README.md b/README.md index edb0fdf6..a04c690b 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,13 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). - Keyboard, mouse and gamepad with remapping, rumble and gyro - Xbox button prompts: LB, RB, LT, RT, Menu and View, and A, B, X and Y as labelled on the button to press, with either button layout - Shaders prepared at install instead of during play +- Mods made for emulators: unpack a mod into the `mods` folder next to `totk.exe`; switch and order them in the menu. RomFS + mods, TKMM's merged output and ExeFS code patches (`.ips`, `.pchtxt`) work; the game rebuilds itself for code patches + ([docs/tester-guide.md](docs/tester-guide.md), "Mods") - Atmosphère cheats: put a cheat file into a mod and switch its cheats in the menu ([docs/cheats.md](docs/cheats.md)) ## Planned -- Mods made for emulators: unpack a mod into a `mods` folder - VRR ## Installing diff --git a/docs/installer.md b/docs/installer.md index f25b27d7..e01793a2 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -53,8 +53,9 @@ loads the saves 1.4.3 wrote. and BFTTF header). Without a firmware folder the setup warns and goes on; the game then lacks those glyphs. 3. **Extract** ExeFS and RomFS into the install folder and verify the hashes. 4. **Lift** the ExeFS to C (`recomp/gen.py`, `recomp/patches-v121.txt` or `patches-v143.txt`) into a temp folder next - to the installer. The code cheats of the mods' cheat files become patch sets first (`tools/cheats.py`, status in - `build-cache/cheats.status`); a cheat file the tool refuses costs the cheats, not the install. + to the installer. The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets first + (`tools/cheats.py`, `tools/exefs_patches.py`, status in `build-cache/cheats.status` and `exefs.status`); a file the + tools refuse costs those cheats or patches, not the install. 5. **Compile** the generated C, `modules.c`, `romfs.c`, `patches.c` and `ov_zstd.c` (they bake the per-install layout, RomFS hashes and the game version's patch page) and link them with the prebuilt runtime library for the host's level (`libtotkrt.a` or `libtotkrt-v2.a`) into `totk.exe` @@ -82,12 +83,12 @@ Ninja recompiles only those: a new or edited code cheat costs the lift (under a and a few dozen chunks, not a full compile, and the shaders stay prepared. `build-cache/rebuild.txt` holds the command that opens the setup window in its rebuild mode on the install folder -(`python -m installer.gui --rebuild `), one argument per line. When an enabled mod has code cheats the -build lacks, `totk.exe` runs it at start and exits (`runtime/cheats.c` `rebuild_if_needed`); the window runs the setup -on the folder at once and starts the game again when it succeeds. A failed or cancelled rebuild offers to start the -game with the code it has; `build-cache/rebuild.skip` then keeps the same cheats from starting it again, until "Rebuild -the game now" on the Cheats tab. `[mods] mods_rebuild = off` turns it off. The rebuild needs the release folder the -setup ran from: a moved or deleted release folder means running a setup again. +(`python -m installer.gui --rebuild `), one argument per line. When an enabled mod has code cheats or +ExeFS patch files the build lacks, `totk.exe` runs it at start and exits (`runtime/cheats.c` `rebuild_if_needed`); the +window runs the setup on the folder at once and starts the game again when it succeeds. A failed or cancelled rebuild +offers to start the game with the code it has; `build-cache/rebuild.skip` then keeps the same files from starting it +again, until "Rebuild the game now" on the Cheats or Mods tab. `[mods] mods_rebuild = off` turns it off. The rebuild +needs the release folder the setup ran from: a moved or deleted release folder means running a setup again. ## Command line diff --git a/docs/tester-guide.md b/docs/tester-guide.md index c36fe03a..5b71cb47 100644 --- a/docs/tester-guide.md +++ b/docs/tester-guide.md @@ -68,8 +68,12 @@ it up or down. When two mods change the same file, the higher one wins. much memory each file needs, is rebuilt at every start with mods enabled: the tables the mods ship are merged and the entries of files a mod made larger or added are raised. The log line `RomFS: resource size table: ...` says how many. A mod's table for another game version still counts for that mod's own files. -- Mods that patch the game code (`exefs/*.ips`, `exefs/*.pchtxt`) are listed but not applied yet; a mod with both kinds - of files stays off as a whole until they are. +- Mods that patch the game code (`exefs/*.ips`, `exefs/*.pchtxt`, or `atmosphere/exefs_patches//*.ips`) work + when the patches are for this game version and change only code: the game rebuilds itself with them at the next + start (the setup's rebuild window, a few minutes) and switches them with the mod. Patches for another version are + counted and left out; a file that writes the game's data, or that the port cannot express, keeps the whole mod off + and the Mods tab says why. A patch over one of the port's own patches turns that built-in patch off while the mod + is on (the Mods tab names it). - Mods with their own code (`exefs/subsdk*` or a replaced `exefs/main`, such as UltraCam) never load, their RomFS files included. - Cheats: an Atmosphère cheat file goes into a mod's `cheats` folder (`mods//cheats/.txt`, the diff --git a/installer/steps.py b/installer/steps.py index 6d96a600..9db5d080 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -14,7 +14,7 @@ logs/. The cleanup keeps gen/, build/ and logs/: the next run lifts into the sam only the chunks whose C changed, and Ninja in build/ recompiles only those. A change of the mods' code cheats then costs the lift (about a minute) and the chunks it touches, not a full compile. -Install folder build-cache/: cheats.status (tools/cheats.py) and rebuild.txt, the command that starts this setup in +Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status (tools/exefs_patches.py) and rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild), one argument per line; totk.exe runs it at start when an enabled mod has code cheats the build lacks (runtime/cheats.c). """ @@ -733,6 +733,30 @@ def cheat_files(ctx): for f in ((mod / where).glob('*.txt') if (mod / where).is_dir() else ())) +EXEFS_PLACES = ('exefs', 'atmosphere/contents/0100F2C0115B6000/exefs', 'contents/0100F2C0115B6000/exefs', + '0100F2C0115B6000/exefs') + + +def exefs_files(ctx): + """The ExeFS patch files of the mods folder (.ips and .pchtxt in each mod's exefs places and its + atmosphere/exefs_patches//, as tools/exefs_patches.py finds them), for the lift key.""" + mods = ctx.dest / 'mods' + if not mods.is_dir(): + return [] + out = [] + for mod in (m for m in mods.iterdir() if m.is_dir()): + folders = [mod / w for w in EXEFS_PLACES if (mod / w).is_dir()] + patches = mod / 'atmosphere' / 'exefs_patches' + folders += [d for d in patches.iterdir() if d.is_dir()] if patches.is_dir() else [] + out += [f for d in folders for f in d.iterdir() if f.is_file() and f.suffix.lower() in ('.ips', '.pchtxt')] + return sorted(out) + + +def exefs_status(ctx): + """The build's ExeFS patch status file (tools/exefs_patches.py --status), which runtime/mods.c reads.""" + return ctx.dest / 'build-cache' / 'exefs.status' + + def cheats_status(ctx): """The build's cheat status file (tools/cheats.py --status), which runtime/cheats.c reads: in build-cache, not in a mod's own cheats folder, which the player (or TKMM's merge) may empty.""" @@ -744,8 +768,9 @@ class Lift(Step): transient = True def key(self, ctx): - return fingerprint('lift-3', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)), - ctx.layout.tools / 'cheats.py', *cheat_files(ctx)) + return fingerprint('lift-4', ctx.keys['extract'], ctx.layout.recomp, ctx.layout.patches(game_version(ctx)), + ctx.layout.tools / 'cheats.py', ctx.layout.tools / 'exefs_patches.py', *cheat_files(ctx), + *exefs_files(ctx)) def current(self, ctx, record): gen = ctx.temp / 'gen' @@ -764,30 +789,38 @@ class Lift(Step): if m: rep.progress('lift', 'emit', int(m.group(1)), int(m.group(2)), 'chunks') - # The code cheats of the mods' cheat files become patch sets (tools/cheats.py); its status file tells the - # runtime which cheats the build has. A cheat file is the player's download, so one the tools refuse costs the - # cheats, not the install: the game is built without them and the setup warns. - plain, status = layout.patches(game_version(ctx)), cheats_status(ctx) - patches, skipped = plain, None - status.unlink(missing_ok=True) # no status: the runtime treats every code cheat as not built - if cheat_files(ctx): - combined = ctx.temp / 'patches-cheats.txt' - code, log = run_tool(ctx, 'lift', layout.tools / 'cheats.py', - ['--exefs', ctx.dest / 'exefs', '--mods', ctx.dest / 'mods', '--patches', plain, - '--out', combined, '--status', status]) - patches, skipped = (plain, log) if code else (combined, None) + # The code cheats of the mods' cheat files and the mods' ExeFS patch files become patch sets (tools/cheats.py, + # tools/exefs_patches.py); their status files tell the runtime what the build has. Both are the player's + # downloads, so files the tools refuse cost those cheats or patches, not the install: the game is built + # without them and the setup warns. + plain = layout.patches(game_version(ctx)) + stages = [('cheats.py', 'patches-cheats.txt', cheats_status(ctx), cheat_files(ctx), 'warn.cheats_skipped'), + ('exefs_patches.py', 'patches-exefs.txt', exefs_status(ctx), exefs_files(ctx), 'warn.exefs_skipped')] + patches, skipped = plain, {} + for tool, out, status, inputs, warn in stages: + status.unlink(missing_ok=True) # no status: the runtime treats every such file as not built + if not inputs: + continue + code, log = run_tool(ctx, 'lift', layout.tools / tool, + ['--exefs', ctx.dest / 'exefs', '--mods', ctx.dest / 'mods', '--patches', patches, + '--out', ctx.temp / out, '--status', status]) if code: status.unlink(missing_ok=True) + skipped[warn] = log + else: + patches = ctx.temp / out args = ['--exefs', ctx.dest / 'exefs', '--out', gen] code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches], parse) if code and patches != plain: - skipped = log - status.unlink(missing_ok=True) + for tool, out, status, inputs, warn in stages: + if inputs and warn not in skipped: + status.unlink(missing_ok=True) + skipped[warn] = log code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', plain], parse) if code: raise InstallError('lift.failed', code=code, log=str(log)) - if skipped: - rep.warning('warn.cheats_skipped', log=str(skipped)) + for warn, log in skipped.items(): + rep.warning(warn, log=str(log)) (gen / STAMP).write_text(ctx.keys['lift'] + '\n') return {} diff --git a/installer/text/en.json b/installer/text/en.json index 5c4c7079..d27a4129 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -95,6 +95,7 @@ "warn.lang_needs_fonts": "Text in {lang} needs the Switch system fonts. Choose the firmware folder (Contents/registered) to install them; without it some text may not show.", "warn.npdm_differs": "main.npdm differs from the reference {version} dump; only the code modules are used.", "warn.cheats_skipped": "The cheats of the mods folder that change the game's code could not be built; the game installs without them. Details: {log}", + "warn.exefs_skipped": "The mods' ExeFS patches (.ips, .pchtxt) could not be built; the game installs without them. Details: {log}", "warn.xbox_prompts_failed": "The Xbox button prompts could not be prepared (exit code {code}); the game keeps the Switch prompts. Details: {log}", "warn.shaders_skipped": "The graphics driver crashed while building some shaders, so {count} were left out of the preparation. The game builds them when it first needs them, and the driver may crash there too; a driver update may fix it. Details: {log}", "warn.kernel_old": "Linux {version} lacks the write tracking the game relies on (6.7 or later); the game may run very slowly. Updating the system fixes this.", diff --git a/installer/text/pt-BR.json b/installer/text/pt-BR.json index 5699f35b..2a0c2d51 100644 --- a/installer/text/pt-BR.json +++ b/installer/text/pt-BR.json @@ -94,6 +94,7 @@ "warn.lang_needs_fonts": "Textos em {lang} precisam das fontes de sistema do Switch, que o instalador não extrai; parte do texto pode não aparecer.", "warn.cheats_skipped": "Não foi possível compilar os cheats da pasta de mods que alteram o código do jogo; o jogo é instalado sem eles. Detalhes: {log}", + "warn.exefs_skipped": "Não foi possível compilar os patches ExeFS dos mods (.ips, .pchtxt); o jogo é instalado sem eles. Detalhes: {log}", "warn.npdm_differs": "O main.npdm difere do dump de referência da versão {version}; só os módulos de código são usados.", "warn.xbox_prompts_failed": "Não foi possível preparar os ícones de botão do Xbox (código de saída {code}); o jogo mantém os ícones do Switch. Detalhes: {log}", "warn.shaders_skipped": "O driver gráfico travou ao compilar alguns shaders, então {count} ficaram fora da preparação. O jogo os compila quando precisar deles pela primeira vez, e o driver pode travar ali também; uma atualização do driver pode resolver. Detalhes: {log}", diff --git a/installer/text/ru.json b/installer/text/ru.json index 4c871a75..d1f4b641 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -95,6 +95,7 @@ "warn.lang_needs_fonts": "Для языка «{lang}» нужны системные шрифты Switch. Выберите папку прошивки (Contents/registered), чтобы установить их; без неё часть текста может не отображаться.", "warn.npdm_differs": "main.npdm отличается от эталонного дампа {version}; используются только модули с кодом.", "warn.cheats_skipped": "Не удалось собрать читы из папки модов, меняющие код игры; игра установится без них. Подробности: {log}", + "warn.exefs_skipped": "Не удалось собрать ExeFS-патчи модов (.ips, .pchtxt); игра установится без них. Подробности: {log}", "warn.xbox_prompts_failed": "Не удалось подготовить подсказки кнопок Xbox (код выхода {code}); игра оставит подсказки Switch. Подробности: {log}", "warn.shaders_skipped": "Графический драйвер аварийно завершился при сборке части шейдеров, поэтому {count} из них пропущены при подготовке. Игра соберёт их, когда они впервые понадобятся, и драйвер может упасть и там; обновление драйвера может это исправить. Подробности: {log}", "warn.kernel_old": "В Linux {version} нет отслеживания записи, на которое опирается игра (нужна версия 6.7 или новее); игра может работать очень медленно. Обновление системы это исправит.", diff --git a/installer/text/zh-Hans.json b/installer/text/zh-Hans.json index df75fd0e..7256a720 100644 --- a/installer/text/zh-Hans.json +++ b/installer/text/zh-Hans.json @@ -94,6 +94,7 @@ "warn.lang_needs_fonts": "{lang}文本需要 Switch 系统字体,而安装程序不会提取这些字体;部分文字可能无法显示。", "warn.cheats_skipped": "无法构建 mods 文件夹中修改游戏代码的金手指;游戏将不含它们进行安装。详情:{log}", + "warn.exefs_skipped": "无法构建模组的 ExeFS 补丁(.ips、.pchtxt);游戏将不含它们进行安装。详情:{log}", "warn.npdm_differs": "main.npdm 与参考的 {version} 导出不同;只使用其中的代码模块。", "warn.xbox_prompts_failed": "无法准备 Xbox 按键提示(退出代码 {code});游戏将继续使用 Switch 按键提示。详情:{log}", "warn.shaders_skipped": "显卡驱动在编译部分着色器时崩溃,因此有 {count} 个着色器未被预先准备。游戏会在首次需要时编译它们,驱动在那时也可能崩溃;更新驱动可能会解决。详情:{log}", diff --git a/runtime/boot.c b/runtime/boot.c index 4c168896..c2b08465 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -18,6 +18,7 @@ #include "cheats.h" #include "fs.h" #include "gfx.h" +#include "mods.h" #include "nvn_api.h" #include "platform/platform.h" #include "vk.h" @@ -343,6 +344,7 @@ void rt_boot(void) { PlatThread *warm = plat_thread_new(prewarm_main, NULL, "prewarm"); ldr_load(rt_config()->exefs); rt_patches_init(); + mods_exefs_switch(); // after the patch sets' settings: an enabled mod's ExeFS patches on frame_step_init(); res_init(); cheats_init(); // after the patch sets: a cheat's set follows its setting diff --git a/runtime/cheats.c b/runtime/cheats.c index b777566c..53f1d6f3 100644 --- a/runtime/cheats.c +++ b/runtime/cheats.c @@ -440,10 +440,11 @@ static void plain_name(char *s) { } } -// ---- rebuild: code cheats the build lacks +// ---- rebuild: code cheats and ExeFS patches the build lacks // The setup writes build-cache\rebuild.txt (installer/steps.py write_rebuild_command), the command that starts its // window in rebuild mode on this install folder, one argument per line. When an enabled mod has code cheats this build -// lacks (CHEAT_REBUILD), the game runs it and exits; the setup lifts and compiles them in and starts the game again. +// lacks (CHEAT_REBUILD) or ExeFS patch files it lacks (mods_exefs_pending), the game runs it and exits; the setup lifts +// and compiles them in and starts the game again. // build-cache\rebuild.skip holds the key of the cheats a rebuild was started for: a rebuild that failed or was // cancelled leaves them unbuilt, and the same set does not start it again. TOTK_MODS_REBUILD=off never starts it. // Scripted runs (--seconds, --boot) never start it. @@ -457,7 +458,8 @@ static void rebuild_if_needed(const char *dir) { key = (key ^ fnv(c->info.key, strlen(c->info.key))) * 0x01000193u; key = (key ^ c->content) * 0x01000193u; } - if (!need) return; + int patches = mods_exefs_pending(&key); + if (!need && !patches) return; char cmd_path[PLAT_PATH_MAX + 64], skip_path[PLAT_PATH_MAX + 64], skip[16] = "", want[16]; snprintf(cmd_path, sizeof cmd_path, "%sbuild-cache" SEP "rebuild.txt", dir); snprintf(skip_path, sizeof skip_path, "%sbuild-cache" SEP "rebuild.skip", dir); @@ -472,7 +474,8 @@ static void rebuild_if_needed(const char *dir) { char *cmd = why ? NULL : read_text(cmd_path); if (!why && !cmd) why = "no build-cache/rebuild.txt (a build without the setup)"; if (why) { - rt_log("cheats: %d code cheats are not built into the game; no rebuild: %s", need, why); + rt_log("cheats: %d code cheats and %d ExeFS patch files are not built into the game; no rebuild: %s", need, + patches, why); free(cmd); return; } @@ -488,11 +491,13 @@ static void rebuild_if_needed(const char *dir) { } argv[argc] = NULL; if (argc < 2 || !plat_file_write(skip_path, want, strlen(want))) { - rt_log("cheats: %d code cheats are not built into the game; cannot start the rebuild (%s)", need, cmd_path); + rt_log("cheats: %d code cheats and %d ExeFS patch files are not built into the game; cannot start the " + "rebuild (%s)", need, patches, cmd_path); free(cmd); return; } - rt_log("cheats: %d code cheats are not built into the game; starting the rebuild: %s", need, argv[0]); + rt_log("cheats: %d code cheats and %d ExeFS patch files are not built into the game; starting the rebuild: %s", + need, patches, argv[0]); int ok = plat_spawn(argv, dir); free(cmd); if (!ok) { @@ -528,6 +533,7 @@ void cheats_init(void) { } if (!g_files) { rt_log("cheats: no enabled mod holds cheats/%s", g_file); + rebuild_if_needed(dir); // an enabled mod's ExeFS patches may still wait for it return; } snprintf(path, sizeof path, "%sbuild-cache" SEP "cheats.status", dir); @@ -592,9 +598,10 @@ void cheats_init(void) { int cheats_rebuild_pending(void) { char path[PLAT_PATH_MAX + 64]; snprintf(path, sizeof path, "%sbuild-cache" SEP "rebuild.txt", g_dir); - for (int i = 0; i < g_n; i++) - if (g_cheats[i].info.kind == CHEAT_REBUILD) return g_dir[0] && plat_file_exists(path); - return 0; + uint32_t key = 0; + int need = mods_exefs_pending(&key); + for (int i = 0; i < g_n && !need; i++) need = g_cheats[i].info.kind == CHEAT_REBUILD; + return need && g_dir[0] && plat_file_exists(path); } void cheats_rebuild_allow(void) { diff --git a/runtime/mods.c b/runtime/mods.c index af350893..4f3ad08c 100644 --- a/runtime/mods.c +++ b/runtime/mods.c @@ -10,6 +10,13 @@ // Windows file name cannot hold '|'; a folder whose name does on Linux is listed as unusable); a folder it does not name // goes below the named ones, by name. TOTK_MODS_DISABLED lists the switched-off names. The menu changes the list in // place and writes both keys; the scan's state is kept to tell the menu a restart is due. +// +// ExeFS patch files: an .ips or .pchtxt file in the exefs folder or in atmosphere/exefs_patches// is for the game +// when its build id is main's (an IPS file's name, a pchtxt file's @nsobid line); others are another version's and +// counted. The setup builds every mod's files into patch sets and writes build-cache/exefs.status next to totk.exe +// (tools/exefs_patches.py has the format): a file whose line is missing or has another content hash waits for the +// rebuild at start (cheats.c), and its mod stays off until then. +#include #include #include #include @@ -119,12 +126,180 @@ static const char *problem_name(ModProblem p) { switch (p) { case MOD_EMPTY: return "empty"; case MOD_PLUGIN: return "has a code module (subsdk or main), not supported; the whole mod stays off"; - case MOD_EXEFS_LATER: return "ExeFS patches need a game code rebuild, not applied; the whole mod stays off"; + case MOD_EXEFS_LATER: return "ExeFS patches wait for a rebuild of the game code; the whole mod stays off"; + case MOD_EXEFS_UNSUPPORTED: return "an ExeFS patch file cannot be applied; the whole mod stays off"; + case MOD_EXEFS_OTHER: return "its ExeFS patches are for another game version"; case MOD_BAD_NAME: return "its folder name holds '|', which the order list uses; rename the folder"; default: return "ok"; } } +// ---- ExeFS patch files + +static uint32_t fnv(const void *p, size_t n) { + uint32_t h = 0x811C9DC5u; + for (size_t i = 0; i < n; i++) h = (h ^ ((const uint8_t *)p)[i]) * 0x01000193u; + return h; +} + +const char *mods_main_build_id(void) { + static char bid[65]; + static int read; + if (read) return bid; + read = 1; + char path[2100]; + snprintf(path, sizeof path, "%s/main", rt_config()->exefs); + PlatFile *f = plat_file_open(path); + uint8_t head[0x60]; + int ok = f && plat_file_pread(f, head, sizeof head, 0) == sizeof head && !memcmp(head, "NSO0", 4); + if (f) plat_file_close(f); + for (int i = 0; ok && i < 32; i++) snprintf(bid + 2 * i, 3, "%02x", head[0x40 + i]); + return bid; +} + +// 1 when hex (any case; trailing zeros left out or not) names main's build id; also when main's is unknown. +static int same_build(const char *hex, size_t n) { + const char *bid = mods_main_build_id(); + if (!*bid) return 1; + if (!n || n > 64) return 0; + for (size_t i = 0; i < 64; i++) { + char c = i < n ? hex[i] : '0'; + if (c >= 'A' && c <= 'F') c = (char)(c - 'A' + 'a'); + if (c != bid[i]) return 0; + } + return 1; +} + +// build-cache/exefs.status next to totk.exe, read once; "" when there is none. +static const char *exefs_status(void) { + static char *text; + if (text) return text; + char path[2100]; + uint64_t n = 0; + if (plat_exe_path(path, sizeof path)) { + char *cut = strrchr(path, '/'), *back = strrchr(path, '\\'); + if (back > cut) cut = back; + if (cut) snprintf(cut + 1, sizeof path - (size_t)(cut + 1 - path), "build-cache/exefs.status"); + text = (char *)plat_file_read(path, &n); + } + if (!text) text = calloc(1, 1); + return text; +} + +// The status line of key: its kind, built-in sets and note, NUL-terminated copies; 0 when none. +static int status_line(const char *key, const char *content, char *kind, char *over, char *note) { + char want[20]; + snprintf(want, sizeof want, "%s\t%s\t", key, content); + for (const char *line = exefs_status(); *line;) { + const char *end = strchr(line, '\n'); + size_t len = end ? (size_t)(end - line) : strlen(line); + if (len > 18 && !memcmp(line, want, 18)) { + char copy[600]; + snprintf(copy, sizeof copy, "%.*s", (int)(len < sizeof copy ? len : sizeof copy - 1), line + 18); + char *t1 = strchr(copy, '\t'), *t2 = t1 ? strchr(t1 + 1, '\t') : NULL; + if (!t1 || !t2) return 0; + *t1 = *t2 = 0; + char *r = strchr(t2 + 1, '\r'); + if (r) *r = 0; + snprintf(kind, 16, "%s", copy); + snprintf(over, 256, "%s", strcmp(t1 + 1, "-") ? t1 + 1 : ""); + snprintf(note, 256, "%s", t2 + 1); + return 1; + } + line = end ? end + 1 : line + len; + } + return 0; +} + +typedef struct { + ModInfo *m; + const char *folder, *rel; // the folder listed and its path in the mod + int unsupported; +} ExefsWalk; + +// One file of an exefs folder: its build, then its status line. +static int exefs_file_cb(const char *leaf, int kind, uint64_t size, void *ctx) { + (void)size; + ExefsWalk *w = ctx; + ModInfo *m = w->m; + int ips = ends_with_ci(leaf, ".ips"); + if (kind != PLAT_FS_FILE || (!ips && !ends_with_ci(leaf, ".pchtxt"))) return 1; + m->kind |= MOD_EXEFS; + char path[4300], rel[600]; + snprintf(path, sizeof path, "%s/%s", w->folder, leaf); + snprintf(rel, sizeof rel, "%s/%s/%s", m->name, w->rel, leaf); + uint64_t n = 0; + char *data = (char *)plat_file_read(path, &n); + if (!data) { + rt_log("mods: %s: cannot read %s", m->name, path); + return 1; + } + int for_main; + if (ips) { + for_main = same_build(leaf, strlen(leaf) - 4); + } else { + char *id = NULL; // after "@nsobid-" at the start of a line + for (char *p = data; !id && p && data + n - p > 8;) { + if (!memcmp(p, "@nsobid-", 8)) id = p + 8; + p = memchr(p, '\n', (size_t)(data + n - p)); + if (p) p++; + } + size_t len = 0; + while (id && id + len < data + n && isxdigit((unsigned char)id[len])) len++; + for_main = id && same_build(id, len); + } + char key[9], content[9], kind_s[16], over[256], note[256]; + snprintf(key, sizeof key, "%08X", fnv(rel, strlen(rel))); + snprintf(content, sizeof content, "%08X", fnv(data, (size_t)n)); + free(data); + if (!for_main) { + m->other_version++; + return 1; + } + if (!status_line(key, content, kind_s, over, note)) { + m->pending++; + m->pending_key = (m->pending_key ^ fnv(key, 8)) * 0x01000193u; + m->pending_key = (m->pending_key ^ fnv(content, 8)) * 0x01000193u; + return 1; + } + if (!strcmp(kind_s, "unsupported")) { + if (!w->unsupported) snprintf(m->note, sizeof m->note, "%s: %s", leaf, note); + w->unsupported = 1; + } else if (!strcmp(kind_s, "code") && m->nsets < (int)(sizeof m->sets / sizeof *m->sets)) { + m->sets[m->nsets++] = (uint32_t)strtoul(key, NULL, 16); + size_t k = strlen(m->overrides); + if (*over) snprintf(m->overrides + k, sizeof m->overrides - k, "%s%s", k ? "," : "", over); + } else if (!strcmp(kind_s, "other")) { + m->other_version++; + } + return 1; +} + +// atmosphere/exefs_patches// of a mod: each folder's files. +static int exefs_patches_cb(const char *leaf, int kind, uint64_t size, void *ctx) { + (void)size; + ExefsWalk *w = ctx; + if (kind != PLAT_FS_DIR) return 1; + char folder[4300], rel[600]; + snprintf(folder, sizeof folder, "%s/%s", w->folder, leaf); + snprintf(rel, sizeof rel, "atmosphere/exefs_patches/%s", leaf); + ExefsWalk sub = {w->m, folder, rel, 0}; + plat_fs_list(folder, exefs_file_cb, &sub); + w->unsupported |= sub.unsupported; + return 1; +} + +// The ExeFS patch files of mod m (folder path): 1 when one cannot be applied. +static int exefs_check(ModInfo *m, const char *path) { + ExefsWalk w = {m, m->exefs, m->exefs + strlen(path) + 1, 0}; + if (m->exefs[0]) plat_fs_list(m->exefs, exefs_file_cb, &w); + char patches[2200]; + snprintf(patches, sizeof patches, "%s/atmosphere/exefs_patches", path); + ExefsWalk at = {m, patches, "", 0}; + if (plat_dir_exists(patches)) plat_fs_list(patches, exefs_patches_cb, &at); + return w.unsupported || at.unsupported; +} + // One mod folder of the scan. static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) { (void)size; @@ -149,12 +324,15 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) { if (m->romfs[0] && has_file(m->romfs)) m->kind |= MOD_ROMFS; if (m->exefs[0]) plat_fs_list(m->exefs, exefs_cb, &m->kind); if (m->cheats[0]) plat_fs_list(m->cheats, cheats_cb, &m->kind); + int unsupported = exefs_check(m, path); // a mod whose code part the port cannot use is off as a whole: its RomFS files may expect that code - m->problem = strchr(m->name, '|') ? MOD_BAD_NAME - : m->kind & MOD_SUBSDK ? MOD_PLUGIN - : m->kind & MOD_EXEFS ? MOD_EXEFS_LATER - : m->kind & (MOD_ROMFS | MOD_CHEATS) ? MOD_OK - : MOD_EMPTY; + m->problem = strchr(m->name, '|') ? MOD_BAD_NAME + : m->kind & MOD_SUBSDK ? MOD_PLUGIN + : unsupported ? MOD_EXEFS_UNSUPPORTED + : m->pending ? MOD_EXEFS_LATER + : m->nsets || m->kind & (MOD_ROMFS | MOD_CHEATS) ? MOD_OK + : m->kind & MOD_EXEFS ? MOD_EXEFS_OTHER + : MOD_EMPTY; m->enabled = !listed(disabled, m->name); nmods++; return 1; @@ -190,13 +368,47 @@ void mods_scan(void) { snprintf(start_order[i], sizeof start_order[i], "%s", mods[i].name); start_enabled[i] = mods[i].enabled; const ModInfo *m = &mods[i]; - rt_log("mods: %s: %s,%s%s%s%s%s, %s", m->name, m->enabled ? "on" : "off", m->kind & MOD_ROMFS ? " romfs" : "", + rt_log("mods: %s: %s,%s%s%s%s%s, %s%s%s", m->name, m->enabled ? "on" : "off", m->kind & MOD_ROMFS ? " romfs" : "", m->kind & MOD_EXEFS ? " exefs" : "", m->kind & MOD_SUBSDK ? " subsdk" : "", - m->kind & MOD_CHEATS ? " cheats" : "", m->kind ? "" : " no files", problem_name(m->problem)); + m->kind & MOD_CHEATS ? " cheats" : "", m->kind ? "" : " no files", problem_name(m->problem), + m->note[0] ? ": " : "", m->note); } rt_log("mods: %d folders in %s", nmods, dir); } +void mods_exefs_switch(void) { + mods_scan(); + for (int i = 0; i < nmods; i++) { + const ModInfo *m = &mods[i]; + if (!start_enabled[i] || m->problem != MOD_OK || !m->nsets) continue; + for (int k = 0; k < m->nsets; k++) { + char name[32]; + snprintf(name, sizeof name, "mod_%08x", m->sets[k]); + int set = rt_patch_set(name); + if (set >= 0) rt_patch_switch(set, 1); + else rt_log("mods: %s: patch set %s is not in this build", m->name, name); + } + char over[256]; + snprintf(over, sizeof over, "%s", m->overrides); + for (char *s = strtok(over, ","); s; s = strtok(NULL, ",")) { + int set = rt_patch_set(s); + if (set >= 0) rt_patch_switch(set, 0); + rt_log("mods: %s replaces the port's patch set %s; it stays off", m->name, s); + } + rt_log("mods: %s: %d ExeFS patch files on", m->name, m->nsets); + } +} + +int mods_exefs_pending(uint32_t *key) { + int n = 0; + for (int i = 0; i < nmods; i++) { + if (!start_enabled[i] || mods[i].problem != MOD_EXEFS_LATER) continue; + n += (int)mods[i].pending; + *key = (*key ^ mods[i].pending_key) * 0x01000193u; + } + return n; +} + const char *mods_dir(void) { return dir; } int mods_count(void) { return nmods; } ModInfo *mods_at(int i) { return i >= 0 && i < nmods ? &mods[i] : NULL; } diff --git a/runtime/mods.h b/runtime/mods.h index f4d77917..98a513db 100644 --- a/runtime/mods.h +++ b/runtime/mods.h @@ -1,6 +1,8 @@ // Player mods (mods.c): one folder per mod under mods/ next to totk.exe (TOTK_MODS_DIR), unpacked as emulators take -// them: /romfs, /exefs and /cheats, the Atmosphere layout (atmosphere/contents//romfs) or -// TKMM's merged output (romfs or romfslite, exefs, cheats). The list is scanned once at start (romfs.c overlay_init); +// them: <mod>/romfs, <mod>/exefs and <mod>/cheats, the Atmosphere layout (atmosphere/contents/<title id>/romfs, +// atmosphere/exefs_patches/<name>/*.ips) or TKMM's merged output (romfs or romfslite, exefs, cheats). ExeFS patch +// files (.ips, .pchtxt) for the game's main are patch sets the setup builds into the game (tools/exefs_patches.py), +// switched with their mod (mods_exefs_switch). The list is scanned once at start (romfs.c overlay_init); // its order and switches live in totk.ini [mods] (TOTK_MODS_ORDER: names top first, separated by '|'; // TOTK_MODS_DISABLED: names switched off). The top mod wins a file two mods replace. #pragma once @@ -12,8 +14,10 @@ typedef enum { MOD_EMPTY, // no RomFS file, no ExeFS patch, no cheat file MOD_PLUGIN, // has code modules (exefs/subsdk*, a replaced exefs/main): the port has no module loader, so the // whole mod stays off (its RomFS files may expect that code) - MOD_EXEFS_LATER, // has ExeFS patches (.ips, .pchtxt): a rebuild of the game code applies them, not yet here; the - // whole mod stays off until then + MOD_EXEFS_LATER, // has ExeFS patches for this game's main that the game was built without: the rebuild at the + // next start builds them in; the whole mod stays off until then + MOD_EXEFS_UNSUPPORTED, // an ExeFS patch file the port cannot apply (note says why): the whole mod stays off + MOD_EXEFS_OTHER, // only ExeFS patches, all for another game version MOD_BAD_NAME, // its folder name holds '|', the separator of the order keys } ModProblem; @@ -25,8 +29,14 @@ typedef struct { int kind, enabled; ModProblem problem; uint32_t replaced, added, overridden; // romfs.c: files served (replacing / added), files a higher mod serves - uint32_t other_version; // romfs.c: files for another game version (*.Product.<n>.*), not served + uint32_t other_version; // files for another game version (RomFS *.Product.<n>.*, ExeFS patches + // for another build), not used char overridden_by[256]; // the first higher mod that took one of its files + char note[256]; // MOD_EXEFS_UNSUPPORTED: why + uint32_t pending, pending_key; // ExeFS patch files the build lacks; FNV-1a of their names and contents + int nsets; + uint32_t sets[16]; // the built patch sets mod_<hash> of its ExeFS patch files + char overrides[256]; // the port's own patch sets its patches replace, comma separated } ModInfo; void mods_scan(void); // once; later calls return at once @@ -38,3 +48,10 @@ ModInfo *mods_at(int i); // 0 = top of the order; NULL out of range void mods_set_enabled(int i, int on); void mods_move(int i, int delta); // -1 up, +1 down; nothing at the ends int mods_changed(void); // 1 when the order or a switch differs from the start +// After rt_patches_init: the patch sets of the enabled mods' ExeFS patches on, the port's own sets they replace off. +void mods_exefs_switch(void); +// The ExeFS patch files of the enabled mods that the build lacks, their FNV-1a mixed into *key (cheats.c starts the +// rebuild for them). +int mods_exefs_pending(uint32_t *key); +// The hex of the game main's build id (64 digits), "" when exefs/main cannot be read. +const char *mods_main_build_id(void); diff --git a/runtime/mods_probe.c b/runtime/mods_probe.c index 1307a7c5..221cb2d7 100644 --- a/runtime/mods_probe.c +++ b/runtime/mods_probe.c @@ -67,32 +67,41 @@ int probe_mods(void) { "K/romfslite/k.bin", "L/romfs/l.bin", "L/cheats/0123456789ABCDEF.txt", "L/exefs/main.npdm", }; for (size_t i = 0; i < sizeof files / sizeof *files; i++) put(root, files[i]); + // an IPS file for this game's main (any file counts as one when main cannot be read), in Atmosphere's layout + char ips[160]; + snprintf(ips, sizeof ips, "M/atmosphere/exefs_patches/x/%s.ips", *mods_main_build_id() ? mods_main_build_id() : "0"); + put(root, ips); cfg_set("TOTK_MODS_DIR", root); cfg_set("TOTK_MODS_ORDER", "B|A|Gone"); cfg_set("TOTK_MODS_DISABLED", "C"); mods_scan(); - static const char *const order[] = {"B", "A", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "Ultra \xD0\x96"}; + static const char *const order[] = {"B", "A", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", + "Ultra \xD0\x96"}; enum { N = sizeof order / sizeof *order }; - CHECK(mods_count() == N, "thirteen mod folders"); + CHECK(mods_count() == N, "fourteen mod folders"); int in_order = mods_count() == N; for (int i = 0; in_order && i < N; i++) in_order = EQ(mods_at(i)->name, order[i]); CHECK(in_order, "order: mods_order first, the rest by name"); if (mods_count() == N && in_order) { ModInfo *a = mods_at(1), *b = mods_at(0), *c = mods_at(2), *d = mods_at(3), *e = mods_at(4), *f = mods_at(5), *g = mods_at(6), *hm = mods_at(7), *im = mods_at(8), *jm = mods_at(9), *km = mods_at(10), - *lm = mods_at(11), *u = mods_at(12); + *lm = mods_at(11), *mm = mods_at(12), *u = mods_at(13); CHECK(hm->kind == (MOD_ROMFS | MOD_SUBSDK) && hm->problem == MOD_PLUGIN, "RomFS + subsdk plugin: unusable"); CHECK(im->kind == (MOD_ROMFS | MOD_SUBSDK) && im->problem == MOD_PLUGIN, "RomFS + replaced exefs/main"); CHECK(ends_with(b->romfs, "B/atmosphere/contents/0100F2C0115B6000/romfs"), "Atmosphere RomFS root"); CHECK(ends_with(c->romfs, "C/contents/0100F2C0115B6000/romfs"), "contents RomFS root"); CHECK(ends_with(a->romfs, "A/romfs") && a->kind == MOD_ROMFS && a->problem == MOD_OK, "plain RomFS mod"); CHECK(!c->enabled && a->enabled && b->enabled && g->enabled && u->enabled, "mods_disabled switches C off"); - CHECK(d->kind == MOD_EXEFS && d->problem == MOD_EXEFS_LATER, "ExeFS patch mod"); + CHECK(d->kind == MOD_EXEFS && d->problem == MOD_EXEFS_OTHER && d->other_version == 1, + "ExeFS patch mod without a build id: another version's"); CHECK(e->problem == MOD_PLUGIN, "subsdk-only mod is a plugin mod"); CHECK(f->problem == MOD_EMPTY, "empty RomFS folder"); - CHECK(g->kind == (MOD_ROMFS | MOD_EXEFS) && g->problem == MOD_EXEFS_LATER, "RomFS + ExeFS mod"); + CHECK(g->kind == (MOD_ROMFS | MOD_EXEFS) && g->problem == MOD_OK && g->other_version == 1, + "RomFS + another version's ExeFS patch: the RomFS is used"); + CHECK(mm->kind == MOD_EXEFS && mm->problem == MOD_EXEFS_LATER && mm->pending == 1, + "an ExeFS patch for main the build lacks waits for the rebuild"); CHECK(jm->kind == MOD_CHEATS && jm->problem == MOD_OK && ends_with(jm->cheats, "J/cheats"), "cheats-only mod"); CHECK(km->kind == MOD_ROMFS && km->problem == MOD_OK && ends_with(km->romfs, "K/romfslite"), "TKMM romfslite"); @@ -102,7 +111,7 @@ int probe_mods(void) { mods_move(1, -1); CHECK(EQ(mods_at(0)->name, "A") && EQ(mods_at(1)->name, "B"), "move up"); - CHECK(EQ(cfg_get("TOTK_MODS_ORDER"), "A|B|C|D|E|F|G|H|I|J|K|L|Ultra \xD0\x96"), + CHECK(EQ(cfg_get("TOTK_MODS_ORDER"), "A|B|C|D|E|F|G|H|I|J|K|L|M|Ultra \xD0\x96"), "order written without the gone name"); mods_move(0, -1); CHECK(EQ(mods_at(0)->name, "A"), "move up at the top does nothing"); diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 7319ac2e..5c239608 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1063,16 +1063,26 @@ static void tab_mods(float scale) { if (count) snprintf(cheat, sizeof cheat, menu_text(UI_MENU_MODS_CHEATS), count); else snprintf(cheat, sizeof cheat, menu_text(UI_MENU_MODS_CHEATS_FILE), cheats_file()); } - UiTextId problem = m->problem == MOD_EMPTY ? UI_MENU_MODS_EMPTY - : m->problem == MOD_PLUGIN ? UI_MENU_MODS_SUBSDK - : m->problem == MOD_EXEFS_LATER ? UI_MENU_MODS_EXEFS_LATER - : m->problem == MOD_BAD_NAME ? UI_MENU_MODS_BAD_NAME - : UI_TEXT_COUNT; + char code[512] = "", problem[512] = ""; + if (m->nsets && m->overrides[0]) + snprintf(code, sizeof code, menu_text(UI_MENU_MODS_REPLACES_BUILTIN), m->nsets, m->overrides); + else if (m->nsets) + snprintf(code, sizeof code, menu_text(UI_MENU_MODS_EXEFS_ON), m->nsets); + switch (m->problem) { + case MOD_EMPTY: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EMPTY)); break; + case MOD_PLUGIN: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_SUBSDK)); break; + case MOD_EXEFS_LATER: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EXEFS_LATER)); break; + case MOD_EXEFS_UNSUPPORTED: + snprintf(problem, sizeof problem, menu_text(UI_MENU_MODS_EXEFS_UNSUPPORTED), m->note); + break; + case MOD_EXEFS_OTHER: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EXEFS_OTHER)); break; + case MOD_BAD_NAME: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_BAD_NAME)); break; + default: break; + } int k = snprintf(desc, sizeof desc, menu_text(UI_MENU_MODS_ROW_DESC), m->replaced, m->added); - if (k >= 0 && k < (int)sizeof desc) - snprintf(desc + k, sizeof desc - k, "%s%s%s%s%s%s%s%s\n\n%s", over[0] ? "\n\n" : "", over, - other[0] ? "\n\n" : "", other, cheat[0] ? "\n\n" : "", cheat, problem != UI_TEXT_COUNT ? "\n\n" : "", - problem != UI_TEXT_COUNT ? menu_text(problem) : "", menu_text(UI_MENU_MODS_MOVE_HINT)); + const char *parts[] = {over, other, cheat, code, problem, menu_text(UI_MENU_MODS_MOVE_HINT)}; + for (const char *part : parts) + if (*part && k >= 0 && k < (int)sizeof desc) k += snprintf(desc + k, sizeof desc - k, "\n\n%s", part); const char *note = m->enabled && m->problem != MOD_OK ? menu_text(UI_MENU_MODS_UNUSED) : after; int s = opt_text_row(id, m->name, menu_text(m->enabled ? UI_MENU_ON : UI_MENU_OFF), note, desc, true); if (s == 2 || s == -1 || s == 1) { @@ -1086,6 +1096,8 @@ static void tab_mods(float scale) { opt_save_soon(); } } + bool rebuild = + cheats_rebuild_pending() && opt_button("##restart", UI_MENU_CHEATS_REBUILD, UI_MENU_CHEATS_REBUILD_DESC); static uint64_t open_failed; // plat_now_ns until which the button says it failed if (opt_button("##mods_open", UI_MENU_MODS_OPEN, UI_MENU_MODS_OPEN_DESC, plat_now_ns() < open_failed ? menu_text(UI_MENU_ACTION_FAILED) : "")) { @@ -1095,6 +1107,11 @@ static void tab_mods(float scale) { } mods_builtin_rows(); opt_end(); + if (rebuild) { + restart_rebuild = true; + ImGui::OpenPopup("##restart_confirm"); + } + restart_confirm(); } // ---- the window: title, tabs (L / R or Q / E switch), the page, and the bottom row with "Continue" and "Quit game". diff --git a/runtime/ui_text.c b/runtime/ui_text.c index e3c21958..5c14ed53 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -171,8 +171,8 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_MODS_EMPTY] = "This folder has no romfs, exefs or cheats files.", [UI_MENU_MODS_SUBSDK] = "This mod brings its own game code (a subsdk plugin or a replaced main). The port cannot " "load code modules, so the whole mod stays off.", - [UI_MENU_MODS_EXEFS_LATER] = "This mod patches the game code (exefs). Code patches are not supported yet, so the " - "whole mod stays off.", + [UI_MENU_MODS_EXEFS_LATER] = "This mod patches the game code (exefs). The game builds the patches in when it " + "starts next (a rebuild of a few minutes); until then the whole mod stays off.", [UI_MENU_MODS_CHEATS] = "Cheats: %d. Switch them on the Cheats tab.", [UI_MENU_MODS_CHEATS_FILE] = "Cheats for this game version are read at start from cheats\\%s in this mod.", [UI_MENU_MODS_BAD_NAME] = "The folder name contains |, which the mod list uses between names. Rename the folder.", @@ -439,6 +439,12 @@ static const char *const text_en[UI_TEXT_COUNT] = { "edited, or their last rebuild failed or was cancelled. Closes the game; the setup " "builds them in (a few minutes) and starts the game again. Progress since your " "last save is lost.", + [UI_MENU_MODS_EXEFS_UNSUPPORTED] = "This mod patches the game code (exefs) in a way the port cannot apply, so the " + "whole mod stays off. %s", + [UI_MENU_MODS_EXEFS_OTHER] = "Its code patches (exefs) are made for another game version.", + [UI_MENU_MODS_EXEFS_ON] = "Code patches in use: %d files.", + [UI_MENU_MODS_REPLACES_BUILTIN] = "Code patches in use: %d files. They replace the port's own patch %s, which " + "stays off while this mod is on.", }; static const char *const text_ru[UI_TEXT_COUNT] = { @@ -588,8 +594,8 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_MODS_EMPTY] = "В этой папке нет файлов romfs, exefs или cheats.", [UI_MENU_MODS_SUBSDK] = "Этот мод приносит свой код игры (плагин subsdk или заменённый main). Порт не загружает " "модули кода, поэтому мод выключен целиком.", - [UI_MENU_MODS_EXEFS_LATER] = "Этот мод меняет код игры (exefs). Патчи кода пока не поддерживаются, поэтому мод " - "выключен целиком.", + [UI_MENU_MODS_EXEFS_LATER] = "Этот мод меняет код игры (exefs). Игра соберёт эти патчи при следующем запуске " + "(пересборка на несколько минут); до тех пор мод выключен целиком.", [UI_MENU_MODS_CHEATS] = "Читов: %d. Они включаются на вкладке «Читы».", [UI_MENU_MODS_CHEATS_FILE] = "Читы для этой версии игры читаются при запуске из cheats\\%s в этом моде.", [UI_MENU_MODS_BAD_NAME] = "В имени папки есть символ |, которым список модов разделяет имена. Переименуйте папку.", @@ -860,6 +866,12 @@ static const char *const text_ru[UI_TEXT_COUNT] = { "либо прошлая пересборка не удалась или была отменена. Закрывает игру; установщик " "собирает их (несколько минут) и снова запускает игру. Прогресс после последнего " "сохранения будет потерян.", + [UI_MENU_MODS_EXEFS_UNSUPPORTED] = "Этот мод меняет код игры (exefs) так, как порт применить не может, поэтому мод " + "выключен целиком. %s", + [UI_MENU_MODS_EXEFS_OTHER] = "Его патчи кода (exefs) сделаны для другой версии игры.", + [UI_MENU_MODS_EXEFS_ON] = "Патчи кода в работе: файлов: %d.", + [UI_MENU_MODS_REPLACES_BUILTIN] = "Патчи кода в работе: файлов: %d. Они заменяют собственный патч порта %s, " + "который выключен, пока этот мод включён.", }; static const char *const text_pt_br[UI_TEXT_COUNT] = { @@ -881,8 +893,9 @@ static const char *const text_pt_br[UI_TEXT_COUNT] = { [UI_MENU_MODS_EMPTY] = "Esta pasta não tem arquivos romfs, exefs ou cheats.", [UI_MENU_MODS_SUBSDK] = "Este mod traz código próprio do jogo (um plugin subsdk ou um main substituído). O port não " "carrega módulos de código, então o mod inteiro fica desligado.", - [UI_MENU_MODS_EXEFS_LATER] = "Este mod altera o código do jogo (exefs). Patches de código ainda não são suportados, " - "então o mod inteiro fica desligado.", + [UI_MENU_MODS_EXEFS_LATER] = "Este mod altera o código do jogo (exefs). O jogo compila esses patches na próxima " + "vez que iniciar (uma recompilação de alguns minutos); até lá o mod inteiro fica " + "desligado.", [UI_MENU_MODS_CHEATS] = "Cheats: %d. Ligue-os na aba Cheats.", [UI_MENU_MODS_CHEATS_FILE] = "Os cheats desta versão do jogo são lidos ao iniciar de cheats\\%s neste mod.", [UI_MENU_MODS_BAD_NAME] = "O nome da pasta contém |, que a lista de mods usa entre os nomes. Renomeie a pasta.", @@ -1309,6 +1322,12 @@ static const char *const text_pt_br[UI_TEXT_COUNT] = { "novos ou editados, ou a última recompilação falhou ou foi cancelada. Fecha o " "jogo; o instalador os compila (alguns minutos) e inicia o jogo de novo. O " "progresso desde o último salvamento é perdido.", + [UI_MENU_MODS_EXEFS_UNSUPPORTED] = "Este mod altera o código do jogo (exefs) de um jeito que o port não consegue " + "aplicar, então o mod inteiro fica desligado. %s", + [UI_MENU_MODS_EXEFS_OTHER] = "Os patches de código dele (exefs) são para outra versão do jogo.", + [UI_MENU_MODS_EXEFS_ON] = "Patches de código em uso: %d arquivos.", + [UI_MENU_MODS_REPLACES_BUILTIN] = "Patches de código em uso: %d arquivos. Eles substituem o patch do próprio port " + "%s, que fica desligado enquanto este mod está ligado.", }; // Simplified Chinese: this row covers the software keyboard and the whole menu, so the menu and the overlay read Chinese @@ -1337,7 +1356,7 @@ static const char *const text_zh_hans[UI_TEXT_COUNT] = { "更改在重启后生效。", [UI_MENU_MODS_EMPTY] = "此文件夹中没有 romfs、exefs 或 cheats 文件。", [UI_MENU_MODS_SUBSDK] = "此模组带有自己的游戏代码(subsdk 插件或替换的 main)。移植版无法加载代码模块,因此整个模组保持关闭。", - [UI_MENU_MODS_EXEFS_LATER] = "此模组修改游戏代码(exefs)。目前还不支持代码补丁,因此整个模组保持关闭。", + [UI_MENU_MODS_EXEFS_LATER] = "此模组修改游戏代码(exefs)。游戏会在下次启动时把这些补丁构建进去(重新构建需要几分钟);在此之前整个模组保持关闭。", [UI_MENU_MODS_CHEATS] = "金手指:%d 个。在“金手指”标签页中开关。", [UI_MENU_MODS_CHEATS_FILE] = "此游戏版本的金手指在启动时从此模组的 cheats\\%s 读取。", [UI_MENU_MODS_BAD_NAME] = "文件夹名称含有 |,模组列表用它分隔名称。请重命名该文件夹。", @@ -1853,6 +1872,10 @@ static const char *const text_zh_hans[UI_TEXT_COUNT] = { [UI_MENU_CHEAT_UNSUPPORTED_DESC] = "此金手指不能用于此游戏版本:", [UI_MENU_CHEATS_REBUILD] = "立即重新构建游戏", [UI_MENU_CHEATS_REBUILD_DESC] = "有些金手指会修改游戏代码,但尚未构建进游戏:它们是新增或修改的,或上次重新构建失败或被取消。将关闭游戏;安装程序会把它们构建进去(几分钟)并重新启动游戏。上次存档之后的进度会丢失。", + [UI_MENU_MODS_EXEFS_UNSUPPORTED] = "此模组修改游戏代码(exefs)的方式本移植无法应用,因此整个模组保持关闭。%s", + [UI_MENU_MODS_EXEFS_OTHER] = "它的代码补丁(exefs)是为其他游戏版本制作的。", + [UI_MENU_MODS_EXEFS_ON] = "正在使用的代码补丁:%d 个文件。", + [UI_MENU_MODS_REPLACES_BUILTIN] = "正在使用的代码补丁:%d 个文件。它们取代本移植自带的补丁 %s,此模组开启时该补丁保持关闭。", }; #define SWKBD_ONLY(name, ...) static const char *const name[UI_TEXT_COUNT] = {SWKBD(__VA_ARGS__)} diff --git a/runtime/ui_text.h b/runtime/ui_text.h index abd1d49c..92856857 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -75,6 +75,10 @@ typedef enum { UI_MENU_MODS_OVERRIDDEN, // printf: files a higher mod serves instead, that mod's name UI_MENU_MODS_MOVE_HINT, UI_MENU_MODS_EMPTY, UI_MENU_MODS_SUBSDK, UI_MENU_MODS_EXEFS_LATER, UI_MENU_MODS_BAD_NAME, + UI_MENU_MODS_EXEFS_UNSUPPORTED, // printf: the file and why + UI_MENU_MODS_EXEFS_OTHER, + UI_MENU_MODS_EXEFS_ON, // printf: its ExeFS patch files in use + UI_MENU_MODS_REPLACES_BUILTIN, // printf: its ExeFS patch files in use, the port's own patch sets they turn off UI_MENU_MODS_OTHER_VERSION, // printf: files made for another game version, left out UI_MENU_MODS_UNUSED, // under the switch of a mod the port cannot use UI_MENU_MODS_CHEATS, // printf: its cheats as read at start diff --git a/tests/installer_test.py b/tests/installer_test.py index 0cba2c9f..3f773c32 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -243,6 +243,17 @@ class LiftKeyTest(unittest.TestCase): self.add_cheats('277178B7DBA1B6D4.txt', '[c]', mod='Other', where='atmosphere/contents/0100F2C0115B6000/cheats') self.assertNotEqual(self.key(), edited) + def test_exefs_patch_files_change_the_key(self): + before = self.key() + self.add_cheats('dynamic-fps.pchtxt', '@nsobid-277178B7DBA1B6D4', mod='Fps', where='exefs') + added = self.key() + self.assertNotEqual(added, before) + self.add_cheats('277178B7DBA1B6D4.ips', 'PATCHEOF', mod='Ips', where='atmosphere/exefs_patches/x') + both = self.key() + self.assertNotEqual(both, added) + self.add_cheats('main.npdm', 'not a patch', mod='Fps', where='exefs') + self.assertEqual(self.key(), both) + def test_status_and_other_files_keep_the_key(self): self.add_cheats('277178B7DBA1B6D4.txt', '[a]') before = self.key() diff --git a/tools/exefs_patches.py b/tools/exefs_patches.py new file mode 100644 index 00000000..7f221a46 --- /dev/null +++ b/tools/exefs_patches.py @@ -0,0 +1,230 @@ +"""ExeFS patches of the mods folder to code patch sets: IPS, IPS32 and IPSwitch (.pchtxt) files for the game's main. + +Lifted code cannot be byte-patched when the game runs, so a patch file's writes into main's code become `site` lines +(recomp/patches.py), one patch set per file, which the runtime switches on while its mod is on (runtime/mods.c). The +writes go through the same rules as the mods' code cheats (tools/cheats.py convert): an instruction replaced in place, +a branch into a code cave the file writes into main's zero padding inlined at the branch. + + python tools/exefs_patches.py --exefs <dir> --mods <mods folder> --patches <patch file> --out <patch file> --status <file> + +The files are <mod>/exefs/*.ips and *.pchtxt (or the Atmosphere places of the exefs folder, as runtime/mods.c finds +them) and <mod>/atmosphere/exefs_patches/<name>/*.ips. A file is for main when its build id matches main's: an IPS +file's name, a pchtxt file's @nsobid line (trailing zeros may be left out). Offsets are as emulators read them: IPS +offsets count the NSO header (0x100 bytes) before main's image; a pchtxt offset adds its `@flag offset_shift` first. +A pchtxt patch line applies between @enabled and @disabled; @stop ends the file. + +--out gets the --patches file followed by one set per usable file (`mod_<hash>`, switch TOTK_MODPATCH_<HASH>, default +off) and its sites. --status gets one line per file, mods by name, tab separated: + + <file hash> <content hash> <kind> <built-in sets> <note> + +file hash: FNV-1a 32 of "<mod folder name>/<path in the mod>" (forward slashes); content hash: the same over the file's +bytes, so the runtime sees a file that changed after the build. kind: + code a set of sites + other for another game version or module: not used + empty no write into main for this version (or only the game's own words) + unsupported <note> says why: a write into main's data, a write the site rules cannot express +built-in sets: the port's own sets (the --patches file's sets that are no cheats) with a site this file patches too, +comma separated, or "-": the runtime switches them off while the mod is on, so the mod's code runs. + +A file the rules cannot express is unsupported as a whole: half of a patch is worse than none. + +Standard library, recomp/ and tools/cheats.py only. +""" +import argparse +import re +import struct +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from cheats import TITLE, Main, convert, fnv # noqa: E402 + +NSO_HEADER = 0x100 +EXEFS_PLACES = ('exefs', f'atmosphere/contents/{TITLE}/exefs', f'contents/{TITLE}/exefs', f'{TITLE}/exefs') + + +def patch_files(mods): + """[(mod folder name, path in the mod, file)] of the folders of mods, by name: the .ips and .pchtxt files of the + first exefs place that exists (runtime/mods.c find_root) and of atmosphere/exefs_patches/<name>/.""" + out = [] + for mod in sorted((d for d in mods.iterdir() if d.is_dir()), key=lambda d: d.name) if mods.is_dir() else []: + folders = [] + exefs = next((mod / w for w in EXEFS_PLACES if (mod / w).is_dir()), None) + if exefs: + folders.append(exefs) + patches = mod / 'atmosphere' / 'exefs_patches' + if patches.is_dir(): + folders += sorted(d for d in patches.iterdir() if d.is_dir()) + for folder in folders: + for f in sorted(folder.iterdir()): + if f.is_file() and f.suffix.lower() in ('.ips', '.pchtxt'): + out.append((mod.name, f.relative_to(mod).as_posix(), f)) + return out + + +def same_build(name, build_id): + """1 when name (hex, any case, trailing zeros left out or not) is main's build id (hex of its 32 bytes).""" + name = name.lower() + return bool(re.fullmatch(r'[0-9a-f]{1,64}', name)) and name.ljust(64, '0') == build_id.ljust(64, '0') + + +# ---- the formats: {offset in main's image: byte} + +def parse_ips(data): + """IPS (PATCH ... EOF, 3-byte offsets) or IPS32 (IPS32 ... EEOF, 4-byte offsets); ValueError when it is neither.""" + if data[:5] == b'PATCH': + width, end, at = 3, b'EOF', 5 + elif data[:5] == b'IPS32': + width, end, at = 4, b'EEOF', 5 + else: + raise ValueError('not an IPS or IPS32 file') + writes = {} + while True: + if data[at:at + len(end)] == end and at + len(end) >= len(data) - 3: + return writes + if at + width + 2 > len(data): + raise ValueError('the IPS file is cut short') + off = int.from_bytes(data[at:at + width], 'big') + size = int.from_bytes(data[at + width:at + width + 2], 'big') + at += width + 2 + if size: + body = data[at:at + size] + at += size + else: # a run: 2-byte length, 1 byte + run = int.from_bytes(data[at:at + 2], 'big') + body = data[at + 2:at + 3] * run + at += 3 + if len(body) != (size or len(body)): + raise ValueError('the IPS file is cut short') + for i, b in enumerate(body): + writes[off - NSO_HEADER + i] = b + + +def pchtxt_value(token): + """The bytes of a pchtxt value: hex digits in byte order, or a quoted string with C escapes.""" + if token.startswith('"'): + if not token.endswith('"') or len(token) < 2: + raise ValueError(f'a string without its closing quote: {token}') + return token[1:-1].encode('utf-8').decode('unicode_escape').encode('latin-1') + if len(token) % 2 or not re.fullmatch(r'[0-9A-Fa-f]+', token): + raise ValueError(f'not a hex value: {token}') + return bytes.fromhex(token) + + +def parse_pchtxt(text): + """(build id or None, {offset in main's image: byte}) of an IPSwitch file.""" + bid, shift, enabled, writes = None, 0, False, {} + for raw in text.splitlines(): + line = raw.split('//', 1)[0].strip() + if not line: + continue + if line.startswith('@stop'): + break + if line.startswith('@nsobid-'): + bid = line[len('@nsobid-'):].strip() + elif line.startswith('@enabled'): + enabled = True + elif line.startswith('@disabled'): + enabled = False + elif line.startswith('@flag'): + f = line.split() + if len(f) >= 3 and f[1] == 'offset_shift': + shift = int(f[2], 0) + elif line.startswith('@') or not enabled: + continue + else: + m = re.fullmatch(r'([0-9A-Fa-f]+)\s+(".*"|\S+)', line) + if not m: + continue # a title in brackets or a line of text + off = int(m.group(1), 16) + shift - NSO_HEADER + for i, b in enumerate(pchtxt_value(m.group(2))): + writes[off + i] = b + return bid, writes + + +# ---- writes to sites + +def word_writes(main, writes): + """{offset: word} of byte writes into main's code (.text and its padding), each word with the bytes the file does + not write kept; ValueError for a write into main's data.""" + words = {} + for off in sorted(writes): + if not main.in_code(off - off % 4): + raise ValueError(f'it writes main\'s data at main+{off:X} (only code patches are supported)') + a = off - off % 4 + w = words.get(a, main.word(a)) + shift = (off % 4) * 8 + words[a] = (w & ~(0xFF << shift) | writes[off] << shift) & 0xFFFFFFFF + return words + + +def builtin_sites(text): + """{offset in main: [set name]} of the patch file's sets that are no cheats or mod patches.""" + out = {} + for line in text.splitlines(): + f = line.split('#', 1)[0].split() + if len(f) >= 4 and f[0] == 'site' and f[2] == 'main' and not f[1].startswith(('cheat_', 'mod_')): + out.setdefault(int(f[3], 16), []).append(f[1]) + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--exefs', type=Path, required=True) + ap.add_argument('--mods', type=Path, required=True, help='the mods folder next to totk.exe') + ap.add_argument('--patches', type=Path, required=True, help='the patch file so far') + ap.add_argument('--out', type=Path, required=True, help='the patch file with the mods\' ExeFS patches added') + ap.add_argument('--status', type=Path, required=True) + args = ap.parse_args() + sys.stdout.reconfigure(errors='replace') + m = Main(args.exefs / 'main') + raw = (args.exefs / 'main').read_bytes() + bid = raw[0x40:0x60].hex() + text = args.patches.read_text() + builtin = builtin_sites(text) + files = patch_files(args.mods) + sets, sites, status, kinds = [], [], [], {} + for mod, rel, f in files: + data = f.read_bytes() + key, content = fnv(f'{mod}/{rel}'.encode()), fnv(data) + kind, note, over, file_sites = 'code', '', [], {} + try: + if f.suffix.lower() == '.ips': + for_main = same_build(f.stem, bid) + writes = parse_ips(data) if for_main else {} + else: + pid, writes = parse_pchtxt(data.decode('utf-8-sig', errors='replace')) + for_main = pid is not None and same_build(pid, bid) + if not for_main: + kind = 'other' + else: + file_sites = convert(m, word_writes(m, writes)) + if not file_sites: + kind = 'empty' + except ValueError as e: + kind, note, file_sites = 'unsupported', str(e), {} + if file_sites: + name = f'mod_{key.lower()}' + label = f'{mod}/{rel}'.encode('ascii', 'replace').decode() + sets.append(f'set {name} TOTK_MODPATCH_{key} 0 # {label}') + for a, repl in sorted(file_sites.items()): + sites.append(f'site {name} main {a:x} {m.word(a):08x} {" ".join(f"{w:08x}" for w in repl)}') + over += [s for s in builtin.get(a, []) if s not in over] + kinds[kind] = kinds.get(kind, 0) + 1 + status.append(f'{key}\t{content}\t{kind}\t{",".join(over) or "-"}\t{note.replace(chr(9), " ")}') + print(f'exefs: {kind:11} {mod}/{rel}' + (f' ({note})' if note else '') + + (f' ({len(file_sites)} sites, over {", ".join(over)})' if over else + f' ({len(file_sites)} sites)' if file_sites else '')) + if sets: + text += ('\n# ExeFS patches of the mods folder (tools/exefs_patches.py)\n' + '\n'.join(sets) + '\n' + + '\n'.join(sites) + '\n') + args.out.write_text(text) + args.status.parent.mkdir(parents=True, exist_ok=True) + args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8') + print(f'exefs: {len(files)} patch files: ' + ', '.join(f'{n} {k}' for k, n in sorted(kinds.items())) + + f'; {len(sites)} sites') + + +if __name__ == '__main__': + main() diff --git a/tools/make_release.py b/tools/make_release.py index da0e1666..08d371ea 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -104,7 +104,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py', 'pack_shader_cache.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', - 'shader_list.py', 'ultrawide_layout.py', 'cheats.py'] + 'shader_list.py', 'ultrawide_layout.py', 'cheats.py', 'exefs_patches.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon