diff --git a/docs/mods.md b/docs/mods.md index 089207b9..9e0f1d76 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -103,12 +103,15 @@ Inventory Expansion's sort fixer, Quick Weapon Durability). On the Switch the pl redirects some of the game's instructions into its own code. The port does the same with the plugin's code compiled ahead of time, after the design in [#845](https://github.com/valentderah/totk-pc-recomp/issues/845) by itaskaev2: -1. The rebuild (the same one as for code patches above) compiles the plugin with the game, starts the game's code - once in a service mode that runs only the plugin's start-up (`totk.exe --plugin-init`), twice, and records which - instructions it redirects. Each redirect becomes a code site of the mod, built into `totk.exe`. -2. At every start the plugin's start-up runs again, before the game's own start-up code, in the same setting. Its - redirects are compared with the record of the rebuild. When they match, the mod's code sites go on. When they do - not, the plugin stays off for this run, the mod's row says why, and the next start runs the rebuild. +1. The rebuild (the same one as for code patches above) compiles the plugins with the game, then starts the game's + code in a service mode that runs only one plugin's start-up (`totk.exe --plugin-init`), twice per plugin, and + records which instructions it redirects. Each redirect becomes a code site of the mod, built into `totk.exe`. Each + plugin is judged on its own: one that is refused, or whose start-up crashes or never returns, stays off with its + reason, and the other plugins are built in. +2. At every start each plugin's start-up runs again, before the game's own start-up code, in the same setting: it + sees the game's own instructions, as in its service run. Its redirects are compared with the record of the + rebuild. When they match, the mod's code sites go on. When they do not, the plugin stays off for this run, the + mod's row says why, and the next start runs the rebuild. 3. The plugin keeps what its start-up made (its memory, its tables), so its code works as on the Switch. The start costs about 50 ms per plugin; the game runs no differently without a plugin. @@ -119,8 +122,9 @@ The rebuild refuses a plugin when its start-up or the code its redirects reach n system calls other than the few its start-up needs (memory queries and maps, the clock, debug output), library calls outside a reviewed list (memory and string functions, the heap, text formatting), threads, services or input; a redirect that is not one of exlaunch's hook shapes (inline hook, replace hook, call-site rewrite); or two runs of its -start-up that differ in code. The row then says **Its plugin ... cannot run in the port** with the reason, and the -mod stays off; **Use without its code** uses the rest of it. UltraCam stays refused: it hooks the graphics API, +start-up that differ in code; or a start-up that crashes the service run or does not return within two minutes. The +row then says **Its plugin ... cannot run in the port** with the reason, and the mod stays off; **Use without its +code** uses the rest of it, including its other plugins that were built in. UltraCam stays refused: it hooks the graphics API, reads input and opens sockets. `plugin_trace = 1` in `totk.ini` logs the first call into each of a plugin's hooks (`plugins: hook entry ... reached`). diff --git a/installer/steps.py b/installer/steps.py index 08ff4baa..2510e376 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -1009,7 +1009,10 @@ class Lift(Step): skipped['warn.plugins_skipped'] = log else: patches = ctx.temp / 'patches-plugins.txt' - lift_plugins = built_plugins(plugins_status(ctx), plugins, ctx.dest / 'mods') + # With one plugin built, the layout keeps every plugin of the service runs: the records hold addresses + # of that layout. A refused plugin stays loaded at play, its init never runs (no patch set). + if built_plugins(plugins_status(ctx), plugins, ctx.dest / 'mods'): + lift_plugins = plugins if lift_plugins: args += ['--plugins', ','.join(str(f) for f in lift_plugins)] code, log = run_tool(ctx, 'lift', layout.recomp / 'gen.py', args + ['--patches', patches], parse) diff --git a/runtime/mods.c b/runtime/mods.c index 5d7667da..0d972f51 100644 --- a/runtime/mods.c +++ b/runtime/mods.c @@ -446,11 +446,12 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) { int unsupported = exefs_check(m, path); // A mod whose code part the port cannot use is off as a whole: its RomFS files may expect that code. Used without // its code (its switch, or mods_without_code = 1 for every mod), the code modules (m->code) and the ExeFS patch - // files the port cannot apply are left out and the rest of the mod is used, when it has a rest. + // files the port cannot apply are left out and the rest of the mod is used, when it has a rest. Each plugin of a + // mod is built or refused on its own (tools/exefs_plugins.py), so a built plugin is such a rest too. m->no_code = listed(cfg_get("TOTK_MODS_NO_CODE"), m->name); m->without_code = m->no_code || cfg_bool("TOTK_MODS_WITHOUT_CODE"); int usable = m->nsets || m->ndata || m->kind & (MOD_ROMFS | MOD_CHEATS); - int code_off = m->code[0] && (!m->without_code || !(m->kind & (MOD_ROMFS | MOD_EXEFS | MOD_CHEATS))); + int code_off = m->code[0] && (!m->without_code || !(m->kind & (MOD_ROMFS | MOD_EXEFS | MOD_CHEATS) || m->nplugins)); m->problem = strchr(m->name, '|') ? MOD_BAD_NAME : code_off ? MOD_PLUGIN : unsupported && (!m->without_code || !usable) ? MOD_EXEFS_UNSUPPORTED diff --git a/runtime/mods_probe.c b/runtime/mods_probe.c index 473ea1e5..0d1e22c0 100644 --- a/runtime/mods_probe.c +++ b/runtime/mods_probe.c @@ -212,6 +212,8 @@ static const char *refused_plugins(void) { // Q an ExeFS patch file on P's site // R a plugin the build has no line for waits for the rebuild // S RomFS + a refused plugin off, the reason kept; used without its code, the rest runs +// T a built plugin + a refused one each judged alone: off for the refused one, its reason kept; used +// without its code, the built one's set runs // Then P's start differs from the build (mods_plugin_off): its set goes off, Q's code runs at the site, the conflict // goes, a stale line marks P's file, and the next scan has P wait for the rebuild. static void probe_plugins(const char *root) { @@ -221,18 +223,22 @@ static void probe_plugins(const char *root) { char q[160]; snprintf(q, sizeof q, "Q/exefs/%s.ips", bid); static const char *const files[] = {"P/romfs/p.bin", "P/exefs/subsdk9", "R/exefs/subsdk9", "S/romfs/s.bin", - "S/exefs/subsdk9"}; + "S/exefs/subsdk9", "T/exefs/subsdk8", "T/exefs/subsdk9"}; for (size_t i = 0; i < sizeof files / sizeof *files; i++) put(root, files[i]); put(root, q); - static char names[3][16]; - static const char *const table[] = {"fps60", names[1], names[2]}; + static char names[4][16]; + static const char *const table[] = {"fps60", names[1], names[2], names[3]}; set_name(names[1], sizeof names[1], "P/exefs/subsdk9"); set_name(names[2], sizeof names[2], q); - char status[256], plugins[256]; + set_name(names[3], sizeof names[3], "T/exefs/subsdk9"); + char status[256], plugins[512]; snprintf(status, sizeof status, "%08X\t%08X\tcode\t-\t\n", fnv(q, strlen(q)), fnv("probe", 5)); snprintf(plugins, sizeof plugins, "%08X\t%08X\tplugin\tfps60\tplugin0, 7 hooks\n" - "%08X\t%08X\tunsupported\t-\tthe code its hooks reach has svc 0x40\n", fnv("P/exefs/subsdk9", 15), - fnv("probe", 5), fnv("S/exefs/subsdk9", 15), fnv("probe", 5)); + "%08X\t%08X\tunsupported\t-\tthe code its hooks reach has svc 0x40\n" + "%08X\t%08X\tplugin\t-\tplugin1, 2 hooks\n" + "%08X\t%08X\tunsupported\t-\tthe run was refused: svc 0x1 at plugin2+0x50a0\n", fnv("P/exefs/subsdk9", 15), + fnv("probe", 5), fnv("S/exefs/subsdk9", 15), fnv("probe", 5), fnv("T/exefs/subsdk9", 15), fnv("probe", 5), + fnv("T/exefs/subsdk8", 15), fnv("probe", 5)); static const int16_t s0[] = {1, 2}; static const RtPatchPick picks[] = {{"main", 0x100, s0, 2}}; mods_probe_cache(status, ""); @@ -244,12 +250,13 @@ static void probe_plugins(const char *root) { cfg_set("TOTK_MODS_ORDER", round ? "Q|P" : "P|Q"); mods_rescan(); mods_scan(); - rt_patch_probe_table(table, 3, picks, 1); + rt_patch_probe_table(table, 4, picks, 1); rt_patch_switch(0, 1); mods_exefs_apply(0, 0, 0, NULL); - const ModInfo *pm = by_name("P"), *qm = by_name("Q"), *rm = by_name("R"), *sm = by_name("S"); - CHECK(pm && qm && rm && sm, "plugin scan: four mods"); - if (!pm || !qm || !rm || !sm) break; + const ModInfo *pm = by_name("P"), *qm = by_name("Q"), *rm = by_name("R"), *sm = by_name("S"), + *tm = by_name("T"); + CHECK(pm && qm && rm && sm && tm, "plugin scan: five mods"); + if (!pm || !qm || !rm || !sm || !tm) break; CHECK(pm->problem == MOD_OK && pm->nsets == 1 && pm->nplugins == 1 && pm->plugin_hooks == 7 && !pm->code[0] && EQ(pm->overrides, "fps60"), "a built plugin: its set, its hooks, the built-in set it replaces"); CHECK(rt_patch_enabled(1) && !rt_patch_enabled(0), "the plugin's set on, the built-in set it replaces off"); @@ -261,25 +268,32 @@ static void probe_plugins(const char *root) { CHECK(rm->problem == MOD_EXEFS_LATER && rm->pending == 1, "a plugin the build lacks waits for the rebuild"); CHECK(sm->problem == MOD_PLUGIN && EQ(sm->code, "subsdk9") && strstr(sm->plugin_note, "svc 0x40"), "a refused plugin keeps its mod off and says why"); + CHECK(tm->problem == MOD_PLUGIN && tm->nplugins == 1 && tm->plugin_hooks == 2 && EQ(tm->code, "subsdk8") && + strstr(tm->plugin_note, "svc 0x1") && !rt_patch_enabled(3), + "a built plugin beside a refused one: the mod off, the refused one's reason on its row"); } // P above Q, then P's start differs from the build cfg_set("TOTK_MODS_ORDER", "P|Q"); mods_rescan(); mods_scan(); - rt_patch_probe_table(table, 3, picks, 1); + rt_patch_probe_table(table, 4, picks, 1); mods_exefs_apply(0, 0, 0, NULL); mods_plugin_off(1, "probe: 1 of the words the build recorded were not written"); const ModInfo *pm = by_name("P"); CHECK(!rt_patch_enabled(1) && rt_patch_pick_winner(0) == 1, "a plugin off at start: the other mod's patch runs"); CHECK(pm && strstr(pm->plugin_off, "were not written"), "its row says why"); CHECK(mods_conflict_count() == 0, "its conflict goes with it"); - cfg_set("TOTK_MODS_NO_CODE", "S"); + cfg_set("TOTK_MODS_NO_CODE", "S|T"); mods_rescan(); mods_scan(); + rt_patch_probe_table(table, 4, picks, 1); + mods_exefs_apply(0, 0, 0, NULL); pm = by_name("P"); - const ModInfo *sm = by_name("S"); + const ModInfo *sm = by_name("S"), *tm = by_name("T"); CHECK(pm && pm->problem == MOD_EXEFS_LATER && pm->pending == 1, "the next start: its file waits for the rebuild"); CHECK(sm && sm->problem == MOD_OK && sm->plugin_note[0], "a refused plugin used without its code: the rest runs"); + CHECK(tm && tm->problem == MOD_OK && tm->plugin_note[0] && rt_patch_enabled(3), + "a built plugin beside a refused one, used without its code: the built one's set runs"); rt_patch_probe_table(NULL, 0, NULL, 0); cfg_set("TOTK_MODS_NO_CODE", ""); cfg_set("TOTK_MODS_ORDER", ""); diff --git a/runtime/plugin_init.c b/runtime/plugin_init.c index 227f6621..ffbfcd3a 100644 --- a/runtime/plugin_init.c +++ b/runtime/plugin_init.c @@ -2,9 +2,11 @@ // this runtime's own kernel and sdk. The design follows #845 by itaskaev2, with the init run natively instead of under // Unicorn, and run again at every start instead of an image taken after one run at build time. // -// The service run: totk.exe --plugin-init DIR runs every plugin's DT_INIT, records what the init did and exits before -// any game module's constructor, nnMain, graphics or audio. The record is the input of tools/exefs_plugins.py, which -// checks it and writes the plugin's patch set (its hook sites, recomp/patches.py `plugin` lines). +// The service run: totk.exe --plugin-init DIR runs the plugins' DT_INITs, records what the inits did and exits before +// any game module's constructor, nnMain, graphics or audio. TOTK_PLUGIN_ONLY names the one plugin module whose init +// runs ("-": none); the others stay loaded, as they are at play. tools/exefs_plugins.py runs each plugin on its own, +// so a plugin that is refused, crashes or hangs costs only itself; it checks each record and writes the plugin's patch +// set (its hook sites, recomp/patches.py `plugin` lines). // // At every start (plugins_start, from the module-init callback in boot.c): when the build has a plugin whose patch set // its mod switched on, sdk's DT_INIT runs first, then the plugin's, in the same environment as in the service run, and @@ -15,8 +17,11 @@ // its GOT, its alias addresses. What the init wrote into the game modules (its hook branches) and the plugin's .text // after it (its stubs and trampolines) are compared with the record the build used (patches.h RT_PLUGIN_WRITES and // RT_PLUGINS): equal, the plugin's set stays on and the generated hook sites call into it; different, the set goes off -// for this run, the reason is logged and the mod waits for a rebuild (mods.c mods_plugin_off). The branches written -// into main's .text are taken back afterwards: the lifted code is the code that runs, and main keeps the game's bytes. +// for this run, the reason is logged and the mod waits for a rebuild (mods.c mods_plugin_off). Each plugin is compared +// right after its own init, and the branches it wrote into main's .text are taken back before the next init: the next +// plugin sees the game's own bytes, as in its own service run (two plugins hooking one site each relocate the game's +// instruction; the generated site runs the one the Mods tab order picks). The lifted code is the code that runs, and +// main keeps the game's bytes. // Without a plugin in the build none of this runs (RT_PLUGIN_COUNT 0). // // What exlaunch's init needs from the process, and how this run gives it: @@ -37,10 +42,10 @@ // Map/UnmapProcessMemory); any other one, such as CreateSession and ReplyAndReceive of exlaunch's process-handle // fallback, ends the run with the reason in the record. // -// The record, in DIR: record.json (modules, the writes into every game module as words, the alias calls, svc and -// import counts, the tick offset, a refusal), plugin.img (the plugin's image after its init, all segments) and -// plugin.pre.img (the same after loading and relocation, before the init). TOTK_PLUGIN_TICK_OFFSET adds to every -// GetSystemTick, so two runs pick different aliases. +// The record, in DIR: record.json (modules, which plugins' inits ran, the writes into every game module as words, the +// alias calls, svc and import counts, the tick offset, a refusal), plugin.img (the plugin's image after its init, +// all segments) and plugin.pre.img (the same after loading and relocation, before the init). +// TOTK_PLUGIN_TICK_OFFSET adds to every GetSystemTick, so two runs pick different aliases. #include #include #include @@ -93,6 +98,7 @@ static uint64_t svc_count[0x80], info_count[16], info_65001; static struct { const char *name; uint64_t calls; } imports[MAX_IMPORTS]; static int nimports; static uint8_t **pre; // per loaded module: its image before the plugins' inits +static uint8_t *ran; // service run, per loaded module: its init ran static uint64_t align_up(uint64_t v, uint64_t a) { return (v + a - 1) & ~(a - 1); } @@ -344,6 +350,7 @@ static void write_record(const char *refused) { i ? "," : "", m->name, (unsigned long long)m->base, (unsigned long long)m->size, (unsigned long long)m->text_end, (unsigned long long)m->data_off, (unsigned long long)m->init, m->plugin ? "true" : "false"); + if (m->plugin) fprintf(f, ", \"ran\": %s", ran && ran[i] ? "true" : "false"); if (m->plugin && !refused) { char img[64], pre_img[64], hex[65]; snprintf(img, sizeof img, "%s.img", m->name); @@ -460,9 +467,17 @@ void plugin_init_run(Cpu *c, const RtLoaded *sdk) { if (!pre[i]) rt_fatal("plugin-init: out of memory for a copy of %s", m->name); memcpy(pre[i], (const void *)(uintptr_t)m->base, m->size); } + const char *only = cfg_get("TOTK_PLUGIN_ONLY"); + ran = calloc((size_t)ldr_count(), 1); + if (!ran) rt_fatal("plugin-init: out of memory"); for (int i = ldr_count() - 1; i >= 0; i--) { const RtLoaded *m = ldr_module(i); if (!m->plugin) continue; + if (only && *only && strcmp(only, m->name)) { + rt_log("plugin-init: %s stays loaded, its init does not run (TOTK_PLUGIN_ONLY=%s)", m->name, only); + continue; + } + ran[i] = 1; if (!m->init) refuse("a plugin without DT_INIT"); rt_log("plugin-init: ENTER %s DT_INIT at %#llx (%llu init_array entries)", m->name, (unsigned long long)m->init, (unsigned long long)(m->init_array_size / 8)); @@ -546,17 +561,59 @@ static void trace_entries(void) { rt_log("plugins: trace: %d hook entries counted", ntraced); } -// Takes back the words the inits changed in the sealed part (.text, .rodata) of a game module: main keeps the game's -// own bytes, the lifted code being what runs. -static void restore_sealed(const RtLoaded *m, const uint8_t *was) { - for (uint64_t off = 0; off < m->data_off; off += PAGE) { - const uint8_t *now = (const uint8_t *)(uintptr_t)(m->base + off); - if (!memcmp(now, was + off, PAGE)) continue; - if (!plat_mem_protect(m->base + off, PAGE, PLAT_PROT_RW)) rt_fatal("plugins: cannot unseal %s+%#llx", m->name, - (unsigned long long)off); - memcpy((void *)(uintptr_t)(m->base + off), was + off, PAGE); - plat_mem_protect(m->base + off, PAGE, PLAT_PROT_READ); +// Takes back a page an init changed in the sealed part (.text, .rodata) of a game module: main keeps the game's own +// bytes, the lifted code being what runs. +static void restore_page(const RtLoaded *m, uint64_t off, const uint8_t *was) { + if (!plat_mem_protect(m->base + off, PAGE, PLAT_PROT_RW)) rt_fatal("plugins: cannot unseal %s+%#llx", m->name, + (unsigned long long)off); + memcpy((void *)(uintptr_t)(m->base + off), was, PAGE); + plat_mem_protect(m->base + off, PAGE, PLAT_PROT_READ); +} + +// After plugin i's init: what it wrote into the game modules, word by word, against the record of the build (its +// entries of RT_PLUGIN_WRITES). The first difference goes into why (unless a reason is there already). Then the game +// modules are made what they were before the init: the sealed part (.text, .rodata) gets the game's bytes back, a +// write elsewhere stays and goes into pre, so the next plugin's init is compared only with what it writes itself. +// The aliases this init mapped (from first_alias on) are flushed first; an earlier init's were flushed after it. +static void check_writes(int plugin, int first_alias, char *why, size_t cap) { + for (int i = first_alias; i < naliases; i++) + if (aliases[i].live) flush(&aliases[i]); + uint8_t seen[NWRITES > 0 ? NWRITES : 1]; + memset(seen, 0, sizeof seen); + int unexpected = 0, missing = 0; + char first[160] = ""; + for (int i = 0; i < ldr_count(); i++) { + const RtLoaded *m = ldr_module(i); + if (!pre[i]) continue; + uint8_t *now = (uint8_t *)(uintptr_t)m->base; + for (uint64_t off = 0; off < m->size; off += PAGE) { + if (!memcmp(now + off, pre[i] + off, PAGE)) continue; + for (uint64_t w = off; w < off + PAGE; w += 4) { + uint32_t a, b; + memcpy(&a, pre[i] + w, 4); + memcpy(&b, now + w, 4); + if (a == b) continue; + int k = 0; + while (k < NWRITES && !(write_tab[k].plugin == plugin && write_tab[k].off == w && + !strcmp(write_tab[k].module, m->name) && write_tab[k].old == a && + write_tab[k].now == b)) + k++; + if (k < NWRITES) { + seen[k] = 1; + } else if (!unexpected++) { + snprintf(first, sizeof first, "%s+%#llx %08x -> %08x", m->name, (unsigned long long)w, a, b); + } + } + if (off < m->data_off) restore_page(m, off, pre[i] + off); + else memcpy(pre[i] + off, now + off, PAGE); + } } + for (int k = 0; k < NWRITES; k++) missing += write_tab[k].plugin == plugin && !seen[k]; + if (why[0]) return; + if (unexpected) + snprintf(why, cap, "its init wrote %d words the build did not record (first %s)", unexpected, first); + else if (missing) + snprintf(why, cap, "%d of the words the build recorded were not written", missing); } #endif @@ -597,62 +654,22 @@ int plugins_start(Cpu *c, const RtLoaded *sdk) { const RtLoaded *m = loaded(plugin_tab[i].module); rt_log("plugins: ENTER %s DT_INIT at %#llx", m->name, (unsigned long long)m->init); failed[0] = 0; + int first_alias = naliases; running = 1; rt_trace |= RT_TRACE_PLUGIN; rt_call(c, m->init, 0); rt_trace &= ~(unsigned)RT_TRACE_PLUGIN; running = 0; if (failed[0]) snprintf(why[i], sizeof why[i], "its init asked for what the port does not serve: %s", failed); - } - for (int i = 0; i < naliases; i++) - if (aliases[i].live) flush(&aliases[i]); - // What the inits wrote into the game modules, word by word, against the record of the build. - uint8_t seen[NWRITES > 0 ? NWRITES : 1]; - memset(seen, 0, sizeof seen); - int unexpected = 0; - char first[160] = ""; - for (int i = 0; i < ldr_count(); i++) { - const RtLoaded *m = ldr_module(i); - if (!pre[i]) continue; - const uint8_t *now = (const uint8_t *)(uintptr_t)m->base; - for (uint64_t off = 0; off < m->size; off += PAGE) { - if (!memcmp(now + off, pre[i] + off, PAGE)) continue; - for (uint64_t w = off; w < off + PAGE; w += 4) { - uint32_t a, b; - memcpy(&a, pre[i] + w, 4); - memcpy(&b, now + w, 4); - if (a == b) continue; - int k = 0; - while (k < NWRITES && !(write_tab[k].off == w && !strcmp(write_tab[k].module, m->name) && - write_tab[k].old == a && write_tab[k].now == b)) - k++; - if (k < NWRITES) { - seen[k] = 1; - } else if (!unexpected++) { - snprintf(first, sizeof first, "%s+%#llx %08x -> %08x", m->name, (unsigned long long)w, a, b); - } - } - } - restore_sealed(m, pre[i]); - free(pre[i]); - } - free(pre); - pre = NULL; - for (int i = 0; i < RT_PLUGIN_COUNT; i++) { - if (!run[i] || why[i][0]) continue; - const RtLoaded *m = loaded(plugin_tab[i].module); - int missing = 0; - for (int k = 0; k < NWRITES; k++) missing += write_tab[k].plugin == i && !seen[k]; + check_writes(i, first_alias, why[i], sizeof why[i]); char hex[65]; sha256_hex((const void *)(uintptr_t)m->base, m->text_end, hex); - if (unexpected) - snprintf(why[i], sizeof why[i], "the inits wrote %d words the build did not record (first %s)", unexpected, - first); - else if (missing) - snprintf(why[i], sizeof why[i], "%d of the words the build recorded were not written", missing); - else if (strcmp(hex, plugin_tab[i].sha256)) + if (!why[i][0] && strcmp(hex, plugin_tab[i].sha256)) snprintf(why[i], sizeof why[i], "its code after the init is not the one the build lifted"); } + for (int i = 0; i < ldr_count(); i++) free(pre[i]); + free(pre); + pre = NULL; mem_query_hook = NULL; // the game's own QueryMemory sees its address space as without plugins rt_process_memory = NULL; // a callback that maps process memory stops with a named fatal (svc.c) int on = 0; diff --git a/tests/exefs_plugins_test.py b/tests/exefs_plugins_test.py index 7687d6c9..5243569f 100644 --- a/tests/exefs_plugins_test.py +++ b/tests/exefs_plugins_test.py @@ -379,6 +379,83 @@ class Lines(unittest.TestCase): self.assertFalse(any(x.startswith(('orig', 'stub')) for x in out)) +# ---- one service run per plugin +class PerPlugin(unittest.TestCase): + """main() with three plugins in three mods and a stand-in for the service run: plugin0 hooks its site, plugin1's + init ends the run without a record (a crash), plugin2's run is refused (an svc). Each gets its own status line; + only plugin0's set goes into the patch file.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.d = Path(tmp.name) + self.files = [] + for name in ('A', 'B', 'C'): + f = self.d / 'mods' / name / 'exefs' / 'subsdk9' + f.parent.mkdir(parents=True) + f.write_bytes(nso(PLUGIN_TEXT) + name.encode()) + self.files.append(f) + (self.d / 'patches.txt').write_text('set fps60 TOTK_FPS60 1\n') + self.runs = [] + saved = xp.run, xp.hook_impls + self.addCleanup(lambda: (setattr(xp, 'run', saved[0]), setattr(xp, 'hook_impls', saved[1]))) + xp.run, xp.hook_impls = self.fake_run, lambda rec: IMPLS + + def fake_run(self, exe, out, tick_offset, timeout, only): + self.runs.append(only) + out.mkdir(parents=True, exist_ok=True) + if only == 'plugin1': + return 0xC0000005, 'its init ended the service run without a record (exit 0xc0000005)' + rec = record(writes=[] if only != 'plugin0' else None) + rec['modules'][2]['file'] = str(self.files[0]) + for i in (1, 2): + rec['modules'].append(dict(rec['modules'][2], name=f'plugin{i}', base=PLUGIN + i * 0x100000, + file=str(self.files[i]))) + for m in rec['modules'][2:]: + m['ran'] = m['name'] == only + if only == 'plugin2': + rec['refused'] = 'svc 0x1 at plugin2+0x50a0' + (out / 'plugin0.img').write_bytes(rec.pop('images')['plugin0']) + rec['modules'][2]['image'] = 'plugin0.img' + (out / 'record.json').write_text(json.dumps(rec)) + return 3 if rec['refused'] else 0, None + + def test_each_plugin_alone(self): + out, status, patched = self.d / 'rec', self.d / 'plugins.status', self.d / 'patches-out.txt' + argv = sys.argv + sys.argv = ['exefs_plugins.py', '--exe', 'totk.exe', '--out', str(out), '--mods', str(self.d / 'mods'), + '--patches', str(self.d / 'patches.txt'), '--patches-out', str(patched), '--status', str(status)] + try: + code = xp.main() + finally: + sys.argv = argv + self.assertEqual(code, 1) + self.assertEqual(self.runs, ['-', 'plugin0', 'plugin0', 'plugin1', 'plugin2']) + lines = {line.split('\t')[0]: line.split('\t') for line in status.read_text().splitlines()} + key = {n: xp.fnv(f'{n}/exefs/subsdk9'.encode()) for n in 'ABC'} + self.assertEqual(lines[key['A']][2:], ['plugin', '-', 'plugin0, 1 hooks']) + self.assertEqual(lines[key['B']][2], 'unsupported') + self.assertIn('ended the service run', lines[key['B']][4]) + self.assertEqual(lines[key['C']][2], 'unsupported') + self.assertIn('svc 0x1', lines[key['C']][4]) + text = patched.read_text() + self.assertIn(f'set mod_{key["A"].lower()} ', text) + self.assertNotIn(f'mod_{key["B"].lower()}', text) + self.assertNotIn(f'mod_{key["C"].lower()}', text) + self.assertIn((out / 'plugin0' / 'run1' / 'plugin0.img').resolve().as_posix(), text) + + def test_ran_view_leaves_out_plugins_that_did_not_run(self): + rec = record() + rec['modules'].append(dict(rec['modules'][2], name='plugin1', base=PLUGIN + 0x100000, ran=False)) + rec['modules'][2]['ran'] = True + rec['images']['plugin1'] = rec['images']['plugin0'] + view = xp.ran_view(rec) + self.assertEqual([m['name'] for m in view['modules'] if m['plugin']], ['plugin0']) + self.assertEqual(set(view['images']), {'plugin0'}) + whole = record() # no "ran": every init ran, the record is its own view + self.assertIs(xp.ran_view(whole), whole) + + # ---- the generator on those lines def code_nso(words_at, size_text=0x5000): """An NSO as nso() whose .text holds words_at {offset: word}.""" diff --git a/tests/installer_test.py b/tests/installer_test.py index f890a1c8..57d09a32 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -383,10 +383,14 @@ class LiftRunTest(unittest.TestCase): self.assertEqual(self.outputs(), [False, False]) self.assertIn('warn.exefs_skipped', self.ctx.reporter.out.getvalue()) - def plugin_setup(self, kind): - """A mod with an exlaunch plugin; the fake service run writes its status line of that kind.""" + def plugin_setup(self, kind, more=()): + """A mod with an exlaunch plugin (SQ), and one more mod per (name, kind) of more; the fake service run writes + each plugin's status line of its kind.""" + kinds = {'SQ': kind, **dict(more)} + for name in kinds: + (self.dest / 'mods' / name / 'exefs').mkdir(parents=True, exist_ok=True) + (self.dest / 'mods' / name / 'exefs' / 'subsdk9').write_bytes(b'NSO0' + name.encode()) plugin = self.dest / 'mods' / 'SQ' / 'exefs' / 'subsdk9' - plugin.write_bytes(b'NSO0') self.ctx.layout = SimpleNamespace(**vars(self.ctx.layout), llvm='llvm', runtime='runtime', ninja='ninja', deps=None, sysroot=None, runtime_lib_for=lambda cpu: None, pgo=lambda version: None, expected=lambda version: 'expected.json') @@ -399,11 +403,14 @@ class LiftRunTest(unittest.TestCase): Path(args[args.index('--patches-out') + 1]).write_text('patches\n') status = Path(args[args.index('--status') + 1]) status.parent.mkdir(parents=True, exist_ok=True) - h = 0x811C9DC5 # FNV-1a 32 of "SQ/exefs/subsdk9", as tools/exefs_plugins.py keys it - for b in b'SQ/exefs/subsdk9': - h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF - status.write_text(f'{h:08X}\t00000000\t{kind}\t-\tnote\n') - return (1 if kind == 'unsupported' else 0), log + lines = [] + for name, k in kinds.items(): + h = 0x811C9DC5 # FNV-1a 32 of "/exefs/subsdk9", as tools/exefs_plugins.py keys it + for b in f'{name}/exefs/subsdk9'.encode(): + h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF + lines.append(f'{h:08X}\t00000000\t{k}\t-\tnote\n') + status.write_text(''.join(lines)) + return (1 if 'unsupported' in kinds.values() else 0), log return code, log steps.run_tool = run_tool return plugin @@ -429,6 +436,18 @@ class LiftRunTest(unittest.TestCase): self.assertTrue(steps.plugins_status(self.ctx).is_file()) self.assertNotIn('warn.plugins_skipped', self.ctx.reporter.out.getvalue()) + def test_refused_plugin_stays_in_the_layout_of_a_built_one(self): + # one plugin built, one refused: the lift with the hook sites keeps both in the layout (the records hold the + # addresses of the service runs' layout); the refused one has no patch set, its status line says why + self.plugin_setup('plugin', [('ZZ', 'unsupported')]) + steps.Lift().run(self.ctx) + last = self.calls[4][1] + mods = self.dest / 'mods' + self.assertEqual(last[last.index('--plugins') + 1], + ','.join(str(mods / n / 'exefs' / 'subsdk9') for n in ('SQ', 'ZZ'))) + self.assertIn('unsupported', steps.plugins_status(self.ctx).read_text()) + self.assertNotIn('warn.plugins_skipped', self.ctx.reporter.out.getvalue()) + def test_failed_plugin_build_warns(self): self.plugin_setup('plugin') self.fail = {'build_game.py'} diff --git a/tools/exefs_plugins.py b/tools/exefs_plugins.py index a0ef82ff..73421bdb 100644 --- a/tools/exefs_plugins.py +++ b/tools/exefs_plugins.py @@ -4,8 +4,8 @@ a plugin's hook sites. After the design in #845 by itaskaev2; the init runs on t same environment (plugin_init.c plugins_start), so the plugin keeps real state (its heap blocks, its GOT, its alias addresses). The record says what the build expects that start-up to write. - python tools/exefs_plugins.py --exe build/-x/totk.exe --out # two runs, their diff, the check - python tools/exefs_plugins.py --check /run1 # the check of one record + python tools/exefs_plugins.py --exe build/-x/totk.exe --out # per plugin: two runs, diff, check + python tools/exefs_plugins.py --check //run1 # the check of one record python tools/exefs_plugins.py --exe --out --mods --patches --patches-out --status # and the patch lines for the generator @@ -20,11 +20,16 @@ file hash: FNV-1a 32 of "/"; conten `plugin` (note: the module and its hook count) or `unsupported` (note: the first reason); built-in sets: the port's own sets with a site the plugin hooks too ("-" none), which the runtime switches off while the mod is on. -The exe must be built with the plugin in its layout (recomp/gen.py --plugins). --out gets run1/ and run2/, one record -each; run2 adds a large offset to every GetSystemTick (TOTK_PLUGIN_TICK_OFFSET), so the init picks other alias -addresses. The words that differ between the two records are listed with what they look like, not refused: an init -keeps its alias addresses and the regions GetInfo gave it in its .data, and a plugin's callbacks must not use them -(the generator's check of the code the callbacks reach, still to come). +The exe must be built with the plugins in its layout (recomp/gen.py --plugins). Each plugin runs in service runs of +its own (TOTK_PLUGIN_ONLY): the other plugins stay loaded, their inits do not run. A plugin that is refused, crashes, +hangs or fails the check is refused alone, with its reason; the others are built in. At every start the game runs each +init against the game's own bytes as well (runtime/plugin_init.c plugins_start takes a plugin's writes back before the +next init), so a plugin's own record is what its start must match. --out gets list/ (a run without inits, which names +the plugin modules), and /run1 and /run2 for each plugin; run2 adds a large offset to every +GetSystemTick (TOTK_PLUGIN_TICK_OFFSET), so the init picks other alias addresses. The words that differ between the two +records are listed with what they look like, not refused: an init keeps its alias addresses and the regions GetInfo +gave it in its .data, and a plugin's callbacks must not use them. A difference in code (a write into a game module, +the plugin's .text or .rodata) refuses the plugin: its start could not be compared with the record. The check (check_record) accepts a record only when every rule holds; each broken rule is one reason: - the run was not refused (an svc outside the served set, such as CreateSession of exlaunch's process-handle @@ -90,6 +95,19 @@ def u(v): return int(v, 16) if isinstance(v, str) else v +def ran_view(rec): + """The record as seen from the plugins whose inits ran: a plugin that only stayed loaded (TOTK_PLUGIN_ONLY) counts + as a module that is no plugin, so its exports and code are not checked as this record's. A record without "ran" + (every init ran) comes back as it is.""" + if all(m.get('ran', True) for m in rec['modules'] if m['plugin']): + return rec + view = dict(rec) + view['modules'] = [dict(m, plugin=False) if m['plugin'] and not m.get('ran', True) else m for m in rec['modules']] + names = {m['name'] for m in view['modules'] if m['plugin']} + view['images'] = {k: v for k, v in rec['images'].items() if k in names} + return view + + # ---- instructions def sext(v, bits): return v - (1 << bits) if v >> (bits - 1) & 1 else v @@ -433,35 +451,35 @@ def plugin_files(mods): return out -def emit(rec, hooks, reasons, record_dir, args): - """--patches-out and --status from the record (accepted or not).""" +def emit(results, args): + """--patches-out and --status from the checked plugins. results: one (module, its NSO file, its record or None, + the record's directory, hooks, reasons) per plugin of the layout; no reasons: accepted.""" text = args.patches.read_text() if args.patches else '' builtin = builtin_sites(text) status, sets = [], [] files = {str(f.resolve()): (m, rel, f) for m, rel, f in plugin_files(args.mods)} - plugins = [m for m in rec['modules'] if m['plugin']] - for p in plugins: - hit = files.get(str(Path(p.get('file', '')).resolve())) + for module, file, rec, record_dir, hooks, reasons in results: + hit = files.get(str(Path(file or '').resolve())) if not hit: - print(f'plugins: {p["name"]}: {p.get("file")} is not under {args.mods}') + print(f'plugins: {module}: {file} is not under {args.mods}') continue mod, rel, f = hit key, content = fnv(f'{mod}/{rel}'.encode()), fnv(f.read_bytes()) if reasons: - note = reasons[0].replace('\t', ' ') + note = ' '.join(reasons[0].split()) status.append(f'{key}\t{content}\tunsupported\t-\t{note}') print(f'plugins: unsupported {mod}/{rel} ({note})') continue name = f'mod_{key.lower()}' - own = hooks_of(hooks, p['name']) + own = hooks_of(hooks, module) sites = [int(h['site'].split('+')[1], 16) for h in own] over = sorted({b for a in sites for b in builtin.get(a, [])}) label = f'{mod}/{rel}'.encode('ascii', 'replace').decode() sets.append([f'set {name} TOTK_MODPATCH_{key} 0 ordered # {label}'] + plugin_lines(rec, own, name, record_dir, - p['name'])) - status.append(f'{key}\t{content}\tplugin\t{",".join(over) or "-"}\t{p["name"]}, {len(own)} hooks') - print(f'plugins: plugin {mod}/{rel} as {p["name"]}, {len(own)} hooks' + (f', over {", ".join(over)}' - if over else '')) + module)) + status.append(f'{key}\t{content}\tplugin\t{",".join(over) or "-"}\t{module}, {len(own)} hooks') + print(f'plugins: plugin {mod}/{rel} as {module}, {len(own)} hooks' + (f', over {", ".join(over)}' + if over else '')) if sets: text += '\n# Mod plugins (tools/exefs_plugins.py)\n' + '\n'.join(line for s in sets for line in s) + '\n' args.patches_out.write_text(text) @@ -469,7 +487,7 @@ def emit(rec, hooks, reasons, record_dir, args): args.status.write_text('\n'.join(status) + ('\n' if status else ''), encoding='utf-8') -# ---- two runs +# ---- the service runs def where_is(v, rec): """What an address-like value points into, for the diff listing.""" for m in rec['modules']: @@ -524,17 +542,34 @@ def diff_records(a, b): return out -def run(exe, out, tick_offset, timeout): +def run(exe, out, tick_offset, timeout, only): + """One service run into out, only module `only`'s init running ("-": none). (exit code or None, why the run gave + no record or None); a refused run writes its record too.""" out.mkdir(parents=True, exist_ok=True) - env = dict(os.environ, TOTK_PLUGIN_TICK_OFFSET=f'{tick_offset:#x}', TOTK_BACKGROUND_WINDOW='1', - TOTK_ASSUME_FOCUS='1') - r = subprocess.run([str(exe), '--plugin-init', str(out)], env=env, capture_output=True, text=True, - errors='replace', timeout=timeout) + for f in out.iterdir(): # an earlier run's record must not pass for this one's + if f.is_file() and (f.name in ('record.json', 'run.log') or f.suffix == '.img'): + f.unlink() + env = dict(os.environ, TOTK_PLUGIN_TICK_OFFSET=f'{tick_offset:#x}', TOTK_PLUGIN_ONLY=only, + TOTK_BACKGROUND_WINDOW='1', TOTK_ASSUME_FOCUS='1') + try: + r = subprocess.run([str(exe), '--plugin-init', str(out)], env=env, capture_output=True, text=True, + errors='replace', timeout=timeout) + except subprocess.TimeoutExpired as e: + log = b''.join(x if isinstance(x, bytes) else x.encode() for x in (e.stdout, e.stderr) if x) + (out / 'run.log').write_text(log.decode(errors='replace'), encoding='utf-8') + return None, f'its init did not return within {timeout} s' (out / 'run.log').write_text(r.stdout + r.stderr, encoding='utf-8') - return r.returncode + if not (out / 'record.json').is_file(): + # the runtime's crash report names the fault in a FATAL line (an access violation, an abort) + fatal = next((line.split('FATAL:', 1)[1].strip() for line in (r.stdout + r.stderr).splitlines() + if 'FATAL:' in line), None) + return r.returncode, (f'its init crashed the service run: {fatal}' if fatal else + f'its init ended the service run without a record (exit {r.returncode & 0xFFFFFFFF:#x})') + return r.returncode, None def report(rec, label): + rec = ran_view(rec) impls = hook_impls(rec) hooks, reasons = check_record(rec, impls) more, notes = reach_check(rec, hooks, impls) @@ -559,12 +594,40 @@ def report(rec, label): return hooks, reasons +def plugin_runs(exe, out, module, timeout): + """The two service runs of one plugin (run2 with the tick offset) into out/: (its first record or None, + hooks, reasons).""" + recs = [] + for i, offset in enumerate((0, TICK_OFFSET), 1): + d = out / module / f'run{i}' + code, why = run(exe, d, offset, timeout, module) + if why: + print(f'{module} run {i}: {why}; see {d / "run.log"}') + return None, [], [why] + recs.append(load_record(d)) + print(f'{module} run {i}: exit {code}, tick offset {offset:#x}') + if recs[0].get('refused'): # refused: a second run says nothing more + hooks, bad = report(recs[0], f'{module} run 1') + return recs[0], hooks, bad + hooks, bad = report(recs[0], f'{module} run 1') + diff = diff_records(ran_view(recs[0]), ran_view(recs[1])) + print(f'{module} run 1 against run 2: {len(diff)} differences' + (':' if diff else '')) + for line in diff: + print(f' {line}') + # Words of a game module or of the plugin's .text that differ between the runs: the start-up could not be compared + # with the record. Its .data and .bss may differ (alias addresses). + unstable = [line for line in diff if line.startswith('write ') or '(.text)' in line or '(.rodata)' in line] + if unstable: + bad = bad + [f'two runs of the init differ in code: {unstable[0]}'] + return recs[0], hooks, bad + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - ap.add_argument('--exe', type=Path, help='totk.exe built with the plugin in its layout') - ap.add_argument('--out', type=Path, help='directory for run1/ and run2/') + ap.add_argument('--exe', type=Path, help='totk.exe built with the plugins in its layout') + ap.add_argument('--out', type=Path, help='directory for list/ and /run1, run2') ap.add_argument('--check', type=Path, help='check the record in this directory only') - ap.add_argument('--timeout', type=int, default=300) + ap.add_argument('--timeout', type=int, default=120, help='seconds a service run may take') ap.add_argument('--mods', type=Path, help='the mods folder the plugins come from') ap.add_argument('--patches', type=Path, help='the patch file so far') ap.add_argument('--patches-out', type=Path, help='the patch file with the plugins\' sets added') @@ -578,32 +641,26 @@ def main(): rec = load_record(args.check) hooks, bad = report(rec, str(args.check)) if emitting: - emit(rec, hooks, bad, args.check, args) + emit([(m['name'], m.get('file'), rec, args.check, hooks, bad) for m in ran_view(rec)['modules'] + if m['plugin']], args) return 1 if bad else 0 if not args.exe or not args.out: ap.error('--exe and --out, or --check') - recs = [] - for i, offset in enumerate((0, TICK_OFFSET), 1): - d = args.out / f'run{i}' - code = run(args.exe, d, offset, args.timeout) - if not (d / 'record.json').is_file(): - print(f'run {i}: no record (exit {code}); see {d / "run.log"}') - return 1 - recs.append(load_record(d)) - print(f'run {i}: exit {code}, tick offset {offset:#x}') - hooks, bad = report(recs[0], 'run 1') - diff = diff_records(*recs) - print(f'run 1 against run 2: {len(diff)} differences' + (':' if diff else '')) - for line in diff: - print(f' {line}') - # Words of a game module or of a plugin's .text that differ between the runs: the start-up could not be compared - # with the record. Its .data and .bss may differ (alias addresses). - unstable = [line for line in diff if line.startswith('write ') or '(.text)' in line or '(.rodata)' in line] - if unstable: - bad = bad + [f'two runs of the init differ in code: {unstable[0]}'] + # A run without any init names the plugin modules of the exe's layout and checks that the service run works at all: + # without it no plugin can be built. + code, why = run(args.exe, args.out / 'list', 0, args.timeout, '-') + if why: + print(f'the service run without inits gave no record: {why}; see {args.out / "list" / "run.log"}') + return 2 + listing = load_record(args.out / 'list') + results = [] + for m in (m for m in listing['modules'] if m['plugin']): + rec, hooks, bad = plugin_runs(args.exe, args.out, m['name'], args.timeout) + print(f'{m["name"]}: ' + (f'refused: {bad[0]}' if bad else f'accepted, {len(hooks)} hooks')) + results.append((m['name'], m.get('file'), rec, args.out / m['name'] / 'run1', hooks, bad)) if emitting: - emit(recs[0], hooks, bad, args.out / 'run1', args) - return 1 if bad else 0 + emit(results, args) + return 1 if any(r[5] for r in results) else 0 if __name__ == '__main__':