diff --git a/tests/exefs_plugins_test.py b/tests/exefs_plugins_test.py new file mode 100644 index 00000000..6bb23495 --- /dev/null +++ b/tests/exefs_plugins_test.py @@ -0,0 +1,282 @@ +"""recomp/layout.py with and without mod plugins, and tools/exefs_plugins.py's check of a plugin service run record: +a synthetic exlaunch-shaped record with one inline hook that passes, and one bad record per rule. No game files. + +Run with .venv (lz4 and numpy, as recomp/): python tests/exefs_plugins_test.py +""" +import json +import struct +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +sys.path.insert(0, str(ROOT / 'recomp')) +import exefs_plugins as xp # noqa: E402 +import layout # noqa: E402 +from nso import Nso # noqa: E402 + + +def nso(size_text, bss=0x1000): + """An uncompressed NSO: .text of size_text bytes (b over MOD0's offset word), .rodata with MOD0 at 0x1000 past + .text, .data with an empty .dynamic.""" + text = bytearray(size_text) + struct.pack_into('> 2) & 0x3FFFFFF + + +def record(original=0x94000010, trampoline_first=None, stub=None, writes=None, **extra): + """A record of one inline hook at main+SITE_OFF whose original is `original` (default bl main+SITE_OFF+0x40).""" + site, stub_at, tramp = MAIN + SITE_OFF, PLUGIN + STUB_OFF, PLUGIN + TRAMP_OFF + img = bytearray(0x14000) + words = stub or [xp.STUR_X30, b(stub_at + 4, PLUGIN + IMPL_OFF, True), xp.LDUR_X30, b(stub_at + 12, tramp)] + struct.pack_into('<4IQ', img, STUB_OFF, *words, PLUGIN + CB_OFF) + if trampoline_first is None: + rel = xp.pc_relative(original, site) + trampoline_first = (b(tramp, rel[1], True) if rel and rel[0][0] == 'b' else original) + struct.pack_into(''], 'compiles its checks with clang'), 'cheats_vm_check': ('.venv', 'cc', ['--cc', ''], 'compiles runtime/cheats.c into a harness with clang'), diff --git a/tools/exefs_plugins.py b/tools/exefs_plugins.py new file mode 100644 index 00000000..8bdaac44 --- /dev/null +++ b/tools/exefs_plugins.py @@ -0,0 +1,358 @@ +"""Run a mod plugin's init in the plugin service run and check what it did: the build step before the generator emits +a plugin's hook sites. After the design in #845 by itaskaev2; the init runs on this port's runtime (totk.exe +--plugin-init, runtime/plugin_init.c) instead of under Unicorn. + + python tools/exefs_plugins.py --exe build/-x/totk.exe --out # two runs, their diff, the check + python tools/exefs_plugins.py --check /run1 # the check of one record + +The exe must be built with the plugin in its layout (recomp/gen.py --plugins). --out gets run1/ and run2/, one record +each; run2 adds a large offset to every GetSystemTick (TOTK_PLUGIN_TICK_OFFSET), so the init picks other alias +addresses. The words that differ between the two records are listed with what they look like, not refused: an init +keeps its alias addresses and the regions GetInfo gave it in its .data, and a plugin's callbacks must not use them +(the generator's check of the code the callbacks reach, still to come). + +The check (check_record) accepts a record only when every rule holds; each broken rule is one reason: + - the run was not refused (an svc outside the served set, such as CreateSession of exlaunch's process-handle + fallback, ends the run with the reason); + - the init called only the svcs ALLOWED_SVCS, GetInfo with ALLOWED_INFO, and the imports ALLOWED_IMPORTS; + - every write into a game module is a word of main's .text whose original is no br, blr or ret, and it is one of: + - an inline hook (MSFix, Full Inventory Expansion): `b ` into a plugin's .text, the stub in exlaunch's + shape (stur x30, [sp, #-0x10]; bl exl_inline_hook_impl or the float one; ldur x30, [sp, #-0x10]; + b ; .quad ), the trampoline the site's original relocated (the + same word, or the same PC-relative target) and `b `; + - a call-site rewrite (Quick Weapon Durability): `bl` straight to a function start of the plugin's lift (the + record lists them), which the generator can emit as a patch site whose replacement is that branch; + - a replace hook (Full Inventory Expansion's Sort Fixer): `b` from a function start of main to a function start of + the plugin's lift; the Orig trampoline (the original relocated, b site+4) is listed when the pool holds one. +Not checked here yet: a site shared with a built-in patch set or another mod, and svcs in the code the callbacks +reach (both need the generator's view). +""" +import argparse +import json +import os +import struct +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'recomp')) +from nso import Nso # noqa: E402 + +ALLOWED_SVCS = {0x06: 'QueryMemory', 0x1E: 'GetSystemTick', 0x27: 'OutputDebugString', 0x29: 'GetInfo', + 0x74: 'MapProcessMemory', 0x75: 'UnmapProcessMemory'} +ALLOWED_INFO = {2, 3, 4, 5, 12, 13, 14, 15, 65001} +# Reviewed one by one: SNPrintf formats into the caller's buffer; GetDisplayVersion is answered by the service run; +# mersenne_twister_engine::_M_gen_rand (Full Inventory Expansion's alias picker) refills the state of the +# engine object it is given, which lives in the plugin's .bss; memcmp and strlen only read. malloc stays out: Quick +# Weapon Durability's init keeps a heap block's address in its .bss, and that block does not exist at play time. +ALLOWED_IMPORTS = {'_ZN2nn4util8SNPrintfEPcmPKcz', '_ZN2nn2oe17GetDisplayVersionEPNS0_14DisplayVersionE', 'memcmp', 'strlen', + '_ZNSt23mersenne_twister_engineImLm64ELm312ELm156ELm31ELm13043109905998158313ELm29ELm6148914691236517205' + 'ELm17ELm8202884508482404352ELm37ELm18444473444759240704ELm43ELm6364136223846793005EE11_M_gen_randEv'} +HOOK_IMPLS = ('exl_inline_hook_impl', 'exl_inline_float_hook_impl') +STUR_X30 = 0xF81F03FE # stur x30, [sp, #-0x10] +LDUR_X30 = 0xF85F03FE # ldur x30, [sp, #-0x10] +STUB_WORDS = 6 # 4 instructions and the callback's .quad +TICK_OFFSET = 0x0123456789AB + + +def u(v): + return int(v, 16) if isinstance(v, str) else v + + +# ---- instructions +def sext(v, bits): + return v - (1 << bits) if v >> (bits - 1) & 1 else v + + +def branch_target(w, pc): + """Target of b / bl at pc, else None.""" + if w & 0x7C000000 == 0x14000000: + return pc + sext(w & 0x3FFFFFF, 26) * 4 + return None + + +def pc_relative(w, pc): + """(kind, absolute value) of a PC-relative instruction at pc, else None: b, bl, b.cond, cbz/cbnz, tbz/tbnz, adr, + adrp, ldr (literal). The kind holds the bits besides the offset, so two encodings match when both agree.""" + if w & 0x7C000000 == 0x14000000: + return ('b', w & 0xFC000000), pc + sext(w & 0x3FFFFFF, 26) * 4 + if w & 0xFF000010 == 0x54000000: + return ('b.cond', w & 0xFF00001F), pc + sext(w >> 5 & 0x7FFFF, 19) * 4 + if w & 0x7E000000 == 0x34000000: + return ('cbz', w & 0xFF00001F), pc + sext(w >> 5 & 0x7FFFF, 19) * 4 + if w & 0x7E000000 == 0x36000000: + return ('tbz', w & 0xFFF8001F), pc + sext(w >> 5 & 0x3FFF, 14) * 4 + if w & 0x9F000000 == 0x10000000: + return ('adr', w & 0x1F), pc + sext((w >> 5 & 0x7FFFF) << 2 | w >> 29 & 3, 21) + if w & 0x9F000000 == 0x90000000: + return ('adrp', w & 0x1F), (pc & ~0xFFF) + (sext((w >> 5 & 0x7FFFF) << 2 | w >> 29 & 3, 21) << 12) + if w & 0x3B000000 == 0x18000000: + return ('ldr', w & 0xFF00001F), pc + sext(w >> 5 & 0x7FFFF, 19) * 4 + return None + + +def is_indirect(w): + """br, blr, ret (and their pointer-authenticated forms): a site exlaunch could hook but the generator cannot.""" + return w & 0xFE1F0000 == 0xD61F0000 and w >> 21 & 0xF in (0, 1, 2) + + +def relocated(orig, site, word, at): + """The trampoline word at `at` does what the original at `site` does.""" + a, b = pc_relative(orig, site), pc_relative(word, at) + if a is None: + return word == orig + return b is not None and a == b + + +def inline_stub(img, o, p): + """The words at image offset o of plugin p have exlaunch's inline-hook entry shape: stur x30; bl; ldur x30; b; + and the .quad after them lies inside the plugin's .text.""" + if o + 4 * STUB_WORDS > p['text_end']: + return False + s = struct.unpack_from('<4I', img, o) + return s[0] == STUR_X30 and s[2] == LDUR_X30 and s[1] >> 26 == 0x25 and s[3] >> 26 == 0x05 + + +def orig_trampoline(img, p, original, site): + """Image offset of a pair in the plugin's .text that runs the site's original instruction relocated, then b site+4 + (exlaunch's Orig trampoline of a replace hook), else None.""" + base = p['base'] + for o in range(0, p['text_end'] - 4, 4): + t1 = struct.unpack_from('> 26 == 0x05 and branch_target(t1, base + o + 4) == site + 4 and \ + relocated(original, site, struct.unpack_from('= main['text_end'] or w['offset'] & 3: + reasons.append(f'a write outside main\'s .text: {where} {w["old"]:#010x} -> {w["new"]:#010x}') + continue + site = main['base'] + w['offset'] + stub = branch_target(w['new'], site) + link = bool(w['new'] >> 31) + if stub is None: + reasons.append(f'{where}: the new word {w["new"]:#010x} is no b or bl') + continue + if is_indirect(w['old']): + reasons.append(f'{where}: the original {w["old"]:#010x} is a br, blr or ret') + p, _ = plugin_word(stub) + if p is None: + reasons.append(f"{where}: {'bl' if link else 'b'} {stub:#x} does not reach into a plugin's .text") + continue + if not (not link and inline_stub(rec['images'][p['name']], stub - p['base'], p)): + # A b or bl straight into the plugin's code (CodePatcher-style call-site rewrite): a site whose replacement + # is that branch, when the target is a function start of the plugin's lift. + # A b replaces the whole main function, so it must sit at a function start of main; exlaunch's replace + # hook keeps the function's first instruction relocated + b site+4 in its pool for the callback's Orig. + o = stub - p['base'] + hook = {'site': f'main+{w["offset"]:#x}', 'original': f'{w["old"]:#010x}', + 'kind': 'call' if link else 'replace', 'target': f'{p["name"]}+{o:#x}'} + orig = orig_trampoline(rec['images'][p['name']], p, w['old'], site) + if orig is not None: + hook['trampoline'] = f'{p["name"]}+{orig:#x}' + hooks.append(hook) + if o not in {u(x) for x in p.get('functions', [])}: + reasons.append(f"{where}: {'bl' if link else 'b'} {p['name']}+{o:#x} is neither an inline-hook stub " + f"nor a function start of {p['name']}'s lift") + if not link and not w.get('fn_start'): + reasons.append(f"{where}: b {p['name']}+{o:#x} from inside a main function (only a function start can " + f"be replaced)") + continue + img, o = rec['images'][p['name']], stub - p['base'] + s = struct.unpack_from('<4IQ', img, o) + impl, tramp, cb = branch_target(s[1], stub + 4), branch_target(s[3], stub + 12), s[4] + hook = {'site': f'main+{w["offset"]:#x}', 'original': f'{w["old"]:#010x}', 'kind': 'inline', + 'stub': f'{p["name"]}+{o:#x}', + 'impl': impls.get(impl, f'{impl:#x}' if impl is not None else None), + 'callback': f'{p["name"]}+{cb - p["base"]:#x}' if p['base'] <= cb < p['base'] + p['text_end'] else f'{cb:#x}', + 'trampoline': f'{p["name"]}+{tramp - p["base"]:#x}' if tramp is not None else None} + hooks.append(hook) + if impl not in impls: + reasons.append(f'{where}: the stub calls {impl:#x}, not {" or ".join(HOOK_IMPLS)}') + if not p['base'] <= cb < p['base'] + p['text_end']: + reasons.append(f'{where}: the stub\'s callback {cb:#x} is outside {p["name"]}\'s .text') + tp, t0 = plugin_word(tramp) + _, t1 = plugin_word(tramp + 4) + if tp is None or t1 is None: + reasons.append(f'{where}: the trampoline {tramp:#x} is outside a plugin\'s .text') + continue + hook['relocated'] = f'{t0:#010x}' + if not relocated(w['old'], site, t0, tramp): + reasons.append(f'{where}: the trampoline {hook["trampoline"]} starts with {t0:#010x}, not the original ' + f'{w["old"]:#010x} relocated') + if branch_target(t1, tramp + 4) != site + 4 or t1 >> 31: + reasons.append(f'{where}: the trampoline\'s second word {t1:#010x} is no b to the site + 4') + return hooks, reasons + + +# ---- two runs +def where_is(v, rec): + """What an address-like value points into, for the diff listing.""" + for m in rec['modules']: + if m['base'] <= v < m['base'] + m['size']: + return f'{m["name"]}+{v - m["base"]:#x}' + for k in ('rtld', 'pool'): + r = rec.get(k) + if r and u(r['base']) <= v < u(r['base']) + u(r['size']): + return f'the {"alias pool" if k == "pool" else k} (+{v - u(r["base"]):#x})' + for a in rec.get('aliases', []): + if u(a['dst']) <= v < u(a['dst']) + u(a['size']): + return f'an alias of {u(a["src"]):#x}' + return None + + +def section(m, off, nso_path): + if off < m['text_end']: + return '.text' + if off < m['data_off']: + return '.rodata' + if nso_path and Path(nso_path).is_file(): + n = Nso(nso_path) + if off < n.data[0] + n.data[1]: + return '.data' + return '.bss' + return '.data/.bss' + + +def diff_records(a, b): + """Lines naming every difference between two records: writes into game modules, plugin image words (8 bytes).""" + out = [] + wa = {(w['module'], w['offset']): w for w in a['writes']} + wb = {(w['module'], w['offset']): w for w in b['writes']} + for k in sorted(set(wa) | set(wb)): + x, y = wa.get(k), wb.get(k) + if not x or not y or x['new'] != y['new']: + out.append(f'write {k[0]}+{k[1]:#x}: run 1 {x and hex(x["new"])}, run 2 {y and hex(y["new"])}') + for m in a['modules']: + if not m['plugin']: + continue + ia, ib = a['images'].get(m['name']), b['images'].get(m['name']) + if ia is None or ib is None or len(ia) != len(ib): + out.append(f'{m["name"]}: image missing or of another size') + continue + for off in range(0, len(ia), 8): + if ia[off:off + 8] == ib[off:off + 8]: + continue + va, vb = struct.unpack_from(' stub {h["stub"]} -> {h["impl"]}, callback ' + f'{h["callback"]}, trampoline {h["trampoline"]} ({h.get("relocated")}; b site+4)') + else: + print(f' {h["kind"]} {h["site"]} (was {h["original"]}) -> {"bl" if h["kind"] == "call" else "b"} ' + f'{h["target"]}' + (f', Orig trampoline {h["trampoline"]}' if h.get('trampoline') else '')) + for r in reasons: + print(f' REFUSED: {r}') + if not reasons: + print(f' accepted: {len(hooks)} hooks') + return reasons + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--exe', type=Path, help='totk.exe built with the plugin in its layout') + ap.add_argument('--out', type=Path, help='directory for run1/ and run2/') + ap.add_argument('--check', type=Path, help='check the record in this directory only') + ap.add_argument('--timeout', type=int, default=300) + args = ap.parse_args() + if args.check: + return 1 if report(load_record(args.check), str(args.check)) else 0 + if not args.exe or not args.out: + ap.error('--exe and --out, or --check') + recs = [] + for i, offset in enumerate((0, TICK_OFFSET), 1): + d = args.out / f'run{i}' + code = run(args.exe, d, offset, args.timeout) + if not (d / 'record.json').is_file(): + print(f'run {i}: no record (exit {code}); see {d / "run.log"}') + return 1 + recs.append(load_record(d)) + print(f'run {i}: exit {code}, tick offset {offset:#x}') + bad = report(recs[0], 'run 1') + diff = diff_records(*recs) + print(f'run 1 against run 2: {len(diff)} differences' + (':' if diff else '')) + for line in diff: + print(f' {line}') + return 1 if bad else 0 + + +if __name__ == '__main__': + sys.exit(main())