diff --git a/recomp/gen.py b/recomp/gen.py index e9f3e16c..29872f4a 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -3,7 +3,7 @@ Usage: gen.py --exefs build/extracted/v121/exefs --out generated/v121 [--modules main,subsdk0,sdk] [--jobs N] [--limit N] [--chunk-fns N] [--chunk-bytes B] [--big-fn B] [--patches FILE] [--v-locals K [--v-trust]] - [--no-cache] + [--no-cache] [--fn-enter-all] Output, under --out: layout.h, layout.json module bases (recomp/layout.py) @@ -29,9 +29,9 @@ the input; a chunk whose text did not change is not rewritten and keeps its mtim A chunk is not lifted again when its inputs did not change. Its text depends only on the generator (recomp/*.py, GENERATED_MARKER, the Emitter options), the module (its file, its base, and the function starts discovered from it), -the chunk's number and functions, and the patch sites and spin waits inside the chunk's address range: the per-module -caches of recomp/jumptables.py are derived from the module's bytes and base, and its per-function state is reset for -each function. The SHA-256 of all of these is the chunk's key. /lift_cache.json records the key of each chunk +the chunk's number and functions, and the patch sites, spin waits and function hooks inside the chunk's address +range (and whether the patch table has hooks at all): the per-module caches of recomp/jumptables.py are derived from +the module's bytes and base, and its per-function state is reset for each function. The SHA-256 of all of these is the chunk's key. /lift_cache.json records the key of each chunk with the SHA-256, size and mtime of the file it wrote and the results the manifest and folded.txt need, under a digest of the entry. A chunk whose entry is intact, whose key matches and whose file still has that size, mtime and hash is reused as it is; any other chunk is lifted. So a rebuild after a change to one cheat lifts only the chunks @@ -77,7 +77,7 @@ MUSTTAIL = '__attribute__((musttail)) return' EST_INT = 62 EST_SIMD = 75 EST_SIMD_LDST = 48 -EST_FN = 38 # per function: signature, FN_ENTER, closing brace +EST_FN = 38 # per function: signature, FN_ENTER (recomp/patches.py `hook`), closing brace EST_BR_LABELS = 10 # per instruction of a function with a `br`: switch cases, or labels if it falls back @@ -229,7 +229,7 @@ class Emitter: the C (an argument here, or anything else it reads) must go into the lift cache's key: chunk_key.""" def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True, spins=None, - v_locals=vlocals.LOCALS, v_trust=False): + v_locals=vlocals.LOCALS, v_trust=False, hooks=None, fn_enter_all=False): self.mod = mod self.starts = starts self.patched = patched or {} # {address: [(set index, [replacement words])]} (recomp/patches.py) @@ -238,6 +238,8 @@ class Emitter: self.kept = regs.CALLEE_SAVED if callee_saved else frozenset() # not reloaded after a call self.v_locals = v_locals # recomp/vlocals.py: V registers per function in C locals (0: all in c->v) self.v_trust = v_trust # with v_locals: V8-V15 are not reloaded after a call + # Functions that start with FN_ENTER (recomp/patches.py `hook`): None is every function + self.hooks = None if fn_enter_all else hooks self.uses_patches = False self.stats = Counter() self.unsupported = Counter() @@ -407,6 +409,7 @@ class Emitter: self.stats['external_branches'] += 1 return f'{publish(i)}{spill}c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);' + enter = [f' FN_ENTER({hx(entry)});'] if self.hooks is None or entry in self.hooks else [] if all_labels: # An unresolved computed br re-enters the function through fb_, which switches on c->pc to any # instruction. A goto to a shared dispatch block inside the body instead turns every instruction into @@ -415,13 +418,13 @@ class Emitter: cases = ' '.join(f'case {hx(t)}: goto L_{t:x};' for t in range(entry, end, 4)) redispatch = f'{MUSTTAIL} {fb}(c);' out = [f'static void {fb}(Cpu *c);', - f'void {fname(entry)}(Cpu *c) {{', f' FN_ENTER({hx(entry)});', f' c->pc = {hx(entry)};', + f'void {fname(entry)}(Cpu *c) {{', *enter, f' c->pc = {hx(entry)};', f' {redispatch}', '}', f'static void {fb}(Cpu *c) {{', f' switch (c->pc) {{ {cases} default: break; }}', f' {MUSTTAIL} rt_dispatch(c);'] else: redispatch = f'{MUSTTAIL} rt_dispatch(c);' - out = [f'void {fname(entry)}(Cpu *c) {{', f' FN_ENTER({hx(entry)});'] + out = [f'void {fname(entry)}(Cpu *c) {{', *enter] if fp.used: out.append(' ' + fp.decl()) @@ -614,7 +617,8 @@ def _emitter(name): mod.read = recorded e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()), _W['patches'].for_module(name, base), - spins=_W['patches'].spins_for(name, base), **_W['opts']) + spins=_W['patches'].spins_for(name, base), + hooks=_W['patches'].hooks_for(name, base), **_W['opts']) return e @@ -729,14 +733,16 @@ def in_span(addrs, lo, hi): return addrs[bisect.bisect_left(addrs, lo):bisect.bisect_left(addrs, hi)] -def chunk_key(mod_key, number, fns, patched, site_addrs, spins, spin_addrs): +def chunk_key(mod_key, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs): """Key of one chunk: its module, number (in its header) and functions, and the patch sites (set index and - replacement words, @page resolved, so the patch page is in them) and spin waits inside its address range. Every - input of the chunk text that is not in generator_key or module_key must be added here, with a case in + replacement words, @page resolved, so the patch page is in them), spin waits and hooked functions inside its + address range; hook_addrs None (FN_ENTER in every function) keys apart from an empty list. Every input of the + chunk text that is not in generator_key or module_key must be added here, with a case in tests/lift_cache_test.py.""" lo, hi = fns[0][0], fns[-1][1] sites = [[a, [[k, words] for k, words in patched[a]]] for a in in_span(site_addrs, lo, hi)] - return _key(mod_key, number, fns, sites, [[b, spins[b]] for b in in_span(spin_addrs, lo, hi)]) + hooks = None if hook_addrs is None else in_span(hook_addrs, lo, hi) + return _key(mod_key, number, fns, sites, [[b, spins[b]] for b in in_span(spin_addrs, lo, hi)], hooks) def _entry_check(file, entry): @@ -855,6 +861,9 @@ def main(): ap.add_argument('--v-trust', action='store_true', help='with --v-locals: do not reload V8-V15 after a call (AAPCS64: the callee preserves their low ' '64 bits)') + ap.add_argument('--fn-enter-all', action='store_true', + help='FN_ENTER (runtime/cpu.h) in every function, also with hook lines in the patch table ' + '(recomp/patches.py `hook`): a whole-game function-entry profile') ap.add_argument('--no-cache', action='store_true', help=f'lift every chunk, also those whose inputs match /{CACHE_FILE} (it is written anew); ' 'so does the environment variable TOTK_LIFT_CACHE=0') @@ -863,7 +872,7 @@ def main(): no_cache = ('--no-cache' if args.no_cache else 'TOTK_LIFT_CACHE=0' if os.environ.get('TOTK_LIFT_CACHE', '').strip() == '0' else None) opts = {'reg_locals': args.reg_locals, 'callee_saved': not args.no_callee_saved_trust, 'v_locals': args.v_locals, - 'v_trust': args.v_trust} + 'v_trust': args.v_trust, 'fn_enter_all': args.fn_enter_all} gen_key = generator_key(opts) t0 = time.perf_counter() @@ -895,9 +904,15 @@ def main(): mkey = module_key(gen_key, name, path, base, abs_starts) patched, spins = table.for_module(name, base), table.spins_for(name, base) site_addrs, spin_addrs = sorted(patched), sorted(spins) + hooks = None if args.fn_enter_all else table.hooks_for(name, base) + stray = sorted(set(hooks or ()) - set(abs_starts.tolist())) + if stray: + raise SystemExit(f'{args.patches}: hook {", ".join(f"{name}+{a - base:#x}" for a in stray)} is no function ' + f'start of {name}') + hook_addrs = None if hooks is None else sorted(hooks) for number, (i0, i1) in enumerate(chunks): fns = list(zip(abs_starts[i0:i1].tolist(), ends[i0:i1].tolist())) - key = chunk_key(mkey, number, fns, patched, site_addrs, spins, spin_addrs) + key = chunk_key(mkey, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs) tasks.append((int(est[i0:i1].sum()), (name, number, str(moddir / f'{name}_{number:04d}.c'), fns, key))) paths[name] = (str(path), base) starts_abs[name] = abs_starts diff --git a/recomp/patches-v143.txt b/recomp/patches-v143.txt index b1d61295..404a3b10 100644 --- a/recomp/patches-v143.txt +++ b/recomp/patches-v143.txt @@ -1155,3 +1155,8 @@ site ability main 1bd01c8 b4000155 aa0903f5 b4000155 # cbz x21 -> mov x21, x9; set report TOTK_REPORT_GUARD 1 site report main a22d8 2a010571 2a010571 37f80341 # orr w17, w11, w1, lsl #1; tbnz w1, #31, 0xa2340 site report main a2310 531f7821 530f7c24 531f7821 35000184 # lsr w4, w1, #15; lsl w1, w1, #1; cbnz w4, 0xa2340 + +# Function-entry hooks (FN_ENTER, runtime/cpu.h): only these functions check g_fnprof on entry, the others carry no +# hook. main+0x585bd0 sets the render size (runtime/resolution.c SET_RENDER_SIZE): res_fn_hit takes the renderer from +# the game's own call at start, so a DLSS mode can change while the game runs. +hook main 585bd0 diff --git a/recomp/patches.py b/recomp/patches.py index 0228c52e..f7ef378b 100644 --- a/recomp/patches.py +++ b/recomp/patches.py @@ -1,13 +1,15 @@ """Code patch table for the generator: guest instructions replaced in the lifted C, switchable at run time. Lifted code cannot be byte-patched when the game runs, so a patch is applied when the C is generated. A patch file -(gen.py --patches) has two kinds of lines; `#` starts a comment, fields are separated by whitespace: +(gen.py --patches) has these kinds of lines; `#` starts a comment, fields are separated by whitespace: set a patch set: a run-time switch =0/1, when it is unset site
[ ...] spin [ ...] + hook [ ...] +
is a module offset (hex), the instruction word the module has there (checked; a mismatch stops the generator), each an instruction word (hex) or `@page:x`, which is `adrp x, ` encoded for the site's address. The patch page is the first 4 KiB page after main's image (layout.py leaves the rest @@ -29,6 +31,11 @@ A `spin` line names a spin wait found by tools/spin_waits.py: a loop with head < and the direct branches back to it (module offsets). Each back-edge (checked: a b, b.cond, cbz / cbnz or tbz / tbnz to ) is lifted as `rt_spin_gate(c, ); goto `, which keeps every register and only adds time (runtime/kernel.c kern_spin_gate). + +A `hook` line names guest functions (module offsets of function starts; gen.py stops on any other address) whose +entry the runtime watches: their lifted C starts with FN_ENTER (runtime/cpu.h), a load of g_fnprof and a predicted +branch to fnprof_hit while the runtime sets it. A table with hook lines puts FN_ENTER into these functions only; a table +without any keeps it in every function, as does gen.py --fn-enter-all (a whole-game function-entry profile). """ from pathlib import Path @@ -49,14 +56,22 @@ def adrp(pc, target, rd): class Table: """sets: [(name, env, default)]; sites: {module: {offset: [(set index, [word | ('page', reg)], original)]}} (the - alternatives of a site in file order); page. What the generated C takes from here (for_module, spins_for) is in - the lift cache's key (recomp/gen.py chunk_key); a new kind of line the Emitter reads must go there too.""" + alternatives of a site in file order); page. What the generated C takes from here (for_module, spins_for, hooks_for) + is in the lift cache's key (recomp/gen.py chunk_key); a new kind of line the Emitter reads must go there too.""" - def __init__(self, sets=(), sites=None, page=0, spins=None): + def __init__(self, sets=(), sites=None, page=0, spins=None, hooks=None): self.sets = list(sets) self.sites = sites or {} self.page = page self.spins = spins or {} # {module: {back-edge offset: head offset}} + self.hooks = hooks or {} # {module: {function offset}}; empty: FN_ENTER in every function + + def hooks_for(self, name, base): + """The absolute addresses of the module's hooked functions, or None when the table has no hook line at all + (FN_ENTER in every function).""" + if not self.hooks: + return None + return {base + o for o in self.hooks.get(name, ())} def spins_for(self, name, base): """{absolute back-edge address: absolute head address}.""" @@ -74,7 +89,7 @@ class Table: def load(path, mods): """Table from a patch file; mods is layout() output [(name, base, size)].""" - sets, index, sites, spins = [], {}, {}, {} + sets, index, sites, spins, hooks = [], {}, {}, {}, {} bases = {n: (b, s) for n, b, s in mods} for no, raw in enumerate(Path(path).read_text().splitlines(), 1): f = raw.split('#', 1)[0].split() @@ -113,6 +128,10 @@ def load(path, mods): if b in mod: raise PatchError(f'{where}: {f[1]}+{b:#x} is a back-edge twice') mod[b] = head + elif f[0] == 'hook' and len(f) >= 3: + if f[1] not in bases: + raise PatchError(f'{where}: module {f[1]} not in the layout') + hooks.setdefault(f[1], set()).update(int(o, 16) for o in f[2:]) else: raise PatchError(f'{where}: cannot parse {raw!r}') base, size = bases['main'][0], bases['main'][1] @@ -120,7 +139,7 @@ def load(path, mods): nxt = min((b for _, b, _ in mods if b > base), default=None) if nxt is not None and page + PAGE > nxt: raise PatchError(f'no room for the patch page after main ({page:#x})') - return Table(sets, sites, page, spins) + return Table(sets, sites, page, spins, hooks) def check(table, mod, name): diff --git a/runtime/cpu.h b/runtime/cpu.h index 207f4901..4d4291b0 100644 --- a/runtime/cpu.h +++ b/runtime/cpu.h @@ -69,7 +69,9 @@ typedef uint32_t __attribute__((aligned(1), may_alias)) u32u; typedef uint64_t __attribute__((aligned(1), may_alias)) u64u; typedef uint8_t __attribute__((may_alias)) u8a; -// Function-entry profiler (runtime/fnprof.c, LARECOMP_FNPROF): one predictable branch per call when off. +// Function-entry hook: one predictable branch per call while g_fnprof is 0. The generator emits it only in the +// functions of the patch table's `hook` lines (recomp/patches.py), or in every function for a table without any and +// with gen.py --fn-enter-all. extern volatile int g_fnprof; void fnprof_hit(uint64_t addr, uint64_t lr, uint64_t x0); #define FN_ENTER(a) do { if (__builtin_expect(g_fnprof, 0)) fnprof_hit(a, c->x[30], c->x[0]); } while (0) diff --git a/runtime/resolution.c b/runtime/resolution.c index 17d3db1e..313967fd 100644 --- a/runtime/resolution.c +++ b/runtime/resolution.c @@ -694,8 +694,9 @@ void res_fn_hit(uint64_t addr, uint64_t x0) { } void res_present(Cpu *c) { - // The hook runs on every guest function entry while it waits: should the game set its render size before - // res_init or never, it stops after 1800 presents (half a minute at 60 fps) and modes change at the next start. + // Should the game set its render size before res_init or never, the hook (FN_ENTER of the hooked functions, + // recomp/patches-v143.txt `hook`) stops after 1800 presents (half a minute at 60 fps) and modes change at the + // next start. static unsigned waited; if (g_fnprof && !renderer && ++waited == 1800) { g_fnprof = 0; diff --git a/tests/lift_cache_test.py b/tests/lift_cache_test.py index b8ba5ec4..78307a36 100644 --- a/tests/lift_cache_test.py +++ b/tests/lift_cache_test.py @@ -1,5 +1,5 @@ """recomp/gen.py's lift cache (/lift_cache.json) on a module it makes itself: a run with unchanged inputs reuses -every chunk, a changed patch site or spin wait lifts only the chunk that holds it, a chunk file or cache entry changed +every chunk, a changed patch site, spin wait or function hook lifts only the chunk that holds it, a chunk file or cache entry changed behind the cache's back is not taken, a cache that cannot be written only warns, TOTK_LIFT_CACHE=0 and a change to the generator's sources or options lift everything, and the output always equals a --no-cache lift's, folded.txt included. No game files. @@ -42,14 +42,16 @@ TEXT = JT_TEXT + struct.pack('<5I', bl(G, H), RET, MOV_W8_100, bl(H + 4, G), RET SPIN = 'spin main 48 c' -def patch_file(*replacements, first=None, spin=False): +def patch_file(*replacements, first=None, spin=False, hooks=()): """A patch file with set a patching h's mov with each replacement (sets a0, a1, ...); first: a set defined before - them, which moves their indices; spin: SPIN too.""" + them, which moves their indices; spin: SPIN too; hooks: a hook line with these function offsets.""" lines = [f'set {first} TEST_{first.upper()} 0'] if first else [] for i, w in enumerate(replacements): lines += [f'set a{i} TEST_A{i} 0', f'site a{i} main {H:x} {MOV_W8_100:08x} {w:08x}'] if spin: lines.append(SPIN) + if hooks: + lines.append('hook main ' + ' '.join(f'{o:x}' for o in hooks)) return '\n'.join(lines) + '\n' @@ -154,6 +156,29 @@ class LiftCache(unittest.TestCase): self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2) self.assertNotIn('rt_spin_gate', self.chunk(0).read_text()) + def test_hooks_are_in_the_key(self): + """No hook line: FN_ENTER in every function. The first hook line takes it out of every other function, so + every chunk is lifted; one more hooked function lifts only its chunk; --fn-enter-all puts it back everywhere; + a hook on an address that is no function start stops the lift.""" + fn_enter = lambda: ['FN_ENTER(' in self.chunk(n).read_text() for n in range(3)] # noqa: E731 + self.gen(patch_file(MOV_W8_200)) + self.assertEqual(fn_enter(), [True, True, True]) + self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G]))['chunks_reused'], 0) + self.assertEqual(fn_enter(), [False, True, False]) + self.assert_like_fresh(patch_file(MOV_W8_200, hooks=[G])) + self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G, H]))['chunks_reused'], 2) + self.assertEqual(fn_enter(), [False, True, True]) + self.assert_like_fresh(patch_file(MOV_W8_200, hooks=[G, H])) + self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G]), 'gen', '--fn-enter-all')['chunks_reused'], 0) + self.assertEqual(fn_enter(), [True, True, True]) + p = self.d / 'patches.txt' + p.write_text(patch_file(MOV_W8_200, hooks=[G + 4])) + r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'), + '--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1', '--patches', str(p)], + capture_output=True, text=True) + self.assertNotEqual(r.returncode, 0) + self.assertIn('is no function start', r.stderr) + def test_changed_cache_entry_is_not_taken(self): """A changed result in lift_cache.json (a folded range, here) fails the entry's digest: the chunk is lifted and folded.txt keeps the jump table's real bytes."""