Keep the six most used general registers of each generated function in C locals

Every lifted instruction read and wrote c->x[n] and c->sp, so each register passed from one guest instruction to the
next went through a store and a load, and clang could not keep it in a host register across a guest store.
recomp/regs.py now copies the six most used X registers and SP of a function into locals at its entry. A forward pass
over the function's flow tracks which locals changed since the last sync, and only those are stored back before a
call, an SVC, a trap, a return or a tail exit. After an SVC every local reloads; after a call only X0-X18 and X30,
since AAPCS64 has the callee preserve X19-X29 and SP. TotK does not break that: main and subsdk0 import neither
longjmp nor the unwinder, sdk's have no direct callers, and the runtime writes no register above X3 except X30.
gen.py --no-callee-saved-trust reloads them too, --reg-locals sets the count. Functions with an unresolved computed br
keep their registers in the Cpu, as they keep NZCV.

All registers in locals was slower than none: clang spills them to the stack and the syncs stay. Over 300 hot
functions, 4-6 locals gave the fewest instructions and memory operands.

1.4.3 .text 299 -> 266 MiB. Fight bench (bench-fight2-2026-10-27b, frames 5400-7500, uncapped): 11.30 and 11.92 ms vs
13.5-13.9 on main; frames 2400-5300 11.24 and 11.36 vs 13.25. Without the callee-saved trust 12.58 ms.
tests/diff/func.py: 0 failing on 1.4.3 (889 functions, 583 with calls) and 1.2.1 (810); a plan with no spills fails
at once. Probes kernel, vkexec, nvn-tex and shader pass in both versions.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-27 22:21:18 +03:00
1 parent fae0faf6dc
commit 0ca11dfba2
3 files changed
+203 -48

No files matched your search

+4 -1
View File
@@ -36,7 +36,10 @@ MacNativeGames cannot run on x86-64 as is: its CPU path executes ARM64 code nati
- jump tables resolved by our own symbolic pass instead of Unicorn, which keeps GPL code out of the shipped generator;
- `LDXP`/`STXP` (128-bit exclusive pairs) supported;
- no 64 MB limit on `main`; `subsdk*` loaded;
- guest registers kept in C locals where liveness allows, instead of memory.
- NZCV and the six most used general registers of each function kept in C locals instead of the `Cpu`
(`recomp/flags.py`, `recomp/regs.py`): registers are stored back where they changed, before calls, SVCs and
exits, and reloaded after them. A call reloads only X0-X18 and X30, as AAPCS64 has the callee preserve X19-X29 and
SP; `gen.py --no-callee-saved-trust` reloads those too.
- Coverage oracle: MacNativeGames' NSO rewriter lists every site where game code touches the OS (SVCs, TLS reads, counter reads, `x18`, exclusive windows, jump tables): 68,968 sites in 1.4.3 `main`. The lifter must handle every one.
- Memory: guest address equals host address, all below 2^39 (IPC encodes 39-bit addresses). The heap, stacks and the alias region are reserved with `MEM_WRITE_WATCH`, so the graphics back end can ask which pages the CPU wrote since the last upload (`runtime/memory.c`).
- Indirect branches go through `rt_dispatch` (`runtime/dispatch.c`), which looks the target up in the address table and tail-calls it.
+92 -47
View File
@@ -41,6 +41,7 @@ sys.path.insert(0, str(ROOT / 'recomp'))
import flags # noqa: E402
import jumptables # noqa: E402
import patches # noqa: E402
import regs # noqa: E402
from check_content import GENERATED_MARKER # noqa: E402
from layout import layout, write as write_layout # noqa: E402
from lifter import hx # noqa: E402
@@ -206,10 +207,12 @@ def partition(est, max_fns, max_bytes, big_fn):
class Emitter:
"""C for the functions of one module. starts is the set of every function start of the module."""
def __init__(self, mod, starts, patched=None):
def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True):
self.mod = mod
self.starts = starts
self.patched = patched or {} # {address: (set index, [replacement words])} (recomp/patches.py)
self.reg_locals = reg_locals # recomp/regs.py: registers per function in C locals
self.kept = regs.CALLEE_SAVED if callee_saved else frozenset() # not reloaded after a call
self.uses_patches = False
self.stats = Counter()
self.unsupported = Counter()
@@ -251,44 +254,48 @@ class Emitter:
"""Lifted replacement words of the patched site a."""
return [self.mod.lifter.lift(w, a) for w in self.patched[a][1]]
def flag_plan(self, insns, entry, end, tables):
"""recomp/flags.py plan for the function: the intra-function control flow and each instruction's flag use.
Index len(insns) stands for falling through into the next function."""
n = len(insns)
succ, calls, effects, exits = [], set(), [], {n}
@staticmethod
def flow(insns, entry, end, tables):
"""Successors of each instruction inside the function [entry, end). Index len(insns) stands for falling
through into the next function and has none."""
def idx(t):
return [(t - entry) // 4] if entry <= t < end else []
succ = []
for i, ins in enumerate(insns):
k, a = ins.kind, entry + 4 * i
nxt = [i + 1]
if k == 'op':
succ.append(nxt)
elif k in ('bl', 'blr'):
succ.append(nxt)
calls.add(i)
k = ins.kind
if k in ('op', 'bl', 'blr'):
succ.append([i + 1])
elif k == 'b':
succ.append(idx(ins.target))
if not idx(ins.target):
exits.add(i)
elif k == 'bcond':
succ.append(idx(ins.target) + nxt)
if not idx(ins.target):
exits.add(i)
succ.append(idx(ins.target) + [i + 1])
elif k == 'br':
jt = tables.get(a)
jt = tables.get(entry + 4 * i)
succ.append([t for x in jt for t in idx(x)] if jt and jt != 'all' else [])
exits.add(i) # through rt_dispatch, also a table's default
else: # ret, trap
succ.append([])
succ.append([])
return succ
def flag_plan(self, insns, entry, end, tables, succ):
"""recomp/flags.py plan for the function: each instruction's flag use over the flow succ (flow())."""
n = len(insns)
calls, effects, exits = set(), [], {n}
for i, ins in enumerate(insns):
k, a = ins.kind, entry + 4 * i
if k in ('bl', 'blr'):
calls.add(i)
elif k in ('b', 'bcond') and not entry <= ins.target < end:
exits.add(i)
elif k == 'br':
exits.add(i) # through rt_dispatch, also a table's default
if a in self.patched:
effects.append(self.patched_effect(ins, a))
if any(r.kind == 'bl' for r in self.replacements(a)):
calls.add(i)
else:
effects.append(flags.effect(ins.cond if k == 'bcond' else ins.code or ''))
succ.append([])
effects.append((False, None))
return flags.plan(effects, succ, calls, exits)
@@ -318,7 +325,11 @@ class Emitter:
labels = set(range(entry, end, 4))
# An unresolved computed br re-enters the function through fb_<entry>, a C function of its own, which local
# flags would not survive; such functions keep NZCV in c->nzcv.
fp = None if all_labels else self.flag_plan(insns, entry, end, tables)
# The same holds for registers in locals (recomp/regs.py): fb_<entry> keeps them in the Cpu too.
local = not all_labels
spill, reload, reload_call = (regs.SPILL, regs.RELOAD, regs.RELOAD_CALL) if local else ('', '', '')
succ = self.flow(insns, entry, end, tables) if local else None
fp = self.flag_plan(insns, entry, end, tables, succ) if local else None
if fp and fp.used:
self.stats['local_flag_functions'] += 1
self.stats['flag_reads_direct'] += sum(1 for k in fp.reader if k and k != 'nz')
@@ -339,9 +350,9 @@ class Emitter:
return f'goto L_{t:x};'
if t in starts:
refs.add(t)
return f'{publish(i)}{MUSTTAIL} {fname(t)}(c);'
return f'{publish(i)}{spill}{MUSTTAIL} {fname(t)}(c);'
self.stats['external_branches'] += 1
return f'{publish(i)}c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);'
return f'{publish(i)}{spill}c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);'
if all_labels:
# An unresolved computed br re-enters the function through fb_<entry>, which switches on c->pc to any
@@ -364,9 +375,13 @@ class Emitter:
def call(t, a):
if t in starts:
refs.add(t)
return f'c->x[30] = {hx(a + 4)}; {fname(t)}(c);'
return f'c->x[30] = {hx(a + 4)}; {spill}{fname(t)}(c);{reload_call}'
self.stats['external_calls'] += 1
return f'c->x[30] = {hx(a + 4)}; c->pc = {hx(t)}; rt_dispatch(c);'
return f'c->x[30] = {hx(a + 4)}; {spill}c->pc = {hx(t)}; rt_dispatch(c);{reload_call}'
def op(i, code):
code = fx(i, code)
return f'{spill}{code}{reload}' if regs.ESCAPE.search(code) else code
def patch(i, ins, a):
"""if (switch) { replacement } else { original } for a patched site (recomp/patches.py)."""
@@ -376,53 +391,76 @@ class Emitter:
repl = []
for j, (w, r) in enumerate(zip(words, self.replacements(a))):
if r.kind == 'op':
repl.append(fx(i, r.code))
repl.append(op(i, r.code))
elif r.kind == 'bl' and j == len(words) - 1:
repl.append(call(r.target, a))
else:
raise patches.PatchError(f'{a:#x}: replacement word {w:08x} is a {r.kind}')
orig = fx(i, ins.code) if ins.kind == 'op' else call(ins.target, a)
orig = op(i, ins.code) if ins.kind == 'op' else call(ins.target, a)
self.uses_patches = True
self.stats['patched_sites'] += 1
return f' if (rt_patch_on[{k}]) {{ {" ".join(repl)} }} else {{ {orig} }}'
body = [] # (instruction index or None for a label, line); a line belongs to the instruction it lifts
def emit(i, line):
body.append((i, line))
k = None
for i, ins in enumerate(insns):
a = entry + 4 * i
if a in labels:
out.append(f'L_{a:x}:;')
emit(None, f'L_{a:x}:;')
k = ins.kind
if a in self.patched:
out.append(patch(i, ins, a))
elif k == 'op' or k == 'trap':
out.append(' ' + fx(i, ins.code))
emit(i, patch(i, ins, a))
elif k == 'op':
emit(i, ' ' + op(i, ins.code))
elif k == 'trap':
emit(i, f' {spill}{fx(i, ins.code)}')
elif k == 'b':
out.append(' ' + jump(ins.target, i))
emit(i, ' ' + jump(ins.target, i))
elif k == 'bcond':
out.append(f' if ({fx(i, ins.cond)}) {{ {jump(ins.target, i)} }}')
emit(i, f' if ({fx(i, ins.cond)}) {{ {jump(ins.target, i)} }}')
elif k == 'bl':
out.append(' ' + call(ins.target, a))
emit(i, ' ' + call(ins.target, a))
elif k == 'blr':
out.append(f' c->pc = {ins.target}; c->x[30] = {hx(a + 4)}; rt_dispatch(c);')
emit(i, f' c->pc = {ins.target}; c->x[30] = {hx(a + 4)}; {spill}rt_dispatch(c);{reload_call}')
elif k == 'br':
jt = tables.get(a)
if jt == 'all':
out.append(f' c->pc = {ins.target}; {redispatch}')
emit(i, f' c->pc = {ins.target}; {redispatch}')
elif jt:
cases = ' '.join(f'case {hx(t)}: goto L_{t:x};' for t in jt)
out.append(f' switch ({ins.target}) {{ {cases} default: break; }}')
out.append(f' {publish(i)}c->pc = {ins.target}; {redispatch}')
emit(i, f' switch ({ins.target}) {{ {cases} default: break; }}')
emit(i, f' {publish(i)}{spill}c->pc = {ins.target}; {redispatch}')
else:
out.append(f' {publish(i)}c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);')
emit(i, f' {publish(i)}{spill}c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);')
elif k == 'ret':
out.append(' return;')
emit(i, f' {spill}return;')
else:
raise ValueError(f'{a:#x}: unknown instruction kind {k!r}')
n = len(insns)
if k not in TERMINATORS: # falls through into the next function
if end < mod.hi:
out.append(' ' + jump(end, len(insns)))
emit(n, ' ' + jump(end, n))
else:
out.append(f' rt_udf(c, {hx(end)});')
emit(n, f' {spill}rt_udf(c, {hx(end)});')
if local:
texts = [''] * (n + 1)
for i, line in body:
if i is not None:
texts[i] += line
clean = {i for i, ins in enumerate(insns) if entry + 4 * i not in self.patched
and (ins.kind in ('bl', 'blr') or (ins.kind == 'op' and regs.RELOAD in texts[i]))}
rp = regs.plan(texts, succ, clean, self.reg_locals, self.kept)
if rp.regs:
out.append(' ' + rp.decl())
self.stats['local_reg_functions'] += bool(rp.regs)
self.stats['reg_spills'] += sum(len(s) for s in rp.spill)
out += [line if i is None else rp.resolve(i, line) for i, line in body]
else:
out += [line for _, line in body]
out.append('}')
self.stats['functions'] += 1
self.stats['instructions'] += len(insns)
@@ -481,10 +519,11 @@ def _retry(fn, *args):
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter}
def _worker_init(mods, starts, table):
def _worker_init(mods, starts, table, opts):
_W['mods'] = mods
_W['starts'] = starts
_W['patches'] = table
_W['opts'] = opts # Emitter keyword arguments
_W['emitters'] = {}
@@ -494,7 +533,7 @@ def _emitter(name):
path, base = _W['mods'][name]
mod = Module(name, path, base, 'c')
e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()),
_W['patches'].for_module(name, base))
_W['patches'].for_module(name, base), **_W['opts'])
return e
@@ -556,7 +595,13 @@ def main():
ap.add_argument('--big-fn', type=int, default=300000, help='estimated bytes of C from which a function gets '
'its own chunk')
ap.add_argument('--patches', type=Path, help='code patch table (recomp/patches.py)')
ap.add_argument('--reg-locals', type=int, default=regs.LOCALS,
help=f'guest registers per function kept in C locals (recomp/regs.py; default {regs.LOCALS}, 0: none)')
ap.add_argument('--no-callee-saved-trust', action='store_true',
help='reload X19-X29 and SP after every call too, instead of trusting AAPCS64 (slower; for '
'telling a broken callee from a generator bug)')
args = ap.parse_args()
opts = {'reg_locals': args.reg_locals, 'callee_saved': not args.no_callee_saved_trust}
t0 = time.perf_counter()
mods = layout(args.exefs)
@@ -602,7 +647,7 @@ def main():
done_at, errors = {}, []
t_emit = time.perf_counter()
ctx = mp.get_context('spawn')
with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table)) as pool:
with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool:
for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)):
results[r['module']].append(r)
if r['error']:
+107
View File
@@ -0,0 +1,107 @@
"""Function-local general registers for recomp/gen.py.
The lifter reads and writes the guest's X0-X30 and SP in the Cpu (c->x[n], c->sp), so every guest instruction loads
and stores memory, and the C compiler cannot keep a register in a host register across a guest store: guest stores go
through pointers that may alias the Cpu. A generated function instead copies the LOCALS registers it uses most into
locals at its entry (xN, sp_), works on those, and syncs with the Cpu only where something else reads or writes them:
before a call, an SVC, a trap, a return or a tail exit it stores the registers it changed since the last sync (spill),
and after an SVC it loads them all again (reload). The other registers stay in the Cpu.
After a call it reloads only X0-X18 and X30: AAPCS64 has the callee preserve X19-X29 and SP, so their locals still
hold what the Cpu holds. What would break that does not occur in TotK: main and subsdk0 import neither longjmp nor
the C++ unwinder, sdk's longjmp, __cxa_throw and _Unwind_* have no direct callers, and the runtime writes no register
above X3 of a running thread's Cpu except X30, which rt_call restores (runtime/hostfn.c). Guest fibers each run on a host fiber of their own with their own Cpu (runtime/kernel.c),
and ExitThread leaves by longjmp without returning. gen.py --no-callee-saved-trust reloads those registers too.
Only plain assignments write registers in the lifted C (`c->x[n] = ...`, `c->sp = ...`), and no code takes a
register's address, so the rewrite is textual. A register counts as dirty from a write until the next spill on some
path: plan() runs that forward over the function's control flow, so a call spills only what changed since the
previous call or the entry. Spilling a register that is not dirty is harmless (the local equals the Cpu), so where a
case is unclear the plan errs on spilling more.
"""
import re
from collections import Counter
REG = re.compile(r'c->x\[(\d+)\]|c->sp\b')
WRITE = re.compile(r'c->(?:x\[(\d+)\]|(sp))\s*=(?!=)')
# Lifted instructions that hand the Cpu to the runtime: SVCs return, BRK and undefined words do not.
ESCAPE = re.compile(r'\brt_(?:svc|brk|udf)\(c\b')
SP = 31 # index of SP in the register sets
# Markers gen.py puts where the Cpu must be current, resolved by Plan.resolve(): spill before; reload all, or reload
# what a call may change, after.
SPILL, RELOAD, RELOAD_CALL = '\x00S', '\x00R', '\x00C'
CALLEE_SAVED = frozenset(range(19, 30)) | {SP}
# Locals per function: the most used registers. More cost more than they save, as clang then spills them to the stack
# and the syncs stay (300 hot TotK functions: fewest instructions and memory operands at 4-6).
LOCALS = 6
def name(r):
return 'sp_' if r == SP else f'x{r}'
def cpu(r):
return 'c->sp' if r == SP else f'c->x[{r}]'
def refs(text):
"""Registers text reads or writes, once per occurrence."""
return [SP if m.group(0) == 'c->sp' else int(m.group(1)) for m in REG.finditer(text)]
def written(text):
return {SP if m.group(2) else int(m.group(1)) for m in WRITE.finditer(text)}
class Plan:
def __init__(self, regs, spill, kept):
self.regs = regs # sorted registers the function keeps in locals
self.local = set(regs)
self.spill = spill # per instruction: registers to store at its sync points
self.kept = kept # registers a call leaves as they were (CALLEE_SAVED, or none)
def decl(self):
"""Locals at the top of the function, loaded from the Cpu."""
return 'uint64_t ' + ', '.join(f'{name(r)} = {cpu(r)}' for r in self.regs) + ';' if self.regs else ''
def resolve(self, i, text):
"""text of instruction i with its registers on the locals and its sync markers expanded."""
def local(m):
r = SP if m.group(0) == 'c->sp' else int(m.group(1))
return name(r) if r in self.local else m.group(0)
text = REG.sub(local, text)
spill = ''.join(f'{cpu(r)} = {name(r)}; ' for r in sorted(self.spill[i]))
reload = ''.join(f' {name(r)} = {cpu(r)};' for r in self.regs)
reload_call = ''.join(f' {name(r)} = {cpu(r)};' for r in self.regs if r not in self.kept)
return text.replace(SPILL, spill).replace(RELOAD, reload).replace(RELOAD_CALL, reload_call)
def plan(texts, succ, clean, n_locals=LOCALS, kept=CALLEE_SAVED):
"""texts[i]: instruction i's C with markers; succ[i]: successor indices inside the function; clean: instructions
after which every local equals the Cpu (a call or an SVC that spills and reloads on every path through it);
n_locals: how many registers to keep in locals; kept: registers a call does not reload."""
n = len(texts)
uses = Counter(r for t in texts for r in refs(t))
regs = {r for r, _ in sorted(uses.items(), key=lambda kv: (-kv[1], kv[0]))[:n_locals]}
writes = [written(t) & regs for t in texts]
dirty = [set() for _ in range(n)]
seen = [False] * n
work = [0] if n else []
if n:
seen[0] = True
while work:
i = work.pop()
out = set() if i in clean else dirty[i] | writes[i]
for s in succ[i]:
if not seen[s] or not out <= dirty[s]:
seen[s] = True
dirty[s] |= out
work.append(s)
every = set().union(*writes) if writes else set()
for i in range(n):
if not seen[i]: # not reached from the entry in the plan's flow: anything may be dirty
dirty[i] = every
# A sync point stores what was dirty on entry to its instruction and what the instruction wrote before it
# (c->x[30] of a call): all the instruction's writes, which is at most a little more.
return Plan(sorted(regs), [dirty[i] | writes[i] if SPILL in texts[i] else set() for i in range(n)], kept)