Merge the generated-code register work: NZCV and the six most used general registers of each function live in C locals, synced with the Cpu only at calls, SVCs and exits; tests/diff/func.py checks the registers at every call

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-27 22:24:24 +03:00
commit adc1cbce69
11 files changed
+841 -72

No files matched your search

+4 -1
View File
@@ -36,7 +36,10 @@ MacNativeGames cannot run on x86-64 as is: its CPU path executes ARM64 code nati
- jump tables resolved by our own symbolic pass instead of Unicorn, which keeps GPL code out of the shipped generator;
- `LDXP`/`STXP` (128-bit exclusive pairs) supported;
- no 64 MB limit on `main`; `subsdk*` loaded;
- guest registers kept in C locals where liveness allows, instead of memory.
- NZCV and the six most used general registers of each function kept in C locals instead of the `Cpu`
(`recomp/flags.py`, `recomp/regs.py`): registers are stored back where they changed, before calls, SVCs and
exits, and reloaded after them. A call reloads only X0-X18 and X30, as AAPCS64 has the callee preserve X19-X29 and
SP; `gen.py --no-callee-saved-trust` reloads those too.
- Coverage oracle: MacNativeGames' NSO rewriter lists every site where game code touches the OS (SVCs, TLS reads, counter reads, `x18`, exclusive windows, jump tables): 68,968 sites in 1.4.3 `main`. The lifter must handle every one.
- Memory: guest address equals host address, all below 2^39 (IPC encodes 39-bit addresses). The heap, stacks and the alias region are reserved with `MEM_WRITE_WATCH`, so the graphics back end can ask which pages the CPU wrote since the last upload (`runtime/memory.c`).
- Indirect branches go through `rt_dispatch` (`runtime/dispatch.c`), which looks the target up in the address table and tail-calls it.
+2 -1
View File
@@ -138,7 +138,8 @@ change passes these, all run with `.venv`:
| `tests\diff\lift.py` | instructions sampled from the extracted modules; `--coverage` lists every word the lifter rejects or traps | yes |
| `tests\diff\simd.py` | the whole FP/SIMD encoding space | no |
| `tests\diff\atomic.py` | exclusive load/store sequences and ARMv8.1 atomics | no |
| `tests\diff\func.py` | whole leaf functions as the generator emits them, loops and jump tables included | yes |
| `tests\diff\func.py` | whole functions as the generator emits them, loops, jump tables and calls included (calls go to a stub that checks the registers at the call) | yes |
| `tests\flags_check.py` | the function-local flag helpers of `runtime\cpu.h` against its packed ones | no |
| `tests\excl_mt.py` | the exclusive monitor helpers under contention and ABA, on several threads | no |
| `tests\jumptables_check.py` | `recomp/jumptables.py` against a Unicorn oracle; must report 0 disagreements, on 1.4.3 too | yes |
+144
View File
@@ -0,0 +1,144 @@
"""Function-local NZCV for recomp/gen.py.
The lifter writes flags the way the guest does: every flag-setting instruction packs N, Z, C and V into c->nzcv
(subs64() and friends in runtime/cpu.h) and every reader unpacks them (COND_xx(c)). Compilers keep NZCV inside a
function, and AAPCS64 neither passes it to a callee nor preserves it across a call. So a generated function keeps its
flags in a local `Flags F` (runtime/cpu.h), which clang holds in registers, and writes c->nzcv only where it leaves
for another function other than by a call or a return (see plan()).
A flag-setting instruction of a known kind (subtract, add or logic result; 32 or 64 bits) stores its operands in F.a
and F.b. A reader all of whose reaching definitions are of one kind computes its condition from those operands, so
`cmp x1, x2; b.lt` becomes a signed compare in C. Instructions whose result is only known packed (CCMP, FCMP, MSR
NZCV, ADCS) write F.nz, the packed form; a reader whose definitions are of several kinds reads F.nz, and those
definitions then also write it.
A read that can see flags a callee left reads the caller's flags from before the call instead. AAPCS64 code never
reads a callee's flags, so such a read only sits on a path that does not run: in TotK it follows calls to noreturn
functions (asserts) that the compiler placed before another branch's target. Plan.after_call counts them.
"""
import re
KIND_DEF = re.compile(r'\b(subs|adds|logic)(32|64)\(c, ')
CARRY_DEF = re.compile(r'\b(adcs|sbcs)(32|64)\(c, ')
CARRY_USE = re.compile(r'\b(adc|sbc)(32|64)\(c, ')
COND = re.compile(r'COND_([A-Z]{2})\(c\)')
NZ_DEF = 'c->nzcv = '
NZ_USE = '(c->nzcv)'
KIND_NAME = {'subs': 'sub', 'adds': 'add', 'logic': 'logic'}
ENTRY, CALL = 'entry', 'call'
class FlagError(Exception):
pass
def effect(code):
"""(reads flags, kind of the flags written or None) of one instruction's C."""
reads = bool(COND.search(code) or CARRY_USE.search(code) or CARRY_DEF.search(code) or NZ_USE in code)
if NZ_DEF in code or CARRY_DEF.search(code):
return reads, 'nz'
defs = KIND_DEF.findall(code)
if len(defs) > 1:
raise FlagError(f'several flag definitions in one instruction: {code}')
if defs:
op, size = defs[0]
return reads, KIND_NAME[op] + size
return reads, None
class Plan:
"""Per instruction: the kind its flag reads use, whether its flag definition also writes F.nz, and what it writes
to c->nzcv before it leaves for another function."""
def __init__(self, n):
self.reader = [None] * n
self.pack = [False] * n
self.exit = [None] * n # kind of the flags to publish in c->nzcv at a tail exit, or None
self.entry = False # F.nz starts as c->nzcv: flags from the entry reach a read or an exit
self.entry_read = False # they reach a read, not only an exit
self.after_call = 0 # reads that can see flags a callee left (see the module docstring)
self.used = False
def decl(self):
"""Declaration of F at the top of the function."""
return 'Flags F = {.nz = c->nzcv};' if self.entry else 'Flags F = {0};'
def publish(self, i):
"""Statement writing the flags to c->nzcv at instruction i's tail exit ('' when it need not)."""
k = self.exit[i]
return f'c->nzcv = fnz_{k}(&F); ' if k else ''
def rewrite(self, i, code):
"""code of instruction i with its flag reads and writes on F."""
rk = self.reader[i]
_, dk = effect(code)
if rk:
code = COND.sub(lambda m: f'fc_{rk}(&F, CC_{m.group(1)})', code)
code = CARRY_DEF.sub(lambda m: f'{m.group(1)}{m.group(2)}f(&F, fc_{rk}(&F, CC_CS), ', code)
code = CARRY_USE.sub(lambda m: f'{m.group(1)}{m.group(2)}f(fc_{rk}(&F, CC_CS), ', code)
code = code.replace(NZ_USE, f'(fnz_{rk}(&F))')
if dk == 'nz':
code = code.replace(NZ_DEF, 'F.nz = ')
code = re.sub(r'\(void\)(subs|adds)(32|64)\(c, ', lambda m: f'F.nz = nz_{KIND_NAME[m.group(1)]}{m.group(2)}(',
code)
elif dk:
code = KIND_DEF.sub(lambda m: f'f{KIND_NAME[m.group(1)]}{m.group(2)}(&F, ', code)
if self.pack[i]:
code += f' F.nz = fnz_{dk}(&F);'
if 'c->nzcv' in code or COND.search(code) or KIND_DEF.search(code) or CARRY_USE.search(code):
raise FlagError(f'flag access left after the rewrite: {code}')
return code
def plan(effects, succ, calls, exits):
"""effects[i]: (reads, kind or None); succ[i]: successor indices inside the function; calls: indices of calls,
whose callee may leave any flags; exits: indices that can leave for another function other than by a call or a
return (a tail branch, falling into the next function, a br through rt_dispatch).
Discovery can start a function inside another one (a branch target, a materialised address), and the code
before it then falls or branches into it with live flags. So a tail exit publishes the flags in c->nzcv, and a
function whose reads can see its entry flags starts F.nz from c->nzcv. Calls and returns publish nothing: AAPCS64
code neither passes flags to a callee nor reads a callee's, so a call's flags count as no definition."""
n = len(effects)
p = Plan(n)
if not any(r or k for r, k in effects):
return p
p.used = True
reach = [set() for _ in range(n)]
reach[0].add(ENTRY)
work = [0]
while work:
i = work.pop()
out = {i} if effects[i][1] else ({CALL} if i in calls else reach[i])
for s in succ[i]:
if not out <= reach[s]:
reach[s] |= out
work.append(s)
def kind(d):
return 'nz' if d == ENTRY else effects[d][1]
def use(r):
"""Kind a use of the definitions r reads; several kinds read the packed form, which they then all write."""
kinds = {kind(d) for d in r}
if ENTRY in r:
p.entry = True
if len(kinds) == 1:
return kinds.pop()
for d in r:
if d != ENTRY:
p.pack[d] = True
return 'nz' # several kinds, or none (unreachable)
for i, (reads, _) in enumerate(effects):
if reads:
p.after_call += CALL in reach[i]
p.reader[i] = use(reach[i] - {CALL})
p.entry_read |= ENTRY in reach[i]
for i in exits:
# Entry flags alone are still in c->nzcv.
r = reach[i] - {CALL}
if r and r != {ENTRY}:
p.exit[i] = use(r)
return p
+161 -29
View File
@@ -38,8 +38,10 @@ ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tools'))
sys.path.insert(0, str(ROOT / 'recomp'))
import flags # noqa: E402
import jumptables # noqa: E402
import patches # noqa: E402
import regs # noqa: E402
from check_content import GENERATED_MARKER # noqa: E402
from layout import layout, write as write_layout # noqa: E402
from lifter import hx # noqa: E402
@@ -205,10 +207,12 @@ def partition(est, max_fns, max_bytes, big_fn):
class Emitter:
"""C for the functions of one module. starts is the set of every function start of the module."""
def __init__(self, mod, starts, patched=None):
def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True):
self.mod = mod
self.starts = starts
self.patched = patched or {} # {address: (set index, [replacement words])} (recomp/patches.py)
self.reg_locals = reg_locals # recomp/regs.py: registers per function in C locals
self.kept = regs.CALLEE_SAVED if callee_saved else frozenset() # not reloaded after a call
self.uses_patches = False
self.stats = Counter()
self.unsupported = Counter()
@@ -246,6 +250,70 @@ class Emitter:
raise ValueError(f'{entry + 4 * i:#x}: asm instruction from the portable lifter')
return labels, tables
def replacements(self, a):
"""Lifted replacement words of the patched site a."""
return [self.mod.lifter.lift(w, a) for w in self.patched[a][1]]
@staticmethod
def flow(insns, entry, end, tables):
"""Successors of each instruction inside the function [entry, end). Index len(insns) stands for falling
through into the next function and has none."""
def idx(t):
return [(t - entry) // 4] if entry <= t < end else []
succ = []
for i, ins in enumerate(insns):
k = ins.kind
if k in ('op', 'bl', 'blr'):
succ.append([i + 1])
elif k == 'b':
succ.append(idx(ins.target))
elif k == 'bcond':
succ.append(idx(ins.target) + [i + 1])
elif k == 'br':
jt = tables.get(entry + 4 * i)
succ.append([t for x in jt for t in idx(x)] if jt and jt != 'all' else [])
else: # ret, trap
succ.append([])
succ.append([])
return succ
def flag_plan(self, insns, entry, end, tables, succ):
"""recomp/flags.py plan for the function: each instruction's flag use over the flow succ (flow())."""
n = len(insns)
calls, effects, exits = set(), [], {n}
for i, ins in enumerate(insns):
k, a = ins.kind, entry + 4 * i
if k in ('bl', 'blr'):
calls.add(i)
elif k in ('b', 'bcond') and not entry <= ins.target < end:
exits.add(i)
elif k == 'br':
exits.add(i) # through rt_dispatch, also a table's default
if a in self.patched:
effects.append(self.patched_effect(ins, a))
if any(r.kind == 'bl' for r in self.replacements(a)):
calls.add(i)
else:
effects.append(flags.effect(ins.cond if k == 'bcond' else ins.code or ''))
effects.append((False, None))
return flags.plan(effects, succ, calls, exits)
def patched_effect(self, ins, a):
"""Flag effect of a patched site: both alternatives must have the same one, in at most one word each."""
alts = []
for words in ([ins], self.replacements(a)):
fx = [flags.effect(w.code or '') for w in words if w.kind == 'op']
fx = [e for e in fx if e[0] or e[1]]
if len(fx) > 1:
raise patches.PatchError(f'{a:#x}: several flag-using words in one alternative')
alts.append(fx[0] if fx else (False, None))
if alts[0][1] != alts[1][1]:
raise patches.PatchError(f'{a:#x}: the alternatives write different flags ({alts[0][1]}, {alts[1][1]})')
if (alts[0][1] or alts[0][0] or alts[1][0]) and ins.kind == 'bl':
raise patches.PatchError(f'{a:#x}: a patched call that uses flags')
return alts[0][0] or alts[1][0], alts[0][1]
def function(self, entry, end):
"""(lines of C, referenced function starts) for the guest function [entry, end)."""
mod, starts = self.mod, self.starts
@@ -255,15 +323,36 @@ class Emitter:
all_labels = 'all' in tables.values()
if all_labels:
labels = set(range(entry, end, 4))
# An unresolved computed br re-enters the function through fb_<entry>, a C function of its own, which local
# flags would not survive; such functions keep NZCV in c->nzcv.
# The same holds for registers in locals (recomp/regs.py): fb_<entry> keeps them in the Cpu too.
local = not all_labels
spill, reload, reload_call = (regs.SPILL, regs.RELOAD, regs.RELOAD_CALL) if local else ('', '', '')
succ = self.flow(insns, entry, end, tables) if local else None
fp = self.flag_plan(insns, entry, end, tables, succ) if local else None
if fp and fp.used:
self.stats['local_flag_functions'] += 1
self.stats['flag_reads_direct'] += sum(1 for k in fp.reader if k and k != 'nz')
self.stats['flag_reads_packed'] += sum(1 for k in fp.reader if k == 'nz')
self.stats['flag_entry_functions'] += fp.entry
self.stats['flag_entry_reads'] += fp.entry_read
self.stats['flag_reads_after_call'] += fp.after_call
self.stats['flag_publishing_exits'] += sum(1 for k in fp.exit if k)
def jump(t):
def fx(i, code):
return fp.rewrite(i, code) if fp and fp.used else code
def publish(i):
return fp.publish(i) if fp and fp.used else ''
def jump(t, i):
if entry <= t < end:
return f'goto L_{t:x};'
if t in starts:
refs.add(t)
return f'{MUSTTAIL} {fname(t)}(c);'
return f'{publish(i)}{spill}{MUSTTAIL} {fname(t)}(c);'
self.stats['external_branches'] += 1
return f'c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);'
return f'{publish(i)}{spill}c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);'
if all_labels:
# An unresolved computed br re-enters the function through fb_<entry>, which switches on c->pc to any
@@ -280,69 +369,98 @@ class Emitter:
else:
redispatch = f'{MUSTTAIL} rt_dispatch(c);'
out = [f'void {fname(entry)}(Cpu *c) {{', f' FN_ENTER({hx(entry)});']
if fp.used:
out.append(' ' + fp.decl())
def call(t, a):
if t in starts:
refs.add(t)
return f'c->x[30] = {hx(a + 4)}; {fname(t)}(c);'
return f'c->x[30] = {hx(a + 4)}; {spill}{fname(t)}(c);{reload_call}'
self.stats['external_calls'] += 1
return f'c->x[30] = {hx(a + 4)}; c->pc = {hx(t)}; rt_dispatch(c);'
return f'c->x[30] = {hx(a + 4)}; {spill}c->pc = {hx(t)}; rt_dispatch(c);{reload_call}'
def patch(ins, a):
def op(i, code):
code = fx(i, code)
return f'{spill}{code}{reload}' if regs.ESCAPE.search(code) else code
def patch(i, ins, a):
"""if (switch) { replacement } else { original } for a patched site (recomp/patches.py)."""
k, words = self.patched[a]
if ins.kind not in ('op', 'bl'):
raise patches.PatchError(f'{a:#x}: patched instruction is a {ins.kind}')
repl = []
for j, w in enumerate(words):
r = mod.lifter.lift(w, a)
for j, (w, r) in enumerate(zip(words, self.replacements(a))):
if r.kind == 'op':
repl.append(r.code)
repl.append(op(i, r.code))
elif r.kind == 'bl' and j == len(words) - 1:
repl.append(call(r.target, a))
else:
raise patches.PatchError(f'{a:#x}: replacement word {w:08x} is a {r.kind}')
orig = ins.code if ins.kind == 'op' else call(ins.target, a)
orig = op(i, ins.code) if ins.kind == 'op' else call(ins.target, a)
self.uses_patches = True
self.stats['patched_sites'] += 1
return f' if (rt_patch_on[{k}]) {{ {" ".join(repl)} }} else {{ {orig} }}'
body = [] # (instruction index or None for a label, line); a line belongs to the instruction it lifts
def emit(i, line):
body.append((i, line))
k = None
for i, ins in enumerate(insns):
a = entry + 4 * i
if a in labels:
out.append(f'L_{a:x}:;')
emit(None, f'L_{a:x}:;')
k = ins.kind
if a in self.patched:
out.append(patch(ins, a))
elif k == 'op' or k == 'trap':
out.append(' ' + ins.code)
emit(i, patch(i, ins, a))
elif k == 'op':
emit(i, ' ' + op(i, ins.code))
elif k == 'trap':
emit(i, f' {spill}{fx(i, ins.code)}')
elif k == 'b':
out.append(' ' + jump(ins.target))
emit(i, ' ' + jump(ins.target, i))
elif k == 'bcond':
out.append(f' if ({ins.cond}) {{ {jump(ins.target)} }}')
emit(i, f' if ({fx(i, ins.cond)}) {{ {jump(ins.target, i)} }}')
elif k == 'bl':
out.append(' ' + call(ins.target, a))
emit(i, ' ' + call(ins.target, a))
elif k == 'blr':
out.append(f' c->pc = {ins.target}; c->x[30] = {hx(a + 4)}; rt_dispatch(c);')
emit(i, f' c->pc = {ins.target}; c->x[30] = {hx(a + 4)}; {spill}rt_dispatch(c);{reload_call}')
elif k == 'br':
jt = tables.get(a)
if jt == 'all':
out.append(f' c->pc = {ins.target}; {redispatch}')
emit(i, f' c->pc = {ins.target}; {redispatch}')
elif jt:
cases = ' '.join(f'case {hx(t)}: goto L_{t:x};' for t in jt)
out.append(f' switch ({ins.target}) {{ {cases} default: break; }}')
out.append(f' c->pc = {ins.target}; {redispatch}')
emit(i, f' switch ({ins.target}) {{ {cases} default: break; }}')
emit(i, f' {publish(i)}{spill}c->pc = {ins.target}; {redispatch}')
else:
out.append(f' c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);')
emit(i, f' {publish(i)}{spill}c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);')
elif k == 'ret':
out.append(' return;')
emit(i, f' {spill}return;')
else:
raise ValueError(f'{a:#x}: unknown instruction kind {k!r}')
n = len(insns)
if k not in TERMINATORS: # falls through into the next function
if end < mod.hi:
out.append(' ' + jump(end))
emit(n, ' ' + jump(end, n))
else:
out.append(f' rt_udf(c, {hx(end)});')
emit(n, f' {spill}rt_udf(c, {hx(end)});')
if local:
texts = [''] * (n + 1)
for i, line in body:
if i is not None:
texts[i] += line
clean = {i for i, ins in enumerate(insns) if entry + 4 * i not in self.patched
and (ins.kind in ('bl', 'blr') or (ins.kind == 'op' and regs.RELOAD in texts[i]))}
rp = regs.plan(texts, succ, clean, self.reg_locals, self.kept)
if rp.regs:
out.append(' ' + rp.decl())
self.stats['local_reg_functions'] += bool(rp.regs)
self.stats['reg_spills'] += sum(len(s) for s in rp.spill)
out += [line if i is None else rp.resolve(i, line) for i, line in body]
else:
out += [line for _, line in body]
out.append('}')
self.stats['functions'] += 1
self.stats['instructions'] += len(insns)
@@ -401,10 +519,11 @@ def _retry(fn, *args):
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter}
def _worker_init(mods, starts, table):
def _worker_init(mods, starts, table, opts):
_W['mods'] = mods
_W['starts'] = starts
_W['patches'] = table
_W['opts'] = opts # Emitter keyword arguments
_W['emitters'] = {}
@@ -414,7 +533,7 @@ def _emitter(name):
path, base = _W['mods'][name]
mod = Module(name, path, base, 'c')
e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()),
_W['patches'].for_module(name, base))
_W['patches'].for_module(name, base), **_W['opts'])
return e
@@ -476,7 +595,13 @@ def main():
ap.add_argument('--big-fn', type=int, default=300000, help='estimated bytes of C from which a function gets '
'its own chunk')
ap.add_argument('--patches', type=Path, help='code patch table (recomp/patches.py)')
ap.add_argument('--reg-locals', type=int, default=regs.LOCALS,
help=f'guest registers per function kept in C locals (recomp/regs.py; default {regs.LOCALS}, 0: none)')
ap.add_argument('--no-callee-saved-trust', action='store_true',
help='reload X19-X29 and SP after every call too, instead of trusting AAPCS64 (slower; for '
'telling a broken callee from a generator bug)')
args = ap.parse_args()
opts = {'reg_locals': args.reg_locals, 'callee_saved': not args.no_callee_saved_trust}
t0 = time.perf_counter()
mods = layout(args.exefs)
@@ -522,7 +647,7 @@ def main():
done_at, errors = {}, []
t_emit = time.perf_counter()
ctx = mp.get_context('spawn')
with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table)) as pool:
with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool:
for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)):
results[r['module']].append(r)
if r['error']:
@@ -562,6 +687,13 @@ def main():
'br_dispatch': stats['br_dispatch'],
'traps': stats['traps'],
'patched_sites': stats['patched_sites'],
'local_flag_functions': stats['local_flag_functions'],
'flag_reads_direct': stats['flag_reads_direct'],
'flag_reads_packed': stats['flag_reads_packed'],
'flag_entry_functions': stats['flag_entry_functions'],
'flag_entry_reads': stats['flag_entry_reads'],
'flag_reads_after_call': stats['flag_reads_after_call'],
'flag_publishing_exits': stats['flag_publishing_exits'],
'unsupported': stats['unsupported'],
'external_branches': stats['external_branches'],
'external_calls': stats['external_calls'],
+107
View File
@@ -0,0 +1,107 @@
"""Function-local general registers for recomp/gen.py.
The lifter reads and writes the guest's X0-X30 and SP in the Cpu (c->x[n], c->sp), so every guest instruction loads
and stores memory, and the C compiler cannot keep a register in a host register across a guest store: guest stores go
through pointers that may alias the Cpu. A generated function instead copies the LOCALS registers it uses most into
locals at its entry (xN, sp_), works on those, and syncs with the Cpu only where something else reads or writes them:
before a call, an SVC, a trap, a return or a tail exit it stores the registers it changed since the last sync (spill),
and after an SVC it loads them all again (reload). The other registers stay in the Cpu.
After a call it reloads only X0-X18 and X30: AAPCS64 has the callee preserve X19-X29 and SP, so their locals still
hold what the Cpu holds. What would break that does not occur in TotK: main and subsdk0 import neither longjmp nor
the C++ unwinder, sdk's longjmp, __cxa_throw and _Unwind_* have no direct callers, and the runtime writes no register
above X3 of a running thread's Cpu except X30, which rt_call restores (runtime/hostfn.c). Guest fibers each run on a host fiber of their own with their own Cpu (runtime/kernel.c),
and ExitThread leaves by longjmp without returning. gen.py --no-callee-saved-trust reloads those registers too.
Only plain assignments write registers in the lifted C (`c->x[n] = ...`, `c->sp = ...`), and no code takes a
register's address, so the rewrite is textual. A register counts as dirty from a write until the next spill on some
path: plan() runs that forward over the function's control flow, so a call spills only what changed since the
previous call or the entry. Spilling a register that is not dirty is harmless (the local equals the Cpu), so where a
case is unclear the plan errs on spilling more.
"""
import re
from collections import Counter
REG = re.compile(r'c->x\[(\d+)\]|c->sp\b')
WRITE = re.compile(r'c->(?:x\[(\d+)\]|(sp))\s*=(?!=)')
# Lifted instructions that hand the Cpu to the runtime: SVCs return, BRK and undefined words do not.
ESCAPE = re.compile(r'\brt_(?:svc|brk|udf)\(c\b')
SP = 31 # index of SP in the register sets
# Markers gen.py puts where the Cpu must be current, resolved by Plan.resolve(): spill before; reload all, or reload
# what a call may change, after.
SPILL, RELOAD, RELOAD_CALL = '\x00S', '\x00R', '\x00C'
CALLEE_SAVED = frozenset(range(19, 30)) | {SP}
# Locals per function: the most used registers. More cost more than they save, as clang then spills them to the stack
# and the syncs stay (300 hot TotK functions: fewest instructions and memory operands at 4-6).
LOCALS = 6
def name(r):
return 'sp_' if r == SP else f'x{r}'
def cpu(r):
return 'c->sp' if r == SP else f'c->x[{r}]'
def refs(text):
"""Registers text reads or writes, once per occurrence."""
return [SP if m.group(0) == 'c->sp' else int(m.group(1)) for m in REG.finditer(text)]
def written(text):
return {SP if m.group(2) else int(m.group(1)) for m in WRITE.finditer(text)}
class Plan:
def __init__(self, regs, spill, kept):
self.regs = regs # sorted registers the function keeps in locals
self.local = set(regs)
self.spill = spill # per instruction: registers to store at its sync points
self.kept = kept # registers a call leaves as they were (CALLEE_SAVED, or none)
def decl(self):
"""Locals at the top of the function, loaded from the Cpu."""
return 'uint64_t ' + ', '.join(f'{name(r)} = {cpu(r)}' for r in self.regs) + ';' if self.regs else ''
def resolve(self, i, text):
"""text of instruction i with its registers on the locals and its sync markers expanded."""
def local(m):
r = SP if m.group(0) == 'c->sp' else int(m.group(1))
return name(r) if r in self.local else m.group(0)
text = REG.sub(local, text)
spill = ''.join(f'{cpu(r)} = {name(r)}; ' for r in sorted(self.spill[i]))
reload = ''.join(f' {name(r)} = {cpu(r)};' for r in self.regs)
reload_call = ''.join(f' {name(r)} = {cpu(r)};' for r in self.regs if r not in self.kept)
return text.replace(SPILL, spill).replace(RELOAD, reload).replace(RELOAD_CALL, reload_call)
def plan(texts, succ, clean, n_locals=LOCALS, kept=CALLEE_SAVED):
"""texts[i]: instruction i's C with markers; succ[i]: successor indices inside the function; clean: instructions
after which every local equals the Cpu (a call or an SVC that spills and reloads on every path through it);
n_locals: how many registers to keep in locals; kept: registers a call does not reload."""
n = len(texts)
uses = Counter(r for t in texts for r in refs(t))
regs = {r for r, _ in sorted(uses.items(), key=lambda kv: (-kv[1], kv[0]))[:n_locals]}
writes = [written(t) & regs for t in texts]
dirty = [set() for _ in range(n)]
seen = [False] * n
work = [0] if n else []
if n:
seen[0] = True
while work:
i = work.pop()
out = set() if i in clean else dirty[i] | writes[i]
for s in succ[i]:
if not seen[s] or not out <= dirty[s]:
seen[s] = True
dirty[s] |= out
work.append(s)
every = set().union(*writes) if writes else set()
for i in range(n):
if not seen[i]: # not reached from the entry in the plan's flow: anything may be dirty
dirty[i] = every
# A sync point stores what was dirty on entry to its instruction and what the instruction wrote before it
# (c->x[30] of a call): all the instruction's writes, which is at most a little more.
return Plan(sorted(regs), [dirty[i] | writes[i] if SPILL in texts[i] else set() for i in range(n)], kept)
+101
View File
@@ -215,6 +215,107 @@ static inline uint32_t logic32(Cpu *c, uint32_t r) { c->nzcv = MKFLAGS(r >> 31,
ADC_IMPL(64, )
ADC_IMPL(32, )
// ---- function-local flags (recomp/flags.py)
// Generated functions keep NZCV in a local Flags F, never in c->nzcv. A definition of a known kind stores its
// operands (a - b, a + b, or a logic result in a); fc_<kind>(&F, CC_xx) computes a condition from them, so a compare
// and its branch become one host compare. F.nz holds the packed form for definitions only known packed (CCMP, FCMP,
// MSR, ADCS) and for readers that several kinds of definition reach. Every helper is always inlined with a constant
// condition, so F stays in registers and only the condition's own arithmetic remains.
typedef struct { uint64_t a, b, nz; } Flags;
enum { CC_EQ, CC_NE, CC_CS, CC_CC, CC_MI, CC_PL, CC_VS, CC_VC, CC_HI, CC_LS, CC_GE, CC_LT, CC_GT, CC_LE };
#define CPU_FL static inline __attribute__((always_inline))
CPU_FL int fl_cond(int cc, int n, int z, int cy, int v) {
switch (cc) {
case CC_EQ: return z;
case CC_NE: return !z;
case CC_CS: return cy;
case CC_CC: return !cy;
case CC_MI: return n;
case CC_PL: return !n;
case CC_VS: return v;
case CC_VC: return !v;
case CC_HI: return cy && !z;
case CC_LS: return !cy || z;
case CC_GE: return n == v;
case CC_LT: return n != v;
case CC_GT: return !z && n == v;
default: return z || n != v;
}
}
// Subtract (SUBS, CMP): conditions as host compares of the operands.
#define FL_SUB(N, U, S) \
CPU_FL U fsub##N(Flags *f, U a, U b) { f->a = a; f->b = b; return (U)(a - b); } \
CPU_FL int fc_sub##N(const Flags *f, int cc) { \
U a = (U)f->a, b = (U)f->b, r = (U)(a - b); \
switch (cc) { \
case CC_EQ: return a == b; \
case CC_NE: return a != b; \
case CC_CS: return a >= b; \
case CC_CC: return a < b; \
case CC_MI: return (S)r < 0; \
case CC_PL: return (S)r >= 0; \
case CC_VS: return (S)((a ^ b) & (a ^ r)) < 0; \
case CC_VC: return (S)((a ^ b) & (a ^ r)) >= 0; \
case CC_HI: return a > b; \
case CC_LS: return a <= b; \
case CC_GE: return (S)a >= (S)b; \
case CC_LT: return (S)a < (S)b; \
case CC_GT: return (S)a > (S)b; \
default: return (S)a <= (S)b; \
} \
} \
CPU_FL uint64_t nz_sub##N(U a, U b) { \
U r = (U)(a - b); \
return MKFLAGS((S)r < 0, r == 0, a >= b, (S)((a ^ b) & (a ^ r)) < 0); \
} \
CPU_FL uint64_t fnz_sub##N(const Flags *f) { return nz_sub##N((U)f->a, (U)f->b); }
FL_SUB(64, uint64_t, int64_t)
FL_SUB(32, uint32_t, int32_t)
// Add (ADDS, CMN) and logic results (ANDS, TST, BICS): conditions from N, Z, C, V.
#define FL_ADD(N, U, S) \
CPU_FL U fadd##N(Flags *f, U a, U b) { f->a = a; f->b = b; return (U)(a + b); } \
CPU_FL uint64_t nz_add##N(U a, U b) { \
U r = (U)(a + b); \
return MKFLAGS((S)r < 0, r == 0, r < a, (S)((a ^ r) & (b ^ r)) < 0); \
} \
CPU_FL int fc_add##N(const Flags *f, int cc) { \
U a = (U)f->a, b = (U)f->b, r = (U)(a + b); \
return fl_cond(cc, (S)r < 0, r == 0, r < a, (S)((a ^ r) & (b ^ r)) < 0); \
} \
CPU_FL uint64_t fnz_add##N(const Flags *f) { return nz_add##N((U)f->a, (U)f->b); } \
CPU_FL U flogic##N(Flags *f, U r) { f->a = r; return r; } \
CPU_FL int fc_logic##N(const Flags *f, int cc) { \
U r = (U)f->a; \
return fl_cond(cc, (S)r < 0, r == 0, 0, 0); \
} \
CPU_FL uint64_t fnz_logic##N(const Flags *f) { return MKFLAGS((S)f->a < 0, (U)f->a == 0, 0, 0); }
FL_ADD(64, uint64_t, int64_t)
FL_ADD(32, uint32_t, int32_t)
// Packed flags.
CPU_FL int fc_nz(const Flags *f, int cc) {
return fl_cond(cc, f->nz >> 31 & 1, f->nz >> 30 & 1, f->nz >> 29 & 1, f->nz >> 28 & 1);
}
CPU_FL uint64_t fnz_nz(const Flags *f) { return f->nz; }
// ADC, SBC, ADCS, SBCS with the carry in from F.
#define FL_ADC(N, U) \
CPU_FL U adc##N##f(int cin, U a, U b) { return (U)(a + b + (U)cin); } \
CPU_FL U sbc##N##f(int cin, U a, U b) { return (U)(a - b - 1 + (U)cin); } \
CPU_FL U adcs##N##f(Flags *f, int cin, U a, U b) { \
U r = (U)(a + b + (U)cin); \
int carry = cin ? r <= a : r < a; \
f->nz = MKFLAGS(r >> (N - 1), r == 0, carry, ((a ^ r) & (b ^ r)) >> (N - 1)); \
return r; \
} \
CPU_FL U sbcs##N##f(Flags *f, int cin, U a, U b) { return adcs##N##f(f, cin, a, (U)~b); }
FL_ADC(64, uint64_t)
FL_ADC(32, uint32_t)
#undef CPU_FL
// ---- bit ops
#define ROR64(v, r) ((uint64_t)(v) >> ((r) & 63) | (uint64_t)(v) << ((64 - ((r) & 63)) & 63))
#define ROR32(v, r) ((uint32_t)(v) >> ((r) & 31) | (uint32_t)(v) << ((32 - ((r) & 31)) & 31))
+7 -7
View File
@@ -226,21 +226,21 @@ def build_harness(work, bodies, jobs, cflags):
return exe
def write_cases(path, cases):
"""cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk)."""
def write_cases(path, cases, stubs=False):
"""cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk). stubs: calls go to harness.c's call stub (rt_stub)."""
with open(path, 'wb') as f:
for k, fpcr, x, sp, nzcv, v, memchk in cases:
f.write(struct.pack(IN_FMT, k, fpcr, *(list(x[:31]) + [0]), sp, nzcv))
for q in v:
f.write(q.to_bytes(16, 'little'))
f.write(struct.pack(IN_TAIL, int(memchk), 0))
f.write(struct.pack(IN_TAIL, int(memchk), int(stubs)))
def run_harness(exe, work, buf0, cases, images):
def run_harness(exe, work, buf0, cases, images, stubs=False):
"""Run the lifted tests. Returns (records, stderr tail or None). images: (base, path) pairs."""
with open(work / 'buf.bin', 'wb') as f:
f.write(buf0)
write_cases(work / 'cases.bin', cases)
write_cases(work / 'cases.bin', cases, stubs)
out = work / 'out.bin'
args = [str(exe), str(work / 'buf.bin'), str(work / 'cases.bin'), str(out)]
for base, path in images:
@@ -258,14 +258,14 @@ def run_harness(exe, work, buf0, cases, images):
return recs, err
def diff_state(got, want, check_pc, check_mem):
def diff_state(got, want, check_pc, check_mem, check_nzcv=True):
"""got: harness record; want: (x, sp, nzcv, pc, v, mem). Returns a list of mismatches."""
hx, hsp, hnz, hpc, hmem, hv = got
ex, esp, enz, epc, ev, emem = want
d = [f'x{j}: got {hx[j]:x} want {ex[j]:x}' for j in range(31) if hx[j] != ex[j]]
if hsp != esp:
d.append(f'sp: got {hsp:x} want {esp:x}')
if hnz != enz:
if check_nzcv and hnz != enz:
d.append(f'nzcv: got {hnz:x} want {enz:x}')
if check_pc and hpc != epc:
d.append(f'pc: got {hpc:x} want {epc:x}')
+124 -30
View File
@@ -1,14 +1,19 @@
"""Differential test of whole emitted functions: recomp/gen.py's C for a guest function against Unicorn running the
same function from its entry until it returns.
Picks leaf functions from the ExeFS modules: no calls, no branches out of the function, no SVC, BRK, system
register, exclusive or atomic instructions; with conditional branches, and loops, jump tables and SIMD where the
module has them. Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each
from random states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into
itself and random data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when
PC reaches the sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are
mapped read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, NZCV, V0-V31
and the scratch buffer.
Picks functions from the ExeFS modules: no branches out of the function, no SVC, BRK, system register, exclusive or
atomic instructions; with conditional branches, and loops, jump tables, SIMD and calls where the module has them.
Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each from random
states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into itself and random
data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when PC reaches the
sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are mapped
read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, V0-V31 and the
scratch buffer; NZCV is local to a generated function (recomp/flags.py).
Every BL and BLR goes to one stub on both sides (rt_stub in harness.c, Runner.stub here) instead of its callee. The
stub folds the guest state at the call into a digest and sets the caller-saved registers from it, so the test sees
whether the emitted code had every register in the Cpu at the call and reads them back after it. Callers of another
picked function are dropped, as the C side would run that function instead of the stub.
Usage: func.py [--exefs DIR] [--modules main,subsdk0,sdk] [--functions N] [--cases K] [--seed S] [--jobs J]
"""
@@ -38,11 +43,24 @@ STACK_TOP = BUF + 0x3F000
MIN_INSNS, MAX_INSNS = 6, 600
FNV = 0x100000001B3
M64 = (1 << 64) - 1
def splitmix(z):
z = (z + 0x9E3779B97F4A7C15) & M64
z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & M64
z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M64
return z ^ (z >> 31)
def call_mask(w):
return ((w & 0xFC000000) == 0x94000000) | ((w & 0xFFFFFC1F) == 0xD63F0000) # BL, BLR
def forbidden_mask(w):
"""Words a picked function must not contain."""
m = (w & 0xFC000000) == 0x94000000 # BL
m |= (w & 0xFFFFFC1F) == 0xD63F0000 # BLR
m |= (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS
m = (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS
sysi = (w & 0xFFC00000) == 0xD5000000 # system: MRS, MSR, SYS (DC ZVA), hints, barriers
hint_or_barrier = ((w & 0xFFFFF000) == 0xD5032000) | ((w & 0xFFFFF000) == 0xD5033000)
m |= sysi & ~hint_or_barrier
@@ -61,7 +79,7 @@ def simd_mask(w):
def pick_functions(name, mod, count, r):
"""[(entry, end, tags)] of up to count leaf functions of mod, plus the Emitter that will emit them."""
"""[(entry, end, tags)] of up to count functions of mod, plus the Emitter that will emit them."""
starts, _ = gen.discover(mod)
w = mod.words
ends = np.append(starts[1:], len(w))
@@ -73,15 +91,18 @@ def pick_functions(name, mod, count, r):
size = ends - starts
ok = (per_fn(forbidden_mask(w)) == 0) & (per_fn(cond_mask(w)) > 0) & (size >= MIN_INSNS) & (size <= MAX_INSNS)
has_br = per_fn((w & 0xFFFFFC1F) == 0xD61F0000) > 0
has_call = per_fn(call_mask(w)) > 0
cand = np.flatnonzero(ok)
with_br = [int(i) for i in cand if has_br[i]]
rest = [int(i) for i in cand if not has_br[i]]
r.shuffle(with_br)
r.shuffle(rest)
with_call = [int(i) for i in cand if has_call[i] and not has_br[i]]
rest = [int(i) for i in cand if not has_br[i] and not has_call[i]]
for pool in (with_br, with_call, rest):
r.shuffle(pool)
emitter = gen.Emitter(mod, set((mod.lo + starts * 4).tolist()))
picks, tags_seen = [], Counter()
# Functions with a br first (jump tables, fallbacks), at most a quarter of the picks.
for pool, quota in ((with_br, count // 4), (rest, count)):
# Functions with a br first (jump tables, fallbacks), at most a quarter of the picks; then functions with calls,
# at most half.
for pool, quota in ((with_br, count // 4), (with_call, count // 2), (rest, count)):
taken = 0
for i in pool:
if len(picks) >= count or taken >= quota:
@@ -91,13 +112,28 @@ def pick_functions(name, mod, count, r):
if tags is None:
continue
picks.append((entry, end, tags))
tags_seen.update(tags)
taken += 1
print(f'{name}: {len(starts)} functions, {len(cand)} leaf candidates, picked {len(picks)}: '
picked = {e for e, _, _ in picks}
picks = [p for p in picks if not callees(emitter, p[0], p[1]) & picked]
for _, _, tags in picks:
tags_seen.update(tags)
print(f'{name}: {len(starts)} functions, {len(cand)} candidates, picked {len(picks)}: '
f'{dict(tags_seen)}', flush=True)
return picks, emitter
def call_sites(mod, entry, end):
"""Addresses of the BL and BLR instructions of [entry, end)."""
return [a for a in range(entry, end, 4) if mod.lift(a).kind in ('bl', 'blr')]
def callees(emitter, entry, end):
"""Function starts the emitted C of [entry, end) calls directly (`f_<addr>(c)`, not through rt_dispatch)."""
mod = emitter.mod
return {mod.lift(a).target for a in call_sites(mod, entry, end)
if mod.lift(a).kind == 'bl' and mod.lift(a).target in emitter.starts}
def check_function(emitter, entry, end):
"""Tags of a function the test can run in isolation, or None."""
mod = emitter.mod
@@ -108,7 +144,7 @@ def check_function(emitter, entry, end):
for k, ins in enumerate(insns):
a = entry + 4 * k
if ins.kind in ('bl', 'blr'):
return None
tags.add('calls')
if ins.kind in ('b', 'bcond'):
if not entry <= ins.target < end:
return None
@@ -147,9 +183,14 @@ def build(work, emitted, jobs, cflags):
path.write_text(gen.chunk_text(name, j // per, emitter, fns[j:j + per]))
srcs.append(path)
table = work / 'table.c'
called = set()
for _, entry, end, emitter in emitted:
called |= callees(emitter, entry, end)
table.write_text('#include "cpu.h"\n' + ''.join(f'void {gen.fname(e)}(Cpu *c);\n' for _, e, _, _ in emitted) +
'GuestFn tests[] = {' + ','.join(gen.fname(e) for _, e, _, _ in emitted) + '};\n' +
f'int ntests = {len(emitted)};\n')
f'int ntests = {len(emitted)};\n' + 'void rt_stub_call(Cpu *c, uint64_t target);\n' +
''.join(f'void {gen.fname(t)}(Cpu *c) {{ rt_stub_call(c, {hex(t)}ull); }}\n'
for t in sorted(called)))
srcs += [table, ROOT / 'tests' / 'diff' / 'harness.c']
objs, procs = [], []
for s in srcs:
@@ -191,9 +232,12 @@ def make_case(r):
class Runner:
"""Unicorn with the module images read-only and a block hook that stops when execution leaves the function."""
"""Unicorn with the module images read-only, a block hook that stops when execution leaves the function, and a
code hook on every call site of the picked functions that stops before the call, for the stub."""
def __init__(self, images, buf0):
MAX_CALLS = 1000
def __init__(self, images, buf0, sites):
self.emu = Emu(images)
mu = self.emu.mu
for base, data in images:
@@ -201,25 +245,70 @@ class Runner:
mu.mem_map(SENTINEL, 0x1000)
mu.mem_write(BUF, buf0)
self.buf0 = buf0
self.lo = self.hi = 0
self.lo = self.hi = self.h = 0
self.left = False
self.at_call = None
mu.hook_add(uc.UC_HOOK_BLOCK, self._block)
# One hook per call site, all added before anything is translated, so no cached block lacks one.
for a in sorted(sites):
mu.hook_add(uc.UC_HOOK_CODE, self._call, begin=a, end=a)
def _block(self, mu, address, size, user):
if not self.lo <= address < self.hi and address != SENTINEL:
self.left = True
mu.emu_stop()
def _call(self, mu, address, size, user):
if self.lo <= address < self.hi:
self.at_call = address
mu.emu_stop()
def stub(self, a):
"""harness.c rt_stub for the call at a, which Unicorn has not executed yet. Returns the address to resume at."""
e, mu = self.emu, self.emu.mu
w = int.from_bytes(mu.mem_read(a, 4), 'little')
if (w & 0xFC000000) == 0x94000000: # BL
imm = w & 0x03FFFFFF
target = (a + ((imm - (1 << 26) if imm >> 25 else imm) << 2)) & M64
else: # BLR
target = mu.reg_read(e.xregs[(w >> 5) & 31])
mu.reg_write(A.UC_ARM64_REG_X30, a + 4)
h = ((self.h ^ target) * FNV) & M64
for val in list(mu.reg_read_batch(e.xregs)) + [mu.reg_read(A.UC_ARM64_REG_SP)]:
h = ((h ^ val) * FNV) & M64
for val in mu.reg_read_batch(e.qregs[:8]):
h = ((h ^ (val & M64)) * FNV) & M64
h = ((h ^ (val >> 64)) * FNV) & M64
self.h = h
regs = []
for i in range(19):
s = splitmix((h + i) & M64)
regs.append((e.xregs[i], PTR_LO + ((s >> 4) % ((PTR_HI - PTR_LO) // 16)) * 16 if i < 8 and s & 3 else s))
regs.append((A.UC_ARM64_REG_NZCV, splitmix((h + 19) & M64) >> 60 << 28))
for i in range(8):
regs.append((e.qregs[i], splitmix((h + 33 + 2 * i) & M64) << 64 | splitmix((h + 32 + 2 * i) & M64)))
mu.reg_write_batch(regs)
top = BUF + BUF_SIZE - 16
n = int.from_bytes(mu.mem_read(top, 8), 'little')
mu.mem_write(top, ((n + 1) & M64).to_bytes(8, 'little') + h.to_bytes(8, 'little'))
return a + 4
def run(self, entry, end, case, steps):
"""(x, sp, nzcv, pc, v, crc of the buffer) or None if the case is skipped."""
fpcr, x, sp, nzcv, v = case
e, mu = self.emu, self.emu.mu
self.lo, self.hi, self.left = entry, end, False
self.lo, self.hi, self.left, self.h = entry, end, False, 0
mu.reg_write_batch(list(zip(e.xregs, x[:31])) + list(zip(e.qregs, v)) +
[(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)])
pc, ok = entry, False
try:
mu.emu_start(entry, SENTINEL, count=steps)
ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL
for _ in range(self.MAX_CALLS):
self.at_call = None
mu.emu_start(pc, SENTINEL, count=steps)
if self.at_call is None:
ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL
break
pc = self.stub(self.at_call)
except uc.UcError:
ok = False
if not ok:
@@ -268,7 +357,10 @@ def main():
print(f'compiled ({time.time() - t1:.0f}s)', flush=True)
buf0 = make_buffer(r)
runner = Runner([(base, data) for base, _, data in images], buf0)
sites = set()
for _, entry, end, emitter in emitted:
sites.update(call_sites(emitter.mod, entry, end))
runner = Runner([(base, data) for base, _, data in images], buf0, sites)
cases, expect, tested, skipped = [], [], set(), 0
t2 = time.time()
for k, (name, entry, end, _) in enumerate(emitted):
@@ -286,7 +378,7 @@ def main():
flush=True)
t3 = time.time()
recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images])
recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images], stubs=True)
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
if err:
k = cases[len(recs)][0] if len(recs) < len(cases) else None
@@ -295,7 +387,9 @@ def main():
fails, nfail = {}, 0
for case, want_state, got in zip(cases, expect, recs):
d = diff_state(got, want_state, check_pc=False, check_mem=True)
# Generated functions keep NZCV in a local (recomp/flags.py) and leave c->nzcv as it was: flags only count
# where the function reads them, which the registers and memory show.
d = diff_state(got, want_state, check_pc=False, check_mem=True, check_nzcv=False)
if d:
nfail += 1
fails.setdefault(case[0], d[:4])
+48 -4
View File
@@ -1,5 +1,6 @@
// Runs lifted single-instruction tests and writes the resulting CPU state per case.
// Shared by tests/diff/lift.py and tests/diff/simd.py; the expected state comes from Unicorn on the Python side.
// Runs lifted tests and writes the resulting CPU state per case.
// Shared by tests/diff/lift.py, simd.py, atomic.py and func.py; the expected state comes from Unicorn on the Python
// side.
//
// argv: buf.bin cases.bin out.bin image base [image base ...]
// Each image is mapped at its guest address (guest == host), the scratch buffer at BUF.
@@ -19,7 +20,48 @@ extern int ntests;
#define BUF 0x7200000000ull
#define BUF_SIZE 0x40000
void rt_dispatch(Cpu *c) { (void)c; }
#define PTR_LO (BUF + 0x8000)
#define PTR_SPAN 0x20000
static int g_stubs; // In.stubs of the running case
static uint64_t g_stub_h; // digest of every call's state so far
static uint64_t splitmix(uint64_t z) {
z += 0x9E3779B97F4A7C15ull;
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
return z ^ (z >> 31);
}
static uint64_t mix(uint64_t h, uint64_t w) { return (h ^ w) * 0x100000001B3ull; }
// The callee of every call in a function under tests/diff/func.py; Runner.stub there is the same function on the
// Unicorn side. Folds the target, X0-X30, SP and V0-V7 into a digest, so a register the caller left unwritten at the
// call changes it, then sets X0-X18, NZCV and V0-V7 from the digest, so a caller reading a register it held across
// the call sees a value the callee could have left. X0-X7 mostly get pointers into the scratch buffer. The digest
// and the call count go to the top of the scratch buffer, which the memory check covers.
static void rt_stub(Cpu *c, uint64_t target) {
uint64_t h = mix(g_stub_h, target);
for (int i = 0; i < 31; i++) h = mix(h, c->x[i]);
h = mix(h, c->sp);
for (int i = 0; i < 8; i++) h = mix(mix(h, (uint64_t)c->v[i]), (uint64_t)(c->v[i] >> 64));
g_stub_h = h;
for (int i = 0; i < 19; i++) {
uint64_t s = splitmix(h + i);
c->x[i] = i < 8 && (s & 3) ? PTR_LO + ((s >> 4) % (PTR_SPAN / 16)) * 16 : s;
}
c->nzcv = splitmix(h + 19) >> 60 << 28;
for (int i = 0; i < 8; i++)
c->v[i] = (__uint128_t)splitmix(h + 33 + 2 * i) << 64 | splitmix(h + 32 + 2 * i);
uint64_t *slot = (uint64_t *)(uintptr_t)(BUF + BUF_SIZE - 16);
slot[0]++;
slot[1] = h;
}
void rt_dispatch(Cpu *c) {
if (g_stubs) rt_stub(c, c->pc);
}
void rt_stub_call(Cpu *c, uint64_t target) { rt_stub(c, target); }
void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; }
void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; abort(); }
void rt_udf(Cpu *c, uint64_t pc) { (void)c; (void)pc; abort(); }
@@ -61,7 +103,7 @@ static uint32_t crc32_buf(const uint8_t *p, size_t n) {
}
// Layouts match IN_FMT / OUT_FMT in tests/diff/common.py.
struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, pad; };
struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, stubs; };
struct Out { uint64_t x[32], sp, nzcv, pc, mem; __uint128_t v[32]; };
int main(int argc, char **argv) {
@@ -96,6 +138,8 @@ int main(int argc, char **argv) {
c.fpcr = in.fpcr;
memcpy(c.v, in.v, sizeof c.v);
c.excl_addr = ~0ull;
g_stubs = (int)in.stubs;
g_stub_h = 0;
tests[in.k](&c);
memcpy(out.x, c.x, sizeof out.x);
out.sp = c.sp;
+111
View File
@@ -0,0 +1,111 @@
// The function-local flag helpers of runtime/cpu.h (Flags, fc_*, fnz_*, nz_*, adc*f) against the packed helpers
// (subs64() and friends, COND_xx), which tests/diff/lift.py checks against Unicorn. Built and run by
// tests/flags_check.py; prints the first mismatches and exits 1 on any.
#include <stdio.h>
#include "cpu.h"
static const uint64_t EDGE[] = {0, 1, 2, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000,
0x80000001, 0xfffffffe, 0xffffffff, 0x100000000ull, 0x7fffffffffffffffull,
0x8000000000000000ull, 0x8000000000000001ull, 0xfffffffffffffffeull,
0xffffffffffffffffull, 0x123456789abcdef0ull, 0xfedcba9876543210ull};
#define NEDGE (sizeof EDGE / sizeof EDGE[0])
static uint64_t rng = 0x9E3779B97F4A7C15ull;
static uint64_t next(void) {
rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17;
return rng;
}
static int fails;
static void bad(const char *what, int cc, uint64_t a, uint64_t b, uint64_t got, uint64_t want) {
if (fails++ < 20)
printf("%s cc %d a %llx b %llx: got %llx want %llx\n", what, cc, (unsigned long long)a, (unsigned long long)b,
(unsigned long long)got, (unsigned long long)want);
}
static int cond(const Cpu *c, int cc) {
switch (cc) {
case CC_EQ: return COND_EQ(c);
case CC_NE: return COND_NE(c);
case CC_CS: return COND_CS(c);
case CC_CC: return COND_CC(c);
case CC_MI: return COND_MI(c);
case CC_PL: return COND_PL(c);
case CC_VS: return COND_VS(c);
case CC_VC: return COND_VC(c);
case CC_HI: return COND_HI(c);
case CC_LS: return COND_LS(c);
case CC_GE: return COND_GE(c);
case CC_LT: return COND_LT(c);
case CC_GT: return COND_GT(c);
default: return COND_LE(c);
}
}
// The condition argument is not constant here, so this also covers the switch default paths.
#define CHECK(NAME, OLD, NEW, FC, FNZ) \
do { \
Cpu c = {0}; \
Flags f = {0}; \
uint64_t ro = OLD, rn = NEW; \
if (ro != rn) bad(NAME " result", -1, a, b, rn, ro); \
for (int cc = 0; cc < 14; cc++) \
if (!FC(&f, cc) != !cond(&c, cc)) bad(NAME, cc, a, b, FC(&f, cc), cond(&c, cc)); \
f.nz = 0; \
if (FNZ(&f) != c.nzcv) bad(NAME " packed", -1, a, b, FNZ(&f), c.nzcv); \
Flags g = {.nz = c.nzcv}; \
for (int cc = 0; cc < 14; cc++) \
if (!fc_nz(&g, cc) != !cond(&c, cc)) bad(NAME " nz", cc, a, b, fc_nz(&g, cc), cond(&c, cc)); \
} while (0)
static void one(uint64_t a, uint64_t b) {
uint32_t a32 = (uint32_t)a, b32 = (uint32_t)b;
CHECK("sub64", subs64(&c, a, b), fsub64(&f, a, b), fc_sub64, fnz_sub64);
CHECK("sub32", subs32(&c, a32, b32), fsub32(&f, a32, b32), fc_sub32, fnz_sub32);
CHECK("add64", adds64(&c, a, b), fadd64(&f, a, b), fc_add64, fnz_add64);
CHECK("add32", adds32(&c, a32, b32), fadd32(&f, a32, b32), fc_add32, fnz_add32);
CHECK("logic64", logic64(&c, a & b), flogic64(&f, a & b), fc_logic64, fnz_logic64);
CHECK("logic32", logic32(&c, a32 & b32), flogic32(&f, a32 & b32), fc_logic32, fnz_logic32);
{
Cpu c = {0};
if (nz_sub64(a, b) != (subs64(&c, a, b), c.nzcv)) bad("nz_sub64", -1, a, b, nz_sub64(a, b), c.nzcv);
if (nz_sub32(a32, b32) != (subs32(&c, a32, b32), c.nzcv)) bad("nz_sub32", -1, a, b, nz_sub32(a32, b32), c.nzcv);
if (nz_add64(a, b) != (adds64(&c, a, b), c.nzcv)) bad("nz_add64", -1, a, b, nz_add64(a, b), c.nzcv);
if (nz_add32(a32, b32) != (adds32(&c, a32, b32), c.nzcv)) bad("nz_add32", -1, a, b, nz_add32(a32, b32), c.nzcv);
}
for (int cin = 0; cin < 2; cin++) {
Cpu c = {.nzcv = (uint64_t)cin << 29};
Flags f = {0};
if (adc64f(cin, a, b) != adc64(&c, a, b)) bad("adc64", cin, a, b, adc64f(cin, a, b), adc64(&c, a, b));
if (sbc64f(cin, a, b) != sbc64(&c, a, b)) bad("sbc64", cin, a, b, sbc64f(cin, a, b), sbc64(&c, a, b));
if (adc32f(cin, a32, b32) != adc32(&c, a32, b32)) bad("adc32", cin, a, b, adc32f(cin, a32, b32), adc32(&c, a32, b32));
if (sbc32f(cin, a32, b32) != sbc32(&c, a32, b32)) bad("sbc32", cin, a, b, sbc32f(cin, a32, b32), sbc32(&c, a32, b32));
uint64_t r;
#define CARRY(NAME, NEW, OLD) \
c.nzcv = (uint64_t)cin << 29; \
r = NEW; \
if (r != OLD || f.nz != c.nzcv) bad(NAME, cin, a, b, f.nz, c.nzcv);
CARRY("adcs64", adcs64f(&f, cin, a, b), adcs64(&c, a, b))
CARRY("sbcs64", sbcs64f(&f, cin, a, b), sbcs64(&c, a, b))
CARRY("adcs32", adcs32f(&f, cin, a32, b32), adcs32(&c, a32, b32))
CARRY("sbcs32", sbcs32f(&f, cin, a32, b32), sbcs32(&c, a32, b32))
}
}
int main(void) {
long n = 0;
for (size_t i = 0; i < NEDGE; i++)
for (size_t j = 0; j < NEDGE; j++, n++) one(EDGE[i], EDGE[j]);
for (int k = 0; k < 2000000; k++, n++) {
uint64_t a = next(), b = next();
switch (k & 3) {
case 1: b = a; break; // equal operands
case 2: b = a + (next() & 3) - 1; break; // neighbours
case 3: a &= 0xff, b &= 0xff; break; // small values
}
one(a, b);
}
printf("%ld operand pairs, %d mismatches\n", n, fails);
return fails != 0;
}
+32
View File
@@ -0,0 +1,32 @@
"""Build and run tests/flags_check.c: runtime/cpu.h's function-local flag helpers against its packed flag helpers.
Usage: flags_check.py
"""
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests' / 'diff'))
from common import MSYS_BIN, compiler # noqa: E402
def main():
work = ROOT / 'build' / 'test' / 'flags'
work.mkdir(parents=True, exist_ok=True)
exe = work / ('flags_check.exe' if os.name == 'nt' else 'flags_check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
for opt in ('-O0', '-O1'):
subprocess.check_call([compiler(), opt, '-fno-strict-aliasing', '-I', str(ROOT / 'runtime'),
str(ROOT / 'tests' / 'flags_check.c'), '-o', str(exe)], env=env)
print(opt, end=': ', flush=True)
if subprocess.run([str(exe)], env=env).returncode:
return 1
return 0
if __name__ == '__main__':
sys.exit(main())