diff --git a/docs/architecture.md b/docs/architecture.md index 87a3d45b..1c7009c6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,7 +36,10 @@ MacNativeGames cannot run on x86-64 as is: its CPU path executes ARM64 code nati - jump tables resolved by our own symbolic pass instead of Unicorn, which keeps GPL code out of the shipped generator; - `LDXP`/`STXP` (128-bit exclusive pairs) supported; - no 64 MB limit on `main`; `subsdk*` loaded; - - guest registers kept in C locals where liveness allows, instead of memory. + - NZCV and the six most used general registers of each function kept in C locals instead of the `Cpu` + (`recomp/flags.py`, `recomp/regs.py`): registers are stored back where they changed, before calls, SVCs and + exits, and reloaded after them. A call reloads only X0-X18 and X30, as AAPCS64 has the callee preserve X19-X29 and + SP; `gen.py --no-callee-saved-trust` reloads those too. - Coverage oracle: MacNativeGames' NSO rewriter lists every site where game code touches the OS (SVCs, TLS reads, counter reads, `x18`, exclusive windows, jump tables): 68,968 sites in 1.4.3 `main`. The lifter must handle every one. - Memory: guest address equals host address, all below 2^39 (IPC encodes 39-bit addresses). The heap, stacks and the alias region are reserved with `MEM_WRITE_WATCH`, so the graphics back end can ask which pages the CPU wrote since the last upload (`runtime/memory.c`). - Indirect branches go through `rt_dispatch` (`runtime/dispatch.c`), which looks the target up in the address table and tail-calls it. diff --git a/docs/building.md b/docs/building.md index 4229ee24..757f60b7 100644 --- a/docs/building.md +++ b/docs/building.md @@ -138,7 +138,8 @@ change passes these, all run with `.venv`: | `tests\diff\lift.py` | instructions sampled from the extracted modules; `--coverage` lists every word the lifter rejects or traps | yes | | `tests\diff\simd.py` | the whole FP/SIMD encoding space | no | | `tests\diff\atomic.py` | exclusive load/store sequences and ARMv8.1 atomics | no | -| `tests\diff\func.py` | whole leaf functions as the generator emits them, loops and jump tables included | yes | +| `tests\diff\func.py` | whole functions as the generator emits them, loops, jump tables and calls included (calls go to a stub that checks the registers at the call) | yes | +| `tests\flags_check.py` | the function-local flag helpers of `runtime\cpu.h` against its packed ones | no | | `tests\excl_mt.py` | the exclusive monitor helpers under contention and ABA, on several threads | no | | `tests\jumptables_check.py` | `recomp/jumptables.py` against a Unicorn oracle; must report 0 disagreements, on 1.4.3 too | yes | diff --git a/recomp/flags.py b/recomp/flags.py new file mode 100644 index 00000000..61188f9d --- /dev/null +++ b/recomp/flags.py @@ -0,0 +1,144 @@ +"""Function-local NZCV for recomp/gen.py. + +The lifter writes flags the way the guest does: every flag-setting instruction packs N, Z, C and V into c->nzcv +(subs64() and friends in runtime/cpu.h) and every reader unpacks them (COND_xx(c)). Compilers keep NZCV inside a +function, and AAPCS64 neither passes it to a callee nor preserves it across a call. So a generated function keeps its +flags in a local `Flags F` (runtime/cpu.h), which clang holds in registers, and writes c->nzcv only where it leaves +for another function other than by a call or a return (see plan()). + +A flag-setting instruction of a known kind (subtract, add or logic result; 32 or 64 bits) stores its operands in F.a +and F.b. A reader all of whose reaching definitions are of one kind computes its condition from those operands, so +`cmp x1, x2; b.lt` becomes a signed compare in C. Instructions whose result is only known packed (CCMP, FCMP, MSR +NZCV, ADCS) write F.nz, the packed form; a reader whose definitions are of several kinds reads F.nz, and those +definitions then also write it. + +A read that can see flags a callee left reads the caller's flags from before the call instead. AAPCS64 code never +reads a callee's flags, so such a read only sits on a path that does not run: in TotK it follows calls to noreturn +functions (asserts) that the compiler placed before another branch's target. Plan.after_call counts them. +""" +import re + +KIND_DEF = re.compile(r'\b(subs|adds|logic)(32|64)\(c, ') +CARRY_DEF = re.compile(r'\b(adcs|sbcs)(32|64)\(c, ') +CARRY_USE = re.compile(r'\b(adc|sbc)(32|64)\(c, ') +COND = re.compile(r'COND_([A-Z]{2})\(c\)') +NZ_DEF = 'c->nzcv = ' +NZ_USE = '(c->nzcv)' +KIND_NAME = {'subs': 'sub', 'adds': 'add', 'logic': 'logic'} + +ENTRY, CALL = 'entry', 'call' + + +class FlagError(Exception): + pass + + +def effect(code): + """(reads flags, kind of the flags written or None) of one instruction's C.""" + reads = bool(COND.search(code) or CARRY_USE.search(code) or CARRY_DEF.search(code) or NZ_USE in code) + if NZ_DEF in code or CARRY_DEF.search(code): + return reads, 'nz' + defs = KIND_DEF.findall(code) + if len(defs) > 1: + raise FlagError(f'several flag definitions in one instruction: {code}') + if defs: + op, size = defs[0] + return reads, KIND_NAME[op] + size + return reads, None + + +class Plan: + """Per instruction: the kind its flag reads use, whether its flag definition also writes F.nz, and what it writes + to c->nzcv before it leaves for another function.""" + + def __init__(self, n): + self.reader = [None] * n + self.pack = [False] * n + self.exit = [None] * n # kind of the flags to publish in c->nzcv at a tail exit, or None + self.entry = False # F.nz starts as c->nzcv: flags from the entry reach a read or an exit + self.entry_read = False # they reach a read, not only an exit + self.after_call = 0 # reads that can see flags a callee left (see the module docstring) + self.used = False + + def decl(self): + """Declaration of F at the top of the function.""" + return 'Flags F = {.nz = c->nzcv};' if self.entry else 'Flags F = {0};' + + def publish(self, i): + """Statement writing the flags to c->nzcv at instruction i's tail exit ('' when it need not).""" + k = self.exit[i] + return f'c->nzcv = fnz_{k}(&F); ' if k else '' + + def rewrite(self, i, code): + """code of instruction i with its flag reads and writes on F.""" + rk = self.reader[i] + _, dk = effect(code) + if rk: + code = COND.sub(lambda m: f'fc_{rk}(&F, CC_{m.group(1)})', code) + code = CARRY_DEF.sub(lambda m: f'{m.group(1)}{m.group(2)}f(&F, fc_{rk}(&F, CC_CS), ', code) + code = CARRY_USE.sub(lambda m: f'{m.group(1)}{m.group(2)}f(fc_{rk}(&F, CC_CS), ', code) + code = code.replace(NZ_USE, f'(fnz_{rk}(&F))') + if dk == 'nz': + code = code.replace(NZ_DEF, 'F.nz = ') + code = re.sub(r'\(void\)(subs|adds)(32|64)\(c, ', lambda m: f'F.nz = nz_{KIND_NAME[m.group(1)]}{m.group(2)}(', + code) + elif dk: + code = KIND_DEF.sub(lambda m: f'f{KIND_NAME[m.group(1)]}{m.group(2)}(&F, ', code) + if self.pack[i]: + code += f' F.nz = fnz_{dk}(&F);' + if 'c->nzcv' in code or COND.search(code) or KIND_DEF.search(code) or CARRY_USE.search(code): + raise FlagError(f'flag access left after the rewrite: {code}') + return code + + +def plan(effects, succ, calls, exits): + """effects[i]: (reads, kind or None); succ[i]: successor indices inside the function; calls: indices of calls, + whose callee may leave any flags; exits: indices that can leave for another function other than by a call or a + return (a tail branch, falling into the next function, a br through rt_dispatch). + + Discovery can start a function inside another one (a branch target, a materialised address), and the code + before it then falls or branches into it with live flags. So a tail exit publishes the flags in c->nzcv, and a + function whose reads can see its entry flags starts F.nz from c->nzcv. Calls and returns publish nothing: AAPCS64 + code neither passes flags to a callee nor reads a callee's, so a call's flags count as no definition.""" + n = len(effects) + p = Plan(n) + if not any(r or k for r, k in effects): + return p + p.used = True + reach = [set() for _ in range(n)] + reach[0].add(ENTRY) + work = [0] + while work: + i = work.pop() + out = {i} if effects[i][1] else ({CALL} if i in calls else reach[i]) + for s in succ[i]: + if not out <= reach[s]: + reach[s] |= out + work.append(s) + + def kind(d): + return 'nz' if d == ENTRY else effects[d][1] + + def use(r): + """Kind a use of the definitions r reads; several kinds read the packed form, which they then all write.""" + kinds = {kind(d) for d in r} + if ENTRY in r: + p.entry = True + if len(kinds) == 1: + return kinds.pop() + for d in r: + if d != ENTRY: + p.pack[d] = True + return 'nz' # several kinds, or none (unreachable) + + for i, (reads, _) in enumerate(effects): + if reads: + p.after_call += CALL in reach[i] + p.reader[i] = use(reach[i] - {CALL}) + p.entry_read |= ENTRY in reach[i] + for i in exits: + # Entry flags alone are still in c->nzcv. + r = reach[i] - {CALL} + if r and r != {ENTRY}: + p.exit[i] = use(r) + return p diff --git a/recomp/gen.py b/recomp/gen.py index a1099c52..69ffbfc3 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -38,8 +38,10 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tools')) sys.path.insert(0, str(ROOT / 'recomp')) +import flags # noqa: E402 import jumptables # noqa: E402 import patches # noqa: E402 +import regs # noqa: E402 from check_content import GENERATED_MARKER # noqa: E402 from layout import layout, write as write_layout # noqa: E402 from lifter import hx # noqa: E402 @@ -205,10 +207,12 @@ def partition(est, max_fns, max_bytes, big_fn): class Emitter: """C for the functions of one module. starts is the set of every function start of the module.""" - def __init__(self, mod, starts, patched=None): + def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True): self.mod = mod self.starts = starts self.patched = patched or {} # {address: (set index, [replacement words])} (recomp/patches.py) + self.reg_locals = reg_locals # recomp/regs.py: registers per function in C locals + self.kept = regs.CALLEE_SAVED if callee_saved else frozenset() # not reloaded after a call self.uses_patches = False self.stats = Counter() self.unsupported = Counter() @@ -246,6 +250,70 @@ class Emitter: raise ValueError(f'{entry + 4 * i:#x}: asm instruction from the portable lifter') return labels, tables + def replacements(self, a): + """Lifted replacement words of the patched site a.""" + return [self.mod.lifter.lift(w, a) for w in self.patched[a][1]] + + @staticmethod + def flow(insns, entry, end, tables): + """Successors of each instruction inside the function [entry, end). Index len(insns) stands for falling + through into the next function and has none.""" + def idx(t): + return [(t - entry) // 4] if entry <= t < end else [] + + succ = [] + for i, ins in enumerate(insns): + k = ins.kind + if k in ('op', 'bl', 'blr'): + succ.append([i + 1]) + elif k == 'b': + succ.append(idx(ins.target)) + elif k == 'bcond': + succ.append(idx(ins.target) + [i + 1]) + elif k == 'br': + jt = tables.get(entry + 4 * i) + succ.append([t for x in jt for t in idx(x)] if jt and jt != 'all' else []) + else: # ret, trap + succ.append([]) + succ.append([]) + return succ + + def flag_plan(self, insns, entry, end, tables, succ): + """recomp/flags.py plan for the function: each instruction's flag use over the flow succ (flow()).""" + n = len(insns) + calls, effects, exits = set(), [], {n} + for i, ins in enumerate(insns): + k, a = ins.kind, entry + 4 * i + if k in ('bl', 'blr'): + calls.add(i) + elif k in ('b', 'bcond') and not entry <= ins.target < end: + exits.add(i) + elif k == 'br': + exits.add(i) # through rt_dispatch, also a table's default + if a in self.patched: + effects.append(self.patched_effect(ins, a)) + if any(r.kind == 'bl' for r in self.replacements(a)): + calls.add(i) + else: + effects.append(flags.effect(ins.cond if k == 'bcond' else ins.code or '')) + effects.append((False, None)) + return flags.plan(effects, succ, calls, exits) + + def patched_effect(self, ins, a): + """Flag effect of a patched site: both alternatives must have the same one, in at most one word each.""" + alts = [] + for words in ([ins], self.replacements(a)): + fx = [flags.effect(w.code or '') for w in words if w.kind == 'op'] + fx = [e for e in fx if e[0] or e[1]] + if len(fx) > 1: + raise patches.PatchError(f'{a:#x}: several flag-using words in one alternative') + alts.append(fx[0] if fx else (False, None)) + if alts[0][1] != alts[1][1]: + raise patches.PatchError(f'{a:#x}: the alternatives write different flags ({alts[0][1]}, {alts[1][1]})') + if (alts[0][1] or alts[0][0] or alts[1][0]) and ins.kind == 'bl': + raise patches.PatchError(f'{a:#x}: a patched call that uses flags') + return alts[0][0] or alts[1][0], alts[0][1] + def function(self, entry, end): """(lines of C, referenced function starts) for the guest function [entry, end).""" mod, starts = self.mod, self.starts @@ -255,15 +323,36 @@ class Emitter: all_labels = 'all' in tables.values() if all_labels: labels = set(range(entry, end, 4)) + # An unresolved computed br re-enters the function through fb_, a C function of its own, which local + # flags would not survive; such functions keep NZCV in c->nzcv. + # The same holds for registers in locals (recomp/regs.py): fb_ keeps them in the Cpu too. + local = not all_labels + spill, reload, reload_call = (regs.SPILL, regs.RELOAD, regs.RELOAD_CALL) if local else ('', '', '') + succ = self.flow(insns, entry, end, tables) if local else None + fp = self.flag_plan(insns, entry, end, tables, succ) if local else None + if fp and fp.used: + self.stats['local_flag_functions'] += 1 + self.stats['flag_reads_direct'] += sum(1 for k in fp.reader if k and k != 'nz') + self.stats['flag_reads_packed'] += sum(1 for k in fp.reader if k == 'nz') + self.stats['flag_entry_functions'] += fp.entry + self.stats['flag_entry_reads'] += fp.entry_read + self.stats['flag_reads_after_call'] += fp.after_call + self.stats['flag_publishing_exits'] += sum(1 for k in fp.exit if k) - def jump(t): + def fx(i, code): + return fp.rewrite(i, code) if fp and fp.used else code + + def publish(i): + return fp.publish(i) if fp and fp.used else '' + + def jump(t, i): if entry <= t < end: return f'goto L_{t:x};' if t in starts: refs.add(t) - return f'{MUSTTAIL} {fname(t)}(c);' + return f'{publish(i)}{spill}{MUSTTAIL} {fname(t)}(c);' self.stats['external_branches'] += 1 - return f'c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);' + return f'{publish(i)}{spill}c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);' if all_labels: # An unresolved computed br re-enters the function through fb_, which switches on c->pc to any @@ -280,69 +369,98 @@ class Emitter: else: redispatch = f'{MUSTTAIL} rt_dispatch(c);' out = [f'void {fname(entry)}(Cpu *c) {{', f' FN_ENTER({hx(entry)});'] + if fp.used: + out.append(' ' + fp.decl()) + def call(t, a): if t in starts: refs.add(t) - return f'c->x[30] = {hx(a + 4)}; {fname(t)}(c);' + return f'c->x[30] = {hx(a + 4)}; {spill}{fname(t)}(c);{reload_call}' self.stats['external_calls'] += 1 - return f'c->x[30] = {hx(a + 4)}; c->pc = {hx(t)}; rt_dispatch(c);' + return f'c->x[30] = {hx(a + 4)}; {spill}c->pc = {hx(t)}; rt_dispatch(c);{reload_call}' - def patch(ins, a): + def op(i, code): + code = fx(i, code) + return f'{spill}{code}{reload}' if regs.ESCAPE.search(code) else code + + def patch(i, ins, a): """if (switch) { replacement } else { original } for a patched site (recomp/patches.py).""" k, words = self.patched[a] if ins.kind not in ('op', 'bl'): raise patches.PatchError(f'{a:#x}: patched instruction is a {ins.kind}') repl = [] - for j, w in enumerate(words): - r = mod.lifter.lift(w, a) + for j, (w, r) in enumerate(zip(words, self.replacements(a))): if r.kind == 'op': - repl.append(r.code) + repl.append(op(i, r.code)) elif r.kind == 'bl' and j == len(words) - 1: repl.append(call(r.target, a)) else: raise patches.PatchError(f'{a:#x}: replacement word {w:08x} is a {r.kind}') - orig = ins.code if ins.kind == 'op' else call(ins.target, a) + orig = op(i, ins.code) if ins.kind == 'op' else call(ins.target, a) self.uses_patches = True self.stats['patched_sites'] += 1 return f' if (rt_patch_on[{k}]) {{ {" ".join(repl)} }} else {{ {orig} }}' + body = [] # (instruction index or None for a label, line); a line belongs to the instruction it lifts + + def emit(i, line): + body.append((i, line)) + k = None for i, ins in enumerate(insns): a = entry + 4 * i if a in labels: - out.append(f'L_{a:x}:;') + emit(None, f'L_{a:x}:;') k = ins.kind if a in self.patched: - out.append(patch(ins, a)) - elif k == 'op' or k == 'trap': - out.append(' ' + ins.code) + emit(i, patch(i, ins, a)) + elif k == 'op': + emit(i, ' ' + op(i, ins.code)) + elif k == 'trap': + emit(i, f' {spill}{fx(i, ins.code)}') elif k == 'b': - out.append(' ' + jump(ins.target)) + emit(i, ' ' + jump(ins.target, i)) elif k == 'bcond': - out.append(f' if ({ins.cond}) {{ {jump(ins.target)} }}') + emit(i, f' if ({fx(i, ins.cond)}) {{ {jump(ins.target, i)} }}') elif k == 'bl': - out.append(' ' + call(ins.target, a)) + emit(i, ' ' + call(ins.target, a)) elif k == 'blr': - out.append(f' c->pc = {ins.target}; c->x[30] = {hx(a + 4)}; rt_dispatch(c);') + emit(i, f' c->pc = {ins.target}; c->x[30] = {hx(a + 4)}; {spill}rt_dispatch(c);{reload_call}') elif k == 'br': jt = tables.get(a) if jt == 'all': - out.append(f' c->pc = {ins.target}; {redispatch}') + emit(i, f' c->pc = {ins.target}; {redispatch}') elif jt: cases = ' '.join(f'case {hx(t)}: goto L_{t:x};' for t in jt) - out.append(f' switch ({ins.target}) {{ {cases} default: break; }}') - out.append(f' c->pc = {ins.target}; {redispatch}') + emit(i, f' switch ({ins.target}) {{ {cases} default: break; }}') + emit(i, f' {publish(i)}{spill}c->pc = {ins.target}; {redispatch}') else: - out.append(f' c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);') + emit(i, f' {publish(i)}{spill}c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);') elif k == 'ret': - out.append(' return;') + emit(i, f' {spill}return;') else: raise ValueError(f'{a:#x}: unknown instruction kind {k!r}') + n = len(insns) if k not in TERMINATORS: # falls through into the next function if end < mod.hi: - out.append(' ' + jump(end)) + emit(n, ' ' + jump(end, n)) else: - out.append(f' rt_udf(c, {hx(end)});') + emit(n, f' {spill}rt_udf(c, {hx(end)});') + if local: + texts = [''] * (n + 1) + for i, line in body: + if i is not None: + texts[i] += line + clean = {i for i, ins in enumerate(insns) if entry + 4 * i not in self.patched + and (ins.kind in ('bl', 'blr') or (ins.kind == 'op' and regs.RELOAD in texts[i]))} + rp = regs.plan(texts, succ, clean, self.reg_locals, self.kept) + if rp.regs: + out.append(' ' + rp.decl()) + self.stats['local_reg_functions'] += bool(rp.regs) + self.stats['reg_spills'] += sum(len(s) for s in rp.spill) + out += [line if i is None else rp.resolve(i, line) for i, line in body] + else: + out += [line for _, line in body] out.append('}') self.stats['functions'] += 1 self.stats['instructions'] += len(insns) @@ -401,10 +519,11 @@ def _retry(fn, *args): _W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter} -def _worker_init(mods, starts, table): +def _worker_init(mods, starts, table, opts): _W['mods'] = mods _W['starts'] = starts _W['patches'] = table + _W['opts'] = opts # Emitter keyword arguments _W['emitters'] = {} @@ -414,7 +533,7 @@ def _emitter(name): path, base = _W['mods'][name] mod = Module(name, path, base, 'c') e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()), - _W['patches'].for_module(name, base)) + _W['patches'].for_module(name, base), **_W['opts']) return e @@ -476,7 +595,13 @@ def main(): ap.add_argument('--big-fn', type=int, default=300000, help='estimated bytes of C from which a function gets ' 'its own chunk') ap.add_argument('--patches', type=Path, help='code patch table (recomp/patches.py)') + ap.add_argument('--reg-locals', type=int, default=regs.LOCALS, + help=f'guest registers per function kept in C locals (recomp/regs.py; default {regs.LOCALS}, 0: none)') + ap.add_argument('--no-callee-saved-trust', action='store_true', + help='reload X19-X29 and SP after every call too, instead of trusting AAPCS64 (slower; for ' + 'telling a broken callee from a generator bug)') args = ap.parse_args() + opts = {'reg_locals': args.reg_locals, 'callee_saved': not args.no_callee_saved_trust} t0 = time.perf_counter() mods = layout(args.exefs) @@ -522,7 +647,7 @@ def main(): done_at, errors = {}, [] t_emit = time.perf_counter() ctx = mp.get_context('spawn') - with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table)) as pool: + with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool: for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)): results[r['module']].append(r) if r['error']: @@ -562,6 +687,13 @@ def main(): 'br_dispatch': stats['br_dispatch'], 'traps': stats['traps'], 'patched_sites': stats['patched_sites'], + 'local_flag_functions': stats['local_flag_functions'], + 'flag_reads_direct': stats['flag_reads_direct'], + 'flag_reads_packed': stats['flag_reads_packed'], + 'flag_entry_functions': stats['flag_entry_functions'], + 'flag_entry_reads': stats['flag_entry_reads'], + 'flag_reads_after_call': stats['flag_reads_after_call'], + 'flag_publishing_exits': stats['flag_publishing_exits'], 'unsupported': stats['unsupported'], 'external_branches': stats['external_branches'], 'external_calls': stats['external_calls'], diff --git a/recomp/regs.py b/recomp/regs.py new file mode 100644 index 00000000..c583faa8 --- /dev/null +++ b/recomp/regs.py @@ -0,0 +1,107 @@ +"""Function-local general registers for recomp/gen.py. + +The lifter reads and writes the guest's X0-X30 and SP in the Cpu (c->x[n], c->sp), so every guest instruction loads +and stores memory, and the C compiler cannot keep a register in a host register across a guest store: guest stores go +through pointers that may alias the Cpu. A generated function instead copies the LOCALS registers it uses most into +locals at its entry (xN, sp_), works on those, and syncs with the Cpu only where something else reads or writes them: +before a call, an SVC, a trap, a return or a tail exit it stores the registers it changed since the last sync (spill), +and after an SVC it loads them all again (reload). The other registers stay in the Cpu. + +After a call it reloads only X0-X18 and X30: AAPCS64 has the callee preserve X19-X29 and SP, so their locals still +hold what the Cpu holds. What would break that does not occur in TotK: main and subsdk0 import neither longjmp nor +the C++ unwinder, sdk's longjmp, __cxa_throw and _Unwind_* have no direct callers, and the runtime writes no register +above X3 of a running thread's Cpu except X30, which rt_call restores (runtime/hostfn.c). Guest fibers each run on a host fiber of their own with their own Cpu (runtime/kernel.c), +and ExitThread leaves by longjmp without returning. gen.py --no-callee-saved-trust reloads those registers too. + +Only plain assignments write registers in the lifted C (`c->x[n] = ...`, `c->sp = ...`), and no code takes a +register's address, so the rewrite is textual. A register counts as dirty from a write until the next spill on some +path: plan() runs that forward over the function's control flow, so a call spills only what changed since the +previous call or the entry. Spilling a register that is not dirty is harmless (the local equals the Cpu), so where a +case is unclear the plan errs on spilling more. +""" +import re +from collections import Counter + +REG = re.compile(r'c->x\[(\d+)\]|c->sp\b') +WRITE = re.compile(r'c->(?:x\[(\d+)\]|(sp))\s*=(?!=)') +# Lifted instructions that hand the Cpu to the runtime: SVCs return, BRK and undefined words do not. +ESCAPE = re.compile(r'\brt_(?:svc|brk|udf)\(c\b') +SP = 31 # index of SP in the register sets +# Markers gen.py puts where the Cpu must be current, resolved by Plan.resolve(): spill before; reload all, or reload +# what a call may change, after. +SPILL, RELOAD, RELOAD_CALL = '\x00S', '\x00R', '\x00C' +CALLEE_SAVED = frozenset(range(19, 30)) | {SP} +# Locals per function: the most used registers. More cost more than they save, as clang then spills them to the stack +# and the syncs stay (300 hot TotK functions: fewest instructions and memory operands at 4-6). +LOCALS = 6 + + +def name(r): + return 'sp_' if r == SP else f'x{r}' + + +def cpu(r): + return 'c->sp' if r == SP else f'c->x[{r}]' + + +def refs(text): + """Registers text reads or writes, once per occurrence.""" + return [SP if m.group(0) == 'c->sp' else int(m.group(1)) for m in REG.finditer(text)] + + +def written(text): + return {SP if m.group(2) else int(m.group(1)) for m in WRITE.finditer(text)} + + +class Plan: + def __init__(self, regs, spill, kept): + self.regs = regs # sorted registers the function keeps in locals + self.local = set(regs) + self.spill = spill # per instruction: registers to store at its sync points + self.kept = kept # registers a call leaves as they were (CALLEE_SAVED, or none) + + def decl(self): + """Locals at the top of the function, loaded from the Cpu.""" + return 'uint64_t ' + ', '.join(f'{name(r)} = {cpu(r)}' for r in self.regs) + ';' if self.regs else '' + + def resolve(self, i, text): + """text of instruction i with its registers on the locals and its sync markers expanded.""" + def local(m): + r = SP if m.group(0) == 'c->sp' else int(m.group(1)) + return name(r) if r in self.local else m.group(0) + + text = REG.sub(local, text) + spill = ''.join(f'{cpu(r)} = {name(r)}; ' for r in sorted(self.spill[i])) + reload = ''.join(f' {name(r)} = {cpu(r)};' for r in self.regs) + reload_call = ''.join(f' {name(r)} = {cpu(r)};' for r in self.regs if r not in self.kept) + return text.replace(SPILL, spill).replace(RELOAD, reload).replace(RELOAD_CALL, reload_call) + + +def plan(texts, succ, clean, n_locals=LOCALS, kept=CALLEE_SAVED): + """texts[i]: instruction i's C with markers; succ[i]: successor indices inside the function; clean: instructions + after which every local equals the Cpu (a call or an SVC that spills and reloads on every path through it); + n_locals: how many registers to keep in locals; kept: registers a call does not reload.""" + n = len(texts) + uses = Counter(r for t in texts for r in refs(t)) + regs = {r for r, _ in sorted(uses.items(), key=lambda kv: (-kv[1], kv[0]))[:n_locals]} + writes = [written(t) & regs for t in texts] + dirty = [set() for _ in range(n)] + seen = [False] * n + work = [0] if n else [] + if n: + seen[0] = True + while work: + i = work.pop() + out = set() if i in clean else dirty[i] | writes[i] + for s in succ[i]: + if not seen[s] or not out <= dirty[s]: + seen[s] = True + dirty[s] |= out + work.append(s) + every = set().union(*writes) if writes else set() + for i in range(n): + if not seen[i]: # not reached from the entry in the plan's flow: anything may be dirty + dirty[i] = every + # A sync point stores what was dirty on entry to its instruction and what the instruction wrote before it + # (c->x[30] of a call): all the instruction's writes, which is at most a little more. + return Plan(sorted(regs), [dirty[i] | writes[i] if SPILL in texts[i] else set() for i in range(n)], kept) diff --git a/runtime/cpu.h b/runtime/cpu.h index 1246ee14..a787cbb0 100644 --- a/runtime/cpu.h +++ b/runtime/cpu.h @@ -215,6 +215,107 @@ static inline uint32_t logic32(Cpu *c, uint32_t r) { c->nzcv = MKFLAGS(r >> 31, ADC_IMPL(64, ) ADC_IMPL(32, ) +// ---- function-local flags (recomp/flags.py) +// Generated functions keep NZCV in a local Flags F, never in c->nzcv. A definition of a known kind stores its +// operands (a - b, a + b, or a logic result in a); fc_(&F, CC_xx) computes a condition from them, so a compare +// and its branch become one host compare. F.nz holds the packed form for definitions only known packed (CCMP, FCMP, +// MSR, ADCS) and for readers that several kinds of definition reach. Every helper is always inlined with a constant +// condition, so F stays in registers and only the condition's own arithmetic remains. +typedef struct { uint64_t a, b, nz; } Flags; +enum { CC_EQ, CC_NE, CC_CS, CC_CC, CC_MI, CC_PL, CC_VS, CC_VC, CC_HI, CC_LS, CC_GE, CC_LT, CC_GT, CC_LE }; +#define CPU_FL static inline __attribute__((always_inline)) + +CPU_FL int fl_cond(int cc, int n, int z, int cy, int v) { + switch (cc) { + case CC_EQ: return z; + case CC_NE: return !z; + case CC_CS: return cy; + case CC_CC: return !cy; + case CC_MI: return n; + case CC_PL: return !n; + case CC_VS: return v; + case CC_VC: return !v; + case CC_HI: return cy && !z; + case CC_LS: return !cy || z; + case CC_GE: return n == v; + case CC_LT: return n != v; + case CC_GT: return !z && n == v; + default: return z || n != v; + } +} + +// Subtract (SUBS, CMP): conditions as host compares of the operands. +#define FL_SUB(N, U, S) \ + CPU_FL U fsub##N(Flags *f, U a, U b) { f->a = a; f->b = b; return (U)(a - b); } \ + CPU_FL int fc_sub##N(const Flags *f, int cc) { \ + U a = (U)f->a, b = (U)f->b, r = (U)(a - b); \ + switch (cc) { \ + case CC_EQ: return a == b; \ + case CC_NE: return a != b; \ + case CC_CS: return a >= b; \ + case CC_CC: return a < b; \ + case CC_MI: return (S)r < 0; \ + case CC_PL: return (S)r >= 0; \ + case CC_VS: return (S)((a ^ b) & (a ^ r)) < 0; \ + case CC_VC: return (S)((a ^ b) & (a ^ r)) >= 0; \ + case CC_HI: return a > b; \ + case CC_LS: return a <= b; \ + case CC_GE: return (S)a >= (S)b; \ + case CC_LT: return (S)a < (S)b; \ + case CC_GT: return (S)a > (S)b; \ + default: return (S)a <= (S)b; \ + } \ + } \ + CPU_FL uint64_t nz_sub##N(U a, U b) { \ + U r = (U)(a - b); \ + return MKFLAGS((S)r < 0, r == 0, a >= b, (S)((a ^ b) & (a ^ r)) < 0); \ + } \ + CPU_FL uint64_t fnz_sub##N(const Flags *f) { return nz_sub##N((U)f->a, (U)f->b); } +FL_SUB(64, uint64_t, int64_t) +FL_SUB(32, uint32_t, int32_t) + +// Add (ADDS, CMN) and logic results (ANDS, TST, BICS): conditions from N, Z, C, V. +#define FL_ADD(N, U, S) \ + CPU_FL U fadd##N(Flags *f, U a, U b) { f->a = a; f->b = b; return (U)(a + b); } \ + CPU_FL uint64_t nz_add##N(U a, U b) { \ + U r = (U)(a + b); \ + return MKFLAGS((S)r < 0, r == 0, r < a, (S)((a ^ r) & (b ^ r)) < 0); \ + } \ + CPU_FL int fc_add##N(const Flags *f, int cc) { \ + U a = (U)f->a, b = (U)f->b, r = (U)(a + b); \ + return fl_cond(cc, (S)r < 0, r == 0, r < a, (S)((a ^ r) & (b ^ r)) < 0); \ + } \ + CPU_FL uint64_t fnz_add##N(const Flags *f) { return nz_add##N((U)f->a, (U)f->b); } \ + CPU_FL U flogic##N(Flags *f, U r) { f->a = r; return r; } \ + CPU_FL int fc_logic##N(const Flags *f, int cc) { \ + U r = (U)f->a; \ + return fl_cond(cc, (S)r < 0, r == 0, 0, 0); \ + } \ + CPU_FL uint64_t fnz_logic##N(const Flags *f) { return MKFLAGS((S)f->a < 0, (U)f->a == 0, 0, 0); } +FL_ADD(64, uint64_t, int64_t) +FL_ADD(32, uint32_t, int32_t) + +// Packed flags. +CPU_FL int fc_nz(const Flags *f, int cc) { + return fl_cond(cc, f->nz >> 31 & 1, f->nz >> 30 & 1, f->nz >> 29 & 1, f->nz >> 28 & 1); +} +CPU_FL uint64_t fnz_nz(const Flags *f) { return f->nz; } + +// ADC, SBC, ADCS, SBCS with the carry in from F. +#define FL_ADC(N, U) \ + CPU_FL U adc##N##f(int cin, U a, U b) { return (U)(a + b + (U)cin); } \ + CPU_FL U sbc##N##f(int cin, U a, U b) { return (U)(a - b - 1 + (U)cin); } \ + CPU_FL U adcs##N##f(Flags *f, int cin, U a, U b) { \ + U r = (U)(a + b + (U)cin); \ + int carry = cin ? r <= a : r < a; \ + f->nz = MKFLAGS(r >> (N - 1), r == 0, carry, ((a ^ r) & (b ^ r)) >> (N - 1)); \ + return r; \ + } \ + CPU_FL U sbcs##N##f(Flags *f, int cin, U a, U b) { return adcs##N##f(f, cin, a, (U)~b); } +FL_ADC(64, uint64_t) +FL_ADC(32, uint32_t) +#undef CPU_FL + // ---- bit ops #define ROR64(v, r) ((uint64_t)(v) >> ((r) & 63) | (uint64_t)(v) << ((64 - ((r) & 63)) & 63)) #define ROR32(v, r) ((uint32_t)(v) >> ((r) & 31) | (uint32_t)(v) << ((32 - ((r) & 31)) & 31)) diff --git a/tests/diff/common.py b/tests/diff/common.py index cbe8a4ee..648e280c 100644 --- a/tests/diff/common.py +++ b/tests/diff/common.py @@ -226,21 +226,21 @@ def build_harness(work, bodies, jobs, cflags): return exe -def write_cases(path, cases): - """cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk).""" +def write_cases(path, cases, stubs=False): + """cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk). stubs: calls go to harness.c's call stub (rt_stub).""" with open(path, 'wb') as f: for k, fpcr, x, sp, nzcv, v, memchk in cases: f.write(struct.pack(IN_FMT, k, fpcr, *(list(x[:31]) + [0]), sp, nzcv)) for q in v: f.write(q.to_bytes(16, 'little')) - f.write(struct.pack(IN_TAIL, int(memchk), 0)) + f.write(struct.pack(IN_TAIL, int(memchk), int(stubs))) -def run_harness(exe, work, buf0, cases, images): +def run_harness(exe, work, buf0, cases, images, stubs=False): """Run the lifted tests. Returns (records, stderr tail or None). images: (base, path) pairs.""" with open(work / 'buf.bin', 'wb') as f: f.write(buf0) - write_cases(work / 'cases.bin', cases) + write_cases(work / 'cases.bin', cases, stubs) out = work / 'out.bin' args = [str(exe), str(work / 'buf.bin'), str(work / 'cases.bin'), str(out)] for base, path in images: @@ -258,14 +258,14 @@ def run_harness(exe, work, buf0, cases, images): return recs, err -def diff_state(got, want, check_pc, check_mem): +def diff_state(got, want, check_pc, check_mem, check_nzcv=True): """got: harness record; want: (x, sp, nzcv, pc, v, mem). Returns a list of mismatches.""" hx, hsp, hnz, hpc, hmem, hv = got ex, esp, enz, epc, ev, emem = want d = [f'x{j}: got {hx[j]:x} want {ex[j]:x}' for j in range(31) if hx[j] != ex[j]] if hsp != esp: d.append(f'sp: got {hsp:x} want {esp:x}') - if hnz != enz: + if check_nzcv and hnz != enz: d.append(f'nzcv: got {hnz:x} want {enz:x}') if check_pc and hpc != epc: d.append(f'pc: got {hpc:x} want {epc:x}') diff --git a/tests/diff/func.py b/tests/diff/func.py index 79a5349c..8e088564 100644 --- a/tests/diff/func.py +++ b/tests/diff/func.py @@ -1,14 +1,19 @@ """Differential test of whole emitted functions: recomp/gen.py's C for a guest function against Unicorn running the same function from its entry until it returns. -Picks leaf functions from the ExeFS modules: no calls, no branches out of the function, no SVC, BRK, system -register, exclusive or atomic instructions; with conditional branches, and loops, jump tables and SIMD where the -module has them. Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each -from random states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into -itself and random data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when -PC reaches the sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are -mapped read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, NZCV, V0-V31 -and the scratch buffer. +Picks functions from the ExeFS modules: no branches out of the function, no SVC, BRK, system register, exclusive or +atomic instructions; with conditional branches, and loops, jump tables, SIMD and calls where the module has them. +Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each from random +states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into itself and random +data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when PC reaches the +sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are mapped +read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, V0-V31 and the +scratch buffer; NZCV is local to a generated function (recomp/flags.py). + +Every BL and BLR goes to one stub on both sides (rt_stub in harness.c, Runner.stub here) instead of its callee. The +stub folds the guest state at the call into a digest and sets the caller-saved registers from it, so the test sees +whether the emitted code had every register in the Cpu at the call and reads them back after it. Callers of another +picked function are dropped, as the C side would run that function instead of the stub. Usage: func.py [--exefs DIR] [--modules main,subsdk0,sdk] [--functions N] [--cases K] [--seed S] [--jobs J] """ @@ -38,11 +43,24 @@ STACK_TOP = BUF + 0x3F000 MIN_INSNS, MAX_INSNS = 6, 600 +FNV = 0x100000001B3 +M64 = (1 << 64) - 1 + + +def splitmix(z): + z = (z + 0x9E3779B97F4A7C15) & M64 + z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & M64 + z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M64 + return z ^ (z >> 31) + + +def call_mask(w): + return ((w & 0xFC000000) == 0x94000000) | ((w & 0xFFFFFC1F) == 0xD63F0000) # BL, BLR + + def forbidden_mask(w): """Words a picked function must not contain.""" - m = (w & 0xFC000000) == 0x94000000 # BL - m |= (w & 0xFFFFFC1F) == 0xD63F0000 # BLR - m |= (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS + m = (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS sysi = (w & 0xFFC00000) == 0xD5000000 # system: MRS, MSR, SYS (DC ZVA), hints, barriers hint_or_barrier = ((w & 0xFFFFF000) == 0xD5032000) | ((w & 0xFFFFF000) == 0xD5033000) m |= sysi & ~hint_or_barrier @@ -61,7 +79,7 @@ def simd_mask(w): def pick_functions(name, mod, count, r): - """[(entry, end, tags)] of up to count leaf functions of mod, plus the Emitter that will emit them.""" + """[(entry, end, tags)] of up to count functions of mod, plus the Emitter that will emit them.""" starts, _ = gen.discover(mod) w = mod.words ends = np.append(starts[1:], len(w)) @@ -73,15 +91,18 @@ def pick_functions(name, mod, count, r): size = ends - starts ok = (per_fn(forbidden_mask(w)) == 0) & (per_fn(cond_mask(w)) > 0) & (size >= MIN_INSNS) & (size <= MAX_INSNS) has_br = per_fn((w & 0xFFFFFC1F) == 0xD61F0000) > 0 + has_call = per_fn(call_mask(w)) > 0 cand = np.flatnonzero(ok) with_br = [int(i) for i in cand if has_br[i]] - rest = [int(i) for i in cand if not has_br[i]] - r.shuffle(with_br) - r.shuffle(rest) + with_call = [int(i) for i in cand if has_call[i] and not has_br[i]] + rest = [int(i) for i in cand if not has_br[i] and not has_call[i]] + for pool in (with_br, with_call, rest): + r.shuffle(pool) emitter = gen.Emitter(mod, set((mod.lo + starts * 4).tolist())) picks, tags_seen = [], Counter() - # Functions with a br first (jump tables, fallbacks), at most a quarter of the picks. - for pool, quota in ((with_br, count // 4), (rest, count)): + # Functions with a br first (jump tables, fallbacks), at most a quarter of the picks; then functions with calls, + # at most half. + for pool, quota in ((with_br, count // 4), (with_call, count // 2), (rest, count)): taken = 0 for i in pool: if len(picks) >= count or taken >= quota: @@ -91,13 +112,28 @@ def pick_functions(name, mod, count, r): if tags is None: continue picks.append((entry, end, tags)) - tags_seen.update(tags) taken += 1 - print(f'{name}: {len(starts)} functions, {len(cand)} leaf candidates, picked {len(picks)}: ' + picked = {e for e, _, _ in picks} + picks = [p for p in picks if not callees(emitter, p[0], p[1]) & picked] + for _, _, tags in picks: + tags_seen.update(tags) + print(f'{name}: {len(starts)} functions, {len(cand)} candidates, picked {len(picks)}: ' f'{dict(tags_seen)}', flush=True) return picks, emitter +def call_sites(mod, entry, end): + """Addresses of the BL and BLR instructions of [entry, end).""" + return [a for a in range(entry, end, 4) if mod.lift(a).kind in ('bl', 'blr')] + + +def callees(emitter, entry, end): + """Function starts the emitted C of [entry, end) calls directly (`f_(c)`, not through rt_dispatch).""" + mod = emitter.mod + return {mod.lift(a).target for a in call_sites(mod, entry, end) + if mod.lift(a).kind == 'bl' and mod.lift(a).target in emitter.starts} + + def check_function(emitter, entry, end): """Tags of a function the test can run in isolation, or None.""" mod = emitter.mod @@ -108,7 +144,7 @@ def check_function(emitter, entry, end): for k, ins in enumerate(insns): a = entry + 4 * k if ins.kind in ('bl', 'blr'): - return None + tags.add('calls') if ins.kind in ('b', 'bcond'): if not entry <= ins.target < end: return None @@ -147,9 +183,14 @@ def build(work, emitted, jobs, cflags): path.write_text(gen.chunk_text(name, j // per, emitter, fns[j:j + per])) srcs.append(path) table = work / 'table.c' + called = set() + for _, entry, end, emitter in emitted: + called |= callees(emitter, entry, end) table.write_text('#include "cpu.h"\n' + ''.join(f'void {gen.fname(e)}(Cpu *c);\n' for _, e, _, _ in emitted) + 'GuestFn tests[] = {' + ','.join(gen.fname(e) for _, e, _, _ in emitted) + '};\n' + - f'int ntests = {len(emitted)};\n') + f'int ntests = {len(emitted)};\n' + 'void rt_stub_call(Cpu *c, uint64_t target);\n' + + ''.join(f'void {gen.fname(t)}(Cpu *c) {{ rt_stub_call(c, {hex(t)}ull); }}\n' + for t in sorted(called))) srcs += [table, ROOT / 'tests' / 'diff' / 'harness.c'] objs, procs = [], [] for s in srcs: @@ -191,9 +232,12 @@ def make_case(r): class Runner: - """Unicorn with the module images read-only and a block hook that stops when execution leaves the function.""" + """Unicorn with the module images read-only, a block hook that stops when execution leaves the function, and a + code hook on every call site of the picked functions that stops before the call, for the stub.""" - def __init__(self, images, buf0): + MAX_CALLS = 1000 + + def __init__(self, images, buf0, sites): self.emu = Emu(images) mu = self.emu.mu for base, data in images: @@ -201,25 +245,70 @@ class Runner: mu.mem_map(SENTINEL, 0x1000) mu.mem_write(BUF, buf0) self.buf0 = buf0 - self.lo = self.hi = 0 + self.lo = self.hi = self.h = 0 self.left = False + self.at_call = None mu.hook_add(uc.UC_HOOK_BLOCK, self._block) + # One hook per call site, all added before anything is translated, so no cached block lacks one. + for a in sorted(sites): + mu.hook_add(uc.UC_HOOK_CODE, self._call, begin=a, end=a) def _block(self, mu, address, size, user): if not self.lo <= address < self.hi and address != SENTINEL: self.left = True mu.emu_stop() + def _call(self, mu, address, size, user): + if self.lo <= address < self.hi: + self.at_call = address + mu.emu_stop() + + def stub(self, a): + """harness.c rt_stub for the call at a, which Unicorn has not executed yet. Returns the address to resume at.""" + e, mu = self.emu, self.emu.mu + w = int.from_bytes(mu.mem_read(a, 4), 'little') + if (w & 0xFC000000) == 0x94000000: # BL + imm = w & 0x03FFFFFF + target = (a + ((imm - (1 << 26) if imm >> 25 else imm) << 2)) & M64 + else: # BLR + target = mu.reg_read(e.xregs[(w >> 5) & 31]) + mu.reg_write(A.UC_ARM64_REG_X30, a + 4) + h = ((self.h ^ target) * FNV) & M64 + for val in list(mu.reg_read_batch(e.xregs)) + [mu.reg_read(A.UC_ARM64_REG_SP)]: + h = ((h ^ val) * FNV) & M64 + for val in mu.reg_read_batch(e.qregs[:8]): + h = ((h ^ (val & M64)) * FNV) & M64 + h = ((h ^ (val >> 64)) * FNV) & M64 + self.h = h + regs = [] + for i in range(19): + s = splitmix((h + i) & M64) + regs.append((e.xregs[i], PTR_LO + ((s >> 4) % ((PTR_HI - PTR_LO) // 16)) * 16 if i < 8 and s & 3 else s)) + regs.append((A.UC_ARM64_REG_NZCV, splitmix((h + 19) & M64) >> 60 << 28)) + for i in range(8): + regs.append((e.qregs[i], splitmix((h + 33 + 2 * i) & M64) << 64 | splitmix((h + 32 + 2 * i) & M64))) + mu.reg_write_batch(regs) + top = BUF + BUF_SIZE - 16 + n = int.from_bytes(mu.mem_read(top, 8), 'little') + mu.mem_write(top, ((n + 1) & M64).to_bytes(8, 'little') + h.to_bytes(8, 'little')) + return a + 4 + def run(self, entry, end, case, steps): """(x, sp, nzcv, pc, v, crc of the buffer) or None if the case is skipped.""" fpcr, x, sp, nzcv, v = case e, mu = self.emu, self.emu.mu - self.lo, self.hi, self.left = entry, end, False + self.lo, self.hi, self.left, self.h = entry, end, False, 0 mu.reg_write_batch(list(zip(e.xregs, x[:31])) + list(zip(e.qregs, v)) + [(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)]) + pc, ok = entry, False try: - mu.emu_start(entry, SENTINEL, count=steps) - ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL + for _ in range(self.MAX_CALLS): + self.at_call = None + mu.emu_start(pc, SENTINEL, count=steps) + if self.at_call is None: + ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL + break + pc = self.stub(self.at_call) except uc.UcError: ok = False if not ok: @@ -268,7 +357,10 @@ def main(): print(f'compiled ({time.time() - t1:.0f}s)', flush=True) buf0 = make_buffer(r) - runner = Runner([(base, data) for base, _, data in images], buf0) + sites = set() + for _, entry, end, emitter in emitted: + sites.update(call_sites(emitter.mod, entry, end)) + runner = Runner([(base, data) for base, _, data in images], buf0, sites) cases, expect, tested, skipped = [], [], set(), 0 t2 = time.time() for k, (name, entry, end, _) in enumerate(emitted): @@ -286,7 +378,7 @@ def main(): flush=True) t3 = time.time() - recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images]) + recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images], stubs=True) print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)') if err: k = cases[len(recs)][0] if len(recs) < len(cases) else None @@ -295,7 +387,9 @@ def main(): fails, nfail = {}, 0 for case, want_state, got in zip(cases, expect, recs): - d = diff_state(got, want_state, check_pc=False, check_mem=True) + # Generated functions keep NZCV in a local (recomp/flags.py) and leave c->nzcv as it was: flags only count + # where the function reads them, which the registers and memory show. + d = diff_state(got, want_state, check_pc=False, check_mem=True, check_nzcv=False) if d: nfail += 1 fails.setdefault(case[0], d[:4]) diff --git a/tests/diff/harness.c b/tests/diff/harness.c index a24f4244..e268d543 100644 --- a/tests/diff/harness.c +++ b/tests/diff/harness.c @@ -1,5 +1,6 @@ -// Runs lifted single-instruction tests and writes the resulting CPU state per case. -// Shared by tests/diff/lift.py and tests/diff/simd.py; the expected state comes from Unicorn on the Python side. +// Runs lifted tests and writes the resulting CPU state per case. +// Shared by tests/diff/lift.py, simd.py, atomic.py and func.py; the expected state comes from Unicorn on the Python +// side. // // argv: buf.bin cases.bin out.bin image base [image base ...] // Each image is mapped at its guest address (guest == host), the scratch buffer at BUF. @@ -19,7 +20,48 @@ extern int ntests; #define BUF 0x7200000000ull #define BUF_SIZE 0x40000 -void rt_dispatch(Cpu *c) { (void)c; } +#define PTR_LO (BUF + 0x8000) +#define PTR_SPAN 0x20000 + +static int g_stubs; // In.stubs of the running case +static uint64_t g_stub_h; // digest of every call's state so far + +static uint64_t splitmix(uint64_t z) { + z += 0x9E3779B97F4A7C15ull; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +static uint64_t mix(uint64_t h, uint64_t w) { return (h ^ w) * 0x100000001B3ull; } + +// The callee of every call in a function under tests/diff/func.py; Runner.stub there is the same function on the +// Unicorn side. Folds the target, X0-X30, SP and V0-V7 into a digest, so a register the caller left unwritten at the +// call changes it, then sets X0-X18, NZCV and V0-V7 from the digest, so a caller reading a register it held across +// the call sees a value the callee could have left. X0-X7 mostly get pointers into the scratch buffer. The digest +// and the call count go to the top of the scratch buffer, which the memory check covers. +static void rt_stub(Cpu *c, uint64_t target) { + uint64_t h = mix(g_stub_h, target); + for (int i = 0; i < 31; i++) h = mix(h, c->x[i]); + h = mix(h, c->sp); + for (int i = 0; i < 8; i++) h = mix(mix(h, (uint64_t)c->v[i]), (uint64_t)(c->v[i] >> 64)); + g_stub_h = h; + for (int i = 0; i < 19; i++) { + uint64_t s = splitmix(h + i); + c->x[i] = i < 8 && (s & 3) ? PTR_LO + ((s >> 4) % (PTR_SPAN / 16)) * 16 : s; + } + c->nzcv = splitmix(h + 19) >> 60 << 28; + for (int i = 0; i < 8; i++) + c->v[i] = (__uint128_t)splitmix(h + 33 + 2 * i) << 64 | splitmix(h + 32 + 2 * i); + uint64_t *slot = (uint64_t *)(uintptr_t)(BUF + BUF_SIZE - 16); + slot[0]++; + slot[1] = h; +} + +void rt_dispatch(Cpu *c) { + if (g_stubs) rt_stub(c, c->pc); +} +void rt_stub_call(Cpu *c, uint64_t target) { rt_stub(c, target); } void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; } void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; abort(); } void rt_udf(Cpu *c, uint64_t pc) { (void)c; (void)pc; abort(); } @@ -61,7 +103,7 @@ static uint32_t crc32_buf(const uint8_t *p, size_t n) { } // Layouts match IN_FMT / OUT_FMT in tests/diff/common.py. -struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, pad; }; +struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, stubs; }; struct Out { uint64_t x[32], sp, nzcv, pc, mem; __uint128_t v[32]; }; int main(int argc, char **argv) { @@ -96,6 +138,8 @@ int main(int argc, char **argv) { c.fpcr = in.fpcr; memcpy(c.v, in.v, sizeof c.v); c.excl_addr = ~0ull; + g_stubs = (int)in.stubs; + g_stub_h = 0; tests[in.k](&c); memcpy(out.x, c.x, sizeof out.x); out.sp = c.sp; diff --git a/tests/flags_check.c b/tests/flags_check.c new file mode 100644 index 00000000..9563925a --- /dev/null +++ b/tests/flags_check.c @@ -0,0 +1,111 @@ +// The function-local flag helpers of runtime/cpu.h (Flags, fc_*, fnz_*, nz_*, adc*f) against the packed helpers +// (subs64() and friends, COND_xx), which tests/diff/lift.py checks against Unicorn. Built and run by +// tests/flags_check.py; prints the first mismatches and exits 1 on any. +#include + +#include "cpu.h" + +static const uint64_t EDGE[] = {0, 1, 2, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000, + 0x80000001, 0xfffffffe, 0xffffffff, 0x100000000ull, 0x7fffffffffffffffull, + 0x8000000000000000ull, 0x8000000000000001ull, 0xfffffffffffffffeull, + 0xffffffffffffffffull, 0x123456789abcdef0ull, 0xfedcba9876543210ull}; +#define NEDGE (sizeof EDGE / sizeof EDGE[0]) + +static uint64_t rng = 0x9E3779B97F4A7C15ull; +static uint64_t next(void) { + rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17; + return rng; +} + +static int fails; +static void bad(const char *what, int cc, uint64_t a, uint64_t b, uint64_t got, uint64_t want) { + if (fails++ < 20) + printf("%s cc %d a %llx b %llx: got %llx want %llx\n", what, cc, (unsigned long long)a, (unsigned long long)b, + (unsigned long long)got, (unsigned long long)want); +} + +static int cond(const Cpu *c, int cc) { + switch (cc) { + case CC_EQ: return COND_EQ(c); + case CC_NE: return COND_NE(c); + case CC_CS: return COND_CS(c); + case CC_CC: return COND_CC(c); + case CC_MI: return COND_MI(c); + case CC_PL: return COND_PL(c); + case CC_VS: return COND_VS(c); + case CC_VC: return COND_VC(c); + case CC_HI: return COND_HI(c); + case CC_LS: return COND_LS(c); + case CC_GE: return COND_GE(c); + case CC_LT: return COND_LT(c); + case CC_GT: return COND_GT(c); + default: return COND_LE(c); + } +} + +// The condition argument is not constant here, so this also covers the switch default paths. +#define CHECK(NAME, OLD, NEW, FC, FNZ) \ + do { \ + Cpu c = {0}; \ + Flags f = {0}; \ + uint64_t ro = OLD, rn = NEW; \ + if (ro != rn) bad(NAME " result", -1, a, b, rn, ro); \ + for (int cc = 0; cc < 14; cc++) \ + if (!FC(&f, cc) != !cond(&c, cc)) bad(NAME, cc, a, b, FC(&f, cc), cond(&c, cc)); \ + f.nz = 0; \ + if (FNZ(&f) != c.nzcv) bad(NAME " packed", -1, a, b, FNZ(&f), c.nzcv); \ + Flags g = {.nz = c.nzcv}; \ + for (int cc = 0; cc < 14; cc++) \ + if (!fc_nz(&g, cc) != !cond(&c, cc)) bad(NAME " nz", cc, a, b, fc_nz(&g, cc), cond(&c, cc)); \ + } while (0) + +static void one(uint64_t a, uint64_t b) { + uint32_t a32 = (uint32_t)a, b32 = (uint32_t)b; + CHECK("sub64", subs64(&c, a, b), fsub64(&f, a, b), fc_sub64, fnz_sub64); + CHECK("sub32", subs32(&c, a32, b32), fsub32(&f, a32, b32), fc_sub32, fnz_sub32); + CHECK("add64", adds64(&c, a, b), fadd64(&f, a, b), fc_add64, fnz_add64); + CHECK("add32", adds32(&c, a32, b32), fadd32(&f, a32, b32), fc_add32, fnz_add32); + CHECK("logic64", logic64(&c, a & b), flogic64(&f, a & b), fc_logic64, fnz_logic64); + CHECK("logic32", logic32(&c, a32 & b32), flogic32(&f, a32 & b32), fc_logic32, fnz_logic32); + { + Cpu c = {0}; + if (nz_sub64(a, b) != (subs64(&c, a, b), c.nzcv)) bad("nz_sub64", -1, a, b, nz_sub64(a, b), c.nzcv); + if (nz_sub32(a32, b32) != (subs32(&c, a32, b32), c.nzcv)) bad("nz_sub32", -1, a, b, nz_sub32(a32, b32), c.nzcv); + if (nz_add64(a, b) != (adds64(&c, a, b), c.nzcv)) bad("nz_add64", -1, a, b, nz_add64(a, b), c.nzcv); + if (nz_add32(a32, b32) != (adds32(&c, a32, b32), c.nzcv)) bad("nz_add32", -1, a, b, nz_add32(a32, b32), c.nzcv); + } + for (int cin = 0; cin < 2; cin++) { + Cpu c = {.nzcv = (uint64_t)cin << 29}; + Flags f = {0}; + if (adc64f(cin, a, b) != adc64(&c, a, b)) bad("adc64", cin, a, b, adc64f(cin, a, b), adc64(&c, a, b)); + if (sbc64f(cin, a, b) != sbc64(&c, a, b)) bad("sbc64", cin, a, b, sbc64f(cin, a, b), sbc64(&c, a, b)); + if (adc32f(cin, a32, b32) != adc32(&c, a32, b32)) bad("adc32", cin, a, b, adc32f(cin, a32, b32), adc32(&c, a32, b32)); + if (sbc32f(cin, a32, b32) != sbc32(&c, a32, b32)) bad("sbc32", cin, a, b, sbc32f(cin, a32, b32), sbc32(&c, a32, b32)); + uint64_t r; +#define CARRY(NAME, NEW, OLD) \ + c.nzcv = (uint64_t)cin << 29; \ + r = NEW; \ + if (r != OLD || f.nz != c.nzcv) bad(NAME, cin, a, b, f.nz, c.nzcv); + CARRY("adcs64", adcs64f(&f, cin, a, b), adcs64(&c, a, b)) + CARRY("sbcs64", sbcs64f(&f, cin, a, b), sbcs64(&c, a, b)) + CARRY("adcs32", adcs32f(&f, cin, a32, b32), adcs32(&c, a32, b32)) + CARRY("sbcs32", sbcs32f(&f, cin, a32, b32), sbcs32(&c, a32, b32)) + } +} + +int main(void) { + long n = 0; + for (size_t i = 0; i < NEDGE; i++) + for (size_t j = 0; j < NEDGE; j++, n++) one(EDGE[i], EDGE[j]); + for (int k = 0; k < 2000000; k++, n++) { + uint64_t a = next(), b = next(); + switch (k & 3) { + case 1: b = a; break; // equal operands + case 2: b = a + (next() & 3) - 1; break; // neighbours + case 3: a &= 0xff, b &= 0xff; break; // small values + } + one(a, b); + } + printf("%ld operand pairs, %d mismatches\n", n, fails); + return fails != 0; +} diff --git a/tests/flags_check.py b/tests/flags_check.py new file mode 100644 index 00000000..9447ff74 --- /dev/null +++ b/tests/flags_check.py @@ -0,0 +1,32 @@ +"""Build and run tests/flags_check.c: runtime/cpu.h's function-local flag helpers against its packed flag helpers. + +Usage: flags_check.py +""" +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests' / 'diff')) +from common import MSYS_BIN, compiler # noqa: E402 + + +def main(): + work = ROOT / 'build' / 'test' / 'flags' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('flags_check.exe' if os.name == 'nt' else 'flags_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + for opt in ('-O0', '-O1'): + subprocess.check_call([compiler(), opt, '-fno-strict-aliasing', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'flags_check.c'), '-o', str(exe)], env=env) + print(opt, end=': ', flush=True) + if subprocess.run([str(exe)], env=env).returncode: + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main())