From e9216b8b4978603b82da340f454413a49ffc8288 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 17:12:34 +0300 Subject: [PATCH 1/4] Test functions with calls in tests/diff/func.py: every BL and BLR goes to one stub on both sides, which folds the registers at the call into a digest kept in the scratch buffer and sets X0-X18, NZCV and V0-V7 from it, so a register missing from the Cpu at a call or held across one fails the case Co-Authored-By: Claude Opus 5.5 --- tests/diff/common.py | 10 +-- tests/diff/func.py | 150 ++++++++++++++++++++++++++++++++++--------- tests/diff/harness.c | 52 +++++++++++++-- 3 files changed, 174 insertions(+), 38 deletions(-) diff --git a/tests/diff/common.py b/tests/diff/common.py index cbe8a4ee..73097311 100644 --- a/tests/diff/common.py +++ b/tests/diff/common.py @@ -226,21 +226,21 @@ def build_harness(work, bodies, jobs, cflags): return exe -def write_cases(path, cases): - """cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk).""" +def write_cases(path, cases, stubs=False): + """cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk). stubs: calls go to harness.c's call stub (rt_stub).""" with open(path, 'wb') as f: for k, fpcr, x, sp, nzcv, v, memchk in cases: f.write(struct.pack(IN_FMT, k, fpcr, *(list(x[:31]) + [0]), sp, nzcv)) for q in v: f.write(q.to_bytes(16, 'little')) - f.write(struct.pack(IN_TAIL, int(memchk), 0)) + f.write(struct.pack(IN_TAIL, int(memchk), int(stubs))) -def run_harness(exe, work, buf0, cases, images): +def run_harness(exe, work, buf0, cases, images, stubs=False): """Run the lifted tests. Returns (records, stderr tail or None). images: (base, path) pairs.""" with open(work / 'buf.bin', 'wb') as f: f.write(buf0) - write_cases(work / 'cases.bin', cases) + write_cases(work / 'cases.bin', cases, stubs) out = work / 'out.bin' args = [str(exe), str(work / 'buf.bin'), str(work / 'cases.bin'), str(out)] for base, path in images: diff --git a/tests/diff/func.py b/tests/diff/func.py index 79a5349c..817a6070 100644 --- a/tests/diff/func.py +++ b/tests/diff/func.py @@ -1,14 +1,19 @@ """Differential test of whole emitted functions: recomp/gen.py's C for a guest function against Unicorn running the same function from its entry until it returns. -Picks leaf functions from the ExeFS modules: no calls, no branches out of the function, no SVC, BRK, system -register, exclusive or atomic instructions; with conditional branches, and loops, jump tables and SIMD where the -module has them. Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each -from random states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into -itself and random data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when -PC reaches the sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are -mapped read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, NZCV, V0-V31 -and the scratch buffer. +Picks functions from the ExeFS modules: no branches out of the function, no SVC, BRK, system register, exclusive or +atomic instructions; with conditional branches, and loops, jump tables, SIMD and calls where the module has them. +Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each from random +states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into itself and random +data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when PC reaches the +sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are mapped +read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, NZCV, V0-V31 and the +scratch buffer. + +Every BL and BLR goes to one stub on both sides (rt_stub in harness.c, Runner.stub here) instead of its callee. The +stub folds the guest state at the call into a digest and sets the caller-saved registers from it, so the test sees +whether the emitted code had every register in the Cpu at the call and reads them back after it. Callers of another +picked function are dropped, as the C side would run that function instead of the stub. Usage: func.py [--exefs DIR] [--modules main,subsdk0,sdk] [--functions N] [--cases K] [--seed S] [--jobs J] """ @@ -38,11 +43,24 @@ STACK_TOP = BUF + 0x3F000 MIN_INSNS, MAX_INSNS = 6, 600 +FNV = 0x100000001B3 +M64 = (1 << 64) - 1 + + +def splitmix(z): + z = (z + 0x9E3779B97F4A7C15) & M64 + z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & M64 + z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M64 + return z ^ (z >> 31) + + +def call_mask(w): + return ((w & 0xFC000000) == 0x94000000) | ((w & 0xFFFFFC1F) == 0xD63F0000) # BL, BLR + + def forbidden_mask(w): """Words a picked function must not contain.""" - m = (w & 0xFC000000) == 0x94000000 # BL - m |= (w & 0xFFFFFC1F) == 0xD63F0000 # BLR - m |= (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS + m = (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS sysi = (w & 0xFFC00000) == 0xD5000000 # system: MRS, MSR, SYS (DC ZVA), hints, barriers hint_or_barrier = ((w & 0xFFFFF000) == 0xD5032000) | ((w & 0xFFFFF000) == 0xD5033000) m |= sysi & ~hint_or_barrier @@ -61,7 +79,7 @@ def simd_mask(w): def pick_functions(name, mod, count, r): - """[(entry, end, tags)] of up to count leaf functions of mod, plus the Emitter that will emit them.""" + """[(entry, end, tags)] of up to count functions of mod, plus the Emitter that will emit them.""" starts, _ = gen.discover(mod) w = mod.words ends = np.append(starts[1:], len(w)) @@ -73,15 +91,18 @@ def pick_functions(name, mod, count, r): size = ends - starts ok = (per_fn(forbidden_mask(w)) == 0) & (per_fn(cond_mask(w)) > 0) & (size >= MIN_INSNS) & (size <= MAX_INSNS) has_br = per_fn((w & 0xFFFFFC1F) == 0xD61F0000) > 0 + has_call = per_fn(call_mask(w)) > 0 cand = np.flatnonzero(ok) with_br = [int(i) for i in cand if has_br[i]] - rest = [int(i) for i in cand if not has_br[i]] - r.shuffle(with_br) - r.shuffle(rest) + with_call = [int(i) for i in cand if has_call[i] and not has_br[i]] + rest = [int(i) for i in cand if not has_br[i] and not has_call[i]] + for pool in (with_br, with_call, rest): + r.shuffle(pool) emitter = gen.Emitter(mod, set((mod.lo + starts * 4).tolist())) picks, tags_seen = [], Counter() - # Functions with a br first (jump tables, fallbacks), at most a quarter of the picks. - for pool, quota in ((with_br, count // 4), (rest, count)): + # Functions with a br first (jump tables, fallbacks), at most a quarter of the picks; then functions with calls, + # at most half. + for pool, quota in ((with_br, count // 4), (with_call, count // 2), (rest, count)): taken = 0 for i in pool: if len(picks) >= count or taken >= quota: @@ -91,13 +112,28 @@ def pick_functions(name, mod, count, r): if tags is None: continue picks.append((entry, end, tags)) - tags_seen.update(tags) taken += 1 - print(f'{name}: {len(starts)} functions, {len(cand)} leaf candidates, picked {len(picks)}: ' + picked = {e for e, _, _ in picks} + picks = [p for p in picks if not callees(emitter, p[0], p[1]) & picked] + for _, _, tags in picks: + tags_seen.update(tags) + print(f'{name}: {len(starts)} functions, {len(cand)} candidates, picked {len(picks)}: ' f'{dict(tags_seen)}', flush=True) return picks, emitter +def call_sites(mod, entry, end): + """Addresses of the BL and BLR instructions of [entry, end).""" + return [a for a in range(entry, end, 4) if mod.lift(a).kind in ('bl', 'blr')] + + +def callees(emitter, entry, end): + """Function starts the emitted C of [entry, end) calls directly (`f_(c)`, not through rt_dispatch).""" + mod = emitter.mod + return {mod.lift(a).target for a in call_sites(mod, entry, end) + if mod.lift(a).kind == 'bl' and mod.lift(a).target in emitter.starts} + + def check_function(emitter, entry, end): """Tags of a function the test can run in isolation, or None.""" mod = emitter.mod @@ -108,7 +144,7 @@ def check_function(emitter, entry, end): for k, ins in enumerate(insns): a = entry + 4 * k if ins.kind in ('bl', 'blr'): - return None + tags.add('calls') if ins.kind in ('b', 'bcond'): if not entry <= ins.target < end: return None @@ -147,9 +183,14 @@ def build(work, emitted, jobs, cflags): path.write_text(gen.chunk_text(name, j // per, emitter, fns[j:j + per])) srcs.append(path) table = work / 'table.c' + called = set() + for _, entry, end, emitter in emitted: + called |= callees(emitter, entry, end) table.write_text('#include "cpu.h"\n' + ''.join(f'void {gen.fname(e)}(Cpu *c);\n' for _, e, _, _ in emitted) + 'GuestFn tests[] = {' + ','.join(gen.fname(e) for _, e, _, _ in emitted) + '};\n' + - f'int ntests = {len(emitted)};\n') + f'int ntests = {len(emitted)};\n' + 'void rt_stub_call(Cpu *c, uint64_t target);\n' + + ''.join(f'void {gen.fname(t)}(Cpu *c) {{ rt_stub_call(c, {hex(t)}ull); }}\n' + for t in sorted(called))) srcs += [table, ROOT / 'tests' / 'diff' / 'harness.c'] objs, procs = [], [] for s in srcs: @@ -191,9 +232,12 @@ def make_case(r): class Runner: - """Unicorn with the module images read-only and a block hook that stops when execution leaves the function.""" + """Unicorn with the module images read-only, a block hook that stops when execution leaves the function, and a + code hook on every call site of the picked functions that stops before the call, for the stub.""" - def __init__(self, images, buf0): + MAX_CALLS = 1000 + + def __init__(self, images, buf0, sites): self.emu = Emu(images) mu = self.emu.mu for base, data in images: @@ -201,25 +245,70 @@ class Runner: mu.mem_map(SENTINEL, 0x1000) mu.mem_write(BUF, buf0) self.buf0 = buf0 - self.lo = self.hi = 0 + self.lo = self.hi = self.h = 0 self.left = False + self.at_call = None mu.hook_add(uc.UC_HOOK_BLOCK, self._block) + # One hook per call site, all added before anything is translated, so no cached block lacks one. + for a in sorted(sites): + mu.hook_add(uc.UC_HOOK_CODE, self._call, begin=a, end=a) def _block(self, mu, address, size, user): if not self.lo <= address < self.hi and address != SENTINEL: self.left = True mu.emu_stop() + def _call(self, mu, address, size, user): + if self.lo <= address < self.hi: + self.at_call = address + mu.emu_stop() + + def stub(self, a): + """harness.c rt_stub for the call at a, which Unicorn has not executed yet. Returns the address to resume at.""" + e, mu = self.emu, self.emu.mu + w = int.from_bytes(mu.mem_read(a, 4), 'little') + if (w & 0xFC000000) == 0x94000000: # BL + imm = w & 0x03FFFFFF + target = (a + ((imm - (1 << 26) if imm >> 25 else imm) << 2)) & M64 + else: # BLR + target = mu.reg_read(e.xregs[(w >> 5) & 31]) + mu.reg_write(A.UC_ARM64_REG_X30, a + 4) + h = ((self.h ^ target) * FNV) & M64 + for val in list(mu.reg_read_batch(e.xregs)) + [mu.reg_read(A.UC_ARM64_REG_SP)]: + h = ((h ^ val) * FNV) & M64 + for val in mu.reg_read_batch(e.qregs[:8]): + h = ((h ^ (val & M64)) * FNV) & M64 + h = ((h ^ (val >> 64)) * FNV) & M64 + self.h = h + regs = [] + for i in range(19): + s = splitmix((h + i) & M64) + regs.append((e.xregs[i], PTR_LO + ((s >> 4) % ((PTR_HI - PTR_LO) // 16)) * 16 if i < 8 and s & 3 else s)) + regs.append((A.UC_ARM64_REG_NZCV, splitmix((h + 19) & M64) >> 60 << 28)) + for i in range(8): + regs.append((e.qregs[i], splitmix((h + 33 + 2 * i) & M64) << 64 | splitmix((h + 32 + 2 * i) & M64))) + mu.reg_write_batch(regs) + top = BUF + BUF_SIZE - 16 + n = int.from_bytes(mu.mem_read(top, 8), 'little') + mu.mem_write(top, ((n + 1) & M64).to_bytes(8, 'little') + h.to_bytes(8, 'little')) + return a + 4 + def run(self, entry, end, case, steps): """(x, sp, nzcv, pc, v, crc of the buffer) or None if the case is skipped.""" fpcr, x, sp, nzcv, v = case e, mu = self.emu, self.emu.mu - self.lo, self.hi, self.left = entry, end, False + self.lo, self.hi, self.left, self.h = entry, end, False, 0 mu.reg_write_batch(list(zip(e.xregs, x[:31])) + list(zip(e.qregs, v)) + [(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)]) + pc, ok = entry, False try: - mu.emu_start(entry, SENTINEL, count=steps) - ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL + for _ in range(self.MAX_CALLS): + self.at_call = None + mu.emu_start(pc, SENTINEL, count=steps) + if self.at_call is None: + ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL + break + pc = self.stub(self.at_call) except uc.UcError: ok = False if not ok: @@ -268,7 +357,10 @@ def main(): print(f'compiled ({time.time() - t1:.0f}s)', flush=True) buf0 = make_buffer(r) - runner = Runner([(base, data) for base, _, data in images], buf0) + sites = set() + for _, entry, end, emitter in emitted: + sites.update(call_sites(emitter.mod, entry, end)) + runner = Runner([(base, data) for base, _, data in images], buf0, sites) cases, expect, tested, skipped = [], [], set(), 0 t2 = time.time() for k, (name, entry, end, _) in enumerate(emitted): @@ -286,7 +378,7 @@ def main(): flush=True) t3 = time.time() - recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images]) + recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images], stubs=True) print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)') if err: k = cases[len(recs)][0] if len(recs) < len(cases) else None diff --git a/tests/diff/harness.c b/tests/diff/harness.c index a24f4244..e268d543 100644 --- a/tests/diff/harness.c +++ b/tests/diff/harness.c @@ -1,5 +1,6 @@ -// Runs lifted single-instruction tests and writes the resulting CPU state per case. -// Shared by tests/diff/lift.py and tests/diff/simd.py; the expected state comes from Unicorn on the Python side. +// Runs lifted tests and writes the resulting CPU state per case. +// Shared by tests/diff/lift.py, simd.py, atomic.py and func.py; the expected state comes from Unicorn on the Python +// side. // // argv: buf.bin cases.bin out.bin image base [image base ...] // Each image is mapped at its guest address (guest == host), the scratch buffer at BUF. @@ -19,7 +20,48 @@ extern int ntests; #define BUF 0x7200000000ull #define BUF_SIZE 0x40000 -void rt_dispatch(Cpu *c) { (void)c; } +#define PTR_LO (BUF + 0x8000) +#define PTR_SPAN 0x20000 + +static int g_stubs; // In.stubs of the running case +static uint64_t g_stub_h; // digest of every call's state so far + +static uint64_t splitmix(uint64_t z) { + z += 0x9E3779B97F4A7C15ull; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +static uint64_t mix(uint64_t h, uint64_t w) { return (h ^ w) * 0x100000001B3ull; } + +// The callee of every call in a function under tests/diff/func.py; Runner.stub there is the same function on the +// Unicorn side. Folds the target, X0-X30, SP and V0-V7 into a digest, so a register the caller left unwritten at the +// call changes it, then sets X0-X18, NZCV and V0-V7 from the digest, so a caller reading a register it held across +// the call sees a value the callee could have left. X0-X7 mostly get pointers into the scratch buffer. The digest +// and the call count go to the top of the scratch buffer, which the memory check covers. +static void rt_stub(Cpu *c, uint64_t target) { + uint64_t h = mix(g_stub_h, target); + for (int i = 0; i < 31; i++) h = mix(h, c->x[i]); + h = mix(h, c->sp); + for (int i = 0; i < 8; i++) h = mix(mix(h, (uint64_t)c->v[i]), (uint64_t)(c->v[i] >> 64)); + g_stub_h = h; + for (int i = 0; i < 19; i++) { + uint64_t s = splitmix(h + i); + c->x[i] = i < 8 && (s & 3) ? PTR_LO + ((s >> 4) % (PTR_SPAN / 16)) * 16 : s; + } + c->nzcv = splitmix(h + 19) >> 60 << 28; + for (int i = 0; i < 8; i++) + c->v[i] = (__uint128_t)splitmix(h + 33 + 2 * i) << 64 | splitmix(h + 32 + 2 * i); + uint64_t *slot = (uint64_t *)(uintptr_t)(BUF + BUF_SIZE - 16); + slot[0]++; + slot[1] = h; +} + +void rt_dispatch(Cpu *c) { + if (g_stubs) rt_stub(c, c->pc); +} +void rt_stub_call(Cpu *c, uint64_t target) { rt_stub(c, target); } void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; } void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; abort(); } void rt_udf(Cpu *c, uint64_t pc) { (void)c; (void)pc; abort(); } @@ -61,7 +103,7 @@ static uint32_t crc32_buf(const uint8_t *p, size_t n) { } // Layouts match IN_FMT / OUT_FMT in tests/diff/common.py. -struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, pad; }; +struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, stubs; }; struct Out { uint64_t x[32], sp, nzcv, pc, mem; __uint128_t v[32]; }; int main(int argc, char **argv) { @@ -96,6 +138,8 @@ int main(int argc, char **argv) { c.fpcr = in.fpcr; memcpy(c.v, in.v, sizeof c.v); c.excl_addr = ~0ull; + g_stubs = (int)in.stubs; + g_stub_h = 0; tests[in.k](&c); memcpy(out.x, c.x, sizeof out.x); out.sp = c.sp; From 27056b212a114a03325185a5f1de33f0947e943c Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 18:07:58 +0300 Subject: [PATCH 2/4] Keep NZCV in a local of each generated function instead of c->nzcv Every flag-setting instruction packed N, Z, C and V into c->nzcv and every reader unpacked them. recomp/flags.py now runs a reaching-definitions pass per function: a compare stores its operands in a local Flags F, and a reader whose definitions are all of one kind computes its condition from them, so cmp and b.cond become one host compare. CCMP, FCMP, MSR NZCV and ADCS write the packed form, which readers reached by several kinds also use. Calls and returns publish nothing (AAPCS64 neither passes nor preserves NZCV); tail branches, falls into the next function and br through rt_dispatch publish c->nzcv, since discovery can start a function inside another one. Functions with an unresolved computed br keep flags in c->nzcv. 1.4.3 main: 492k direct and 89k packed flag reads, .text 342 -> 299 MiB. Fight bench (bench-fight2-2026-10-27b, frames 5400-7500, uncapped) 13.15 and 13.63 ms vs 13.9 before. tests/flags_check.py checks the new helpers against the packed ones; tests/diff/func.py no longer compares NZCV, which generated code leaves untouched. Co-Authored-By: Claude Opus 5.5 --- docs/building.md | 2 +- recomp/flags.py | 144 +++++++++++++++++++++++++++++++++++++++++++ recomp/gen.py | 117 ++++++++++++++++++++++++++++++----- runtime/cpu.h | 101 ++++++++++++++++++++++++++++++ tests/diff/common.py | 4 +- tests/diff/func.py | 8 ++- tests/flags_check.c | 111 +++++++++++++++++++++++++++++++++ tests/flags_check.py | 32 ++++++++++ 8 files changed, 498 insertions(+), 21 deletions(-) create mode 100644 recomp/flags.py create mode 100644 tests/flags_check.c create mode 100644 tests/flags_check.py diff --git a/docs/building.md b/docs/building.md index 4229ee24..645b82b5 100644 --- a/docs/building.md +++ b/docs/building.md @@ -138,7 +138,7 @@ change passes these, all run with `.venv`: | `tests\diff\lift.py` | instructions sampled from the extracted modules; `--coverage` lists every word the lifter rejects or traps | yes | | `tests\diff\simd.py` | the whole FP/SIMD encoding space | no | | `tests\diff\atomic.py` | exclusive load/store sequences and ARMv8.1 atomics | no | -| `tests\diff\func.py` | whole leaf functions as the generator emits them, loops and jump tables included | yes | +| `tests\diff\func.py` | whole functions as the generator emits them, loops, jump tables and calls included (calls go to a stub that checks the registers at the call) | yes | | `tests\excl_mt.py` | the exclusive monitor helpers under contention and ABA, on several threads | no | | `tests\jumptables_check.py` | `recomp/jumptables.py` against a Unicorn oracle; must report 0 disagreements, on 1.4.3 too | yes | diff --git a/recomp/flags.py b/recomp/flags.py new file mode 100644 index 00000000..61188f9d --- /dev/null +++ b/recomp/flags.py @@ -0,0 +1,144 @@ +"""Function-local NZCV for recomp/gen.py. + +The lifter writes flags the way the guest does: every flag-setting instruction packs N, Z, C and V into c->nzcv +(subs64() and friends in runtime/cpu.h) and every reader unpacks them (COND_xx(c)). Compilers keep NZCV inside a +function, and AAPCS64 neither passes it to a callee nor preserves it across a call. So a generated function keeps its +flags in a local `Flags F` (runtime/cpu.h), which clang holds in registers, and writes c->nzcv only where it leaves +for another function other than by a call or a return (see plan()). + +A flag-setting instruction of a known kind (subtract, add or logic result; 32 or 64 bits) stores its operands in F.a +and F.b. A reader all of whose reaching definitions are of one kind computes its condition from those operands, so +`cmp x1, x2; b.lt` becomes a signed compare in C. Instructions whose result is only known packed (CCMP, FCMP, MSR +NZCV, ADCS) write F.nz, the packed form; a reader whose definitions are of several kinds reads F.nz, and those +definitions then also write it. + +A read that can see flags a callee left reads the caller's flags from before the call instead. AAPCS64 code never +reads a callee's flags, so such a read only sits on a path that does not run: in TotK it follows calls to noreturn +functions (asserts) that the compiler placed before another branch's target. Plan.after_call counts them. +""" +import re + +KIND_DEF = re.compile(r'\b(subs|adds|logic)(32|64)\(c, ') +CARRY_DEF = re.compile(r'\b(adcs|sbcs)(32|64)\(c, ') +CARRY_USE = re.compile(r'\b(adc|sbc)(32|64)\(c, ') +COND = re.compile(r'COND_([A-Z]{2})\(c\)') +NZ_DEF = 'c->nzcv = ' +NZ_USE = '(c->nzcv)' +KIND_NAME = {'subs': 'sub', 'adds': 'add', 'logic': 'logic'} + +ENTRY, CALL = 'entry', 'call' + + +class FlagError(Exception): + pass + + +def effect(code): + """(reads flags, kind of the flags written or None) of one instruction's C.""" + reads = bool(COND.search(code) or CARRY_USE.search(code) or CARRY_DEF.search(code) or NZ_USE in code) + if NZ_DEF in code or CARRY_DEF.search(code): + return reads, 'nz' + defs = KIND_DEF.findall(code) + if len(defs) > 1: + raise FlagError(f'several flag definitions in one instruction: {code}') + if defs: + op, size = defs[0] + return reads, KIND_NAME[op] + size + return reads, None + + +class Plan: + """Per instruction: the kind its flag reads use, whether its flag definition also writes F.nz, and what it writes + to c->nzcv before it leaves for another function.""" + + def __init__(self, n): + self.reader = [None] * n + self.pack = [False] * n + self.exit = [None] * n # kind of the flags to publish in c->nzcv at a tail exit, or None + self.entry = False # F.nz starts as c->nzcv: flags from the entry reach a read or an exit + self.entry_read = False # they reach a read, not only an exit + self.after_call = 0 # reads that can see flags a callee left (see the module docstring) + self.used = False + + def decl(self): + """Declaration of F at the top of the function.""" + return 'Flags F = {.nz = c->nzcv};' if self.entry else 'Flags F = {0};' + + def publish(self, i): + """Statement writing the flags to c->nzcv at instruction i's tail exit ('' when it need not).""" + k = self.exit[i] + return f'c->nzcv = fnz_{k}(&F); ' if k else '' + + def rewrite(self, i, code): + """code of instruction i with its flag reads and writes on F.""" + rk = self.reader[i] + _, dk = effect(code) + if rk: + code = COND.sub(lambda m: f'fc_{rk}(&F, CC_{m.group(1)})', code) + code = CARRY_DEF.sub(lambda m: f'{m.group(1)}{m.group(2)}f(&F, fc_{rk}(&F, CC_CS), ', code) + code = CARRY_USE.sub(lambda m: f'{m.group(1)}{m.group(2)}f(fc_{rk}(&F, CC_CS), ', code) + code = code.replace(NZ_USE, f'(fnz_{rk}(&F))') + if dk == 'nz': + code = code.replace(NZ_DEF, 'F.nz = ') + code = re.sub(r'\(void\)(subs|adds)(32|64)\(c, ', lambda m: f'F.nz = nz_{KIND_NAME[m.group(1)]}{m.group(2)}(', + code) + elif dk: + code = KIND_DEF.sub(lambda m: f'f{KIND_NAME[m.group(1)]}{m.group(2)}(&F, ', code) + if self.pack[i]: + code += f' F.nz = fnz_{dk}(&F);' + if 'c->nzcv' in code or COND.search(code) or KIND_DEF.search(code) or CARRY_USE.search(code): + raise FlagError(f'flag access left after the rewrite: {code}') + return code + + +def plan(effects, succ, calls, exits): + """effects[i]: (reads, kind or None); succ[i]: successor indices inside the function; calls: indices of calls, + whose callee may leave any flags; exits: indices that can leave for another function other than by a call or a + return (a tail branch, falling into the next function, a br through rt_dispatch). + + Discovery can start a function inside another one (a branch target, a materialised address), and the code + before it then falls or branches into it with live flags. So a tail exit publishes the flags in c->nzcv, and a + function whose reads can see its entry flags starts F.nz from c->nzcv. Calls and returns publish nothing: AAPCS64 + code neither passes flags to a callee nor reads a callee's, so a call's flags count as no definition.""" + n = len(effects) + p = Plan(n) + if not any(r or k for r, k in effects): + return p + p.used = True + reach = [set() for _ in range(n)] + reach[0].add(ENTRY) + work = [0] + while work: + i = work.pop() + out = {i} if effects[i][1] else ({CALL} if i in calls else reach[i]) + for s in succ[i]: + if not out <= reach[s]: + reach[s] |= out + work.append(s) + + def kind(d): + return 'nz' if d == ENTRY else effects[d][1] + + def use(r): + """Kind a use of the definitions r reads; several kinds read the packed form, which they then all write.""" + kinds = {kind(d) for d in r} + if ENTRY in r: + p.entry = True + if len(kinds) == 1: + return kinds.pop() + for d in r: + if d != ENTRY: + p.pack[d] = True + return 'nz' # several kinds, or none (unreachable) + + for i, (reads, _) in enumerate(effects): + if reads: + p.after_call += CALL in reach[i] + p.reader[i] = use(reach[i] - {CALL}) + p.entry_read |= ENTRY in reach[i] + for i in exits: + # Entry flags alone are still in c->nzcv. + r = reach[i] - {CALL} + if r and r != {ENTRY}: + p.exit[i] = use(r) + return p diff --git a/recomp/gen.py b/recomp/gen.py index a1099c52..34da1f47 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -38,6 +38,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tools')) sys.path.insert(0, str(ROOT / 'recomp')) +import flags # noqa: E402 import jumptables # noqa: E402 import patches # noqa: E402 from check_content import GENERATED_MARKER # noqa: E402 @@ -246,6 +247,66 @@ class Emitter: raise ValueError(f'{entry + 4 * i:#x}: asm instruction from the portable lifter') return labels, tables + def replacements(self, a): + """Lifted replacement words of the patched site a.""" + return [self.mod.lifter.lift(w, a) for w in self.patched[a][1]] + + def flag_plan(self, insns, entry, end, tables): + """recomp/flags.py plan for the function: the intra-function control flow and each instruction's flag use. + Index len(insns) stands for falling through into the next function.""" + n = len(insns) + succ, calls, effects, exits = [], set(), [], {n} + + def idx(t): + return [(t - entry) // 4] if entry <= t < end else [] + + for i, ins in enumerate(insns): + k, a = ins.kind, entry + 4 * i + nxt = [i + 1] + if k == 'op': + succ.append(nxt) + elif k in ('bl', 'blr'): + succ.append(nxt) + calls.add(i) + elif k == 'b': + succ.append(idx(ins.target)) + if not idx(ins.target): + exits.add(i) + elif k == 'bcond': + succ.append(idx(ins.target) + nxt) + if not idx(ins.target): + exits.add(i) + elif k == 'br': + jt = tables.get(a) + succ.append([t for x in jt for t in idx(x)] if jt and jt != 'all' else []) + exits.add(i) # through rt_dispatch, also a table's default + else: # ret, trap + succ.append([]) + if a in self.patched: + effects.append(self.patched_effect(ins, a)) + if any(r.kind == 'bl' for r in self.replacements(a)): + calls.add(i) + else: + effects.append(flags.effect(ins.cond if k == 'bcond' else ins.code or '')) + succ.append([]) + effects.append((False, None)) + return flags.plan(effects, succ, calls, exits) + + def patched_effect(self, ins, a): + """Flag effect of a patched site: both alternatives must have the same one, in at most one word each.""" + alts = [] + for words in ([ins], self.replacements(a)): + fx = [flags.effect(w.code or '') for w in words if w.kind == 'op'] + fx = [e for e in fx if e[0] or e[1]] + if len(fx) > 1: + raise patches.PatchError(f'{a:#x}: several flag-using words in one alternative') + alts.append(fx[0] if fx else (False, None)) + if alts[0][1] != alts[1][1]: + raise patches.PatchError(f'{a:#x}: the alternatives write different flags ({alts[0][1]}, {alts[1][1]})') + if (alts[0][1] or alts[0][0] or alts[1][0]) and ins.kind == 'bl': + raise patches.PatchError(f'{a:#x}: a patched call that uses flags') + return alts[0][0] or alts[1][0], alts[0][1] + def function(self, entry, end): """(lines of C, referenced function starts) for the guest function [entry, end).""" mod, starts = self.mod, self.starts @@ -255,15 +316,32 @@ class Emitter: all_labels = 'all' in tables.values() if all_labels: labels = set(range(entry, end, 4)) + # An unresolved computed br re-enters the function through fb_, a C function of its own, which local + # flags would not survive; such functions keep NZCV in c->nzcv. + fp = None if all_labels else self.flag_plan(insns, entry, end, tables) + if fp and fp.used: + self.stats['local_flag_functions'] += 1 + self.stats['flag_reads_direct'] += sum(1 for k in fp.reader if k and k != 'nz') + self.stats['flag_reads_packed'] += sum(1 for k in fp.reader if k == 'nz') + self.stats['flag_entry_functions'] += fp.entry + self.stats['flag_entry_reads'] += fp.entry_read + self.stats['flag_reads_after_call'] += fp.after_call + self.stats['flag_publishing_exits'] += sum(1 for k in fp.exit if k) - def jump(t): + def fx(i, code): + return fp.rewrite(i, code) if fp and fp.used else code + + def publish(i): + return fp.publish(i) if fp and fp.used else '' + + def jump(t, i): if entry <= t < end: return f'goto L_{t:x};' if t in starts: refs.add(t) - return f'{MUSTTAIL} {fname(t)}(c);' + return f'{publish(i)}{MUSTTAIL} {fname(t)}(c);' self.stats['external_branches'] += 1 - return f'c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);' + return f'{publish(i)}c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);' if all_labels: # An unresolved computed br re-enters the function through fb_, which switches on c->pc to any @@ -280,6 +358,9 @@ class Emitter: else: redispatch = f'{MUSTTAIL} rt_dispatch(c);' out = [f'void {fname(entry)}(Cpu *c) {{', f' FN_ENTER({hx(entry)});'] + if fp.used: + out.append(' ' + fp.decl()) + def call(t, a): if t in starts: refs.add(t) @@ -287,21 +368,20 @@ class Emitter: self.stats['external_calls'] += 1 return f'c->x[30] = {hx(a + 4)}; c->pc = {hx(t)}; rt_dispatch(c);' - def patch(ins, a): + def patch(i, ins, a): """if (switch) { replacement } else { original } for a patched site (recomp/patches.py).""" k, words = self.patched[a] if ins.kind not in ('op', 'bl'): raise patches.PatchError(f'{a:#x}: patched instruction is a {ins.kind}') repl = [] - for j, w in enumerate(words): - r = mod.lifter.lift(w, a) + for j, (w, r) in enumerate(zip(words, self.replacements(a))): if r.kind == 'op': - repl.append(r.code) + repl.append(fx(i, r.code)) elif r.kind == 'bl' and j == len(words) - 1: repl.append(call(r.target, a)) else: raise patches.PatchError(f'{a:#x}: replacement word {w:08x} is a {r.kind}') - orig = ins.code if ins.kind == 'op' else call(ins.target, a) + orig = fx(i, ins.code) if ins.kind == 'op' else call(ins.target, a) self.uses_patches = True self.stats['patched_sites'] += 1 return f' if (rt_patch_on[{k}]) {{ {" ".join(repl)} }} else {{ {orig} }}' @@ -313,13 +393,13 @@ class Emitter: out.append(f'L_{a:x}:;') k = ins.kind if a in self.patched: - out.append(patch(ins, a)) + out.append(patch(i, ins, a)) elif k == 'op' or k == 'trap': - out.append(' ' + ins.code) + out.append(' ' + fx(i, ins.code)) elif k == 'b': - out.append(' ' + jump(ins.target)) + out.append(' ' + jump(ins.target, i)) elif k == 'bcond': - out.append(f' if ({ins.cond}) {{ {jump(ins.target)} }}') + out.append(f' if ({fx(i, ins.cond)}) {{ {jump(ins.target, i)} }}') elif k == 'bl': out.append(' ' + call(ins.target, a)) elif k == 'blr': @@ -331,16 +411,16 @@ class Emitter: elif jt: cases = ' '.join(f'case {hx(t)}: goto L_{t:x};' for t in jt) out.append(f' switch ({ins.target}) {{ {cases} default: break; }}') - out.append(f' c->pc = {ins.target}; {redispatch}') + out.append(f' {publish(i)}c->pc = {ins.target}; {redispatch}') else: - out.append(f' c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);') + out.append(f' {publish(i)}c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);') elif k == 'ret': out.append(' return;') else: raise ValueError(f'{a:#x}: unknown instruction kind {k!r}') if k not in TERMINATORS: # falls through into the next function if end < mod.hi: - out.append(' ' + jump(end)) + out.append(' ' + jump(end, len(insns))) else: out.append(f' rt_udf(c, {hx(end)});') out.append('}') @@ -562,6 +642,13 @@ def main(): 'br_dispatch': stats['br_dispatch'], 'traps': stats['traps'], 'patched_sites': stats['patched_sites'], + 'local_flag_functions': stats['local_flag_functions'], + 'flag_reads_direct': stats['flag_reads_direct'], + 'flag_reads_packed': stats['flag_reads_packed'], + 'flag_entry_functions': stats['flag_entry_functions'], + 'flag_entry_reads': stats['flag_entry_reads'], + 'flag_reads_after_call': stats['flag_reads_after_call'], + 'flag_publishing_exits': stats['flag_publishing_exits'], 'unsupported': stats['unsupported'], 'external_branches': stats['external_branches'], 'external_calls': stats['external_calls'], diff --git a/runtime/cpu.h b/runtime/cpu.h index 1246ee14..a787cbb0 100644 --- a/runtime/cpu.h +++ b/runtime/cpu.h @@ -215,6 +215,107 @@ static inline uint32_t logic32(Cpu *c, uint32_t r) { c->nzcv = MKFLAGS(r >> 31, ADC_IMPL(64, ) ADC_IMPL(32, ) +// ---- function-local flags (recomp/flags.py) +// Generated functions keep NZCV in a local Flags F, never in c->nzcv. A definition of a known kind stores its +// operands (a - b, a + b, or a logic result in a); fc_(&F, CC_xx) computes a condition from them, so a compare +// and its branch become one host compare. F.nz holds the packed form for definitions only known packed (CCMP, FCMP, +// MSR, ADCS) and for readers that several kinds of definition reach. Every helper is always inlined with a constant +// condition, so F stays in registers and only the condition's own arithmetic remains. +typedef struct { uint64_t a, b, nz; } Flags; +enum { CC_EQ, CC_NE, CC_CS, CC_CC, CC_MI, CC_PL, CC_VS, CC_VC, CC_HI, CC_LS, CC_GE, CC_LT, CC_GT, CC_LE }; +#define CPU_FL static inline __attribute__((always_inline)) + +CPU_FL int fl_cond(int cc, int n, int z, int cy, int v) { + switch (cc) { + case CC_EQ: return z; + case CC_NE: return !z; + case CC_CS: return cy; + case CC_CC: return !cy; + case CC_MI: return n; + case CC_PL: return !n; + case CC_VS: return v; + case CC_VC: return !v; + case CC_HI: return cy && !z; + case CC_LS: return !cy || z; + case CC_GE: return n == v; + case CC_LT: return n != v; + case CC_GT: return !z && n == v; + default: return z || n != v; + } +} + +// Subtract (SUBS, CMP): conditions as host compares of the operands. +#define FL_SUB(N, U, S) \ + CPU_FL U fsub##N(Flags *f, U a, U b) { f->a = a; f->b = b; return (U)(a - b); } \ + CPU_FL int fc_sub##N(const Flags *f, int cc) { \ + U a = (U)f->a, b = (U)f->b, r = (U)(a - b); \ + switch (cc) { \ + case CC_EQ: return a == b; \ + case CC_NE: return a != b; \ + case CC_CS: return a >= b; \ + case CC_CC: return a < b; \ + case CC_MI: return (S)r < 0; \ + case CC_PL: return (S)r >= 0; \ + case CC_VS: return (S)((a ^ b) & (a ^ r)) < 0; \ + case CC_VC: return (S)((a ^ b) & (a ^ r)) >= 0; \ + case CC_HI: return a > b; \ + case CC_LS: return a <= b; \ + case CC_GE: return (S)a >= (S)b; \ + case CC_LT: return (S)a < (S)b; \ + case CC_GT: return (S)a > (S)b; \ + default: return (S)a <= (S)b; \ + } \ + } \ + CPU_FL uint64_t nz_sub##N(U a, U b) { \ + U r = (U)(a - b); \ + return MKFLAGS((S)r < 0, r == 0, a >= b, (S)((a ^ b) & (a ^ r)) < 0); \ + } \ + CPU_FL uint64_t fnz_sub##N(const Flags *f) { return nz_sub##N((U)f->a, (U)f->b); } +FL_SUB(64, uint64_t, int64_t) +FL_SUB(32, uint32_t, int32_t) + +// Add (ADDS, CMN) and logic results (ANDS, TST, BICS): conditions from N, Z, C, V. +#define FL_ADD(N, U, S) \ + CPU_FL U fadd##N(Flags *f, U a, U b) { f->a = a; f->b = b; return (U)(a + b); } \ + CPU_FL uint64_t nz_add##N(U a, U b) { \ + U r = (U)(a + b); \ + return MKFLAGS((S)r < 0, r == 0, r < a, (S)((a ^ r) & (b ^ r)) < 0); \ + } \ + CPU_FL int fc_add##N(const Flags *f, int cc) { \ + U a = (U)f->a, b = (U)f->b, r = (U)(a + b); \ + return fl_cond(cc, (S)r < 0, r == 0, r < a, (S)((a ^ r) & (b ^ r)) < 0); \ + } \ + CPU_FL uint64_t fnz_add##N(const Flags *f) { return nz_add##N((U)f->a, (U)f->b); } \ + CPU_FL U flogic##N(Flags *f, U r) { f->a = r; return r; } \ + CPU_FL int fc_logic##N(const Flags *f, int cc) { \ + U r = (U)f->a; \ + return fl_cond(cc, (S)r < 0, r == 0, 0, 0); \ + } \ + CPU_FL uint64_t fnz_logic##N(const Flags *f) { return MKFLAGS((S)f->a < 0, (U)f->a == 0, 0, 0); } +FL_ADD(64, uint64_t, int64_t) +FL_ADD(32, uint32_t, int32_t) + +// Packed flags. +CPU_FL int fc_nz(const Flags *f, int cc) { + return fl_cond(cc, f->nz >> 31 & 1, f->nz >> 30 & 1, f->nz >> 29 & 1, f->nz >> 28 & 1); +} +CPU_FL uint64_t fnz_nz(const Flags *f) { return f->nz; } + +// ADC, SBC, ADCS, SBCS with the carry in from F. +#define FL_ADC(N, U) \ + CPU_FL U adc##N##f(int cin, U a, U b) { return (U)(a + b + (U)cin); } \ + CPU_FL U sbc##N##f(int cin, U a, U b) { return (U)(a - b - 1 + (U)cin); } \ + CPU_FL U adcs##N##f(Flags *f, int cin, U a, U b) { \ + U r = (U)(a + b + (U)cin); \ + int carry = cin ? r <= a : r < a; \ + f->nz = MKFLAGS(r >> (N - 1), r == 0, carry, ((a ^ r) & (b ^ r)) >> (N - 1)); \ + return r; \ + } \ + CPU_FL U sbcs##N##f(Flags *f, int cin, U a, U b) { return adcs##N##f(f, cin, a, (U)~b); } +FL_ADC(64, uint64_t) +FL_ADC(32, uint32_t) +#undef CPU_FL + // ---- bit ops #define ROR64(v, r) ((uint64_t)(v) >> ((r) & 63) | (uint64_t)(v) << ((64 - ((r) & 63)) & 63)) #define ROR32(v, r) ((uint32_t)(v) >> ((r) & 31) | (uint32_t)(v) << ((32 - ((r) & 31)) & 31)) diff --git a/tests/diff/common.py b/tests/diff/common.py index 73097311..648e280c 100644 --- a/tests/diff/common.py +++ b/tests/diff/common.py @@ -258,14 +258,14 @@ def run_harness(exe, work, buf0, cases, images, stubs=False): return recs, err -def diff_state(got, want, check_pc, check_mem): +def diff_state(got, want, check_pc, check_mem, check_nzcv=True): """got: harness record; want: (x, sp, nzcv, pc, v, mem). Returns a list of mismatches.""" hx, hsp, hnz, hpc, hmem, hv = got ex, esp, enz, epc, ev, emem = want d = [f'x{j}: got {hx[j]:x} want {ex[j]:x}' for j in range(31) if hx[j] != ex[j]] if hsp != esp: d.append(f'sp: got {hsp:x} want {esp:x}') - if hnz != enz: + if check_nzcv and hnz != enz: d.append(f'nzcv: got {hnz:x} want {enz:x}') if check_pc and hpc != epc: d.append(f'pc: got {hpc:x} want {epc:x}') diff --git a/tests/diff/func.py b/tests/diff/func.py index 817a6070..8e088564 100644 --- a/tests/diff/func.py +++ b/tests/diff/func.py @@ -7,8 +7,8 @@ Emits them with the generator's own Emitter, compiles them with tests/diff/harne states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into itself and random data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when PC reaches the sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are mapped -read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, NZCV, V0-V31 and the -scratch buffer. +read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, V0-V31 and the +scratch buffer; NZCV is local to a generated function (recomp/flags.py). Every BL and BLR goes to one stub on both sides (rt_stub in harness.c, Runner.stub here) instead of its callee. The stub folds the guest state at the call into a digest and sets the caller-saved registers from it, so the test sees @@ -387,7 +387,9 @@ def main(): fails, nfail = {}, 0 for case, want_state, got in zip(cases, expect, recs): - d = diff_state(got, want_state, check_pc=False, check_mem=True) + # Generated functions keep NZCV in a local (recomp/flags.py) and leave c->nzcv as it was: flags only count + # where the function reads them, which the registers and memory show. + d = diff_state(got, want_state, check_pc=False, check_mem=True, check_nzcv=False) if d: nfail += 1 fails.setdefault(case[0], d[:4]) diff --git a/tests/flags_check.c b/tests/flags_check.c new file mode 100644 index 00000000..9563925a --- /dev/null +++ b/tests/flags_check.c @@ -0,0 +1,111 @@ +// The function-local flag helpers of runtime/cpu.h (Flags, fc_*, fnz_*, nz_*, adc*f) against the packed helpers +// (subs64() and friends, COND_xx), which tests/diff/lift.py checks against Unicorn. Built and run by +// tests/flags_check.py; prints the first mismatches and exits 1 on any. +#include + +#include "cpu.h" + +static const uint64_t EDGE[] = {0, 1, 2, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000, + 0x80000001, 0xfffffffe, 0xffffffff, 0x100000000ull, 0x7fffffffffffffffull, + 0x8000000000000000ull, 0x8000000000000001ull, 0xfffffffffffffffeull, + 0xffffffffffffffffull, 0x123456789abcdef0ull, 0xfedcba9876543210ull}; +#define NEDGE (sizeof EDGE / sizeof EDGE[0]) + +static uint64_t rng = 0x9E3779B97F4A7C15ull; +static uint64_t next(void) { + rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17; + return rng; +} + +static int fails; +static void bad(const char *what, int cc, uint64_t a, uint64_t b, uint64_t got, uint64_t want) { + if (fails++ < 20) + printf("%s cc %d a %llx b %llx: got %llx want %llx\n", what, cc, (unsigned long long)a, (unsigned long long)b, + (unsigned long long)got, (unsigned long long)want); +} + +static int cond(const Cpu *c, int cc) { + switch (cc) { + case CC_EQ: return COND_EQ(c); + case CC_NE: return COND_NE(c); + case CC_CS: return COND_CS(c); + case CC_CC: return COND_CC(c); + case CC_MI: return COND_MI(c); + case CC_PL: return COND_PL(c); + case CC_VS: return COND_VS(c); + case CC_VC: return COND_VC(c); + case CC_HI: return COND_HI(c); + case CC_LS: return COND_LS(c); + case CC_GE: return COND_GE(c); + case CC_LT: return COND_LT(c); + case CC_GT: return COND_GT(c); + default: return COND_LE(c); + } +} + +// The condition argument is not constant here, so this also covers the switch default paths. +#define CHECK(NAME, OLD, NEW, FC, FNZ) \ + do { \ + Cpu c = {0}; \ + Flags f = {0}; \ + uint64_t ro = OLD, rn = NEW; \ + if (ro != rn) bad(NAME " result", -1, a, b, rn, ro); \ + for (int cc = 0; cc < 14; cc++) \ + if (!FC(&f, cc) != !cond(&c, cc)) bad(NAME, cc, a, b, FC(&f, cc), cond(&c, cc)); \ + f.nz = 0; \ + if (FNZ(&f) != c.nzcv) bad(NAME " packed", -1, a, b, FNZ(&f), c.nzcv); \ + Flags g = {.nz = c.nzcv}; \ + for (int cc = 0; cc < 14; cc++) \ + if (!fc_nz(&g, cc) != !cond(&c, cc)) bad(NAME " nz", cc, a, b, fc_nz(&g, cc), cond(&c, cc)); \ + } while (0) + +static void one(uint64_t a, uint64_t b) { + uint32_t a32 = (uint32_t)a, b32 = (uint32_t)b; + CHECK("sub64", subs64(&c, a, b), fsub64(&f, a, b), fc_sub64, fnz_sub64); + CHECK("sub32", subs32(&c, a32, b32), fsub32(&f, a32, b32), fc_sub32, fnz_sub32); + CHECK("add64", adds64(&c, a, b), fadd64(&f, a, b), fc_add64, fnz_add64); + CHECK("add32", adds32(&c, a32, b32), fadd32(&f, a32, b32), fc_add32, fnz_add32); + CHECK("logic64", logic64(&c, a & b), flogic64(&f, a & b), fc_logic64, fnz_logic64); + CHECK("logic32", logic32(&c, a32 & b32), flogic32(&f, a32 & b32), fc_logic32, fnz_logic32); + { + Cpu c = {0}; + if (nz_sub64(a, b) != (subs64(&c, a, b), c.nzcv)) bad("nz_sub64", -1, a, b, nz_sub64(a, b), c.nzcv); + if (nz_sub32(a32, b32) != (subs32(&c, a32, b32), c.nzcv)) bad("nz_sub32", -1, a, b, nz_sub32(a32, b32), c.nzcv); + if (nz_add64(a, b) != (adds64(&c, a, b), c.nzcv)) bad("nz_add64", -1, a, b, nz_add64(a, b), c.nzcv); + if (nz_add32(a32, b32) != (adds32(&c, a32, b32), c.nzcv)) bad("nz_add32", -1, a, b, nz_add32(a32, b32), c.nzcv); + } + for (int cin = 0; cin < 2; cin++) { + Cpu c = {.nzcv = (uint64_t)cin << 29}; + Flags f = {0}; + if (adc64f(cin, a, b) != adc64(&c, a, b)) bad("adc64", cin, a, b, adc64f(cin, a, b), adc64(&c, a, b)); + if (sbc64f(cin, a, b) != sbc64(&c, a, b)) bad("sbc64", cin, a, b, sbc64f(cin, a, b), sbc64(&c, a, b)); + if (adc32f(cin, a32, b32) != adc32(&c, a32, b32)) bad("adc32", cin, a, b, adc32f(cin, a32, b32), adc32(&c, a32, b32)); + if (sbc32f(cin, a32, b32) != sbc32(&c, a32, b32)) bad("sbc32", cin, a, b, sbc32f(cin, a32, b32), sbc32(&c, a32, b32)); + uint64_t r; +#define CARRY(NAME, NEW, OLD) \ + c.nzcv = (uint64_t)cin << 29; \ + r = NEW; \ + if (r != OLD || f.nz != c.nzcv) bad(NAME, cin, a, b, f.nz, c.nzcv); + CARRY("adcs64", adcs64f(&f, cin, a, b), adcs64(&c, a, b)) + CARRY("sbcs64", sbcs64f(&f, cin, a, b), sbcs64(&c, a, b)) + CARRY("adcs32", adcs32f(&f, cin, a32, b32), adcs32(&c, a32, b32)) + CARRY("sbcs32", sbcs32f(&f, cin, a32, b32), sbcs32(&c, a32, b32)) + } +} + +int main(void) { + long n = 0; + for (size_t i = 0; i < NEDGE; i++) + for (size_t j = 0; j < NEDGE; j++, n++) one(EDGE[i], EDGE[j]); + for (int k = 0; k < 2000000; k++, n++) { + uint64_t a = next(), b = next(); + switch (k & 3) { + case 1: b = a; break; // equal operands + case 2: b = a + (next() & 3) - 1; break; // neighbours + case 3: a &= 0xff, b &= 0xff; break; // small values + } + one(a, b); + } + printf("%ld operand pairs, %d mismatches\n", n, fails); + return fails != 0; +} diff --git a/tests/flags_check.py b/tests/flags_check.py new file mode 100644 index 00000000..9447ff74 --- /dev/null +++ b/tests/flags_check.py @@ -0,0 +1,32 @@ +"""Build and run tests/flags_check.c: runtime/cpu.h's function-local flag helpers against its packed flag helpers. + +Usage: flags_check.py +""" +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests' / 'diff')) +from common import MSYS_BIN, compiler # noqa: E402 + + +def main(): + work = ROOT / 'build' / 'test' / 'flags' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('flags_check.exe' if os.name == 'nt' else 'flags_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + for opt in ('-O0', '-O1'): + subprocess.check_call([compiler(), opt, '-fno-strict-aliasing', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'flags_check.c'), '-o', str(exe)], env=env) + print(opt, end=': ', flush=True) + if subprocess.run([str(exe)], env=env).returncode: + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From fae0faf6dc0211d1b7a670a4ada7b1745dc91064 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 18:08:11 +0300 Subject: [PATCH 3/4] List tests/flags_check.py among the lifter checks in docs/building.md Co-Authored-By: Claude Opus 5.5 --- docs/building.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/building.md b/docs/building.md index 645b82b5..757f60b7 100644 --- a/docs/building.md +++ b/docs/building.md @@ -139,6 +139,7 @@ change passes these, all run with `.venv`: | `tests\diff\simd.py` | the whole FP/SIMD encoding space | no | | `tests\diff\atomic.py` | exclusive load/store sequences and ARMv8.1 atomics | no | | `tests\diff\func.py` | whole functions as the generator emits them, loops, jump tables and calls included (calls go to a stub that checks the registers at the call) | yes | +| `tests\flags_check.py` | the function-local flag helpers of `runtime\cpu.h` against its packed ones | no | | `tests\excl_mt.py` | the exclusive monitor helpers under contention and ABA, on several threads | no | | `tests\jumptables_check.py` | `recomp/jumptables.py` against a Unicorn oracle; must report 0 disagreements, on 1.4.3 too | yes | From 0ca11dfba28a93f9784807f44897393cf186fc63 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Tue, 27 Oct 2026 22:21:18 +0300 Subject: [PATCH 4/4] Keep the six most used general registers of each generated function in C locals Every lifted instruction read and wrote c->x[n] and c->sp, so each register passed from one guest instruction to the next went through a store and a load, and clang could not keep it in a host register across a guest store. recomp/regs.py now copies the six most used X registers and SP of a function into locals at its entry. A forward pass over the function's flow tracks which locals changed since the last sync, and only those are stored back before a call, an SVC, a trap, a return or a tail exit. After an SVC every local reloads; after a call only X0-X18 and X30, since AAPCS64 has the callee preserve X19-X29 and SP. TotK does not break that: main and subsdk0 import neither longjmp nor the unwinder, sdk's have no direct callers, and the runtime writes no register above X3 except X30. gen.py --no-callee-saved-trust reloads them too, --reg-locals sets the count. Functions with an unresolved computed br keep their registers in the Cpu, as they keep NZCV. All registers in locals was slower than none: clang spills them to the stack and the syncs stay. Over 300 hot functions, 4-6 locals gave the fewest instructions and memory operands. 1.4.3 .text 299 -> 266 MiB. Fight bench (bench-fight2-2026-10-27b, frames 5400-7500, uncapped): 11.30 and 11.92 ms vs 13.5-13.9 on main; frames 2400-5300 11.24 and 11.36 vs 13.25. Without the callee-saved trust 12.58 ms. tests/diff/func.py: 0 failing on 1.4.3 (889 functions, 583 with calls) and 1.2.1 (810); a plan with no spills fails at once. Probes kernel, vkexec, nvn-tex and shader pass in both versions. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 5 +- recomp/gen.py | 139 ++++++++++++++++++++++++++++--------------- recomp/regs.py | 107 +++++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 48 deletions(-) create mode 100644 recomp/regs.py diff --git a/docs/architecture.md b/docs/architecture.md index 87a3d45b..1c7009c6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,7 +36,10 @@ MacNativeGames cannot run on x86-64 as is: its CPU path executes ARM64 code nati - jump tables resolved by our own symbolic pass instead of Unicorn, which keeps GPL code out of the shipped generator; - `LDXP`/`STXP` (128-bit exclusive pairs) supported; - no 64 MB limit on `main`; `subsdk*` loaded; - - guest registers kept in C locals where liveness allows, instead of memory. + - NZCV and the six most used general registers of each function kept in C locals instead of the `Cpu` + (`recomp/flags.py`, `recomp/regs.py`): registers are stored back where they changed, before calls, SVCs and + exits, and reloaded after them. A call reloads only X0-X18 and X30, as AAPCS64 has the callee preserve X19-X29 and + SP; `gen.py --no-callee-saved-trust` reloads those too. - Coverage oracle: MacNativeGames' NSO rewriter lists every site where game code touches the OS (SVCs, TLS reads, counter reads, `x18`, exclusive windows, jump tables): 68,968 sites in 1.4.3 `main`. The lifter must handle every one. - Memory: guest address equals host address, all below 2^39 (IPC encodes 39-bit addresses). The heap, stacks and the alias region are reserved with `MEM_WRITE_WATCH`, so the graphics back end can ask which pages the CPU wrote since the last upload (`runtime/memory.c`). - Indirect branches go through `rt_dispatch` (`runtime/dispatch.c`), which looks the target up in the address table and tail-calls it. diff --git a/recomp/gen.py b/recomp/gen.py index 34da1f47..69ffbfc3 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -41,6 +41,7 @@ sys.path.insert(0, str(ROOT / 'recomp')) import flags # noqa: E402 import jumptables # noqa: E402 import patches # noqa: E402 +import regs # noqa: E402 from check_content import GENERATED_MARKER # noqa: E402 from layout import layout, write as write_layout # noqa: E402 from lifter import hx # noqa: E402 @@ -206,10 +207,12 @@ def partition(est, max_fns, max_bytes, big_fn): class Emitter: """C for the functions of one module. starts is the set of every function start of the module.""" - def __init__(self, mod, starts, patched=None): + def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True): self.mod = mod self.starts = starts self.patched = patched or {} # {address: (set index, [replacement words])} (recomp/patches.py) + self.reg_locals = reg_locals # recomp/regs.py: registers per function in C locals + self.kept = regs.CALLEE_SAVED if callee_saved else frozenset() # not reloaded after a call self.uses_patches = False self.stats = Counter() self.unsupported = Counter() @@ -251,44 +254,48 @@ class Emitter: """Lifted replacement words of the patched site a.""" return [self.mod.lifter.lift(w, a) for w in self.patched[a][1]] - def flag_plan(self, insns, entry, end, tables): - """recomp/flags.py plan for the function: the intra-function control flow and each instruction's flag use. - Index len(insns) stands for falling through into the next function.""" - n = len(insns) - succ, calls, effects, exits = [], set(), [], {n} - + @staticmethod + def flow(insns, entry, end, tables): + """Successors of each instruction inside the function [entry, end). Index len(insns) stands for falling + through into the next function and has none.""" def idx(t): return [(t - entry) // 4] if entry <= t < end else [] + succ = [] for i, ins in enumerate(insns): - k, a = ins.kind, entry + 4 * i - nxt = [i + 1] - if k == 'op': - succ.append(nxt) - elif k in ('bl', 'blr'): - succ.append(nxt) - calls.add(i) + k = ins.kind + if k in ('op', 'bl', 'blr'): + succ.append([i + 1]) elif k == 'b': succ.append(idx(ins.target)) - if not idx(ins.target): - exits.add(i) elif k == 'bcond': - succ.append(idx(ins.target) + nxt) - if not idx(ins.target): - exits.add(i) + succ.append(idx(ins.target) + [i + 1]) elif k == 'br': - jt = tables.get(a) + jt = tables.get(entry + 4 * i) succ.append([t for x in jt for t in idx(x)] if jt and jt != 'all' else []) - exits.add(i) # through rt_dispatch, also a table's default else: # ret, trap succ.append([]) + succ.append([]) + return succ + + def flag_plan(self, insns, entry, end, tables, succ): + """recomp/flags.py plan for the function: each instruction's flag use over the flow succ (flow()).""" + n = len(insns) + calls, effects, exits = set(), [], {n} + for i, ins in enumerate(insns): + k, a = ins.kind, entry + 4 * i + if k in ('bl', 'blr'): + calls.add(i) + elif k in ('b', 'bcond') and not entry <= ins.target < end: + exits.add(i) + elif k == 'br': + exits.add(i) # through rt_dispatch, also a table's default if a in self.patched: effects.append(self.patched_effect(ins, a)) if any(r.kind == 'bl' for r in self.replacements(a)): calls.add(i) else: effects.append(flags.effect(ins.cond if k == 'bcond' else ins.code or '')) - succ.append([]) effects.append((False, None)) return flags.plan(effects, succ, calls, exits) @@ -318,7 +325,11 @@ class Emitter: labels = set(range(entry, end, 4)) # An unresolved computed br re-enters the function through fb_, a C function of its own, which local # flags would not survive; such functions keep NZCV in c->nzcv. - fp = None if all_labels else self.flag_plan(insns, entry, end, tables) + # The same holds for registers in locals (recomp/regs.py): fb_ keeps them in the Cpu too. + local = not all_labels + spill, reload, reload_call = (regs.SPILL, regs.RELOAD, regs.RELOAD_CALL) if local else ('', '', '') + succ = self.flow(insns, entry, end, tables) if local else None + fp = self.flag_plan(insns, entry, end, tables, succ) if local else None if fp and fp.used: self.stats['local_flag_functions'] += 1 self.stats['flag_reads_direct'] += sum(1 for k in fp.reader if k and k != 'nz') @@ -339,9 +350,9 @@ class Emitter: return f'goto L_{t:x};' if t in starts: refs.add(t) - return f'{publish(i)}{MUSTTAIL} {fname(t)}(c);' + return f'{publish(i)}{spill}{MUSTTAIL} {fname(t)}(c);' self.stats['external_branches'] += 1 - return f'{publish(i)}c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);' + return f'{publish(i)}{spill}c->pc = {hx(t)}; {MUSTTAIL} rt_dispatch(c);' if all_labels: # An unresolved computed br re-enters the function through fb_, which switches on c->pc to any @@ -364,9 +375,13 @@ class Emitter: def call(t, a): if t in starts: refs.add(t) - return f'c->x[30] = {hx(a + 4)}; {fname(t)}(c);' + return f'c->x[30] = {hx(a + 4)}; {spill}{fname(t)}(c);{reload_call}' self.stats['external_calls'] += 1 - return f'c->x[30] = {hx(a + 4)}; c->pc = {hx(t)}; rt_dispatch(c);' + return f'c->x[30] = {hx(a + 4)}; {spill}c->pc = {hx(t)}; rt_dispatch(c);{reload_call}' + + def op(i, code): + code = fx(i, code) + return f'{spill}{code}{reload}' if regs.ESCAPE.search(code) else code def patch(i, ins, a): """if (switch) { replacement } else { original } for a patched site (recomp/patches.py).""" @@ -376,53 +391,76 @@ class Emitter: repl = [] for j, (w, r) in enumerate(zip(words, self.replacements(a))): if r.kind == 'op': - repl.append(fx(i, r.code)) + repl.append(op(i, r.code)) elif r.kind == 'bl' and j == len(words) - 1: repl.append(call(r.target, a)) else: raise patches.PatchError(f'{a:#x}: replacement word {w:08x} is a {r.kind}') - orig = fx(i, ins.code) if ins.kind == 'op' else call(ins.target, a) + orig = op(i, ins.code) if ins.kind == 'op' else call(ins.target, a) self.uses_patches = True self.stats['patched_sites'] += 1 return f' if (rt_patch_on[{k}]) {{ {" ".join(repl)} }} else {{ {orig} }}' + body = [] # (instruction index or None for a label, line); a line belongs to the instruction it lifts + + def emit(i, line): + body.append((i, line)) + k = None for i, ins in enumerate(insns): a = entry + 4 * i if a in labels: - out.append(f'L_{a:x}:;') + emit(None, f'L_{a:x}:;') k = ins.kind if a in self.patched: - out.append(patch(i, ins, a)) - elif k == 'op' or k == 'trap': - out.append(' ' + fx(i, ins.code)) + emit(i, patch(i, ins, a)) + elif k == 'op': + emit(i, ' ' + op(i, ins.code)) + elif k == 'trap': + emit(i, f' {spill}{fx(i, ins.code)}') elif k == 'b': - out.append(' ' + jump(ins.target, i)) + emit(i, ' ' + jump(ins.target, i)) elif k == 'bcond': - out.append(f' if ({fx(i, ins.cond)}) {{ {jump(ins.target, i)} }}') + emit(i, f' if ({fx(i, ins.cond)}) {{ {jump(ins.target, i)} }}') elif k == 'bl': - out.append(' ' + call(ins.target, a)) + emit(i, ' ' + call(ins.target, a)) elif k == 'blr': - out.append(f' c->pc = {ins.target}; c->x[30] = {hx(a + 4)}; rt_dispatch(c);') + emit(i, f' c->pc = {ins.target}; c->x[30] = {hx(a + 4)}; {spill}rt_dispatch(c);{reload_call}') elif k == 'br': jt = tables.get(a) if jt == 'all': - out.append(f' c->pc = {ins.target}; {redispatch}') + emit(i, f' c->pc = {ins.target}; {redispatch}') elif jt: cases = ' '.join(f'case {hx(t)}: goto L_{t:x};' for t in jt) - out.append(f' switch ({ins.target}) {{ {cases} default: break; }}') - out.append(f' {publish(i)}c->pc = {ins.target}; {redispatch}') + emit(i, f' switch ({ins.target}) {{ {cases} default: break; }}') + emit(i, f' {publish(i)}{spill}c->pc = {ins.target}; {redispatch}') else: - out.append(f' {publish(i)}c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);') + emit(i, f' {publish(i)}{spill}c->pc = {ins.target}; {MUSTTAIL} rt_dispatch(c);') elif k == 'ret': - out.append(' return;') + emit(i, f' {spill}return;') else: raise ValueError(f'{a:#x}: unknown instruction kind {k!r}') + n = len(insns) if k not in TERMINATORS: # falls through into the next function if end < mod.hi: - out.append(' ' + jump(end, len(insns))) + emit(n, ' ' + jump(end, n)) else: - out.append(f' rt_udf(c, {hx(end)});') + emit(n, f' {spill}rt_udf(c, {hx(end)});') + if local: + texts = [''] * (n + 1) + for i, line in body: + if i is not None: + texts[i] += line + clean = {i for i, ins in enumerate(insns) if entry + 4 * i not in self.patched + and (ins.kind in ('bl', 'blr') or (ins.kind == 'op' and regs.RELOAD in texts[i]))} + rp = regs.plan(texts, succ, clean, self.reg_locals, self.kept) + if rp.regs: + out.append(' ' + rp.decl()) + self.stats['local_reg_functions'] += bool(rp.regs) + self.stats['reg_spills'] += sum(len(s) for s in rp.spill) + out += [line if i is None else rp.resolve(i, line) for i, line in body] + else: + out += [line for _, line in body] out.append('}') self.stats['functions'] += 1 self.stats['instructions'] += len(insns) @@ -481,10 +519,11 @@ def _retry(fn, *args): _W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter} -def _worker_init(mods, starts, table): +def _worker_init(mods, starts, table, opts): _W['mods'] = mods _W['starts'] = starts _W['patches'] = table + _W['opts'] = opts # Emitter keyword arguments _W['emitters'] = {} @@ -494,7 +533,7 @@ def _emitter(name): path, base = _W['mods'][name] mod = Module(name, path, base, 'c') e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()), - _W['patches'].for_module(name, base)) + _W['patches'].for_module(name, base), **_W['opts']) return e @@ -556,7 +595,13 @@ def main(): ap.add_argument('--big-fn', type=int, default=300000, help='estimated bytes of C from which a function gets ' 'its own chunk') ap.add_argument('--patches', type=Path, help='code patch table (recomp/patches.py)') + ap.add_argument('--reg-locals', type=int, default=regs.LOCALS, + help=f'guest registers per function kept in C locals (recomp/regs.py; default {regs.LOCALS}, 0: none)') + ap.add_argument('--no-callee-saved-trust', action='store_true', + help='reload X19-X29 and SP after every call too, instead of trusting AAPCS64 (slower; for ' + 'telling a broken callee from a generator bug)') args = ap.parse_args() + opts = {'reg_locals': args.reg_locals, 'callee_saved': not args.no_callee_saved_trust} t0 = time.perf_counter() mods = layout(args.exefs) @@ -602,7 +647,7 @@ def main(): done_at, errors = {}, [] t_emit = time.perf_counter() ctx = mp.get_context('spawn') - with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table)) as pool: + with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool: for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)): results[r['module']].append(r) if r['error']: diff --git a/recomp/regs.py b/recomp/regs.py new file mode 100644 index 00000000..c583faa8 --- /dev/null +++ b/recomp/regs.py @@ -0,0 +1,107 @@ +"""Function-local general registers for recomp/gen.py. + +The lifter reads and writes the guest's X0-X30 and SP in the Cpu (c->x[n], c->sp), so every guest instruction loads +and stores memory, and the C compiler cannot keep a register in a host register across a guest store: guest stores go +through pointers that may alias the Cpu. A generated function instead copies the LOCALS registers it uses most into +locals at its entry (xN, sp_), works on those, and syncs with the Cpu only where something else reads or writes them: +before a call, an SVC, a trap, a return or a tail exit it stores the registers it changed since the last sync (spill), +and after an SVC it loads them all again (reload). The other registers stay in the Cpu. + +After a call it reloads only X0-X18 and X30: AAPCS64 has the callee preserve X19-X29 and SP, so their locals still +hold what the Cpu holds. What would break that does not occur in TotK: main and subsdk0 import neither longjmp nor +the C++ unwinder, sdk's longjmp, __cxa_throw and _Unwind_* have no direct callers, and the runtime writes no register +above X3 of a running thread's Cpu except X30, which rt_call restores (runtime/hostfn.c). Guest fibers each run on a host fiber of their own with their own Cpu (runtime/kernel.c), +and ExitThread leaves by longjmp without returning. gen.py --no-callee-saved-trust reloads those registers too. + +Only plain assignments write registers in the lifted C (`c->x[n] = ...`, `c->sp = ...`), and no code takes a +register's address, so the rewrite is textual. A register counts as dirty from a write until the next spill on some +path: plan() runs that forward over the function's control flow, so a call spills only what changed since the +previous call or the entry. Spilling a register that is not dirty is harmless (the local equals the Cpu), so where a +case is unclear the plan errs on spilling more. +""" +import re +from collections import Counter + +REG = re.compile(r'c->x\[(\d+)\]|c->sp\b') +WRITE = re.compile(r'c->(?:x\[(\d+)\]|(sp))\s*=(?!=)') +# Lifted instructions that hand the Cpu to the runtime: SVCs return, BRK and undefined words do not. +ESCAPE = re.compile(r'\brt_(?:svc|brk|udf)\(c\b') +SP = 31 # index of SP in the register sets +# Markers gen.py puts where the Cpu must be current, resolved by Plan.resolve(): spill before; reload all, or reload +# what a call may change, after. +SPILL, RELOAD, RELOAD_CALL = '\x00S', '\x00R', '\x00C' +CALLEE_SAVED = frozenset(range(19, 30)) | {SP} +# Locals per function: the most used registers. More cost more than they save, as clang then spills them to the stack +# and the syncs stay (300 hot TotK functions: fewest instructions and memory operands at 4-6). +LOCALS = 6 + + +def name(r): + return 'sp_' if r == SP else f'x{r}' + + +def cpu(r): + return 'c->sp' if r == SP else f'c->x[{r}]' + + +def refs(text): + """Registers text reads or writes, once per occurrence.""" + return [SP if m.group(0) == 'c->sp' else int(m.group(1)) for m in REG.finditer(text)] + + +def written(text): + return {SP if m.group(2) else int(m.group(1)) for m in WRITE.finditer(text)} + + +class Plan: + def __init__(self, regs, spill, kept): + self.regs = regs # sorted registers the function keeps in locals + self.local = set(regs) + self.spill = spill # per instruction: registers to store at its sync points + self.kept = kept # registers a call leaves as they were (CALLEE_SAVED, or none) + + def decl(self): + """Locals at the top of the function, loaded from the Cpu.""" + return 'uint64_t ' + ', '.join(f'{name(r)} = {cpu(r)}' for r in self.regs) + ';' if self.regs else '' + + def resolve(self, i, text): + """text of instruction i with its registers on the locals and its sync markers expanded.""" + def local(m): + r = SP if m.group(0) == 'c->sp' else int(m.group(1)) + return name(r) if r in self.local else m.group(0) + + text = REG.sub(local, text) + spill = ''.join(f'{cpu(r)} = {name(r)}; ' for r in sorted(self.spill[i])) + reload = ''.join(f' {name(r)} = {cpu(r)};' for r in self.regs) + reload_call = ''.join(f' {name(r)} = {cpu(r)};' for r in self.regs if r not in self.kept) + return text.replace(SPILL, spill).replace(RELOAD, reload).replace(RELOAD_CALL, reload_call) + + +def plan(texts, succ, clean, n_locals=LOCALS, kept=CALLEE_SAVED): + """texts[i]: instruction i's C with markers; succ[i]: successor indices inside the function; clean: instructions + after which every local equals the Cpu (a call or an SVC that spills and reloads on every path through it); + n_locals: how many registers to keep in locals; kept: registers a call does not reload.""" + n = len(texts) + uses = Counter(r for t in texts for r in refs(t)) + regs = {r for r, _ in sorted(uses.items(), key=lambda kv: (-kv[1], kv[0]))[:n_locals]} + writes = [written(t) & regs for t in texts] + dirty = [set() for _ in range(n)] + seen = [False] * n + work = [0] if n else [] + if n: + seen[0] = True + while work: + i = work.pop() + out = set() if i in clean else dirty[i] | writes[i] + for s in succ[i]: + if not seen[s] or not out <= dirty[s]: + seen[s] = True + dirty[s] |= out + work.append(s) + every = set().union(*writes) if writes else set() + for i in range(n): + if not seen[i]: # not reached from the entry in the plan's flow: anything may be dirty + dirty[i] = every + # A sync point stores what was dirty on entry to its instruction and what the instruction wrote before it + # (c->x[30] of a call): all the instruction's writes, which is at most a little more. + return Plan(sorted(regs), [dirty[i] | writes[i] if SPILL in texts[i] else set() for i in range(n)], kept)