mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-21 21:05:56 +00:00
Add the recompiler, Windows runtime, tools and docs
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3d73290520
commit
19c9b648d1
558 files changed
+119174
-10
No files matched your search
@@ -0,0 +1,221 @@
|
||||
"""Differential test of exclusives and atomics against Unicorn (max CPU model, which has the ARMv8.1 LSE atomics).
|
||||
|
||||
Single steps cannot cover these: a store exclusive only succeeds after a load exclusive, and the A57 model rejects
|
||||
LSE. So this test enumerates every variant and runs short sequences, with the monitor cleared before each case:
|
||||
|
||||
- LDXR/LDAXR then STXR/STLXR to the same address (succeeds), all sizes;
|
||||
- LDXP/LDAXP then STXP/STLXP to the same address (succeeds) or 16 bytes further (fails), W and X pairs;
|
||||
- STXR and STXP alone (fail without a monitor);
|
||||
- LDADD, LDCLR, LDEOR, LDSET, LDSMAX, LDSMIN, LDUMAX, LDUMIN and SWP, every size and ordering;
|
||||
- CAS and CASP, every size and ordering, with the compare value equal to memory in half of the cases.
|
||||
|
||||
Needs no game files. Usage: atomic.py [--cases K] [--seed S] [--jobs J]
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
from common import BUF, BUF_SIZE, ROOT, Emu, build_harness, diff_state, dis, rand_fpcr, rand_v, rand_x, run_harness
|
||||
from lifter import Lifter
|
||||
|
||||
BUF_MID = BUF + 0x10000
|
||||
CODE = 0x7220000000
|
||||
SLOT = 16 # bytes of code per test
|
||||
CLREX = 0xD5033F5F
|
||||
|
||||
|
||||
def lse(size, a, r, rs, o3, opc, rn, rt):
|
||||
return 0x38200000 | size << 30 | a << 23 | r << 22 | rs << 16 | o3 << 15 | opc << 12 | rn << 5 | rt
|
||||
|
||||
|
||||
def cas(size, l, rs, o0, rn, rt):
|
||||
return 0x08A07C00 | size << 30 | l << 22 | rs << 16 | o0 << 15 | rn << 5 | rt
|
||||
|
||||
|
||||
def casp(sz, l, rs, o0, rn, rt):
|
||||
return 0x08207C00 | sz << 30 | l << 22 | rs << 16 | o0 << 15 | rn << 5 | rt
|
||||
|
||||
|
||||
def ldxp(sz, o0, rt2, rn, rt):
|
||||
return 0x887F0000 | sz << 30 | o0 << 15 | rt2 << 10 | rn << 5 | rt
|
||||
|
||||
|
||||
def stxp(sz, o0, rs, rt2, rn, rt):
|
||||
return 0x88200000 | sz << 30 | rs << 16 | o0 << 15 | rt2 << 10 | rn << 5 | rt
|
||||
|
||||
|
||||
def ldxr(size, o0, rn, rt):
|
||||
return 0x085F7C00 | size << 30 | o0 << 15 | rn << 5 | rt
|
||||
|
||||
|
||||
def stxr(size, o0, rs, rn, rt):
|
||||
return 0x08007C00 | size << 30 | rs << 16 | o0 << 15 | rn << 5 | rt
|
||||
|
||||
|
||||
def regs(r, n, exclude=()):
|
||||
"""n distinct general registers 0-30 outside exclude."""
|
||||
return r.sample([i for i in range(31) if i not in exclude], n)
|
||||
|
||||
|
||||
def tests(r):
|
||||
"""[(words, rn, kind)]: kind 'cas'/'casp' marks compare-and-swap tests (state gets a matching compare value)."""
|
||||
out = []
|
||||
for size in range(4):
|
||||
for a in (0, 1):
|
||||
for rel in (0, 1):
|
||||
for o3, opc in [(0, k) for k in range(8)] + [(1, 0)]:
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rs, rt = r.randrange(32), r.randrange(32)
|
||||
out.append(([lse(size, a, rel, rs, o3, opc, rn, rt)], rn, 'lse'))
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rs, rt = regs(r, 2, {rn})
|
||||
out.append(([cas(size, a, rs, rel, rn, rt)], rn, 'cas'))
|
||||
for sz in (0, 1):
|
||||
for l in (0, 1):
|
||||
for o0 in (0, 1):
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rs, rt = (2 * i for i in r.sample(range(15), 2))
|
||||
if rn in (rs, rs + 1):
|
||||
rs = 30 if rn < 30 else 0 # Rs+1 = XZR or away from Rn
|
||||
out.append(([casp(sz, l, rs, o0, rn, rt)], rn, 'casp'))
|
||||
for size in range(4):
|
||||
for o0 in (0, 1):
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rt, rs, rt_st = regs(r, 3, {rn})
|
||||
out.append(([ldxr(size, o0, rn, rt), stxr(size, o0, rs, rn, rt_st)], rn, 'excl'))
|
||||
out.append(([stxr(size, o0, rs, rn, rt_st)], rn, 'excl'))
|
||||
for sz in (0, 1):
|
||||
for o0 in (0, 1):
|
||||
rn = r.choice(regs(r, 1) + [31])
|
||||
rt, rt2, rs, d1, d2 = regs(r, 5, {rn})
|
||||
out.append(([ldxp(sz, o0, rt2, rn, rt)], rn, 'excl'))
|
||||
out.append(([ldxp(sz, o0, rt2, rn, rt), stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
|
||||
out.append(([stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
|
||||
# A store exclusive to another address fails; Rn is advanced between the two by an ADD.
|
||||
if rn != 31:
|
||||
add = 0x91004000 | rn << 5 | rn # add xn, xn, #16
|
||||
out.append(([ldxp(sz, o0, rt2, rn, rt), add, stxp(sz, o0, rs, d2, rn, d1)], rn, 'excl'))
|
||||
return out
|
||||
|
||||
|
||||
def unicorn_wrong(w):
|
||||
"""LDSMAX/LDSMIN on bytes, halfwords and words: Unicorn 2.1 (QEMU 5.0) compares them unsigned."""
|
||||
return (w & 0x3F200C00) == 0x38200000 and (w >> 30) < 3 and not (w >> 15) & 1 and (w >> 12) & 7 in (4, 5)
|
||||
|
||||
|
||||
def model_signed_minmax(w, x, sp, nzcv, v, buf0):
|
||||
"""Reference for unicorn_wrong(w) words, from the ARM pseudocode: returns the expected state and memory CRC."""
|
||||
nb = 1 << (w >> 30)
|
||||
rs, rn, rt = (w >> 16) & 31, (w >> 5) & 31, w & 31
|
||||
addr = sp if rn == 31 else x[rn]
|
||||
off = addr - BUF
|
||||
old = int.from_bytes(buf0[off:off + nb], 'little')
|
||||
operand = (x[rs] if rs != 31 else 0) & ((1 << 8 * nb) - 1)
|
||||
sign = lambda u: u - (1 << 8 * nb) if u >> (8 * nb - 1) else u
|
||||
pick = max if (w >> 12) & 7 == 4 else min
|
||||
new = old if pick(sign(old), sign(operand)) == sign(old) else operand
|
||||
mem = bytearray(buf0)
|
||||
mem[off:off + nb] = new.to_bytes(nb, 'little')
|
||||
ox = list(x[:31])
|
||||
if rt != 31:
|
||||
ox[rt] = old
|
||||
return (ox, sp, nzcv, 0, list(v), zlib.crc32(bytes(mem)))
|
||||
|
||||
|
||||
def make_case(r, words, rn, kind, buf0):
|
||||
addr = BUF_MID + r.randrange(0, 0x100, 16)
|
||||
x = [rand_x(r) for _ in range(31)]
|
||||
sp = addr if rn == 31 else BUF_MID + r.randrange(0, 0x100, 16)
|
||||
if rn != 31:
|
||||
x[rn] = addr
|
||||
w = words[0]
|
||||
if kind in ('cas', 'casp') and r.random() < 0.5:
|
||||
off = addr - BUF
|
||||
rs = (w >> 16) & 31
|
||||
if kind == 'cas':
|
||||
nb = 1 << (w >> 30)
|
||||
if rs != 31:
|
||||
x[rs] = int.from_bytes(buf0[off:off + nb], 'little')
|
||||
else:
|
||||
nb = 8 if (w >> 30) & 1 else 4
|
||||
for i in (0, 1):
|
||||
if rs + i != 31:
|
||||
x[rs + i] = int.from_bytes(buf0[off + i * nb:off + (i + 1) * nb], 'little')
|
||||
return rand_fpcr(r), x, sp, r.getrandbits(4) << 28, [rand_v(r) for _ in range(32)]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--cases', type=int, default=64)
|
||||
ap.add_argument('--seed', type=int, default=1)
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
|
||||
args = ap.parse_args()
|
||||
r = random.Random(args.seed)
|
||||
t0 = time.time()
|
||||
|
||||
ts = tests(r)
|
||||
code = [CLREX, 0, 0, 0]
|
||||
bodies = []
|
||||
lifter = Lifter(0, 'c')
|
||||
for i, (words, _, _) in enumerate(ts):
|
||||
pc = CODE + SLOT * (i + 1)
|
||||
code += words + [0] * (SLOT // 4 - len(words))
|
||||
bodies.append(' '.join(lifter.lift(w, pc + 4 * j).code for j, w in enumerate(words)))
|
||||
code_bytes = struct.pack(f'<{len(code)}I', *code)
|
||||
work = ROOT / 'build' / 'test' / 'atomic'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
code_path = work / 'code.bin'
|
||||
code_path.write_bytes(code_bytes)
|
||||
exe = build_harness(work, bodies, args.jobs, args.cflags)
|
||||
|
||||
emu = Emu([(CODE, code_bytes)], A.UC_CPU_ARM64_MAX)
|
||||
mu = emu.mu
|
||||
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases, expect = [], []
|
||||
for k, (words, rn, kind) in enumerate(ts):
|
||||
pc = CODE + SLOT * (k + 1)
|
||||
for _ in range(args.cases):
|
||||
fpcr, x, sp, nzcv, v = make_case(r, words, rn, kind, buf0)
|
||||
if unicorn_wrong(words[0]):
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, True))
|
||||
expect.append(model_signed_minmax(words[0], x, sp, nzcv, v, buf0))
|
||||
continue
|
||||
mu.emu_start(CODE, CODE + 4, count=1) # clear the exclusive monitor
|
||||
try:
|
||||
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v, count=len(words))
|
||||
except uc.UcError as e:
|
||||
raise SystemExit(f'unicorn rejected {" ; ".join(dis(w) for w in words)}: {e}')
|
||||
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, True))
|
||||
expect.append((ox, osp, onz, opc, ov, mh))
|
||||
|
||||
recs, err = run_harness(exe, work, buf0, cases, [(CODE, code_path)])
|
||||
if err:
|
||||
k = cases[len(recs)][0] if len(recs) < len(cases) else None
|
||||
print(f'harness failed: {err}' + (f', stopped at {" ; ".join(dis(w) for w in ts[k][0])}' if k is not None else ''))
|
||||
fails, nfail = {}, 0
|
||||
for case, want, got in zip(cases, expect, recs):
|
||||
d = diff_state(got, want, check_pc=False, check_mem=True)
|
||||
if d:
|
||||
nfail += 1
|
||||
fails.setdefault(case[0], d[:4])
|
||||
print(f'{len(ts)} tests, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing tests '
|
||||
f'({time.time() - t0:.0f}s)')
|
||||
for k, d in fails.items():
|
||||
print(f' {" ; ".join(dis(w) for w in ts[k][0]):70} {"; ".join(d)}')
|
||||
return 1 if fails or err else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,275 @@
|
||||
"""Shared parts of the differential tests: Unicorn reference, operand generators, build and run of harness.c.
|
||||
|
||||
Adapted from LAS Recomp tools/test_lift.py and tools/test_simd.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import capstone
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'recomp'))
|
||||
|
||||
BUF, BUF_SIZE = 0x7200000000, 0x40000
|
||||
UC_INVALID = (uc.UC_ERR_INSN_INVALID, uc.UC_ERR_EXCEPTION)
|
||||
|
||||
# Record layouts of struct In / struct Out in harness.c.
|
||||
IN_FMT = '<QQ32QQQ'
|
||||
IN_TAIL = '<QQ'
|
||||
OUT_SIZE = 32 * 8 + 4 * 8 + 32 * 16
|
||||
|
||||
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||
|
||||
F32 = [0, 0x80000000, 0x7f800000, 0xff800000, 0x7fc00000, 0xffc00000, 0x7fa00001, 0xff812345, 0x7fd00abc,
|
||||
0x00000001, 0x80000001, 0x007fffff, 0x807fffff, 0x00400000, 0x00800000, 0x80800000, 0x00800001, 0x7f7fffff,
|
||||
0xff7fffff, 0x3f800000, 0xbf800000, 0x3f000000, 0xbf000000, 0x40000000, 0x40400000, 0x3fc00000, 0xbfc00000,
|
||||
0x40200000, 0x40100000, 0x4f000000, 0xcf000000, 0x4f800000, 0x5f000000, 0xdf000000, 0x5f800000, 0x4effffff,
|
||||
0x4f7fffff, 0x5effffff, 0x5f7fffff, 0x3effffff, 0x3f7fffff, 0x7effffff, 0x01000000, 0x3e800000, 0x7e800000,
|
||||
0x7e7fffff, 0x00200000, 0x001fffff, 0x00200001, 0x47800000, 0x477fe000, 0x477ff000, 0x38800000, 0x33800000,
|
||||
0x33000000, 0x387fc000, 0x3f7ff000, 0x46fffe00, 0x80000000 | 0x00200000, 0x7f000000, 0x7f7ffffe]
|
||||
F64 = [0, 1 << 63, 0x7ff0000000000000, 0xfff0000000000000, 0x7ff8000000000000, 0xfff8000000000000,
|
||||
0x7ff4000000000001, 0xfff0000000012345, 0x7ffc00000abcdef0, 1, (1 << 63) | 1, 0x000fffffffffffff,
|
||||
0x0008000000000000, 0x0010000000000000, 0x8010000000000000, 0x7fefffffffffffff, 0xffefffffffffffff,
|
||||
0x3ff0000000000000, 0xbff0000000000000, 0x3fe0000000000000, 0x4000000000000000, 0x4008000000000000,
|
||||
0x3ff8000000000000, 0xbff8000000000000, 0x4004000000000000, 0x41e0000000000000, 0xc1e0000000000000,
|
||||
0x41f0000000000000, 0x43e0000000000000, 0xc3e0000000000000, 0x43f0000000000000, 0x41dfffffffc00000,
|
||||
0x41efffffffe00000, 0x43dfffffffffffff, 0x43efffffffffffff, 0x3fdfffffffffffff, 0x7fdfffffffffffff,
|
||||
0x7fd0000000000000, 0x0004000000000000, 0x0003ffffffffffff, 0x47efffffe0000000, 0x47efffff00000000,
|
||||
0x36a0000000000000, 0x3810000000000000, 0x380fffffffffffff, 0x47f0000000000000, 0x3e70000000000000,
|
||||
0x3f0ffc0000000000, 0x40effc0000000000, 0x40effe0000000000, 0x3e6ffffffff00000, 0x369fffffffffffff,
|
||||
0x3810000010000000, 0x381000000fffffff, 0x3e80000000000000]
|
||||
F16 = [0, 0x8000, 0x7c00, 0xfc00, 0x7e00, 0xfe00, 0x7d00, 0x7e55, 0x0001, 0x8001, 0x03ff, 0x0400, 0x7bff, 0xfbff,
|
||||
0x3c00, 0xbc00, 0x3800, 0x4000, 0x3e00, 0x7fff, 0xffff]
|
||||
|
||||
_md = capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
|
||||
|
||||
|
||||
def dis(w, pc=0):
|
||||
d = next(_md.disasm(struct.pack('<I', w), pc), None)
|
||||
return f'{d.mnemonic} {d.op_str}' if d else '?'
|
||||
|
||||
|
||||
def mnemonic(w):
|
||||
return dis(w).split(' ')[0]
|
||||
|
||||
|
||||
def rand_lane(r, sz):
|
||||
"""Random interesting integer lane of 8 << sz bits."""
|
||||
bits_ = 8 << sz
|
||||
k = r.random()
|
||||
if k < 0.35:
|
||||
return r.choice([0, 1, 2, 3, (1 << bits_) - 1, (1 << bits_) - 2, 1 << (bits_ - 1), (1 << (bits_ - 1)) - 1,
|
||||
(1 << (bits_ - 1)) + 1, 0x80 % (1 << bits_), 0x7f, bits_ - 1, bits_, bits_ + 1,
|
||||
(1 << bits_) - bits_, 1 << (bits_ // 2)])
|
||||
if k < 0.5:
|
||||
return r.getrandbits(r.randint(1, bits_))
|
||||
if k < 0.6:
|
||||
return ((1 << bits_) - r.getrandbits(r.randint(1, bits_ - 1))) & ((1 << bits_) - 1)
|
||||
return r.getrandbits(bits_)
|
||||
|
||||
|
||||
def rand_f32(r):
|
||||
k = r.random()
|
||||
if k < 0.4:
|
||||
return r.choice(F32)
|
||||
if k < 0.8:
|
||||
e = r.choice([r.randint(110, 145), r.randint(0, 255), r.randint(120, 135)])
|
||||
return (r.getrandbits(1) << 31) | (e << 23) | (r.getrandbits(23) if r.random() < 0.7 else
|
||||
r.getrandbits(4) << 19)
|
||||
return struct.unpack('<I', struct.pack('<f', r.choice([r.uniform(-4, 4), float(r.randint(-300, 300)) + 0.5,
|
||||
float(r.randint(-5, 5)), r.uniform(-1e6, 1e6)])))[0]
|
||||
|
||||
|
||||
def rand_f64(r):
|
||||
k = r.random()
|
||||
if k < 0.4:
|
||||
return r.choice(F64)
|
||||
if k < 0.8:
|
||||
e = r.choice([r.randint(1000, 1050), r.randint(0, 2047), r.randint(1020, 1030), r.randint(0, 40),
|
||||
r.randint(2000, 2047), r.randint(860, 920)])
|
||||
return (r.getrandbits(1) << 63) | (e << 52) | (r.getrandbits(52) if r.random() < 0.7 else
|
||||
r.getrandbits(6) << 46)
|
||||
return struct.unpack('<Q', struct.pack('<d', r.choice([r.uniform(-4, 4), float(r.randint(-300, 300)) + 0.5,
|
||||
r.uniform(-1e12, 1e12)])))[0]
|
||||
|
||||
|
||||
def rand_v(r):
|
||||
k = r.random()
|
||||
out = 0
|
||||
if k < 0.3:
|
||||
for i in range(4):
|
||||
out |= rand_f32(r) << (32 * i)
|
||||
elif k < 0.55:
|
||||
for i in range(2):
|
||||
out |= rand_f64(r) << (64 * i)
|
||||
elif k < 0.6:
|
||||
for i in range(8):
|
||||
out |= (r.choice(F16) if r.random() < 0.5 else r.getrandbits(16)) << (16 * i)
|
||||
elif k < 0.9:
|
||||
sz = r.randrange(4)
|
||||
for i in range(16 >> sz):
|
||||
out |= rand_lane(r, sz) << ((8 << sz) * i)
|
||||
else:
|
||||
out = r.getrandbits(128)
|
||||
if r.random() < 0.15: # upper half zero, like most 64-bit results
|
||||
out &= (1 << 64) - 1
|
||||
return out
|
||||
|
||||
|
||||
def rand_x(r):
|
||||
k = r.random()
|
||||
if k < 0.3:
|
||||
return r.choice([0, 1, 2, 0xFFFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF, 0x8000000000000000, 0xFFFFFFFF,
|
||||
0x7FFFFFFF, 0x80000000, 0x100000000, 0xFFFFFFFF80000000, 0x8000000000000001, 0xFFFFFF,
|
||||
0x1000001, 0x20000000000001, 0xFFFFFFFFFF000001, 63, 64, 31, 32, 0xFF, 0xFFFF])
|
||||
if k < 0.45:
|
||||
return rand_f32(r) | (r.getrandbits(32) << 32 if r.random() < 0.3 else 0)
|
||||
if k < 0.6:
|
||||
return rand_f64(r)
|
||||
if k < 0.75:
|
||||
return r.getrandbits(r.randint(1, 64))
|
||||
if k < 0.8:
|
||||
return (-r.getrandbits(r.randint(1, 63))) & 0xFFFFFFFFFFFFFFFF
|
||||
return r.getrandbits(64)
|
||||
|
||||
|
||||
def rand_fpcr(r):
|
||||
k = r.random()
|
||||
if k < 0.55:
|
||||
return 0
|
||||
v = 0
|
||||
if r.random() < 0.4:
|
||||
v |= 1 << 25 # DN
|
||||
if r.random() < 0.4:
|
||||
v |= 1 << 24 # FZ
|
||||
if r.random() < 0.4:
|
||||
v |= r.randrange(4) << 22 # RMode
|
||||
if r.random() < 0.15:
|
||||
v |= 1 << 26 # AHP
|
||||
return v
|
||||
|
||||
|
||||
class Emu:
|
||||
"""Unicorn as a Cortex-A57, the Switch's CPU, unless another model is given (LSE atomics need
|
||||
UC_CPU_ARM64_MAX). Images are (base, bytes) pairs mapped at their guest address."""
|
||||
|
||||
def __init__(self, images, model=A.UC_CPU_ARM64_A57):
|
||||
mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM)
|
||||
mu.ctl_set_cpu_model(model)
|
||||
mu.mem_map(BUF, BUF_SIZE)
|
||||
for base, data in images:
|
||||
mu.mem_map(base, (len(data) + 0xFFFF) & ~0xFFFF)
|
||||
mu.mem_write(base, bytes(data))
|
||||
mu.reg_write(A.UC_ARM64_REG_CPACR_EL1, 0x300000) # enable FP/SIMD
|
||||
self.mu = mu
|
||||
self.xregs = [A.UC_ARM64_REG_X0 + i for i in range(29)] + [A.UC_ARM64_REG_X29, A.UC_ARM64_REG_X30]
|
||||
self.qregs = [A.UC_ARM64_REG_Q0 + i for i in range(32)]
|
||||
self.out_regs = self.xregs + [A.UC_ARM64_REG_SP, A.UC_ARM64_REG_NZCV, A.UC_ARM64_REG_PC] + self.qregs
|
||||
|
||||
def run(self, pc, fpcr, x, sp, nzcv, v, count=1):
|
||||
"""Run count instructions from pc. Returns (x[31], sp, nzcv, pc, v[32])."""
|
||||
mu = self.mu
|
||||
mu.reg_write_batch(list(zip(self.xregs, x[:31])) + list(zip(self.qregs, v)) +
|
||||
[(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)])
|
||||
mu.emu_start(pc, pc + 4 * count + 4, count=count)
|
||||
o = mu.reg_read_batch(self.out_regs)
|
||||
return list(o[:31]), o[31], o[32] & 0xF0000000, o[33], list(o[34:])
|
||||
|
||||
|
||||
def compiler():
|
||||
cc = os.environ.get('CC')
|
||||
if cc:
|
||||
return cc
|
||||
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
|
||||
return str(MSYS_BIN / 'clang.exe')
|
||||
return shutil.which('clang') or 'clang'
|
||||
|
||||
|
||||
def build_harness(work, bodies, jobs, cflags):
|
||||
"""Compile one C function per test body plus harness.c into work/harness[.exe]."""
|
||||
cc = compiler()
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
flags = ['-O1', '-fno-strict-aliasing', '-Wno-unused-label', '-I', str(ROOT / 'runtime')] + cflags.split()
|
||||
n = max(1, min(jobs, len(bodies) // 200 + 1))
|
||||
per = (len(bodies) + n - 1) // n
|
||||
srcs = []
|
||||
for j in range(n):
|
||||
path = work / f'tests_{j}.c'
|
||||
with open(path, 'w') as f:
|
||||
f.write('#include "cpu.h"\n')
|
||||
for k in range(j * per, min(len(bodies), (j + 1) * per)):
|
||||
f.write(f'void t{k}(Cpu *c) {{ {bodies[k]} }}\n')
|
||||
srcs.append(path)
|
||||
with open(work / 'table.c', 'w') as f:
|
||||
f.write('#include "cpu.h"\n')
|
||||
f.write(''.join(f'void t{k}(Cpu *c);\n' for k in range(len(bodies))))
|
||||
f.write('GuestFn tests[] = {' + ','.join(f't{k}' for k in range(len(bodies))) + '};\n')
|
||||
f.write(f'int ntests = {len(bodies)};\n')
|
||||
srcs += [work / 'table.c', ROOT / 'tests' / 'diff' / 'harness.c']
|
||||
objs, procs = [], []
|
||||
for s in srcs:
|
||||
o = work / (s.stem + '.o')
|
||||
objs.append(o)
|
||||
procs.append(subprocess.Popen([cc] + flags + ['-c', str(s), '-o', str(o)], env=env))
|
||||
if any(p.wait() for p in procs):
|
||||
raise SystemExit('compile failed')
|
||||
exe = work / ('harness.exe' if os.name == 'nt' else 'harness')
|
||||
subprocess.check_call([cc, '-fuse-ld=lld', '-o', str(exe)] + [str(o) for o in objs], env=env)
|
||||
return exe
|
||||
|
||||
|
||||
def write_cases(path, cases):
|
||||
"""cases: (k, fpcr, x[31], sp, nzcv, v[32], memchk)."""
|
||||
with open(path, 'wb') as f:
|
||||
for k, fpcr, x, sp, nzcv, v, memchk in cases:
|
||||
f.write(struct.pack(IN_FMT, k, fpcr, *(list(x[:31]) + [0]), sp, nzcv))
|
||||
for q in v:
|
||||
f.write(q.to_bytes(16, 'little'))
|
||||
f.write(struct.pack(IN_TAIL, int(memchk), 0))
|
||||
|
||||
|
||||
def run_harness(exe, work, buf0, cases, images):
|
||||
"""Run the lifted tests. Returns (records, stderr tail or None). images: (base, path) pairs."""
|
||||
with open(work / 'buf.bin', 'wb') as f:
|
||||
f.write(buf0)
|
||||
write_cases(work / 'cases.bin', cases)
|
||||
out = work / 'out.bin'
|
||||
args = [str(exe), str(work / 'buf.bin'), str(work / 'cases.bin'), str(out)]
|
||||
for base, path in images:
|
||||
args += [str(path), f'{base:x}']
|
||||
p = subprocess.run(args, capture_output=True)
|
||||
err = None if p.returncode == 0 else f'exit {p.returncode:#x}: {p.stderr.decode(errors="replace")[-2000:]}'
|
||||
data = out.read_bytes() if out.exists() else b''
|
||||
recs = []
|
||||
for i in range(len(data) // OUT_SIZE):
|
||||
rec = data[i * OUT_SIZE:(i + 1) * OUT_SIZE]
|
||||
x = struct.unpack_from('<31Q', rec, 0)
|
||||
sp, nzcv, pc, mem = struct.unpack_from('<4Q', rec, 256)
|
||||
v = [int.from_bytes(rec[288 + 16 * j:304 + 16 * j], 'little') for j in range(32)]
|
||||
recs.append((x, sp, nzcv, pc, mem, v))
|
||||
return recs, err
|
||||
|
||||
|
||||
def diff_state(got, want, check_pc, check_mem):
|
||||
"""got: harness record; want: (x, sp, nzcv, pc, v, mem). Returns a list of mismatches."""
|
||||
hx, hsp, hnz, hpc, hmem, hv = got
|
||||
ex, esp, enz, epc, ev, emem = want
|
||||
d = [f'x{j}: got {hx[j]:x} want {ex[j]:x}' for j in range(31) if hx[j] != ex[j]]
|
||||
if hsp != esp:
|
||||
d.append(f'sp: got {hsp:x} want {esp:x}')
|
||||
if hnz != enz:
|
||||
d.append(f'nzcv: got {hnz:x} want {enz:x}')
|
||||
if check_pc and hpc != epc:
|
||||
d.append(f'pc: got {hpc:x} want {epc:x}')
|
||||
if check_mem and hmem != emem:
|
||||
d.append('memory differs')
|
||||
d += [f'v{j}: got {hv[j]:032x} want {ev[j]:032x}' for j in range(32) if hv[j] != ev[j]]
|
||||
return d
|
||||
@@ -0,0 +1,317 @@
|
||||
"""Differential test of whole emitted functions: recomp/gen.py's C for a guest function against Unicorn running the
|
||||
same function from its entry until it returns.
|
||||
|
||||
Picks leaf functions from the ExeFS modules: no calls, no branches out of the function, no SVC, BRK, system
|
||||
register, exclusive or atomic instructions; with conditional branches, and loops, jump tables and SIMD where the
|
||||
module has them. Emits them with the generator's own Emitter, compiles them with tests/diff/harness.c, and runs each
|
||||
from random states: x0-x7 point into the scratch buffer (or hold small integers), the buffer holds pointers into
|
||||
itself and random data, SP sits near the buffer's top and LR is a sentinel address. Unicorn (Cortex-A57) stops when
|
||||
PC reaches the sentinel. A case is skipped when Unicorn faults, writes outside the scratch buffer (module images are
|
||||
mapped read-only), leaves the function, or runs longer than --steps instructions. Compares X0-X30, SP, NZCV, V0-V31
|
||||
and the scratch buffer.
|
||||
|
||||
Usage: func.py [--exefs DIR] [--modules main,subsdk0,sdk] [--functions N] [--cases K] [--seed S] [--jobs J]
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
from common import BUF, BUF_SIZE, MSYS_BIN, ROOT, Emu, compiler, diff_state, rand_fpcr, rand_v, rand_x, run_harness
|
||||
import gen
|
||||
from layout import layout
|
||||
from module import Module
|
||||
|
||||
SENTINEL = 0x7210000000 # LR on entry; never executed
|
||||
PTR_LO, PTR_HI = BUF + 0x8000, BUF + 0x28000
|
||||
STACK_TOP = BUF + 0x3F000
|
||||
MIN_INSNS, MAX_INSNS = 6, 600
|
||||
|
||||
|
||||
def forbidden_mask(w):
|
||||
"""Words a picked function must not contain."""
|
||||
m = (w & 0xFC000000) == 0x94000000 # BL
|
||||
m |= (w & 0xFFFFFC1F) == 0xD63F0000 # BLR
|
||||
m |= (w & 0xFF000000) == 0xD4000000 # SVC, HVC, SMC, BRK, HLT, DCPS
|
||||
sysi = (w & 0xFFC00000) == 0xD5000000 # system: MRS, MSR, SYS (DC ZVA), hints, barriers
|
||||
hint_or_barrier = ((w & 0xFFFFF000) == 0xD5032000) | ((w & 0xFFFFF000) == 0xD5033000)
|
||||
m |= sysi & ~hint_or_barrier
|
||||
m |= (w & 0x3F000000) == 0x08000000 # exclusives, load-acquire / store-release, CAS
|
||||
m |= (w & 0x3F200C00) == 0x38200000 # LSE atomics
|
||||
return m
|
||||
|
||||
|
||||
def cond_mask(w):
|
||||
return (((w & 0xFE000010) == 0x54000000) | ((w & 0x7E000000) == 0x34000000) |
|
||||
((w & 0x7E000000) == 0x36000000))
|
||||
|
||||
|
||||
def simd_mask(w):
|
||||
return ((w & 0x0E000000) == 0x0E000000) | (((w & 0x0A000000) == 0x08000000) & (((w >> 26) & 1) == 1))
|
||||
|
||||
|
||||
def pick_functions(name, mod, count, r):
|
||||
"""[(entry, end, tags)] of up to count leaf functions of mod, plus the Emitter that will emit them."""
|
||||
starts, _ = gen.discover(mod)
|
||||
w = mod.words
|
||||
ends = np.append(starts[1:], len(w))
|
||||
|
||||
def per_fn(mask):
|
||||
cs = np.concatenate([[0], np.cumsum(mask.astype(np.int64))])
|
||||
return cs[ends] - cs[starts]
|
||||
|
||||
size = ends - starts
|
||||
ok = (per_fn(forbidden_mask(w)) == 0) & (per_fn(cond_mask(w)) > 0) & (size >= MIN_INSNS) & (size <= MAX_INSNS)
|
||||
has_br = per_fn((w & 0xFFFFFC1F) == 0xD61F0000) > 0
|
||||
cand = np.flatnonzero(ok)
|
||||
with_br = [int(i) for i in cand if has_br[i]]
|
||||
rest = [int(i) for i in cand if not has_br[i]]
|
||||
r.shuffle(with_br)
|
||||
r.shuffle(rest)
|
||||
emitter = gen.Emitter(mod, set((mod.lo + starts * 4).tolist()))
|
||||
picks, tags_seen = [], Counter()
|
||||
# Functions with a br first (jump tables, fallbacks), at most a quarter of the picks.
|
||||
for pool, quota in ((with_br, count // 4), (rest, count)):
|
||||
taken = 0
|
||||
for i in pool:
|
||||
if len(picks) >= count or taken >= quota:
|
||||
break
|
||||
entry, end = mod.lo + int(starts[i]) * 4, mod.lo + int(ends[i]) * 4
|
||||
tags = check_function(emitter, entry, end)
|
||||
if tags is None:
|
||||
continue
|
||||
picks.append((entry, end, tags))
|
||||
tags_seen.update(tags)
|
||||
taken += 1
|
||||
print(f'{name}: {len(starts)} functions, {len(cand)} leaf candidates, picked {len(picks)}: '
|
||||
f'{dict(tags_seen)}', flush=True)
|
||||
return picks, emitter
|
||||
|
||||
|
||||
def check_function(emitter, entry, end):
|
||||
"""Tags of a function the test can run in isolation, or None."""
|
||||
mod = emitter.mod
|
||||
insns = [mod.lift(a) for a in range(entry, end, 4)]
|
||||
if insns[-1].kind not in gen.TERMINATORS or not any(i.kind == 'ret' for i in insns):
|
||||
return None
|
||||
tags = set()
|
||||
for k, ins in enumerate(insns):
|
||||
a = entry + 4 * k
|
||||
if ins.kind in ('bl', 'blr'):
|
||||
return None
|
||||
if ins.kind in ('b', 'bcond'):
|
||||
if not entry <= ins.target < end:
|
||||
return None
|
||||
if ins.target <= a:
|
||||
tags.add('loop')
|
||||
if ins.kind == 'br':
|
||||
jt = gen.jumptables.resolve(mod, a, entry, end)
|
||||
if jt:
|
||||
tags.add('jump_table')
|
||||
elif gen.jumptables.is_computed(mod, a, entry):
|
||||
tags.add('all_label_fallback')
|
||||
else:
|
||||
return None
|
||||
if simd_mask(mod.words[(entry - mod.lo) >> 2:(end - mod.lo) >> 2]).any():
|
||||
tags.add('simd')
|
||||
return tags
|
||||
|
||||
|
||||
def build(work, emitted, jobs, cflags):
|
||||
"""Compile the emitted functions with harness.c into work/func[.exe]; tests[k] is emitted[k]."""
|
||||
cc = compiler()
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
flags = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-I', str(ROOT / 'runtime')]
|
||||
flags += cflags.split()
|
||||
groups = {}
|
||||
for name, entry, end, emitter in emitted:
|
||||
groups.setdefault(name, []).append((entry, end))
|
||||
srcs, n = [], max(1, jobs)
|
||||
for name, fns in groups.items():
|
||||
emitter = next(e for m, _, _, e in emitted if m == name)
|
||||
per = (len(fns) + n - 1) // n
|
||||
for j in range(0, len(fns), per):
|
||||
path = work / f'{name}_{j // per:03d}.c'
|
||||
path.write_text(gen.chunk_text(name, j // per, emitter, fns[j:j + per]))
|
||||
srcs.append(path)
|
||||
table = work / 'table.c'
|
||||
table.write_text('#include "cpu.h"\n' + ''.join(f'void {gen.fname(e)}(Cpu *c);\n' for _, e, _, _ in emitted) +
|
||||
'GuestFn tests[] = {' + ','.join(gen.fname(e) for _, e, _, _ in emitted) + '};\n' +
|
||||
f'int ntests = {len(emitted)};\n')
|
||||
srcs += [table, ROOT / 'tests' / 'diff' / 'harness.c']
|
||||
objs, procs = [], []
|
||||
for s in srcs:
|
||||
o = work / (s.stem + '.o')
|
||||
objs.append(o)
|
||||
procs.append(subprocess.Popen([cc] + flags + ['-c', str(s), '-o', str(o)], env=env))
|
||||
if any(p.wait() for p in procs):
|
||||
raise SystemExit('compile failed')
|
||||
exe = work / ('func.exe' if os.name == 'nt' else 'func')
|
||||
subprocess.check_call([cc, '-fuse-ld=lld', '-o', str(exe)] + [str(o) for o in objs], env=env)
|
||||
return exe
|
||||
|
||||
|
||||
def make_buffer(r):
|
||||
out = bytearray()
|
||||
for _ in range(BUF_SIZE // 8):
|
||||
k = r.random()
|
||||
if k < 0.4:
|
||||
v = r.randrange(PTR_LO, PTR_HI, 8)
|
||||
elif k < 0.7:
|
||||
v = r.randrange(0, 256)
|
||||
else:
|
||||
v = r.getrandbits(64)
|
||||
out += v.to_bytes(8, 'little')
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def make_case(r):
|
||||
x = [rand_x(r) for _ in range(31)]
|
||||
for i in range(8):
|
||||
k = r.random()
|
||||
if k < 0.7:
|
||||
x[i] = r.randrange(PTR_LO, PTR_HI, 16)
|
||||
elif k < 0.9:
|
||||
x[i] = r.randrange(0, 64)
|
||||
x[30] = SENTINEL
|
||||
sp = STACK_TOP - r.randrange(0, 0x100, 16)
|
||||
return rand_fpcr(r), x, sp, r.getrandbits(4) << 28, [rand_v(r) for _ in range(32)]
|
||||
|
||||
|
||||
class Runner:
|
||||
"""Unicorn with the module images read-only and a block hook that stops when execution leaves the function."""
|
||||
|
||||
def __init__(self, images, buf0):
|
||||
self.emu = Emu(images)
|
||||
mu = self.emu.mu
|
||||
for base, data in images:
|
||||
mu.mem_protect(base, (len(data) + 0xFFFF) & ~0xFFFF, uc.UC_PROT_READ | uc.UC_PROT_EXEC)
|
||||
mu.mem_map(SENTINEL, 0x1000)
|
||||
mu.mem_write(BUF, buf0)
|
||||
self.buf0 = buf0
|
||||
self.lo = self.hi = 0
|
||||
self.left = False
|
||||
mu.hook_add(uc.UC_HOOK_BLOCK, self._block)
|
||||
|
||||
def _block(self, mu, address, size, user):
|
||||
if not self.lo <= address < self.hi and address != SENTINEL:
|
||||
self.left = True
|
||||
mu.emu_stop()
|
||||
|
||||
def run(self, entry, end, case, steps):
|
||||
"""(x, sp, nzcv, pc, v, crc of the buffer) or None if the case is skipped."""
|
||||
fpcr, x, sp, nzcv, v = case
|
||||
e, mu = self.emu, self.emu.mu
|
||||
self.lo, self.hi, self.left = entry, end, False
|
||||
mu.reg_write_batch(list(zip(e.xregs, x[:31])) + list(zip(e.qregs, v)) +
|
||||
[(A.UC_ARM64_REG_SP, sp), (A.UC_ARM64_REG_NZCV, nzcv), (A.UC_ARM64_REG_FPCR, fpcr)])
|
||||
try:
|
||||
mu.emu_start(entry, SENTINEL, count=steps)
|
||||
ok = not self.left and mu.reg_read(A.UC_ARM64_REG_PC) == SENTINEL
|
||||
except uc.UcError:
|
||||
ok = False
|
||||
if not ok:
|
||||
mu.mem_write(BUF, self.buf0)
|
||||
return None
|
||||
o = mu.reg_read_batch(e.out_regs)
|
||||
crc = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
|
||||
mu.mem_write(BUF, self.buf0)
|
||||
return list(o[:31]), o[31], o[32] & 0xF0000000, o[33], list(o[34:]), crc
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, default=ROOT / 'build' / 'extracted' / 'v121' / 'exefs')
|
||||
ap.add_argument('--modules', default='main,subsdk0,sdk')
|
||||
ap.add_argument('--functions', type=int, default=400, help='per module')
|
||||
ap.add_argument('--cases', type=int, default=12, help='random states tried per function')
|
||||
ap.add_argument('--steps', type=int, default=200000, help='Unicorn instruction cap per case')
|
||||
ap.add_argument('--seed', type=int, default=1)
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--show', type=int, default=20)
|
||||
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
|
||||
args = ap.parse_args()
|
||||
r = random.Random(args.seed)
|
||||
t0 = time.time()
|
||||
want = args.modules.split(',')
|
||||
|
||||
work = ROOT / 'build' / 'test' / 'func'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
images, emitted, tags = [], [], []
|
||||
for name, base, _ in layout(args.exefs):
|
||||
if name not in want:
|
||||
continue
|
||||
mod = Module(name, args.exefs / name, base, 'c')
|
||||
img = work / f'{name}.img'
|
||||
img.write_bytes(mod.image)
|
||||
images.append((base, img, mod.image))
|
||||
picks, emitter = pick_functions(name, mod, args.functions, r)
|
||||
for entry, end, t in picks:
|
||||
emitted.append((name, entry, end, emitter))
|
||||
tags.append(t)
|
||||
print(f'picked {len(emitted)} functions ({time.time() - t0:.0f}s)', flush=True)
|
||||
|
||||
t1 = time.time()
|
||||
exe = build(work, emitted, args.jobs, args.cflags)
|
||||
print(f'compiled ({time.time() - t1:.0f}s)', flush=True)
|
||||
|
||||
buf0 = make_buffer(r)
|
||||
runner = Runner([(base, data) for base, _, data in images], buf0)
|
||||
cases, expect, tested, skipped = [], [], set(), 0
|
||||
t2 = time.time()
|
||||
for k, (name, entry, end, _) in enumerate(emitted):
|
||||
for _ in range(args.cases):
|
||||
case = make_case(r)
|
||||
want_state = runner.run(entry, end, case, args.steps)
|
||||
if want_state is None:
|
||||
skipped += 1
|
||||
continue
|
||||
fpcr, x, sp, nzcv, v = case
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, True))
|
||||
expect.append(want_state)
|
||||
tested.add(k)
|
||||
print(f'unicorn: {len(cases)} cases on {len(tested)} functions, {skipped} skipped ({time.time() - t2:.0f}s)',
|
||||
flush=True)
|
||||
|
||||
t3 = time.time()
|
||||
recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images])
|
||||
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
|
||||
if err:
|
||||
k = cases[len(recs)][0] if len(recs) < len(cases) else None
|
||||
where = f', stopped in {emitted[k][0]} {emitted[k][1]:#x}' if k is not None else ''
|
||||
print(f'harness failed: {err}{where}')
|
||||
|
||||
fails, nfail = {}, 0
|
||||
for case, want_state, got in zip(cases, expect, recs):
|
||||
d = diff_state(got, want_state, check_pc=False, check_mem=True)
|
||||
if d:
|
||||
nfail += 1
|
||||
fails.setdefault(case[0], d[:4])
|
||||
by_tag, fail_tag = Counter(), Counter()
|
||||
for k in tested:
|
||||
by_tag.update(tags[k] or {'plain'})
|
||||
if k in fails:
|
||||
fail_tag.update(tags[k] or {'plain'})
|
||||
print(f'{len(tested)} functions, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing functions '
|
||||
f'({time.time() - t0:.0f}s total)')
|
||||
print(f' tested by tag: {dict(by_tag)}; failing by tag: {dict(fail_tag)}')
|
||||
for k, d in list(fails.items())[:args.show]:
|
||||
name, entry, end, _ = emitted[k]
|
||||
print(f' {name:8} {gen.fname(entry)} ({(end - entry) // 4} insns, {sorted(tags[k])}): {"; ".join(d)}')
|
||||
return 1 if fails or err else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,116 @@
|
||||
// Runs lifted single-instruction tests and writes the resulting CPU state per case.
|
||||
// Shared by tests/diff/lift.py and tests/diff/simd.py; the expected state comes from Unicorn on the Python side.
|
||||
//
|
||||
// argv: buf.bin cases.bin out.bin image base [image base ...]
|
||||
// Each image is mapped at its guest address (guest == host), the scratch buffer at BUF.
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
|
||||
#include "cpu.h"
|
||||
|
||||
extern GuestFn tests[];
|
||||
extern int ntests;
|
||||
|
||||
#define BUF 0x7200000000ull
|
||||
#define BUF_SIZE 0x40000
|
||||
|
||||
void rt_dispatch(Cpu *c) { (void)c; }
|
||||
void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; }
|
||||
void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) { (void)c; (void)imm; (void)pc; abort(); }
|
||||
void rt_udf(Cpu *c, uint64_t pc) { (void)c; (void)pc; abort(); }
|
||||
uint64_t rt_ticks(void) { return 0; }
|
||||
volatile int g_fnprof;
|
||||
void fnprof_hit(uint64_t addr, uint64_t lr, uint64_t x0) { (void)addr; (void)lr; (void)x0; }
|
||||
|
||||
static void *map_at(uint64_t addr, size_t len) {
|
||||
len = (len + 0xffff) & ~(size_t)0xffff;
|
||||
#ifdef _WIN32
|
||||
void *p = VirtualAlloc((void *)addr, len, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
#else
|
||||
void *p = mmap((void *)addr, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
|
||||
#endif
|
||||
if (p != (void *)addr) {
|
||||
fprintf(stderr, "map %llx (+%zx) failed\n", (unsigned long long)addr, len);
|
||||
exit(1);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static void *load_at(const char *path, uint64_t addr) {
|
||||
FILE *f = fopen(path, "rb");
|
||||
if (!f) { perror(path); exit(1); }
|
||||
fseek(f, 0, SEEK_END);
|
||||
size_t size = (size_t)ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
void *p = map_at(addr, size);
|
||||
if (fread(p, 1, size, f) != size) { perror(path); exit(1); }
|
||||
fclose(f);
|
||||
return p;
|
||||
}
|
||||
|
||||
static uint32_t crc_tab[256];
|
||||
static uint32_t crc32_buf(const uint8_t *p, size_t n) {
|
||||
uint32_t c = 0xffffffffu;
|
||||
for (size_t i = 0; i < n; i++) c = crc_tab[(c ^ p[i]) & 0xff] ^ (c >> 8);
|
||||
return c ^ 0xffffffffu;
|
||||
}
|
||||
|
||||
// Layouts match IN_FMT / OUT_FMT in tests/diff/common.py.
|
||||
struct In { uint64_t k, fpcr, x[32], sp, nzcv; __uint128_t v[32]; uint64_t memchk, pad; };
|
||||
struct Out { uint64_t x[32], sp, nzcv, pc, mem; __uint128_t v[32]; };
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 6 || (argc - 4) % 2) {
|
||||
fprintf(stderr, "usage: %s buf.bin cases.bin out.bin image base [image base ...]\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
for (uint32_t i = 0; i < 256; i++) {
|
||||
uint32_t c = i;
|
||||
for (int k = 0; k < 8; k++) c = c & 1 ? (c >> 1) ^ 0xEDB88320u : c >> 1;
|
||||
crc_tab[i] = c;
|
||||
}
|
||||
for (int i = 4; i < argc; i += 2) load_at(argv[i], strtoull(argv[i + 1], 0, 16));
|
||||
uint8_t *buf = map_at(BUF, BUF_SIZE);
|
||||
FILE *bf = fopen(argv[1], "rb");
|
||||
if (!bf || fread(buf, 1, BUF_SIZE, bf) != BUF_SIZE) { perror(argv[1]); return 1; }
|
||||
fclose(bf);
|
||||
uint8_t *pristine = malloc(BUF_SIZE);
|
||||
memcpy(pristine, buf, BUF_SIZE);
|
||||
|
||||
FILE *f = fopen(argv[2], "rb"), *o = fopen(argv[3], "wb");
|
||||
if (!f || !o) return 1;
|
||||
static struct In in;
|
||||
static struct Out out;
|
||||
static Cpu c;
|
||||
while (fread(&in, sizeof in, 1, f) == 1) {
|
||||
if (in.k >= (uint64_t)ntests) { fprintf(stderr, "bad test index %llu\n", (unsigned long long)in.k); return 1; }
|
||||
memset(&c, 0, sizeof c);
|
||||
memcpy(c.x, in.x, sizeof c.x);
|
||||
c.sp = in.sp;
|
||||
c.nzcv = in.nzcv;
|
||||
c.fpcr = in.fpcr;
|
||||
memcpy(c.v, in.v, sizeof c.v);
|
||||
c.excl_addr = ~0ull;
|
||||
tests[in.k](&c);
|
||||
memcpy(out.x, c.x, sizeof out.x);
|
||||
out.sp = c.sp;
|
||||
out.nzcv = c.nzcv & 0xf0000000ull;
|
||||
out.pc = c.pc;
|
||||
out.mem = 0;
|
||||
if (in.memchk) {
|
||||
out.mem = crc32_buf(buf, BUF_SIZE);
|
||||
memcpy(buf, pristine, BUF_SIZE);
|
||||
}
|
||||
memcpy(out.v, c.v, sizeof out.v);
|
||||
fwrite(&out, sizeof out, 1, o);
|
||||
// Flushed per case so a crash leaves the completed cases readable.
|
||||
fflush(o);
|
||||
}
|
||||
fclose(o);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Differential test of the lifter on the game's own instructions: lifted C against Unicorn, one instruction at a time.
|
||||
|
||||
Samples distinct instruction words from the .text of each ExeFS module, one per opcode signature first, lifts each
|
||||
with the portable FP/SIMD lowering, runs it from random register state, and compares X0-X30, SP, NZCV, PC (for
|
||||
branches), V0-V31 and the scratch buffer (for loads and stores) with Unicorn as a Cortex-A57.
|
||||
|
||||
Skipped: system instructions and exceptions, whose single step cannot be compared, and exclusives and ARMv8.1
|
||||
atomics, which atomic.py covers. Encodings the lifter rejects are counted per module and mnemonic; --coverage lifts
|
||||
every distinct word instead of testing.
|
||||
|
||||
Adapted from LAS Recomp tools/test_lift.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
|
||||
|
||||
Usage: lift.py [--exefs DIR] [--modules main,sdk,subsdk0] [--encodings N] [--cases K] [--seed S] [--jobs J]
|
||||
lift.py --coverage
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import unicorn as uc
|
||||
|
||||
from common import BUF, BUF_SIZE, ROOT, Emu, build_harness, diff_state, dis, mnemonic, rand_fpcr, rand_v, rand_x, \
|
||||
run_harness
|
||||
from lifter import Lifter, Unsupported, bits
|
||||
from nso import Nso
|
||||
|
||||
BUF_MID = BUF + 0x10000
|
||||
# Test load addresses, below 2^39 and clear of BUF.
|
||||
BASES = {'main': 0x7000000000, 'subsdk0': 0x7008000000, 'sdk': 0x7010000000}
|
||||
|
||||
|
||||
def is_mem(w):
|
||||
return bits(w, 25, 1) == 0 and bits(w, 27, 1) == 1
|
||||
|
||||
|
||||
def classify(w):
|
||||
"""'skip' for instructions whose single-step behaviour cannot be compared."""
|
||||
if (w & 0xFFC00000) == 0xD5000000: # system: hints, barriers, MRS/MSR, cache maintenance
|
||||
return 'skip'
|
||||
if (w & 0x3F000000) == 0x08000000 and (bits(w, 23, 1) == 0 or bits(w, 21, 1)):
|
||||
return 'skip' # exclusives need a paired load; CAS is ARMv8.1; both are covered by atomic.py
|
||||
if (w & 0x3F200C00) == 0x38200000: # LSE atomics, ARMv8.1: atomic.py
|
||||
return 'skip'
|
||||
if (w & 0xFF000000) == 0xD4000000: # SVC, BRK, HLT
|
||||
return 'skip'
|
||||
if (w & 0xFFFF0000) == 0: # UDF
|
||||
return 'skip'
|
||||
return 'ok'
|
||||
|
||||
|
||||
def class_keys(words):
|
||||
"""Opcode signature per word: register fields (Rd, Rn, Rm) cleared."""
|
||||
return words & np.uint32(0xFFE0FC00)
|
||||
|
||||
|
||||
def to_test_body(ins, pc):
|
||||
if ins.kind == 'op':
|
||||
return ins.code
|
||||
nxt = f'0x{pc + 4:x}ull'
|
||||
if ins.kind == 'b':
|
||||
return f'c->pc = 0x{ins.target:x}ull;'
|
||||
if ins.kind == 'bl':
|
||||
return f'c->x[30] = {nxt}; c->pc = 0x{ins.target:x}ull;'
|
||||
if ins.kind == 'bcond':
|
||||
return f'c->pc = ({ins.cond}) ? 0x{ins.target:x}ull : {nxt};'
|
||||
if ins.kind == 'br':
|
||||
return f'c->pc = {ins.target};'
|
||||
if ins.kind == 'blr':
|
||||
return f'{{ uint64_t t = {ins.target}; c->x[30] = {nxt}; c->pc = t; }}'
|
||||
if ins.kind == 'ret':
|
||||
return 'c->pc = c->x[30];'
|
||||
raise Unsupported(ins.kind)
|
||||
|
||||
|
||||
def pick_module(name, nso, base, count, r, unsupported):
|
||||
"""Distinct words of .text, one per opcode signature first, lifted. Returns [(word, pc, body)]."""
|
||||
lo, size = nso.text
|
||||
words = np.frombuffer(bytes(nso.image[lo:lo + size]), dtype='<u4')
|
||||
uniq, first = np.unique(words, return_index=True)
|
||||
keys = class_keys(uniq)
|
||||
order = list(range(len(uniq)))
|
||||
r.shuffle(order)
|
||||
seen, primary, rest = set(), [], []
|
||||
for i in order:
|
||||
k = int(keys[i])
|
||||
(rest if k in seen else primary).append(i)
|
||||
seen.add(k)
|
||||
lifter = Lifter(base, 'c')
|
||||
picks = []
|
||||
for i in primary + rest:
|
||||
if len(picks) >= count:
|
||||
break
|
||||
w = int(uniq[i])
|
||||
if classify(w) == 'skip':
|
||||
continue
|
||||
pc = base + lo + int(first[i]) * 4
|
||||
try:
|
||||
body = to_test_body(lifter.lift(w, pc), pc)
|
||||
except Unsupported:
|
||||
unsupported[(name, mnemonic(w))] += 1
|
||||
continue
|
||||
picks.append((w, pc, body))
|
||||
print(f'{name}: {len(words)} words, {len(uniq)} distinct, {len(seen)} signatures, {len(picks)} picked')
|
||||
return picks
|
||||
|
||||
|
||||
def coverage(name, nso, base):
|
||||
"""Lift every distinct word of .text. Returns {kind mnemonic: [distinct words, occurrences]} of words the lifter
|
||||
rejects (Unsupported) or turns into an undefined-instruction trap (rt_udf)."""
|
||||
lo, size = nso.text
|
||||
words = np.frombuffer(bytes(nso.image[lo:lo + size]), dtype='<u4')
|
||||
uniq, first, counts = np.unique(words, return_index=True, return_counts=True)
|
||||
lifter = Lifter(base, 'c')
|
||||
rejected = {}
|
||||
for w, i, n in zip(uniq.tolist(), first.tolist(), counts.tolist()):
|
||||
try:
|
||||
ins = lifter.lift(w, base + lo + i * 4)
|
||||
if ins.kind != 'trap' or not ins.code.startswith('rt_udf'):
|
||||
continue
|
||||
key = f'udf {mnemonic(w)}'
|
||||
except Unsupported:
|
||||
key = f'unsupported {mnemonic(w)}'
|
||||
e = rejected.setdefault(key, [0, 0])
|
||||
e[0] += 1
|
||||
e[1] += n
|
||||
total = sum(e[1] for e in rejected.values())
|
||||
print(f'{name}: {len(words)} words, {len(uniq)} distinct, {total} rejected '
|
||||
f'({sum(e[0] for e in rejected.values())} distinct)')
|
||||
for mn, (d, n) in sorted(rejected.items(), key=lambda kv: -kv[1][1]):
|
||||
print(f' {mn:24} {n:8} occurrences {d:6} distinct')
|
||||
return rejected
|
||||
|
||||
|
||||
def make_case(r, w):
|
||||
if is_mem(w):
|
||||
x = [BUF_MID + r.randrange(0, 0x100, 8) for _ in range(31)]
|
||||
if (bits(w, 21, 1) and bits(w, 10, 2) == 2) or (w & 0xBFA00000) == 0x0C800000: # register offset
|
||||
rm = bits(w, 16, 5)
|
||||
if rm != bits(w, 5, 5) and rm < 31:
|
||||
x[rm] = r.randrange(0, 0x100)
|
||||
else:
|
||||
x = [rand_x(r) for _ in range(31)]
|
||||
sp = BUF_MID + r.randrange(0, 0x100, 16)
|
||||
nzcv = r.getrandbits(4) << 28
|
||||
v = [rand_v(r) for _ in range(32)]
|
||||
fpcr = rand_fpcr(r)
|
||||
return fpcr, x, sp, nzcv, v
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, default=ROOT / 'build' / 'extracted' / 'v121' / 'exefs')
|
||||
ap.add_argument('--modules', default='main,sdk,subsdk0')
|
||||
ap.add_argument('--encodings', type=int, default=20000, help='per module')
|
||||
ap.add_argument('--cases', type=int, default=3, help='random states per encoding')
|
||||
ap.add_argument('--seed', type=int, default=1)
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--show', type=int, default=40)
|
||||
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
|
||||
ap.add_argument('--coverage', action='store_true',
|
||||
help='only lift every distinct word and list the rejected ones (capstone "?" = undecodable data)')
|
||||
args = ap.parse_args()
|
||||
r = random.Random(args.seed)
|
||||
t0 = time.time()
|
||||
names = args.modules.split(',')
|
||||
if args.coverage:
|
||||
for name in names:
|
||||
coverage(name, Nso(str(args.exefs / name)), BASES[name])
|
||||
print(f'({time.time() - t0:.0f}s)')
|
||||
return 0
|
||||
|
||||
work = ROOT / 'build' / 'test' / 'lift'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
images, picks, unsupported = [], [], Counter()
|
||||
for name in names:
|
||||
nso = Nso(str(args.exefs / name))
|
||||
base = BASES[name]
|
||||
img = work / f'{name}.img'
|
||||
img.write_bytes(bytes(nso.image))
|
||||
images.append((base, img, nso.image))
|
||||
picks += [(name,) + p for p in pick_module(name, nso, base, args.encodings, r, unsupported)]
|
||||
print(f'lifted {len(picks)} encodings ({time.time() - t0:.0f}s)')
|
||||
|
||||
t1 = time.time()
|
||||
exe = build_harness(work, [p[3] for p in picks], args.jobs, args.cflags)
|
||||
print(f'compiled ({time.time() - t1:.0f}s)')
|
||||
|
||||
emu = Emu([(base, data) for base, _, data in images])
|
||||
mu = emu.mu
|
||||
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases, expect = [], []
|
||||
t2 = time.time()
|
||||
for k, (name, w, pc, _) in enumerate(picks):
|
||||
mem = is_mem(w)
|
||||
for _ in range(args.cases):
|
||||
fpcr, x, sp, nzcv, v = make_case(r, w)
|
||||
try:
|
||||
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v)
|
||||
except uc.UcError:
|
||||
if mem:
|
||||
mu.mem_write(BUF, buf0)
|
||||
continue
|
||||
mh = 0
|
||||
if mem:
|
||||
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, mem))
|
||||
expect.append((ox, osp, onz, opc, ov, mh))
|
||||
print(f'unicorn: {len(cases)} cases ({time.time() - t2:.0f}s)')
|
||||
|
||||
t3 = time.time()
|
||||
recs, err = run_harness(exe, work, buf0, cases, [(base, img) for base, img, _ in images])
|
||||
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
|
||||
if err:
|
||||
k = cases[len(recs)][0] if len(recs) < len(cases) else None
|
||||
where = f', stopped at {picks[k][1]:08x} {dis(picks[k][1])}' if k is not None else ''
|
||||
print(f'harness failed: {err}{where}')
|
||||
|
||||
fails, nfail = {}, 0
|
||||
for case, want, got in zip(cases, expect, recs):
|
||||
k = case[0]
|
||||
name, w, pc, _ = picks[k]
|
||||
d = diff_state(got, want, check_pc=got[3] != 0, check_mem=case[6])
|
||||
if d:
|
||||
nfail += 1
|
||||
fails.setdefault(w, (name, pc, d[:4]))
|
||||
|
||||
print('unsupported encodings (distinct words) by module and mnemonic:')
|
||||
for (name, mn), n in sorted(unsupported.items(), key=lambda kv: (kv[0][0], -kv[1])):
|
||||
print(f' {name:8} {mn:10} {n}')
|
||||
print(f'{len(picks)} encodings, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing encodings '
|
||||
f'({time.time() - t0:.0f}s total)')
|
||||
if fails:
|
||||
print(' failing by mnemonic:', dict(Counter(mnemonic(w) for w in fails).most_common(40)))
|
||||
for w, (name, pc, d) in list(fails.items())[:args.show]:
|
||||
print(f' {name:8} {w:08x} {dis(w, pc):40} {"; ".join(d)}')
|
||||
return 1 if fails or err else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Differential test of the portable FP/SIMD lowering (lifter simd='c') against Unicorn as a Cortex-A57.
|
||||
|
||||
Needs no game files. Enumerates every opcode bit pattern of the FP/SIMD data-processing space (bits 27..25 = 111),
|
||||
the FP/SIMD load/store space and CRC32 with random register fields, and compares which encodings the lifter and
|
||||
Unicorn accept. Then a stratified sample of encodings both accept is compiled and run with random interesting
|
||||
operands (NaNs, infinities, denormals, +-0, saturation edges, random FPCR.DN/FZ/RMode/AHP): X0-X30, SP, NZCV, V0-V31
|
||||
and, for stores, memory must match. FPSR is not compared; the lowering does not model the cumulative exception bits
|
||||
or QC.
|
||||
|
||||
Adapted from LAS Recomp tools/test_simd.py @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
|
||||
|
||||
Usage: simd.py [--encodings N] [--cases K] [--seed S] [--jobs J] [--no-validity] [--filter HEX/MASK]
|
||||
"""
|
||||
import argparse
|
||||
import functools
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
import unicorn as uc
|
||||
from unicorn import arm64_const as A
|
||||
|
||||
from common import BUF, BUF_SIZE, ROOT, UC_INVALID, Emu, build_harness, diff_state, dis, mnemonic, rand_fpcr, \
|
||||
rand_v, rand_x, run_harness
|
||||
from lifter import Lifter, Unsupported
|
||||
|
||||
print = functools.partial(print, flush=True)
|
||||
BASE_AREA = BUF + 0x8000
|
||||
CODE, CODE_SIZE = 0x7210000000, 0x400000
|
||||
|
||||
|
||||
def is_ldst(w):
|
||||
return (w >> 25) & 1 == 0 and (w >> 27) & 1 == 1
|
||||
|
||||
|
||||
def is_store(w):
|
||||
if not is_ldst(w):
|
||||
return False
|
||||
if (w & 0x3B000000) == 0x18000000: # literal
|
||||
return False
|
||||
return not (w >> 22) & 1 # L bit, same position for structures, pairs and single registers
|
||||
|
||||
|
||||
def is_literal(w):
|
||||
return (w & 0x3F000000) == 0x1C000000
|
||||
|
||||
|
||||
def class_key(w):
|
||||
"""Opcode signature: register fields (Rd, Rn, Rm) cleared."""
|
||||
if is_ldst(w):
|
||||
return w & 0xFFE0FC00 if (w & 0xBE000000) == 0x0C000000 else w & 0xFFFFFC00
|
||||
return w & 0xFFE0FC00
|
||||
|
||||
|
||||
def candidates(r):
|
||||
"""Every opcode bit pattern of the FP/SIMD data-processing and load/store spaces, random registers."""
|
||||
out = []
|
||||
for p in range(1 << 19):
|
||||
hi, mid = p >> 15, p & 0x7FFF
|
||||
rr = r.getrandbits(10)
|
||||
out.append((hi << 28) | (0b111 << 25) | (mid << 10) | rr) # data processing
|
||||
out.append((hi << 28) | (1 << 27) | (1 << 26) | (mid << 10) | rr) # loads/stores
|
||||
for sf in (0, 1):
|
||||
for C in (0, 1):
|
||||
for sz in range(4):
|
||||
for _ in range(4):
|
||||
out.append((sf << 31) | (0b0011010110 << 21) | (r.getrandbits(5) << 16) | (0b010 << 13) |
|
||||
(C << 12) | (sz << 10) | r.getrandbits(10))
|
||||
return out
|
||||
|
||||
|
||||
def validity(lifter, ws, emu):
|
||||
"""Map word -> (lifter accepts, Unicorn accepts); words are placed at CODE + 4i."""
|
||||
mu = emu.mu
|
||||
res = {}
|
||||
step = CODE_SIZE // 4
|
||||
for base in range(0, len(ws), step):
|
||||
chunk = ws[base:base + step]
|
||||
mu.mem_write(CODE, struct.pack(f'<{len(chunk)}I', *chunk))
|
||||
mu.ctl_remove_cache(CODE, CODE + CODE_SIZE)
|
||||
state = list(zip(emu.xregs, [BASE_AREA] * 31)) + [(A.UC_ARM64_REG_SP, BASE_AREA)]
|
||||
for i, w in enumerate(chunk):
|
||||
pc = CODE + 4 * i
|
||||
try:
|
||||
lifter.lift(w, pc)
|
||||
lok = True
|
||||
except Unsupported:
|
||||
lok = False
|
||||
if i % 32 == 0:
|
||||
mu.reg_write_batch(state)
|
||||
try:
|
||||
mu.emu_start(pc, pc + 4, count=1)
|
||||
uok = True
|
||||
except uc.UcError as e:
|
||||
uok = e.errno not in UC_INVALID
|
||||
res[w] = (lok, uok)
|
||||
return res
|
||||
|
||||
|
||||
def make_case(r, w):
|
||||
fpcr = rand_fpcr(r)
|
||||
v = [rand_v(r) for _ in range(32)]
|
||||
nzcv = r.getrandbits(4) << 28
|
||||
sp = BASE_AREA + r.randrange(0, 0x100, 16)
|
||||
if is_ldst(w):
|
||||
x = [BASE_AREA + r.randrange(0, 0x100) for _ in range(31)]
|
||||
rm = (w >> 16) & 31
|
||||
if rm < 31 and rm != (w >> 5) & 31:
|
||||
reg_off = (w & 0x3B200C00) == 0x38200800 # register offset
|
||||
post_reg = (w & 0xBF800000) in (0x0C800000, 0x0D800000)
|
||||
if reg_off or post_reg:
|
||||
x[rm] = r.randrange(0, 0x100)
|
||||
else:
|
||||
x = [rand_x(r) for _ in range(31)]
|
||||
return fpcr, x, sp, nzcv, v
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--encodings', type=int, default=20000)
|
||||
ap.add_argument('--cases', type=int, default=8)
|
||||
ap.add_argument('--seed', type=int, default=1)
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--no-validity', action='store_true', help='skip the full validity enumeration')
|
||||
ap.add_argument('--filter', help='only encodings with (w & MASK) == VAL, as VAL/MASK in hex')
|
||||
ap.add_argument('--show', type=int, default=40)
|
||||
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
|
||||
args = ap.parse_args()
|
||||
r = random.Random(args.seed)
|
||||
t0 = time.time()
|
||||
lifter = Lifter(0, 'c')
|
||||
emu = Emu([(CODE, bytes(CODE_SIZE))])
|
||||
cands = candidates(r)
|
||||
if args.filter:
|
||||
val, mask = (int(x, 16) for x in args.filter.split('/'))
|
||||
cands = [w for w in cands if (w & mask) == val]
|
||||
if args.no_validity:
|
||||
val = validity(lifter, r.sample(cands, min(len(cands), args.encodings * 4)), emu)
|
||||
else:
|
||||
val = validity(lifter, cands, emu)
|
||||
agree = Counter()
|
||||
only = defaultdict(list)
|
||||
for w, (lok, uok) in val.items():
|
||||
agree[(lok, uok)] += 1
|
||||
if lok != uok:
|
||||
only['lifter' if lok else 'unicorn'].append(w)
|
||||
print(f'validity: {len(val)} encodings, both valid {agree[(True, True)]}, both invalid {agree[(False, False)]}, '
|
||||
f'lifter-only {agree[(True, False)]}, unicorn-only {agree[(False, True)]} ({time.time() - t0:.0f}s)')
|
||||
for kind in ('lifter', 'unicorn'):
|
||||
ws = only[kind]
|
||||
if ws:
|
||||
print(f' {kind}-only by mnemonic: {dict(Counter(mnemonic(w) for w in ws).most_common(30))}')
|
||||
for w in ws[:args.show // 2]:
|
||||
print(f' {w:08x} {dis(w)}')
|
||||
|
||||
# Stratified sample of valid encodings: one per opcode signature first.
|
||||
valid = [w for w, (lok, uok) in val.items() if lok and uok]
|
||||
r.shuffle(valid)
|
||||
seen, pick = set(), []
|
||||
for w in valid:
|
||||
k = class_key(w)
|
||||
if k not in seen:
|
||||
seen.add(k)
|
||||
pick.append(w)
|
||||
picked = set(pick)
|
||||
pick = (pick + [w for w in valid if w not in picked])[:args.encodings]
|
||||
# Placed at CODE + 4i; literal loads get an offset into the code region.
|
||||
for i, w in enumerate(pick):
|
||||
pc = CODE + 4 * i
|
||||
if is_literal(w):
|
||||
tgt = CODE + r.randrange(0, (len(pick) * 4 + 0x1000) & ~3, 4)
|
||||
pick[i] = (w & ~(0x7FFFF << 5)) | ((((tgt - pc) >> 2) & 0x7FFFF) << 5)
|
||||
code_bytes = struct.pack(f'<{len(pick)}I', *pick) + bytes(0x1000 + 64)
|
||||
bodies = [lifter.lift(w, CODE + 4 * i).code for i, w in enumerate(pick)]
|
||||
work = ROOT / 'build' / 'test' / 'simd'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
t1 = time.time()
|
||||
exe = build_harness(work, bodies, args.jobs, args.cflags)
|
||||
print(f'compiled {len(bodies)} tests ({time.time() - t1:.0f}s)')
|
||||
|
||||
mu = emu.mu
|
||||
mu.mem_write(CODE, code_bytes)
|
||||
mu.ctl_remove_cache(CODE, CODE + CODE_SIZE)
|
||||
buf0 = bytes(r.getrandbits(8) for _ in range(4096)) * (BUF_SIZE // 4096)
|
||||
mu.mem_write(BUF, buf0)
|
||||
code_path = work / 'code.bin'
|
||||
code_path.write_bytes(code_bytes)
|
||||
cases, expect = [], []
|
||||
t2 = time.time()
|
||||
for k, w in enumerate(pick):
|
||||
pc = CODE + 4 * k
|
||||
st = is_store(w)
|
||||
for _ in range(args.cases):
|
||||
fpcr, x, sp, nzcv, v = make_case(r, w)
|
||||
try:
|
||||
ox, osp, onz, opc, ov = emu.run(pc, fpcr, x, sp, nzcv, v)
|
||||
except uc.UcError:
|
||||
if st:
|
||||
mu.mem_write(BUF, buf0)
|
||||
continue
|
||||
mh = 0
|
||||
if st:
|
||||
mh = zlib.crc32(bytes(mu.mem_read(BUF, BUF_SIZE)))
|
||||
mu.mem_write(BUF, buf0)
|
||||
cases.append((k, fpcr, x, sp, nzcv, v, st))
|
||||
expect.append((ox, osp, onz, opc, ov, mh))
|
||||
print(f'unicorn: {len(cases)} cases ({time.time() - t2:.0f}s)')
|
||||
|
||||
t3 = time.time()
|
||||
recs, err = run_harness(exe, work, buf0, cases, [(CODE, code_path)])
|
||||
print(f'harness: {len(recs)} of {len(cases)} cases ({time.time() - t3:.0f}s)')
|
||||
if err:
|
||||
k = cases[len(recs)][0] if len(recs) < len(cases) else None
|
||||
print(f'harness failed: {err}' + (f', stopped at {pick[k]:08x} {dis(pick[k])}' if k is not None else ''))
|
||||
|
||||
fails, nfail = {}, 0
|
||||
for case, want, got in zip(cases, expect, recs):
|
||||
k, v, st = case[0], case[5], case[6]
|
||||
d = diff_state(got, want, check_pc=False, check_mem=st)
|
||||
if d:
|
||||
nfail += 1
|
||||
w = pick[k]
|
||||
if w not in fails:
|
||||
ins = [f'v{j}={v[j]:032x}' for j in sorted({(w >> 5) & 31, (w >> 16) & 31, w & 31})]
|
||||
fails[w] = (case[1], d[:3], ins)
|
||||
print(f'{len(pick)} encodings, {len(cases)} cases: {nfail} failing cases, {len(fails)} failing encodings '
|
||||
f'({time.time() - t0:.0f}s total)')
|
||||
if fails:
|
||||
print(' failing by mnemonic:', dict(Counter(mnemonic(w) for w in fails).most_common(40)))
|
||||
for w, (fpcr, d, ins) in list(fails.items())[:args.show]:
|
||||
print(f' {w:08x} {dis(w):40} fpcr={fpcr:x} {"; ".join(d)}')
|
||||
print(f' {" ".join(ins)}')
|
||||
return 1 if fails or err else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user