Merge task/vloc: lifted functions keep their 8 most used V registers in host registers, 4-6% less guest time in a town

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-13 20:42:31 +03:00
commit 673229727d
6 files changed
+413 -67

No files matched your search

+4
View File
@@ -40,6 +40,10 @@ MacNativeGames cannot run on x86-64 as is: its CPU path executes ARM64 code nati
(`recomp/flags.py`, `recomp/regs.py`): registers are stored back where they changed, before calls, SVCs and
exits, and reloaded after them. A call reloads only X0-X18 and X30, as AAPCS64 has the callee preserve X19-X29 and
SP; `gen.py --no-callee-saved-trust` reloads those too.
- `gen.py --v-locals K` keeps the K most used V registers of each function in a local `u128 vr[32]` that clang
holds in XMM registers (`recomp/vlocals.py`, the `RT_VLOCALS` accessors of `runtime/simd.h`): loaded at entry,
stored where they may have changed before calls and exits, reloaded after calls; `--v-trust` does not reload
V8-V15, whose low halves AAPCS64 has the callee preserve. K is 8 by default; K = 0 keeps every V register in the `Cpu`.
- Coverage oracle: MacNativeGames' NSO rewriter lists every site where game code touches the OS (SVCs, TLS reads, counter reads, `x18`, exclusive windows, jump tables): 68,968 sites in 1.4.3 `main`. The lifter must handle every one.
- Memory: guest address equals host address, all below 2^39 (IPC encodes 39-bit addresses). The heap, stacks and the alias region are reserved with `MEM_WRITE_WATCH`, so the graphics back end can ask which pages the CPU wrote since the last upload (`runtime/memory.c`).
- Indirect branches go through `rt_dispatch` (`runtime/dispatch.c`), which looks the target up in the address table and tail-calls it.
+20 -5
View File
@@ -2,7 +2,7 @@
"""Generate C for the ExeFS modules: function discovery, then parallel emission into chunked C files.
Usage: gen.py --exefs build/extracted/v121/exefs --out generated/v121 [--modules main,subsdk0,sdk] [--jobs N]
[--limit N] [--chunk-fns N] [--chunk-bytes B] [--big-fn B] [--patches FILE]
[--limit N] [--chunk-fns N] [--chunk-bytes B] [--big-fn B] [--patches FILE] [--v-locals K [--v-trust]]
Output, under --out:
layout.h, layout.json module bases (recomp/layout.py)
@@ -46,6 +46,7 @@ import hostmem # noqa: E402
import jumptables # noqa: E402
import patches # noqa: E402
import regs # noqa: E402
import vlocals # noqa: E402
from check_content import GENERATED_MARKER # noqa: E402
from layout import layout, write as write_layout # noqa: E402
from lifter import hx # noqa: E402
@@ -211,13 +212,16 @@ def partition(est, max_fns, max_bytes, big_fn):
class Emitter:
"""C for the functions of one module. starts is the set of every function start of the module."""
def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True, spins=None):
def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True, spins=None,
v_locals=vlocals.LOCALS, v_trust=False):
self.mod = mod
self.starts = starts
self.patched = patched or {} # {address: [(set index, [replacement words])]} (recomp/patches.py)
self.spins = spins or {} # {spin wait back-edge: its head} (recomp/patches.py `spin`)
self.reg_locals = reg_locals # recomp/regs.py: registers per function in C locals
self.kept = regs.CALLEE_SAVED if callee_saved else frozenset() # not reloaded after a call
self.v_locals = v_locals # recomp/vlocals.py: V registers per function in C locals (0: all in c->v)
self.v_trust = v_trust # with v_locals: V8-V15 are not reloaded after a call
self.uses_patches = False
self.stats = Counter()
self.unsupported = Counter()
@@ -504,6 +508,8 @@ class Emitter:
else:
out += [line for _, line in body]
out.append('}')
if self.v_locals:
out = vlocals.rewrite('\n'.join(out), self.v_locals, self.v_trust, self.stats).split('\n')
self.stats['functions'] += 1
self.stats['instructions'] += len(insns)
return out, refs
@@ -521,8 +527,9 @@ def chunk_text(name, number, emitter, fns):
protos = ''.join(f'void {fname(r)}(Cpu *c);\n' for r in sorted(refs))
if emitter.uses_patches:
protos += 'extern uint8_t rt_patch_on[];\n'
return (header(f'{name} chunk {number:04d}') + '#include "cpu.h"\n#include "simd.h"\n\n' + protos + '\n'
+ '\n\n'.join(body) + '\n')
vdef = '#define RT_VLOCALS 1\n' if emitter.v_locals else '' # runtime/simd.h: the V-locals accessors
return (header(f'{name} chunk {number:04d}') + vdef + '#include "cpu.h"\n#include "simd.h"\n\n' + protos
+ '\n' + '\n\n'.join(body) + '\n')
def write_if_changed(path, text):
@@ -705,8 +712,15 @@ def main():
ap.add_argument('--no-callee-saved-trust', action='store_true',
help='reload X19-X29 and SP after every call too, instead of trusting AAPCS64 (slower; for '
'telling a broken callee from a generator bug)')
ap.add_argument('--v-locals', type=int, default=vlocals.LOCALS,
help='V registers per function kept in C locals that clang holds in XMM registers, the most used '
f'ones (recomp/vlocals.py; default {vlocals.LOCALS}, 0: all in the Cpu)')
ap.add_argument('--v-trust', action='store_true',
help='with --v-locals: do not reload V8-V15 after a call (AAPCS64: the callee preserves their low '
'64 bits)')
args = ap.parse_args()
opts = {'reg_locals': args.reg_locals, 'callee_saved': not args.no_callee_saved_trust}
opts = {'reg_locals': args.reg_locals, 'callee_saved': not args.no_callee_saved_trust, 'v_locals': args.v_locals,
'v_trust': args.v_trust}
t0 = time.perf_counter()
mods = layout(args.exefs)
@@ -828,6 +842,7 @@ def main():
'largest_chunk_bytes': max((r['bytes'] for r in rs), default=0),
'chunks_written': sum(r['written'] for r in rs) + table_written,
'stale_chunks_removed': removed,
**{k: v for k, v in sorted(stats.items()) if k.startswith('v_')}, # --v-locals only
},
'unsupported_census': dict(unsupported.most_common(50)),
'timings': {
+246
View File
@@ -0,0 +1,246 @@
"""V registers in C locals (gen.py --v-locals K): a rewrite of one emitted function's C.
The lifter writes V registers as `c->v[n]`, lane forms `((T *)&c->v[n])[i]`, the register macros of runtime/simd.h
(VREG, VQ, VW, VS32, ...) and helper calls that name registers by number (fp_op_s(c, op, d, n, m), ...). A chunk
generated with K > 0 defines RT_VLOCALS, under which those macros and helpers reach a register through a local array
`u128 vr[32]` when its bit in the function's VMASK is set, else through c->v; every access is a whole-vector load or
store, so clang's SROA keeps each local register in an XMM register. This module
- turns the direct forms into the accessors (c->v[n] = e -> VSET, c->v[n] -> VGET, lane forms -> VLANE/VLSET),
- picks the K registers the function names most often (VMASK),
- loads them at entry; before every call, rt_dispatch, rt_svc/rt_udf/rt_brk, rt_spin_gate and exit stores the ones
the function writes (VSYNC_OUT_D: only if the dirty mask may have the bit; the mask folds to a constant on most
paths); after a call reloads them, except V8-V15 with trust (AAPCS64: a callee preserves their low 64 bits, and a
conforming caller does not read their upper halves after the call).
Why the cap: 32 guest registers on 16 XMM registers spill (all 32 cost +20% instructions, the stack traffic tripled),
while K = 6..8 removed a quarter of the c->v traffic and kept dependent FP chains in registers; build/v143-vloc
NOTES.md has the measurements.
"""
import re
from collections import Counter
# Default K: 8 cut guest time 3.9-6.2% in the town on a 7900X against K = 0 (build/v143-vloc NOTES.md, series s1 and g1).
LOCALS = 8
# runtime/simd.h helpers that take VCTX, with their parameters after c. Register-number parameters are d (written,
# also read by some), n, m, a (read) and t (the first of up to 4 consecutive registers; written by the sl_ld* loads).
HELPERS = {
'fp_op_s': ['op', 'd', 'n', 'm'],
'fp_op_d': ['op', 'd', 'n', 'm'],
'fp_fma_s': ['na', 'nn', 'd', 'n', 'm', 'a'],
'fp_fma_d': ['na', 'nn', 'd', 'n', 'm', 'a'],
'sl_gen': ['op', 'sz', 'ne', 'd', 'szd', 'offd', 'keep', 'n', 'szn', 'offn', 'm', 'szm', 'offm', 'bidx', 'bimm'],
'sl_addlp': ['u', 'acc', 'Q', 'sz', 'd', 'n'],
'sl_across': ['op', 'u', 'Q', 'sz', 'd', 'n'],
'sl_pmull64': ['part', 'd', 'n', 'm'],
'sv_fred_s': ['op', 'd', 'n'],
'sv_fcvtl': ['sz', 'part', 'd', 'n'],
'sv_fcvtn': ['sz', 'part', 'odd', 'd', 'n'],
'sv_tbl': ['Q', 'len', 'tbx', 'd', 'n', 'm'],
'sv_aes': ['op', 'd', 'n'],
'sv_sha1hash': ['op', 'd', 'n', 'm'],
'sv_sha256hash': ['part1', 'd', 'n', 'm'],
'sv_sha_misc': ['op', 'd', 'n', 'm'],
'sl_ldm': ['Q', 'sz', 'selem', 'rpt', 't', 'ea'],
'sl_stm': ['Q', 'sz', 'selem', 'rpt', 't', 'ea'],
'sl_lds': ['sz', 'selem', 'idx', 't', 'ea'],
'sl_sts': ['sz', 'selem', 'idx', 't', 'ea'],
'sl_ldr': ['Q', 'sz', 'selem', 't', 'ea'],
}
REG_PARAMS = ('d', 'n', 'm', 'a', 't')
LANE_T = {'sa16': 'vu16', 'sa32': 'vu32', 'sa64': 'vu64', 'saf32': 'vf32', 'saf64': 'vf64', 'uint8_t': 'vu8',
'uint16_t': 'vu16', 'uint32_t': 'vu32', 'uint64_t': 'vu64', 'float': 'vf32', 'double': 'vf64'}
CALLEE_SAVED = frozenset(range(8, 16))
PLAIN = '\n const unsigned VMASK = 0; u128 *const vr = c->v; unsigned vd_ = 0, *const vdp = &vd_; (void)vr; (void)vdp;'
FN = re.compile(r'^(?:static )?void (?:f|fb)_[0-9a-f]+\(Cpu \*c\) \{$', re.M)
CALL = re.compile(r'\b(?:f|fb)_[0-9a-f]+\(c\);|\brt_dispatch\(c\);|\brt_(?:svc|udf|brk|spin_gate)\(c,[^;]*;')
RET = re.compile(r'(?:__attribute__\(\(musttail\)\)\s*)?\breturn\b[^;]*;')
RET_BLOCK = re.compile(r'(\{ [^{}]*?(?:__attribute__\(\(musttail\)\) )?return\b[^;]*; \})')
LANE = re.compile(r'\(\((\w+) \*\)&c->v\[')
WHOLE = re.compile(r'c->v\[')
USES = re.compile(r'\b(' + '|'.join(['VSET', 'VLSET', 'VGET', 'VLANE', 'VS8', 'VS16', 'VS32', 'VS64', 'VSF32', 'VSF64',
'VHI64', 'VREG', 'VQ', 'VW', 'FP_SETS', 'FP_SETD', 'SV_FOP_S', 'SV_FOP_D']
+ sorted(HELPERS)) + r')\(')
class Unsupported(ValueError):
pass
def _close(s, i):
"""s[i-1] opens a bracket; the index after its match"""
d = 1
while d:
d += {'[': 1, ']': -1}.get(s[i], 0)
i += 1
return i
def _stmt_end(s, i):
"""index of the ';' that ends the expression starting at s[i]"""
d = 0
while True:
ch = s[i]
if ch in '([{':
d += 1
elif ch in ')]}':
d -= 1
elif ch == ';' and d == 0:
return i
i += 1
def _args(s, i):
"""arguments of the call whose '(' is s[i]"""
d, cur, out = 0, '', []
while True:
ch = s[i]
if ch == '(':
d += 1
if d == 1:
i += 1
continue
elif ch == ')':
d -= 1
if d == 0:
out.append(cur.strip())
return out
elif ch == ',' and d == 1:
out.append(cur.strip())
cur = ''
i += 1
continue
cur += ch
i += 1
def accessors(t):
"""c->v forms of emitted C -> the RT_VLOCALS accessors of runtime/simd.h"""
out, i = [], 0
while True:
m1, m2 = LANE.search(t, i), WHOLE.search(t, i)
m = min((x for x in (m1, m2) if x), key=lambda x: x.start(), default=None)
if not m:
out.append(t[i:])
return ''.join(out)
out.append(t[i:m.start()])
if m is m1:
e = _close(t, m.end())
if m.group(1) not in LANE_T or t[e:e + 2] != ')[':
raise Unsupported(f'V register pointer form {t[m.start():e + 10]!r}')
je = _close(t, e + 2)
reg, lane, vt = accessors(t[m.end():e - 1]), t[e + 2:je - 1], LANE_T[m.group(1)]
a = re.match(r'\s*=(?!=)', t[je:])
if a:
se = _stmt_end(t, je + a.end())
out.append(f'VLSET({reg}, {vt}, {lane}, {accessors(t[je + a.end():se]).strip()})')
i = se
else:
out.append(f'VLANE({reg}, {vt}, {lane})')
i = je
continue
if t[m.start() - 1:m.start()] == '&':
raise Unsupported(f'V register address {t[m.start() - 1:m.start() + 20]!r}')
e = _close(t, m.end())
reg = accessors(t[m.end():e - 1])
a = re.match(r'\s*=(?!=)', t[e:])
if a:
se = _stmt_end(t, e + a.end())
out.append(f'VSET({reg}, {accessors(t[e + a.end():se]).strip()})')
i = se
else:
out.append(f'VGET({reg})')
i = e
def _lit(a):
a = a.strip().rstrip('uUlL')
if re.fullmatch(r'-\d+', a):
return -1
return int(a, 0) if re.fullmatch(r'0x[0-9a-fA-F]+|\d+', a) else None
def registers(body):
"""(Counter of register uses, set of written registers) of a body in accessor form"""
used, wr = Counter(), set()
for m in USES.finditer(body):
name = m.group(1)
args = _args(body, m.end() - 1)
if name in HELPERS:
for p, a in zip(['c'] + HELPERS[name], args):
if p not in REG_PARAMS:
continue
v = _lit(a)
if v == -1:
continue
if v is None:
raise Unsupported(f'register argument {a!r} of {name}')
span = 4 if p == 't' or (name == 'sv_tbl' and p == 'n') else 1
rs = [(v + k) & 31 for k in range(span)]
used.update(rs)
if p == 'd' or (p == 't' and name.startswith('sl_ld')):
wr.update(rs)
continue
pos, write = {'VREG': (1, False), 'VQ': (2, False), 'VW': (2, True), 'SV_FOP_S': (2, True),
'SV_FOP_D': (2, True), 'VSET': (0, True), 'VLSET': (0, True), 'FP_SETS': (0, True),
'FP_SETD': (0, True)}.get(name, (0, False))
v = _lit(args[pos])
if v is None or v < 0:
raise Unsupported(f'register {args[pos]!r} of {name}')
used[v] += 1
if write:
wr.add(v)
return used, wr
def rewrite_one(body, k, trust, stats):
"""Body of one C function (text between its braces)."""
try:
new = accessors(body)
cnt, written = registers(new)
except Unsupported:
# a form this rewrite does not know: every register stays in c->v (VMASK 0)
stats['v_unsupported_functions'] += 1
return PLAIN + body
body = new
if not cnt:
return body
local = {r for r, _ in sorted(cnt.items(), key=lambda x: (-x[1], x[0]))[:k]}
written &= local
load = ''.join(f'VSYNC_IN({r}); ' for r in sorted(local))
reload = ''.join(f'VSYNC_IN({r}); ' for r in sorted(local - (CALLEE_SAVED if trust else set())))
store = ''.join(f'VSYNC_OUT_D({r}); ' for r in sorted(written))
stats['v_local_functions'] += 1
stats['v_local_registers'] += len(local)
def at_ret(m):
stats['v_exit_syncs'] += 1
return '{ ' + store + m.group(0) + ' }'
def at_call(m):
stats['v_call_syncs'] += 1
return '{ ' + store + m.group(0) + ' ' + reload + 'vd_ = 0; }'
body = RET.sub(at_ret, body)
parts = RET_BLOCK.split(body)
body = ''.join(p if i % 2 else CALL.sub(at_call, p) for i, p in enumerate(parts))
mask = sum(1 << r for r in local)
return (f'\n const unsigned VMASK = {mask:#x}u; u128 vr[32]; unsigned vd_ = 0, *const vdp = &vd_; {load.rstrip()}'
+ body + ('\n ' + store.rstrip() if store else ''))
def rewrite(text, k, trust, stats):
"""C text of one emitted guest function (an f_ and, for a computed branch, its fb_) with K registers local."""
out, pos = [], 0
for m in FN.finditer(text):
d, i = 1, m.end()
while d:
d += {'{': 1, '}': -1}.get(text[i], 0)
i += 1
out.append(text[pos:m.end()])
out.append(rewrite_one(text[m.end():i - 1], k, trust, stats) + text[i - 1])
pos = i
out.append(text[pos:])
return ''.join(out)
+128 -58
View File
@@ -34,6 +34,45 @@ typedef u128 __attribute__((aligned(1), may_alias)) u128u;
// ---- registers: VREG = whole V register as a vector type; VQ = source with the upper half cleared when
// Q=0; VW = write, zeroing the upper half when Q=0. Scalar views of lane 0: VS8..VS64, VSF32/VSF64.
// The helpers below reach registers through VGET/VSET (whole register), VP (address), VPR (address of a copy to read)
// and VPW (address to write), and take VCTX instead of Cpu *c.
// RT_VLOCALS (a chunk generated with gen.py --v-locals K defines it): a function keeps some V registers in `u128 vr[32]`;
// VMASK (a constant in the function) has a bit per register held there, the rest stay in c->v, and every write sets
// the register's bit in *vdp (the dirty mask that lets the syncs at calls and exits skip clean registers). Each access
// is a whole-vector load or store, so clang's SROA turns the used elements of vr into SSA vectors in XMM registers.
#ifdef RT_VLOCALS
#define VCTX Cpu *c, u128 *vr, unsigned VMASK, unsigned *vdp
#define VP(n) ((VMASK >> (n) & 1u) ? &vr[n] : &c->v[n])
#define VV(n) (*(vu64 *)VP(n))
#define VDIRTY(n) (*vdp |= 1u << ((n) & 31))
#define VGET(n) ((u128)VV(n))
#define VSET(n, x) ({ u128 _vx = (u128)(x); VDIRTY(n); VV(n) = (vu64){(uint64_t)_vx, (uint64_t)(_vx >> 64)}; })
#define VPR(n) (&(u128){VGET(n)})
#define VPW(n) (VDIRTY(n), VP(n))
#define VREG(T, n) ((T)VV(n))
#define VQ(T, Q, n) ((T)((Q) ? VV(n) : VV(n) & (vu64){~0ull, 0}))
#define VW(T, Q, d, e) do { T _w = (e); VDIRTY(d); VV(d) = (Q) ? (vu64)_w : (vu64)_w & (vu64){~0ull, 0}; } while (0)
#define VLANE(n, VT, i) (((VT)VV(n))[i])
#define VLSET(n, VT, i, x) do { VT _l = (VT)VV(n); _l[i] = (x); VDIRTY(n); VV(n) = (vu64)_l; } while (0)
#define VS8(n) VLANE(n, vu8, 0)
#define VS16(n) VLANE(n, vu16, 0)
#define VS32(n) VLANE(n, vu32, 0)
#define VS64(n) VLANE(n, vu64, 0)
#define VSF32(n) VLANE(n, vf32, 0)
#define VSF64(n) VLANE(n, vf64, 0)
#define VHI64(n) VLANE(n, vu64, 1)
// the generated syncs: copy a local register in from c->v, out to c->v, out only if it may be dirty (the mask folds
// to a constant on most paths, and a clean register's store then disappears)
#define VSYNC_IN(r) (*(vu64 *)&vr[r] = *(vu64 *)&c->v[r])
#define VSYNC_OUT(r) (*(vu64 *)&c->v[r] = *(vu64 *)&vr[r])
#define VSYNC_OUT_D(r) do { if (!__builtin_constant_p(*vdp >> (r) & 1) || (*vdp >> (r) & 1)) VSYNC_OUT(r); } while (0)
#else
#define VCTX Cpu *c
#define VP(n) (&c->v[n])
#define VGET(n) (c->v[n])
#define VSET(n, x) (c->v[n] = (x))
#define VPR(n) VP(n)
#define VPW(n) VP(n)
#define VREG(T, n) (*(T *)&c->v[n])
#define VQ(T, Q, n) ((T)((Q) ? c->v[n] : (u128)(uint64_t)c->v[n]))
#define VW(T, Q, d, e) do { T _w = (e); c->v[d] = (Q) ? (u128)_w : (u128)(uint64_t)(u128)_w; } while (0)
@@ -44,6 +83,7 @@ typedef u128 __attribute__((aligned(1), may_alias)) u128u;
#define VSF32(n) (*(saf32 *)&c->v[n])
#define VSF64(n) (*(saf64 *)&c->v[n])
#define VHI64(n) (((sa64 *)&c->v[n])[1])
#endif
#define LD128(a) (*(const u128u *)(uintptr_t)(a))
#define ST128(a, v) (*(u128u *)(uintptr_t)(a) = (u128)(v))
#define SHUF __builtin_shufflevector
@@ -384,18 +424,23 @@ SI float simd_h2s(Cpu *c, uint16_t h) {
}
// ---- scalar FP instructions (FP data processing)
#ifdef RT_VLOCALS
#define FP_SETS(d, x) (VDIRTY(d), VV(d) = (vu64)(vu32){fb_s(x), 0, 0, 0})
#define FP_SETD(d, x) (VDIRTY(d), VV(d) = (vu64){fb_d(x), 0})
#else
#define FP_SETS(d, x) (c->v[d] = fb_s(x))
#define FP_SETD(d, x) (c->v[d] = fb_d(x))
SI void fp_op_s(Cpu *c, int op, int d, int n, int m) { FP_SETS(d, op_s(c, op, VSF32(n), VSF32(m), 0)); }
SI void fp_op_d(Cpu *c, int op, int d, int n, int m) { FP_SETD(d, op_d(c, op, VSF64(n), VSF64(m), 0)); }
#endif
SI void fp_op_s(VCTX, int op, int d, int n, int m) { FP_SETS(d, op_s(c, op, VSF32(n), VSF32(m), 0)); }
SI void fp_op_d(VCTX, int op, int d, int n, int m) { FP_SETD(d, op_d(c, op, VSF64(n), VSF64(m), 0)); }
// FMADD family: neg_a / neg_n negate the addend / first factor before the fused op
SI void fp_fma_s(Cpu *c, int na, int nn, int d, int n, int m, int a) {
SI void fp_fma_s(VCTX, int na, int nn, int d, int n, int m, int a) {
float x = VSF32(a), y = VSF32(n);
if (na) x = neg_s(x);
if (nn) y = neg_s(y);
FP_SETS(d, op_s(c, FOP_MADD, x, y, VSF32(m)));
}
SI void fp_fma_d(Cpu *c, int na, int nn, int d, int n, int m, int a) {
SI void fp_fma_d(VCTX, int na, int nn, int d, int n, int m, int a) {
double x = VSF64(a), y = VSF64(n);
if (na) x = neg_d(x);
if (nn) y = neg_d(y);
@@ -561,28 +606,28 @@ SI uint64_t lop(int op, int sz, uint64_t a, uint64_t b, uint64_t d) {
// Generic lane loop: ne lanes; result lane i (size szd) goes to lane offd+i of d (other lanes kept when keep,
// else zero); a = n lane offn+i (size szn), b = m lane offm+i (size szm), or lane bidx of m when bidx >= 0,
// or the constant bimm when m < 0; the accumulator is d's old lane offd+i.
SI void sl_gen(Cpu *c, int op, int sz, int ne, int d, int szd, int offd, int keep, int n, int szn, int offn, int m,
SI void sl_gen(VCTX, int op, int sz, int ne, int d, int szd, int offd, int keep, int n, int szn, int offn, int m,
int szm, int offm, int bidx, uint64_t bimm) {
u128 a = c->v[n], b = m >= 0 ? c->v[m] : 0, o = c->v[d], r = keep ? o : 0;
u128 a = VGET(n), b = m >= 0 ? VGET(m) : 0, o = VGET(d), r = keep ? o : 0;
for (int i = 0; i < ne; i++) {
uint64_t y = m < 0 ? bimm : lget(&b, szm, bidx >= 0 ? bidx : offm + i);
lset(&r, szd, offd + i, lop(op, sz, lget(&a, szn, offn + i), y, lget(&o, szd, offd + i)));
}
c->v[d] = r;
VSET(d, r);
}
// SADDLP/UADDLP/SADALP/UADALP
SI void sl_addlp(Cpu *c, int u, int acc, int Q, int sz, int d, int n) {
u128 a = c->v[n], o = c->v[d], r = 0;
SI void sl_addlp(VCTX, int u, int acc, int Q, int sz, int d, int n) {
u128 a = VGET(n), o = VGET(d), r = 0;
for (int i = 0; i < (4 << Q) >> sz; i++) {
uint64_t x = lget(&a, sz, 2 * i), y = lget(&a, sz, 2 * i + 1);
uint64_t s = u ? x + y : (uint64_t)(lsx(x, sz) + lsx(y, sz));
lset(&r, sz + 1, i, s + (acc ? lget(&o, sz + 1, i) : 0));
}
c->v[d] = r;
VSET(d, r);
}
// ADDV, SADDLV/UADDLV, SMAXV/UMAXV, SMINV/UMINV (op 0..3; u = unsigned)
SI void sl_across(Cpu *c, int op, int u, int Q, int sz, int d, int n) {
u128 a = c->v[n];
SI void sl_across(VCTX, int op, int u, int Q, int sz, int d, int n) {
u128 a = VGET(n);
int ne = (8 << Q) >> sz;
int64_t acc = u ? (int64_t)lget(&a, sz, 0) : lsx(lget(&a, sz, 0), sz);
for (int i = 1; i < ne; i++) {
@@ -591,12 +636,12 @@ SI void sl_across(Cpu *c, int op, int u, int Q, int sz, int d, int n) {
else if (op == 2) acc = u ? ((uint64_t)x > (uint64_t)acc ? x : acc) : (x > acc ? x : acc);
else acc = u ? ((uint64_t)x < (uint64_t)acc ? x : acc) : (x < acc ? x : acc);
}
c->v[d] = (uint64_t)acc & lmask(op == 1 ? sz + 1 : sz);
VSET(d, (uint64_t)acc & lmask(op == 1 ? sz + 1 : sz));
}
// PMULL2/PMULL 1Q (64x64 -> 128 carry-less)
SI void sl_pmull64(Cpu *c, int part, int d, int n, int m) {
uint64_t hi, lo = pmul64(((sa64 *)&c->v[n])[part], ((sa64 *)&c->v[m])[part], &hi);
c->v[d] = ((u128)hi << 64) | lo;
SI void sl_pmull64(VCTX, int part, int d, int n, int m) {
uint64_t hi, lo = pmul64(((sa64 *)VP(n))[part], ((sa64 *)VP(m))[part], &hi);
VSET(d, ((u128)hi << 64) | lo);
}
// ---- FP vector helpers: lanes of the 64-bit form are computed on a zero upper half
@@ -609,9 +654,9 @@ SI vf64 vneg_d(vf64 v) { return (vf64)((vu64)v ^ 0x8000000000000000ull); }
SI float pair_s(Cpu *c, int op, float a, float b) { return op_s(c, op, a, b, 0); }
SI double pair_d(Cpu *c, int op, double a, double b) { return op_d(c, op, a, b, 0); }
// FMAXV/FMINV/FMAXNMV/FMINNMV (4S): pairwise tree
SI void sv_fred_s(Cpu *c, int op, int d, int n) {
SI void sv_fred_s(VCTX, int op, int d, int n) {
vf32 v = VREG(vf32, n);
c->v[d] = fb_s(pair_s(c, op, pair_s(c, op, v[0], v[1]), pair_s(c, op, v[2], v[3])));
VSET(d, fb_s(pair_s(c, op, pair_s(c, op, v[0], v[1]), pair_s(c, op, v[2], v[3]))));
}
// lane-wise FP conversions
SI vu32 vtoint_s(Cpu *c, vf32 v, int rm, int fbits, int uns) {
@@ -645,8 +690,8 @@ SI vf32 vrsqrte_s(Cpu *c, vf32 v) { for (int i = 0; i < 4; i++) v[i] = rsqrte_s(
SI vf64 vrsqrte_d(Cpu *c, vf64 v) { for (int i = 0; i < 2; i++) v[i] = rsqrte_d(c, v[i]); return v; }
// FCVTL/FCVTL2 (h->s, s->d), FCVTN/FCVTN2 (s->h, d->s), FCVTXN/FCVTXN2
SI void sv_fcvtl(Cpu *c, int sz, int part, int d, int n) {
u128 a = c->v[n], r = 0;
SI void sv_fcvtl(VCTX, int sz, int part, int d, int n) {
u128 a = VGET(n), r = 0;
if (sz == 0) {
vf32 o;
for (int i = 0; i < 4; i++) o[i] = simd_h2s(c, (uint16_t)lget(&a, 1, 4 * part + i));
@@ -656,10 +701,10 @@ SI void sv_fcvtl(Cpu *c, int sz, int part, int d, int n) {
for (int i = 0; i < 2; i++) o[i] = simd_s2d(c, fv_s((uint32_t)lget(&a, 2, 2 * part + i)));
r = (u128)o;
}
c->v[d] = r;
VSET(d, r);
}
SI void sv_fcvtn(Cpu *c, int sz, int part, int odd, int d, int n) {
u128 a = c->v[n], r = part ? (uint64_t)c->v[d] : 0;
SI void sv_fcvtn(VCTX, int sz, int part, int odd, int d, int n) {
u128 a = VGET(n), r = part ? (uint64_t)VGET(d) : 0;
if (sz == 0) {
for (int i = 0; i < 4; i++) lset(&r, 1, 4 * part + i, simd_s2h(c, fv_s((uint32_t)lget(&a, 2, i))));
} else {
@@ -668,19 +713,19 @@ SI void sv_fcvtn(Cpu *c, int sz, int part, int odd, int d, int n) {
lset(&r, 2, 2 * part + i, fb_s(odd ? simd_d2s_odd(c, x) : simd_d2s(c, x)));
}
}
c->v[d] = r;
VSET(d, r);
}
// TBL / TBX with len+1 table registers
SI void sv_tbl(Cpu *c, int Q, int len, int tbx, int d, int n, int m) {
SI void sv_tbl(VCTX, int Q, int len, int tbx, int d, int n, int m) {
uint8_t tab[64];
for (int i = 0; i <= len; i++) memcpy(tab + 16 * i, &c->v[(n + i) & 31], 16);
u128 idx = c->v[m], o = c->v[d], r = 0;
for (int i = 0; i <= len; i++) memcpy(tab + 16 * i, VP((n + i) & 31), 16);
u128 idx = VGET(m), o = VGET(d), r = 0;
for (int i = 0; i < 8 << Q; i++) {
unsigned k = (unsigned)lget(&idx, 0, i);
lset(&r, 0, i, k < 16u * (len + 1) ? tab[k] : tbx ? lget(&o, 0, i) : 0);
}
c->v[d] = r;
VSET(d, r);
}
// ---- CRC32 / CRC32C (bytes = 1, 2, 4, 8)
@@ -697,9 +742,9 @@ SI uint8_t aes_mul(uint8_t x, int k) { // k in {9, 11, 13, 14}
uint8_t x2 = aes_xt(x), x4 = aes_xt(x2), x8 = aes_xt(x4);
return (uint8_t)(x8 ^ (k & 4 ? x4 : 0) ^ (k & 2 ? x2 : 0) ^ (k & 1 ? x : 0));
}
SI void sv_aes(Cpu *c, int op, int d, int n) { // 0 AESE, 1 AESD, 2 AESMC, 3 AESIMC
SI void sv_aes(VCTX, int op, int d, int n) { // 0 AESE, 1 AESD, 2 AESMC, 3 AESIMC
uint8_t s[16], r[16];
u128 v = op < 2 ? c->v[d] ^ c->v[n] : c->v[n];
u128 v = op < 2 ? VGET(d) ^ VGET(n) : VGET(n);
memcpy(s, &v, 16);
if (op == 0) {
for (int i = 0; i < 16; i++) r[i] = simd_aes_sbox[s[(i + 4 * (i & 3)) & 15]];
@@ -716,14 +761,14 @@ SI void sv_aes(Cpu *c, int op, int d, int n) { // 0 AESE, 1 AESD, 2 AESMC, 3 AES
}
}
memcpy(&v, r, 16);
c->v[d] = v;
VSET(d, v);
}
SI uint32_t rol32_(uint32_t x, int s) { return (x << s) | (x >> (32 - s)); }
SI uint32_t ror32_(uint32_t x, int s) { return (x >> s) | (x << (32 - s)); }
SI void sv_sha1hash(Cpu *c, int op, int d, int n, int m) { // 0 SHA1C, 1 SHA1P, 2 SHA1M
uint32_t x[4], w[4], y = (uint32_t)c->v[n];
memcpy(x, &c->v[d], 16);
memcpy(w, &c->v[m], 16);
SI void sv_sha1hash(VCTX, int op, int d, int n, int m) { // 0 SHA1C, 1 SHA1P, 2 SHA1M
uint32_t x[4], w[4], y = (uint32_t)VGET(n);
memcpy(x, VP(d), 16);
memcpy(w, VP(m), 16);
for (int e = 0; e < 4; e++) {
uint32_t t = op == 0 ? (((x[2] ^ x[3]) & x[1]) ^ x[3]) : op == 1 ? (x[1] ^ x[2] ^ x[3])
: ((x[1] & x[2]) | ((x[1] | x[2]) & x[3]));
@@ -732,13 +777,13 @@ SI void sv_sha1hash(Cpu *c, int op, int d, int n, int m) { // 0 SHA1C, 1 SHA1P,
uint32_t top = x[3];
x[3] = x[2], x[2] = x[1], x[1] = x[0], x[0] = y, y = top;
}
memcpy(&c->v[d], x, 16);
memcpy(VPW(d), x, 16);
}
SI void sv_sha256hash(Cpu *c, int part1, int d, int n, int m) { // SHA256H (part1), SHA256H2
SI void sv_sha256hash(VCTX, int part1, int d, int n, int m) { // SHA256H (part1), SHA256H2
uint32_t x[4], y[4], w[4];
memcpy(part1 ? x : y, &c->v[d], 16);
memcpy(part1 ? y : x, &c->v[n], 16);
memcpy(w, &c->v[m], 16);
memcpy(part1 ? x : y, VP(d), 16);
memcpy(part1 ? y : x, VP(n), 16);
memcpy(w, VP(m), 16);
for (int e = 0; e < 4; e++) {
uint32_t chs = ((y[1] ^ y[2]) & y[0]) ^ y[2];
uint32_t maj = (x[0] & x[1]) | ((x[0] | x[1]) & x[2]);
@@ -749,13 +794,13 @@ SI void sv_sha256hash(Cpu *c, int part1, int d, int n, int m) { // SHA256H (part
y[3] = y[2], y[2] = y[1], y[1] = y[0], y[0] = x[3];
x[3] = x[2], x[2] = x[1], x[1] = x[0], x[0] = t;
}
memcpy(&c->v[d], part1 ? x : y, 16);
memcpy(VPW(d), part1 ? x : y, 16);
}
SI void sv_sha_misc(Cpu *c, int op, int d, int n, int m) { // 0 SHA1SU0, 1 SHA1SU1, 2 SHA256SU0, 3 SHA256SU1, 4 SHA1H
SI void sv_sha_misc(VCTX, int op, int d, int n, int m) { // 0 SHA1SU0, 1 SHA1SU1, 2 SHA256SU0, 3 SHA256SU1, 4 SHA1H
uint32_t a[4], b[4], k[4], r[4], t[4];
memcpy(a, &c->v[d], 16);
memcpy(b, &c->v[n], 16);
memcpy(k, &c->v[m], 16);
memcpy(a, VP(d), 16);
memcpy(b, VP(n), 16);
memcpy(k, VP(m), 16);
switch (op) {
case 0:
t[0] = a[2], t[1] = a[3], t[2] = b[0], t[3] = b[1];
@@ -779,10 +824,10 @@ SI void sv_sha_misc(Cpu *c, int op, int d, int n, int m) { // 0 SHA1SU0, 1 SHA1S
break;
}
default:
c->v[d] = rol32_(b[0], 30);
VSET(d, rol32_(b[0], 30));
return;
}
memcpy(&c->v[d], r, 16);
memcpy(VPW(d), r, 16);
}
// ---- loads / stores of lanes
@@ -803,39 +848,64 @@ SI void simd_st(uint64_t a, int sz, uint64_t v) {
}
}
// LDn/STn (multiple structures): selem interleaved registers, rpt repeats (LD1 with 2-4 registers)
SI void sl_ldm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) {
SI void sl_ldm(VCTX, int Q, int sz, int selem, int rpt, int t, uint64_t ea) {
u128 r[4] = {0, 0, 0, 0};
int ne = (8 << Q) >> sz;
for (int k = 0; k < rpt; k++)
for (int e = 0; e < ne; e++)
for (int s = 0; s < selem; s++, ea += 1 << sz) lset(&r[k + s], sz, e, simd_ld(ea, sz));
for (int i = 0; i < rpt + selem - 1; i++) c->v[(t + i) & 31] = r[i];
for (int i = 0; i < rpt + selem - 1; i++) VSET((t + i) & 31, r[i]);
}
SI void sl_stm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) {
SI void sl_stm(VCTX, int Q, int sz, int selem, int rpt, int t, uint64_t ea) {
u128 r[4];
int ne = (8 << Q) >> sz;
for (int i = 0; i < rpt + selem - 1; i++) r[i] = c->v[(t + i) & 31];
for (int i = 0; i < rpt + selem - 1; i++) r[i] = VGET((t + i) & 31);
for (int k = 0; k < rpt; k++)
for (int e = 0; e < ne; e++)
for (int s = 0; s < selem; s++, ea += 1 << sz) simd_st(ea, sz, lget(&r[k + s], sz, e));
}
// single structure: lane idx of selem registers; LDnR replicates into every lane
SI void sl_lds(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) {
SI void sl_lds(VCTX, int sz, int selem, int idx, int t, uint64_t ea) {
for (int s = 0; s < selem; s++) {
u128 v = c->v[(t + s) & 31];
u128 v = VGET((t + s) & 31);
lset(&v, sz, idx, simd_ld(ea + (s << sz), sz));
c->v[(t + s) & 31] = v;
VSET((t + s) & 31, v);
}
}
SI void sl_sts(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) {
for (int s = 0; s < selem; s++) simd_st(ea + (s << sz), sz, lget(&c->v[(t + s) & 31], sz, idx));
SI void sl_sts(VCTX, int sz, int selem, int idx, int t, uint64_t ea) {
for (int s = 0; s < selem; s++) simd_st(ea + (s << sz), sz, lget(VPR((t + s) & 31), sz, idx));
}
SI void sl_ldr(Cpu *c, int Q, int sz, int selem, int t, uint64_t ea) {
SI void sl_ldr(VCTX, int Q, int sz, int selem, int t, uint64_t ea) {
uint64_t x[4];
for (int s = 0; s < selem; s++) x[s] = simd_ld(ea + (s << sz), sz);
for (int s = 0; s < selem; s++) {
u128 v = 0;
for (int e = 0; e < (8 << Q) >> sz; e++) lset(&v, sz, e, x[s]);
c->v[(t + s) & 31] = v;
VSET((t + s) & 31, v);
}
}
// RT_VLOCALS: the generated code calls the helpers above with c alone; they also take the locals
#ifdef RT_VLOCALS
#define fp_op_s(c, ...) fp_op_s(c, vr, VMASK, vdp, __VA_ARGS__)
#define fp_op_d(c, ...) fp_op_d(c, vr, VMASK, vdp, __VA_ARGS__)
#define fp_fma_s(c, ...) fp_fma_s(c, vr, VMASK, vdp, __VA_ARGS__)
#define fp_fma_d(c, ...) fp_fma_d(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_gen(c, ...) sl_gen(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_addlp(c, ...) sl_addlp(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_across(c, ...) sl_across(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_pmull64(c, ...) sl_pmull64(c, vr, VMASK, vdp, __VA_ARGS__)
#define sv_fred_s(c, ...) sv_fred_s(c, vr, VMASK, vdp, __VA_ARGS__)
#define sv_fcvtl(c, ...) sv_fcvtl(c, vr, VMASK, vdp, __VA_ARGS__)
#define sv_fcvtn(c, ...) sv_fcvtn(c, vr, VMASK, vdp, __VA_ARGS__)
#define sv_tbl(c, ...) sv_tbl(c, vr, VMASK, vdp, __VA_ARGS__)
#define sv_aes(c, ...) sv_aes(c, vr, VMASK, vdp, __VA_ARGS__)
#define sv_sha1hash(c, ...) sv_sha1hash(c, vr, VMASK, vdp, __VA_ARGS__)
#define sv_sha256hash(c, ...) sv_sha256hash(c, vr, VMASK, vdp, __VA_ARGS__)
#define sv_sha_misc(c, ...) sv_sha_misc(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_ldm(c, ...) sl_ldm(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_stm(c, ...) sl_stm(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_lds(c, ...) sl_lds(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_sts(c, ...) sl_sts(c, vr, VMASK, vdp, __VA_ARGS__)
#define sl_ldr(c, ...) sl_ldr(c, vr, VMASK, vdp, __VA_ARGS__)
#endif
+11 -4
View File
@@ -39,6 +39,7 @@ from unicorn import arm64_const as A
from common import BUF, BUF_SIZE, MSYS_BIN, ROOT, Emu, compiler, diff_state, rand_fpcr, rand_v, rand_x, run_harness
import gen
import patches
import vlocals
from layout import layout
from module import Module
@@ -83,7 +84,7 @@ def simd_mask(w):
return ((w & 0x0E000000) == 0x0E000000) | (((w & 0x0A000000) == 0x08000000) & (((w >> 26) & 1) == 1))
def pick_functions(name, mod, count, r, spins=None):
def pick_functions(name, mod, count, r, spins=None, **opts):
"""[(entry, end, tags)] of up to count functions of mod, plus the Emitter that will emit them. spins: {spin wait
back-edge: head}, emitted with gates."""
starts, _ = gen.discover(mod)
@@ -107,7 +108,7 @@ def pick_functions(name, mod, count, r, spins=None):
with_spin = sorted({int(i) for i in gated} & set(cand.tolist()))
for pool in (with_spin, with_br, with_call, rest):
r.shuffle(pool)
emitter = gen.Emitter(mod, set((mod.lo + starts * 4).tolist()), spins=spins)
emitter = gen.Emitter(mod, set((mod.lo + starts * 4).tolist()), spins=spins, **opts)
picks, tags_seen = [], Counter()
# Functions with a spin wait first, then with a br (jump tables, fallbacks), at most a quarter of the picks each;
# then functions with calls, at most half.
@@ -346,12 +347,17 @@ def main():
ap.add_argument('--show', type=int, default=20)
ap.add_argument('--patches', type=Path, help='patch table whose spin waits are emitted with their gates')
ap.add_argument('--cflags', default='-march=x86-64-v3' if platform.machine().lower() in ('amd64', 'x86_64') else '')
ap.add_argument('--v-locals', type=int, default=vlocals.LOCALS,
help='gen.py --v-locals: V registers per function in locals')
ap.add_argument('--v-trust', action='store_true', help='gen.py --v-trust: V8-V15 not reloaded after a call')
ap.add_argument('--work', type=Path, default=ROOT / 'build' / 'test' / 'func',
help='work directory (chunks, harness, images); one per concurrent run')
args = ap.parse_args()
r = random.Random(args.seed)
t0 = time.time()
want = args.modules.split(',')
work = ROOT / 'build' / 'test' / 'func'
work = args.work
work.mkdir(parents=True, exist_ok=True)
images, emitted, tags = [], [], []
mods = layout(args.exefs)
@@ -363,7 +369,8 @@ def main():
img = work / f'{name}.img'
img.write_bytes(mod.image)
images.append((base, img, mod.image))
picks, emitter = pick_functions(name, mod, args.functions, r, table.spins_for(name, base))
picks, emitter = pick_functions(name, mod, args.functions, r, table.spins_for(name, base),
v_locals=args.v_locals, v_trust=args.v_trust)
for entry, end, t in picks:
emitted.append((name, entry, end, emitter))
tags.append(t)
+4
View File
@@ -845,6 +845,8 @@ def main():
help='profile-guided optimisation, generated code at -O2: gen builds an instrumented totk.exe '
'(needs --toolchain on Windows: MSYS2 clang ships no profile runtime); FILE builds with that '
'.profdata (tools/pgo.py merge)')
ap.add_argument('--gen-o2', action='store_true',
help='compile the generated code at -O2 without a profile (the --pgo level), for A/B builds')
ap.add_argument('--expected', type=Path,
help='expected hashes (default tools/data/expected-<display version>.json)')
args = ap.parse_args()
@@ -857,6 +859,8 @@ def main():
sys.exit('--pgo gen: MSYS2 UCRT64 clang ships no profile runtime (clang_rt.profile); pass --toolchain with '
'an llvm-mingw directory')
set_pgo(args.pgo, out)
elif args.gen_o2:
GEN_FLAGS[GEN_FLAGS.index('-O1')] = '-O2'
extracted = None if args.exefs and args.romfs else work_dir() / 'extracted' / args.version
paths = {'exefs': (args.exefs or extracted / 'exefs').resolve(),
'romfs': (args.romfs or extracted / 'romfs').resolve(),