Merge task/fpslim: the FP fast paths of the game's code are smaller with the same results, 8% fewer instructions in the hottest functions (#381)

The directed-RMode test folds into the result compare through a per-thread threshold, vector slow paths take
their operands through the Cpu instead of stack copies (___chkstk_ms leaves the large lifted functions), the
static FP helpers use preserve_most, and the slow paths rerun only the failing lanes. Bit-identical to the old
headers: 14.85M edge cases under every FPCR mode on v3 and v2, and 13.4G FP ops compared in a check-mode build in
the town and fight2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-15 08:54:37 +03:00
commit b642c849ef
9 files changed
+90 -12

No files matched your search

+23 -3
View File
@@ -18,7 +18,11 @@ typedef struct __attribute__((aligned(16))) Cpu {
uint64_t fpcr; // 320: guest FPCR/FPSR (portable FP/SIMD lowering only; the asm path uses the host's)
uint64_t fpsr; // 328
uint64_t excl_val_hi; // 336: upper half of a 128-bit exclusive pair (LDXP/LDAXP of two X registers)
uint8_t _pad[512 - 344];
uint64_t _pad0; // 344
float fp_th_s[4]; // 352: FP result check threshold, every lane (rt_fpcr_threshold, runtime/simd_fp.h FOKT)
double fp_th_d[2]; // 368
__uint128_t fp_arg[3]; // 384: operands of a vector FP op handed to its slow path (runtime/simd_fp.h FN(vredo))
uint8_t _pad[512 - 432];
__uint128_t v[32]; // 512
} Cpu;
@@ -27,6 +31,8 @@ _Static_assert(offsetof(Cpu, pc) == 264, "pc");
_Static_assert(offsetof(Cpu, nzcv) == 272, "nzcv");
_Static_assert(offsetof(Cpu, fpcr) == 320, "fpcr");
_Static_assert(offsetof(Cpu, fpsr) == 328, "fpsr");
_Static_assert(offsetof(Cpu, fp_th_s) == 352 && offsetof(Cpu, fp_th_d) == 368, "fp_th");
_Static_assert(offsetof(Cpu, fp_arg) == 384, "fp_arg");
_Static_assert(offsetof(Cpu, v) == 512, "v");
typedef void (*GuestFn)(Cpu *);
@@ -39,12 +45,24 @@ void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) __attribute__((noreturn));
void rt_udf(Cpu *c, uint64_t pc) __attribute__((noreturn));
uint64_t rt_ticks(void);
// The FP fast paths of runtime/simd_fp.h test FPCR.RMode and the result in one compare: a result is good when its
// magnitude differs from c->fp_th_*. In round-to-nearest the threshold is the smallest normal, so NaN and +-smallest
// normal fail as before; under a directed RMode it is NaN, both compares are unordered and every result goes through
// the redo path, which rounds in the FPCR mode. It must follow every change of c->fpcr: rt_fpcr_sync sets it, and so
// does code that creates a Cpu or copies fpcr into one. A zeroed threshold is wrong, not merely slow: it passes
// +-smallest normal.
static inline void rt_fpcr_threshold(Cpu *c) {
int directed = (c->fpcr & (3u << 22)) != 0;
for (int i = 0; i < 4; i++) c->fp_th_s[i] = directed ? __builtin_nanf("") : 0x1p-126f;
for (int i = 0; i < 2; i++) c->fp_th_d[i] = directed ? __builtin_nan("") : 0x1p-1022;
}
#if defined(__aarch64__)
static inline uint64_t rt_get_fpcr(void) { uint64_t v; __asm__ volatile("mrs %0, fpcr" : "=r"(v)); return v; }
static inline uint64_t rt_get_fpsr(void) { uint64_t v; __asm__ volatile("mrs %0, fpsr" : "=r"(v)); return v; }
static inline void rt_set_fpcr(uint64_t v) { __asm__ volatile("msr fpcr, %0" :: "r"(v)); }
static inline void rt_set_fpsr(uint64_t v) { __asm__ volatile("msr fpsr, %0" :: "r"(v)); }
static inline void rt_fpcr_sync(const Cpu *c) { (void)c; }
static inline void rt_fpcr_sync(Cpu *c) { rt_fpcr_threshold(c); }
#else
// Host FP state is not the guest's here: code lifted with --simd=c keeps FPCR/FPSR in Cpu (c->fpcr, zeroed
// with the Cpu), so the host-side reset in thread start-up has nothing to do.
@@ -56,7 +74,9 @@ static inline void rt_set_fpsr(uint64_t v) { (void)v; }
// the slow paths of runtime/simd.h, which switch RC around one operation). The FP helpers rely on it without checking,
// so it is set wherever FPCR or the host FP context changes: MSR FPCR (recomp/lifter.py), rt_call (thread and fiber
// entry, host code calling guest code), fiber resume (runtime/kernel.c fiber_resumed; fibers keep their own MXCSR).
static inline void rt_fpcr_sync(const Cpu *c) {
// It also sets the FP result check threshold (rt_fpcr_threshold).
static inline void rt_fpcr_sync(Cpu *c) {
rt_fpcr_threshold(c);
unsigned cur = __builtin_ia32_stmxcsr();
unsigned want = (cur & ~0xE040u) | (c->fpcr & (1u << 24) ? 0x8040u : 0); // FTZ 0x8000, DAZ 0x40, RC 0x6000
if (cur != want) __builtin_ia32_ldmxcsr(want);
+1
View File
@@ -1770,6 +1770,7 @@ int probe_fs(void) {
pcpu = plat_aligned_alloc(sizeof(Cpu), 64);
need(pcpu != NULL, "probe Cpu");
memset(pcpu, 0, sizeof *pcpu);
rt_fpcr_threshold(pcpu);
uint8_t *stack = mem_kernel_alloc(0x10000, 16);
pcpu->sp = (uint64_t)(uintptr_t)stack + 0x10000;
pcpu->excl_addr = ~0ull;
+3
View File
@@ -1445,6 +1445,7 @@ static RtThread *thr_new(int prio, int core, uint64_t mask) {
Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64);
if (!t || !c) rt_fatal("kernel: thread allocation");
memset(c, 0, sizeof *c);
rt_fpcr_threshold(c);
plat_cond_init(&t->cv);
t->prio = prio;
t->core = core;
@@ -2259,6 +2260,7 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) {
dst->tpidr = from->tpidr;
dst->tpidrro = from->tpidrro;
dst->fpcr = from->fpcr;
rt_fpcr_threshold(dst); // fiber_resumed syncs MXCSR on the other side; the threshold follows fpcr at once
dst->fpsr = from->fpsr;
dst->excl_addr = ~0ull;
dst->thread = t;
@@ -2291,6 +2293,7 @@ void kern_fiber_initialize(Cpu *c) {
plat_fiber_free(f->host);
}
memset(f->cpu, 0, sizeof *f->cpu);
rt_fpcr_threshold(f->cpu);
f->cpu->excl_addr = ~0ull;
f->addr = addr;
f->func = c->x[1];
+1
View File
@@ -643,6 +643,7 @@ static void check_got(void) {
hostfn_seal();
Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64);
memset(c, 0, sizeof *c);
rt_fpcr_threshold(c);
uint8_t *stack = mem_kernel_alloc(0x10000, 16);
c->sp = (uint64_t)(uintptr_t)stack + 0x10000;
c->excl_addr = ~0ull;
+1
View File
@@ -568,6 +568,7 @@ static int selftest(void) {
CHECK(s.functions > 0, "no functions linked");
Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64);
memset(c, 0, sizeof *c);
rt_fpcr_threshold(c);
uint64_t r = rt_call(c, probe, 1, (uint64_t)7);
CHECK(rt_dispatch_lookup(probe) && r == 42 && host_probe_hits == 1 && c->x[30] == 0,
"a registered host function does not dispatch through rt_call");
+1
View File
@@ -435,6 +435,7 @@ int probe_movie(void) {
C = plat_aligned_alloc(sizeof(Cpu), 64);
CHECK(C, "Cpu allocation");
memset(C, 0, sizeof *C);
rt_fpcr_threshold(C);
uint64_t obj_mem[2] = {0}, obj = U(obj_mem);
volatile int32_t ev[4] = {0};
+1
View File
@@ -919,6 +919,7 @@ int probe_nvn(void) {
if (child && *child) return run_as_child(child);
// The probe checks the null back end whatever TOTK_GPU says; the choice is made once, at the first object call.
((RtConfig *)rt_config())->gpu = "null";
rt_fpcr_threshold(&cpu); // the FP result check threshold of a fresh Cpu (cpu.h): a zeroed one is wrong, not slow
nvn_procs_register();
hostfn_seal();
kern_run_on_guest_main(probe_main);
+15
View File
@@ -17,7 +17,17 @@
typedef __uint128_t u128;
#define SI static inline __attribute__((always_inline))
// The noinline FP helpers are the slow paths of inline fast paths, called from cold blocks inside the lifted
// functions. preserve_most makes the helper save the caller-saved GPRs it clobbers, so a call site does not spill and
// reload its live values around the call. The helpers are static, so the convention never crosses a module boundary.
#if defined(__has_attribute)
#if __has_attribute(preserve_most)
#define SN static __attribute__((noinline, preserve_most))
#endif
#endif
#ifndef SN
#define SN static __attribute__((noinline))
#endif
#define SIMD_CAT2(a, b) a##_##b
#define SIMD_CAT(a, b) SIMD_CAT2(a, b)
@@ -274,6 +284,11 @@ SI void simd_rm_leave(unsigned old) {
// plain host FP gives the ARM result unless FPCR.RMode asks for directed rounding, which the slow paths handle.
SI int simd_directed(const Cpu *c) { return __builtin_expect((c->fpcr & FPCR_RM) != 0, 0); }
// FP result check threshold of the thread (cpu.h rt_fpcr_threshold), for one lane and for a whole vector.
SI float simd_th_s(const Cpu *c) { return c->fp_th_s[0]; }
SI double simd_th_d(const Cpu *c) { return c->fp_th_d[0]; }
SI vf32 simd_thv_s(const Cpu *c) { return *(const vf32 *)c->fp_th_s; }
SI vf64 simd_thv_d(const Cpu *c) { return *(const vf64 *)c->fp_th_d; }
#define F s
#define T float
#define U uint32_t
+44 -9
View File
@@ -204,21 +204,34 @@ SN T FN(slow)(Cpu *c, int op, T a, T b, T d) {
// The complement of FREDO for every op but RSQRTS, as one compare: |r| ordered and not the smallest normal (LLVM folds
// it to fcmp one, a single ucomiss + jne for scalars and one vcmpps for vectors).
#define FOK(r) ((__builtin_elementwise_abs(r) < FN(fv)(MINN)) | (__builtin_elementwise_abs(r) > FN(fv)(MINN)))
// FOK against the thread's threshold (cpu.h rt_fpcr_threshold), the check of the fast paths: the same as FOK in
// round-to-nearest, where the threshold is the smallest normal, and false for every result under a directed RMode,
// where the threshold is NaN, so that RMode needs no test of its own. The threshold is a memory operand of the compare.
#define FOKT(r, th) ((__builtin_elementwise_abs(r) < (th)) | (__builtin_elementwise_abs(r) > (th)))
// A result FOKT rejected. A NaN result in round-to-nearest goes straight to fix: the host already ran the operation
// on the operands FPCR.FZ flushes (MXCSR.DAZ), so slow would compute the same NaN and return fix of the flushed
// operands, and fix's result for these ops is never tiny, so slow's FZ result flush would leave it alone. RSQRTS
// keeps the slow path because its fix result can be tiny.
SN T FN(redo)(Cpu *c, int op, T a, T b, T d, T r) {
if (c->fpcr & (FPCR_FZ | FPCR_RM)) return FN(slow)(c, op, a, b, d);
uint64_t fpcr = c->fpcr;
if (op != FOP_RSQRTS && !(fpcr & FPCR_RM) && r != r) {
if (fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b), d = FN(flush)(d);
return FN(fix)(c, op, a, b, d);
}
if (fpcr & (FPCR_FZ | FPCR_RM)) return FN(slow)(c, op, a, b, d);
return FBAD(op, r) ? FN(fix)(c, op, a, b, d) : r;
}
// Scalar FP op (constant op folds the selection). RMode = nearest runs on the host, with or without FZ. A directed
// RMode is rare, so it shares the result check's branch: the host result, computed in round-to-nearest, is then
// thrown away and redo recomputes in the FPCR mode. Min and max do not round, so RMode does not matter to them.
// RMode is rare, so it shares the result check (the FOKT threshold): the host result, computed in round-to-nearest,
// is then thrown away and redo recomputes in the FPCR mode. Min and max do not round, so RMode does not matter to them.
SI T FN(op)(Cpu *c, int op, T a, T b, T d) {
if (FMINMAX(op)) {
if (c->fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b);
return FN(minmax)(c, op, a, b);
}
T r = FCALC(op, a, b, d, K, FMA_S);
if (__builtin_expect((op == FOP_RSQRTS ? FREDO(op, r) : !FOK(r)) | simd_directed(c), 0))
if (__builtin_expect(op == FOP_RSQRTS ? FREDO(op, r) | simd_directed(c) : !FOKT(r, SIMD_CAT(simd_th, F)(c)), 0))
r = FN(redo)(c, op, a, b, d, r);
return r;
}
@@ -228,6 +241,24 @@ SN VT FN(vslow)(Cpu *c, int op, VT a, VT b, VT d) {
for (int i = 0; i < NL; i++) r[i] = FN(op)(c, op, a[i], b[i], d[i]);
return r;
}
// Slow path of FN(vop) for a vector with flagged lanes. The operands come in c->fp_arg, stored by the caller just
// before the call: passing three vectors by value costs every call site a stack copy of each (Win64 passes them by
// reference), and the frames of the large lifted functions grow past a page (a ___chkstk_ms call at every entry). The
// scratch is per thread and nothing runs between the store and the loads here. In round-to-nearest a lane that passed
// the check equals what the scalar op returns for it (same host operation, same MXCSR), so only the flagged lanes go
// through FN(op); the vector result is recomputed here rather than passed in, which keeps the call sites small. A
// directed RMode reruns every lane.
SN VT FN(vredo)(Cpu *c, int op) {
VT a, b, d;
memcpy(&a, &c->fp_arg[0], sizeof a);
memcpy(&b, &c->fp_arg[1], sizeof b);
memcpy(&d, &c->fp_arg[2], sizeof d);
if (c->fpcr & FPCR_RM) return FN(vslow)(c, op, a, b, d);
VT r = FCALC(op, a, b, d, VK, FMA_V);
for (int i = 0; i < NL; i++)
if (op == FOP_RSQRTS ? FREDO(op, r[i]) : !FOK(r[i])) r[i] = FN(op)(c, op, a[i], b[i], d[i]);
return r;
}
SI VTU FN(vsel)(VTS m, VTU x, VTU y) { return (x & (VTU)m) | (y & ~(VTU)m); }
@@ -239,8 +270,8 @@ SI VT FN(vflush)(VT a) {
return (VT)FN(vsel)(den, u & SGN, u);
}
// Vector FP op: native lanes; any lane FREDO flags (or a directed FPCR.RMode, checked in the same branch) reruns all
// lanes through the scalar path. Min and max ignore RMode, as in FN(op).
// Vector FP op: native lanes; a lane FREDO flags, or a directed FPCR.RMode (the FOKT threshold, or the RSQRTS test),
// takes FN(vredo). Min and max ignore RMode, as in FN(op).
SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) {
VT r;
if (FMINMAX(op)) {
@@ -254,9 +285,13 @@ SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) {
return r;
}
r = FCALC(op, a, b, d, VK, FMA_V);
if (__builtin_expect((op == FOP_RSQRTS ? __builtin_reduce_or(FREDO(op, r)) != 0 : __builtin_reduce_and(FOK(r)) == 0)
| simd_directed(c), 0))
return FN(vslow)(c, op, a, b, d);
if (__builtin_expect(op == FOP_RSQRTS ? (__builtin_reduce_or(FREDO(op, r)) != 0) | simd_directed(c)
: __builtin_reduce_and(FOKT(r, SIMD_CAT(simd_thv, F)(c))) == 0, 0)) {
memcpy(&c->fp_arg[0], &a, sizeof a);
memcpy(&c->fp_arg[1], &b, sizeof b);
memcpy(&c->fp_arg[2], &d, sizeof d);
return FN(vredo)(c, op);
}
return r;
}