From 5c48f62fcacdaa69652c52857b6efd031a3f6b67 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 15 Nov 2026 00:31:00 +0300 Subject: [PATCH 1/2] The FP fast paths of the lifted code check FPCR.RMode inside the result compare and pass vector slow-path operands through the Cpu The result check of every scalar and vector FP op compares |r| against a per-thread threshold in the Cpu (fp_th_s, fp_th_d): the smallest normal in round-to-nearest, NaN under a directed RMode, so a directed mode sends every result to the redo path and the separate load and test of c->fpcr per op goes. rt_fpcr_threshold sets it; rt_fpcr_sync calls it on every host, and the Cpu inits and the fiber switch's fpcr copy call it too. The vector slow path FN(vredo) takes its operands from c->fp_arg instead of three stack copies per call site, which takes ___chkstk_ms out of the large lifted functions (275 call sites in main to 5). It reruns only the lanes that failed the check. A NaN result in round-to-nearest goes from redo straight to fix with the FZ-flushed operands. The static FP helpers use preserve_most, so their cold call sites do not spill the caller-saved registers. Results are bit-identical to the old headers (equivalence harness of build/v143-genko/static2, 200M cases). On the 42 hottest 1.4.3 functions: instructions -8.2%, bytes -8.8%, hot-path instructions -4.3%. Co-Authored-By: Claude Opus 5.5 --- runtime/cpu.h | 26 ++++++++++++++++++--- runtime/fs_host.c | 1 + runtime/kernel.c | 3 +++ runtime/loader.c | 1 + runtime/main.c | 1 + runtime/movie_probe.c | 1 + runtime/simd.h | 15 ++++++++++++ runtime/simd_fp.h | 53 +++++++++++++++++++++++++++++++++++-------- 8 files changed, 89 insertions(+), 12 deletions(-) diff --git a/runtime/cpu.h b/runtime/cpu.h index 7bacacad..1a3717ec 100644 --- a/runtime/cpu.h +++ b/runtime/cpu.h @@ -18,7 +18,11 @@ typedef struct __attribute__((aligned(16))) Cpu { uint64_t fpcr; // 320: guest FPCR/FPSR (portable FP/SIMD lowering only; the asm path uses the host's) uint64_t fpsr; // 328 uint64_t excl_val_hi; // 336: upper half of a 128-bit exclusive pair (LDXP/LDAXP of two X registers) - uint8_t _pad[512 - 344]; + uint64_t _pad0; // 344 + float fp_th_s[4]; // 352: FP result check threshold, every lane (rt_fpcr_threshold, runtime/simd_fp.h FOKT) + double fp_th_d[2]; // 368 + __uint128_t fp_arg[3]; // 384: operands of a vector FP op handed to its slow path (runtime/simd_fp.h FN(vredo)) + uint8_t _pad[512 - 432]; __uint128_t v[32]; // 512 } Cpu; @@ -27,6 +31,8 @@ _Static_assert(offsetof(Cpu, pc) == 264, "pc"); _Static_assert(offsetof(Cpu, nzcv) == 272, "nzcv"); _Static_assert(offsetof(Cpu, fpcr) == 320, "fpcr"); _Static_assert(offsetof(Cpu, fpsr) == 328, "fpsr"); +_Static_assert(offsetof(Cpu, fp_th_s) == 352 && offsetof(Cpu, fp_th_d) == 368, "fp_th"); +_Static_assert(offsetof(Cpu, fp_arg) == 384, "fp_arg"); _Static_assert(offsetof(Cpu, v) == 512, "v"); typedef void (*GuestFn)(Cpu *); @@ -39,12 +45,24 @@ void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) __attribute__((noreturn)); void rt_udf(Cpu *c, uint64_t pc) __attribute__((noreturn)); uint64_t rt_ticks(void); +// The FP fast paths of runtime/simd_fp.h test FPCR.RMode and the result in one compare: a result is good when its +// magnitude differs from c->fp_th_*. In round-to-nearest the threshold is the smallest normal, so NaN and +-smallest +// normal fail as before; under a directed RMode it is NaN, both compares are unordered and every result goes through +// the redo path, which rounds in the FPCR mode. It must follow every change of c->fpcr: rt_fpcr_sync sets it, and so +// does code that creates a Cpu or copies fpcr into one. A zeroed threshold is wrong, not merely slow: it passes +// +-smallest normal. +static inline void rt_fpcr_threshold(Cpu *c) { + int directed = (c->fpcr & (3u << 22)) != 0; + for (int i = 0; i < 4; i++) c->fp_th_s[i] = directed ? __builtin_nanf("") : 0x1p-126f; + for (int i = 0; i < 2; i++) c->fp_th_d[i] = directed ? __builtin_nan("") : 0x1p-1022; +} + #if defined(__aarch64__) static inline uint64_t rt_get_fpcr(void) { uint64_t v; __asm__ volatile("mrs %0, fpcr" : "=r"(v)); return v; } static inline uint64_t rt_get_fpsr(void) { uint64_t v; __asm__ volatile("mrs %0, fpsr" : "=r"(v)); return v; } static inline void rt_set_fpcr(uint64_t v) { __asm__ volatile("msr fpcr, %0" :: "r"(v)); } static inline void rt_set_fpsr(uint64_t v) { __asm__ volatile("msr fpsr, %0" :: "r"(v)); } -static inline void rt_fpcr_sync(const Cpu *c) { (void)c; } +static inline void rt_fpcr_sync(Cpu *c) { rt_fpcr_threshold(c); } #else // Host FP state is not the guest's here: code lifted with --simd=c keeps FPCR/FPSR in Cpu (c->fpcr, zeroed // with the Cpu), so the host-side reset in thread start-up has nothing to do. @@ -56,7 +74,9 @@ static inline void rt_set_fpsr(uint64_t v) { (void)v; } // the slow paths of runtime/simd.h, which switch RC around one operation). The FP helpers rely on it without checking, // so it is set wherever FPCR or the host FP context changes: MSR FPCR (recomp/lifter.py), rt_call (thread and fiber // entry, host code calling guest code), fiber resume (runtime/kernel.c fiber_resumed; fibers keep their own MXCSR). -static inline void rt_fpcr_sync(const Cpu *c) { +// It also sets the FP result check threshold (rt_fpcr_threshold). +static inline void rt_fpcr_sync(Cpu *c) { + rt_fpcr_threshold(c); unsigned cur = __builtin_ia32_stmxcsr(); unsigned want = (cur & ~0xE040u) | (c->fpcr & (1u << 24) ? 0x8040u : 0); // FTZ 0x8000, DAZ 0x40, RC 0x6000 if (cur != want) __builtin_ia32_ldmxcsr(want); diff --git a/runtime/fs_host.c b/runtime/fs_host.c index 3af390a7..8671ceb6 100644 --- a/runtime/fs_host.c +++ b/runtime/fs_host.c @@ -1770,6 +1770,7 @@ int probe_fs(void) { pcpu = plat_aligned_alloc(sizeof(Cpu), 64); need(pcpu != NULL, "probe Cpu"); memset(pcpu, 0, sizeof *pcpu); + rt_fpcr_threshold(pcpu); uint8_t *stack = mem_kernel_alloc(0x10000, 16); pcpu->sp = (uint64_t)(uintptr_t)stack + 0x10000; pcpu->excl_addr = ~0ull; diff --git a/runtime/kernel.c b/runtime/kernel.c index 4b5114c1..eb16a323 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -1445,6 +1445,7 @@ static RtThread *thr_new(int prio, int core, uint64_t mask) { Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64); if (!t || !c) rt_fatal("kernel: thread allocation"); memset(c, 0, sizeof *c); + rt_fpcr_threshold(c); plat_cond_init(&t->cv); t->prio = prio; t->core = core; @@ -2259,6 +2260,7 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) { dst->tpidr = from->tpidr; dst->tpidrro = from->tpidrro; dst->fpcr = from->fpcr; + rt_fpcr_threshold(dst); // fiber_resumed syncs MXCSR on the other side; the threshold follows fpcr at once dst->fpsr = from->fpsr; dst->excl_addr = ~0ull; dst->thread = t; @@ -2291,6 +2293,7 @@ void kern_fiber_initialize(Cpu *c) { plat_fiber_free(f->host); } memset(f->cpu, 0, sizeof *f->cpu); + rt_fpcr_threshold(f->cpu); f->cpu->excl_addr = ~0ull; f->addr = addr; f->func = c->x[1]; diff --git a/runtime/loader.c b/runtime/loader.c index 4aa030f5..970c5fb7 100644 --- a/runtime/loader.c +++ b/runtime/loader.c @@ -643,6 +643,7 @@ static void check_got(void) { hostfn_seal(); Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64); memset(c, 0, sizeof *c); + rt_fpcr_threshold(c); uint8_t *stack = mem_kernel_alloc(0x10000, 16); c->sp = (uint64_t)(uintptr_t)stack + 0x10000; c->excl_addr = ~0ull; diff --git a/runtime/main.c b/runtime/main.c index 2d0e9827..9ce8517a 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -568,6 +568,7 @@ static int selftest(void) { CHECK(s.functions > 0, "no functions linked"); Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64); memset(c, 0, sizeof *c); + rt_fpcr_threshold(c); uint64_t r = rt_call(c, probe, 1, (uint64_t)7); CHECK(rt_dispatch_lookup(probe) && r == 42 && host_probe_hits == 1 && c->x[30] == 0, "a registered host function does not dispatch through rt_call"); diff --git a/runtime/movie_probe.c b/runtime/movie_probe.c index a944d569..57bd7a7f 100644 --- a/runtime/movie_probe.c +++ b/runtime/movie_probe.c @@ -435,6 +435,7 @@ int probe_movie(void) { C = plat_aligned_alloc(sizeof(Cpu), 64); CHECK(C, "Cpu allocation"); memset(C, 0, sizeof *C); + rt_fpcr_threshold(C); uint64_t obj_mem[2] = {0}, obj = U(obj_mem); volatile int32_t ev[4] = {0}; diff --git a/runtime/simd.h b/runtime/simd.h index 7e906a6f..f8a14b39 100644 --- a/runtime/simd.h +++ b/runtime/simd.h @@ -17,7 +17,17 @@ typedef __uint128_t u128; #define SI static inline __attribute__((always_inline)) +// The noinline FP helpers are the slow paths of inline fast paths, called from cold blocks inside the lifted +// functions. preserve_most makes the helper save the caller-saved GPRs it clobbers, so a call site does not spill and +// reload its live values around the call. The helpers are static, so the convention never crosses a module boundary. +#if defined(__has_attribute) +#if __has_attribute(preserve_most) +#define SN static __attribute__((noinline, preserve_most)) +#endif +#endif +#ifndef SN #define SN static __attribute__((noinline)) +#endif #define SIMD_CAT2(a, b) a##_##b #define SIMD_CAT(a, b) SIMD_CAT2(a, b) @@ -274,6 +284,11 @@ SI void simd_rm_leave(unsigned old) { // plain host FP gives the ARM result unless FPCR.RMode asks for directed rounding, which the slow paths handle. SI int simd_directed(const Cpu *c) { return __builtin_expect((c->fpcr & FPCR_RM) != 0, 0); } +// FP result check threshold of the thread (cpu.h rt_fpcr_threshold), for one lane and for a whole vector. +SI float simd_th_s(const Cpu *c) { return c->fp_th_s[0]; } +SI double simd_th_d(const Cpu *c) { return c->fp_th_d[0]; } +SI vf32 simd_thv_s(const Cpu *c) { return *(const vf32 *)c->fp_th_s; } +SI vf64 simd_thv_d(const Cpu *c) { return *(const vf64 *)c->fp_th_d; } #define F s #define T float #define U uint32_t diff --git a/runtime/simd_fp.h b/runtime/simd_fp.h index 1e7721ed..d24f28c5 100644 --- a/runtime/simd_fp.h +++ b/runtime/simd_fp.h @@ -204,21 +204,34 @@ SN T FN(slow)(Cpu *c, int op, T a, T b, T d) { // The complement of FREDO for every op but RSQRTS, as one compare: |r| ordered and not the smallest normal (LLVM folds // it to fcmp one, a single ucomiss + jne for scalars and one vcmpps for vectors). #define FOK(r) ((__builtin_elementwise_abs(r) < FN(fv)(MINN)) | (__builtin_elementwise_abs(r) > FN(fv)(MINN))) +// FOK against the thread's threshold (cpu.h rt_fpcr_threshold), the check of the fast paths: the same as FOK in +// round-to-nearest, where the threshold is the smallest normal, and false for every result under a directed RMode, +// where the threshold is NaN, so that RMode needs no test of its own. The threshold is a memory operand of the compare. +#define FOKT(r, th) ((__builtin_elementwise_abs(r) < (th)) | (__builtin_elementwise_abs(r) > (th))) +// A result FOKT rejected. A NaN result in round-to-nearest goes straight to fix: the host already ran the operation +// on the operands FPCR.FZ flushes (MXCSR.DAZ), so slow would compute the same NaN and return fix of the flushed +// operands, and fix's result for these ops is never tiny, so slow's FZ result flush would leave it alone. RSQRTS +// keeps the slow path because its fix result can be tiny. SN T FN(redo)(Cpu *c, int op, T a, T b, T d, T r) { - if (c->fpcr & (FPCR_FZ | FPCR_RM)) return FN(slow)(c, op, a, b, d); + uint64_t fpcr = c->fpcr; + if (op != FOP_RSQRTS && !(fpcr & FPCR_RM) && r != r) { + if (fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b), d = FN(flush)(d); + return FN(fix)(c, op, a, b, d); + } + if (fpcr & (FPCR_FZ | FPCR_RM)) return FN(slow)(c, op, a, b, d); return FBAD(op, r) ? FN(fix)(c, op, a, b, d) : r; } // Scalar FP op (constant op folds the selection). RMode = nearest runs on the host, with or without FZ. A directed -// RMode is rare, so it shares the result check's branch: the host result, computed in round-to-nearest, is then -// thrown away and redo recomputes in the FPCR mode. Min and max do not round, so RMode does not matter to them. +// RMode is rare, so it shares the result check (the FOKT threshold): the host result, computed in round-to-nearest, +// is then thrown away and redo recomputes in the FPCR mode. Min and max do not round, so RMode does not matter to them. SI T FN(op)(Cpu *c, int op, T a, T b, T d) { if (FMINMAX(op)) { if (c->fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b); return FN(minmax)(c, op, a, b); } T r = FCALC(op, a, b, d, K, FMA_S); - if (__builtin_expect((op == FOP_RSQRTS ? FREDO(op, r) : !FOK(r)) | simd_directed(c), 0)) + if (__builtin_expect(op == FOP_RSQRTS ? FREDO(op, r) | simd_directed(c) : !FOKT(r, SIMD_CAT(simd_th, F)(c)), 0)) r = FN(redo)(c, op, a, b, d, r); return r; } @@ -228,6 +241,24 @@ SN VT FN(vslow)(Cpu *c, int op, VT a, VT b, VT d) { for (int i = 0; i < NL; i++) r[i] = FN(op)(c, op, a[i], b[i], d[i]); return r; } +// Slow path of FN(vop) for a vector with flagged lanes. The operands come in c->fp_arg, stored by the caller just +// before the call: passing three vectors by value costs every call site a stack copy of each (Win64 passes them by +// reference), and the frames of the large lifted functions grow past a page (a ___chkstk_ms call at every entry). The +// scratch is per thread and nothing runs between the store and the loads here. In round-to-nearest a lane that passed +// the check equals what the scalar op returns for it (same host operation, same MXCSR), so only the flagged lanes go +// through FN(op); the vector result is recomputed here rather than passed in, which keeps the call sites small. A +// directed RMode reruns every lane. +SN VT FN(vredo)(Cpu *c, int op) { + VT a, b, d; + memcpy(&a, &c->fp_arg[0], sizeof a); + memcpy(&b, &c->fp_arg[1], sizeof b); + memcpy(&d, &c->fp_arg[2], sizeof d); + if (c->fpcr & FPCR_RM) return FN(vslow)(c, op, a, b, d); + VT r = FCALC(op, a, b, d, VK, FMA_V); + for (int i = 0; i < NL; i++) + if (op == FOP_RSQRTS ? FREDO(op, r[i]) : !FOK(r[i])) r[i] = FN(op)(c, op, a[i], b[i], d[i]); + return r; +} SI VTU FN(vsel)(VTS m, VTU x, VTU y) { return (x & (VTU)m) | (y & ~(VTU)m); } @@ -239,8 +270,8 @@ SI VT FN(vflush)(VT a) { return (VT)FN(vsel)(den, u & SGN, u); } -// Vector FP op: native lanes; any lane FREDO flags (or a directed FPCR.RMode, checked in the same branch) reruns all -// lanes through the scalar path. Min and max ignore RMode, as in FN(op). +// Vector FP op: native lanes; a lane FREDO flags, or a directed FPCR.RMode (the FOKT threshold, or the RSQRTS test), +// takes FN(vredo). Min and max ignore RMode, as in FN(op). SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) { VT r; if (FMINMAX(op)) { @@ -254,9 +285,13 @@ SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) { return r; } r = FCALC(op, a, b, d, VK, FMA_V); - if (__builtin_expect((op == FOP_RSQRTS ? __builtin_reduce_or(FREDO(op, r)) != 0 : __builtin_reduce_and(FOK(r)) == 0) - | simd_directed(c), 0)) - return FN(vslow)(c, op, a, b, d); + if (__builtin_expect(op == FOP_RSQRTS ? (__builtin_reduce_or(FREDO(op, r)) != 0) | simd_directed(c) + : __builtin_reduce_and(FOKT(r, SIMD_CAT(simd_thv, F)(c))) == 0, 0)) { + memcpy(&c->fp_arg[0], &a, sizeof a); + memcpy(&c->fp_arg[1], &b, sizeof b); + memcpy(&c->fp_arg[2], &d, sizeof d); + return FN(vredo)(c, op); + } return r; } From 2ab53299097ed4ef255672407ac4b463a9a91bc4 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 15 Nov 2026 08:54:22 +0300 Subject: [PATCH 2/2] The NVN probe's Cpu gets the FP result check threshold too Its static Cpu ran guest code with a zeroed threshold, which lets +-smallest normal pass the check unexamined. Co-Authored-By: Claude Opus 5.5 --- runtime/nvn_probe.c | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/nvn_probe.c b/runtime/nvn_probe.c index 8431d61a..a3996aed 100644 --- a/runtime/nvn_probe.c +++ b/runtime/nvn_probe.c @@ -919,6 +919,7 @@ int probe_nvn(void) { if (child && *child) return run_as_child(child); // The probe checks the null back end whatever TOTK_GPU says; the choice is made once, at the first object call. ((RtConfig *)rt_config())->gpu = "null"; + rt_fpcr_threshold(&cpu); // the FP result check threshold of a fresh Cpu (cpu.h): a zeroed one is wrong, not slow nvn_procs_register(); hostfn_seal(); kern_run_on_guest_main(probe_main);