diff --git a/recomp/lifter.py b/recomp/lifter.py index 230923e7..5dd506cd 100644 --- a/recomp/lifter.py +++ b/recomp/lifter.py @@ -352,7 +352,7 @@ class Lifter: dst = { (3, 3, 13, 0, 2): 'c->tpidr = {v};', (3, 3, 4, 2, 0): 'c->nzcv = {v} & 0xf0000000ull;', - (3, 3, 4, 4, 0): 'c->fpcr = {v} & 0x7c00000ull;' if self.sc else 'rt_set_fpcr({v});', + (3, 3, 4, 4, 0): 'c->fpcr = {v} & 0x7c00000ull; rt_fpcr_sync(c);' if self.sc else 'rt_set_fpcr({v});', (3, 3, 4, 4, 1): 'c->fpsr = {v} & 0xf800009full;' if self.sc else 'rt_set_fpsr({v});', }.get(key) if dst is None: diff --git a/runtime/cpu.h b/runtime/cpu.h index a787cbb0..26d4daea 100644 --- a/runtime/cpu.h +++ b/runtime/cpu.h @@ -18,8 +18,7 @@ typedef struct __attribute__((aligned(16))) Cpu { uint64_t fpcr; // 320: guest FPCR/FPSR (portable FP/SIMD lowering only; the asm path uses the host's) uint64_t fpsr; // 328 uint64_t excl_val_hi; // 336: upper half of a 128-bit exclusive pair (LDXP/LDAXP of two X registers) - uint64_t fpcr_host; // 344: ~fpcr while this host thread's MXCSR mirrors it (simd.h simd_fast); 0 = not synced - uint8_t _pad[512 - 352]; + uint8_t _pad[512 - 344]; __uint128_t v[32]; // 512 } Cpu; @@ -28,7 +27,6 @@ _Static_assert(offsetof(Cpu, pc) == 264, "pc"); _Static_assert(offsetof(Cpu, nzcv) == 272, "nzcv"); _Static_assert(offsetof(Cpu, fpcr) == 320, "fpcr"); _Static_assert(offsetof(Cpu, fpsr) == 328, "fpsr"); -_Static_assert(offsetof(Cpu, fpcr_host) == 344, "fpcr_host"); _Static_assert(offsetof(Cpu, v) == 512, "v"); typedef void (*GuestFn)(Cpu *); @@ -45,6 +43,7 @@ static inline uint64_t rt_get_fpcr(void) { uint64_t v; __asm__ volatile("mrs %0, static inline uint64_t rt_get_fpsr(void) { uint64_t v; __asm__ volatile("mrs %0, fpsr" : "=r"(v)); return v; } static inline void rt_set_fpcr(uint64_t v) { __asm__ volatile("msr fpcr, %0" :: "r"(v)); } static inline void rt_set_fpsr(uint64_t v) { __asm__ volatile("msr fpsr, %0" :: "r"(v)); } +static inline void rt_fpcr_sync(const Cpu *c) { (void)c; } #else // Host FP state is not the guest's here: code lifted with --simd=c keeps FPCR/FPSR in Cpu (c->fpcr, zeroed // with the Cpu), so the host-side reset in thread start-up has nothing to do. @@ -52,6 +51,15 @@ static inline uint64_t rt_get_fpcr(void) { return 0; } static inline uint64_t rt_get_fpsr(void) { return 0; } static inline void rt_set_fpcr(uint64_t v) { (void)v; } static inline void rt_set_fpsr(uint64_t v) { (void)v; } +// The MXCSR of a host thread running guest code mirrors c->fpcr: FTZ|DAZ = FPCR.FZ, RC = nearest (FPCR.RMode takes +// the slow paths of runtime/simd.h, which switch RC around one operation). The FP helpers rely on it without checking, +// so it is set wherever FPCR or the host FP context changes: MSR FPCR (recomp/lifter.py), rt_call (thread and fiber +// entry, host code calling guest code), fiber resume (runtime/kernel.c fiber_resumed; fibers keep their own MXCSR). +static inline void rt_fpcr_sync(const Cpu *c) { + unsigned cur = __builtin_ia32_stmxcsr(); + unsigned want = (cur & ~0xE040u) | (c->fpcr & (1u << 24) ? 0x8040u : 0); // FTZ 0x8000, DAZ 0x40, RC 0x6000 + if (cur != want) __builtin_ia32_ldmxcsr(want); +} #endif // ---- memory: guest addresses are host addresses diff --git a/runtime/hostfn.c b/runtime/hostfn.c index bfba009d..d3ef85dd 100644 --- a/runtime/hostfn.c +++ b/runtime/hostfn.c @@ -197,6 +197,7 @@ uint64_t rt_call(Cpu *c, uint64_t fn, int n, ...) { uint64_t lr = c->x[30], pc = c->pc; c->x[30] = RT_RET_HOST; c->pc = fn; + rt_fpcr_sync(c); // guest code may start here on a host thread or fiber that has not run it rt_dispatch(c); c->x[30] = lr; c->pc = pc; diff --git a/runtime/kernel.c b/runtime/kernel.c index a7a4f96f..4afb9d52 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -1940,6 +1940,7 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to); // After a host fiber switch: release the context left behind, and finish an ExitThread issued on a fiber. static void fiber_resumed(void) { RtThread *t = tcur; + rt_fpcr_sync(t->cur); // the host fiber resumes with its own MXCSR (FIBER_FLAG_FLOAT_SWITCH) if (t->leaving) { __atomic_store_n(&t->leaving->running, 0, __ATOMIC_RELEASE); t->leaving = NULL; @@ -1981,7 +1982,6 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) { dst->tpidr = from->tpidr; dst->tpidrro = from->tpidrro; dst->fpcr = from->fpcr; - dst->fpcr_host = 0; // the target host fiber resumes with its own MXCSR; simd_fast resyncs it dst->fpsr = from->fpsr; dst->excl_addr = ~0ull; dst->thread = t; diff --git a/runtime/simd.h b/runtime/simd.h index 49ae1fbf..535cb106 100644 --- a/runtime/simd.h +++ b/runtime/simd.h @@ -6,7 +6,7 @@ // for arithmetic / conversions / FRINTX/I, saturating FP->int conversions, fused multiply-add, ARM estimate // tables. Not modelled: FPSR cumulative exception bits and QC, FPCR trap enables, FP16 arithmetic (ARMv8.2). // FPCR.AHP is honoured by the half-precision conversions. Host assumptions: IEEE binary32/64, MXCSR in -// round-to-nearest outside the slow paths, with FTZ|DAZ equal to FPCR.FZ (simd_fast keeps it so); fma must be +// round-to-nearest outside the slow paths, with FTZ|DAZ equal to FPCR.FZ (cpu.h rt_fpcr_sync keeps it so); fma must be // correctly rounded (build x86 with -mfma). #pragma once #include @@ -230,22 +230,9 @@ SI void simd_rm_leave(unsigned old) { if ((cur & SIMD_RC_MASK) != (old & SIMD_RC_MASK)) _mm_setcsr((cur & ~SIMD_RC_MASK) | (old & SIMD_RC_MASK)); } -// The thread's MXCSR mirrors the guest FPCR: FTZ|DAZ = FPCR.FZ, RC = nearest (the slow paths switch RC and -// restore it). MSR FPCR is lowered inline (c->fpcr = v), so every helper that runs host FP checks the mirror -// lazily: c->fpcr_host holds ~fpcr once this thread's MXCSR matches it. A zeroed Cpu (0) never matches, and code -// that hands a Cpu to another host context (fiber switch, runtime/kernel.c) clears it. -#define SIMD_FTZ_DAZ 0x8040u -SN int simd_fpcr_sync(Cpu *c) { - uint64_t fpcr = c->fpcr; - unsigned cur = _mm_getcsr(); - unsigned want = (cur & ~(SIMD_FTZ_DAZ | SIMD_RC_MASK)) | (fpcr & FPCR_FZ ? SIMD_FTZ_DAZ : 0); - if (cur != want) _mm_setcsr(want); - if (fpcr & FPCR_RM) return 0; // directed rounding: slow paths only, so not recorded - c->fpcr_host = ~fpcr; - return 1; -} -// Nonzero when plain host FP gives the ARM result (MXCSR in sync, RMode = nearest). Always leaves FTZ|DAZ = FPCR.FZ. -SI int simd_fast(Cpu *c) { return __builtin_expect(c->fpcr == ~c->fpcr_host, 1) || simd_fpcr_sync(c); } +// The thread's MXCSR mirrors the guest FPCR.FZ with RC = nearest whenever guest code runs (cpu.h rt_fpcr_sync), so +// plain host FP gives the ARM result unless FPCR.RMode asks for directed rounding, which the slow paths handle. +SI int simd_directed(const Cpu *c) { return __builtin_expect((c->fpcr & FPCR_RM) != 0, 0); } #define F s #define T float diff --git a/runtime/simd_fp.h b/runtime/simd_fp.h index e089a58e..7020c254 100644 --- a/runtime/simd_fp.h +++ b/runtime/simd_fp.h @@ -26,8 +26,8 @@ SI T FN(flush)(T x) { __asm__("" : "+r"(u)); return FN(fv)(FN(isden)(u) ? u & SGN : u); } -// FPCR.FZ input flush for helpers that run host FP; also brings the thread's MXCSR in line with FPCR. -SI T FN(fz)(Cpu *c, T x) { (void)simd_fast(c); return c->fpcr & FPCR_FZ ? FN(flush)(x) : x; } +// FPCR.FZ input flush for helpers that run host FP. +SI T FN(fz)(Cpu *c, T x) { return c->fpcr & FPCR_FZ ? FN(flush)(x) : x; } SI T FN(neg)(T x) { return FN(fv)(FN(fb)(x) ^ SGN); } // FPProcessNaN(s): first signalling NaN (quietened), else first quiet NaN, else (invalid op) default NaN @@ -157,6 +157,9 @@ SN T FN(slow)(Cpu *c, int op, T a, T b, T d) { // an exact result just below the smallest normal that rounds up to it is flushed on ARM only. The slow path // decides both cases with the operands flushed. #define FREDO(op, r) (FBAD(op, r) | (__builtin_elementwise_abs(r) == FN(fv)(MINN))) +// The complement of FREDO for every op but RSQRTS, as one compare: |r| ordered and not the smallest normal (LLVM folds +// it to fcmp one, a single ucomiss + jne for scalars and one vcmpps for vectors). +#define FOK(r) ((__builtin_elementwise_abs(r) < FN(fv)(MINN)) | (__builtin_elementwise_abs(r) > FN(fv)(MINN))) SN T FN(redo)(Cpu *c, int op, T a, T b, T d, T r) { if (c->fpcr & FPCR_FZ) return FN(slow)(c, op, a, b, d); return FBAD(op, r) ? FN(fix)(c, op, a, b, d) : r; @@ -164,13 +167,13 @@ SN T FN(redo)(Cpu *c, int op, T a, T b, T d, T r) { // Scalar FP op (constant op folds the selection). RMode = nearest runs on the host, with or without FZ. SI T FN(op)(Cpu *c, int op, T a, T b, T d) { - if (__builtin_expect(!simd_fast(c), 0)) return FN(slow)(c, op, a, b, d); + if (simd_directed(c)) return FN(slow)(c, op, a, b, d); if (FMINMAX(op)) { if (c->fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b); return FN(minmax)(c, op, a, b); } T r = FCALC(op, a, b, d, K); - if (__builtin_expect(FREDO(op, r), 0)) r = FN(redo)(c, op, a, b, d, r); + if (__builtin_expect(op == FOP_RSQRTS ? FREDO(op, r) : !FOK(r), 0)) r = FN(redo)(c, op, a, b, d, r); return r; } @@ -192,7 +195,7 @@ SI VT FN(vflush)(VT a) { // Vector FP op: native lanes; any lane FREDO flags (or FPCR.RMode) reruns all lanes through the scalar path. SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) { - if (__builtin_expect(!simd_fast(c), 0)) return FN(vslow)(c, op, a, b, d); + if (simd_directed(c)) return FN(vslow)(c, op, a, b, d); VT r; if (FMINMAX(op)) { if (c->fpcr & FPCR_FZ) a = FN(vflush)(a), b = FN(vflush)(b); @@ -205,11 +208,12 @@ SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) { return r; } r = FCALC(op, a, b, d, VK); - if (__builtin_expect(__builtin_reduce_or(FREDO(op, r)) != 0, 0)) return FN(vslow)(c, op, a, b, d); + if (__builtin_expect(op == FOP_RSQRTS ? __builtin_reduce_or(FREDO(op, r)) != 0 : __builtin_reduce_and(FOK(r)) == 0, 0)) + return FN(vslow)(c, op, a, b, d); return r; } -SI VT FN(vfz)(Cpu *c, VT a) { (void)simd_fast(c); return c->fpcr & FPCR_FZ ? FN(vflush)(a) : a; } +SI VT FN(vfz)(Cpu *c, VT a) { return c->fpcr & FPCR_FZ ? FN(vflush)(a) : a; } // ---- compares SI uint64_t FN(cmp)(Cpu *c, T a, T b) { // FCMP NZCV diff --git a/tests/diff/harness.c b/tests/diff/harness.c index e268d543..8468596e 100644 --- a/tests/diff/harness.c +++ b/tests/diff/harness.c @@ -136,6 +136,7 @@ int main(int argc, char **argv) { c.sp = in.sp; c.nzcv = in.nzcv; c.fpcr = in.fpcr; + rt_fpcr_sync(&c); memcpy(c.v, in.v, sizeof c.v); c.excl_addr = ~0ull; g_stubs = (int)in.stubs;