mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-18 12:08:38 +00:00
Merge the FP fast path: MXCSR follows FPCR where FPCR changes (rt_fpcr_sync at MSR FPCR, rt_call, fiber resume) instead of a mirror check in every FP op; hot guest code 25% smaller
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
608bfed6f0
7 files changed
+30
-29
No files matched your search
+1
-1
@@ -352,7 +352,7 @@ class Lifter:
|
||||
dst = {
|
||||
(3, 3, 13, 0, 2): 'c->tpidr = {v};',
|
||||
(3, 3, 4, 2, 0): 'c->nzcv = {v} & 0xf0000000ull;',
|
||||
(3, 3, 4, 4, 0): 'c->fpcr = {v} & 0x7c00000ull;' if self.sc else 'rt_set_fpcr({v});',
|
||||
(3, 3, 4, 4, 0): 'c->fpcr = {v} & 0x7c00000ull; rt_fpcr_sync(c);' if self.sc else 'rt_set_fpcr({v});',
|
||||
(3, 3, 4, 4, 1): 'c->fpsr = {v} & 0xf800009full;' if self.sc else 'rt_set_fpsr({v});',
|
||||
}.get(key)
|
||||
if dst is None:
|
||||
|
||||
+11
-3
@@ -18,8 +18,7 @@ typedef struct __attribute__((aligned(16))) Cpu {
|
||||
uint64_t fpcr; // 320: guest FPCR/FPSR (portable FP/SIMD lowering only; the asm path uses the host's)
|
||||
uint64_t fpsr; // 328
|
||||
uint64_t excl_val_hi; // 336: upper half of a 128-bit exclusive pair (LDXP/LDAXP of two X registers)
|
||||
uint64_t fpcr_host; // 344: ~fpcr while this host thread's MXCSR mirrors it (simd.h simd_fast); 0 = not synced
|
||||
uint8_t _pad[512 - 352];
|
||||
uint8_t _pad[512 - 344];
|
||||
__uint128_t v[32]; // 512
|
||||
} Cpu;
|
||||
|
||||
@@ -28,7 +27,6 @@ _Static_assert(offsetof(Cpu, pc) == 264, "pc");
|
||||
_Static_assert(offsetof(Cpu, nzcv) == 272, "nzcv");
|
||||
_Static_assert(offsetof(Cpu, fpcr) == 320, "fpcr");
|
||||
_Static_assert(offsetof(Cpu, fpsr) == 328, "fpsr");
|
||||
_Static_assert(offsetof(Cpu, fpcr_host) == 344, "fpcr_host");
|
||||
_Static_assert(offsetof(Cpu, v) == 512, "v");
|
||||
|
||||
typedef void (*GuestFn)(Cpu *);
|
||||
@@ -45,6 +43,7 @@ static inline uint64_t rt_get_fpcr(void) { uint64_t v; __asm__ volatile("mrs %0,
|
||||
static inline uint64_t rt_get_fpsr(void) { uint64_t v; __asm__ volatile("mrs %0, fpsr" : "=r"(v)); return v; }
|
||||
static inline void rt_set_fpcr(uint64_t v) { __asm__ volatile("msr fpcr, %0" :: "r"(v)); }
|
||||
static inline void rt_set_fpsr(uint64_t v) { __asm__ volatile("msr fpsr, %0" :: "r"(v)); }
|
||||
static inline void rt_fpcr_sync(const Cpu *c) { (void)c; }
|
||||
#else
|
||||
// Host FP state is not the guest's here: code lifted with --simd=c keeps FPCR/FPSR in Cpu (c->fpcr, zeroed
|
||||
// with the Cpu), so the host-side reset in thread start-up has nothing to do.
|
||||
@@ -52,6 +51,15 @@ static inline uint64_t rt_get_fpcr(void) { return 0; }
|
||||
static inline uint64_t rt_get_fpsr(void) { return 0; }
|
||||
static inline void rt_set_fpcr(uint64_t v) { (void)v; }
|
||||
static inline void rt_set_fpsr(uint64_t v) { (void)v; }
|
||||
// The MXCSR of a host thread running guest code mirrors c->fpcr: FTZ|DAZ = FPCR.FZ, RC = nearest (FPCR.RMode takes
|
||||
// the slow paths of runtime/simd.h, which switch RC around one operation). The FP helpers rely on it without checking,
|
||||
// so it is set wherever FPCR or the host FP context changes: MSR FPCR (recomp/lifter.py), rt_call (thread and fiber
|
||||
// entry, host code calling guest code), fiber resume (runtime/kernel.c fiber_resumed; fibers keep their own MXCSR).
|
||||
static inline void rt_fpcr_sync(const Cpu *c) {
|
||||
unsigned cur = __builtin_ia32_stmxcsr();
|
||||
unsigned want = (cur & ~0xE040u) | (c->fpcr & (1u << 24) ? 0x8040u : 0); // FTZ 0x8000, DAZ 0x40, RC 0x6000
|
||||
if (cur != want) __builtin_ia32_ldmxcsr(want);
|
||||
}
|
||||
#endif
|
||||
|
||||
// ---- memory: guest addresses are host addresses
|
||||
|
||||
@@ -197,6 +197,7 @@ uint64_t rt_call(Cpu *c, uint64_t fn, int n, ...) {
|
||||
uint64_t lr = c->x[30], pc = c->pc;
|
||||
c->x[30] = RT_RET_HOST;
|
||||
c->pc = fn;
|
||||
rt_fpcr_sync(c); // guest code may start here on a host thread or fiber that has not run it
|
||||
rt_dispatch(c);
|
||||
c->x[30] = lr;
|
||||
c->pc = pc;
|
||||
|
||||
+1
-1
@@ -1940,6 +1940,7 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to);
|
||||
// After a host fiber switch: release the context left behind, and finish an ExitThread issued on a fiber.
|
||||
static void fiber_resumed(void) {
|
||||
RtThread *t = tcur;
|
||||
rt_fpcr_sync(t->cur); // the host fiber resumes with its own MXCSR (FIBER_FLAG_FLOAT_SWITCH)
|
||||
if (t->leaving) {
|
||||
__atomic_store_n(&t->leaving->running, 0, __ATOMIC_RELEASE);
|
||||
t->leaving = NULL;
|
||||
@@ -1981,7 +1982,6 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) {
|
||||
dst->tpidr = from->tpidr;
|
||||
dst->tpidrro = from->tpidrro;
|
||||
dst->fpcr = from->fpcr;
|
||||
dst->fpcr_host = 0; // the target host fiber resumes with its own MXCSR; simd_fast resyncs it
|
||||
dst->fpsr = from->fpsr;
|
||||
dst->excl_addr = ~0ull;
|
||||
dst->thread = t;
|
||||
|
||||
+4
-17
@@ -6,7 +6,7 @@
|
||||
// for arithmetic / conversions / FRINTX/I, saturating FP->int conversions, fused multiply-add, ARM estimate
|
||||
// tables. Not modelled: FPSR cumulative exception bits and QC, FPCR trap enables, FP16 arithmetic (ARMv8.2).
|
||||
// FPCR.AHP is honoured by the half-precision conversions. Host assumptions: IEEE binary32/64, MXCSR in
|
||||
// round-to-nearest outside the slow paths, with FTZ|DAZ equal to FPCR.FZ (simd_fast keeps it so); fma must be
|
||||
// round-to-nearest outside the slow paths, with FTZ|DAZ equal to FPCR.FZ (cpu.h rt_fpcr_sync keeps it so); fma must be
|
||||
// correctly rounded (build x86 with -mfma).
|
||||
#pragma once
|
||||
#include <fenv.h>
|
||||
@@ -230,22 +230,9 @@ SI void simd_rm_leave(unsigned old) {
|
||||
if ((cur & SIMD_RC_MASK) != (old & SIMD_RC_MASK)) _mm_setcsr((cur & ~SIMD_RC_MASK) | (old & SIMD_RC_MASK));
|
||||
}
|
||||
|
||||
// The thread's MXCSR mirrors the guest FPCR: FTZ|DAZ = FPCR.FZ, RC = nearest (the slow paths switch RC and
|
||||
// restore it). MSR FPCR is lowered inline (c->fpcr = v), so every helper that runs host FP checks the mirror
|
||||
// lazily: c->fpcr_host holds ~fpcr once this thread's MXCSR matches it. A zeroed Cpu (0) never matches, and code
|
||||
// that hands a Cpu to another host context (fiber switch, runtime/kernel.c) clears it.
|
||||
#define SIMD_FTZ_DAZ 0x8040u
|
||||
SN int simd_fpcr_sync(Cpu *c) {
|
||||
uint64_t fpcr = c->fpcr;
|
||||
unsigned cur = _mm_getcsr();
|
||||
unsigned want = (cur & ~(SIMD_FTZ_DAZ | SIMD_RC_MASK)) | (fpcr & FPCR_FZ ? SIMD_FTZ_DAZ : 0);
|
||||
if (cur != want) _mm_setcsr(want);
|
||||
if (fpcr & FPCR_RM) return 0; // directed rounding: slow paths only, so not recorded
|
||||
c->fpcr_host = ~fpcr;
|
||||
return 1;
|
||||
}
|
||||
// Nonzero when plain host FP gives the ARM result (MXCSR in sync, RMode = nearest). Always leaves FTZ|DAZ = FPCR.FZ.
|
||||
SI int simd_fast(Cpu *c) { return __builtin_expect(c->fpcr == ~c->fpcr_host, 1) || simd_fpcr_sync(c); }
|
||||
// The thread's MXCSR mirrors the guest FPCR.FZ with RC = nearest whenever guest code runs (cpu.h rt_fpcr_sync), so
|
||||
// plain host FP gives the ARM result unless FPCR.RMode asks for directed rounding, which the slow paths handle.
|
||||
SI int simd_directed(const Cpu *c) { return __builtin_expect((c->fpcr & FPCR_RM) != 0, 0); }
|
||||
|
||||
#define F s
|
||||
#define T float
|
||||
|
||||
+11
-7
@@ -26,8 +26,8 @@ SI T FN(flush)(T x) {
|
||||
__asm__("" : "+r"(u));
|
||||
return FN(fv)(FN(isden)(u) ? u & SGN : u);
|
||||
}
|
||||
// FPCR.FZ input flush for helpers that run host FP; also brings the thread's MXCSR in line with FPCR.
|
||||
SI T FN(fz)(Cpu *c, T x) { (void)simd_fast(c); return c->fpcr & FPCR_FZ ? FN(flush)(x) : x; }
|
||||
// FPCR.FZ input flush for helpers that run host FP.
|
||||
SI T FN(fz)(Cpu *c, T x) { return c->fpcr & FPCR_FZ ? FN(flush)(x) : x; }
|
||||
SI T FN(neg)(T x) { return FN(fv)(FN(fb)(x) ^ SGN); }
|
||||
|
||||
// FPProcessNaN(s): first signalling NaN (quietened), else first quiet NaN, else (invalid op) default NaN
|
||||
@@ -157,6 +157,9 @@ SN T FN(slow)(Cpu *c, int op, T a, T b, T d) {
|
||||
// an exact result just below the smallest normal that rounds up to it is flushed on ARM only. The slow path
|
||||
// decides both cases with the operands flushed.
|
||||
#define FREDO(op, r) (FBAD(op, r) | (__builtin_elementwise_abs(r) == FN(fv)(MINN)))
|
||||
// The complement of FREDO for every op but RSQRTS, as one compare: |r| ordered and not the smallest normal (LLVM folds
|
||||
// it to fcmp one, a single ucomiss + jne for scalars and one vcmpps for vectors).
|
||||
#define FOK(r) ((__builtin_elementwise_abs(r) < FN(fv)(MINN)) | (__builtin_elementwise_abs(r) > FN(fv)(MINN)))
|
||||
SN T FN(redo)(Cpu *c, int op, T a, T b, T d, T r) {
|
||||
if (c->fpcr & FPCR_FZ) return FN(slow)(c, op, a, b, d);
|
||||
return FBAD(op, r) ? FN(fix)(c, op, a, b, d) : r;
|
||||
@@ -164,13 +167,13 @@ SN T FN(redo)(Cpu *c, int op, T a, T b, T d, T r) {
|
||||
|
||||
// Scalar FP op (constant op folds the selection). RMode = nearest runs on the host, with or without FZ.
|
||||
SI T FN(op)(Cpu *c, int op, T a, T b, T d) {
|
||||
if (__builtin_expect(!simd_fast(c), 0)) return FN(slow)(c, op, a, b, d);
|
||||
if (simd_directed(c)) return FN(slow)(c, op, a, b, d);
|
||||
if (FMINMAX(op)) {
|
||||
if (c->fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b);
|
||||
return FN(minmax)(c, op, a, b);
|
||||
}
|
||||
T r = FCALC(op, a, b, d, K);
|
||||
if (__builtin_expect(FREDO(op, r), 0)) r = FN(redo)(c, op, a, b, d, r);
|
||||
if (__builtin_expect(op == FOP_RSQRTS ? FREDO(op, r) : !FOK(r), 0)) r = FN(redo)(c, op, a, b, d, r);
|
||||
return r;
|
||||
}
|
||||
|
||||
@@ -192,7 +195,7 @@ SI VT FN(vflush)(VT a) {
|
||||
|
||||
// Vector FP op: native lanes; any lane FREDO flags (or FPCR.RMode) reruns all lanes through the scalar path.
|
||||
SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) {
|
||||
if (__builtin_expect(!simd_fast(c), 0)) return FN(vslow)(c, op, a, b, d);
|
||||
if (simd_directed(c)) return FN(vslow)(c, op, a, b, d);
|
||||
VT r;
|
||||
if (FMINMAX(op)) {
|
||||
if (c->fpcr & FPCR_FZ) a = FN(vflush)(a), b = FN(vflush)(b);
|
||||
@@ -205,11 +208,12 @@ SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) {
|
||||
return r;
|
||||
}
|
||||
r = FCALC(op, a, b, d, VK);
|
||||
if (__builtin_expect(__builtin_reduce_or(FREDO(op, r)) != 0, 0)) return FN(vslow)(c, op, a, b, d);
|
||||
if (__builtin_expect(op == FOP_RSQRTS ? __builtin_reduce_or(FREDO(op, r)) != 0 : __builtin_reduce_and(FOK(r)) == 0, 0))
|
||||
return FN(vslow)(c, op, a, b, d);
|
||||
return r;
|
||||
}
|
||||
|
||||
SI VT FN(vfz)(Cpu *c, VT a) { (void)simd_fast(c); return c->fpcr & FPCR_FZ ? FN(vflush)(a) : a; }
|
||||
SI VT FN(vfz)(Cpu *c, VT a) { return c->fpcr & FPCR_FZ ? FN(vflush)(a) : a; }
|
||||
|
||||
// ---- compares
|
||||
SI uint64_t FN(cmp)(Cpu *c, T a, T b) { // FCMP NZCV
|
||||
|
||||
@@ -136,6 +136,7 @@ int main(int argc, char **argv) {
|
||||
c.sp = in.sp;
|
||||
c.nzcv = in.nzcv;
|
||||
c.fpcr = in.fpcr;
|
||||
rt_fpcr_sync(&c);
|
||||
memcpy(c.v, in.v, sizeof c.v);
|
||||
c.excl_addr = ~0ull;
|
||||
g_stubs = (int)in.stubs;
|
||||
|
||||
Reference in new issue
Block a user