From ec9029def487b8ae202c4f258d1d05f108dd02e5 Mon Sep 17 00:00:00 2001 From: deanzx06 Date: Mon, 2 Nov 2026 05:17:58 +0700 Subject: [PATCH] runtime: vectorized AVX round-to-odd FMA emulation for non-FMA/AVX-only CPUs --- runtime/simd_fp.h | 69 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/runtime/simd_fp.h b/runtime/simd_fp.h index 7020c254..5593bb5b 100644 --- a/runtime/simd_fp.h +++ b/runtime/simd_fp.h @@ -59,16 +59,60 @@ SN T FN(nan3)(Cpu *c, T a, T b, T d) { return FN(fv)(DNAN); } +// ---- FMA flavours. FMA_S / FMA_V: fast paths, host MXCSR is round-to-nearest there (simd_directed() is checked first). +// FMA_X: scalar FMA for the slow paths, any host rounding mode. +// Without hardware FMA (Ivy Bridge, no -mfma) clang turns __builtin_elementwise_fma into a call to the C runtime's +// fmaf/fma for every lane, which is very slow and keeps the thread outside generated code. For float the product of +// two floats is exact in a double (24 x 24 = 48 bits), so the fused op is done in double. A plain +// (float)(x * y + z) rounds twice and can differ by one ulp; rounding the double sum to odd first (Boldo-Melquiond) +// makes the final float rounding exact: TwoSum gives the error e of s = fl(p + z), and when e != 0 and s has an even +// last bit, s is moved one ulp toward the exact value (the odd neighbour). Inf/NaN lanes are left alone (e is NaN +// there, so the ordered compares below are false). Double has no wider type: it keeps the C runtime's fma. +#if defined(__FMA__) || defined(__aarch64__) || NB == 64 +#define FMA_S(x, y, z) __builtin_elementwise_fma(x, y, z) +#define FMA_V(x, y, z) __builtin_elementwise_fma(x, y, z) +#define FMA_X(x, y, z) __builtin_elementwise_fma(x, y, z) +#else +typedef double simd_vf64x4 __attribute__((vector_size(32))); +typedef uint64_t simd_vu64x4 __attribute__((vector_size(32))); +SI float fma_rn_s(float x, float y, float z) { + double p = (double)x * (double)y, zd = (double)z, s = p + zd; + double bb = s - p, e = (p - (s - bb)) + (zd - bb); // TwoSum: s + e == p + zd exactly + uint64_t u; + memcpy(&u, &s, sizeof u); + if (((e < 0.0) | (e > 0.0)) && !(u & 1)) u += (e > 0.0) == (s > 0.0) ? 1 : (uint64_t)-1; + memcpy(&s, &u, sizeof s); + return (float)s; +} +SI vf32 fma_rn_v(vf32 x, vf32 y, vf32 z) { + simd_vf64x4 p = CVT(x, simd_vf64x4) * CVT(y, simd_vf64x4), zd = CVT(z, simd_vf64x4), s = p + zd; + simd_vf64x4 bb = s - p, e = (p - (s - bb)) + (zd - bb); + simd_vu64x4 u = (simd_vu64x4)s; + simd_vu64x4 fix = (simd_vu64x4)((e < 0.0) | (e > 0.0)) & (simd_vu64x4)((u & 1) == 0); + simd_vu64x4 up = (simd_vu64x4)((e > 0.0) == (s > 0.0)); + u += fix & ((up & 1) | ~up); // +1 toward the exact value, i.e. +1 if e and s share a sign, else -1 + return CVT((simd_vf64x4)u, vf32); +} +// Any rounding mode: directed modes are exact with a plain double (same-direction double rounding is harmless). +SI float fma_any_s(float x, float y, float z) { + if (__builtin_expect(_mm_getcsr() & 0x6000u, 0)) return (float)((double)x * (double)y + (double)z); + return fma_rn_s(x, y, z); +} +#define FMA_S(x, y, z) fma_rn_s(x, y, z) +#define FMA_V(x, y, z) fma_rn_v(x, y, z) +#define FMA_X(x, y, z) fma_any_s(x, y, z) +#endif + // The arithmetic itself, for scalars (KK = K) and vectors (KK = VK). MADD: a + b*d (a = addend). // RECPS/RSQRTS get their first operand already negated (FPNeg happens before NaN processing). -#define FCALC(op, a, b, d, KK) \ +#define FCALC(op, a, b, d, KK, FMA) \ ((op) == FOP_ADD ? (a) + (b) : (op) == FOP_SUB ? (a) - (b) \ : (op) == FOP_MUL || (op) == FOP_MULX ? (a) * (b) : (op) == FOP_NMUL ? -((a) * (b)) \ : (op) == FOP_DIV ? (a) / (b) : (op) == FOP_ABD ? __builtin_elementwise_abs((a) - (b)) \ : (op) == FOP_SQRT ? __builtin_elementwise_sqrt(a) \ - : (op) == FOP_MADD ? __builtin_elementwise_fma(b, d, a) \ - : (op) == FOP_RECPS ? __builtin_elementwise_fma(a, b, KK(2)) \ - : (op) == FOP_RSQRTS ? __builtin_elementwise_fma(a, b, KK(3)) * KK(0.5) : (a)) + : (op) == FOP_MADD ? FMA(b, d, a) \ + : (op) == FOP_RECPS ? FMA(a, b, KK(2)) \ + : (op) == FOP_RSQRTS ? FMA(a, b, KK(3)) * KK(0.5) : (a)) // Lanes whose fast result needs fixing: NaN results (NaN operand or invalid op), RSQRTS overflow. #define FBAD(op, r) ((op) == FOP_RSQRTS ? ((r) != (r)) | (__builtin_elementwise_abs(r) == K(INFINITY)) : (r) != (r)) @@ -76,9 +120,9 @@ SN T FN(nan3)(Cpu *c, T a, T b, T d) { // (3 + a*b) / 2 with one rounding (a already negated): halve a normal operand first, so a product beyond // the format's range (which the fast path rounds to inf, or to the largest normal when rounding down) is exact. SN T FN(rsqrts)(T a, T b) { - if (__builtin_elementwise_abs(a) >= FN(fv)(2 * MINN)) return __builtin_elementwise_fma(a * K(0.5), b, K(1.5)); - if (__builtin_elementwise_abs(b) >= FN(fv)(2 * MINN)) return __builtin_elementwise_fma(a, b * K(0.5), K(1.5)); - return __builtin_elementwise_fma(a, b, K(3)) * K(0.5); + if (__builtin_elementwise_abs(a) >= FN(fv)(2 * MINN)) return FMA_X(a * K(0.5), b, K(1.5)); + if (__builtin_elementwise_abs(b) >= FN(fv)(2 * MINN)) return FMA_X(a, b * K(0.5), K(1.5)); + return FMA_X(a, b, K(3)) * K(0.5); } // ARM result for a lane flagged by FBAD (operands already flushed). @@ -136,13 +180,13 @@ SN T FN(slow)(Cpu *c, int op, T a, T b, T d) { if (FMINMAX(op)) return FN(minmax)(c, op, a, b); unsigned old = simd_rm_enter(fpcr); volatile T va = a, vb = b, vd = d; - volatile T vr = op == FOP_RSQRTS ? FN(rsqrts)(va, vb) : FCALC(op, va, vb, vd, K); + volatile T vr = op == FOP_RSQRTS ? FN(rsqrts)(va, vb) : FCALC(op, va, vb, vd, K, FMA_X); T r = vr; if (FBAD(op, r)) vr = FN(fix)(c, op, a, b, d), r = vr; if ((fpcr & FPCR_FZ) && (FN(fb)(r) & ~SGN) <= MINN && !FN(iszero)(FN(fb)(r))) { if ((FN(fb)(r) & ~SGN) == MINN) { simd_rm_rz(); - volatile T vz = FCALC(op, va, vb, vd, K); + volatile T vz = FCALC(op, va, vb, vd, K, FMA_X); if ((FN(fb)(vz) & ~SGN) < MINN) r = FN(fv)(FN(fb)(r) & SGN); } else { r = FN(fv)(FN(fb)(r) & SGN); @@ -172,7 +216,7 @@ SI T FN(op)(Cpu *c, int op, T a, T b, T d) { if (c->fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b); return FN(minmax)(c, op, a, b); } - T r = FCALC(op, a, b, d, K); + T r = FCALC(op, a, b, d, K, FMA_S); if (__builtin_expect(op == FOP_RSQRTS ? FREDO(op, r) : !FOK(r), 0)) r = FN(redo)(c, op, a, b, d, r); return r; } @@ -207,7 +251,7 @@ SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) { r = (VT)FN(vsel)(a < b, x, FN(vsel)(a == b, x | y, y)); return r; } - r = FCALC(op, a, b, d, VK); + r = FCALC(op, a, b, d, VK, FMA_V); if (__builtin_expect(op == FOP_RSQRTS ? __builtin_reduce_or(FREDO(op, r)) != 0 : __builtin_reduce_and(FOK(r)) == 0, 0)) return FN(vslow)(c, op, a, b, d); return r; @@ -363,3 +407,6 @@ SI T FN(recpx)(Cpu *c, T a) { #undef MINN #undef VK #undef K +#undef FMA_S +#undef FMA_V +#undef FMA_X