runtime: vectorized AVX round-to-odd FMA emulation for non-FMA/AVX-only CPUs

This commit is contained in:
deanzx06 committed 2026-11-02 05:17:58 +07:00
1 parent 6c2595275d
commit ec9029def4
1 file changed
+58 -11
+58 -11
View File
@@ -59,16 +59,60 @@ SN T FN(nan3)(Cpu *c, T a, T b, T d) {
return FN(fv)(DNAN);
}
// ---- FMA flavours. FMA_S / FMA_V: fast paths, host MXCSR is round-to-nearest there (simd_directed() is checked first).
// FMA_X: scalar FMA for the slow paths, any host rounding mode.
// Without hardware FMA (Ivy Bridge, no -mfma) clang turns __builtin_elementwise_fma into a call to the C runtime's
// fmaf/fma for every lane, which is very slow and keeps the thread outside generated code. For float the product of
// two floats is exact in a double (24 x 24 = 48 bits), so the fused op is done in double. A plain
// (float)(x * y + z) rounds twice and can differ by one ulp; rounding the double sum to odd first (Boldo-Melquiond)
// makes the final float rounding exact: TwoSum gives the error e of s = fl(p + z), and when e != 0 and s has an even
// last bit, s is moved one ulp toward the exact value (the odd neighbour). Inf/NaN lanes are left alone (e is NaN
// there, so the ordered compares below are false). Double has no wider type: it keeps the C runtime's fma.
#if defined(__FMA__) || defined(__aarch64__) || NB == 64
#define FMA_S(x, y, z) __builtin_elementwise_fma(x, y, z)
#define FMA_V(x, y, z) __builtin_elementwise_fma(x, y, z)
#define FMA_X(x, y, z) __builtin_elementwise_fma(x, y, z)
#else
typedef double simd_vf64x4 __attribute__((vector_size(32)));
typedef uint64_t simd_vu64x4 __attribute__((vector_size(32)));
SI float fma_rn_s(float x, float y, float z) {
double p = (double)x * (double)y, zd = (double)z, s = p + zd;
double bb = s - p, e = (p - (s - bb)) + (zd - bb); // TwoSum: s + e == p + zd exactly
uint64_t u;
memcpy(&u, &s, sizeof u);
if (((e < 0.0) | (e > 0.0)) && !(u & 1)) u += (e > 0.0) == (s > 0.0) ? 1 : (uint64_t)-1;
memcpy(&s, &u, sizeof s);
return (float)s;
}
SI vf32 fma_rn_v(vf32 x, vf32 y, vf32 z) {
simd_vf64x4 p = CVT(x, simd_vf64x4) * CVT(y, simd_vf64x4), zd = CVT(z, simd_vf64x4), s = p + zd;
simd_vf64x4 bb = s - p, e = (p - (s - bb)) + (zd - bb);
simd_vu64x4 u = (simd_vu64x4)s;
simd_vu64x4 fix = (simd_vu64x4)((e < 0.0) | (e > 0.0)) & (simd_vu64x4)((u & 1) == 0);
simd_vu64x4 up = (simd_vu64x4)((e > 0.0) == (s > 0.0));
u += fix & ((up & 1) | ~up); // +1 toward the exact value, i.e. +1 if e and s share a sign, else -1
return CVT((simd_vf64x4)u, vf32);
}
// Any rounding mode: directed modes are exact with a plain double (same-direction double rounding is harmless).
SI float fma_any_s(float x, float y, float z) {
if (__builtin_expect(_mm_getcsr() & 0x6000u, 0)) return (float)((double)x * (double)y + (double)z);
return fma_rn_s(x, y, z);
}
#define FMA_S(x, y, z) fma_rn_s(x, y, z)
#define FMA_V(x, y, z) fma_rn_v(x, y, z)
#define FMA_X(x, y, z) fma_any_s(x, y, z)
#endif
// The arithmetic itself, for scalars (KK = K) and vectors (KK = VK). MADD: a + b*d (a = addend).
// RECPS/RSQRTS get their first operand already negated (FPNeg happens before NaN processing).
#define FCALC(op, a, b, d, KK) \
#define FCALC(op, a, b, d, KK, FMA) \
((op) == FOP_ADD ? (a) + (b) : (op) == FOP_SUB ? (a) - (b) \
: (op) == FOP_MUL || (op) == FOP_MULX ? (a) * (b) : (op) == FOP_NMUL ? -((a) * (b)) \
: (op) == FOP_DIV ? (a) / (b) : (op) == FOP_ABD ? __builtin_elementwise_abs((a) - (b)) \
: (op) == FOP_SQRT ? __builtin_elementwise_sqrt(a) \
: (op) == FOP_MADD ? __builtin_elementwise_fma(b, d, a) \
: (op) == FOP_RECPS ? __builtin_elementwise_fma(a, b, KK(2)) \
: (op) == FOP_RSQRTS ? __builtin_elementwise_fma(a, b, KK(3)) * KK(0.5) : (a))
: (op) == FOP_MADD ? FMA(b, d, a) \
: (op) == FOP_RECPS ? FMA(a, b, KK(2)) \
: (op) == FOP_RSQRTS ? FMA(a, b, KK(3)) * KK(0.5) : (a))
// Lanes whose fast result needs fixing: NaN results (NaN operand or invalid op), RSQRTS overflow.
#define FBAD(op, r) ((op) == FOP_RSQRTS ? ((r) != (r)) | (__builtin_elementwise_abs(r) == K(INFINITY)) : (r) != (r))
@@ -76,9 +120,9 @@ SN T FN(nan3)(Cpu *c, T a, T b, T d) {
// (3 + a*b) / 2 with one rounding (a already negated): halve a normal operand first, so a product beyond
// the format's range (which the fast path rounds to inf, or to the largest normal when rounding down) is exact.
SN T FN(rsqrts)(T a, T b) {
if (__builtin_elementwise_abs(a) >= FN(fv)(2 * MINN)) return __builtin_elementwise_fma(a * K(0.5), b, K(1.5));
if (__builtin_elementwise_abs(b) >= FN(fv)(2 * MINN)) return __builtin_elementwise_fma(a, b * K(0.5), K(1.5));
return __builtin_elementwise_fma(a, b, K(3)) * K(0.5);
if (__builtin_elementwise_abs(a) >= FN(fv)(2 * MINN)) return FMA_X(a * K(0.5), b, K(1.5));
if (__builtin_elementwise_abs(b) >= FN(fv)(2 * MINN)) return FMA_X(a, b * K(0.5), K(1.5));
return FMA_X(a, b, K(3)) * K(0.5);
}
// ARM result for a lane flagged by FBAD (operands already flushed).
@@ -136,13 +180,13 @@ SN T FN(slow)(Cpu *c, int op, T a, T b, T d) {
if (FMINMAX(op)) return FN(minmax)(c, op, a, b);
unsigned old = simd_rm_enter(fpcr);
volatile T va = a, vb = b, vd = d;
volatile T vr = op == FOP_RSQRTS ? FN(rsqrts)(va, vb) : FCALC(op, va, vb, vd, K);
volatile T vr = op == FOP_RSQRTS ? FN(rsqrts)(va, vb) : FCALC(op, va, vb, vd, K, FMA_X);
T r = vr;
if (FBAD(op, r)) vr = FN(fix)(c, op, a, b, d), r = vr;
if ((fpcr & FPCR_FZ) && (FN(fb)(r) & ~SGN) <= MINN && !FN(iszero)(FN(fb)(r))) {
if ((FN(fb)(r) & ~SGN) == MINN) {
simd_rm_rz();
volatile T vz = FCALC(op, va, vb, vd, K);
volatile T vz = FCALC(op, va, vb, vd, K, FMA_X);
if ((FN(fb)(vz) & ~SGN) < MINN) r = FN(fv)(FN(fb)(r) & SGN);
} else {
r = FN(fv)(FN(fb)(r) & SGN);
@@ -172,7 +216,7 @@ SI T FN(op)(Cpu *c, int op, T a, T b, T d) {
if (c->fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b);
return FN(minmax)(c, op, a, b);
}
T r = FCALC(op, a, b, d, K);
T r = FCALC(op, a, b, d, K, FMA_S);
if (__builtin_expect(op == FOP_RSQRTS ? FREDO(op, r) : !FOK(r), 0)) r = FN(redo)(c, op, a, b, d, r);
return r;
}
@@ -207,7 +251,7 @@ SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) {
r = (VT)FN(vsel)(a < b, x, FN(vsel)(a == b, x | y, y));
return r;
}
r = FCALC(op, a, b, d, VK);
r = FCALC(op, a, b, d, VK, FMA_V);
if (__builtin_expect(op == FOP_RSQRTS ? __builtin_reduce_or(FREDO(op, r)) != 0 : __builtin_reduce_and(FOK(r)) == 0, 0))
return FN(vslow)(c, op, a, b, d);
return r;
@@ -363,3 +407,6 @@ SI T FN(recpx)(Cpu *c, T a) {
#undef MINN
#undef VK
#undef K
#undef FMA_S
#undef FMA_V
#undef FMA_X