mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 04:35:57 +00:00
Run the game on x86-64-v2 CPUs without AVX2
-march=x86-64-v3 kept the port and the installer to CPUs with AVX2 and FMA. --cpu v2 compiles the generated code and the runtime for SSE4.2 instead. Without FMA3 the FP lowering calls fma/fmaf, so runtime/fma.c provides correctly rounded ones (musl, MIT) that honour MXCSR's rounding mode, FTZ and DAZ; the UCRT's software fallback rounds incorrectly by up to half an ulp. simd.h's fast paths keep their assumptions. The installer's host check asks for x86-64-v2 and selects libtotkrt.a or libtotkrt-v2.a; both release scripts build and ship the pair. The differential tests compile fma.c as well, so a --cflags=-march=x86-64-v2 run tests it. Tested on an i7-3770K (no AVX2, FMA or BMI) with an RTX 3060 Ti, game 1.4.3: the installer ran end to end, --selftest passed and play holds 40-60 fps at 1080p. fma and fmaf matched 152,288 exact vectors over two seeds, including ties, subnormals and every rounding mode, FTZ and DAZ combination. The v3 build is unchanged: fma.c is an empty translation unit there.
This commit is contained in:
1 parent
7d62cbfa85
commit
34b4faee7c
19 files changed
+402
-58
No files matched your search
+1
-1
@@ -109,7 +109,7 @@ EXCL(16, uint16_t)
|
||||
EXCL(32, uint32_t)
|
||||
EXCL(64, uint64_t)
|
||||
|
||||
// Exclusive pair of two X registers: one 16-byte-aligned 128-bit access (cmpxchg16b; x86-64-v3 has CX16).
|
||||
// Exclusive pair of two X registers: one 16-byte-aligned 128-bit access (cmpxchg16b; x86-64-v2 has CX16).
|
||||
// A pair of two W registers is a 64-bit exclusive and uses ldx64/stx64.
|
||||
static inline __uint128_t ldxp128(Cpu *c, uint64_t a) {
|
||||
__uint128_t v = __atomic_load_n((__uint128_t *)(uintptr_t)a, __ATOMIC_ACQUIRE);
|
||||
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
// Correctly rounded fma/fmaf for x86-64 builds without FMA3: the -march=x86-64-v2 build (docs/building.md).
|
||||
//
|
||||
// runtime/simd.h lowers AArch64 FMADD/FMSUB and the RECPS/RSQRTS helpers to __builtin_elementwise_fma, which clang
|
||||
// turns into a call to fma/fmaf when the target has no FMA instruction. The C runtime is not a substitute: on CPUs
|
||||
// without FMA3 the UCRT's software fallback is documented as rounding incorrectly by up to half an ulp, while the FP
|
||||
// lowering needs the single rounding the ARM instruction gives. This file supplies it.
|
||||
//
|
||||
// Adapted from musl src/math/fma.c and src/math/fmaf.c (MIT, Copyright (c) 2005-2020 Rich Felker and contributors).
|
||||
// The result is rounded in the MXCSR rounding mode the caller set (simd.h simd_rm_enter), and MXCSR FTZ and DAZ are
|
||||
// honoured as the hardware FMA would, so runtime/simd.h's fast paths keep their assumptions: denormal inputs read as
|
||||
// zero and denormal results as zero under FPCR.FZ, without a trip through the slow path.
|
||||
//
|
||||
// The definitions exist only where the compiler would emit the calls. A -march=x86-64-v3 build defines __FMA__ and
|
||||
// emits vfmadd instead, so this file contributes nothing there.
|
||||
#include <stdint.h>
|
||||
|
||||
#if defined(__x86_64__) && !defined(__FMA__)
|
||||
|
||||
#include <float.h>
|
||||
#include <immintrin.h>
|
||||
|
||||
#define MXCSR_DAZ 0x0040u
|
||||
#define MXCSR_FTZ 0x8000u
|
||||
|
||||
static double fma_flush_d(double x) {
|
||||
union { double f; uint64_t i; } u = {x};
|
||||
if (!(u.i & 0x7ff0000000000000ull) && (u.i & 0x000fffffffffffffull)) u.i &= 0x8000000000000000ull;
|
||||
return u.f;
|
||||
}
|
||||
|
||||
static float fma_flush_f(float x) {
|
||||
union { float f; uint32_t i; } u = {x};
|
||||
if (!(u.i & 0x7f800000u) && (u.i & 0x007fffffu)) u.i &= 0x80000000u;
|
||||
return u.f;
|
||||
}
|
||||
|
||||
// ---- float: the exact product fits in a double, so one extra term and a round-to-odd step decide the float
|
||||
// rounding. Halfway and tiny results are the only ones where rounding the double first can change the float result.
|
||||
float fmaf(float x, float y, float z) {
|
||||
unsigned csr = _mm_getcsr();
|
||||
if (csr & MXCSR_DAZ) x = fma_flush_f(x), y = fma_flush_f(y), z = fma_flush_f(z);
|
||||
double xy = (double)x * y; // exact
|
||||
union { double r; uint64_t i; } u = {xy + z};
|
||||
int e = u.i >> 52 & 0x7ff;
|
||||
int halfway = (u.i & 0x1fffffff) == 0x10000000;
|
||||
int tiny = e <= 0x3ff - 126 && e >= 0x3ff - 149;
|
||||
if (halfway || tiny) {
|
||||
if (e != 0x7ff) {
|
||||
// r+t is the exact sum in nearest rounding; otherwise t keeps its sign and whether it is zero, which
|
||||
// is all the round-to-odd step needs. An odd sticky bit makes the final conversion round once, in
|
||||
// whichever mode MXCSR asks for.
|
||||
int s = u.i >> 63;
|
||||
double t = s == (xy < z) ? xy - u.r + z : z - u.r + xy;
|
||||
if (t) {
|
||||
u.i -= s ^ (t < 0);
|
||||
u.i |= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
float r = (float)u.r;
|
||||
if (csr & MXCSR_FTZ) r = fma_flush_f(r);
|
||||
return r;
|
||||
}
|
||||
|
||||
// ---- double: the significands are multiplied and aligned as integers, then converted back with one rounding.
|
||||
#define ASUINT64(x) ((union { double f; uint64_t i; }){x}).i
|
||||
#define ZEROINFNAN (0x7ff - 0x3ff - 52 - 1)
|
||||
|
||||
struct fma_num { uint64_t m; int e; int sign; };
|
||||
|
||||
static struct fma_num fma_normalize(double x) {
|
||||
uint64_t ix = ASUINT64(x);
|
||||
int e = ix >> 52;
|
||||
int sign = e & 0x800;
|
||||
e &= 0x7ff;
|
||||
if (!e) {
|
||||
ix = ASUINT64(x * 0x1p63);
|
||||
e = ix >> 52 & 0x7ff;
|
||||
e = e ? e - 63 : 0x800;
|
||||
}
|
||||
ix &= (1ull << 52) - 1;
|
||||
ix |= 1ull << 52;
|
||||
ix <<= 1;
|
||||
e -= 0x3ff + 52 + 1;
|
||||
return (struct fma_num){ix, e, sign};
|
||||
}
|
||||
|
||||
static void fma_mul(uint64_t *hi, uint64_t *lo, uint64_t x, uint64_t y) {
|
||||
uint64_t xlo = (uint32_t)x, xhi = x >> 32;
|
||||
uint64_t ylo = (uint32_t)y, yhi = y >> 32;
|
||||
uint64_t t1 = xlo * ylo;
|
||||
uint64_t t2 = xlo * yhi + xhi * ylo;
|
||||
uint64_t t3 = xhi * yhi;
|
||||
*lo = t1 + (t2 << 32);
|
||||
*hi = t3 + (t2 >> 32) + (t1 > *lo);
|
||||
}
|
||||
|
||||
// scalbn from musl src/math/scalbn.c (MIT): the fma's final scaling, kept here so the result does not depend on the
|
||||
// C runtime's version. Its multiplications honour MXCSR like any other host FP operation.
|
||||
static double fma_scalbn(double x, int n) {
|
||||
union { double f; uint64_t i; } u;
|
||||
double y = x;
|
||||
if (n > 1023) {
|
||||
y *= 0x1p1023;
|
||||
n -= 1023;
|
||||
if (n > 1023) {
|
||||
y *= 0x1p1023;
|
||||
n -= 1023;
|
||||
if (n > 1023) n = 1023;
|
||||
}
|
||||
} else if (n < -1022) {
|
||||
// Make sure the final n < -53, so the subnormal range rounds only once.
|
||||
y *= 0x1p-1022 * 0x1p53;
|
||||
n += 1022 - 53;
|
||||
if (n < -1022) {
|
||||
y *= 0x1p-1022 * 0x1p53;
|
||||
n += 1022 - 53;
|
||||
if (n < -1022) n = -1022;
|
||||
}
|
||||
}
|
||||
u.i = (uint64_t)(0x3ff + n) << 52;
|
||||
return y * u.f;
|
||||
}
|
||||
|
||||
double fma(double x, double y, double z) {
|
||||
#pragma STDC FENV_ACCESS ON
|
||||
unsigned csr = _mm_getcsr();
|
||||
if (csr & MXCSR_DAZ) x = fma_flush_d(x), y = fma_flush_d(y), z = fma_flush_d(z);
|
||||
|
||||
struct fma_num nx = fma_normalize(x), ny = fma_normalize(y), nz = fma_normalize(z);
|
||||
if (nx.e >= ZEROINFNAN || ny.e >= ZEROINFNAN) return x * y + z;
|
||||
if (nz.e >= ZEROINFNAN) {
|
||||
if (nz.e > ZEROINFNAN) return x * y; // z == 0
|
||||
return z;
|
||||
}
|
||||
|
||||
uint64_t rhi, rlo, zhi, zlo;
|
||||
fma_mul(&rhi, &rlo, nx.m, ny.m);
|
||||
|
||||
// Align the exponents: shift z left by kz and r right by kr with kz + kr == d.
|
||||
int e = nx.e + ny.e;
|
||||
int d = nz.e - e;
|
||||
if (d > 0) {
|
||||
if (d < 64) {
|
||||
zlo = nz.m << d;
|
||||
zhi = nz.m >> (64 - d);
|
||||
} else {
|
||||
zlo = 0;
|
||||
zhi = nz.m;
|
||||
e = nz.e - 64;
|
||||
d -= 64;
|
||||
if (d == 0) {
|
||||
} else if (d < 64) {
|
||||
rlo = rhi << (64 - d) | rlo >> d | !!(rlo << (64 - d));
|
||||
rhi = rhi >> d;
|
||||
} else {
|
||||
rlo = 1;
|
||||
rhi = 0;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
zhi = 0;
|
||||
d = -d;
|
||||
if (d == 0) {
|
||||
zlo = nz.m;
|
||||
} else if (d < 64) {
|
||||
zlo = nz.m >> d | !!(nz.m << (64 - d));
|
||||
} else {
|
||||
zlo = 1;
|
||||
}
|
||||
}
|
||||
|
||||
int sign = nx.sign ^ ny.sign;
|
||||
int samesign = !(sign ^ nz.sign);
|
||||
int nonzero = 1;
|
||||
if (samesign) {
|
||||
rlo += zlo;
|
||||
rhi += zhi + (rlo < zlo);
|
||||
} else {
|
||||
uint64_t t = rlo;
|
||||
rlo -= zlo;
|
||||
rhi = rhi - zhi - (t < rlo);
|
||||
if (rhi >> 63) {
|
||||
rlo = -rlo;
|
||||
rhi = -rhi - !!rlo;
|
||||
sign = !sign;
|
||||
}
|
||||
nonzero = !!rhi;
|
||||
}
|
||||
|
||||
// Keep the top 63 bits in rhi; the last bit is sticky.
|
||||
if (nonzero) {
|
||||
e += 64;
|
||||
d = __builtin_clzll(rhi) - 1;
|
||||
rhi = rhi << d | rlo >> (64 - d) | !!(rlo << d);
|
||||
} else if (rlo) {
|
||||
d = __builtin_clzll(rlo) - 1;
|
||||
if (d < 0)
|
||||
rhi = rlo >> 1 | (rlo & 1);
|
||||
else
|
||||
rhi = rlo << d;
|
||||
} else {
|
||||
return x * y + z; // exact +-0
|
||||
}
|
||||
e -= d;
|
||||
|
||||
int64_t i = rhi;
|
||||
if (sign) i = -i;
|
||||
double r = i; // |r| is in [0x1p62, 0x1p63]
|
||||
|
||||
if (e < -1022 - 62) {
|
||||
// The result is subnormal before rounding.
|
||||
if (e == -1022 - 63) {
|
||||
double c = 0x1p63;
|
||||
if (sign) c = -c;
|
||||
if (r == c) {
|
||||
// The smallest normal after rounding: whether it underflows depends on the architecture. A double
|
||||
// to float conversion imitates the before-rounding rule the guest has.
|
||||
float fltmin = 0x0.ffffff8p-63 * FLT_MIN * r;
|
||||
r = DBL_MIN / FLT_MIN * fltmin;
|
||||
return (csr & MXCSR_FTZ) ? fma_flush_d(r) : r;
|
||||
}
|
||||
// One bit is lost when scaled: add another top bit so the conversion rounds only once if inexact.
|
||||
if (rhi << 53) {
|
||||
i = rhi >> 1 | (rhi & 1) | 1ull << 62;
|
||||
if (sign) i = -i;
|
||||
r = i;
|
||||
r = 2 * r - c;
|
||||
// Raise underflow portably, so the compiler cannot optimize the expression away.
|
||||
double tiny = DBL_MIN / FLT_MIN * r;
|
||||
r += (double)(tiny * tiny) * (r - r);
|
||||
}
|
||||
} else {
|
||||
// Only round once when scaled.
|
||||
d = 10;
|
||||
i = (rhi >> d | !!(rhi << (64 - d))) << d;
|
||||
if (sign) i = -i;
|
||||
r = i;
|
||||
}
|
||||
}
|
||||
r = fma_scalbn(r, e);
|
||||
if (csr & MXCSR_FTZ) r = fma_flush_d(r);
|
||||
return r;
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -249,11 +249,22 @@ static double now_s(void) { return (double)plat_now_ns() / 1e9; }
|
||||
static volatile int host_probe_hits;
|
||||
static void host_probe(Cpu *c) { host_probe_hits++; c->x[0] = 42; }
|
||||
|
||||
// The instruction set build_game.py --cpu compiled for: v3 (AVX2/FMA/BMI2, the default) or v2 (SSE4.2/POPCNT).
|
||||
#ifndef RT_CPU_LEVEL
|
||||
#define RT_CPU_LEVEL 3
|
||||
#endif
|
||||
|
||||
static void check_cpu(void) {
|
||||
__builtin_cpu_init();
|
||||
#if RT_CPU_LEVEL >= 3
|
||||
int ok = __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma") && __builtin_cpu_supports("bmi2");
|
||||
printf("cpu: x86-64-v3 (AVX2, FMA, BMI2) %s\n", ok ? "present" : "MISSING");
|
||||
CHECK(ok, "the lifted code is built for x86-64-v3");
|
||||
#else
|
||||
int ok = __builtin_cpu_supports("sse4.2") && __builtin_cpu_supports("popcnt");
|
||||
printf("cpu: x86-64-v2 (SSE4.2, POPCNT) %s\n", ok ? "present" : "MISSING");
|
||||
CHECK(ok, "the lifted code is built for x86-64-v2");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Layout: ascending, no overlap, below 2^39, and inside the window mem_init reserved.
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@
|
||||
// tables. Not modelled: FPSR cumulative exception bits and QC, FPCR trap enables, FP16 arithmetic (ARMv8.2).
|
||||
// FPCR.AHP is honoured by the half-precision conversions. Host assumptions: IEEE binary32/64, MXCSR in
|
||||
// round-to-nearest outside the slow paths, with FTZ|DAZ equal to FPCR.FZ (cpu.h rt_fpcr_sync keeps it so); fma must be
|
||||
// correctly rounded (build x86 with -mfma).
|
||||
// correctly rounded: -march=x86-64-v3 has the FMA instruction, and the v2 build links runtime/fma.c for it.
|
||||
#pragma once
|
||||
#include <fenv.h>
|
||||
#include <immintrin.h>
|
||||
|
||||
Reference in new issue
Block a user