mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-18 12:08:38 +00:00
Merge task/fpslim: the FP fast paths of the game's code are smaller with the same results, 8% fewer instructions in the hottest functions (#381)
The directed-RMode test folds into the result compare through a per-thread threshold, vector slow paths take their operands through the Cpu instead of stack copies (___chkstk_ms leaves the large lifted functions), the static FP helpers use preserve_most, and the slow paths rerun only the failing lanes. Bit-identical to the old headers: 14.85M edge cases under every FPCR mode on v3 and v2, and 13.4G FP ops compared in a check-mode build in the town and fight2. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
b642c849ef
9 files changed
+90
-12
No files matched your search
+23
-3
@@ -18,7 +18,11 @@ typedef struct __attribute__((aligned(16))) Cpu {
|
||||
uint64_t fpcr; // 320: guest FPCR/FPSR (portable FP/SIMD lowering only; the asm path uses the host's)
|
||||
uint64_t fpsr; // 328
|
||||
uint64_t excl_val_hi; // 336: upper half of a 128-bit exclusive pair (LDXP/LDAXP of two X registers)
|
||||
uint8_t _pad[512 - 344];
|
||||
uint64_t _pad0; // 344
|
||||
float fp_th_s[4]; // 352: FP result check threshold, every lane (rt_fpcr_threshold, runtime/simd_fp.h FOKT)
|
||||
double fp_th_d[2]; // 368
|
||||
__uint128_t fp_arg[3]; // 384: operands of a vector FP op handed to its slow path (runtime/simd_fp.h FN(vredo))
|
||||
uint8_t _pad[512 - 432];
|
||||
__uint128_t v[32]; // 512
|
||||
} Cpu;
|
||||
|
||||
@@ -27,6 +31,8 @@ _Static_assert(offsetof(Cpu, pc) == 264, "pc");
|
||||
_Static_assert(offsetof(Cpu, nzcv) == 272, "nzcv");
|
||||
_Static_assert(offsetof(Cpu, fpcr) == 320, "fpcr");
|
||||
_Static_assert(offsetof(Cpu, fpsr) == 328, "fpsr");
|
||||
_Static_assert(offsetof(Cpu, fp_th_s) == 352 && offsetof(Cpu, fp_th_d) == 368, "fp_th");
|
||||
_Static_assert(offsetof(Cpu, fp_arg) == 384, "fp_arg");
|
||||
_Static_assert(offsetof(Cpu, v) == 512, "v");
|
||||
|
||||
typedef void (*GuestFn)(Cpu *);
|
||||
@@ -39,12 +45,24 @@ void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) __attribute__((noreturn));
|
||||
void rt_udf(Cpu *c, uint64_t pc) __attribute__((noreturn));
|
||||
uint64_t rt_ticks(void);
|
||||
|
||||
// The FP fast paths of runtime/simd_fp.h test FPCR.RMode and the result in one compare: a result is good when its
|
||||
// magnitude differs from c->fp_th_*. In round-to-nearest the threshold is the smallest normal, so NaN and +-smallest
|
||||
// normal fail as before; under a directed RMode it is NaN, both compares are unordered and every result goes through
|
||||
// the redo path, which rounds in the FPCR mode. It must follow every change of c->fpcr: rt_fpcr_sync sets it, and so
|
||||
// does code that creates a Cpu or copies fpcr into one. A zeroed threshold is wrong, not merely slow: it passes
|
||||
// +-smallest normal.
|
||||
static inline void rt_fpcr_threshold(Cpu *c) {
|
||||
int directed = (c->fpcr & (3u << 22)) != 0;
|
||||
for (int i = 0; i < 4; i++) c->fp_th_s[i] = directed ? __builtin_nanf("") : 0x1p-126f;
|
||||
for (int i = 0; i < 2; i++) c->fp_th_d[i] = directed ? __builtin_nan("") : 0x1p-1022;
|
||||
}
|
||||
|
||||
#if defined(__aarch64__)
|
||||
static inline uint64_t rt_get_fpcr(void) { uint64_t v; __asm__ volatile("mrs %0, fpcr" : "=r"(v)); return v; }
|
||||
static inline uint64_t rt_get_fpsr(void) { uint64_t v; __asm__ volatile("mrs %0, fpsr" : "=r"(v)); return v; }
|
||||
static inline void rt_set_fpcr(uint64_t v) { __asm__ volatile("msr fpcr, %0" :: "r"(v)); }
|
||||
static inline void rt_set_fpsr(uint64_t v) { __asm__ volatile("msr fpsr, %0" :: "r"(v)); }
|
||||
static inline void rt_fpcr_sync(const Cpu *c) { (void)c; }
|
||||
static inline void rt_fpcr_sync(Cpu *c) { rt_fpcr_threshold(c); }
|
||||
#else
|
||||
// Host FP state is not the guest's here: code lifted with --simd=c keeps FPCR/FPSR in Cpu (c->fpcr, zeroed
|
||||
// with the Cpu), so the host-side reset in thread start-up has nothing to do.
|
||||
@@ -56,7 +74,9 @@ static inline void rt_set_fpsr(uint64_t v) { (void)v; }
|
||||
// the slow paths of runtime/simd.h, which switch RC around one operation). The FP helpers rely on it without checking,
|
||||
// so it is set wherever FPCR or the host FP context changes: MSR FPCR (recomp/lifter.py), rt_call (thread and fiber
|
||||
// entry, host code calling guest code), fiber resume (runtime/kernel.c fiber_resumed; fibers keep their own MXCSR).
|
||||
static inline void rt_fpcr_sync(const Cpu *c) {
|
||||
// It also sets the FP result check threshold (rt_fpcr_threshold).
|
||||
static inline void rt_fpcr_sync(Cpu *c) {
|
||||
rt_fpcr_threshold(c);
|
||||
unsigned cur = __builtin_ia32_stmxcsr();
|
||||
unsigned want = (cur & ~0xE040u) | (c->fpcr & (1u << 24) ? 0x8040u : 0); // FTZ 0x8000, DAZ 0x40, RC 0x6000
|
||||
if (cur != want) __builtin_ia32_ldmxcsr(want);
|
||||
|
||||
@@ -1770,6 +1770,7 @@ int probe_fs(void) {
|
||||
pcpu = plat_aligned_alloc(sizeof(Cpu), 64);
|
||||
need(pcpu != NULL, "probe Cpu");
|
||||
memset(pcpu, 0, sizeof *pcpu);
|
||||
rt_fpcr_threshold(pcpu);
|
||||
uint8_t *stack = mem_kernel_alloc(0x10000, 16);
|
||||
pcpu->sp = (uint64_t)(uintptr_t)stack + 0x10000;
|
||||
pcpu->excl_addr = ~0ull;
|
||||
|
||||
@@ -1445,6 +1445,7 @@ static RtThread *thr_new(int prio, int core, uint64_t mask) {
|
||||
Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64);
|
||||
if (!t || !c) rt_fatal("kernel: thread allocation");
|
||||
memset(c, 0, sizeof *c);
|
||||
rt_fpcr_threshold(c);
|
||||
plat_cond_init(&t->cv);
|
||||
t->prio = prio;
|
||||
t->core = core;
|
||||
@@ -2259,6 +2260,7 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) {
|
||||
dst->tpidr = from->tpidr;
|
||||
dst->tpidrro = from->tpidrro;
|
||||
dst->fpcr = from->fpcr;
|
||||
rt_fpcr_threshold(dst); // fiber_resumed syncs MXCSR on the other side; the threshold follows fpcr at once
|
||||
dst->fpsr = from->fpsr;
|
||||
dst->excl_addr = ~0ull;
|
||||
dst->thread = t;
|
||||
@@ -2291,6 +2293,7 @@ void kern_fiber_initialize(Cpu *c) {
|
||||
plat_fiber_free(f->host);
|
||||
}
|
||||
memset(f->cpu, 0, sizeof *f->cpu);
|
||||
rt_fpcr_threshold(f->cpu);
|
||||
f->cpu->excl_addr = ~0ull;
|
||||
f->addr = addr;
|
||||
f->func = c->x[1];
|
||||
|
||||
@@ -643,6 +643,7 @@ static void check_got(void) {
|
||||
hostfn_seal();
|
||||
Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64);
|
||||
memset(c, 0, sizeof *c);
|
||||
rt_fpcr_threshold(c);
|
||||
uint8_t *stack = mem_kernel_alloc(0x10000, 16);
|
||||
c->sp = (uint64_t)(uintptr_t)stack + 0x10000;
|
||||
c->excl_addr = ~0ull;
|
||||
|
||||
@@ -568,6 +568,7 @@ static int selftest(void) {
|
||||
CHECK(s.functions > 0, "no functions linked");
|
||||
Cpu *c = plat_aligned_alloc(sizeof(Cpu), 64);
|
||||
memset(c, 0, sizeof *c);
|
||||
rt_fpcr_threshold(c);
|
||||
uint64_t r = rt_call(c, probe, 1, (uint64_t)7);
|
||||
CHECK(rt_dispatch_lookup(probe) && r == 42 && host_probe_hits == 1 && c->x[30] == 0,
|
||||
"a registered host function does not dispatch through rt_call");
|
||||
|
||||
@@ -435,6 +435,7 @@ int probe_movie(void) {
|
||||
C = plat_aligned_alloc(sizeof(Cpu), 64);
|
||||
CHECK(C, "Cpu allocation");
|
||||
memset(C, 0, sizeof *C);
|
||||
rt_fpcr_threshold(C);
|
||||
|
||||
uint64_t obj_mem[2] = {0}, obj = U(obj_mem);
|
||||
volatile int32_t ev[4] = {0};
|
||||
|
||||
@@ -919,6 +919,7 @@ int probe_nvn(void) {
|
||||
if (child && *child) return run_as_child(child);
|
||||
// The probe checks the null back end whatever TOTK_GPU says; the choice is made once, at the first object call.
|
||||
((RtConfig *)rt_config())->gpu = "null";
|
||||
rt_fpcr_threshold(&cpu); // the FP result check threshold of a fresh Cpu (cpu.h): a zeroed one is wrong, not slow
|
||||
nvn_procs_register();
|
||||
hostfn_seal();
|
||||
kern_run_on_guest_main(probe_main);
|
||||
|
||||
@@ -17,7 +17,17 @@
|
||||
|
||||
typedef __uint128_t u128;
|
||||
#define SI static inline __attribute__((always_inline))
|
||||
// The noinline FP helpers are the slow paths of inline fast paths, called from cold blocks inside the lifted
|
||||
// functions. preserve_most makes the helper save the caller-saved GPRs it clobbers, so a call site does not spill and
|
||||
// reload its live values around the call. The helpers are static, so the convention never crosses a module boundary.
|
||||
#if defined(__has_attribute)
|
||||
#if __has_attribute(preserve_most)
|
||||
#define SN static __attribute__((noinline, preserve_most))
|
||||
#endif
|
||||
#endif
|
||||
#ifndef SN
|
||||
#define SN static __attribute__((noinline))
|
||||
#endif
|
||||
#define SIMD_CAT2(a, b) a##_##b
|
||||
#define SIMD_CAT(a, b) SIMD_CAT2(a, b)
|
||||
|
||||
@@ -274,6 +284,11 @@ SI void simd_rm_leave(unsigned old) {
|
||||
// plain host FP gives the ARM result unless FPCR.RMode asks for directed rounding, which the slow paths handle.
|
||||
SI int simd_directed(const Cpu *c) { return __builtin_expect((c->fpcr & FPCR_RM) != 0, 0); }
|
||||
|
||||
// FP result check threshold of the thread (cpu.h rt_fpcr_threshold), for one lane and for a whole vector.
|
||||
SI float simd_th_s(const Cpu *c) { return c->fp_th_s[0]; }
|
||||
SI double simd_th_d(const Cpu *c) { return c->fp_th_d[0]; }
|
||||
SI vf32 simd_thv_s(const Cpu *c) { return *(const vf32 *)c->fp_th_s; }
|
||||
SI vf64 simd_thv_d(const Cpu *c) { return *(const vf64 *)c->fp_th_d; }
|
||||
#define F s
|
||||
#define T float
|
||||
#define U uint32_t
|
||||
|
||||
+44
-9
@@ -204,21 +204,34 @@ SN T FN(slow)(Cpu *c, int op, T a, T b, T d) {
|
||||
// The complement of FREDO for every op but RSQRTS, as one compare: |r| ordered and not the smallest normal (LLVM folds
|
||||
// it to fcmp one, a single ucomiss + jne for scalars and one vcmpps for vectors).
|
||||
#define FOK(r) ((__builtin_elementwise_abs(r) < FN(fv)(MINN)) | (__builtin_elementwise_abs(r) > FN(fv)(MINN)))
|
||||
// FOK against the thread's threshold (cpu.h rt_fpcr_threshold), the check of the fast paths: the same as FOK in
|
||||
// round-to-nearest, where the threshold is the smallest normal, and false for every result under a directed RMode,
|
||||
// where the threshold is NaN, so that RMode needs no test of its own. The threshold is a memory operand of the compare.
|
||||
#define FOKT(r, th) ((__builtin_elementwise_abs(r) < (th)) | (__builtin_elementwise_abs(r) > (th)))
|
||||
// A result FOKT rejected. A NaN result in round-to-nearest goes straight to fix: the host already ran the operation
|
||||
// on the operands FPCR.FZ flushes (MXCSR.DAZ), so slow would compute the same NaN and return fix of the flushed
|
||||
// operands, and fix's result for these ops is never tiny, so slow's FZ result flush would leave it alone. RSQRTS
|
||||
// keeps the slow path because its fix result can be tiny.
|
||||
SN T FN(redo)(Cpu *c, int op, T a, T b, T d, T r) {
|
||||
if (c->fpcr & (FPCR_FZ | FPCR_RM)) return FN(slow)(c, op, a, b, d);
|
||||
uint64_t fpcr = c->fpcr;
|
||||
if (op != FOP_RSQRTS && !(fpcr & FPCR_RM) && r != r) {
|
||||
if (fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b), d = FN(flush)(d);
|
||||
return FN(fix)(c, op, a, b, d);
|
||||
}
|
||||
if (fpcr & (FPCR_FZ | FPCR_RM)) return FN(slow)(c, op, a, b, d);
|
||||
return FBAD(op, r) ? FN(fix)(c, op, a, b, d) : r;
|
||||
}
|
||||
|
||||
// Scalar FP op (constant op folds the selection). RMode = nearest runs on the host, with or without FZ. A directed
|
||||
// RMode is rare, so it shares the result check's branch: the host result, computed in round-to-nearest, is then
|
||||
// thrown away and redo recomputes in the FPCR mode. Min and max do not round, so RMode does not matter to them.
|
||||
// RMode is rare, so it shares the result check (the FOKT threshold): the host result, computed in round-to-nearest,
|
||||
// is then thrown away and redo recomputes in the FPCR mode. Min and max do not round, so RMode does not matter to them.
|
||||
SI T FN(op)(Cpu *c, int op, T a, T b, T d) {
|
||||
if (FMINMAX(op)) {
|
||||
if (c->fpcr & FPCR_FZ) a = FN(flush)(a), b = FN(flush)(b);
|
||||
return FN(minmax)(c, op, a, b);
|
||||
}
|
||||
T r = FCALC(op, a, b, d, K, FMA_S);
|
||||
if (__builtin_expect((op == FOP_RSQRTS ? FREDO(op, r) : !FOK(r)) | simd_directed(c), 0))
|
||||
if (__builtin_expect(op == FOP_RSQRTS ? FREDO(op, r) | simd_directed(c) : !FOKT(r, SIMD_CAT(simd_th, F)(c)), 0))
|
||||
r = FN(redo)(c, op, a, b, d, r);
|
||||
return r;
|
||||
}
|
||||
@@ -228,6 +241,24 @@ SN VT FN(vslow)(Cpu *c, int op, VT a, VT b, VT d) {
|
||||
for (int i = 0; i < NL; i++) r[i] = FN(op)(c, op, a[i], b[i], d[i]);
|
||||
return r;
|
||||
}
|
||||
// Slow path of FN(vop) for a vector with flagged lanes. The operands come in c->fp_arg, stored by the caller just
|
||||
// before the call: passing three vectors by value costs every call site a stack copy of each (Win64 passes them by
|
||||
// reference), and the frames of the large lifted functions grow past a page (a ___chkstk_ms call at every entry). The
|
||||
// scratch is per thread and nothing runs between the store and the loads here. In round-to-nearest a lane that passed
|
||||
// the check equals what the scalar op returns for it (same host operation, same MXCSR), so only the flagged lanes go
|
||||
// through FN(op); the vector result is recomputed here rather than passed in, which keeps the call sites small. A
|
||||
// directed RMode reruns every lane.
|
||||
SN VT FN(vredo)(Cpu *c, int op) {
|
||||
VT a, b, d;
|
||||
memcpy(&a, &c->fp_arg[0], sizeof a);
|
||||
memcpy(&b, &c->fp_arg[1], sizeof b);
|
||||
memcpy(&d, &c->fp_arg[2], sizeof d);
|
||||
if (c->fpcr & FPCR_RM) return FN(vslow)(c, op, a, b, d);
|
||||
VT r = FCALC(op, a, b, d, VK, FMA_V);
|
||||
for (int i = 0; i < NL; i++)
|
||||
if (op == FOP_RSQRTS ? FREDO(op, r[i]) : !FOK(r[i])) r[i] = FN(op)(c, op, a[i], b[i], d[i]);
|
||||
return r;
|
||||
}
|
||||
|
||||
SI VTU FN(vsel)(VTS m, VTU x, VTU y) { return (x & (VTU)m) | (y & ~(VTU)m); }
|
||||
|
||||
@@ -239,8 +270,8 @@ SI VT FN(vflush)(VT a) {
|
||||
return (VT)FN(vsel)(den, u & SGN, u);
|
||||
}
|
||||
|
||||
// Vector FP op: native lanes; any lane FREDO flags (or a directed FPCR.RMode, checked in the same branch) reruns all
|
||||
// lanes through the scalar path. Min and max ignore RMode, as in FN(op).
|
||||
// Vector FP op: native lanes; a lane FREDO flags, or a directed FPCR.RMode (the FOKT threshold, or the RSQRTS test),
|
||||
// takes FN(vredo). Min and max ignore RMode, as in FN(op).
|
||||
SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) {
|
||||
VT r;
|
||||
if (FMINMAX(op)) {
|
||||
@@ -254,9 +285,13 @@ SI VT FN(vop)(Cpu *c, int op, VT a, VT b, VT d) {
|
||||
return r;
|
||||
}
|
||||
r = FCALC(op, a, b, d, VK, FMA_V);
|
||||
if (__builtin_expect((op == FOP_RSQRTS ? __builtin_reduce_or(FREDO(op, r)) != 0 : __builtin_reduce_and(FOK(r)) == 0)
|
||||
| simd_directed(c), 0))
|
||||
return FN(vslow)(c, op, a, b, d);
|
||||
if (__builtin_expect(op == FOP_RSQRTS ? (__builtin_reduce_or(FREDO(op, r)) != 0) | simd_directed(c)
|
||||
: __builtin_reduce_and(FOKT(r, SIMD_CAT(simd_thv, F)(c))) == 0, 0)) {
|
||||
memcpy(&c->fp_arg[0], &a, sizeof a);
|
||||
memcpy(&c->fp_arg[1], &b, sizeof b);
|
||||
memcpy(&c->fp_arg[2], &d, sizeof d);
|
||||
return FN(vredo)(c, op);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user