mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-11 09:51:02 +00:00
Valiantsin DzerakhandClaude Opus 5.5
2fe6a6d4c1
The FP helpers compared c->fpcr with a cached copy before every operation to resync MXCSR lazily. MXCSR now follows FPCR.FZ at once: MSR FPCR calls rt_fpcr_sync, and so do rt_call (thread, fiber and host entry into guest code), fiber resume (FIBER_FLAG_FLOAT_SWITCH fibers keep their own MXCSR) and the differential test harness per case. op and vop only test FPCR.RMode for the slow path, and their redo check is one ordered-not-equal compare of |r| with the smallest normal, the same NaN-or-smallest-normal set as before. Hot guest code shrinks by about a quarter (132 functions holding half the guest cycles, 1.4.3); fight frame time moves 1-3%, inside the run-to-run spread. lift, simd, func, atomic and flags_check pass on 1.4.3; simd.py fails without the harness sync. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
842 lines
42 KiB
C
842 lines
42 KiB
C
// Adapted from LAS Recomp runtime/simd.h @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
|
|
// Portable C lowering of AArch64 FP / AdvSIMD / crypto / CRC32 (tools/simd_c.py emits calls with constant
|
|
// fields; everything is always_inline so the size/lane/op switches fold at -O1).
|
|
// Semantics follow the ARM pseudocode, not the host: NaN propagation (first signalling NaN quietened, then
|
|
// first quiet NaN), FPCR.DN default NaN, FPCR.FZ flushing (inputs and pre-rounding-tiny results), FPCR.RMode
|
|
// for arithmetic / conversions / FRINTX/I, saturating FP->int conversions, fused multiply-add, ARM estimate
|
|
// tables. Not modelled: FPSR cumulative exception bits and QC, FPCR trap enables, FP16 arithmetic (ARMv8.2).
|
|
// FPCR.AHP is honoured by the half-precision conversions. Host assumptions: IEEE binary32/64, MXCSR in
|
|
// round-to-nearest outside the slow paths, with FTZ|DAZ equal to FPCR.FZ (cpu.h rt_fpcr_sync keeps it so); fma must be
|
|
// correctly rounded (build x86 with -mfma).
|
|
#pragma once
|
|
#include <fenv.h>
|
|
#include <immintrin.h>
|
|
#include <math.h>
|
|
|
|
#include "cpu.h"
|
|
|
|
typedef __uint128_t u128;
|
|
#define SI static inline __attribute__((always_inline))
|
|
#define SN static __attribute__((noinline))
|
|
#define SIMD_CAT2(a, b) a##_##b
|
|
#define SIMD_CAT(a, b) SIMD_CAT2(a, b)
|
|
|
|
#define SIMD_VT(name, T) typedef T name __attribute__((vector_size(16), may_alias));
|
|
SIMD_VT(vu8, uint8_t) SIMD_VT(vs8, int8_t) SIMD_VT(vu16, uint16_t) SIMD_VT(vs16, int16_t)
|
|
SIMD_VT(vu32, uint32_t) SIMD_VT(vs32, int32_t) SIMD_VT(vu64, uint64_t) SIMD_VT(vs64, int64_t)
|
|
SIMD_VT(vf32, float) SIMD_VT(vf64, double)
|
|
typedef uint16_t __attribute__((may_alias)) sa16;
|
|
typedef uint32_t __attribute__((may_alias)) sa32;
|
|
typedef uint64_t __attribute__((may_alias)) sa64;
|
|
typedef float __attribute__((may_alias)) saf32;
|
|
typedef double __attribute__((may_alias)) saf64;
|
|
typedef u128 __attribute__((aligned(1), may_alias)) u128u;
|
|
|
|
// ---- registers: VREG = whole V register as a vector type; VQ = source with the upper half cleared when
|
|
// Q=0; VW = write, zeroing the upper half when Q=0. Scalar views of lane 0: VS8..VS64, VSF32/VSF64.
|
|
#define VREG(T, n) (*(T *)&c->v[n])
|
|
#define VQ(T, Q, n) ((T)((Q) ? c->v[n] : (u128)(uint64_t)c->v[n]))
|
|
#define VW(T, Q, d, e) do { T _w = (e); c->v[d] = (Q) ? (u128)_w : (u128)(uint64_t)(u128)_w; } while (0)
|
|
#define VS8(n) (*(uint8_t *)&c->v[n])
|
|
#define VS16(n) (*(sa16 *)&c->v[n])
|
|
#define VS32(n) (*(sa32 *)&c->v[n])
|
|
#define VS64(n) (*(sa64 *)&c->v[n])
|
|
#define VSF32(n) (*(saf32 *)&c->v[n])
|
|
#define VSF64(n) (*(saf64 *)&c->v[n])
|
|
#define VHI64(n) (((sa64 *)&c->v[n])[1])
|
|
#define LD128(a) (*(const u128u *)(uintptr_t)(a))
|
|
#define ST128(a, v) (*(u128u *)(uintptr_t)(a) = (u128)(v))
|
|
#define SHUF __builtin_shufflevector
|
|
#define CVT __builtin_convertvector
|
|
|
|
#define FPCR_AHP (1u << 26)
|
|
#define FPCR_DN (1u << 25)
|
|
#define FPCR_FZ (1u << 24)
|
|
#define FPCR_RM (3u << 22)
|
|
|
|
enum { FOP_ADD, FOP_SUB, FOP_MUL, FOP_MULX, FOP_NMUL, FOP_DIV, FOP_ABD, FOP_SQRT, FOP_MADD, FOP_RECPS, FOP_RSQRTS,
|
|
FOP_MAX, FOP_MIN, FOP_MAXNM, FOP_MINNM };
|
|
|
|
static const uint8_t simd_recip_tab[256] = {
|
|
255, 253, 251, 249, 247, 245, 243, 241, 240, 238, 236, 234, 232, 230, 229, 227,
|
|
225, 223, 221, 220, 218, 216, 215, 213, 211, 210, 208, 206, 205, 203, 201, 200,
|
|
198, 197, 195, 194, 192, 191, 189, 188, 186, 185, 183, 182, 180, 179, 177, 176,
|
|
174, 173, 172, 170, 169, 167, 166, 165, 163, 162, 161, 159, 158, 157, 156, 154,
|
|
153, 152, 150, 149, 148, 147, 145, 144, 143, 142, 141, 139, 138, 137, 136, 135,
|
|
134, 132, 131, 130, 129, 128, 127, 126, 124, 123, 122, 121, 120, 119, 118, 117,
|
|
116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101,
|
|
100, 99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 88, 87, 86,
|
|
85, 84, 83, 82, 81, 81, 80, 79, 78, 77, 76, 75, 75, 74, 73, 72,
|
|
71, 70, 70, 69, 68, 67, 66, 66, 65, 64, 63, 63, 62, 61, 60, 59,
|
|
59, 58, 57, 56, 56, 55, 54, 53, 53, 52, 51, 51, 50, 49, 48, 48,
|
|
47, 46, 46, 45, 44, 44, 43, 42, 42, 41, 40, 40, 39, 38, 38, 37,
|
|
36, 36, 35, 34, 34, 33, 32, 32, 31, 30, 30, 29, 29, 28, 27, 27,
|
|
26, 26, 25, 24, 24, 23, 23, 22, 21, 21, 20, 20, 19, 18, 18, 17,
|
|
17, 16, 16, 15, 15, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9,
|
|
8, 7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0,
|
|
};
|
|
static const uint8_t simd_rsqrt_tab[384] = {
|
|
255, 253, 251, 249, 247, 245, 243, 242, 240, 238, 236, 234, 233, 231, 229, 228,
|
|
226, 224, 223, 221, 219, 218, 216, 215, 213, 212, 210, 209, 207, 206, 204, 203,
|
|
201, 200, 198, 197, 196, 194, 193, 192, 190, 189, 188, 186, 185, 184, 183, 181,
|
|
180, 179, 178, 176, 175, 174, 173, 172, 170, 169, 168, 167, 166, 165, 164, 163,
|
|
162, 160, 159, 158, 157, 156, 155, 154, 153, 152, 151, 150, 149, 148, 147, 146,
|
|
145, 144, 143, 142, 141, 140, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131,
|
|
131, 130, 129, 128, 127, 126, 126, 125, 124, 123, 122, 121, 121, 120, 119, 118,
|
|
118, 117, 116, 115, 114, 114, 113, 112, 111, 111, 110, 109, 109, 108, 107, 106,
|
|
105, 105, 104, 104, 103, 103, 101, 101, 100, 100, 99, 99, 97, 97, 96, 96,
|
|
95, 95, 93, 93, 92, 92, 91, 91, 90, 90, 88, 88, 87, 87, 86, 86,
|
|
85, 85, 84, 84, 82, 82, 81, 81, 80, 80, 79, 79, 78, 78, 77, 77,
|
|
76, 76, 75, 75, 74, 74, 72, 72, 71, 71, 70, 70, 69, 69, 68, 68,
|
|
67, 67, 66, 66, 65, 65, 64, 64, 63, 63, 62, 62, 61, 61, 60, 60,
|
|
60, 60, 59, 59, 58, 58, 57, 57, 56, 56, 55, 55, 54, 54, 53, 53,
|
|
52, 52, 51, 51, 51, 51, 50, 50, 49, 49, 48, 48, 47, 47, 46, 46,
|
|
46, 46, 45, 45, 44, 44, 43, 43, 42, 42, 42, 42, 41, 41, 40, 40,
|
|
39, 39, 38, 38, 38, 38, 37, 37, 36, 36, 35, 35, 35, 35, 34, 34,
|
|
33, 33, 33, 33, 32, 32, 31, 31, 30, 30, 30, 30, 29, 29, 28, 28,
|
|
28, 28, 27, 27, 26, 26, 26, 26, 25, 25, 24, 24, 24, 24, 23, 23,
|
|
22, 22, 22, 22, 21, 21, 20, 20, 20, 20, 19, 19, 19, 19, 18, 18,
|
|
17, 17, 17, 17, 16, 16, 16, 16, 15, 15, 14, 14, 14, 14, 13, 13,
|
|
13, 13, 12, 12, 11, 11, 11, 11, 10, 10, 10, 10, 9, 9, 9, 9,
|
|
8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4,
|
|
4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0,
|
|
};
|
|
static const uint32_t simd_crc32_tab[256] = {
|
|
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
|
|
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
|
|
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
|
|
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
|
|
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
|
|
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
|
|
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
|
|
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
|
|
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
|
|
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
|
|
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
|
|
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
|
|
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
|
|
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
|
|
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
|
|
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
|
|
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
|
|
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
|
|
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
|
|
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
|
|
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
|
|
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
|
|
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
|
|
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
|
|
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
|
|
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
|
|
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
|
|
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
|
|
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
|
|
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
|
|
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
|
|
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
|
|
};
|
|
static const uint32_t simd_crc32c_tab[256] = {
|
|
0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb,
|
|
0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24,
|
|
0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384,
|
|
0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b,
|
|
0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35,
|
|
0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa,
|
|
0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a,
|
|
0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595,
|
|
0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,
|
|
0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198,
|
|
0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38,
|
|
0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7,
|
|
0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789,
|
|
0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46,
|
|
0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6,
|
|
0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829,
|
|
0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93,
|
|
0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c,
|
|
0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc,
|
|
0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033,
|
|
0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d,
|
|
0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982,
|
|
0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622,
|
|
0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed,
|
|
0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f,
|
|
0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0,
|
|
0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540,
|
|
0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f,
|
|
0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1,
|
|
0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e,
|
|
0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e,
|
|
0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,
|
|
};
|
|
static const uint8_t simd_aes_sbox[256] = {
|
|
0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
|
|
0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
|
|
0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
|
|
0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
|
|
0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
|
|
0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
|
|
0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
|
|
0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
|
|
0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
|
|
0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
|
|
0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
|
|
0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
|
|
0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
|
|
0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
|
|
0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
|
|
0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
|
|
};
|
|
static const uint8_t simd_aes_isbox[256] = {
|
|
0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
|
|
0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
|
|
0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
|
|
0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
|
|
0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
|
|
0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
|
|
0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
|
|
0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
|
|
0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
|
|
0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
|
|
0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
|
|
0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
|
|
0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
|
|
0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
|
|
0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
|
|
0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d,
|
|
};
|
|
|
|
SI int simd_fe_mode(uint64_t fpcr) {
|
|
static const int m[4] = {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO};
|
|
return m[(fpcr >> 22) & 3];
|
|
}
|
|
|
|
// MXCSR rounding control (bits 13-14) without fesetround's x87 + SSE double write; ldmxcsr only on change.
|
|
#define SIMD_RC_MASK 0x6000u
|
|
SI unsigned simd_rm_set_rc(unsigned cur, unsigned rc) {
|
|
if ((cur & SIMD_RC_MASK) != rc) _mm_setcsr((cur & ~SIMD_RC_MASK) | rc);
|
|
return cur;
|
|
}
|
|
// Set MXCSR RC from FPCR.RMode (RN 00, RP 10, RM 01, RZ 11); returns the previous MXCSR.
|
|
SI unsigned simd_rm_enter(uint64_t fpcr) {
|
|
static const unsigned rc[4] = {0x0000u, 0x4000u, 0x2000u, 0x6000u};
|
|
return simd_rm_set_rc(_mm_getcsr(), rc[(fpcr >> 22) & 3]);
|
|
}
|
|
// Force toward-zero.
|
|
SI void simd_rm_rz(void) { simd_rm_set_rc(_mm_getcsr(), 0x6000u); }
|
|
// Restore the MXCSR saved by simd_rm_enter, only if it changed.
|
|
SI void simd_rm_leave(unsigned old) {
|
|
unsigned cur = _mm_getcsr();
|
|
if ((cur & SIMD_RC_MASK) != (old & SIMD_RC_MASK)) _mm_setcsr((cur & ~SIMD_RC_MASK) | (old & SIMD_RC_MASK));
|
|
}
|
|
|
|
// The thread's MXCSR mirrors the guest FPCR.FZ with RC = nearest whenever guest code runs (cpu.h rt_fpcr_sync), so
|
|
// plain host FP gives the ARM result unless FPCR.RMode asks for directed rounding, which the slow paths handle.
|
|
SI int simd_directed(const Cpu *c) { return __builtin_expect((c->fpcr & FPCR_RM) != 0, 0); }
|
|
|
|
#define F s
|
|
#define T float
|
|
#define U uint32_t
|
|
#define NB 32
|
|
#define EB 8
|
|
#define MB 23
|
|
#define VT vf32
|
|
#define VTU vu32
|
|
#define VTS vs32
|
|
#define NL 4
|
|
#include "simd_fp.h"
|
|
#undef F
|
|
#undef T
|
|
#undef U
|
|
#undef NB
|
|
#undef EB
|
|
#undef MB
|
|
#undef VT
|
|
#undef VTU
|
|
#undef VTS
|
|
#undef NL
|
|
#define F d
|
|
#define T double
|
|
#define U uint64_t
|
|
#define NB 64
|
|
#define EB 11
|
|
#define MB 52
|
|
#define VT vf64
|
|
#define VTU vu64
|
|
#define VTS vs64
|
|
#define NL 2
|
|
#include "simd_fp.h"
|
|
#undef F
|
|
#undef T
|
|
#undef U
|
|
#undef NB
|
|
#undef EB
|
|
#undef MB
|
|
#undef VT
|
|
#undef VTU
|
|
#undef VTS
|
|
#undef NL
|
|
|
|
// ---- precision conversions (FCVT, FCVTL/N, FCVTXN); FZ flushes single/double inputs, never halves
|
|
SI double simd_s2d(Cpu *c, float x) {
|
|
uint32_t u = fb_s(fz_s(c, x));
|
|
if (isnan_s(u)) {
|
|
if (c->fpcr & FPCR_DN) return fv_d(0x7ff8000000000000ull);
|
|
return fv_d(((uint64_t)(u >> 31) << 63) | 0x7ff8000000000000ull | ((uint64_t)(u & 0x3fffff) << 29));
|
|
}
|
|
return (double)fv_s(u);
|
|
}
|
|
SN float simd_d2s_slow(Cpu *c, double x) {
|
|
#pragma STDC FENV_ACCESS ON
|
|
unsigned old = simd_rm_enter(c->fpcr);
|
|
volatile double vx = x;
|
|
volatile float r = (float)vx;
|
|
simd_rm_leave(old);
|
|
return r;
|
|
}
|
|
SI float simd_d2s(Cpu *c, double x) {
|
|
uint64_t u = fb_d(fz_d(c, x));
|
|
if (isnan_d(u)) {
|
|
if (c->fpcr & FPCR_DN) return fv_s(0x7fc00000u);
|
|
return fv_s((uint32_t)(u >> 63) << 31 | 0x7fc00000u | (uint32_t)((u >> 29) & 0x3fffff));
|
|
}
|
|
x = fv_d(u);
|
|
// FZ: tiny before rounding (host FTZ would keep a result rounded up to the smallest normal)
|
|
if ((c->fpcr & FPCR_FZ) && __builtin_fabs(x) < 0x1p-126 && x != 0) return fv_s((uint32_t)(u >> 63) << 31);
|
|
if (__builtin_expect(c->fpcr & FPCR_RM, 0)) return simd_d2s_slow(c, x);
|
|
return (float)x;
|
|
}
|
|
// round to odd (FCVTXN)
|
|
SI float simd_d2s_odd(Cpu *c, double x) {
|
|
uint64_t u = fb_d(fz_d(c, x));
|
|
if (isnan_d(u)) return simd_d2s(c, x);
|
|
x = fv_d(u);
|
|
if ((c->fpcr & FPCR_FZ) && __builtin_fabs(x) < 0x1p-126 && x != 0) return fv_s((uint32_t)(u >> 63) << 31);
|
|
float f = (float)x;
|
|
if ((double)f != x && x == x) {
|
|
uint32_t b = fb_s(f);
|
|
if (__builtin_fabs((double)f) > __builtin_fabs(x)) b--;
|
|
return fv_s(b | 1);
|
|
}
|
|
return f;
|
|
}
|
|
SI uint64_t simd_rnd_shift(uint64_t m, int sh, int rm, int neg) {
|
|
if (sh >= 64) return (rm == 1 && !neg && m) || (rm == 2 && neg && m);
|
|
uint64_t q = m >> sh, rem = m & ((1ull << sh) - 1), half = 1ull << (sh - 1);
|
|
if (rm == 0) q += rem > half || (rem == half && (q & 1));
|
|
else if (rm == 1) q += rem && !neg;
|
|
else if (rm == 2) q += rem && neg;
|
|
return q;
|
|
}
|
|
// double (exact image of the single/double source, NaN payload left-aligned) -> half, FPCR rounding
|
|
SI uint16_t simd_d2h(Cpu *c, uint64_t u) {
|
|
uint64_t fpcr = c->fpcr;
|
|
int ahp = !!(fpcr & FPCR_AHP), rm = (fpcr >> 22) & 3, neg = (int)(u >> 63);
|
|
uint16_t s = (uint16_t)(neg << 15);
|
|
int e = (int)((u >> 52) & 0x7ff);
|
|
uint64_t f = u & ((1ull << 52) - 1);
|
|
if (e == 0x7ff) {
|
|
if (f) return ahp ? s : (fpcr & FPCR_DN) ? 0x7e00 : (uint16_t)(s | 0x7e00 | (f >> 42));
|
|
return ahp ? (uint16_t)(s | 0x7fff) : (uint16_t)(s | 0x7c00);
|
|
}
|
|
if (!e && !f) return s;
|
|
uint64_t m = e ? f | (1ull << 52) : f;
|
|
int E = e ? e - 1075 : -1074, ue = e ? e - 1023 : -1023;
|
|
uint64_t q;
|
|
int bits;
|
|
if (ue < -14) {
|
|
q = simd_rnd_shift(m, -24 - E, rm, neg);
|
|
bits = (int)q;
|
|
} else {
|
|
q = simd_rnd_shift(m, 42, rm, neg);
|
|
bits = ((ue + 14) << 10) + (int)q;
|
|
}
|
|
if (ahp) return (uint16_t)(s | (bits > 0x7fff ? 0x7fff : bits));
|
|
if (bits >= 0x7c00) {
|
|
int inf = rm == 0 || (rm == 1 && !neg) || (rm == 2 && neg);
|
|
return (uint16_t)(s | (inf ? 0x7c00 : 0x7bff));
|
|
}
|
|
return (uint16_t)(s | bits);
|
|
}
|
|
SI uint16_t simd_s2h(Cpu *c, float x) {
|
|
uint32_t u = fb_s(fz_s(c, x));
|
|
uint64_t d = isnan_s(u) ? ((uint64_t)(u >> 31) << 63) | 0x7ff0000000000000ull | ((uint64_t)(u & 0x7fffff) << 29)
|
|
: fb_d((double)fv_s(u));
|
|
return simd_d2h(c, d);
|
|
}
|
|
SI uint16_t simd_dd2h(Cpu *c, double x) { return simd_d2h(c, fb_d(fz_d(c, x))); }
|
|
SI uint64_t simd_h2dbits(Cpu *c, uint16_t h) {
|
|
uint64_t s = (uint64_t)(h >> 15) << 63;
|
|
int e = (h >> 10) & 31, f = h & 0x3ff;
|
|
if (e == 31 && !(c->fpcr & FPCR_AHP)) {
|
|
if (!f) return s | 0x7ff0000000000000ull;
|
|
if (c->fpcr & FPCR_DN) return 0x7ff8000000000000ull;
|
|
return s | 0x7ff8000000000000ull | ((uint64_t)f << 42);
|
|
}
|
|
if (!e) return fb_d((s ? -1.0 : 1.0) * f * 0x1p-24);
|
|
return s | ((uint64_t)(e - 15 + 1023) << 52) | ((uint64_t)f << 42);
|
|
}
|
|
SI double simd_h2d(Cpu *c, uint16_t h) { return fv_d(simd_h2dbits(c, h)); }
|
|
SI float simd_h2s(Cpu *c, uint16_t h) {
|
|
uint64_t d = simd_h2dbits(c, h);
|
|
if (isnan_d(d)) return fv_s((uint32_t)(d >> 63) << 31 | 0x7fc00000u | (uint32_t)((d >> 29) & 0x3fffff));
|
|
return (float)fv_d(d);
|
|
}
|
|
|
|
// ---- scalar FP instructions (FP data processing)
|
|
#define FP_SETS(d, x) (c->v[d] = fb_s(x))
|
|
#define FP_SETD(d, x) (c->v[d] = fb_d(x))
|
|
SI void fp_op_s(Cpu *c, int op, int d, int n, int m) { FP_SETS(d, op_s(c, op, VSF32(n), VSF32(m), 0)); }
|
|
SI void fp_op_d(Cpu *c, int op, int d, int n, int m) { FP_SETD(d, op_d(c, op, VSF64(n), VSF64(m), 0)); }
|
|
// FMADD family: neg_a / neg_n negate the addend / first factor before the fused op
|
|
SI void fp_fma_s(Cpu *c, int na, int nn, int d, int n, int m, int a) {
|
|
float x = VSF32(a), y = VSF32(n);
|
|
if (na) x = neg_s(x);
|
|
if (nn) y = neg_s(y);
|
|
FP_SETS(d, op_s(c, FOP_MADD, x, y, VSF32(m)));
|
|
}
|
|
SI void fp_fma_d(Cpu *c, int na, int nn, int d, int n, int m, int a) {
|
|
double x = VSF64(a), y = VSF64(n);
|
|
if (na) x = neg_d(x);
|
|
if (nn) y = neg_d(y);
|
|
FP_SETD(d, op_d(c, FOP_MADD, x, y, VSF64(m)));
|
|
}
|
|
|
|
// ---- integer lanes (complex ops run lane by lane; constant sizes fold to plain loads/stores)
|
|
SI uint64_t lmask(int sz) { return sz == 3 ? ~0ull : (1ull << (8 << sz)) - 1; }
|
|
SI int64_t lsx(uint64_t v, int sz) { int s = 64 - (8 << sz); return (int64_t)(v << s) >> s; }
|
|
SI uint64_t lget(const u128 *v, int sz, int i) {
|
|
const uint8_t *p = (const uint8_t *)v + (i << sz);
|
|
switch (sz) {
|
|
case 0: return *p;
|
|
case 1: return *(const sa16 *)p;
|
|
case 2: return *(const sa32 *)p;
|
|
default: return *(const sa64 *)p;
|
|
}
|
|
}
|
|
SI void lset(u128 *v, int sz, int i, uint64_t x) {
|
|
uint8_t *p = (uint8_t *)v + (i << sz);
|
|
switch (sz) {
|
|
case 0: *p = (uint8_t)x; break;
|
|
case 1: *(sa16 *)p = (uint16_t)x; break;
|
|
case 2: *(sa32 *)p = (uint32_t)x; break;
|
|
default: *(sa64 *)p = x; break;
|
|
}
|
|
}
|
|
SI uint64_t sat_s(__int128 v, int sz) {
|
|
__int128 hi = ((__int128)1 << ((8 << sz) - 1)) - 1, lo = -hi - 1;
|
|
return (uint64_t)(v > hi ? hi : v < lo ? lo : v) & lmask(sz);
|
|
}
|
|
SI uint64_t sat_u(__int128 v, int sz) {
|
|
__int128 hi = ((__int128)1 << (8 << sz)) - 1;
|
|
return (uint64_t)(v > hi ? hi : v < 0 ? 0 : v);
|
|
}
|
|
SI uint64_t sat_uu(unsigned __int128 v, int sz) { return v > lmask(sz) ? lmask(sz) : (uint64_t)v; }
|
|
SI __int128 shl128(__int128 v, int s) { return (__int128)((unsigned __int128)v << s); }
|
|
// right shift by s >= 1 with optional rounding, as an infinite-precision value
|
|
SI __int128 lshr_s(int64_t v, int s, int rnd) {
|
|
if (s > 100) s = 100;
|
|
return rnd ? ((__int128)v + ((__int128)1 << (s - 1))) >> s : (__int128)v >> s;
|
|
}
|
|
SI unsigned __int128 lshr_u(uint64_t v, int s, int rnd) {
|
|
if (s > 100) s = 100;
|
|
return rnd ? ((unsigned __int128)v + ((unsigned __int128)1 << (s - 1))) >> s : (unsigned __int128)v >> s;
|
|
}
|
|
SI uint64_t pmul64(uint64_t a, uint64_t b, uint64_t *hi) { // carry-less
|
|
uint64_t lo = 0, h = 0;
|
|
for (int i = 0; i < 64; i++)
|
|
if ((b >> i) & 1) lo ^= a << i, h ^= i ? a >> (64 - i) : 0;
|
|
if (hi) *hi = h;
|
|
return lo;
|
|
}
|
|
|
|
enum {
|
|
LOP_SQADD, LOP_UQADD, LOP_SQSUB, LOP_UQSUB, LOP_SUQADD, LOP_USQADD,
|
|
LOP_SHL, LOP_RSHL, LOP_QSHL, LOP_QRSHL, // register shifts (b = signed byte), U selects unsigned
|
|
LOP_SHLI, LOP_QSHLI, LOP_QSHLUI, LOP_SHRI, LOP_RSHRI, LOP_SRAI, LOP_RSRAI, LOP_SRI, LOP_SLI,
|
|
LOP_SQDMULH, LOP_SQRDMULH, LOP_PMUL,
|
|
LOP_SQABS, LOP_SQNEG, LOP_ABS, LOP_NEG, LOP_CLS, LOP_CLZ, LOP_CNT, LOP_RBIT, LOP_URECPE, LOP_URSQRTE,
|
|
LOP_HADD, LOP_RHADD, LOP_HSUB, LOP_ABD, LOP_ABA,
|
|
// long: result lanes sz+1
|
|
LOP_ADDL, LOP_SUBL, LOP_ABDL, LOP_ABAL, LOP_MLAL, LOP_MLSL, LOP_MULL, LOP_ADDW, LOP_SUBW,
|
|
LOP_SQDMULL, LOP_SQDMLAL, LOP_SQDMLSL, LOP_PMULL, LOP_SHLL,
|
|
// narrow: sz = destination lane size, a/b are sz+1
|
|
LOP_XTN, LOP_SQXTN, LOP_UQXTN, LOP_SQXTUN, LOP_SHRN, LOP_RSHRN, LOP_QSHRN, LOP_QRSHRN, LOP_QSHRUN,
|
|
LOP_QRSHRUN, LOP_ADDHN, LOP_RADDHN, LOP_SUBHN, LOP_RSUBHN,
|
|
LOP_SHLLI, // SSHLL/USHLL: extend, shift (long)
|
|
};
|
|
#define LOP_U 256 // unsigned variant flag
|
|
|
|
SI uint64_t lop(int op, int sz, uint64_t a, uint64_t b, uint64_t d) {
|
|
int u = op >> 8, es = 8 << sz;
|
|
op &= 255;
|
|
int64_t sa = lsx(a, sz), sb = lsx(b, sz);
|
|
__int128 xa = u ? (__int128)a : (__int128)sa, xb = u ? (__int128)b : (__int128)sb; // extended operands
|
|
int sh = (int8_t)b;
|
|
switch (op) {
|
|
case LOP_SQADD: return sat_s((__int128)sa + sb, sz);
|
|
case LOP_UQADD: return sat_u((__int128)a + b, sz);
|
|
case LOP_SQSUB: return sat_s((__int128)sa - sb, sz);
|
|
case LOP_UQSUB: return sat_u((__int128)a - b, sz);
|
|
case LOP_SUQADD: return sat_s((__int128)lsx(d, sz) + a, sz);
|
|
case LOP_USQADD: return sat_u((__int128)d + sa, sz);
|
|
case LOP_SHL: case LOP_RSHL: case LOP_QSHL: case LOP_QRSHL: {
|
|
int sat = op == LOP_QSHL || op == LOP_QRSHL, rnd = op == LOP_RSHL || op == LOP_QRSHL;
|
|
if (sh >= 0) {
|
|
if (sat) return u ? sat_uu((unsigned __int128)a << (sh > 64 ? 64 : sh), sz)
|
|
: sat_s(shl128(sa, sh > 64 ? 64 : sh), sz);
|
|
return sh >= es ? 0 : a << sh;
|
|
}
|
|
return u ? (uint64_t)lshr_u(a, -sh, rnd) : (uint64_t)lshr_s(sa, -sh, rnd);
|
|
}
|
|
case LOP_SHLI: return a << b;
|
|
case LOP_QSHLI: return u ? sat_uu((unsigned __int128)a << b, sz) : sat_s(shl128(sa, (int)b), sz);
|
|
case LOP_QSHLUI: return sat_u(shl128(sa, (int)b), sz);
|
|
case LOP_SHRI: return u ? (uint64_t)lshr_u(a, (int)b, 0) : (uint64_t)lshr_s(sa, (int)b, 0);
|
|
case LOP_RSHRI: return u ? (uint64_t)lshr_u(a, (int)b, 1) : (uint64_t)lshr_s(sa, (int)b, 1);
|
|
case LOP_SRAI: return d + (u ? (uint64_t)lshr_u(a, (int)b, 0) : (uint64_t)lshr_s(sa, (int)b, 0));
|
|
case LOP_RSRAI: return d + (u ? (uint64_t)lshr_u(a, (int)b, 1) : (uint64_t)lshr_s(sa, (int)b, 1));
|
|
case LOP_SRI: {
|
|
uint64_t m = (int)b >= es ? 0 : lmask(sz) >> b;
|
|
return (d & ~m) | ((int)b >= es ? 0 : a >> b);
|
|
}
|
|
case LOP_SLI: { uint64_t m = (lmask(sz) << b) & lmask(sz); return (d & ~m) | ((a << b) & m); }
|
|
case LOP_SQDMULH: return sat_s(((__int128)sa * sb * 2) >> es, sz);
|
|
case LOP_SQRDMULH: return sat_s(((__int128)sa * sb * 2 + ((__int128)1 << (es - 1))) >> es, sz);
|
|
case LOP_PMUL: return pmul64(a, b, 0);
|
|
case LOP_SQABS: return sat_s(sa < 0 ? -(__int128)sa : sa, sz);
|
|
case LOP_SQNEG: return sat_s(-(__int128)sa, sz);
|
|
case LOP_ABS: return sa < 0 ? (uint64_t)-sa : (uint64_t)sa;
|
|
case LOP_NEG: return -a;
|
|
case LOP_CLS: {
|
|
uint64_t x = (uint64_t)(sa ^ (sa >> 1)) & lmask(sz);
|
|
return (x ? (uint64_t)__builtin_clzll(x << (64 - es)) : (uint64_t)es) - 1;
|
|
}
|
|
case LOP_CLZ: return a ? (uint64_t)__builtin_clzll(a << (64 - es)) : (uint64_t)es;
|
|
case LOP_CNT: return (uint64_t)__builtin_popcountll(a);
|
|
case LOP_RBIT: return __builtin_bitreverse8((uint8_t)a);
|
|
case LOP_URECPE: return (a >> 31) ? (uint64_t)(256 | simd_recip_tab[(a >> 23) & 255]) << 23 : 0xffffffffull;
|
|
case LOP_URSQRTE: return (a >> 30) ? (uint64_t)(256 | simd_rsqrt_tab[((a >> 23) & 511) - 128]) << 23 : 0xffffffffull;
|
|
case LOP_HADD: return (uint64_t)((xa + xb) >> 1);
|
|
case LOP_RHADD: return (uint64_t)((xa + xb + 1) >> 1);
|
|
case LOP_HSUB: return (uint64_t)((xa - xb) >> 1);
|
|
case LOP_ABD: return (uint64_t)(xa > xb ? xa - xb : xb - xa);
|
|
case LOP_ABA: return d + (uint64_t)(xa > xb ? xa - xb : xb - xa);
|
|
case LOP_ADDL: return (uint64_t)(xa + xb);
|
|
case LOP_SUBL: return (uint64_t)(xa - xb);
|
|
case LOP_ABDL: return (uint64_t)(xa > xb ? xa - xb : xb - xa);
|
|
case LOP_ABAL: return d + (uint64_t)(xa > xb ? xa - xb : xb - xa);
|
|
case LOP_MLAL: return d + (uint64_t)(xa * xb);
|
|
case LOP_MLSL: return d - (uint64_t)(xa * xb);
|
|
case LOP_MULL: return (uint64_t)(xa * xb);
|
|
case LOP_ADDW: return a + (uint64_t)xb;
|
|
case LOP_SUBW: return a - (uint64_t)xb;
|
|
case LOP_SQDMULL: return sat_s((__int128)sa * sb * 2, sz + 1);
|
|
case LOP_SQDMLAL: return sat_s((__int128)lsx(d, sz + 1) + (__int128)lsx(sat_s((__int128)sa * sb * 2, sz + 1), sz + 1), sz + 1);
|
|
case LOP_SQDMLSL: return sat_s((__int128)lsx(d, sz + 1) - (__int128)lsx(sat_s((__int128)sa * sb * 2, sz + 1), sz + 1), sz + 1);
|
|
case LOP_PMULL: return pmul64(a, b, 0);
|
|
case LOP_SHLL: return a << es;
|
|
// narrowing: a (and b for xHN) are 2*es wide
|
|
case LOP_XTN: return a;
|
|
case LOP_SQXTN: return sat_s(lsx(a, sz + 1), sz);
|
|
case LOP_UQXTN: return sat_u(a, sz);
|
|
case LOP_SQXTUN: return sat_u(lsx(a, sz + 1), sz);
|
|
case LOP_SHRN: return a >> b;
|
|
case LOP_RSHRN: return (uint64_t)lshr_u(a, (int)b, 1);
|
|
case LOP_QSHRN: case LOP_QRSHRN: {
|
|
int rnd = op == LOP_QRSHRN;
|
|
return u ? sat_u((__int128)lshr_u(a, (int)b, rnd), sz) : sat_s(lshr_s(lsx(a, sz + 1), (int)b, rnd), sz);
|
|
}
|
|
case LOP_QSHRUN: return sat_u(lshr_s(lsx(a, sz + 1), (int)b, 0), sz);
|
|
case LOP_QRSHRUN: return sat_u(lshr_s(lsx(a, sz + 1), (int)b, 1), sz);
|
|
case LOP_ADDHN: return (a + b) >> es;
|
|
case LOP_RADDHN: return (a + b + (1ull << (es - 1))) >> es;
|
|
case LOP_SUBHN: return (a - b) >> es;
|
|
case LOP_RSUBHN: return (a - b + (1ull << (es - 1))) >> es;
|
|
case LOP_SHLLI: return (uint64_t)xa << b;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Generic lane loop: ne lanes; result lane i (size szd) goes to lane offd+i of d (other lanes kept when keep,
|
|
// else zero); a = n lane offn+i (size szn), b = m lane offm+i (size szm), or lane bidx of m when bidx >= 0,
|
|
// or the constant bimm when m < 0; the accumulator is d's old lane offd+i.
|
|
SI void sl_gen(Cpu *c, int op, int sz, int ne, int d, int szd, int offd, int keep, int n, int szn, int offn, int m,
|
|
int szm, int offm, int bidx, uint64_t bimm) {
|
|
u128 a = c->v[n], b = m >= 0 ? c->v[m] : 0, o = c->v[d], r = keep ? o : 0;
|
|
for (int i = 0; i < ne; i++) {
|
|
uint64_t y = m < 0 ? bimm : lget(&b, szm, bidx >= 0 ? bidx : offm + i);
|
|
lset(&r, szd, offd + i, lop(op, sz, lget(&a, szn, offn + i), y, lget(&o, szd, offd + i)));
|
|
}
|
|
c->v[d] = r;
|
|
}
|
|
// SADDLP/UADDLP/SADALP/UADALP
|
|
SI void sl_addlp(Cpu *c, int u, int acc, int Q, int sz, int d, int n) {
|
|
u128 a = c->v[n], o = c->v[d], r = 0;
|
|
for (int i = 0; i < (4 << Q) >> sz; i++) {
|
|
uint64_t x = lget(&a, sz, 2 * i), y = lget(&a, sz, 2 * i + 1);
|
|
uint64_t s = u ? x + y : (uint64_t)(lsx(x, sz) + lsx(y, sz));
|
|
lset(&r, sz + 1, i, s + (acc ? lget(&o, sz + 1, i) : 0));
|
|
}
|
|
c->v[d] = r;
|
|
}
|
|
// ADDV, SADDLV/UADDLV, SMAXV/UMAXV, SMINV/UMINV (op 0..3; u = unsigned)
|
|
SI void sl_across(Cpu *c, int op, int u, int Q, int sz, int d, int n) {
|
|
u128 a = c->v[n];
|
|
int ne = (8 << Q) >> sz;
|
|
int64_t acc = u ? (int64_t)lget(&a, sz, 0) : lsx(lget(&a, sz, 0), sz);
|
|
for (int i = 1; i < ne; i++) {
|
|
int64_t x = u ? (int64_t)lget(&a, sz, i) : lsx(lget(&a, sz, i), sz);
|
|
if (op <= 1) acc += x;
|
|
else if (op == 2) acc = u ? ((uint64_t)x > (uint64_t)acc ? x : acc) : (x > acc ? x : acc);
|
|
else acc = u ? ((uint64_t)x < (uint64_t)acc ? x : acc) : (x < acc ? x : acc);
|
|
}
|
|
c->v[d] = (uint64_t)acc & lmask(op == 1 ? sz + 1 : sz);
|
|
}
|
|
// PMULL2/PMULL 1Q (64x64 -> 128 carry-less)
|
|
SI void sl_pmull64(Cpu *c, int part, int d, int n, int m) {
|
|
uint64_t hi, lo = pmul64(((sa64 *)&c->v[n])[part], ((sa64 *)&c->v[m])[part], &hi);
|
|
c->v[d] = ((u128)hi << 64) | lo;
|
|
}
|
|
|
|
// ---- FP vector helpers: lanes of the 64-bit form are computed on a zero upper half
|
|
SI vf32 vsplat_s(vf32 v, int i, int Q) { float x = v[i]; return Q ? (vf32){x, x, x, x} : (vf32){x, x, 0, 0}; }
|
|
SI vf64 vsplat_d(vf64 v, int i, int Q) { double x = v[i]; (void)Q; return (vf64){x, x}; }
|
|
#define SV_FOP_S(op, Q, d, n, m, a) VW(vf32, Q, d, vop_s(c, op, n, m, a))
|
|
#define SV_FOP_D(op, Q, d, n, m, a) VW(vf64, Q, d, vop_d(c, op, n, m, a))
|
|
SI vf32 vneg_s(vf32 v) { return (vf32)((vu32)v ^ 0x80000000u); }
|
|
SI vf64 vneg_d(vf64 v) { return (vf64)((vu64)v ^ 0x8000000000000000ull); }
|
|
SI float pair_s(Cpu *c, int op, float a, float b) { return op_s(c, op, a, b, 0); }
|
|
SI double pair_d(Cpu *c, int op, double a, double b) { return op_d(c, op, a, b, 0); }
|
|
// FMAXV/FMINV/FMAXNMV/FMINNMV (4S): pairwise tree
|
|
SI void sv_fred_s(Cpu *c, int op, int d, int n) {
|
|
vf32 v = VREG(vf32, n);
|
|
c->v[d] = fb_s(pair_s(c, op, pair_s(c, op, v[0], v[1]), pair_s(c, op, v[2], v[3])));
|
|
}
|
|
// lane-wise FP conversions
|
|
SI vu32 vtoint_s(Cpu *c, vf32 v, int rm, int fbits, int uns) {
|
|
vu32 r;
|
|
for (int i = 0; i < 4; i++) r[i] = (uint32_t)toint_s(c, v[i], rm, fbits, 32, uns);
|
|
return r;
|
|
}
|
|
SI vu64 vtoint_d(Cpu *c, vf64 v, int rm, int fbits, int uns) {
|
|
vu64 r;
|
|
for (int i = 0; i < 2; i++) r[i] = toint_d(c, v[i], rm, fbits, 64, uns);
|
|
return r;
|
|
}
|
|
SI vf32 vfromint_s(Cpu *c, vu32 v, int fbits, int uns) {
|
|
vf32 r;
|
|
if (!(c->fpcr & FPCR_RM)) {
|
|
r = uns ? CVT(v, vf32) : CVT((vs32)v, vf32);
|
|
if (fbits) r = r / (float)(1ull << (fbits - 1)) / 2.0f;
|
|
return r;
|
|
}
|
|
for (int i = 0; i < 4; i++) r[i] = fromint_s(c, v[i], 32, uns, fbits);
|
|
return r;
|
|
}
|
|
SI vf64 vfromint_d(Cpu *c, vu64 v, int fbits, int uns) {
|
|
vf64 r;
|
|
for (int i = 0; i < 2; i++) r[i] = fromint_d(c, v[i], 64, uns, fbits);
|
|
return r;
|
|
}
|
|
SI vf32 vrecpe_s(Cpu *c, vf32 v) { for (int i = 0; i < 4; i++) v[i] = recpe_s(c, v[i]); return v; }
|
|
SI vf64 vrecpe_d(Cpu *c, vf64 v) { for (int i = 0; i < 2; i++) v[i] = recpe_d(c, v[i]); return v; }
|
|
SI vf32 vrsqrte_s(Cpu *c, vf32 v) { for (int i = 0; i < 4; i++) v[i] = rsqrte_s(c, v[i]); return v; }
|
|
SI vf64 vrsqrte_d(Cpu *c, vf64 v) { for (int i = 0; i < 2; i++) v[i] = rsqrte_d(c, v[i]); return v; }
|
|
|
|
// FCVTL/FCVTL2 (h->s, s->d), FCVTN/FCVTN2 (s->h, d->s), FCVTXN/FCVTXN2
|
|
SI void sv_fcvtl(Cpu *c, int sz, int part, int d, int n) {
|
|
u128 a = c->v[n], r = 0;
|
|
if (sz == 0) {
|
|
vf32 o;
|
|
for (int i = 0; i < 4; i++) o[i] = simd_h2s(c, (uint16_t)lget(&a, 1, 4 * part + i));
|
|
r = (u128)o;
|
|
} else {
|
|
vf64 o;
|
|
for (int i = 0; i < 2; i++) o[i] = simd_s2d(c, fv_s((uint32_t)lget(&a, 2, 2 * part + i)));
|
|
r = (u128)o;
|
|
}
|
|
c->v[d] = r;
|
|
}
|
|
SI void sv_fcvtn(Cpu *c, int sz, int part, int odd, int d, int n) {
|
|
u128 a = c->v[n], r = part ? (uint64_t)c->v[d] : 0;
|
|
if (sz == 0) {
|
|
for (int i = 0; i < 4; i++) lset(&r, 1, 4 * part + i, simd_s2h(c, fv_s((uint32_t)lget(&a, 2, i))));
|
|
} else {
|
|
for (int i = 0; i < 2; i++) {
|
|
double x = fv_d(lget(&a, 3, i));
|
|
lset(&r, 2, 2 * part + i, fb_s(odd ? simd_d2s_odd(c, x) : simd_d2s(c, x)));
|
|
}
|
|
}
|
|
c->v[d] = r;
|
|
}
|
|
|
|
// TBL / TBX with len+1 table registers
|
|
SI void sv_tbl(Cpu *c, int Q, int len, int tbx, int d, int n, int m) {
|
|
uint8_t tab[64];
|
|
for (int i = 0; i <= len; i++) memcpy(tab + 16 * i, &c->v[(n + i) & 31], 16);
|
|
u128 idx = c->v[m], o = c->v[d], r = 0;
|
|
for (int i = 0; i < 8 << Q; i++) {
|
|
unsigned k = (unsigned)lget(&idx, 0, i);
|
|
lset(&r, 0, i, k < 16u * (len + 1) ? tab[k] : tbx ? lget(&o, 0, i) : 0);
|
|
}
|
|
c->v[d] = r;
|
|
}
|
|
|
|
// ---- CRC32 / CRC32C (bytes = 1, 2, 4, 8)
|
|
SI uint32_t simd_crc(const uint32_t *t, uint32_t acc, uint64_t v, int bytes) {
|
|
for (int i = 0; i < bytes; i++) acc = t[(acc ^ (uint8_t)(v >> (8 * i))) & 0xff] ^ (acc >> 8);
|
|
return acc;
|
|
}
|
|
#define CRC32(acc, v, n) simd_crc(simd_crc32_tab, (uint32_t)(acc), (v), n)
|
|
#define CRC32C(acc, v, n) simd_crc(simd_crc32c_tab, (uint32_t)(acc), (v), n)
|
|
|
|
// ---- crypto (AES, SHA1, SHA256)
|
|
SI uint8_t aes_xt(uint8_t x) { return (uint8_t)((x << 1) ^ ((x >> 7) * 0x1b)); }
|
|
SI uint8_t aes_mul(uint8_t x, int k) { // k in {9, 11, 13, 14}
|
|
uint8_t x2 = aes_xt(x), x4 = aes_xt(x2), x8 = aes_xt(x4);
|
|
return (uint8_t)(x8 ^ (k & 4 ? x4 : 0) ^ (k & 2 ? x2 : 0) ^ (k & 1 ? x : 0));
|
|
}
|
|
SI void sv_aes(Cpu *c, int op, int d, int n) { // 0 AESE, 1 AESD, 2 AESMC, 3 AESIMC
|
|
uint8_t s[16], r[16];
|
|
u128 v = op < 2 ? c->v[d] ^ c->v[n] : c->v[n];
|
|
memcpy(s, &v, 16);
|
|
if (op == 0) {
|
|
for (int i = 0; i < 16; i++) r[i] = simd_aes_sbox[s[(i + 4 * (i & 3)) & 15]];
|
|
} else if (op == 1) {
|
|
for (int i = 0; i < 16; i++) r[i] = simd_aes_isbox[s[(i - 4 * (i & 3)) & 15]];
|
|
} else {
|
|
for (int col = 0; col < 4; col++) {
|
|
uint8_t *a = s + 4 * col, *o = r + 4 * col;
|
|
for (int i = 0; i < 4; i++) {
|
|
uint8_t a0 = a[i], a1 = a[(i + 1) & 3], a2 = a[(i + 2) & 3], a3 = a[(i + 3) & 3];
|
|
o[i] = op == 2 ? (uint8_t)(aes_xt(a0) ^ aes_xt(a1) ^ a1 ^ a2 ^ a3)
|
|
: (uint8_t)(aes_mul(a0, 14) ^ aes_mul(a1, 11) ^ aes_mul(a2, 13) ^ aes_mul(a3, 9));
|
|
}
|
|
}
|
|
}
|
|
memcpy(&v, r, 16);
|
|
c->v[d] = v;
|
|
}
|
|
SI uint32_t rol32_(uint32_t x, int s) { return (x << s) | (x >> (32 - s)); }
|
|
SI uint32_t ror32_(uint32_t x, int s) { return (x >> s) | (x << (32 - s)); }
|
|
SI void sv_sha1hash(Cpu *c, int op, int d, int n, int m) { // 0 SHA1C, 1 SHA1P, 2 SHA1M
|
|
uint32_t x[4], w[4], y = (uint32_t)c->v[n];
|
|
memcpy(x, &c->v[d], 16);
|
|
memcpy(w, &c->v[m], 16);
|
|
for (int e = 0; e < 4; e++) {
|
|
uint32_t t = op == 0 ? (((x[2] ^ x[3]) & x[1]) ^ x[3]) : op == 1 ? (x[1] ^ x[2] ^ x[3])
|
|
: ((x[1] & x[2]) | ((x[1] | x[2]) & x[3]));
|
|
y = y + rol32_(x[0], 5) + t + w[e];
|
|
x[1] = rol32_(x[1], 30);
|
|
uint32_t top = x[3];
|
|
x[3] = x[2], x[2] = x[1], x[1] = x[0], x[0] = y, y = top;
|
|
}
|
|
memcpy(&c->v[d], x, 16);
|
|
}
|
|
SI void sv_sha256hash(Cpu *c, int part1, int d, int n, int m) { // SHA256H (part1), SHA256H2
|
|
uint32_t x[4], y[4], w[4];
|
|
memcpy(part1 ? x : y, &c->v[d], 16);
|
|
memcpy(part1 ? y : x, &c->v[n], 16);
|
|
memcpy(w, &c->v[m], 16);
|
|
for (int e = 0; e < 4; e++) {
|
|
uint32_t chs = ((y[1] ^ y[2]) & y[0]) ^ y[2];
|
|
uint32_t maj = (x[0] & x[1]) | ((x[0] | x[1]) & x[2]);
|
|
uint32_t t1 = y[3] + (ror32_(y[0], 6) ^ ror32_(y[0], 11) ^ ror32_(y[0], 25)) + chs + w[e];
|
|
x[3] = t1 + x[3];
|
|
y[3] = t1 + (ror32_(x[0], 2) ^ ror32_(x[0], 13) ^ ror32_(x[0], 22)) + maj;
|
|
uint32_t t = y[3];
|
|
y[3] = y[2], y[2] = y[1], y[1] = y[0], y[0] = x[3];
|
|
x[3] = x[2], x[2] = x[1], x[1] = x[0], x[0] = t;
|
|
}
|
|
memcpy(&c->v[d], part1 ? x : y, 16);
|
|
}
|
|
SI void sv_sha_misc(Cpu *c, int op, int d, int n, int m) { // 0 SHA1SU0, 1 SHA1SU1, 2 SHA256SU0, 3 SHA256SU1, 4 SHA1H
|
|
uint32_t a[4], b[4], k[4], r[4], t[4];
|
|
memcpy(a, &c->v[d], 16);
|
|
memcpy(b, &c->v[n], 16);
|
|
memcpy(k, &c->v[m], 16);
|
|
switch (op) {
|
|
case 0:
|
|
t[0] = a[2], t[1] = a[3], t[2] = b[0], t[3] = b[1];
|
|
for (int i = 0; i < 4; i++) r[i] = t[i] ^ a[i] ^ k[i];
|
|
break;
|
|
case 1:
|
|
for (int i = 0; i < 4; i++) t[i] = a[i] ^ (i < 3 ? b[i + 1] : 0);
|
|
for (int i = 0; i < 4; i++) r[i] = rol32_(t[i], 1);
|
|
r[3] ^= rol32_(t[0], 2);
|
|
break;
|
|
case 2:
|
|
t[0] = a[1], t[1] = a[2], t[2] = a[3], t[3] = b[0];
|
|
for (int i = 0; i < 4; i++) r[i] = (ror32_(t[i], 7) ^ ror32_(t[i], 18) ^ (t[i] >> 3)) + a[i];
|
|
break;
|
|
case 3: {
|
|
uint32_t t0[4] = {b[1], b[2], b[3], k[0]};
|
|
for (int e = 0; e < 4; e++) {
|
|
uint32_t x = e < 2 ? k[e + 2] : r[e - 2];
|
|
r[e] = (ror32_(x, 17) ^ ror32_(x, 19) ^ (x >> 10)) + a[e] + t0[e];
|
|
}
|
|
break;
|
|
}
|
|
default:
|
|
c->v[d] = rol32_(b[0], 30);
|
|
return;
|
|
}
|
|
memcpy(&c->v[d], r, 16);
|
|
}
|
|
|
|
// ---- loads / stores of lanes
|
|
SI uint64_t simd_ld(uint64_t a, int sz) {
|
|
switch (sz) {
|
|
case 0: return LD8(a);
|
|
case 1: return LD16(a);
|
|
case 2: return LD32(a);
|
|
default: return LD64(a);
|
|
}
|
|
}
|
|
SI void simd_st(uint64_t a, int sz, uint64_t v) {
|
|
switch (sz) {
|
|
case 0: ST8(a, v); break;
|
|
case 1: ST16(a, v); break;
|
|
case 2: ST32(a, v); break;
|
|
default: ST64(a, v); break;
|
|
}
|
|
}
|
|
// LDn/STn (multiple structures): selem interleaved registers, rpt repeats (LD1 with 2-4 registers)
|
|
SI void sl_ldm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) {
|
|
u128 r[4] = {0, 0, 0, 0};
|
|
int ne = (8 << Q) >> sz;
|
|
for (int k = 0; k < rpt; k++)
|
|
for (int e = 0; e < ne; e++)
|
|
for (int s = 0; s < selem; s++, ea += 1 << sz) lset(&r[k + s], sz, e, simd_ld(ea, sz));
|
|
for (int i = 0; i < rpt + selem - 1; i++) c->v[(t + i) & 31] = r[i];
|
|
}
|
|
SI void sl_stm(Cpu *c, int Q, int sz, int selem, int rpt, int t, uint64_t ea) {
|
|
u128 r[4];
|
|
int ne = (8 << Q) >> sz;
|
|
for (int i = 0; i < rpt + selem - 1; i++) r[i] = c->v[(t + i) & 31];
|
|
for (int k = 0; k < rpt; k++)
|
|
for (int e = 0; e < ne; e++)
|
|
for (int s = 0; s < selem; s++, ea += 1 << sz) simd_st(ea, sz, lget(&r[k + s], sz, e));
|
|
}
|
|
// single structure: lane idx of selem registers; LDnR replicates into every lane
|
|
SI void sl_lds(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) {
|
|
for (int s = 0; s < selem; s++) {
|
|
u128 v = c->v[(t + s) & 31];
|
|
lset(&v, sz, idx, simd_ld(ea + (s << sz), sz));
|
|
c->v[(t + s) & 31] = v;
|
|
}
|
|
}
|
|
SI void sl_sts(Cpu *c, int sz, int selem, int idx, int t, uint64_t ea) {
|
|
for (int s = 0; s < selem; s++) simd_st(ea + (s << sz), sz, lget(&c->v[(t + s) & 31], sz, idx));
|
|
}
|
|
SI void sl_ldr(Cpu *c, int Q, int sz, int selem, int t, uint64_t ea) {
|
|
uint64_t x[4];
|
|
for (int s = 0; s < selem; s++) x[s] = simd_ld(ea + (s << sz), sz);
|
|
for (int s = 0; s < selem; s++) {
|
|
u128 v = 0;
|
|
for (int e = 0; e < (8 << Q) >> sz; e++) lset(&v, sz, e, x[s]);
|
|
c->v[(t + s) & 31] = v;
|
|
}
|
|
}
|