diff --git a/runtime/kernel.c b/runtime/kernel.c index 2e86a005..e8fb48b4 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -415,6 +415,18 @@ static int ap_disabled(const RtThread *t) { return g_ap > 0 && ap_tlr(t)[0]; } // in kern_ap_yield and jumps back to kern_ap_rip(), whose value it first stores in the return slot it reserved at // entry, so a stack walk sees the interrupted function as its caller. No ret: the jump keeps call/return pairs // balanced. The waiter never writes the owner's stack. +// An x86-64-v2 build (no __AVX__) saves XMM0-15 with SSE moves into the same 32-byte slots instead: its game code +// keeps nothing in the upper YMM halves, and the CPUs it is for have no VEX encoding, so a vmovdqa there faults with +// an illegal instruction the first time a core is interrupted. +#ifdef __AVX__ +#define AP_VMOV "vmovdqa" +#define AP_V "%ymm" +#define AP_VZEROUPPER " vzeroupper\n" +#else +#define AP_VMOV "movdqa" +#define AP_V "%xmm" +#define AP_VZEROUPPER "" +#endif #ifdef _WIN32 __asm__(".text\n" ".globl kern_ap_stub\n" @@ -446,48 +458,48 @@ __asm__(".text\n" ".seh_setframe %rbp, 0\n" ".seh_endprologue\n" " andq $-32, %rsp\n" - " subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 - " vmovdqa %ymm0, 64(%rsp)\n" - " vmovdqa %ymm1, 96(%rsp)\n" - " vmovdqa %ymm2, 128(%rsp)\n" - " vmovdqa %ymm3, 160(%rsp)\n" - " vmovdqa %ymm4, 192(%rsp)\n" - " vmovdqa %ymm5, 224(%rsp)\n" - " vmovdqa %ymm6, 256(%rsp)\n" - " vmovdqa %ymm7, 288(%rsp)\n" - " vmovdqa %ymm8, 320(%rsp)\n" - " vmovdqa %ymm9, 352(%rsp)\n" - " vmovdqa %ymm10, 384(%rsp)\n" - " vmovdqa %ymm11, 416(%rsp)\n" - " vmovdqa %ymm12, 448(%rsp)\n" - " vmovdqa %ymm13, 480(%rsp)\n" - " vmovdqa %ymm14, 512(%rsp)\n" - " vmovdqa %ymm15, 544(%rsp)\n" + " subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) + " " AP_VMOV " " AP_V "0, 64(%rsp)\n" + " " AP_VMOV " " AP_V "1, 96(%rsp)\n" + " " AP_VMOV " " AP_V "2, 128(%rsp)\n" + " " AP_VMOV " " AP_V "3, 160(%rsp)\n" + " " AP_VMOV " " AP_V "4, 192(%rsp)\n" + " " AP_VMOV " " AP_V "5, 224(%rsp)\n" + " " AP_VMOV " " AP_V "6, 256(%rsp)\n" + " " AP_VMOV " " AP_V "7, 288(%rsp)\n" + " " AP_VMOV " " AP_V "8, 320(%rsp)\n" + " " AP_VMOV " " AP_V "9, 352(%rsp)\n" + " " AP_VMOV " " AP_V "10, 384(%rsp)\n" + " " AP_VMOV " " AP_V "11, 416(%rsp)\n" + " " AP_VMOV " " AP_V "12, 448(%rsp)\n" + " " AP_VMOV " " AP_V "13, 480(%rsp)\n" + " " AP_VMOV " " AP_V "14, 512(%rsp)\n" + " " AP_VMOV " " AP_V "15, 544(%rsp)\n" " stmxcsr 32(%rsp)\n" " movl $0x1f80, 36(%rsp)\n" " ldmxcsr 36(%rsp)\n" " cld\n" - " vzeroupper\n" + AP_VZEROUPPER " call kern_ap_rip\n" " movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq " call kern_ap_yield\n" " ldmxcsr 32(%rsp)\n" - " vmovdqa 64(%rsp), %ymm0\n" - " vmovdqa 96(%rsp), %ymm1\n" - " vmovdqa 128(%rsp), %ymm2\n" - " vmovdqa 160(%rsp), %ymm3\n" - " vmovdqa 192(%rsp), %ymm4\n" - " vmovdqa 224(%rsp), %ymm5\n" - " vmovdqa 256(%rsp), %ymm6\n" - " vmovdqa 288(%rsp), %ymm7\n" - " vmovdqa 320(%rsp), %ymm8\n" - " vmovdqa 352(%rsp), %ymm9\n" - " vmovdqa 384(%rsp), %ymm10\n" - " vmovdqa 416(%rsp), %ymm11\n" - " vmovdqa 448(%rsp), %ymm12\n" - " vmovdqa 480(%rsp), %ymm13\n" - " vmovdqa 512(%rsp), %ymm14\n" - " vmovdqa 544(%rsp), %ymm15\n" + " " AP_VMOV " 64(%rsp), " AP_V "0\n" + " " AP_VMOV " 96(%rsp), " AP_V "1\n" + " " AP_VMOV " 128(%rsp), " AP_V "2\n" + " " AP_VMOV " 160(%rsp), " AP_V "3\n" + " " AP_VMOV " 192(%rsp), " AP_V "4\n" + " " AP_VMOV " 224(%rsp), " AP_V "5\n" + " " AP_VMOV " 256(%rsp), " AP_V "6\n" + " " AP_VMOV " 288(%rsp), " AP_V "7\n" + " " AP_VMOV " 320(%rsp), " AP_V "8\n" + " " AP_VMOV " 352(%rsp), " AP_V "9\n" + " " AP_VMOV " 384(%rsp), " AP_V "10\n" + " " AP_VMOV " 416(%rsp), " AP_V "11\n" + " " AP_VMOV " 448(%rsp), " AP_V "12\n" + " " AP_VMOV " 480(%rsp), " AP_V "13\n" + " " AP_VMOV " 512(%rsp), " AP_V "14\n" + " " AP_VMOV " 544(%rsp), " AP_V "15\n" " movq %rbp, %rsp\n" " popq %rbp\n" " popq %r11\n" @@ -525,48 +537,48 @@ __asm__(".text\n" " pushq %rbp\n" " movq %rsp, %rbp\n" " andq $-32, %rsp\n" - " subq $576, %rsp\n" // 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 - " vmovdqa %ymm0, 64(%rsp)\n" - " vmovdqa %ymm1, 96(%rsp)\n" - " vmovdqa %ymm2, 128(%rsp)\n" - " vmovdqa %ymm3, 160(%rsp)\n" - " vmovdqa %ymm4, 192(%rsp)\n" - " vmovdqa %ymm5, 224(%rsp)\n" - " vmovdqa %ymm6, 256(%rsp)\n" - " vmovdqa %ymm7, 288(%rsp)\n" - " vmovdqa %ymm8, 320(%rsp)\n" - " vmovdqa %ymm9, 352(%rsp)\n" - " vmovdqa %ymm10, 384(%rsp)\n" - " vmovdqa %ymm11, 416(%rsp)\n" - " vmovdqa %ymm12, 448(%rsp)\n" - " vmovdqa %ymm13, 480(%rsp)\n" - " vmovdqa %ymm14, 512(%rsp)\n" - " vmovdqa %ymm15, 544(%rsp)\n" + " subq $576, %rsp\n" // 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) + " " AP_VMOV " " AP_V "0, 64(%rsp)\n" + " " AP_VMOV " " AP_V "1, 96(%rsp)\n" + " " AP_VMOV " " AP_V "2, 128(%rsp)\n" + " " AP_VMOV " " AP_V "3, 160(%rsp)\n" + " " AP_VMOV " " AP_V "4, 192(%rsp)\n" + " " AP_VMOV " " AP_V "5, 224(%rsp)\n" + " " AP_VMOV " " AP_V "6, 256(%rsp)\n" + " " AP_VMOV " " AP_V "7, 288(%rsp)\n" + " " AP_VMOV " " AP_V "8, 320(%rsp)\n" + " " AP_VMOV " " AP_V "9, 352(%rsp)\n" + " " AP_VMOV " " AP_V "10, 384(%rsp)\n" + " " AP_VMOV " " AP_V "11, 416(%rsp)\n" + " " AP_VMOV " " AP_V "12, 448(%rsp)\n" + " " AP_VMOV " " AP_V "13, 480(%rsp)\n" + " " AP_VMOV " " AP_V "14, 512(%rsp)\n" + " " AP_VMOV " " AP_V "15, 544(%rsp)\n" " stmxcsr 32(%rsp)\n" " movl $0x1f80, 36(%rsp)\n" " ldmxcsr 36(%rsp)\n" " cld\n" - " vzeroupper\n" + AP_VZEROUPPER " call kern_ap_rip\n" " movq %rax, 88(%rbp)\n" // the return slot: rbp + 10 pushes + pushfq " call kern_ap_yield\n" " ldmxcsr 32(%rsp)\n" - " vmovdqa 64(%rsp), %ymm0\n" - " vmovdqa 96(%rsp), %ymm1\n" - " vmovdqa 128(%rsp), %ymm2\n" - " vmovdqa 160(%rsp), %ymm3\n" - " vmovdqa 192(%rsp), %ymm4\n" - " vmovdqa 224(%rsp), %ymm5\n" - " vmovdqa 256(%rsp), %ymm6\n" - " vmovdqa 288(%rsp), %ymm7\n" - " vmovdqa 320(%rsp), %ymm8\n" - " vmovdqa 352(%rsp), %ymm9\n" - " vmovdqa 384(%rsp), %ymm10\n" - " vmovdqa 416(%rsp), %ymm11\n" - " vmovdqa 448(%rsp), %ymm12\n" - " vmovdqa 480(%rsp), %ymm13\n" - " vmovdqa 512(%rsp), %ymm14\n" - " vmovdqa 544(%rsp), %ymm15\n" + " " AP_VMOV " 64(%rsp), " AP_V "0\n" + " " AP_VMOV " 96(%rsp), " AP_V "1\n" + " " AP_VMOV " 128(%rsp), " AP_V "2\n" + " " AP_VMOV " 160(%rsp), " AP_V "3\n" + " " AP_VMOV " 192(%rsp), " AP_V "4\n" + " " AP_VMOV " 224(%rsp), " AP_V "5\n" + " " AP_VMOV " 256(%rsp), " AP_V "6\n" + " " AP_VMOV " 288(%rsp), " AP_V "7\n" + " " AP_VMOV " 320(%rsp), " AP_V "8\n" + " " AP_VMOV " 352(%rsp), " AP_V "9\n" + " " AP_VMOV " 384(%rsp), " AP_V "10\n" + " " AP_VMOV " 416(%rsp), " AP_V "11\n" + " " AP_VMOV " 448(%rsp), " AP_V "12\n" + " " AP_VMOV " 480(%rsp), " AP_V "13\n" + " " AP_VMOV " 512(%rsp), " AP_V "14\n" + " " AP_VMOV " 544(%rsp), " AP_V "15\n" " movq %rbp, %rsp\n" " popq %rbp\n" " popq %r11\n"