Mirrors/totk-pc-recomp
1
0 Fork 0
mirror of https://github.com/valentderah/totk-pc-recomp.git synced 2026-11-11 09:51:02 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
5da6c5bb8933679eb221ddcde98656d59f034df7
totk-pc-recomp/runtime/kernel.c
T

2126 lines
82 KiB
C
Raw Blame History

// Adapted from MacNativeGames games/totk/runtime/threads.c @ b75be8c (MIT, Copyright (c) 2026 MacNativeGames
// contributors; itself from d2r-mobile dm/host/hle_thread.c), d2r-mobile dm/host/thread.c @ 53e7c5b (MIT, Copyright
// (c) 2026 jhaizhou) and LAS Recomp runtime/kernel.c @ da35b83 (MIT, Copyright (c) 2026 Peiikai); the preemption of
// virtual cores from MacNativeGames threads.c @ 33e8fa8 (MIT, as above).
// Threads, synchronization, virtual cores and fibers of the Horizon kernel.
//
// One guest thread = one Windows thread with a 256 MiB reserved host stack (every guest BL is a host call) and its own
// Cpu. The SDK's nn::os runs as lifted code; only the kernel under it is the host. Every wait hangs on one global lock
// G with one condition variable per thread; wake order is the kernel's: lower priority value first, then first come.
// Semantics follow the public Horizon kernel (KConditionVariable, KAddressArbiter, KSynchronization).
#include <windows.h>
#define __USE_MINGW_SETJMP_NON_SEH // setjmp(b) = _setjmp(b, NULL): longjmp restores registers, no SEH unwinding
#include <setjmp.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cfg.h"
#include "host.h"
#include "kernel.h"
#include "platform/platform.h"
#define R_OK 0u
#define R_TIMEOUT KERN_R_TIMEOUT
#define R_CANCELLED KERN_R_CANCELLED
#define R_BAD_HANDLE KERN_R_BAD_HANDLE
#define R_BAD_STATE KERN_R_BAD_STATE
#define R_INVALID_ENUM KERN_R_INVALID_ENUM
#define WAITERS_BIT KERN_WAITERS_BIT
#define H_CUR_THREAD KERN_H_CURRENT_THREAD
#define MAX_H 4096
#define NCORE 3
#define HOST_STACK (256ull << 20) // reserved, committed on demand
#define TLR_SIZE 0x200
#define THREAD_OBJ_SIZE 0x400 // "current thread object" at TLR+0x1F8 (d2r-mobile thread.c)
#define TLS_SIZE 0x2000
#define MAIN_STACK (1ull << 20) // main.npdm META+0x1C
#define MAIN_PRIO 44 // META+0xE
#define MAIN_CORE 0 // META+0xF
#define FIBER_MAX 1024
enum { O_FREE, O_THREAD, O_EVENT_R, O_EVENT_W, O_SESSION = KERN_SESSION, O_TMEM = KERN_TMEM };
_Static_assert(O_EVENT_W < O_SESSION, "handle kinds");
enum { W_NONE = KERN_WAIT_NONE, W_LOCK = KERN_WAIT_LOCK, W_CONDVAR = KERN_WAIT_CONDVAR, W_ADDRESS = KERN_WAIT_ADDRESS,
W_SYNC = KERN_WAIT_SYNC };
// Scheduler accounting states (TOTK_TRACE=sched).
enum { S_OFF, S_RUN, S_CORE, S_HOST, S_SLEEP, S_SYNC, S_CV, S_LOCK, S_ADDR, S_N };
static const char *const s_name[S_N] = {"off", "run", "core", "host", "sleep", "sync", "cv", "lock", "addr"};
typedef struct Event { int signaled, refs; } Event;
typedef struct Fiber Fiber;
struct RtThread {
int preempt; // first member: rt_dispatch reads it (host.h kern_preempt_pending)
int host_depth; // second member: host calls and SVCs in progress, while kern_host_track (host.h)
uint32_t handle;
uint64_t id;
Cpu *cpu; // the thread's own context
Cpu *cur; // the context running now: cpu, or the current fiber's
uint64_t entry, arg, stack_top;
int prio, core;
uint64_t mask;
int started, exited, closed, cancel, is_main;
CONDITION_VARIABLE cv;
// current wait
int wkind, woke;
uint64_t waddr, wkey;
uint32_t wtag, wres;
uint64_t wseq; // registration order: equal priority wakes first come first
RtThread *next; // wait list
RtThread *all; // every thread
// virtual core
int cur_core; // core held while running guest code, -1 while blocked
uint64_t cseq;
RtThread *cnext;
uint8_t *tlr, *tls;
jmp_buf exit_jmp; // the thread start; ExitThread longjmps to it
// fibers
void *base_fiber; // the host thread as a host fiber, after the first switch
Fiber *fiber; // current guest fiber, NULL on the thread's own context
Fiber *leaving; // fiber just switched away from, released by whoever resumes next
int exit_pending; // ExitThread on a fiber: unwind once back on the thread's own context
HANDLE timer; // high-resolution waitable timer for SleepThread
// TOTK_TRACE=sched accounting (holding G)
DWORD tid; // host thread id
int st; // S_* state since st_since
uint64_t st_since, st_ns[S_N], nblk, npre, pre_since, cpu_last;
uint64_t wobj; // what the current wait is on: owner handle, key, address or first handle
uint32_t waker; // the thread whose signal ended the last wait (0: a host thread or a timeout)
uint64_t wake_at, grant_at; // when a wait was signalled and a core granted, until the thread runs again
uint64_t lw_ns, lw_n, lw_max, lg_ns, lg_n, lg_max, sl_n, sl_req, sl_act; // those latencies; SleepThread asked/took
RtThread *target; // as a core waiter: the owner core_kick asked to give its core up for it (holding G)
// TOTK_ASYNC_PREEMPT (holding G)
HANDLE hthr; // the host thread, for SuspendThread / Get/SetThreadContext; open while it may own a core
uint64_t ap_rip; // the interrupted host rip kern_ap_stub returns to
uint64_t ap_last; // the last redirect attempt on this owner
int ap_pending; // redirected to kern_ap_stub, which has not reached kern_ap_yield yet
// TOTK_CORE_DIAG (holding G, except dg_host_end, which only the thread itself writes)
uint64_t dg_since; // as a waiter: core wait start
int dg_cls; // as a waiter: DG_* class at the start of the wait
uint64_t dg_kick; // as an owner: the preempt request
uint64_t dg_host_end; // as an owner: the return from the host call or SVC it was in at the request
uint64_t dg_pc, dg_lr; // as an owner: guest pc and lr at the checkpoint that gave the core up
int dg_host_kick; // as an owner: in a host call or SVC at the request
uint32_t dg_svc; // the thread's latest SVC (TOTK_CORE_DIAG: what an owner in a host call is in)
struct DgRel { uint64_t kick, host_end, pc, lr; int host, pre, prio; uint32_t h; } dg_rel; // as a waiter: the
// owner that released
};
static SRWLOCK G = SRWLOCK_INIT;
static struct { int type; void *obj; } g_h[MAX_H];
static _Thread_local RtThread *tcur;
_Static_assert(offsetof(RtThread, preempt) == 0, "host.h kern_preempt_pending reads the first int");
_Static_assert(offsetof(RtThread, host_depth) == sizeof(int), "host.h kern_host_depth reads the second int");
static RtThread *g_all, *g_wait, *g_main;
static uint64_t g_next_id = 0x51, g_wseq, g_created;
static uint64_t g_main_stack;
static void lock(void) { AcquireSRWLockExclusive(&G); }
static void unlock(void) { ReleaseSRWLockExclusive(&G); }
uint64_t kern_now_ns(void) { return plat_now_ns(); }
// ---- TOTK_CRIT=<first>-<last frame> with TOTK_TRACE=sched: every state change and wake of the guest threads and
// every present over those frames, written to TOTK_CRIT_OUT (default crit.bin) at the last one, for
// tools/crit_path.py, which walks the chain of waits back from each present (holding G).
typedef struct { uint64_t ts, obj; uint32_t a, b; uint16_t kind, st; uint32_t pad; } CritEv;
enum { CRIT_STATE, CRIT_WAKE, CRIT_FRAME }; // a: thread; b: its waker (STATE), the waker (WAKE); st: new S_* state
#define CRIT_N (1u << 22) // 128 MiB, about 2000 fight frames
static CritEv *g_crit;
static uint32_t g_crit_n;
static int g_crit_on;
static uint64_t g_crit_lo, g_crit_hi;
static void crit_add(int kind, uint32_t a, uint32_t b, int st, uint64_t obj, uint64_t ts) {
if (g_crit_n < CRIT_N) g_crit[g_crit_n++] = (CritEv){ts, obj, a, b, (uint16_t)kind, (uint16_t)st, 0};
}
// ---- TOTK_TRACE=sched accounting (holding G)
#define RW_N 32
static RtThread *g_render; // the thread that presents (sched trace, core diag)
typedef struct { int st; uint32_t waker; uint64_t obj, ns; } WaitRec;
static WaitRec g_rw[RW_N], g_mw[RW_N]; // the render and main thread's waits by object and waker
static uint64_t g_pre_req, g_pre_n, g_pre_ns, g_pre_max, g_pre_stale; // preempt requests; releases after one and their latency
static uint64_t g_sync_calls, g_sync_blocks; // WaitSynchronization calls and the blocks inside them
static void wait_add(WaitRec *w, int st, uint64_t obj, uint32_t waker, uint64_t ns) {
for (int i = 0; i < RW_N; i++)
if (!w[i].ns || (w[i].st == st && w[i].obj == obj && w[i].waker == waker)) {
w[i].st = st;
w[i].obj = obj;
w[i].waker = waker;
w[i].ns += ns;
return;
}
}
// Close the current state's interval and enter st.
static void acct(RtThread *t, int st) {
if (!(rt_trace & RT_TRACE_SCHED)) return;
uint64_t now = kern_now_ns();
if (t->st_since) {
t->st_ns[t->st] += now - t->st_since;
if (t->st >= S_HOST && (t == g_render || t->is_main)) {
uint32_t waker = t->st == S_HOST || t->st == S_SLEEP ? 0 : t->waker;
wait_add(t == g_render ? g_rw : g_mw, t->st, t->wobj, waker, now - t->st_since);
}
}
if (g_crit_on) crit_add(CRIT_STATE, t->handle, t->waker, st, t->wobj, now);
t->st = st;
t->st_since = now;
}
static int st_of(int kind) {
return kind == W_LOCK ? S_LOCK : kind == W_CONDVAR ? S_CV : kind == W_ADDRESS ? S_ADDR : S_SYNC;
}
// ---- handles (caller holds G)
static uint32_t h_new(int type, void *obj) {
for (int i = 1; i < MAX_H; i++)
if (!g_h[i].type) {
g_h[i].type = type;
g_h[i].obj = obj;
return 0x1000 + i;
}
rt_fatal("kernel: handle table full (%d handles)", MAX_H);
}
static void *h_peek(uint32_t h, int type) {
if (h == H_CUR_THREAD && type == O_THREAD) return tcur;
unsigned i = h - 0x1000;
if (i == 0 || i >= MAX_H || g_h[i].type != type) return NULL;
return g_h[i].obj;
}
static void *h_get(uint32_t h, int type) {
void *o = h_peek(h, type);
if (!o) rt_log("kernel: invalid handle %#x (want kind %d)", h, type);
return o;
}
static RtThread *cur(uint32_t svc) {
if (!tcur) rt_fatal("kernel: svc %#x on a host thread (no guest thread)", svc);
return tcur;
}
// ---- wait primitives (holding G)
static void wait_unlink(RtThread *t) {
for (RtThread **p = &g_wait; *p; p = &(*p)->next)
if (*p == t) {
*p = t->next;
break;
}
t->next = NULL;
}
// The waiter to wake in kernel order. Lock waits match by lock address, condvar waits by key, arbiter waits by
// address.
static RtThread *pick(int kind, uint64_t a) {
RtThread *b = NULL;
for (RtThread *t = g_wait; t; t = t->next) {
if (t->wkind != kind || (kind == W_CONDVAR ? t->wkey : t->waddr) != a) continue;
if (!b || t->prio < b->prio || (t->prio == b->prio && t->wseq < b->wseq)) b = t;
}
return b;
}
static void wake(RtThread *t, uint32_t res) {
wait_unlink(t);
t->wkind = W_NONE;
t->woke = 1;
t->wres = res;
t->waker = tcur ? tcur->handle : 0;
if (rt_trace & RT_TRACE_SCHED) t->wake_at = kern_now_ns();
if (g_crit_on) crit_add(CRIT_WAKE, t->handle, t->waker, 0, 0, t->wake_at);
WakeConditionVariable(&t->cv);
}
static void wake_sync_waiters(void) {
for (RtThread *w = g_wait; w;) {
RtThread *n = w->next;
if (w->wkind == W_SYNC) wake(w, (uint32_t)-1);
w = n;
}
}
// SleepConditionVariableSRW counts milliseconds: round up (late is legal, early is not) and re-check the deadline
// against the performance counter, because the wait may also end early.
static uint32_t block_inner(RtThread *t, int kind, int64_t ns) {
t->wkind = kind;
t->woke = 0;
t->wseq = ++g_wseq;
t->next = g_wait;
g_wait = t;
if (ns == 0) {
wait_unlink(t);
t->wkind = W_NONE;
return R_TIMEOUT;
}
uint64_t now = kern_now_ns();
int forever = ns < 0 || (uint64_t)ns > UINT64_MAX - now;
uint64_t deadline = forever ? 0 : now + (uint64_t)ns;
while (!t->woke) {
if (forever) {
SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
continue;
}
now = kern_now_ns();
if (now >= deadline) {
wait_unlink(t);
t->wkind = W_NONE;
return R_TIMEOUT;
}
uint64_t ms = (deadline - now + 999999) / 1000000;
SleepConditionVariableSRW(&t->cv, &G, ms > 0xFFFFFFF0u ? 0xFFFFFFF0u : (DWORD)ms, 0);
}
if (t->wake_at) {
uint64_t d = kern_now_ns() - t->wake_at;
t->lw_ns += d;
t->lw_n++;
if (d > t->lw_max) t->lw_max = d;
t->wake_at = 0;
}
return t->wres;
}
// ---- virtual cores
// Horizon runs at most one thread per core, and TotK keys per-core data (allocators, job queues) by the current core
// number without locks. A guest thread therefore holds one of the three application cores while it runs guest code
// and gives it up only where Horizon would switch threads (waits, sleep, yield, exit, blocking host calls); a free
// core goes to the best waiting thread. A waiter better than a holder asks one holder to give its core up (core_kick),
// which it does at its next SVC or indirect call (kern_checkpoint, rt_dispatch), as Horizon runs the better thread at
// once; guest code without either keeps the core until it reaches one. TOTK_FREE_CORES=1 turns the gate off.
static RtThread *g_core_owner[NCORE], *g_core_wait;
static int g_cores_on = -1, g_held_max;
static void diag_init(void);
static void ap_init(void);
static int cores_enabled(void) {
if (g_cores_on < 0) {
g_cores_on = !cfg_bool("TOTK_FREE_CORES"); // "0" keeps the gate, like unset
diag_init();
ap_init();
}
return g_cores_on;
}
static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) & 1); }
// ---- asynchronous preemption (TOTK_ASYNC_PREEMPT, on unless 0)
// Adapted from MacNativeGames games/totk/runtime/threads.c @ 33e8fa8 (8bf0893, PR #488; MIT, Copyright (c) 2026
// MacNativeGames contributors): Horizon gives a core to a strictly better thread the moment that thread is ready.
// Here a better waiter only marks the owner, which gives the core up at its next SVC or indirect call; an owner in a
// stretch of game code with neither keeps it (TOTK_CORE_DIAG: 0.5-1.1 ms of thread-time per frame, waits to 16 ms and
// more). With it on, three parts, all of which MacNativeGames found necessary:
// - The waiter interrupts the owner (SuspendThread, GetThreadContext) when it has waited AP_FIRST_NS, then every
// millisecond: if the owner's host rip is in the lifted modules' code (g_gen_lo..g_gen_hi) and its host depth is 0
// (not in a host call or SVC, nor in game code a host call made), it holds no host lock, so its rip is moved to
// kern_ap_stub, which saves every volatile register, gives the core up like a checkpoint (kern_ap_yield) and jumps
// back. Otherwise it is only marked, as before.
// - Priority inheritance (eff_prio): a thread owning a guest mutex counts at the priority of the best thread blocked on
// it, for granting, kicking, preempting and yielding. Without it MacNativeGames' game stopped at its first
// "Continue" (a better thread took the core of the low-priority owner of a mutex it then waited for).
// - Horizon's preemption disable: the SDK's spin locks (1.2.1 sdk f_70051242a8, f_7005124360 and siblings) count
// TLR+0x100 (disable_count) while they try or hold the lock, and after clearing it call SVC 0x36 when TLR+0x102
// (interrupt_flag) is set. Such an owner is not preempted, at a checkpoint either: interrupt_flag is set and the
// switch happens at that SVC. Preempting it anyway hung MacNativeGames' game at "Continue" (the lock holder lost its
// core to a thread that then spun on that lock).
// TOTK_ASYNC_PREEMPT=0 keeps the old scheduling exactly: raw priorities, no interruption, no disable_count; =2 keeps
// inheritance and the preemption disable without interrupting owners.
#define AP_FIRST_NS 250000ull // the waiter spins this long before the first interruption (most waits end sooner)
#define AP_GAP_NS 1000000ull // and interrupts the same owner at most this often
static int g_ap = -1, g_ap_int; // g_ap_int: interrupt owners (TOTK_ASYNC_PREEMPT=2: inheritance and disable only)
static int g_ap_log; // redirects logged so far (AP_LOG_N)
#define AP_LOG_N 40
static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start
static uint64_t g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok;
void kern_ap_stub(void);
static void ap_init(void) {
if (g_ap >= 0) return;
const char *e = cfg_value("TOTK_ASYNC_PREEMPT");
g_ap = strcmp(e, "0") != 0;
g_ap_int = g_ap && strcmp(e, "2") != 0;
if (g_ap) {
RtModule m[16];
int n = rt_linked_modules(m, 16);
uintptr_t lo = UINTPTR_MAX, hi = 0;
for (int i = 0; i < n; i++)
for (uint64_t k = 0; k < m[i].count; k++) {
uintptr_t f = (uintptr_t)m[i].fns[k];
if (f < lo) lo = f;
if (f > hi) hi = f;
}
const uintptr_t host[] = {(uintptr_t)kern_ap_stub, (uintptr_t)kern_checkpoint, (uintptr_t)rt_svc,
(uintptr_t)rt_dispatch, (uintptr_t)rt_call};
int bad = !n || lo >= hi;
for (unsigned i = 0; i < sizeof host / sizeof *host; i++) bad |= host[i] >= lo && host[i] <= hi;
if (bad) {
rt_log("kernel: asynchronous preemption off: the lifted code range %#llx-%#llx holds host code",
(unsigned long long)lo, (unsigned long long)hi);
g_ap = 0;
} else {
g_gen_lo = lo;
g_gen_hi = hi;
kern_host_track = 1;
}
}
rt_log("kernel: asynchronous preemption of virtual cores %s, priority inheritance and preemption disable %s; lifted "
"code exe+%#llx-exe+%#llx", g_ap_int ? "on" : "off", g_ap ? "on (TOTK_ASYNC_PREEMPT=0 turns all off)" : "off",
(unsigned long long)(g_gen_lo ? g_gen_lo - (uintptr_t)GetModuleHandleW(NULL) : 0),
(unsigned long long)(g_gen_hi ? g_gen_hi - (uintptr_t)GetModuleHandleW(NULL) : 0));
}
// Horizon's priority inheritance: o counts at the priority of the best thread blocked on a guest mutex it owns (the
// owner is read from the mutex word, so it follows hand-offs). Holding G.
static int eff_prio(const RtThread *o) {
int p = o->prio;
for (const RtThread *w = g_wait; w; w = w->next)
if (w->wkind == W_LOCK && w->prio < p && w->waddr &&
(__atomic_load_n((const uint32_t *)(uintptr_t)w->waddr, __ATOMIC_RELAXED) & ~WAITERS_BIT) == o->handle)
p = w->prio;
return p;
}
// The priority the virtual cores go by: raw with TOTK_ASYNC_PREEMPT=0, as before.
static int prio_of(const RtThread *t) { return g_ap > 0 ? eff_prio(t) : t->prio; }
// TLR+0x100 disable_count, +0x102 interrupt_flag (any thread may read them; the flag is set from outside as the
// kernel does).
static volatile uint16_t *ap_tlr(const RtThread *t) { return (volatile uint16_t *)(t->tlr + 0x100); }
static int ap_disabled(const RtThread *t) { return g_ap > 0 && ap_tlr(t)[0]; }
// kern_ap_stub: entered in place of the interrupted instruction of top-level game code. It keeps every register the
// game code may have live (volatile GPRs, flags, YMM0-15, MXCSR; the C calls preserve the rest), gives the core up
// in kern_ap_yield and jumps back to kern_ap_rip(), whose value it first stores in the return slot it reserved at
// entry, so a stack walk sees the interrupted function as its caller. No ret: the jump keeps call/return pairs
// balanced. The waiter never writes the owner's stack.
__asm__(".text\n"
".globl kern_ap_stub\n"
".def kern_ap_stub; .scl 2; .type 32; .endef\n"
".p2align 4\n"
"kern_ap_stub:\n"
".seh_proc kern_ap_stub\n"
" leaq -8(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq
".seh_stackalloc 8\n"
" pushfq\n"
".seh_stackalloc 8\n"
" pushq %rax\n"
".seh_pushreg %rax\n"
" pushq %rcx\n"
".seh_pushreg %rcx\n"
" pushq %rdx\n"
".seh_pushreg %rdx\n"
" pushq %r8\n"
".seh_pushreg %r8\n"
" pushq %r9\n"
".seh_pushreg %r9\n"
" pushq %r10\n"
".seh_pushreg %r10\n"
" pushq %r11\n"
".seh_pushreg %r11\n"
" pushq %rbp\n"
".seh_pushreg %rbp\n"
" movq %rsp, %rbp\n"
".seh_setframe %rbp, 0\n"
".seh_endprologue\n"
" andq $-32, %rsp\n"
" subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15
" vmovdqa %ymm0, 64(%rsp)\n"
" vmovdqa %ymm1, 96(%rsp)\n"
" vmovdqa %ymm2, 128(%rsp)\n"
" vmovdqa %ymm3, 160(%rsp)\n"
" vmovdqa %ymm4, 192(%rsp)\n"
" vmovdqa %ymm5, 224(%rsp)\n"
" vmovdqa %ymm6, 256(%rsp)\n"
" vmovdqa %ymm7, 288(%rsp)\n"
" vmovdqa %ymm8, 320(%rsp)\n"
" vmovdqa %ymm9, 352(%rsp)\n"
" vmovdqa %ymm10, 384(%rsp)\n"
" vmovdqa %ymm11, 416(%rsp)\n"
" vmovdqa %ymm12, 448(%rsp)\n"
" vmovdqa %ymm13, 480(%rsp)\n"
" vmovdqa %ymm14, 512(%rsp)\n"
" vmovdqa %ymm15, 544(%rsp)\n"
" stmxcsr 32(%rsp)\n"
" movl $0x1f80, 36(%rsp)\n"
" ldmxcsr 36(%rsp)\n"
" cld\n"
" vzeroupper\n"
" call kern_ap_rip\n"
" movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq
" call kern_ap_yield\n"
" ldmxcsr 32(%rsp)\n"
" vmovdqa 64(%rsp), %ymm0\n"
" vmovdqa 96(%rsp), %ymm1\n"
" vmovdqa 128(%rsp), %ymm2\n"
" vmovdqa 160(%rsp), %ymm3\n"
" vmovdqa 192(%rsp), %ymm4\n"
" vmovdqa 224(%rsp), %ymm5\n"
" vmovdqa 256(%rsp), %ymm6\n"
" vmovdqa 288(%rsp), %ymm7\n"
" vmovdqa 320(%rsp), %ymm8\n"
" vmovdqa 352(%rsp), %ymm9\n"
" vmovdqa 384(%rsp), %ymm10\n"
" vmovdqa 416(%rsp), %ymm11\n"
" vmovdqa 448(%rsp), %ymm12\n"
" vmovdqa 480(%rsp), %ymm13\n"
" vmovdqa 512(%rsp), %ymm14\n"
" vmovdqa 544(%rsp), %ymm15\n"
" movq %rbp, %rsp\n"
" popq %rbp\n"
" popq %r11\n"
" popq %r10\n"
" popq %r9\n"
" popq %r8\n"
" popq %rdx\n"
" popq %rcx\n"
" popq %rax\n"
" popfq\n"
" leaq 8(%rsp), %rsp\n"
" jmpq *-8(%rsp)\n"
".seh_endproc\n");
enum { AP_NONE, AP_RETRY, AP_SENT };
// The waiter w interrupts its target owner if that runs top-level game code (holding G; the owner is suspended only
// between SuspendThread and ResumeThread, during which nothing here takes a lock).
static int ap_try(RtThread *w) {
RtThread *o = w->target;
if (!o || o->cur_core < 0 || !o->preempt) return AP_NONE;
if (!o->hthr || o->ap_pending) return AP_RETRY;
volatile uint16_t *tl = ap_tlr(o);
if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0) {
g_ap_host++;
return AP_RETRY;
}
if (tl[0]) { // preemption disabled: SVC 0x36 switches
tl[1] = 1;
g_ap_deferred++;
return AP_RETRY;
}
uint64_t now = kern_now_ns();
if (o->ap_last && now - o->ap_last < AP_GAP_NS) return AP_RETRY;
o->ap_last = now;
if (SuspendThread(o->hthr) == (DWORD)-1) {
g_ap_fail++;
return AP_RETRY;
}
CONTEXT ctx;
memset(&ctx, 0, sizeof ctx);
ctx.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; // set back whole, so the kernel restores every GPR
int r = AP_RETRY;
if (!GetThreadContext(o->hthr, &ctx))
g_ap_fail++;
else if (ctx.Rip < g_gen_lo || ctx.Rip > g_gen_hi)
g_ap_pc++;
else if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0)
g_ap_host++;
else if (tl[0]) {
tl[1] = 1;
g_ap_deferred++;
} else {
o->ap_rip = ctx.Rip;
o->ap_pending = 1;
ctx.Rip = (DWORD64)(uintptr_t)kern_ap_stub;
if (SetThreadContext(o->hthr, &ctx)) {
g_ap_sent++;
r = AP_SENT;
} else {
o->ap_pending = 0;
g_ap_fail++;
}
}
ResumeThread(o->hthr);
if (r == AP_SENT && g_ap_log < AP_LOG_N) {
g_ap_log++;
rt_log("kernel: interrupt %#x p%d at exe+%#llx rsp %#llx for %#x p%d", o->handle, o->prio,
(unsigned long long)(o->ap_rip - (uintptr_t)GetModuleHandleW(NULL)), (unsigned long long)ctx.Rsp,
w->handle, w->prio);
}
return r;
}
// ---- TOTK_CORE_DIAG=1: why ready threads wait for a virtual core, and for how long
// Each core wait is split into classes: the owner asked to give the core up ran game code (DG_GAME, the part an
// asynchronous preemption would remove), was in a host call or SVC that kept the core (DG_HOST), every owner the waiter
// may displace was of better or equal priority (DG_EQ: Horizon waits too), or the worse owners were claimed by better
// waiters (DG_NOCORE). Per 60 presents: time per frame and a wait-length histogram per class, the owners that cost
// most; a line per present over 25 ms; a line per wait with over 2 ms of DG_GAME (where the owner reached its
// checkpoint), and one host RIP sample of the owner per such wait (exe RVA, for totk.map).
enum { DG_GAME, DG_HOST, DG_EQ, DG_NOCORE, DG_N };
static const char *const dg_name[DG_N] = {"game", "host", "eq", "nocore"};
#define DG_NB 8 // wait lengths: < 0.05, 0.25, 1, 2, 4, 8, 16 ms, longer
static const uint64_t dg_edge[DG_NB - 1] = {50000, 250000, 1000000, 2000000, 4000000, 8000000, 16000000};
#define DG_LONG_LINES 400
#define DG_SAMPLES 600
#define DG_OWN 32
int kern_host_track;
static int g_diag = -1;
typedef struct {
uint64_t ns[DG_N], n[DG_N], hist[DG_N][DG_NB]; // this window
struct { uint32_t h; int prio, cls; uint64_t ns; } own[DG_OWN]; // this window: owners by class
uint64_t long_frames, long_game_frames;
} DgWindow;
static DgWindow g_dgw;
static uint64_t g_dgf[DG_N], g_dgfr[DG_N]; // this frame: all threads, the render thread
static uint64_t g_dg_samples, g_dg_lines, g_sync36, g_sync36_flag;
static void diag_init(void) {
if (g_diag >= 0) return;
const char *e = cfg_get("TOTK_CORE_DIAG");
g_diag = e && *e && strcmp(e, "0");
if (g_diag) {
kern_host_track = 1;
rt_log("core-diag: on (TOTK_CORE_DIAG)");
}
}
int kern_core_diag_on(void) { return g_diag > 0; }
// A tracked host call or SVC returned to depth 0 with a preempt request pending (host.h); on the thread itself.
void kern_host_left(void *thread) {
RtThread *t = thread;
if (g_diag > 0 && t->dg_kick && t->dg_host_kick && !t->dg_host_end) t->dg_host_end = kern_now_ns();
}
static int dg_in_host(const RtThread *o) { return __atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0; }
static void dg_add(const RtThread *w, int cls, uint64_t ns, uint32_t oh, int oprio) {
if (!ns) return;
g_dgw.ns[cls] += ns;
g_dgf[cls] += ns;
if (w == g_render) g_dgfr[cls] += ns;
if (!oh) return;
int i = 0;
for (; i < DG_OWN && g_dgw.own[i].ns; i++)
if (g_dgw.own[i].h == oh && g_dgw.own[i].cls == cls) break;
if (i == DG_OWN) return;
g_dgw.own[i].h = oh;
g_dgw.own[i].prio = oprio;
g_dgw.own[i].cls = cls;
g_dgw.own[i].ns += ns;
}
// The waiter's class at the start of its wait, after core_kick claimed (or did not claim) an owner for it.
static int dg_class0(const RtThread *t) {
if (t->target) return dg_in_host(t->target) ? DG_HOST : DG_GAME;
for (int i = 0; i < NCORE; i++)
if (core_allowed(t, i) && g_core_owner[i] && g_core_owner[i]->prio > t->prio) return DG_NOCORE;
return DG_EQ;
}
// The waiter got a core: split [dg_since, now] by what the owner that released it did after its preempt request.
static void dg_done(RtThread *t) {
uint64_t now = kern_now_ns(), t0 = t->dg_since, w = now - t0;
t->dg_since = 0;
if (!t0) return;
const struct DgRel *r = &t->dg_rel;
uint64_t part[DG_N] = {0};
uint32_t oh = t->target ? t->target->handle : r->h;
int oprio = t->target ? t->target->prio : r->prio;
if (r->pre && r->kick) {
uint64_t k = r->kick > t0 ? (r->kick < now ? r->kick : now) : t0;
part[t->dg_cls] += k - t0;
if (r->host) {
uint64_t he = r->host_end > k && r->host_end < now ? r->host_end : now;
part[DG_HOST] += he - k;
part[DG_GAME] += now - he;
} else
part[DG_GAME] += now - k;
oh = r->h;
oprio = r->prio;
} else
part[t->dg_cls] += w;
int top = 0;
for (int c = 0; c < DG_N; c++) {
dg_add(t, c, part[c], oh, oprio);
if (part[c] > part[top]) top = c;
}
int b = 0;
while (b < DG_NB - 1 && w >= dg_edge[b]) b++;
g_dgw.n[top]++;
g_dgw.hist[top][b]++;
if ((part[DG_GAME] >= 2000000 || part[DG_HOST] >= 2000000) && g_dg_lines < DG_LONG_LINES) {
g_dg_lines++;
char a[128], l[128];
rt_log("core-diag: wait %.2f ms (game %.2f host %.2f): %#x p%d mask %llx; released by %#x p%d%s at %s lr %s",
w / 1e6, part[DG_GAME] / 1e6, part[DG_HOST] / 1e6, t->handle, t->prio, (unsigned long long)t->mask, r->h,
r->prio, r->pre ? " (preempted)" : "", r->pc ? ldr_describe(r->pc, a, sizeof a) : "-",
r->lr ? ldr_describe(r->lr, l, sizeof l) : "-");
}
t->target = NULL;
memset(&t->dg_rel, 0, sizeof t->dg_rel);
}
// Once per long DG_GAME or DG_HOST wait: where the owner's host thread runs (module and offset), its latest SVC and
// its guest lr. Holding G; the owner is resumed before anything that could take a lock it holds.
static void dg_sample(RtThread *t) {
RtThread *o = t->target;
if (!o || o->cur_core < 0 || !o->tid || g_dg_samples >= DG_SAMPLES) return;
HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, o->tid);
if (!h) return;
CONTEXT ctx;
memset(&ctx, 0, sizeof ctx);
ctx.ContextFlags = CONTEXT_CONTROL;
int depth = 0, ok = 0;
if (SuspendThread(h) != (DWORD)-1) {
ok = GetThreadContext(h, &ctx);
depth = __atomic_load_n(&o->host_depth, __ATOMIC_RELAXED);
ResumeThread(h);
}
CloseHandle(h);
if (!ok) return;
g_dg_samples++;
HMODULE m = NULL;
wchar_t mw[MAX_PATH] = L"?";
if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
(LPCWSTR)(uintptr_t)ctx.Rip, &m) && m)
GetModuleFileNameW(m, mw, MAX_PATH);
const wchar_t *mn = wcsrchr(mw, L'\\') ? wcsrchr(mw, L'\\') + 1 : mw;
char lr[128];
rt_log("core-diag: sample owner %#x p%d depth %d rip %ls+%#llx, last svc %#x, guest lr %s (waiter %#x p%d, %s, "
"%.2f ms)", o->handle, o->prio, depth, mn, (unsigned long long)(ctx.Rip - (uintptr_t)m), o->dg_svc,
o->cur ? ldr_describe(o->cur->x[30], lr, sizeof lr) : "-", t->handle, t->prio,
t->dg_cls == DG_HOST ? "host" : "game", (kern_now_ns() - t->dg_since) / 1e6);
}
// perf_present, every present: a line for a present over 25 ms, the 60-present summary.
void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns) {
if (g_ap > 0 && frame && frame % 600 == 0) { // TOTK_ASYNC_PREEMPT counters since the start
lock();
uint64_t v[8] = {g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok, g_sync36};
unlock();
rt_log("kernel: frame %llu: asynchronous preemption: %llu interrupted, %llu cores given up there; refused: "
"%llu in a host call, %llu outside game code, %llu preemption disabled, %llu failed; %llu waits ended "
"in the first spin; SVC 0x36 %llu",
(unsigned long long)frame, (unsigned long long)v[0], (unsigned long long)v[1], (unsigned long long)v[2],
(unsigned long long)v[3], (unsigned long long)v[4], (unsigned long long)v[5], (unsigned long long)v[6],
(unsigned long long)v[7]);
}
if (g_diag <= 0) return;
DgWindow w;
uint64_t f[DG_N], fr[DG_N], s36, s36f;
lock();
g_render = tcur;
memcpy(f, g_dgf, sizeof f);
memcpy(fr, g_dgfr, sizeof fr);
memset(g_dgf, 0, sizeof g_dgf);
memset(g_dgfr, 0, sizeof g_dgfr);
int lng = interval_ns > 25000000ull;
if (lng) {
g_dgw.long_frames++;
if (f[DG_GAME] > 1000000) g_dgw.long_game_frames++;
}
int rep = frame && frame % 60 == 0;
if (rep) {
w = g_dgw;
memset(&g_dgw, 0, sizeof g_dgw);
}
s36 = g_sync36;
s36f = g_sync36_flag;
unlock();
if (lng)
rt_log("core-diag: frame %llu: %.1f ms; core wait game %.2f host %.2f eq %.2f nocore %.2f ms; render thread game "
"%.2f host %.2f eq %.2f nocore %.2f ms",
(unsigned long long)frame, interval_ns / 1e6, f[0] / 1e6, f[1] / 1e6, f[2] / 1e6, f[3] / 1e6,
fr[0] / 1e6, fr[1] / 1e6, fr[2] / 1e6, fr[3] / 1e6);
if (!rep) return;
char s[1024];
int o = 0;
for (int c = 0; c < DG_N; c++) {
o += snprintf(s + o, sizeof s - o, "%s %.3f ms/frame %llu waits [", dg_name[c], w.ns[c] / 60e6,
(unsigned long long)w.n[c]);
for (int b = 0; b < DG_NB; b++)
o += snprintf(s + o, sizeof s - o, "%s%llu", b ? " " : "", (unsigned long long)w.hist[c][b]);
o += snprintf(s + o, sizeof s - o, "]%s", c < DG_N - 1 ? "; " : "");
}
rt_log("core-diag: frames %llu-%llu: %s; long frames %llu (game wait > 1 ms in %llu); SVC 0x36 %llu (flag set %llu)",
(unsigned long long)frame - 59, (unsigned long long)frame, s, (unsigned long long)w.long_frames,
(unsigned long long)w.long_game_frames, (unsigned long long)s36, (unsigned long long)s36f);
o = 0;
for (int k = 0; k < 6; k++) {
int bi = -1;
for (int i = 0; i < DG_OWN; i++)
if (w.own[i].ns && (bi < 0 || w.own[i].ns > w.own[bi].ns)) bi = i;
if (bi < 0) break;
o += snprintf(s + o, sizeof s - o, " %#x p%d %s %.3f;", w.own[bi].h, w.own[bi].prio, dg_name[w.own[bi].cls],
w.own[bi].ns / 60e6);
w.own[bi].ns = 0;
}
if (o) rt_log("core-diag: owners (ms/frame):%s", s);
}
static void core_count(void) {
int n = 0;
for (int i = 0; i < NCORE; i++) n += g_core_owner[i] != NULL;
if (n > g_held_max) g_held_max = n;
}
static struct DgRel g_dg_rel; // TOTK_CORE_DIAG: the owner core_put is releasing, for core_grant
static void core_grant(int c) {
RtThread *b = NULL, **bp = NULL;
int bpri = 0;
for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext) {
if (!core_allowed(*p, c)) continue;
int pr = prio_of(*p);
if (!b || pr < bpri || (pr == bpri && (*p)->cseq < b->cseq)) {
b = *p;
bp = p;
bpri = pr;
}
}
if (!b) return;
*bp = b->cnext;
b->cnext = NULL;
b->cur_core = c;
g_core_owner[c] = b;
core_count();
if (g_diag > 0) b->dg_rel = g_dg_rel;
if (rt_trace & RT_TRACE_SCHED) b->grant_at = kern_now_ns();
WakeConditionVariable(&b->cv);
}
static void core_put(RtThread *t) {
if (t->cur_core < 0) return;
int c = t->cur_core;
if (t->preempt && t->pre_since) { // sched trace: time from the preempt request to this release
uint64_t d = kern_now_ns() - t->pre_since;
g_pre_n++;
g_pre_ns += d;
if (d > g_pre_max) g_pre_max = d;
t->npre++;
}
t->pre_since = 0;
if (g_diag > 0) {
g_dg_rel = (struct DgRel){.kick = t->dg_kick, .host_end = t->dg_host_end, .pc = t->dg_pc, .lr = t->dg_lr,
.host = t->dg_host_kick, .pre = t->preempt, .prio = t->prio, .h = t->handle};
t->dg_kick = t->dg_host_end = t->dg_pc = t->dg_lr = 0;
t->dg_host_kick = 0;
}
t->cur_core = -1;
t->preempt = 0;
g_core_owner[c] = NULL;
core_grant(c);
}
// Whether a waiter better than t may run on core c.
static int core_better_waiter(const RtThread *t, int c) {
int tp = prio_of(t);
for (const RtThread *w = g_core_wait; w; w = w->cnext)
if (core_allowed(w, c) && prio_of(w) < tp) return 1;
return 0;
}
// Asks holders to give their cores up, one holder per waiter: waiters best first, each claims the worst holder worse
// than itself on a core it may run on that no better waiter claimed (a request already pending is reused). Asking
// every worse holder made several drop their cores for one waiter; the spare cores went to worse waiters, which asked
// again.
static void core_kick(void) {
RtThread *w[64];
int wp[64], op[NCORE], n = 0, claimed = 0;
for (int i = 0; i < NCORE; i++) op[i] = g_core_owner[i] ? prio_of(g_core_owner[i]) : 0;
for (RtThread *p = g_core_wait; p && n < 64; p = p->cnext) {
int i = n++, pp = prio_of(p);
for (; i > 0 && (wp[i - 1] > pp || (wp[i - 1] == pp && w[i - 1]->cseq > p->cseq)); i--) {
w[i] = w[i - 1];
wp[i] = wp[i - 1];
}
w[i] = p;
wp[i] = pp;
}
for (int k = 0; k < n && claimed != (1 << NCORE) - 1; k++) {
int v = -1;
for (int i = 0; i < NCORE; i++) {
RtThread *o = g_core_owner[i];
if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || op[i] <= wp[k]) continue;
RtThread *b = v < 0 ? NULL : g_core_owner[v];
if (!b || op[i] > op[v] || (op[i] == op[v] && o->preempt && !b->preempt)) v = i;
}
if (v < 0) continue;
claimed |= 1 << v;
RtThread *o = g_core_owner[v];
w[k]->target = o;
if (o->preempt) continue;
if (rt_trace & RT_TRACE_SCHED) {
o->pre_since = kern_now_ns();
g_pre_req++;
}
if (g_diag > 0) {
o->dg_kick = kern_now_ns();
o->dg_host_kick = dg_in_host(o);
o->dg_host_end = 0;
}
__atomic_store_n(&o->preempt, 1, __ATOMIC_RELEASE);
}
}
static void core_take(RtThread *t) {
if (t->cur_core >= 0) return;
if (!cores_enabled()) {
acct(t, S_RUN);
return;
}
if (!(t->mask & ((1u << NCORE) - 1)))
rt_fatal("kernel: thread %#x core mask %#llx has no application core", t->handle, (unsigned long long)t->mask);
int c = t->core >= 0 && t->core < NCORE && core_allowed(t, t->core) && !g_core_owner[t->core] ? t->core : -1;
for (int i = 0; c < 0 && i < NCORE; i++)
if (core_allowed(t, i) && !g_core_owner[i]) c = i;
if (c >= 0) {
t->cur_core = c;
g_core_owner[c] = t;
core_count();
acct(t, S_RUN);
return;
}
acct(t, S_CORE);
t->cseq = ++g_wseq;
t->cnext = g_core_wait;
g_core_wait = t;
t->target = NULL;
if (g_diag > 0) {
t->dg_since = kern_now_ns();
memset(&t->dg_rel, 0, sizeof t->dg_rel);
}
core_kick();
if (g_diag > 0) t->dg_cls = dg_class0(t);
if (g_ap_int && t->target && t->cur_core < 0) { // most waits end within microseconds: spin before interrupting
uint64_t end = kern_now_ns() + AP_FIRST_NS;
unlock();
while (__atomic_load_n(&t->cur_core, __ATOMIC_RELAXED) < 0 && kern_now_ns() < end) YieldProcessor();
lock();
if (t->cur_core >= 0) g_ap_spin_ok++;
}
int sampled = 0;
while (t->cur_core < 0) {
DWORD ms = INFINITE;
if (g_ap_int && ap_try(t) != AP_NONE) ms = 1; // the owner keeps its core: again in a millisecond
int sample = g_diag > 0 && !sampled && (t->dg_cls == DG_GAME || t->dg_cls == DG_HOST);
if (sample && ms == INFINITE) ms = 2;
SleepConditionVariableSRW(&t->cv, &G, ms, 0);
if (sample && t->cur_core < 0 && kern_now_ns() - t->dg_since >= 2000000) {
sampled = 1;
dg_sample(t);
}
}
if (g_diag > 0) dg_done(t);
if (t->grant_at) {
uint64_t d = kern_now_ns() - t->grant_at;
t->lg_ns += d;
t->lg_n++;
if (d > t->lg_max) t->lg_max = d;
t->grant_at = 0;
}
t->target = NULL;
acct(t, S_RUN);
}
static uint32_t block(RtThread *t, int kind, int64_t ns) {
t->waker = 0;
if (ns == 0) return block_inner(t, kind, ns);
core_put(t);
acct(t, st_of(kind));
t->nblk++;
uint32_t r = block_inner(t, kind, ns);
core_take(t);
return r;
}
int kern_core_away(void) {
RtThread *t = tcur;
if (!t || t->cur_core < 0) return 0;
lock();
core_put(t);
acct(t, S_HOST);
if (rt_trace & RT_TRACE_SCHED) // the caller as an exe RVA (totk.map)
t->wobj = (uintptr_t)__builtin_return_address(0) - (uintptr_t)GetModuleHandleW(NULL);
unlock();
return 1;
}
void kern_core_back(int *away) {
RtThread *t = tcur;
if (!*away || !t) return;
lock();
core_take(t);
unlock();
*away = 0;
}
// A pending preempt request, at an SVC or an indirect call. The core goes over only if a better thread still waits
// for it, so core_put grants it to a waiter better than t; a stale request (that waiter got another core) is dropped.
void kern_checkpoint(void) {
RtThread *t = tcur;
if (!t || !__atomic_load_n(&t->preempt, __ATOMIC_RELAXED)) return;
lock();
if (g_diag > 0 && t->cur) {
t->dg_pc = t->cur->pc;
t->dg_lr = t->cur->x[30];
}
int better = t->cur_core >= 0 && core_better_waiter(t, t->cur_core);
if (better && ap_disabled(t)) { // preemption disabled: the switch comes at SVC 0x36
ap_tlr(t)[1] = 1;
t->preempt = 0;
g_ap_deferred++;
} else if (better) {
core_put(t); // clears preempt (after the sched trace reads it)
core_take(t);
} else {
t->preempt = 0;
t->pre_since = 0;
t->dg_kick = t->dg_host_end = 0;
t->dg_host_kick = 0;
g_pre_stale++;
}
unlock();
}
// kern_ap_stub, on the interrupted owner: where to jump back to, then the core given up as at a checkpoint.
uint64_t kern_ap_rip(void) { return tcur->ap_rip; }
void kern_ap_yield(void) {
RtThread *t = tcur;
lock();
t->ap_pending = 0;
if (t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) {
if (g_diag > 0) {
t->dg_pc = t->cur->pc;
t->dg_lr = t->cur->x[30];
}
g_ap_taken++;
core_put(t);
core_take(t);
}
unlock();
}
// SVC 0x36 SynchronizePreemptionState: a preemption deferred by disable_count happens here.
static void ap_sync(RtThread *t) {
lock();
if (g_ap > 0 && t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) {
core_put(t);
core_take(t);
}
unlock();
}
void kern_core_stats(int *held_now, int *held_max) {
lock();
int n = 0;
for (int i = 0; i < NCORE; i++) n += g_core_owner[i] != NULL;
*held_now = n;
*held_max = g_held_max;
unlock();
}
void kern_core_stats_reset(void) {
lock();
g_held_max = 0;
core_count();
unlock();
}
// TOTK_TRACE=sched: every 120 presents, the time split of the busiest threads (running, waiting for a core, host
// calls) and of the render thread, the render and the main thread's longest waits by object and by the thread whose
// signal ended them, and the preemption latency.
#define REP_N 8
// At each present (holding G): starts recording at TOTK_CRIT's first frame, stops and writes the file at its last.
// File: "CRIT", thread count, event count (u32 each); per thread handle, priority, flags (1 main, 2 presents), host
// tid (u32 each); then the CritEv records.
static void crit_frame(RtThread *self, uint64_t frame) {
static int parsed;
if (!parsed) {
parsed = 1;
const char *e = cfg_get("TOTK_CRIT");
unsigned long long lo, hi;
if (e && sscanf(e, "%llu-%llu", &lo, &hi) == 2 && lo < hi && (g_crit = malloc(CRIT_N * sizeof *g_crit))) {
g_crit_lo = lo;
g_crit_hi = hi;
rt_log("crit: recording frames %llu-%llu", lo, hi);
}
}
if (!g_crit || frame < g_crit_lo || frame > g_crit_hi) return;
g_crit_on = 1;
crit_add(CRIT_FRAME, self ? self->handle : 0, 0, 0, frame, kern_now_ns());
if (frame < g_crit_hi) return;
g_crit_on = 0;
const char *path = cfg_get("TOTK_CRIT_OUT");
FILE *f = fopen(path && *path ? path : "crit.bin", "wb");
if (!f) {
rt_log("crit: cannot write %s", path && *path ? path : "crit.bin");
return;
}
uint32_t nt = 0;
for (RtThread *t = g_all; t; t = t->all) nt++;
fwrite("CRIT", 1, 4, f);
fwrite(&nt, 4, 1, f);
fwrite(&g_crit_n, 4, 1, f);
for (RtThread *t = g_all; t; t = t->all) {
uint32_t row[4] = {t->handle, (uint32_t)t->prio, (uint32_t)(t->is_main | (t == g_render) << 1), t->tid};
fwrite(row, 4, 4, f);
}
fwrite(g_crit, sizeof *g_crit, g_crit_n, f);
fclose(f);
rt_log("crit: %u events, %u threads, frames %llu-%llu%s", g_crit_n, nt, (unsigned long long)g_crit_lo,
(unsigned long long)g_crit_hi, g_crit_n >= CRIT_N ? " (buffer full: the last frames are incomplete)" : "");
}
void kern_sched_report(uint64_t frame) {
static uint64_t last;
typedef struct {
uint32_t h;
int prio, render;
uint64_t mask, entry, ns[S_N], nblk, npre, cpu, lw_ns, lw_n, lw_max, lg_ns, lg_n, lg_max, sl_n, sl_req, sl_act;
} Row;
Row rows[REP_N + 1];
WaitRec rw[2][4] = {0}; // the render and the main thread's longest waits
int nr = 0;
RtThread *self = tcur;
lock();
if (g_render != self) {
g_render = self;
memset(g_rw, 0, sizeof g_rw);
}
crit_frame(self, frame);
if (frame % 120) {
unlock();
return;
}
uint64_t now = kern_now_ns(), span = last ? now - last : 0;
last = now;
RtThread *top[REP_N + 1] = {0};
uint64_t busy[REP_N + 1] = {0}, all_run = 0, all_core = 0;
for (RtThread *t = g_all; t; t = t->all) {
acct(t, t->st); // close the open interval
uint64_t b = t->st_ns[S_RUN] + t->st_ns[S_CORE] + t->st_ns[S_HOST];
all_run += t->st_ns[S_RUN];
all_core += t->st_ns[S_CORE];
if (t == self) continue;
for (int i = 0; i < REP_N; i++)
if (!top[i] || b > busy[i]) {
memmove(top + i + 1, top + i, (REP_N - 1 - i) * sizeof *top);
memmove(busy + i + 1, busy + i, (REP_N - 1 - i) * sizeof *busy);
top[i] = t;
busy[i] = b;
break;
}
}
if (self) { // the render thread first
memmove(top + 1, top, REP_N * sizeof *top);
top[0] = self;
}
for (int i = 0; i <= REP_N && top[i]; i++) {
RtThread *t = top[i];
Row *r = &rows[nr++];
r->h = t->handle;
r->prio = t->prio;
r->render = t == self;
r->mask = t->mask;
r->entry = t->entry;
memcpy(r->ns, t->st_ns, sizeof r->ns);
r->nblk = t->nblk;
r->npre = t->npre;
r->lw_ns = t->lw_ns, r->lw_n = t->lw_n, r->lw_max = t->lw_max;
r->lg_ns = t->lg_ns, r->lg_n = t->lg_n, r->lg_max = t->lg_max;
r->sl_n = t->sl_n, r->sl_req = t->sl_req, r->sl_act = t->sl_act;
r->cpu = 0;
HANDLE th = t->tid ? OpenThread(THREAD_QUERY_LIMITED_INFORMATION, FALSE, t->tid) : NULL;
FILETIME a, b, k, u;
if (th && GetThreadTimes(th, &a, &b, &k, &u)) {
uint64_t c = (((uint64_t)k.dwHighDateTime << 32 | k.dwLowDateTime) +
((uint64_t)u.dwHighDateTime << 32 | u.dwLowDateTime)) * 100;
r->cpu = t->cpu_last ? c - t->cpu_last : 0;
t->cpu_last = c;
}
if (th) CloseHandle(th);
}
for (int k = 0; k < 2; k++) {
const WaitRec *src = k ? g_mw : g_rw;
for (int i = 0; i < RW_N && src[i].ns; i++)
for (int j = 0; j < 4; j++)
if (src[i].ns > rw[k][j].ns) {
memmove(rw[k] + j + 1, rw[k] + j, (3 - j) * sizeof **rw);
rw[k][j] = src[i];
break;
}
}
uint64_t pre_stale = g_pre_stale, pre_req = g_pre_req, pre_n = g_pre_n, pre_ns = g_pre_ns, pre_max = g_pre_max;
uint64_t sc = g_sync_calls, sb = g_sync_blocks;
for (RtThread *t = g_all; t; t = t->all) {
memset(t->st_ns, 0, sizeof t->st_ns);
t->nblk = t->npre = 0;
t->lw_ns = t->lw_n = t->lw_max = t->lg_ns = t->lg_n = t->lg_max = t->sl_n = t->sl_req = t->sl_act = 0;
}
memset(g_rw, 0, sizeof g_rw);
memset(g_mw, 0, sizeof g_mw);
g_pre_req = g_pre_n = g_pre_ns = g_pre_max = g_pre_stale = 0;
g_sync_calls = g_sync_blocks = 0;
unlock();
if (!span) return;
rt_log("sched: frame %llu: %.2f s, cores busy %.2f of %d, core wait %.2f thread-s, preempt req %llu released %llu "
"avg %.2f ms max %.2f ms, WaitSynchronization %llu calls %llu blocks, stale preempt %llu%s",
(unsigned long long)frame, span / 1e9, (double)all_run / span, NCORE, all_core / 1e9,
(unsigned long long)pre_req, (unsigned long long)pre_n, pre_n ? pre_ns / 1e6 / pre_n : 0.0, pre_max / 1e6,
(unsigned long long)sc, (unsigned long long)sb, (unsigned long long)pre_stale, self ? "" : " (render thread is not a guest thread)");
for (int i = 0; i < nr; i++) {
Row *r = &rows[i];
char d[128], s[256];
int o = 0;
for (int k = S_RUN; k < S_N; k++)
o += snprintf(s + o, sizeof s - o, " %s %.0f%%", s_name[k], 100.0 * r->ns[k] / span);
ldr_describe(r->entry, d, sizeof d);
rt_log("sched: %s%#x p%d m%llx%s cpu %.0f%% blocks %llu preempted %llu (%s)", r->render ? "render " : "",
r->h, r->prio, (unsigned long long)r->mask, s, 100.0 * r->cpu / span, (unsigned long long)r->nblk,
(unsigned long long)r->npre, r->entry ? d : "main");
rt_log("sched: %#x wake to run avg %.1f us max %.0f us (%llu), core grant to run avg %.1f us max %.0f us "
"(%llu), sleep %llu asked avg %.1f us took avg %.1f us", r->h, r->lw_n ? r->lw_ns / 1e3 / r->lw_n : 0.0,
r->lw_max / 1e3, (unsigned long long)r->lw_n, r->lg_n ? r->lg_ns / 1e3 / r->lg_n : 0.0, r->lg_max / 1e3,
(unsigned long long)r->lg_n, (unsigned long long)r->sl_n, r->sl_n ? r->sl_req / 1e3 / r->sl_n : 0.0,
r->sl_n ? r->sl_act / 1e3 / r->sl_n : 0.0);
}
for (int k = 0; k < 2; k++)
for (int j = 0; j < 4 && rw[k][j].ns; j++) {
char by[32] = "";
if (rw[k][j].waker) snprintf(by, sizeof by, ", woken by %#x", rw[k][j].waker);
rt_log("sched: %s wait %d: %s on %s%#llx%s: %.1f%%", k ? "main" : "render", j + 1, s_name[rw[k][j].st],
rw[k][j].st == S_HOST ? "exe+" : "", (unsigned long long)rw[k][j].obj, by, 100.0 * rw[k][j].ns / span);
}
}
// ---- locks: tag word = owner handle | WAITERS_BIT
static inline uint32_t ld32(uint64_t a) { return __atomic_load_n((uint32_t *)(uintptr_t)a, __ATOMIC_SEQ_CST); }
static inline void st32(uint64_t a, uint32_t v) { __atomic_store_n((uint32_t *)(uintptr_t)a, v, __ATOMIC_SEQ_CST); }
static inline int cas32(uint64_t a, uint32_t *exp, uint32_t v) {
return __atomic_compare_exchange_n((uint32_t *)(uintptr_t)a, exp, v, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
}
// Release the lock at addr: hand it to the best waiter, or clear it.
static void lock_handoff(uint64_t addr) {
RtThread *w = pick(W_LOCK, addr);
int more = 0;
if (!w) {
st32(addr, 0);
return;
}
for (RtThread *t = g_wait; t; t = t->next)
if (t != w && t->wkind == W_LOCK && t->waddr == addr) {
more = 1;
break;
}
st32(addr, w->wtag | (more ? WAITERS_BIT : 0));
wake(w, 0);
}
// A condvar waiter takes its lock: directly if free (and wakes), else it becomes a lock waiter behind the others.
static void lock_acquire_for(RtThread *t, uint64_t addr, uint32_t tag) {
uint32_t v = ld32(addr);
for (;;) {
if (v == 0) {
if (cas32(addr, &v, tag)) {
wake(t, 0);
return;
}
continue;
}
if (cas32(addr, &v, v | WAITERS_BIT)) break;
}
acct(t, S_LOCK);
t->wobj = v & ~WAITERS_BIT;
t->wkind = W_LOCK;
t->waddr = addr;
t->wtag = tag;
t->wseq = ++g_wseq;
}
// ---- threads
static RtThread *thr_new(int prio, int core, uint64_t mask) {
RtThread *t = calloc(1, sizeof *t);
Cpu *c = _aligned_malloc(sizeof(Cpu), 64);
if (!t || !c) rt_fatal("kernel: thread allocation");
memset(c, 0, sizeof *c);
InitializeConditionVariable(&t->cv);
t->prio = prio;
t->core = core;
t->mask = mask;
t->cur_core = -1;
// TLR (0x200) and the "current thread object" the SDK reads through TLR+0x1F8, in one guest-visible block.
t->tlr = mem_kernel_alloc(TLR_SIZE + THREAD_OBJ_SIZE, TLR_SIZE);
*(uint64_t *)(t->tlr + 0x1F8) = (uint64_t)(uintptr_t)(t->tlr + TLR_SIZE);
t->tls = mem_kernel_alloc(TLS_SIZE, 16);
c->tpidrro = (uint64_t)(uintptr_t)t->tlr;
c->tpidr = (uint64_t)(uintptr_t)t->tls;
c->excl_addr = ~0ull;
c->thread = t;
t->cpu = t->cur = c;
return t;
}
static void thr_link(RtThread *t) { // holding G
t->handle = h_new(O_THREAD, t);
t->id = g_next_id++;
t->all = g_all;
g_all = t;
g_created++;
}
static void thr_free(RtThread *t) { // holding G; t has exited and its handle is closed
for (RtThread **p = &g_all; *p; p = &(*p)->all)
if (*p == t) {
*p = t->all;
break;
}
mem_kernel_free(t->tlr, TLR_SIZE + THREAD_OBJ_SIZE);
mem_kernel_free(t->tls, TLS_SIZE);
_aligned_free(t->cpu);
free(t);
}
// The end of every guest thread, on its own host context after the unwind: give the core up, signal the thread
// object, free it if its handle is already closed. t is not touched after the unlock.
static void thread_finish(RtThread *t) {
if (t->timer) CloseHandle(t->timer);
t->timer = NULL;
if (t->base_fiber) ConvertFiberToThread();
t->base_fiber = NULL;
tcur = NULL;
lock();
core_put(t);
acct(t, S_OFF);
if (t->hthr) CloseHandle(t->hthr); // no waiter interrupts a thread without a core
t->hthr = NULL;
t->exited = 1;
wake_sync_waiters();
if (t->closed) thr_free(t);
unlock();
}
// ExitThread: back to the thread start with a non-unwinding longjmp; no SEH unwinding through lifted frames.
// (__builtin_setjmp is not usable: on Win64 LLVM saves the frame address, which is not rbp when the frame register
// is offset, and the landing code then reads its locals through the wrong base.)
static __attribute__((noreturn)) void exit_now(RtThread *t) { longjmp(t->exit_jmp, 1); }
// The calling host thread, for ap_try (TOTK_ASYNC_PREEMPT only).
static HANDLE ap_open_self(void) {
if (!cores_enabled() || !g_ap_int) return NULL;
HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_SET_CONTEXT | THREAD_QUERY_INFORMATION,
FALSE, GetCurrentThreadId());
if (!h) rt_log("kernel: OpenThread for asynchronous preemption failed (error %lu)", GetLastError());
return h;
}
static DWORD WINAPI thread_main(void *p) {
RtThread *t = p;
tcur = t;
t->tid = GetCurrentThreadId();
HANDLE self = ap_open_self();
lock();
t->hthr = self;
while (!t->started) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
core_take(t);
unlock();
Cpu *c = t->cpu;
c->sp = t->stack_top & ~15ull;
if (!setjmp(t->exit_jmp)) {
rt_call(c, t->entry, 1, t->arg);
char d[128];
rt_fatal("guest thread %#x: entry %s returned (Horizon threads end with ExitThread)", t->handle,
ldr_describe(t->entry, d, sizeof d));
}
thread_finish(tcur);
return 0;
}
uint64_t kern_threads_created(void) {
lock();
uint64_t n = g_created;
unlock();
return n;
}
Cpu *kern_cpu(void) { return tcur ? tcur->cur : NULL; }
uint32_t kern_main_thread(Cpu **cpu) {
if (tcur) {
if (!tcur->is_main) rt_fatal("kern_main_thread on guest thread %#x, not the main thread", tcur->handle);
*cpu = tcur->cpu;
return tcur->handle;
}
if (g_main) rt_fatal("kern_main_thread: the main thread already runs on another host thread");
const RtGuestLayout *L = mem_layout();
uint64_t stack = L->stack.base + L->stack.size - MAIN_STACK;
if (!VirtualAlloc((void *)(uintptr_t)stack, MAIN_STACK, MEM_COMMIT, PAGE_READWRITE))
rt_fatal("kernel: commit the main thread stack at %#llx failed (error %lu)", (unsigned long long)stack,
GetLastError());
RtThread *t = thr_new(MAIN_PRIO, MAIN_CORE, 7);
t->started = 1;
t->is_main = 1;
t->stack_top = stack + MAIN_STACK;
t->cpu->sp = t->stack_top;
t->tid = GetCurrentThreadId();
perf_main_thread();
HANDLE self = ap_open_self();
lock();
t->hthr = self;
thr_link(t);
g_main = t;
g_main_stack = stack;
core_take(t);
unlock();
tcur = t;
*cpu = t->cpu;
return t->handle;
}
void kern_main_stack(uint64_t *base, uint64_t *size) {
uint64_t b = __atomic_load_n(&g_main_stack, __ATOMIC_ACQUIRE);
*base = b;
*size = b ? MAIN_STACK : 0;
}
static DWORD WINAPI guest_main(void *p) {
void (*fn)(Cpu *) = (void (*)(Cpu *))p;
Cpu *c;
kern_main_thread(&c);
if (!setjmp(tcur->exit_jmp)) fn(c);
thread_finish(tcur);
return 0;
}
void kern_run_on_guest_main(void (*fn)(Cpu *c)) {
HANDLE h = CreateThread(NULL, HOST_STACK, guest_main, (void *)fn, STACK_SIZE_PARAM_IS_A_RESERVATION, NULL);
if (!h) rt_fatal("kernel: CreateThread for the guest main thread failed (error %lu)", GetLastError());
WaitForSingleObject(h, INFINITE);
CloseHandle(h);
}
// The high-resolution waitable timer ends a wait 0.3-0.5 ms late at best: the SDK's 20 us polling sleeps (main thread
// and job workers, about 1.5 per frame each) took 460-515 us. Sleeps up to SLEEP_SPIN_NS spin on the performance
// counter instead; the virtual core is already given up. Linux: clock_nanosleep wakes within about 50 us (timer slack),
// so a Linux host spins only below that, or not at all.
#define SLEEP_SPIN_NS 1000000
static void sleep_ns(RtThread *t, int64_t ns) {
if (ns <= SLEEP_SPIN_NS) {
uint64_t end = kern_now_ns() + (uint64_t)ns;
while (kern_now_ns() < end) YieldProcessor();
return;
}
if (!t->timer) {
t->timer = CreateWaitableTimerExW(NULL, NULL, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS);
if (!t->timer) t->timer = CreateWaitableTimerExW(NULL, NULL, 0, TIMER_ALL_ACCESS);
if (!t->timer) rt_fatal("kernel: CreateWaitableTimerExW failed (error %lu)", GetLastError());
}
LARGE_INTEGER due;
int64_t units = ns / 100 + (ns % 100 != 0); // 100 ns units, rounded up
due.QuadPart = -(units > 0 ? units : 1);
if (!SetWaitableTimer(t->timer, &due, 0, NULL, NULL, FALSE))
rt_fatal("kernel: SetWaitableTimer failed (error %lu)", GetLastError());
WaitForSingleObject(t->timer, INFINITE);
}
// ---- synchronization objects
static int sig_obj(uint32_t h, int *bad) { // 1 = signaled (holding G)
unsigned i = h - 0x1000;
if (h == H_CUR_THREAD) return 0;
if (i == 0 || i >= MAX_H) {
*bad = 1;
return 0;
}
switch (g_h[i].type) {
case O_THREAD: return ((RtThread *)g_h[i].obj)->exited;
case O_EVENT_R: return ((Event *)g_h[i].obj)->signaled;
default: *bad = 1; return 0;
}
}
int kern_thread_state(uint32_t h) {
lock();
RtThread *t = h_peek(h, O_THREAD);
int r = !t ? -1 : t->exited ? KERN_THREAD_EXITED : t->wkind;
unlock();
return r;
}
// CloseHandle: an event goes with its last end; a thread object once it has also exited.
static uint32_t handle_close(uint32_t h) {
unsigned i = h - 0x1000;
int ok = 0, kind = 0;
void *ipc = NULL;
lock();
if (i > 0 && i < MAX_H) {
int type = g_h[i].type;
void *o = g_h[i].obj;
if (type == O_EVENT_R || type == O_EVENT_W) {
Event *e = o;
if (--e->refs == 0) free(e);
ok = 1;
} else if (type == O_SESSION || type == O_TMEM) {
kind = type;
ipc = o;
ok = 1;
} else if (type == O_THREAD) {
RtThread *t = o;
t->closed = 1;
if (t->exited) thr_free(t);
ok = 1;
}
if (ok) {
g_h[i].type = O_FREE;
g_h[i].obj = NULL;
}
}
unlock();
if (ipc) ipc_handle_closed(kind, ipc);
return ok ? R_OK : R_BAD_HANDLE;
}
void kern_handle_close(uint32_t h) {
if (handle_close(h)) rt_fatal("kern_handle_close(%#x): bad handle", h);
}
// Hooks of the probe's condvar negative control.
volatile int kern_test_cv_old_order;
void (*volatile kern_test_cv_hook)(void);
static void cv_hook(void) {
void (*h)(void) = kern_test_cv_hook;
if (h) h();
}
int kern_svc(Cpu *c, uint32_t imm) {
uint64_t *x = c->x;
if (g_diag > 0 && tcur) tcur->dg_svc = imm;
switch (imm) {
case 0x08: { // CreateThread(-> w1 handle; x1 entry, x2 arg, x3 stack top, w4 priority, w5 core)
int prio = (int)x[4], core = (int)x[5];
if (!x[1] || !x[3] || prio < 28 || prio > 59 || core < -3 || core > 2) {
x[0] = R_BAD_STATE;
return 1;
}
RtThread *t = thr_new(prio, core >= 0 ? core : MAIN_CORE, core >= 0 ? 1ull << core : 7);
t->entry = x[1];
t->arg = x[2];
t->stack_top = x[3] & ~15ull;
lock();
thr_link(t);
unlock();
HANDLE h = CreateThread(NULL, HOST_STACK, thread_main, t, STACK_SIZE_PARAM_IS_A_RESERVATION, NULL);
if (!h) rt_fatal("kernel: CreateThread failed (error %lu)", GetLastError());
CloseHandle(h);
if (rt_trace & RT_TRACE_SVC) {
char d[128];
rt_log("CreateThread %#x entry %s stack %#llx prio %d core %d", t->handle,
ldr_describe(t->entry, d, sizeof d), (unsigned long long)t->stack_top, prio, core);
}
x[0] = R_OK;
x[1] = t->handle;
return 1;
}
case 0x09: { // StartThread(w0)
lock();
RtThread *t = h_get((uint32_t)x[0], O_THREAD);
if (t) {
t->started = 1;
WakeConditionVariable(&t->cv);
}
unlock();
x[0] = t ? R_OK : R_BAD_HANDLE;
return 1;
}
case 0x0A: { // ExitThread
RtThread *t = cur(imm);
if (t->fiber) { // on a guest fiber: unwind from the thread's own context (kern_fiber_switch resumes it)
t->exit_pending = 1;
t->leaving = t->fiber;
t->fiber = NULL;
t->cur = t->cpu;
SwitchToFiber(t->base_fiber);
rt_fatal("kernel: ExitThread on a fiber came back");
}
exit_now(t);
}
case 0x0B: { // SleepThread(x0 ns); 0, -1, -2 yield
int64_t ns = (int64_t)x[0];
RtThread *t = cur(imm);
if (ns > 0) {
lock();
core_put(t);
acct(t, S_SLEEP);
t->wobj = (uint64_t)ns;
t->nblk++;
unlock();
uint64_t t0 = kern_now_ns();
sleep_ns(t, ns);
uint64_t took = kern_now_ns() - t0;
lock();
t->sl_n++;
t->sl_req += (uint64_t)ns;
t->sl_act += took;
core_take(t);
unlock();
return 1;
}
// Yield: hand the core to a waiting thread of equal or better priority.
lock();
int give = 0;
int tp = prio_of(t);
for (RtThread *w = g_core_wait; w && !give; w = w->cnext)
give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && prio_of(w) <= tp;
if (give) {
core_put(t);
core_take(t);
}
unlock();
if (!give) SwitchToThread();
return 1;
}
case 0x0C: { // GetThreadPriority(w1 handle -> w1)
lock();
RtThread *t = h_get((uint32_t)x[1], O_THREAD);
x[0] = t ? R_OK : R_BAD_HANDLE;
if (t) x[1] = (uint64_t)t->prio;
unlock();
return 1;
}
case 0x0D: { // SetThreadPriority(w0 handle, w1 priority)
if ((uint32_t)x[1] < 28 || (uint32_t)x[1] > 59) {
x[0] = R_BAD_STATE;
return 1;
}
lock();
RtThread *t = h_get((uint32_t)x[0], O_THREAD);
if (t) t->prio = (int)x[1];
x[0] = t ? R_OK : R_BAD_HANDLE;
unlock();
return 1;
}
case 0x0E: { // GetThreadCoreMask(w2 handle -> w1 core, x2 mask)
lock();
RtThread *t = h_get((uint32_t)x[2], O_THREAD);
x[0] = t ? R_OK : R_BAD_HANDLE;
if (t) {
x[1] = (uint64_t)(uint32_t)t->core;
x[2] = t->mask;
}
unlock();
return 1;
}
case 0x0F: { // SetThreadCoreMask(w0 handle, w1 ideal core, x2 mask): -1 process default, -2/-3 keep the ideal core
lock();
RtThread *t = h_get((uint32_t)x[0], O_THREAD);
if (t) {
int core = (int)x[1];
if (core >= 0) t->core = core;
else if (core == -1) t->core = MAIN_CORE;
if (x[2]) t->mask = x[2];
if (t == tcur && t->cur_core >= 0 && !core_allowed(t, t->cur_core)) { // move off a forbidden core
core_put(t);
core_take(t);
}
}
x[0] = t ? R_OK : R_BAD_HANDLE;
unlock();
return 1;
}
case 0x10: { // GetCurrentProcessorNumber: the virtual core held
RtThread *t = cur(imm);
x[0] = (uint64_t)(uint32_t)(t->cur_core >= 0 ? t->cur_core : t->core);
return 1;
}
case 0x36: { // SynchronizePreemptionState: the SDK's spin locks call it after clearing TLR+0x100 (disable_count)
// when TLR+0x102 (interrupt_flag) is set; rt_svc's checkpoint has already honoured a pending preempt
RtThread *t = cur(imm);
volatile uint16_t *flag = (volatile uint16_t *)(t->tlr + 0x102);
__atomic_fetch_add(&g_sync36, 1, __ATOMIC_RELAXED);
if (*flag) __atomic_fetch_add(&g_sync36_flag, 1, __ATOMIC_RELAXED);
*flag = 0;
ap_sync(t);
x[0] = R_OK;
return 1;
}
case 0x25: { // GetThreadId(w1 handle -> x1)
lock();
RtThread *t = h_get((uint32_t)x[1], O_THREAD);
x[0] = t ? R_OK : R_BAD_HANDLE;
if (t) x[1] = t->id;
unlock();
return 1;
}
case 0x1A: { // ArbitrateLock(w0 owner handle, x1 address, w2 tag)
RtThread *t = cur(imm);
uint64_t addr = x[1];
uint32_t r = R_OK;
lock();
if (ld32(addr) == ((uint32_t)x[0] | WAITERS_BIT)) {
t->waddr = addr;
t->wtag = (uint32_t)x[2];
t->wobj = (uint32_t)x[0];
r = block(t, W_LOCK, -1);
}
unlock();
x[0] = r;
return 1;
}
case 0x1B: // ArbitrateUnlock(x0 address)
lock();
lock_handoff(x[0]);
unlock();
x[0] = R_OK;
return 1;
case 0x1C: { // WaitProcessWideKeyAtomic(x0 lock, x1 key, w2 tag, x3 timeout)
// The key is set before the lock is released. The SDK's Signal reads the key in user mode and skips svc 0x1D
// when it is 0, and user mode takes the lock without G: releasing first lets another thread take the lock,
// change the condition, read key 0 and wake nobody, after which this thread waits forever (MNG threads.c
// 439-446). Linking into the wait list happens in block, still holding G, so svc 0x1D cannot come between.
RtThread *t = cur(imm);
uint64_t addr = x[0], key = x[1];
lock();
if (!kern_test_cv_old_order) {
st32(key, 1);
t->wkey = key;
t->waddr = addr;
t->wtag = (uint32_t)x[2];
lock_handoff(addr);
cv_hook();
} else { // the probe's negative control: MNG's pre-fix order
lock_handoff(addr);
cv_hook();
st32(key, 1);
t->wkey = key;
t->waddr = addr;
t->wtag = (uint32_t)x[2];
}
t->wobj = key;
uint32_t r = block(t, W_CONDVAR, (int64_t)x[3]); // on timeout user mode takes the lock itself
unlock();
x[0] = r;
return 1;
}
case 0x1D: { // SignalProcessWideKey(x0 key, w1 count; <= 0 all)
uint64_t key = x[0];
int cnt = (int)x[1], n = 0;
lock();
for (RtThread *w; (cnt <= 0 || n < cnt) && (w = pick(W_CONDVAR, key)); n++) lock_acquire_for(w, w->waddr, w->wtag);
int left = 0;
for (RtThread *w = g_wait; w && !left; w = w->next) left = w->wkind == W_CONDVAR && w->wkey == key;
if (!left) st32(key, 0);
unlock();
x[0] = R_OK;
return 1;
}
case 0x34: { // WaitForAddress(x0 address, w1 type, w2 value, x3 timeout): 0 if less, 1 decrement if less, 2 if equal
RtThread *t = cur(imm);
uint64_t addr = x[0];
int type = (int)x[1];
int32_t val = (int32_t)x[2];
if (type < 0 || type > 2) {
x[0] = R_INVALID_ENUM;
return 1;
}
lock();
uint32_t r, v = ld32(addr);
int ok = type == 2 ? (int32_t)v == val : (int32_t)v < val;
if (ok && type == 1)
while (!cas32(addr, &v, v - 1))
if (!((int32_t)v < val)) {
ok = 0;
break;
}
if (!ok) r = R_BAD_STATE;
else {
t->waddr = addr;
t->wobj = addr;
r = block(t, W_ADDRESS, (int64_t)x[3]);
}
unlock();
x[0] = r;
return 1;
}
case 0x35: { // SignalToAddress(x0 address, w1 type, w2 value, w3 count): 0 wake, 1 increment if equal,
// 2 if equal modify by the number of waiters
uint64_t addr = x[0];
int type = (int)x[1], cnt = (int)x[3];
int32_t val = (int32_t)x[2];
if (type < 0 || type > 2) {
x[0] = R_INVALID_ENUM;
return 1;
}
lock();
int nw = 0;
for (RtThread *w = g_wait; w; w = w->next) nw += w->wkind == W_ADDRESS && w->waddr == addr;
uint32_t r = R_OK;
if (type != 0) {
int32_t nv = val + 1;
if (type == 2) {
if (cnt <= 0) nv = nw > 0 ? val - 1 : val + 1;
else nv = nw == 0 ? val + 1 : nw <= cnt ? val - 1 : val;
}
uint32_t e = (uint32_t)val;
if (!cas32(addr, &e, (uint32_t)nv)) r = R_BAD_STATE;
}
if (!r)
for (int n = 0; cnt <= 0 || n < cnt; n++) {
RtThread *w = pick(W_ADDRESS, addr);
if (!w) break;
wake(w, 0);
}
unlock();
x[0] = r;
return 1;
}
case 0x45: { // CreateEvent(-> w1 writable, w2 readable)
Event *e = calloc(1, sizeof *e);
if (!e) rt_fatal("kernel: event allocation");
e->refs = 2;
lock();
uint32_t hw = h_new(O_EVENT_W, e), hr = h_new(O_EVENT_R, e);
unlock();
x[0] = R_OK;
x[1] = hw;
x[2] = hr;
return 1;
}
case 0x11: case 0x12: case 0x17: { // SignalEvent(writable), ClearEvent(either end), ResetSignal(readable)
uint32_t h = (uint32_t)x[0];
uint32_t r = R_OK;
lock();
Event *e = imm == 0x11 ? h_peek(h, O_EVENT_W) : imm == 0x17 ? h_peek(h, O_EVENT_R)
: h_peek(h, O_EVENT_W) ? h_peek(h, O_EVENT_W) : h_peek(h, O_EVENT_R);
if (!e) {
r = imm == 0x17 && h_peek(h, O_THREAD) ? R_BAD_STATE : R_BAD_HANDLE;
if (r == R_BAD_HANDLE) rt_log("kernel: event svc %#x: invalid handle %#x", imm, h);
} else if (imm == 0x11) {
e->signaled = 1;
wake_sync_waiters();
} else if (imm == 0x17 && !e->signaled) {
r = R_BAD_STATE;
} else {
e->signaled = 0;
}
unlock();
x[0] = r;
return 1;
}
case 0x18: { // WaitSynchronization(-> w1 index; x1 handles, w2 count, x3 timeout)
RtThread *t = cur(imm);
const uint32_t *hs = (const uint32_t *)(uintptr_t)x[1];
int n = (int)x[2];
int64_t ns = (int64_t)x[3];
if (n < 0 || n > 64 || (!hs && n)) {
x[0] = R_BAD_STATE;
return 1;
}
uint64_t start = kern_now_ns();
uint32_t r;
int idx = -1;
lock();
g_sync_calls++;
for (;;) {
if (t->cancel) {
t->cancel = 0;
r = R_CANCELLED;
break;
}
int bad = 0;
for (int i = 0; i < n && idx < 0; i++)
if (sig_obj(hs[i], &bad)) idx = i;
if (bad) {
r = R_BAD_HANDLE;
break;
}
if (idx >= 0) {
r = R_OK;
break;
}
int64_t left = ns;
if (ns > 0) {
uint64_t spent = kern_now_ns() - start;
if (spent >= (uint64_t)ns) {
r = R_TIMEOUT;
break;
}
left = ns - (int64_t)spent;
}
t->wobj = n ? hs[0] | (uint64_t)n << 32 : 0;
g_sync_blocks++;
if (block(t, W_SYNC, left) == R_TIMEOUT) {
r = R_TIMEOUT;
break;
}
}
unlock();
x[0] = r;
x[1] = idx >= 0 ? (uint64_t)idx : 0;
return 1;
}
case 0x19: { // CancelSynchronization(w0 thread)
lock();
RtThread *t = h_get((uint32_t)x[0], O_THREAD);
if (t) {
t->cancel = 1;
if (t->wkind == W_SYNC) wake(t, (uint32_t)-1);
}
unlock();
x[0] = t ? R_OK : R_BAD_HANDLE;
return 1;
}
case 0x16: // CloseHandle(w0)
x[0] = handle_close((uint32_t)x[0]);
return 1;
}
return 0;
}
// ---- handles and events for host services (ipc.c and the service overrides)
uint32_t kern_handle_new(int kind, void *obj) {
if (kind != KERN_SESSION && kind != KERN_TMEM) rt_fatal("kern_handle_new: kind %d", kind);
lock();
uint32_t h = h_new(kind, obj);
unlock();
return h;
}
void *kern_handle_get(uint32_t h, int kind) {
lock();
void *o = h_peek(h, kind);
unlock();
return o;
}
uint32_t kern_event_new(void) {
Event *e = calloc(1, sizeof *e);
if (!e) rt_fatal("kernel: event allocation");
e->refs = 1;
lock();
uint32_t hw = h_new(O_EVENT_W, e);
unlock();
return hw;
}
// Copy-handle transfer of the readable end: each call gives the client its own handle to the same event.
uint32_t kern_event_readable(uint32_t hw) {
lock();
Event *e = h_peek(hw, O_EVENT_W);
if (!e) rt_fatal("kern_event_readable: %#x is not a writable event handle", hw);
e->refs++;
uint32_t hr = h_new(O_EVENT_R, e);
unlock();
return hr;
}
void kern_event_signal(uint32_t hw) {
lock();
Event *e = h_peek(hw, O_EVENT_W);
if (!e) rt_fatal("kern_event_signal: %#x is not a writable event handle", hw);
e->signaled = 1;
wake_sync_waiters();
unlock();
}
// System events of host services: the SDK's own nn::os::CreateSystemEvent creates the kernel event (SVC 0x45) on a
// guest thread; the writable handle is kept, so any thread (host audio tick included) can signal it later.
static struct { uint64_t ev; uint32_t wh; } g_se[512];
static int g_nse;
static SRWLOCK g_se_lock = SRWLOCK_INIT;
static uint32_t se_find(uint64_t ev) {
AcquireSRWLockShared(&g_se_lock);
uint32_t wh = 0;
for (int i = 0; i < g_nse && !wh; i++)
if (g_se[i].ev == ev) wh = g_se[i].wh;
ReleaseSRWLockShared(&g_se_lock);
return wh;
}
void kern_bind_system_event(Cpu *c, uint64_t ev, int autoclear) {
if (se_find(ev)) return;
if (!tcur || !c) rt_fatal("kern_bind_system_event(%#llx) on a host thread", (unsigned long long)ev);
static uint64_t create, getw;
if (!create) {
create = ldr_symbol("_ZN2nn2os17CreateSystemEventEPNS0_15SystemEventTypeENS0_14EventClearModeEb");
getw = ldr_symbol("_ZN2nn2os30GetWritableHandleOfSystemEventEPKNS0_15SystemEventTypeE");
if (!create || !getw) rt_fatal("kern_bind_system_event: the SDK's system event functions are not loaded");
}
uint32_t r = (uint32_t)rt_call(c, create, 3, ev, (uint64_t)(autoclear ? 1 : 0), (uint64_t)1);
if (r) rt_fatal("kern_bind_system_event: CreateSystemEvent(%#llx) failed %#x", (unsigned long long)ev, r);
uint32_t wh = (uint32_t)rt_call(c, getw, 1, ev);
AcquireSRWLockExclusive(&g_se_lock);
if (g_nse == (int)(sizeof g_se / sizeof *g_se)) rt_fatal("kern_bind_system_event: table full");
g_se[g_nse].ev = ev;
g_se[g_nse++].wh = wh;
ReleaseSRWLockExclusive(&g_se_lock);
}
// An event not bound yet is bound on the spot with manual clear, which needs a guest thread.
void kern_signal_system_event(uint64_t ev) {
uint32_t wh = se_find(ev);
if (!wh) {
kern_bind_system_event(kern_cpu(), ev, 0);
wh = se_find(ev);
}
kern_event_signal(wh);
}
// ---- fibers
// The SDK switches fibers by swapping register context, which breaks "RET = C return". Each guest fiber gets a host
// fiber (own 256 MiB reserved host stack) and its own Cpu, so a suspended fiber's registers and host frames stay
// untouched while others run, and a fiber may resume on another thread. A switch carries the thread state (TLR, TLS
// pointer, FPCR/FPSR) over to the target's Cpu and clears the exclusive monitor.
enum { F_NEW, F_STARTED, F_DONE };
struct Fiber {
uint64_t addr, func, arg, stack, size;
Cpu *cpu;
void *host;
int state;
volatile int running;
};
static Fiber *g_fibers[FIBER_MAX];
static SRWLOCK g_fiber_lock = SRWLOCK_INIT;
static Fiber *fiber_find(uint64_t addr) {
Fiber *f = NULL;
AcquireSRWLockShared(&g_fiber_lock);
for (int i = 0; i < FIBER_MAX && !f; i++)
if (g_fibers[i] && g_fibers[i]->addr == addr) f = g_fibers[i];
ReleaseSRWLockShared(&g_fiber_lock);
return f;
}
static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to);
// After a host fiber switch: release the context left behind, and finish an ExitThread issued on a fiber.
static void fiber_resumed(void) {
RtThread *t = tcur;
rt_fpcr_sync(t->cur); // the host fiber resumes with its own MXCSR (FIBER_FLAG_FLOAT_SWITCH)
if (t->leaving) {
__atomic_store_n(&t->leaving->running, 0, __ATOMIC_RELEASE);
t->leaving = NULL;
}
if (t->exit_pending && !t->fiber) exit_now(t);
}
static void WINAPI fiber_main(void *p) {
Fiber *f = p;
fiber_resumed();
Cpu *c = f->cpu;
tcur->host_depth = 0; // a new fiber starts at the top level; each context restores its own depth on resume
uint64_t next = rt_call(c, f->func, 1, f->arg);
f->state = F_DONE;
Fiber *n = NULL;
if (next && !(n = fiber_find(next)))
rt_fatal("fiber %#llx returned %#llx, which is not an initialized fiber", (unsigned long long)f->addr,
(unsigned long long)next);
fiber_switch_to(tcur, c, n); // NULL: back to the thread's own context
rt_fatal("fiber %#llx resumed after it finished", (unsigned long long)f->addr);
}
static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) {
if (!t->base_fiber) {
t->base_fiber = ConvertThreadToFiberEx(NULL, FIBER_FLAG_FLOAT_SWITCH);
if (!t->base_fiber) rt_fatal("kernel: ConvertThreadToFiberEx failed (error %lu)", GetLastError());
}
Cpu *dst = to ? to->cpu : t->cpu;
if (to) {
if (to->state == F_DONE) rt_fatal("SwitchToFiber(%#llx): the fiber has finished", (unsigned long long)to->addr);
int idle = 0;
if (!__atomic_compare_exchange_n(&to->running, &idle, 1, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE))
rt_fatal("SwitchToFiber(%#llx): the fiber is running", (unsigned long long)to->addr);
if (to->state == F_NEW) {
dst->sp = (to->stack + to->size) & ~15ull;
to->state = F_STARTED;
}
}
dst->tpidr = from->tpidr;
dst->tpidrro = from->tpidrro;
dst->fpcr = from->fpcr;
dst->fpsr = from->fpsr;
dst->excl_addr = ~0ull;
dst->thread = t;
t->leaving = t->fiber;
t->fiber = to;
t->cur = dst;
int depth = t->host_depth;
SwitchToFiber(to ? to->host : t->base_fiber);
t->host_depth = depth;
fiber_resumed();
}
// InitializeFiber(x0 FiberType*, x1 function, x2 argument, x3 stack, x4 stack size, w5 flags)
void kern_fiber_initialize(Cpu *c) {
uint64_t addr = c->x[0];
Fiber *f = fiber_find(addr);
if (f && f->running) rt_fatal("InitializeFiber(%#llx): the fiber is running", (unsigned long long)addr);
if (!f) {
f = calloc(1, sizeof *f);
Cpu *fc = _aligned_malloc(sizeof(Cpu), 64);
if (!f || !fc) rt_fatal("kernel: fiber allocation");
f->cpu = fc;
AcquireSRWLockExclusive(&g_fiber_lock);
int i = 0;
while (i < FIBER_MAX && g_fibers[i]) i++;
if (i == FIBER_MAX) rt_fatal("kernel: more than %d fibers", FIBER_MAX);
g_fibers[i] = f;
ReleaseSRWLockExclusive(&g_fiber_lock);
} else if (f->host) {
DeleteFiber(f->host);
}
memset(f->cpu, 0, sizeof *f->cpu);
f->cpu->excl_addr = ~0ull;
f->addr = addr;
f->func = c->x[1];
f->arg = c->x[2];
f->stack = c->x[3];
f->size = c->x[4];
f->state = F_NEW;
f->running = 0;
f->host = CreateFiberEx(0x10000, HOST_STACK, FIBER_FLAG_FLOAT_SWITCH, fiber_main, f);
if (!f->host) rt_fatal("kernel: CreateFiberEx failed (error %lu)", GetLastError());
}
// FinalizeFiber(x0 FiberType*)
void kern_fiber_finalize(Cpu *c) {
uint64_t addr = c->x[0];
AcquireSRWLockExclusive(&g_fiber_lock);
Fiber *f = NULL;
for (int i = 0; i < FIBER_MAX && !f; i++)
if (g_fibers[i] && g_fibers[i]->addr == addr) {
f = g_fibers[i];
g_fibers[i] = NULL;
}
ReleaseSRWLockExclusive(&g_fiber_lock);
if (!f) rt_fatal("FinalizeFiber(%#llx): not an initialized fiber", (unsigned long long)addr);
if (f->running) rt_fatal("FinalizeFiber(%#llx): the fiber is running", (unsigned long long)addr);
DeleteFiber(f->host);
_aligned_free(f->cpu);
free(f);
}
// SwitchToFiber(x0 FiberType*)
void kern_fiber_switch(Cpu *c) {
RtThread *t = tcur;
if (!t) rt_fatal("SwitchToFiber on a host thread");
Fiber *to = fiber_find(c->x[0]);
if (!to) rt_fatal("SwitchToFiber(%#llx): not an initialized fiber", (unsigned long long)c->x[0]);
if (to == t->fiber) return;
fiber_switch_to(t, c, to);
}
// GetCurrentFiber() -> x0, NULL on the thread's own context
void kern_fiber_current(Cpu *c) {
RtThread *t = tcur;
c->x[0] = t && t->fiber ? t->fiber->addr : 0;
}
HOST_OVERRIDE(KERN_SYM_INITIALIZE_FIBER, kern_fiber_initialize);
HOST_OVERRIDE(KERN_SYM_FINALIZE_FIBER, kern_fiber_finalize);
HOST_OVERRIDE(KERN_SYM_SWITCH_TO_FIBER, kern_fiber_switch);
HOST_OVERRIDE(KERN_SYM_GET_CURRENT_FIBER, kern_fiber_current);
Reference in new issue View Git Blame Copy Permalink