mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-11 01:41:02 +00:00
Valiantsin DzerakhandClaude Opus 5.5
ce150130df
Comments cited codes from local notes (port-lessons P1, backlog), task labels (T6c1-T6c3, M4), bare upstream commit hashes and issue numbers, a file that no longer exists (nvn_api.c, now nvn_api_*.c), a generated file (main_0103.c) and line numbers into upstream files that refs/ has moved past, so they pointed at unrelated code. Those now name the file, function or project and say what the cited code does. Attribution of copied code keeps its pinned commits. Corrected facts: TOTK_REPLAY_ASYNC defaults to 0, and the shader cache, pipeline cache and pipeline keys live next to totk.exe, not in build/v121. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2196 lines
86 KiB
C
2196 lines
86 KiB
C
// Adapted from MacNativeGames games/totk/runtime/threads.c @ b75be8c (MIT, Copyright (c) 2026 MacNativeGames
|
|
// contributors; itself from d2r-mobile dm/host/hle_thread.c), d2r-mobile dm/host/thread.c @ 53e7c5b (MIT, Copyright
|
|
// (c) 2026 jhaizhou) and LAS Recomp runtime/kernel.c @ da35b83 (MIT, Copyright (c) 2026 Peiikai); the preemption of
|
|
// virtual cores from MacNativeGames threads.c @ 33e8fa8 (MIT, as above).
|
|
// Threads, synchronization, virtual cores and fibers of the Horizon kernel.
|
|
//
|
|
// One guest thread = one Windows thread with a 256 MiB reserved host stack (every guest BL is a host call) and its own
|
|
// Cpu. The SDK's nn::os runs as lifted code; only the kernel under it is the host. Every wait hangs on one global lock
|
|
// G with one condition variable per thread; wake order is the kernel's: lower priority value first, then first come.
|
|
// Semantics follow the public Horizon kernel (KConditionVariable, KAddressArbiter, KSynchronization).
|
|
#include <windows.h>
|
|
#define __USE_MINGW_SETJMP_NON_SEH // setjmp(b) = _setjmp(b, NULL): longjmp restores registers, no SEH unwinding
|
|
#include <setjmp.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "host.h"
|
|
#include "kernel.h"
|
|
#include "platform/platform.h"
|
|
|
|
#define R_OK 0u
|
|
#define R_TIMEOUT KERN_R_TIMEOUT
|
|
#define R_CANCELLED KERN_R_CANCELLED
|
|
#define R_BAD_HANDLE KERN_R_BAD_HANDLE
|
|
#define R_BAD_STATE KERN_R_BAD_STATE
|
|
#define R_INVALID_ENUM KERN_R_INVALID_ENUM
|
|
#define WAITERS_BIT KERN_WAITERS_BIT
|
|
#define H_CUR_THREAD KERN_H_CURRENT_THREAD
|
|
|
|
#define MAX_H 4096
|
|
#define NCORE 3
|
|
#define HOST_STACK (256ull << 20) // reserved, committed on demand
|
|
#define TLR_SIZE 0x200
|
|
#define THREAD_OBJ_SIZE 0x400 // "current thread object" at TLR+0x1F8 (d2r-mobile thread.c)
|
|
#define TLS_SIZE 0x2000
|
|
#define MAIN_STACK (1ull << 20) // main.npdm META+0x1C
|
|
#define MAIN_PRIO 44 // META+0xE
|
|
#define MAIN_CORE 0 // META+0xF
|
|
#define FIBER_MAX 1024
|
|
|
|
enum { O_FREE, O_THREAD, O_EVENT_R, O_EVENT_W, O_SESSION = KERN_SESSION, O_TMEM = KERN_TMEM };
|
|
_Static_assert(O_EVENT_W < O_SESSION, "handle kinds");
|
|
enum { W_NONE = KERN_WAIT_NONE, W_LOCK = KERN_WAIT_LOCK, W_CONDVAR = KERN_WAIT_CONDVAR, W_ADDRESS = KERN_WAIT_ADDRESS,
|
|
W_SYNC = KERN_WAIT_SYNC };
|
|
|
|
// Scheduler accounting states (TOTK_TRACE=sched).
|
|
enum { S_OFF, S_RUN, S_CORE, S_HOST, S_SLEEP, S_SYNC, S_CV, S_LOCK, S_ADDR, S_N };
|
|
static const char *const s_name[S_N] = {"off", "run", "core", "host", "sleep", "sync", "cv", "lock", "addr"};
|
|
|
|
typedef struct Event { int signaled, refs; } Event;
|
|
typedef struct Fiber Fiber;
|
|
|
|
struct RtThread {
|
|
int preempt; // first member: rt_dispatch reads it (host.h kern_preempt_pending)
|
|
int host_depth; // second member: host calls and SVCs in progress, while kern_host_track (host.h)
|
|
uint32_t handle;
|
|
uint64_t id;
|
|
Cpu *cpu; // the thread's own context
|
|
Cpu *cur; // the context running now: cpu, or the current fiber's
|
|
uint64_t entry, arg, stack_top;
|
|
int prio, core;
|
|
uint64_t mask;
|
|
int started, exited, closed, cancel, is_main;
|
|
CONDITION_VARIABLE cv;
|
|
// current wait
|
|
int wkind, woke;
|
|
uint64_t waddr, wkey;
|
|
uint32_t wtag, wres;
|
|
uint64_t wseq; // registration order: equal priority wakes first come first
|
|
RtThread *next; // wait list
|
|
RtThread *all; // every thread
|
|
// virtual core
|
|
int cur_core; // core held while running guest code, -1 while blocked
|
|
uint64_t cseq;
|
|
RtThread *cnext;
|
|
uint8_t *tlr, *tls;
|
|
jmp_buf exit_jmp; // the thread start; ExitThread longjmps to it
|
|
// fibers
|
|
void *base_fiber; // the host thread as a host fiber, after the first switch
|
|
Fiber *fiber; // current guest fiber, NULL on the thread's own context
|
|
Fiber *leaving; // fiber just switched away from, released by whoever resumes next
|
|
int exit_pending; // ExitThread on a fiber: unwind once back on the thread's own context
|
|
HANDLE timer; // high-resolution waitable timer for SleepThread
|
|
// TOTK_TRACE=sched accounting (holding G)
|
|
DWORD tid; // host thread id
|
|
int st; // S_* state since st_since
|
|
uint64_t st_since, st_ns[S_N], nblk, npre, pre_since, cpu_last;
|
|
uint64_t wobj; // what the current wait is on: owner handle, key, address or first handle
|
|
uint32_t waker; // the thread whose signal ended the last wait (0: a host thread or a timeout)
|
|
uint64_t wake_at, grant_at; // when a wait was signalled and a core granted, until the thread runs again
|
|
uint64_t lw_ns, lw_n, lw_max, lg_ns, lg_n, lg_max, sl_n, sl_req, sl_act; // those latencies; SleepThread asked/took
|
|
uint64_t sp_n, sp_ho, sp_site; // spin wait passes and core handoffs, the last loop head (the thread writes them)
|
|
RtThread *target; // as a core waiter: the owner core_kick asked to give its core up for it (holding G)
|
|
// TOTK_ASYNC_PREEMPT (holding G)
|
|
HANDLE hthr; // the host thread, for SuspendThread / Get/SetThreadContext; open while it may own a core
|
|
uint64_t ap_rip; // the interrupted host rip kern_ap_stub returns to
|
|
uint64_t ap_last; // the last redirect attempt on this owner
|
|
int ap_pending; // redirected to kern_ap_stub, which has not reached kern_ap_yield yet
|
|
// TOTK_CORE_DIAG (holding G, except dg_host_end, which only the thread itself writes)
|
|
uint64_t dg_since; // as a waiter: core wait start
|
|
int dg_cls; // as a waiter: DG_* class at the start of the wait
|
|
uint64_t dg_kick; // as an owner: the preempt request
|
|
uint64_t dg_host_end; // as an owner: the return from the host call or SVC it was in at the request
|
|
uint64_t dg_pc, dg_lr; // as an owner: guest pc and lr at the checkpoint that gave the core up
|
|
int dg_host_kick; // as an owner: in a host call or SVC at the request
|
|
uint32_t dg_svc; // the thread's latest SVC (TOTK_CORE_DIAG: what an owner in a host call is in)
|
|
struct DgRel { uint64_t kick, host_end, pc, lr; int host, pre, prio; uint32_t h; } dg_rel; // as a waiter: the
|
|
// owner that released
|
|
};
|
|
|
|
static SRWLOCK G = SRWLOCK_INIT;
|
|
static struct { int type; void *obj; } g_h[MAX_H];
|
|
static _Thread_local RtThread *tcur;
|
|
_Static_assert(offsetof(RtThread, preempt) == 0, "host.h kern_preempt_pending reads the first int");
|
|
_Static_assert(offsetof(RtThread, host_depth) == sizeof(int), "host.h kern_host_depth reads the second int");
|
|
static RtThread *g_all, *g_wait, *g_main;
|
|
static uint64_t g_next_id = 0x51, g_wseq, g_created;
|
|
static uint64_t g_main_stack;
|
|
|
|
static void lock(void) { AcquireSRWLockExclusive(&G); }
|
|
static void unlock(void) { ReleaseSRWLockExclusive(&G); }
|
|
|
|
uint64_t kern_now_ns(void) { return plat_now_ns(); }
|
|
|
|
// ---- TOTK_CRIT=<first>-<last frame> with TOTK_TRACE=sched: every state change and wake of the guest threads and
|
|
// every present over those frames, written to TOTK_CRIT_OUT (default crit.bin) at the last one, for
|
|
// tools/crit_path.py, which walks the chain of waits back from each present (holding G). TOTK_CRIT=w<lo>-<hi> takes
|
|
// world-clock seconds instead (frame_step_world_us): from the first present at or after lo to the first after hi, so a
|
|
// session covers the same game time at any frame rate.
|
|
typedef struct { uint64_t ts, obj; uint32_t a, b; uint16_t kind, st; uint32_t pad; } CritEv;
|
|
enum { CRIT_STATE, CRIT_WAKE, CRIT_FRAME }; // a: thread; b: its waker (STATE), the waker (WAKE); st: new S_* state
|
|
#define CRIT_N (1u << 22) // 128 MiB, about 2000 fight frames
|
|
static CritEv *g_crit;
|
|
static uint32_t g_crit_n;
|
|
static int g_crit_on;
|
|
static uint64_t g_crit_lo, g_crit_hi; // frames; UINT64_MAX in world mode until the world clock fixes them
|
|
static int64_t g_crit_wlo, g_crit_whi; // world mode: the window in microseconds of world clock, else 0
|
|
|
|
static void crit_add(int kind, uint32_t a, uint32_t b, int st, uint64_t obj, uint64_t ts) {
|
|
if (g_crit_n < CRIT_N) g_crit[g_crit_n++] = (CritEv){ts, obj, a, b, (uint16_t)kind, (uint16_t)st, 0};
|
|
}
|
|
|
|
// ---- TOTK_TRACE=sched accounting (holding G)
|
|
#define RW_N 32
|
|
static RtThread *g_render; // the thread that presents (sched trace, core diag)
|
|
typedef struct { int st; uint32_t waker; uint64_t obj, ns; } WaitRec;
|
|
static WaitRec g_rw[RW_N], g_mw[RW_N]; // the render and main thread's waits by object and waker
|
|
static uint64_t g_pre_req, g_pre_n, g_pre_ns, g_pre_max, g_pre_stale; // preempt requests; releases after one and their latency
|
|
static uint64_t g_sync_calls, g_sync_blocks; // WaitSynchronization calls and the blocks inside them
|
|
|
|
static void wait_add(WaitRec *w, int st, uint64_t obj, uint32_t waker, uint64_t ns) {
|
|
for (int i = 0; i < RW_N; i++)
|
|
if (!w[i].ns || (w[i].st == st && w[i].obj == obj && w[i].waker == waker)) {
|
|
w[i].st = st;
|
|
w[i].obj = obj;
|
|
w[i].waker = waker;
|
|
w[i].ns += ns;
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Close the current state's interval and enter st.
|
|
static void acct(RtThread *t, int st) {
|
|
if (!(rt_trace & RT_TRACE_SCHED)) return;
|
|
uint64_t now = kern_now_ns();
|
|
if (t->st_since) {
|
|
t->st_ns[t->st] += now - t->st_since;
|
|
if (t->st >= S_HOST && (t == g_render || t->is_main)) {
|
|
uint32_t waker = t->st == S_HOST || t->st == S_SLEEP ? 0 : t->waker;
|
|
wait_add(t == g_render ? g_rw : g_mw, t->st, t->wobj, waker, now - t->st_since);
|
|
}
|
|
}
|
|
if (g_crit_on) crit_add(CRIT_STATE, t->handle, t->waker, st, t->wobj, now);
|
|
t->st = st;
|
|
t->st_since = now;
|
|
}
|
|
|
|
static int st_of(int kind) {
|
|
return kind == W_LOCK ? S_LOCK : kind == W_CONDVAR ? S_CV : kind == W_ADDRESS ? S_ADDR : S_SYNC;
|
|
}
|
|
|
|
// ---- handles (caller holds G)
|
|
static uint32_t h_new(int type, void *obj) {
|
|
for (int i = 1; i < MAX_H; i++)
|
|
if (!g_h[i].type) {
|
|
g_h[i].type = type;
|
|
g_h[i].obj = obj;
|
|
return 0x1000 + i;
|
|
}
|
|
rt_fatal("kernel: handle table full (%d handles)", MAX_H);
|
|
}
|
|
|
|
static void *h_peek(uint32_t h, int type) {
|
|
if (h == H_CUR_THREAD && type == O_THREAD) return tcur;
|
|
unsigned i = h - 0x1000;
|
|
if (i == 0 || i >= MAX_H || g_h[i].type != type) return NULL;
|
|
return g_h[i].obj;
|
|
}
|
|
|
|
static void *h_get(uint32_t h, int type) {
|
|
void *o = h_peek(h, type);
|
|
if (!o) rt_log("kernel: invalid handle %#x (want kind %d)", h, type);
|
|
return o;
|
|
}
|
|
|
|
static RtThread *cur(uint32_t svc) {
|
|
if (!tcur) rt_fatal("kernel: svc %#x on a host thread (no guest thread)", svc);
|
|
return tcur;
|
|
}
|
|
|
|
// ---- wait primitives (holding G)
|
|
static void wait_unlink(RtThread *t) {
|
|
for (RtThread **p = &g_wait; *p; p = &(*p)->next)
|
|
if (*p == t) {
|
|
*p = t->next;
|
|
break;
|
|
}
|
|
t->next = NULL;
|
|
}
|
|
|
|
// The waiter to wake in kernel order. Lock waits match by lock address, condvar waits by key, arbiter waits by
|
|
// address.
|
|
static RtThread *pick(int kind, uint64_t a) {
|
|
RtThread *b = NULL;
|
|
for (RtThread *t = g_wait; t; t = t->next) {
|
|
if (t->wkind != kind || (kind == W_CONDVAR ? t->wkey : t->waddr) != a) continue;
|
|
if (!b || t->prio < b->prio || (t->prio == b->prio && t->wseq < b->wseq)) b = t;
|
|
}
|
|
return b;
|
|
}
|
|
|
|
static void wake(RtThread *t, uint32_t res) {
|
|
wait_unlink(t);
|
|
t->wkind = W_NONE;
|
|
t->woke = 1;
|
|
t->wres = res;
|
|
t->waker = tcur ? tcur->handle : 0;
|
|
if (rt_trace & RT_TRACE_SCHED) t->wake_at = kern_now_ns();
|
|
if (g_crit_on) crit_add(CRIT_WAKE, t->handle, t->waker, 0, 0, t->wake_at);
|
|
WakeConditionVariable(&t->cv);
|
|
}
|
|
|
|
static void wake_sync_waiters(void) {
|
|
for (RtThread *w = g_wait; w;) {
|
|
RtThread *n = w->next;
|
|
if (w->wkind == W_SYNC) wake(w, (uint32_t)-1);
|
|
w = n;
|
|
}
|
|
}
|
|
|
|
// SleepConditionVariableSRW counts milliseconds: round up (late is legal, early is not) and re-check the deadline
|
|
// against the performance counter, because the wait may also end early.
|
|
static uint32_t block_inner(RtThread *t, int kind, int64_t ns) {
|
|
t->wkind = kind;
|
|
t->woke = 0;
|
|
t->wseq = ++g_wseq;
|
|
t->next = g_wait;
|
|
g_wait = t;
|
|
if (ns == 0) {
|
|
wait_unlink(t);
|
|
t->wkind = W_NONE;
|
|
return R_TIMEOUT;
|
|
}
|
|
uint64_t now = kern_now_ns();
|
|
int forever = ns < 0 || (uint64_t)ns > UINT64_MAX - now;
|
|
uint64_t deadline = forever ? 0 : now + (uint64_t)ns;
|
|
while (!t->woke) {
|
|
if (forever) {
|
|
SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
|
|
continue;
|
|
}
|
|
now = kern_now_ns();
|
|
if (now >= deadline) {
|
|
wait_unlink(t);
|
|
t->wkind = W_NONE;
|
|
return R_TIMEOUT;
|
|
}
|
|
uint64_t ms = (deadline - now + 999999) / 1000000;
|
|
SleepConditionVariableSRW(&t->cv, &G, ms > 0xFFFFFFF0u ? 0xFFFFFFF0u : (DWORD)ms, 0);
|
|
}
|
|
if (t->wake_at) {
|
|
uint64_t d = kern_now_ns() - t->wake_at;
|
|
t->lw_ns += d;
|
|
t->lw_n++;
|
|
if (d > t->lw_max) t->lw_max = d;
|
|
t->wake_at = 0;
|
|
}
|
|
return t->wres;
|
|
}
|
|
|
|
// ---- virtual cores
|
|
// Horizon runs at most one thread per core, and TotK keys per-core data (allocators, job queues) by the current core
|
|
// number without locks. A guest thread therefore holds one of the three application cores while it runs guest code
|
|
// and gives it up only where Horizon would switch threads (waits, sleep, yield, exit, blocking host calls); a free
|
|
// core goes to the best waiting thread. A waiter better than a holder asks one holder to give its core up (core_kick),
|
|
// which it does at its next SVC or indirect call (kern_checkpoint, rt_dispatch), as Horizon runs the better thread at
|
|
// once; guest code without either keeps the core until it reaches one. TOTK_FREE_CORES=1 turns the gate off.
|
|
static RtThread *g_core_owner[NCORE], *g_core_wait;
|
|
static int g_cores_on = -1, g_held_max;
|
|
|
|
static void diag_init(void);
|
|
static void ap_init(void);
|
|
|
|
static int cores_enabled(void) {
|
|
if (g_cores_on < 0) {
|
|
g_cores_on = !cfg_bool("TOTK_FREE_CORES"); // "0" keeps the gate, like unset
|
|
diag_init();
|
|
ap_init();
|
|
}
|
|
return g_cores_on;
|
|
}
|
|
|
|
static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) & 1); }
|
|
|
|
// ---- asynchronous preemption (TOTK_ASYNC_PREEMPT, on unless 0)
|
|
// Adapted from MacNativeGames games/totk/runtime/threads.c @ 33e8fa8 (8bf0893, PR #488; MIT, Copyright (c) 2026
|
|
// MacNativeGames contributors): Horizon gives a core to a strictly better thread the moment that thread is ready.
|
|
// Here a better waiter only marks the owner, which gives the core up at its next SVC or indirect call; an owner in a
|
|
// stretch of game code with neither keeps it (TOTK_CORE_DIAG: 0.5-1.1 ms of thread-time per frame, waits to 16 ms and
|
|
// more). With it on, three parts, all of which MacNativeGames found necessary:
|
|
// - The waiter interrupts the owner (SuspendThread, GetThreadContext) when it has waited AP_FIRST_NS, then every
|
|
// millisecond: if the owner's host rip is in the lifted modules' code (g_gen_lo..g_gen_hi) and its host depth is 0
|
|
// (not in a host call or SVC, nor in game code a host call made), it holds no host lock, so its rip is moved to
|
|
// kern_ap_stub, which saves every volatile register, gives the core up like a checkpoint (kern_ap_yield) and jumps
|
|
// back. Otherwise it is only marked, as before.
|
|
// - Priority inheritance (eff_prio): a thread owning a guest mutex counts at the priority of the best thread blocked on
|
|
// it, for granting, kicking, preempting and yielding. Without it MacNativeGames' game stopped at its first
|
|
// "Continue" (a better thread took the core of the low-priority owner of a mutex it then waited for).
|
|
// - Horizon's preemption disable: the SDK's spin locks (1.2.1 sdk f_70051242a8, f_7005124360 and siblings) count
|
|
// TLR+0x100 (disable_count) while they try or hold the lock, and after clearing it call SVC 0x36 when TLR+0x102
|
|
// (interrupt_flag) is set. Such an owner is not preempted, at a checkpoint either: interrupt_flag is set and the
|
|
// switch happens at that SVC. Preempting it anyway hung MacNativeGames' game at "Continue" (the lock holder lost its
|
|
// core to a thread that then spun on that lock).
|
|
// TOTK_ASYNC_PREEMPT=0 keeps the old scheduling exactly: raw priorities, no interruption, no disable_count; =2 keeps
|
|
// inheritance and the preemption disable without interrupting owners.
|
|
#define AP_FIRST_NS 250000ull // the waiter spins this long before the first interruption (most waits end sooner)
|
|
#define AP_GAP_NS 1000000ull // and interrupts the same owner at most this often
|
|
static int g_ap = -1, g_ap_int; // g_ap_int: interrupt owners (TOTK_ASYNC_PREEMPT=2: inheritance and disable only)
|
|
static int g_ap_log; // redirects logged so far (AP_LOG_N)
|
|
#define AP_LOG_N 40
|
|
static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start
|
|
static uint64_t g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok;
|
|
void kern_ap_stub(void);
|
|
|
|
static void ap_init(void) {
|
|
if (g_ap >= 0) return;
|
|
const char *e = cfg_value("TOTK_ASYNC_PREEMPT");
|
|
g_ap = strcmp(e, "0") != 0;
|
|
g_ap_int = g_ap && strcmp(e, "2") != 0;
|
|
if (g_ap) {
|
|
RtModule m[16];
|
|
int n = rt_linked_modules(m, 16);
|
|
uintptr_t lo = UINTPTR_MAX, hi = 0;
|
|
for (int i = 0; i < n; i++)
|
|
for (uint64_t k = 0; k < m[i].count; k++) {
|
|
uintptr_t f = (uintptr_t)m[i].fns[k];
|
|
if (f < lo) lo = f;
|
|
if (f > hi) hi = f;
|
|
}
|
|
const uintptr_t host[] = {(uintptr_t)kern_ap_stub, (uintptr_t)kern_checkpoint, (uintptr_t)rt_svc,
|
|
(uintptr_t)rt_dispatch, (uintptr_t)rt_call};
|
|
int bad = !n || lo >= hi;
|
|
for (unsigned i = 0; i < sizeof host / sizeof *host; i++) bad |= host[i] >= lo && host[i] <= hi;
|
|
if (bad) {
|
|
rt_log("kernel: asynchronous preemption off: the lifted code range %#llx-%#llx holds host code",
|
|
(unsigned long long)lo, (unsigned long long)hi);
|
|
g_ap = 0;
|
|
} else {
|
|
g_gen_lo = lo;
|
|
g_gen_hi = hi;
|
|
kern_host_track = 1;
|
|
}
|
|
}
|
|
rt_log("kernel: asynchronous preemption of virtual cores %s, priority inheritance and preemption disable %s; lifted "
|
|
"code exe+%#llx-exe+%#llx", g_ap_int ? "on" : "off", g_ap ? "on (TOTK_ASYNC_PREEMPT=0 turns all off)" : "off",
|
|
(unsigned long long)(g_gen_lo ? g_gen_lo - (uintptr_t)GetModuleHandleW(NULL) : 0),
|
|
(unsigned long long)(g_gen_hi ? g_gen_hi - (uintptr_t)GetModuleHandleW(NULL) : 0));
|
|
}
|
|
|
|
// Horizon's priority inheritance: o counts at the priority of the best thread blocked on a guest mutex it owns (the
|
|
// owner is read from the mutex word, so it follows hand-offs). Holding G.
|
|
static int eff_prio(const RtThread *o) {
|
|
int p = o->prio;
|
|
for (const RtThread *w = g_wait; w; w = w->next)
|
|
if (w->wkind == W_LOCK && w->prio < p && w->waddr &&
|
|
(__atomic_load_n((const uint32_t *)(uintptr_t)w->waddr, __ATOMIC_RELAXED) & ~WAITERS_BIT) == o->handle)
|
|
p = w->prio;
|
|
return p;
|
|
}
|
|
|
|
// The priority the virtual cores go by: raw with TOTK_ASYNC_PREEMPT=0, as before.
|
|
static int prio_of(const RtThread *t) { return g_ap > 0 ? eff_prio(t) : t->prio; }
|
|
|
|
// TLR+0x100 disable_count, +0x102 interrupt_flag (any thread may read them; the flag is set from outside as the
|
|
// kernel does).
|
|
static volatile uint16_t *ap_tlr(const RtThread *t) { return (volatile uint16_t *)(t->tlr + 0x100); }
|
|
static int ap_disabled(const RtThread *t) { return g_ap > 0 && ap_tlr(t)[0]; }
|
|
|
|
// kern_ap_stub: entered in place of the interrupted instruction of top-level game code. It keeps every register the
|
|
// game code may have live (volatile GPRs, flags, YMM0-15, MXCSR; the C calls preserve the rest), gives the core up
|
|
// in kern_ap_yield and jumps back to kern_ap_rip(), whose value it first stores in the return slot it reserved at
|
|
// entry, so a stack walk sees the interrupted function as its caller. No ret: the jump keeps call/return pairs
|
|
// balanced. The waiter never writes the owner's stack.
|
|
__asm__(".text\n"
|
|
".globl kern_ap_stub\n"
|
|
".def kern_ap_stub; .scl 2; .type 32; .endef\n"
|
|
".p2align 4\n"
|
|
"kern_ap_stub:\n"
|
|
".seh_proc kern_ap_stub\n"
|
|
" leaq -8(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq
|
|
".seh_stackalloc 8\n"
|
|
" pushfq\n"
|
|
".seh_stackalloc 8\n"
|
|
" pushq %rax\n"
|
|
".seh_pushreg %rax\n"
|
|
" pushq %rcx\n"
|
|
".seh_pushreg %rcx\n"
|
|
" pushq %rdx\n"
|
|
".seh_pushreg %rdx\n"
|
|
" pushq %r8\n"
|
|
".seh_pushreg %r8\n"
|
|
" pushq %r9\n"
|
|
".seh_pushreg %r9\n"
|
|
" pushq %r10\n"
|
|
".seh_pushreg %r10\n"
|
|
" pushq %r11\n"
|
|
".seh_pushreg %r11\n"
|
|
" pushq %rbp\n"
|
|
".seh_pushreg %rbp\n"
|
|
" movq %rsp, %rbp\n"
|
|
".seh_setframe %rbp, 0\n"
|
|
".seh_endprologue\n"
|
|
" andq $-32, %rsp\n"
|
|
" subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15
|
|
" vmovdqa %ymm0, 64(%rsp)\n"
|
|
" vmovdqa %ymm1, 96(%rsp)\n"
|
|
" vmovdqa %ymm2, 128(%rsp)\n"
|
|
" vmovdqa %ymm3, 160(%rsp)\n"
|
|
" vmovdqa %ymm4, 192(%rsp)\n"
|
|
" vmovdqa %ymm5, 224(%rsp)\n"
|
|
" vmovdqa %ymm6, 256(%rsp)\n"
|
|
" vmovdqa %ymm7, 288(%rsp)\n"
|
|
" vmovdqa %ymm8, 320(%rsp)\n"
|
|
" vmovdqa %ymm9, 352(%rsp)\n"
|
|
" vmovdqa %ymm10, 384(%rsp)\n"
|
|
" vmovdqa %ymm11, 416(%rsp)\n"
|
|
" vmovdqa %ymm12, 448(%rsp)\n"
|
|
" vmovdqa %ymm13, 480(%rsp)\n"
|
|
" vmovdqa %ymm14, 512(%rsp)\n"
|
|
" vmovdqa %ymm15, 544(%rsp)\n"
|
|
" stmxcsr 32(%rsp)\n"
|
|
" movl $0x1f80, 36(%rsp)\n"
|
|
" ldmxcsr 36(%rsp)\n"
|
|
" cld\n"
|
|
" vzeroupper\n"
|
|
" call kern_ap_rip\n"
|
|
" movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq
|
|
" call kern_ap_yield\n"
|
|
" ldmxcsr 32(%rsp)\n"
|
|
" vmovdqa 64(%rsp), %ymm0\n"
|
|
" vmovdqa 96(%rsp), %ymm1\n"
|
|
" vmovdqa 128(%rsp), %ymm2\n"
|
|
" vmovdqa 160(%rsp), %ymm3\n"
|
|
" vmovdqa 192(%rsp), %ymm4\n"
|
|
" vmovdqa 224(%rsp), %ymm5\n"
|
|
" vmovdqa 256(%rsp), %ymm6\n"
|
|
" vmovdqa 288(%rsp), %ymm7\n"
|
|
" vmovdqa 320(%rsp), %ymm8\n"
|
|
" vmovdqa 352(%rsp), %ymm9\n"
|
|
" vmovdqa 384(%rsp), %ymm10\n"
|
|
" vmovdqa 416(%rsp), %ymm11\n"
|
|
" vmovdqa 448(%rsp), %ymm12\n"
|
|
" vmovdqa 480(%rsp), %ymm13\n"
|
|
" vmovdqa 512(%rsp), %ymm14\n"
|
|
" vmovdqa 544(%rsp), %ymm15\n"
|
|
" movq %rbp, %rsp\n"
|
|
" popq %rbp\n"
|
|
" popq %r11\n"
|
|
" popq %r10\n"
|
|
" popq %r9\n"
|
|
" popq %r8\n"
|
|
" popq %rdx\n"
|
|
" popq %rcx\n"
|
|
" popq %rax\n"
|
|
" popfq\n"
|
|
" leaq 8(%rsp), %rsp\n"
|
|
" jmpq *-8(%rsp)\n"
|
|
".seh_endproc\n");
|
|
|
|
enum { AP_NONE, AP_RETRY, AP_SENT };
|
|
|
|
// The waiter w interrupts its target owner if that runs top-level game code (holding G; the owner is suspended only
|
|
// between SuspendThread and ResumeThread, during which nothing here takes a lock).
|
|
static int ap_try(RtThread *w) {
|
|
RtThread *o = w->target;
|
|
if (!o || o->cur_core < 0 || !o->preempt) return AP_NONE;
|
|
if (!o->hthr || o->ap_pending) return AP_RETRY;
|
|
volatile uint16_t *tl = ap_tlr(o);
|
|
if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0) {
|
|
g_ap_host++;
|
|
return AP_RETRY;
|
|
}
|
|
if (tl[0]) { // preemption disabled: SVC 0x36 switches
|
|
tl[1] = 1;
|
|
g_ap_deferred++;
|
|
return AP_RETRY;
|
|
}
|
|
uint64_t now = kern_now_ns();
|
|
if (o->ap_last && now - o->ap_last < AP_GAP_NS) return AP_RETRY;
|
|
o->ap_last = now;
|
|
if (SuspendThread(o->hthr) == (DWORD)-1) {
|
|
g_ap_fail++;
|
|
return AP_RETRY;
|
|
}
|
|
CONTEXT ctx;
|
|
memset(&ctx, 0, sizeof ctx);
|
|
ctx.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; // set back whole, so the kernel restores every GPR
|
|
int r = AP_RETRY;
|
|
if (!GetThreadContext(o->hthr, &ctx))
|
|
g_ap_fail++;
|
|
else if (ctx.Rip < g_gen_lo || ctx.Rip > g_gen_hi)
|
|
g_ap_pc++;
|
|
else if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0)
|
|
g_ap_host++;
|
|
else if (tl[0]) {
|
|
tl[1] = 1;
|
|
g_ap_deferred++;
|
|
} else {
|
|
o->ap_rip = ctx.Rip;
|
|
o->ap_pending = 1;
|
|
ctx.Rip = (DWORD64)(uintptr_t)kern_ap_stub;
|
|
if (SetThreadContext(o->hthr, &ctx)) {
|
|
g_ap_sent++;
|
|
r = AP_SENT;
|
|
} else {
|
|
o->ap_pending = 0;
|
|
g_ap_fail++;
|
|
}
|
|
}
|
|
ResumeThread(o->hthr);
|
|
if (r == AP_SENT && g_ap_log < AP_LOG_N) {
|
|
g_ap_log++;
|
|
rt_log("kernel: interrupt %#x p%d at exe+%#llx rsp %#llx for %#x p%d", o->handle, o->prio,
|
|
(unsigned long long)(o->ap_rip - (uintptr_t)GetModuleHandleW(NULL)), (unsigned long long)ctx.Rsp,
|
|
w->handle, w->prio);
|
|
}
|
|
return r;
|
|
}
|
|
|
|
// ---- TOTK_CORE_DIAG=1: why ready threads wait for a virtual core, and for how long
|
|
// Each core wait is split into classes: the owner asked to give the core up ran game code (DG_GAME, the part an
|
|
// asynchronous preemption would remove), was in a host call or SVC that kept the core (DG_HOST), every owner the waiter
|
|
// may displace was of better or equal priority (DG_EQ: Horizon waits too), or the worse owners were claimed by better
|
|
// waiters (DG_NOCORE). Per 60 presents: time per frame and a wait-length histogram per class, the owners that cost
|
|
// most; a line per present over 25 ms; a line per wait with over 2 ms of DG_GAME (where the owner reached its
|
|
// checkpoint), and one host RIP sample of the owner per such wait (exe RVA, for totk.map).
|
|
enum { DG_GAME, DG_HOST, DG_EQ, DG_NOCORE, DG_N };
|
|
static const char *const dg_name[DG_N] = {"game", "host", "eq", "nocore"};
|
|
#define DG_NB 8 // wait lengths: < 0.05, 0.25, 1, 2, 4, 8, 16 ms, longer
|
|
static const uint64_t dg_edge[DG_NB - 1] = {50000, 250000, 1000000, 2000000, 4000000, 8000000, 16000000};
|
|
#define DG_LONG_LINES 400
|
|
#define DG_SAMPLES 600
|
|
#define DG_OWN 32
|
|
int kern_host_track;
|
|
static int g_diag = -1;
|
|
typedef struct {
|
|
uint64_t ns[DG_N], n[DG_N], hist[DG_N][DG_NB]; // this window
|
|
struct { uint32_t h; int prio, cls; uint64_t ns; } own[DG_OWN]; // this window: owners by class
|
|
uint64_t long_frames, long_game_frames;
|
|
} DgWindow;
|
|
static DgWindow g_dgw;
|
|
static uint64_t g_dgf[DG_N], g_dgfr[DG_N]; // this frame: all threads, the render thread
|
|
static uint64_t g_dg_samples, g_dg_lines, g_sync36, g_sync36_flag;
|
|
|
|
static void diag_init(void) {
|
|
if (g_diag >= 0) return;
|
|
const char *e = cfg_get("TOTK_CORE_DIAG");
|
|
g_diag = e && *e && strcmp(e, "0");
|
|
if (g_diag) {
|
|
kern_host_track = 1;
|
|
rt_log("core-diag: on (TOTK_CORE_DIAG)");
|
|
}
|
|
}
|
|
|
|
int kern_core_diag_on(void) { return g_diag > 0; }
|
|
|
|
// A tracked host call or SVC returned to depth 0 with a preempt request pending (host.h); on the thread itself.
|
|
void kern_host_left(void *thread) {
|
|
RtThread *t = thread;
|
|
if (g_diag > 0 && t->dg_kick && t->dg_host_kick && !t->dg_host_end) t->dg_host_end = kern_now_ns();
|
|
}
|
|
|
|
static int dg_in_host(const RtThread *o) { return __atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0; }
|
|
|
|
static void dg_add(const RtThread *w, int cls, uint64_t ns, uint32_t oh, int oprio) {
|
|
if (!ns) return;
|
|
g_dgw.ns[cls] += ns;
|
|
g_dgf[cls] += ns;
|
|
if (w == g_render) g_dgfr[cls] += ns;
|
|
if (!oh) return;
|
|
int i = 0;
|
|
for (; i < DG_OWN && g_dgw.own[i].ns; i++)
|
|
if (g_dgw.own[i].h == oh && g_dgw.own[i].cls == cls) break;
|
|
if (i == DG_OWN) return;
|
|
g_dgw.own[i].h = oh;
|
|
g_dgw.own[i].prio = oprio;
|
|
g_dgw.own[i].cls = cls;
|
|
g_dgw.own[i].ns += ns;
|
|
}
|
|
|
|
// The waiter's class at the start of its wait, after core_kick claimed (or did not claim) an owner for it.
|
|
static int dg_class0(const RtThread *t) {
|
|
if (t->target) return dg_in_host(t->target) ? DG_HOST : DG_GAME;
|
|
for (int i = 0; i < NCORE; i++)
|
|
if (core_allowed(t, i) && g_core_owner[i] && g_core_owner[i]->prio > t->prio) return DG_NOCORE;
|
|
return DG_EQ;
|
|
}
|
|
|
|
// The waiter got a core: split [dg_since, now] by what the owner that released it did after its preempt request.
|
|
static void dg_done(RtThread *t) {
|
|
uint64_t now = kern_now_ns(), t0 = t->dg_since, w = now - t0;
|
|
t->dg_since = 0;
|
|
if (!t0) return;
|
|
const struct DgRel *r = &t->dg_rel;
|
|
uint64_t part[DG_N] = {0};
|
|
uint32_t oh = t->target ? t->target->handle : r->h;
|
|
int oprio = t->target ? t->target->prio : r->prio;
|
|
if (r->pre && r->kick) {
|
|
uint64_t k = r->kick > t0 ? (r->kick < now ? r->kick : now) : t0;
|
|
part[t->dg_cls] += k - t0;
|
|
if (r->host) {
|
|
uint64_t he = r->host_end > k && r->host_end < now ? r->host_end : now;
|
|
part[DG_HOST] += he - k;
|
|
part[DG_GAME] += now - he;
|
|
} else
|
|
part[DG_GAME] += now - k;
|
|
oh = r->h;
|
|
oprio = r->prio;
|
|
} else
|
|
part[t->dg_cls] += w;
|
|
int top = 0;
|
|
for (int c = 0; c < DG_N; c++) {
|
|
dg_add(t, c, part[c], oh, oprio);
|
|
if (part[c] > part[top]) top = c;
|
|
}
|
|
int b = 0;
|
|
while (b < DG_NB - 1 && w >= dg_edge[b]) b++;
|
|
g_dgw.n[top]++;
|
|
g_dgw.hist[top][b]++;
|
|
if ((part[DG_GAME] >= 2000000 || part[DG_HOST] >= 2000000) && g_dg_lines < DG_LONG_LINES) {
|
|
g_dg_lines++;
|
|
char a[128], l[128];
|
|
rt_log("core-diag: wait %.2f ms (game %.2f host %.2f): %#x p%d mask %llx; released by %#x p%d%s at %s lr %s",
|
|
w / 1e6, part[DG_GAME] / 1e6, part[DG_HOST] / 1e6, t->handle, t->prio, (unsigned long long)t->mask, r->h,
|
|
r->prio, r->pre ? " (preempted)" : "", r->pc ? ldr_describe(r->pc, a, sizeof a) : "-",
|
|
r->lr ? ldr_describe(r->lr, l, sizeof l) : "-");
|
|
}
|
|
t->target = NULL;
|
|
memset(&t->dg_rel, 0, sizeof t->dg_rel);
|
|
}
|
|
|
|
// Once per long DG_GAME or DG_HOST wait: where the owner's host thread runs (module and offset), its latest SVC and
|
|
// its guest lr. Holding G; the owner is resumed before anything that could take a lock it holds.
|
|
static void dg_sample(RtThread *t) {
|
|
RtThread *o = t->target;
|
|
if (!o || o->cur_core < 0 || !o->tid || g_dg_samples >= DG_SAMPLES) return;
|
|
HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, o->tid);
|
|
if (!h) return;
|
|
CONTEXT ctx;
|
|
memset(&ctx, 0, sizeof ctx);
|
|
ctx.ContextFlags = CONTEXT_CONTROL;
|
|
int depth = 0, ok = 0;
|
|
if (SuspendThread(h) != (DWORD)-1) {
|
|
ok = GetThreadContext(h, &ctx);
|
|
depth = __atomic_load_n(&o->host_depth, __ATOMIC_RELAXED);
|
|
ResumeThread(h);
|
|
}
|
|
CloseHandle(h);
|
|
if (!ok) return;
|
|
g_dg_samples++;
|
|
HMODULE m = NULL;
|
|
wchar_t mw[MAX_PATH] = L"?";
|
|
if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
|
|
(LPCWSTR)(uintptr_t)ctx.Rip, &m) && m)
|
|
GetModuleFileNameW(m, mw, MAX_PATH);
|
|
const wchar_t *mn = wcsrchr(mw, L'\\') ? wcsrchr(mw, L'\\') + 1 : mw;
|
|
char lr[128];
|
|
rt_log("core-diag: sample owner %#x p%d depth %d rip %ls+%#llx, last svc %#x, guest lr %s (waiter %#x p%d, %s, "
|
|
"%.2f ms)", o->handle, o->prio, depth, mn, (unsigned long long)(ctx.Rip - (uintptr_t)m), o->dg_svc,
|
|
o->cur ? ldr_describe(o->cur->x[30], lr, sizeof lr) : "-", t->handle, t->prio,
|
|
t->dg_cls == DG_HOST ? "host" : "game", (kern_now_ns() - t->dg_since) / 1e6);
|
|
}
|
|
|
|
|
|
// perf_present, every present: a line for a present over 25 ms, the 60-present summary.
|
|
void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns) {
|
|
if (g_ap > 0 && frame && frame % 600 == 0) { // TOTK_ASYNC_PREEMPT counters since the start
|
|
lock();
|
|
uint64_t v[8] = {g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok, g_sync36};
|
|
unlock();
|
|
rt_log("kernel: frame %llu: asynchronous preemption: %llu interrupted, %llu cores given up there; refused: "
|
|
"%llu in a host call, %llu outside game code, %llu preemption disabled, %llu failed; %llu waits ended "
|
|
"in the first spin; SVC 0x36 %llu",
|
|
(unsigned long long)frame, (unsigned long long)v[0], (unsigned long long)v[1], (unsigned long long)v[2],
|
|
(unsigned long long)v[3], (unsigned long long)v[4], (unsigned long long)v[5], (unsigned long long)v[6],
|
|
(unsigned long long)v[7]);
|
|
}
|
|
if (g_diag <= 0) return;
|
|
DgWindow w;
|
|
uint64_t f[DG_N], fr[DG_N], s36, s36f;
|
|
lock();
|
|
g_render = tcur;
|
|
memcpy(f, g_dgf, sizeof f);
|
|
memcpy(fr, g_dgfr, sizeof fr);
|
|
memset(g_dgf, 0, sizeof g_dgf);
|
|
memset(g_dgfr, 0, sizeof g_dgfr);
|
|
int lng = interval_ns > 25000000ull;
|
|
if (lng) {
|
|
g_dgw.long_frames++;
|
|
if (f[DG_GAME] > 1000000) g_dgw.long_game_frames++;
|
|
}
|
|
int rep = frame && frame % 60 == 0;
|
|
if (rep) {
|
|
w = g_dgw;
|
|
memset(&g_dgw, 0, sizeof g_dgw);
|
|
}
|
|
s36 = g_sync36;
|
|
s36f = g_sync36_flag;
|
|
unlock();
|
|
if (lng)
|
|
rt_log("core-diag: frame %llu: %.1f ms; core wait game %.2f host %.2f eq %.2f nocore %.2f ms; render thread game "
|
|
"%.2f host %.2f eq %.2f nocore %.2f ms",
|
|
(unsigned long long)frame, interval_ns / 1e6, f[0] / 1e6, f[1] / 1e6, f[2] / 1e6, f[3] / 1e6,
|
|
fr[0] / 1e6, fr[1] / 1e6, fr[2] / 1e6, fr[3] / 1e6);
|
|
if (!rep) return;
|
|
char s[1024];
|
|
int o = 0;
|
|
for (int c = 0; c < DG_N; c++) {
|
|
o += snprintf(s + o, sizeof s - o, "%s %.3f ms/frame %llu waits [", dg_name[c], w.ns[c] / 60e6,
|
|
(unsigned long long)w.n[c]);
|
|
for (int b = 0; b < DG_NB; b++)
|
|
o += snprintf(s + o, sizeof s - o, "%s%llu", b ? " " : "", (unsigned long long)w.hist[c][b]);
|
|
o += snprintf(s + o, sizeof s - o, "]%s", c < DG_N - 1 ? "; " : "");
|
|
}
|
|
rt_log("core-diag: frames %llu-%llu: %s; long frames %llu (game wait > 1 ms in %llu); SVC 0x36 %llu (flag set %llu)",
|
|
(unsigned long long)frame - 59, (unsigned long long)frame, s, (unsigned long long)w.long_frames,
|
|
(unsigned long long)w.long_game_frames, (unsigned long long)s36, (unsigned long long)s36f);
|
|
o = 0;
|
|
for (int k = 0; k < 6; k++) {
|
|
int bi = -1;
|
|
for (int i = 0; i < DG_OWN; i++)
|
|
if (w.own[i].ns && (bi < 0 || w.own[i].ns > w.own[bi].ns)) bi = i;
|
|
if (bi < 0) break;
|
|
o += snprintf(s + o, sizeof s - o, " %#x p%d %s %.3f;", w.own[bi].h, w.own[bi].prio, dg_name[w.own[bi].cls],
|
|
w.own[bi].ns / 60e6);
|
|
w.own[bi].ns = 0;
|
|
}
|
|
if (o) rt_log("core-diag: owners (ms/frame):%s", s);
|
|
}
|
|
|
|
static void core_count(void) {
|
|
int n = 0;
|
|
for (int i = 0; i < NCORE; i++) n += g_core_owner[i] != NULL;
|
|
if (n > g_held_max) g_held_max = n;
|
|
}
|
|
|
|
static struct DgRel g_dg_rel; // TOTK_CORE_DIAG: the owner core_put is releasing, for core_grant
|
|
|
|
static void core_grant(int c) {
|
|
RtThread *b = NULL, **bp = NULL;
|
|
int bpri = 0;
|
|
for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext) {
|
|
if (!core_allowed(*p, c)) continue;
|
|
int pr = prio_of(*p);
|
|
if (!b || pr < bpri || (pr == bpri && (*p)->cseq < b->cseq)) {
|
|
b = *p;
|
|
bp = p;
|
|
bpri = pr;
|
|
}
|
|
}
|
|
if (!b) return;
|
|
*bp = b->cnext;
|
|
b->cnext = NULL;
|
|
b->cur_core = c;
|
|
g_core_owner[c] = b;
|
|
core_count();
|
|
if (g_diag > 0) b->dg_rel = g_dg_rel;
|
|
if (rt_trace & RT_TRACE_SCHED) b->grant_at = kern_now_ns();
|
|
WakeConditionVariable(&b->cv);
|
|
}
|
|
|
|
static void core_put(RtThread *t) {
|
|
if (t->cur_core < 0) return;
|
|
int c = t->cur_core;
|
|
if (t->preempt && t->pre_since) { // sched trace: time from the preempt request to this release
|
|
uint64_t d = kern_now_ns() - t->pre_since;
|
|
g_pre_n++;
|
|
g_pre_ns += d;
|
|
if (d > g_pre_max) g_pre_max = d;
|
|
t->npre++;
|
|
}
|
|
t->pre_since = 0;
|
|
if (g_diag > 0) {
|
|
g_dg_rel = (struct DgRel){.kick = t->dg_kick, .host_end = t->dg_host_end, .pc = t->dg_pc, .lr = t->dg_lr,
|
|
.host = t->dg_host_kick, .pre = t->preempt, .prio = t->prio, .h = t->handle};
|
|
t->dg_kick = t->dg_host_end = t->dg_pc = t->dg_lr = 0;
|
|
t->dg_host_kick = 0;
|
|
}
|
|
t->cur_core = -1;
|
|
t->preempt = 0;
|
|
g_core_owner[c] = NULL;
|
|
core_grant(c);
|
|
}
|
|
|
|
// Whether a waiter better than t may run on core c.
|
|
static int core_better_waiter(const RtThread *t, int c) {
|
|
int tp = prio_of(t);
|
|
for (const RtThread *w = g_core_wait; w; w = w->cnext)
|
|
if (core_allowed(w, c) && prio_of(w) < tp) return 1;
|
|
return 0;
|
|
}
|
|
|
|
// Asks holders to give their cores up, one holder per waiter: waiters best first, each claims the worst holder worse
|
|
// than itself on a core it may run on that no better waiter claimed (a request already pending is reused). Asking
|
|
// every worse holder made several drop their cores for one waiter; the spare cores went to worse waiters, which asked
|
|
// again.
|
|
static void core_kick(void) {
|
|
RtThread *w[64];
|
|
int wp[64], op[NCORE], n = 0, claimed = 0;
|
|
for (int i = 0; i < NCORE; i++) op[i] = g_core_owner[i] ? prio_of(g_core_owner[i]) : 0;
|
|
for (RtThread *p = g_core_wait; p && n < 64; p = p->cnext) {
|
|
int i = n++, pp = prio_of(p);
|
|
for (; i > 0 && (wp[i - 1] > pp || (wp[i - 1] == pp && w[i - 1]->cseq > p->cseq)); i--) {
|
|
w[i] = w[i - 1];
|
|
wp[i] = wp[i - 1];
|
|
}
|
|
w[i] = p;
|
|
wp[i] = pp;
|
|
}
|
|
for (int k = 0; k < n && claimed != (1 << NCORE) - 1; k++) {
|
|
int v = -1;
|
|
for (int i = 0; i < NCORE; i++) {
|
|
RtThread *o = g_core_owner[i];
|
|
if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || op[i] <= wp[k]) continue;
|
|
RtThread *b = v < 0 ? NULL : g_core_owner[v];
|
|
if (!b || op[i] > op[v] || (op[i] == op[v] && o->preempt && !b->preempt)) v = i;
|
|
}
|
|
if (v < 0) continue;
|
|
claimed |= 1 << v;
|
|
RtThread *o = g_core_owner[v];
|
|
w[k]->target = o;
|
|
if (o->preempt) continue;
|
|
if (rt_trace & RT_TRACE_SCHED) {
|
|
o->pre_since = kern_now_ns();
|
|
g_pre_req++;
|
|
}
|
|
if (g_diag > 0) {
|
|
o->dg_kick = kern_now_ns();
|
|
o->dg_host_kick = dg_in_host(o);
|
|
o->dg_host_end = 0;
|
|
}
|
|
__atomic_store_n(&o->preempt, 1, __ATOMIC_RELEASE);
|
|
}
|
|
}
|
|
|
|
static void core_take(RtThread *t) {
|
|
if (t->cur_core >= 0) return;
|
|
if (!cores_enabled()) {
|
|
acct(t, S_RUN);
|
|
return;
|
|
}
|
|
if (!(t->mask & ((1u << NCORE) - 1)))
|
|
rt_fatal("kernel: thread %#x core mask %#llx has no application core", t->handle, (unsigned long long)t->mask);
|
|
int c = t->core >= 0 && t->core < NCORE && core_allowed(t, t->core) && !g_core_owner[t->core] ? t->core : -1;
|
|
for (int i = 0; c < 0 && i < NCORE; i++)
|
|
if (core_allowed(t, i) && !g_core_owner[i]) c = i;
|
|
if (c >= 0) {
|
|
t->cur_core = c;
|
|
g_core_owner[c] = t;
|
|
core_count();
|
|
acct(t, S_RUN);
|
|
return;
|
|
}
|
|
acct(t, S_CORE);
|
|
t->cseq = ++g_wseq;
|
|
t->cnext = g_core_wait;
|
|
g_core_wait = t;
|
|
t->target = NULL;
|
|
if (g_diag > 0) {
|
|
t->dg_since = kern_now_ns();
|
|
memset(&t->dg_rel, 0, sizeof t->dg_rel);
|
|
}
|
|
core_kick();
|
|
if (g_diag > 0) t->dg_cls = dg_class0(t);
|
|
if (g_ap_int && t->target && t->cur_core < 0) { // most waits end within microseconds: spin before interrupting
|
|
uint64_t end = kern_now_ns() + AP_FIRST_NS;
|
|
unlock();
|
|
while (__atomic_load_n(&t->cur_core, __ATOMIC_RELAXED) < 0 && kern_now_ns() < end) YieldProcessor();
|
|
lock();
|
|
if (t->cur_core >= 0) g_ap_spin_ok++;
|
|
}
|
|
int sampled = 0;
|
|
while (t->cur_core < 0) {
|
|
DWORD ms = INFINITE;
|
|
if (g_ap_int && ap_try(t) != AP_NONE) ms = 1; // the owner keeps its core: again in a millisecond
|
|
int sample = g_diag > 0 && !sampled && (t->dg_cls == DG_GAME || t->dg_cls == DG_HOST);
|
|
if (sample && ms == INFINITE) ms = 2;
|
|
SleepConditionVariableSRW(&t->cv, &G, ms, 0);
|
|
if (sample && t->cur_core < 0 && kern_now_ns() - t->dg_since >= 2000000) {
|
|
sampled = 1;
|
|
dg_sample(t);
|
|
}
|
|
}
|
|
if (g_diag > 0) dg_done(t);
|
|
if (t->grant_at) {
|
|
uint64_t d = kern_now_ns() - t->grant_at;
|
|
t->lg_ns += d;
|
|
t->lg_n++;
|
|
if (d > t->lg_max) t->lg_max = d;
|
|
t->grant_at = 0;
|
|
}
|
|
t->target = NULL;
|
|
acct(t, S_RUN);
|
|
}
|
|
|
|
static uint32_t block(RtThread *t, int kind, int64_t ns) {
|
|
t->waker = 0;
|
|
if (ns == 0) return block_inner(t, kind, ns);
|
|
core_put(t);
|
|
acct(t, st_of(kind));
|
|
t->nblk++;
|
|
uint32_t r = block_inner(t, kind, ns);
|
|
core_take(t);
|
|
return r;
|
|
}
|
|
|
|
int kern_core_away(void) {
|
|
RtThread *t = tcur;
|
|
if (!t || t->cur_core < 0) return 0;
|
|
lock();
|
|
core_put(t);
|
|
acct(t, S_HOST);
|
|
if (rt_trace & RT_TRACE_SCHED) // the caller as an exe RVA (totk.map)
|
|
t->wobj = (uintptr_t)__builtin_return_address(0) - (uintptr_t)GetModuleHandleW(NULL);
|
|
unlock();
|
|
return 1;
|
|
}
|
|
|
|
void kern_core_back(int *away) {
|
|
RtThread *t = tcur;
|
|
if (!*away || !t) return;
|
|
lock();
|
|
core_take(t);
|
|
unlock();
|
|
*away = 0;
|
|
}
|
|
|
|
// ---- spin waits (recomp/patches.py `spin` lines, found by tools/spin_waits.py)
|
|
// Adapted from MacNativeGames games/totk/runtime/threads.c runtime_spin_wait @ a038e6b and nxnative dm/host/hle_thread.c
|
|
// dm_spin_wait @ 6d34984 (MIT, Copyright (c) 2026 MacNativeGames contributors, Copyright (c) 2026 seedds).
|
|
// A pass of a guest loop that polls memory until another thread changes it. On the Switch it fills only its own core,
|
|
// where a better thread pinned there preempts it; here it holds a virtual core and a host core, and pollers that take a
|
|
// lock on every pass hand its cache line between their host cores so the thread whose item it is rarely gets it
|
|
// (nxnative found this in Breath of the Wild's job queue). Every pass pauses the host core once. A wait is a run of
|
|
// passes through the same loop at most SPIN_GAP_NS apart; once one has lasted TOTK_SPIN_HANDOFF_US (default 50) and a
|
|
// thread waits for a virtual core, each pass gives the core to the waiter, whatever its priority, yields the host CPU
|
|
// and takes a core again. Shorter waits keep the core: a lock held for a moment would otherwise make the next pass wait
|
|
// for another thread's next SVC (MacNativeGames). TOTK_SPIN=0 keeps the core always (the gate's own cost only).
|
|
#define SPIN_GAP_NS 20000ull
|
|
static int g_spin_on = -1;
|
|
static uint64_t g_spin_handoff_ns;
|
|
static _Thread_local uint64_t sw_head, sw_t0, sw_last;
|
|
|
|
void rt_spin_gate(Cpu *c, uint64_t head) {
|
|
(void)c;
|
|
YieldProcessor();
|
|
RtThread *t = tcur;
|
|
if (!t) return;
|
|
if (g_spin_on < 0) {
|
|
const char *e = cfg_get("TOTK_SPIN_HANDOFF_US");
|
|
g_spin_handoff_ns = (uint64_t)(e && *e ? atoll(e) : 50) * 1000;
|
|
g_spin_on = !(cfg_get("TOTK_SPIN") && !cfg_bool("TOTK_SPIN"));
|
|
}
|
|
t->sp_n++;
|
|
t->sp_site = head;
|
|
uint64_t now = kern_now_ns();
|
|
if (head != sw_head || now - sw_last > SPIN_GAP_NS) { // a new wait
|
|
sw_head = head;
|
|
sw_t0 = now;
|
|
}
|
|
sw_last = now;
|
|
if (!g_spin_on || t->cur_core < 0 || now - sw_t0 < g_spin_handoff_ns ||
|
|
!__atomic_load_n(&g_core_wait, __ATOMIC_RELAXED)) // a racy look: a waiter missed now is seen at the next pass
|
|
return;
|
|
t->sp_ho++;
|
|
int away = kern_core_away();
|
|
SwitchToThread();
|
|
kern_core_back(&away);
|
|
sw_last = kern_now_ns();
|
|
}
|
|
|
|
// A pending preempt request, at an SVC or an indirect call. The core goes over only if a better thread still waits
|
|
// for it, so core_put grants it to a waiter better than t; a stale request (that waiter got another core) is dropped.
|
|
void kern_checkpoint(void) {
|
|
RtThread *t = tcur;
|
|
if (!t || !__atomic_load_n(&t->preempt, __ATOMIC_RELAXED)) return;
|
|
lock();
|
|
if (g_diag > 0 && t->cur) {
|
|
t->dg_pc = t->cur->pc;
|
|
t->dg_lr = t->cur->x[30];
|
|
}
|
|
int better = t->cur_core >= 0 && core_better_waiter(t, t->cur_core);
|
|
if (better && ap_disabled(t)) { // preemption disabled: the switch comes at SVC 0x36
|
|
ap_tlr(t)[1] = 1;
|
|
t->preempt = 0;
|
|
g_ap_deferred++;
|
|
} else if (better) {
|
|
core_put(t); // clears preempt (after the sched trace reads it)
|
|
core_take(t);
|
|
} else {
|
|
t->preempt = 0;
|
|
t->pre_since = 0;
|
|
t->dg_kick = t->dg_host_end = 0;
|
|
t->dg_host_kick = 0;
|
|
g_pre_stale++;
|
|
}
|
|
unlock();
|
|
}
|
|
|
|
// kern_ap_stub, on the interrupted owner: where to jump back to, then the core given up as at a checkpoint.
|
|
uint64_t kern_ap_rip(void) { return tcur->ap_rip; }
|
|
|
|
void kern_ap_yield(void) {
|
|
RtThread *t = tcur;
|
|
lock();
|
|
t->ap_pending = 0;
|
|
if (t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) {
|
|
if (g_diag > 0) {
|
|
t->dg_pc = t->cur->pc;
|
|
t->dg_lr = t->cur->x[30];
|
|
}
|
|
g_ap_taken++;
|
|
core_put(t);
|
|
core_take(t);
|
|
}
|
|
unlock();
|
|
}
|
|
|
|
// SVC 0x36 SynchronizePreemptionState: a preemption deferred by disable_count happens here.
|
|
static void ap_sync(RtThread *t) {
|
|
lock();
|
|
if (g_ap > 0 && t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) {
|
|
core_put(t);
|
|
core_take(t);
|
|
}
|
|
unlock();
|
|
}
|
|
|
|
void kern_core_stats(int *held_now, int *held_max) {
|
|
lock();
|
|
int n = 0;
|
|
for (int i = 0; i < NCORE; i++) n += g_core_owner[i] != NULL;
|
|
*held_now = n;
|
|
*held_max = g_held_max;
|
|
unlock();
|
|
}
|
|
|
|
void kern_core_stats_reset(void) {
|
|
lock();
|
|
g_held_max = 0;
|
|
core_count();
|
|
unlock();
|
|
}
|
|
|
|
// TOTK_TRACE=sched: every 120 presents, the time split of the busiest threads (running, waiting for a core, host
|
|
// calls) and of the render thread, the render and the main thread's longest waits by object and by the thread whose
|
|
// signal ended them, and the preemption latency.
|
|
#define REP_N 8
|
|
// At each present (holding G): starts recording at TOTK_CRIT's first frame, stops and writes the file at its last.
|
|
// File: "CRIT", thread count, event count (u32 each); per thread handle, priority, flags (1 main, 2 presents), host
|
|
// tid (u32 each); then the CritEv records.
|
|
static void crit_frame(RtThread *self, uint64_t frame) {
|
|
static int parsed;
|
|
if (!parsed) {
|
|
parsed = 1;
|
|
const char *e = cfg_get("TOTK_CRIT");
|
|
unsigned long long lo, hi;
|
|
double wl, wh;
|
|
if (e && e[0] == 'w' && sscanf(e + 1, "%lf-%lf", &wl, &wh) == 2 && wl >= 0 && wl < wh &&
|
|
(g_crit = malloc(CRIT_N * sizeof *g_crit))) {
|
|
g_crit_wlo = (int64_t)(wl * 1e6), g_crit_whi = (int64_t)(wh * 1e6);
|
|
g_crit_lo = g_crit_hi = UINT64_MAX;
|
|
rt_log("crit: recording world seconds %g-%g", wl, wh);
|
|
} else if (e && sscanf(e, "%llu-%llu", &lo, &hi) == 2 && lo < hi && (g_crit = malloc(CRIT_N * sizeof *g_crit))) {
|
|
g_crit_lo = lo;
|
|
g_crit_hi = hi;
|
|
rt_log("crit: recording frames %llu-%llu", lo, hi);
|
|
}
|
|
}
|
|
if (g_crit && g_crit_whi && g_crit_hi == UINT64_MAX) { // world mode: fix the frames as the clock passes lo and hi
|
|
int64_t w = frame_step_world_us();
|
|
if (g_crit_lo == UINT64_MAX) {
|
|
if (w < g_crit_wlo) return;
|
|
g_crit_lo = frame;
|
|
}
|
|
if (w > g_crit_whi) g_crit_hi = frame;
|
|
}
|
|
if (!g_crit || frame < g_crit_lo || frame > g_crit_hi) return;
|
|
g_crit_on = 1;
|
|
crit_add(CRIT_FRAME, self ? self->handle : 0, 0, 0, frame, kern_now_ns());
|
|
if (frame < g_crit_hi) return;
|
|
g_crit_on = 0;
|
|
const char *path = cfg_get("TOTK_CRIT_OUT");
|
|
FILE *f = fopen(path && *path ? path : "crit.bin", "wb");
|
|
if (!f) {
|
|
rt_log("crit: cannot write %s", path && *path ? path : "crit.bin");
|
|
return;
|
|
}
|
|
uint32_t nt = 0;
|
|
for (RtThread *t = g_all; t; t = t->all) nt++;
|
|
fwrite("CRIT", 1, 4, f);
|
|
fwrite(&nt, 4, 1, f);
|
|
fwrite(&g_crit_n, 4, 1, f);
|
|
for (RtThread *t = g_all; t; t = t->all) {
|
|
uint32_t row[4] = {t->handle, (uint32_t)t->prio, (uint32_t)(t->is_main | (t == g_render) << 1), t->tid};
|
|
fwrite(row, 4, 4, f);
|
|
}
|
|
fwrite(g_crit, sizeof *g_crit, g_crit_n, f);
|
|
fclose(f);
|
|
rt_log("crit: %u events, %u threads, frames %llu-%llu%s", g_crit_n, nt, (unsigned long long)g_crit_lo,
|
|
(unsigned long long)g_crit_hi, g_crit_n >= CRIT_N ? " (buffer full: the last frames are incomplete)" : "");
|
|
}
|
|
|
|
void kern_sched_report(uint64_t frame) {
|
|
static uint64_t last;
|
|
typedef struct {
|
|
uint32_t h;
|
|
int prio, render;
|
|
uint64_t mask, entry, ns[S_N], nblk, npre, cpu, lw_ns, lw_n, lw_max, lg_ns, lg_n, lg_max, sl_n, sl_req, sl_act;
|
|
uint64_t sp_n, sp_ho, sp_site;
|
|
} Row;
|
|
Row rows[REP_N + 1];
|
|
WaitRec rw[2][4] = {0}; // the render and the main thread's longest waits
|
|
int nr = 0;
|
|
RtThread *self = tcur;
|
|
lock();
|
|
if (g_render != self) {
|
|
g_render = self;
|
|
memset(g_rw, 0, sizeof g_rw);
|
|
}
|
|
crit_frame(self, frame);
|
|
if (frame % 120) {
|
|
unlock();
|
|
return;
|
|
}
|
|
uint64_t now = kern_now_ns(), span = last ? now - last : 0;
|
|
last = now;
|
|
RtThread *top[REP_N + 1] = {0};
|
|
uint64_t busy[REP_N + 1] = {0}, all_run = 0, all_core = 0;
|
|
for (RtThread *t = g_all; t; t = t->all) {
|
|
acct(t, t->st); // close the open interval
|
|
uint64_t b = t->st_ns[S_RUN] + t->st_ns[S_CORE] + t->st_ns[S_HOST];
|
|
all_run += t->st_ns[S_RUN];
|
|
all_core += t->st_ns[S_CORE];
|
|
if (t == self) continue;
|
|
for (int i = 0; i < REP_N; i++)
|
|
if (!top[i] || b > busy[i]) {
|
|
memmove(top + i + 1, top + i, (REP_N - 1 - i) * sizeof *top);
|
|
memmove(busy + i + 1, busy + i, (REP_N - 1 - i) * sizeof *busy);
|
|
top[i] = t;
|
|
busy[i] = b;
|
|
break;
|
|
}
|
|
}
|
|
if (self) { // the render thread first
|
|
memmove(top + 1, top, REP_N * sizeof *top);
|
|
top[0] = self;
|
|
}
|
|
for (int i = 0; i <= REP_N && top[i]; i++) {
|
|
RtThread *t = top[i];
|
|
Row *r = &rows[nr++];
|
|
r->h = t->handle;
|
|
r->prio = t->prio;
|
|
r->render = t == self;
|
|
r->mask = t->mask;
|
|
r->entry = t->entry;
|
|
memcpy(r->ns, t->st_ns, sizeof r->ns);
|
|
r->nblk = t->nblk;
|
|
r->npre = t->npre;
|
|
r->lw_ns = t->lw_ns, r->lw_n = t->lw_n, r->lw_max = t->lw_max;
|
|
r->lg_ns = t->lg_ns, r->lg_n = t->lg_n, r->lg_max = t->lg_max;
|
|
r->sl_n = t->sl_n, r->sl_req = t->sl_req, r->sl_act = t->sl_act;
|
|
r->sp_n = t->sp_n, r->sp_ho = t->sp_ho, r->sp_site = t->sp_site;
|
|
r->cpu = 0;
|
|
HANDLE th = t->tid ? OpenThread(THREAD_QUERY_LIMITED_INFORMATION, FALSE, t->tid) : NULL;
|
|
FILETIME a, b, k, u;
|
|
if (th && GetThreadTimes(th, &a, &b, &k, &u)) {
|
|
uint64_t c = (((uint64_t)k.dwHighDateTime << 32 | k.dwLowDateTime) +
|
|
((uint64_t)u.dwHighDateTime << 32 | u.dwLowDateTime)) * 100;
|
|
r->cpu = t->cpu_last ? c - t->cpu_last : 0;
|
|
t->cpu_last = c;
|
|
}
|
|
if (th) CloseHandle(th);
|
|
}
|
|
for (int k = 0; k < 2; k++) {
|
|
const WaitRec *src = k ? g_mw : g_rw;
|
|
for (int i = 0; i < RW_N && src[i].ns; i++)
|
|
for (int j = 0; j < 4; j++)
|
|
if (src[i].ns > rw[k][j].ns) {
|
|
memmove(rw[k] + j + 1, rw[k] + j, (3 - j) * sizeof **rw);
|
|
rw[k][j] = src[i];
|
|
break;
|
|
}
|
|
}
|
|
uint64_t pre_stale = g_pre_stale, pre_req = g_pre_req, pre_n = g_pre_n, pre_ns = g_pre_ns, pre_max = g_pre_max;
|
|
uint64_t sc = g_sync_calls, sb = g_sync_blocks;
|
|
for (RtThread *t = g_all; t; t = t->all) {
|
|
memset(t->st_ns, 0, sizeof t->st_ns);
|
|
t->nblk = t->npre = 0;
|
|
t->lw_ns = t->lw_n = t->lw_max = t->lg_ns = t->lg_n = t->lg_max = t->sl_n = t->sl_req = t->sl_act = 0;
|
|
t->sp_n = t->sp_ho = 0; // racy against the thread's own increments: a few passes may be lost
|
|
}
|
|
memset(g_rw, 0, sizeof g_rw);
|
|
memset(g_mw, 0, sizeof g_mw);
|
|
g_pre_req = g_pre_n = g_pre_ns = g_pre_max = g_pre_stale = 0;
|
|
g_sync_calls = g_sync_blocks = 0;
|
|
unlock();
|
|
if (!span) return;
|
|
rt_log("sched: frame %llu: %.2f s, cores busy %.2f of %d, core wait %.2f thread-s, preempt req %llu released %llu "
|
|
"avg %.2f ms max %.2f ms, WaitSynchronization %llu calls %llu blocks, stale preempt %llu%s",
|
|
(unsigned long long)frame, span / 1e9, (double)all_run / span, NCORE, all_core / 1e9,
|
|
(unsigned long long)pre_req, (unsigned long long)pre_n, pre_n ? pre_ns / 1e6 / pre_n : 0.0, pre_max / 1e6,
|
|
(unsigned long long)sc, (unsigned long long)sb, (unsigned long long)pre_stale, self ? "" : " (render thread is not a guest thread)");
|
|
for (int i = 0; i < nr; i++) {
|
|
Row *r = &rows[i];
|
|
char d[128], s[256];
|
|
int o = 0;
|
|
for (int k = S_RUN; k < S_N; k++)
|
|
o += snprintf(s + o, sizeof s - o, " %s %.0f%%", s_name[k], 100.0 * r->ns[k] / span);
|
|
ldr_describe(r->entry, d, sizeof d);
|
|
rt_log("sched: %s%#x p%d m%llx%s cpu %.0f%% blocks %llu preempted %llu (%s)", r->render ? "render " : "",
|
|
r->h, r->prio, (unsigned long long)r->mask, s, 100.0 * r->cpu / span, (unsigned long long)r->nblk,
|
|
(unsigned long long)r->npre, r->entry ? d : "main");
|
|
rt_log("sched: %#x wake to run avg %.1f us max %.0f us (%llu), core grant to run avg %.1f us max %.0f us "
|
|
"(%llu), sleep %llu asked avg %.1f us took avg %.1f us", r->h, r->lw_n ? r->lw_ns / 1e3 / r->lw_n : 0.0,
|
|
r->lw_max / 1e3, (unsigned long long)r->lw_n, r->lg_n ? r->lg_ns / 1e3 / r->lg_n : 0.0, r->lg_max / 1e3,
|
|
(unsigned long long)r->lg_n, (unsigned long long)r->sl_n, r->sl_n ? r->sl_req / 1e3 / r->sl_n : 0.0,
|
|
r->sl_n ? r->sl_act / 1e3 / r->sl_n : 0.0);
|
|
if (r->sp_n) {
|
|
ldr_describe(r->sp_site, d, sizeof d);
|
|
rt_log("sched: %#x spin wait passes %llu, core handoffs %llu, last at %s", r->h,
|
|
(unsigned long long)r->sp_n, (unsigned long long)r->sp_ho, d);
|
|
}
|
|
}
|
|
for (int k = 0; k < 2; k++)
|
|
for (int j = 0; j < 4 && rw[k][j].ns; j++) {
|
|
char by[32] = "";
|
|
if (rw[k][j].waker) snprintf(by, sizeof by, ", woken by %#x", rw[k][j].waker);
|
|
rt_log("sched: %s wait %d: %s on %s%#llx%s: %.1f%%", k ? "main" : "render", j + 1, s_name[rw[k][j].st],
|
|
rw[k][j].st == S_HOST ? "exe+" : "", (unsigned long long)rw[k][j].obj, by, 100.0 * rw[k][j].ns / span);
|
|
}
|
|
}
|
|
|
|
// ---- locks: tag word = owner handle | WAITERS_BIT
|
|
static inline uint32_t ld32(uint64_t a) { return __atomic_load_n((uint32_t *)(uintptr_t)a, __ATOMIC_SEQ_CST); }
|
|
static inline void st32(uint64_t a, uint32_t v) { __atomic_store_n((uint32_t *)(uintptr_t)a, v, __ATOMIC_SEQ_CST); }
|
|
static inline int cas32(uint64_t a, uint32_t *exp, uint32_t v) {
|
|
return __atomic_compare_exchange_n((uint32_t *)(uintptr_t)a, exp, v, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
|
|
}
|
|
|
|
// Release the lock at addr: hand it to the best waiter, or clear it.
|
|
static void lock_handoff(uint64_t addr) {
|
|
RtThread *w = pick(W_LOCK, addr);
|
|
int more = 0;
|
|
if (!w) {
|
|
st32(addr, 0);
|
|
return;
|
|
}
|
|
for (RtThread *t = g_wait; t; t = t->next)
|
|
if (t != w && t->wkind == W_LOCK && t->waddr == addr) {
|
|
more = 1;
|
|
break;
|
|
}
|
|
st32(addr, w->wtag | (more ? WAITERS_BIT : 0));
|
|
wake(w, 0);
|
|
}
|
|
|
|
// A condvar waiter takes its lock: directly if free (and wakes), else it becomes a lock waiter behind the others.
|
|
static void lock_acquire_for(RtThread *t, uint64_t addr, uint32_t tag) {
|
|
uint32_t v = ld32(addr);
|
|
for (;;) {
|
|
if (v == 0) {
|
|
if (cas32(addr, &v, tag)) {
|
|
wake(t, 0);
|
|
return;
|
|
}
|
|
continue;
|
|
}
|
|
if (cas32(addr, &v, v | WAITERS_BIT)) break;
|
|
}
|
|
acct(t, S_LOCK);
|
|
t->wobj = v & ~WAITERS_BIT;
|
|
t->wkind = W_LOCK;
|
|
t->waddr = addr;
|
|
t->wtag = tag;
|
|
t->wseq = ++g_wseq;
|
|
}
|
|
|
|
// ---- threads
|
|
static RtThread *thr_new(int prio, int core, uint64_t mask) {
|
|
RtThread *t = calloc(1, sizeof *t);
|
|
Cpu *c = _aligned_malloc(sizeof(Cpu), 64);
|
|
if (!t || !c) rt_fatal("kernel: thread allocation");
|
|
memset(c, 0, sizeof *c);
|
|
InitializeConditionVariable(&t->cv);
|
|
t->prio = prio;
|
|
t->core = core;
|
|
t->mask = mask;
|
|
t->cur_core = -1;
|
|
// TLR (0x200) and the "current thread object" the SDK reads through TLR+0x1F8, in one guest-visible block.
|
|
t->tlr = mem_kernel_alloc(TLR_SIZE + THREAD_OBJ_SIZE, TLR_SIZE);
|
|
*(uint64_t *)(t->tlr + 0x1F8) = (uint64_t)(uintptr_t)(t->tlr + TLR_SIZE);
|
|
t->tls = mem_kernel_alloc(TLS_SIZE, 16);
|
|
c->tpidrro = (uint64_t)(uintptr_t)t->tlr;
|
|
c->tpidr = (uint64_t)(uintptr_t)t->tls;
|
|
c->excl_addr = ~0ull;
|
|
c->thread = t;
|
|
t->cpu = t->cur = c;
|
|
return t;
|
|
}
|
|
|
|
static void thr_link(RtThread *t) { // holding G
|
|
t->handle = h_new(O_THREAD, t);
|
|
t->id = g_next_id++;
|
|
t->all = g_all;
|
|
g_all = t;
|
|
g_created++;
|
|
}
|
|
|
|
static void thr_free(RtThread *t) { // holding G; t has exited and its handle is closed
|
|
for (RtThread **p = &g_all; *p; p = &(*p)->all)
|
|
if (*p == t) {
|
|
*p = t->all;
|
|
break;
|
|
}
|
|
mem_kernel_free(t->tlr, TLR_SIZE + THREAD_OBJ_SIZE);
|
|
mem_kernel_free(t->tls, TLS_SIZE);
|
|
_aligned_free(t->cpu);
|
|
free(t);
|
|
}
|
|
|
|
// The end of every guest thread, on its own host context after the unwind: give the core up, signal the thread
|
|
// object, free it if its handle is already closed. t is not touched after the unlock.
|
|
static void thread_finish(RtThread *t) {
|
|
if (t->timer) CloseHandle(t->timer);
|
|
t->timer = NULL;
|
|
if (t->base_fiber) ConvertFiberToThread();
|
|
t->base_fiber = NULL;
|
|
tcur = NULL;
|
|
lock();
|
|
core_put(t);
|
|
acct(t, S_OFF);
|
|
if (t->hthr) CloseHandle(t->hthr); // no waiter interrupts a thread without a core
|
|
t->hthr = NULL;
|
|
t->exited = 1;
|
|
wake_sync_waiters();
|
|
if (t->closed) thr_free(t);
|
|
unlock();
|
|
}
|
|
|
|
// ExitThread: back to the thread start with a non-unwinding longjmp; no SEH unwinding through lifted frames.
|
|
// (__builtin_setjmp is not usable: on Win64 LLVM saves the frame address, which is not rbp when the frame register
|
|
// is offset, and the landing code then reads its locals through the wrong base.)
|
|
static __attribute__((noreturn)) void exit_now(RtThread *t) { longjmp(t->exit_jmp, 1); }
|
|
|
|
// The calling host thread, for ap_try (TOTK_ASYNC_PREEMPT only).
|
|
static HANDLE ap_open_self(void) {
|
|
if (!cores_enabled() || !g_ap_int) return NULL;
|
|
HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_SET_CONTEXT | THREAD_QUERY_INFORMATION,
|
|
FALSE, GetCurrentThreadId());
|
|
if (!h) rt_log("kernel: OpenThread for asynchronous preemption failed (error %lu)", GetLastError());
|
|
return h;
|
|
}
|
|
|
|
static DWORD WINAPI thread_main(void *p) {
|
|
RtThread *t = p;
|
|
tcur = t;
|
|
t->tid = GetCurrentThreadId();
|
|
HANDLE self = ap_open_self();
|
|
lock();
|
|
t->hthr = self;
|
|
while (!t->started) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
|
|
core_take(t);
|
|
unlock();
|
|
Cpu *c = t->cpu;
|
|
c->sp = t->stack_top & ~15ull;
|
|
if (!setjmp(t->exit_jmp)) {
|
|
rt_call(c, t->entry, 1, t->arg);
|
|
char d[128];
|
|
rt_fatal("guest thread %#x: entry %s returned (Horizon threads end with ExitThread)", t->handle,
|
|
ldr_describe(t->entry, d, sizeof d));
|
|
}
|
|
thread_finish(tcur);
|
|
return 0;
|
|
}
|
|
|
|
uint64_t kern_threads_created(void) {
|
|
lock();
|
|
uint64_t n = g_created;
|
|
unlock();
|
|
return n;
|
|
}
|
|
|
|
Cpu *kern_cpu(void) { return tcur ? tcur->cur : NULL; }
|
|
|
|
uint32_t kern_main_thread(Cpu **cpu) {
|
|
if (tcur) {
|
|
if (!tcur->is_main) rt_fatal("kern_main_thread on guest thread %#x, not the main thread", tcur->handle);
|
|
*cpu = tcur->cpu;
|
|
return tcur->handle;
|
|
}
|
|
if (g_main) rt_fatal("kern_main_thread: the main thread already runs on another host thread");
|
|
const RtGuestLayout *L = mem_layout();
|
|
uint64_t stack = L->stack.base + L->stack.size - MAIN_STACK;
|
|
if (!VirtualAlloc((void *)(uintptr_t)stack, MAIN_STACK, MEM_COMMIT, PAGE_READWRITE))
|
|
rt_fatal("kernel: commit the main thread stack at %#llx failed (error %lu)", (unsigned long long)stack,
|
|
GetLastError());
|
|
RtThread *t = thr_new(MAIN_PRIO, MAIN_CORE, 7);
|
|
t->started = 1;
|
|
t->is_main = 1;
|
|
t->stack_top = stack + MAIN_STACK;
|
|
t->cpu->sp = t->stack_top;
|
|
t->tid = GetCurrentThreadId();
|
|
perf_main_thread();
|
|
HANDLE self = ap_open_self();
|
|
lock();
|
|
t->hthr = self;
|
|
thr_link(t);
|
|
g_main = t;
|
|
g_main_stack = stack;
|
|
core_take(t);
|
|
unlock();
|
|
tcur = t;
|
|
*cpu = t->cpu;
|
|
return t->handle;
|
|
}
|
|
|
|
void kern_main_stack(uint64_t *base, uint64_t *size) {
|
|
uint64_t b = __atomic_load_n(&g_main_stack, __ATOMIC_ACQUIRE);
|
|
*base = b;
|
|
*size = b ? MAIN_STACK : 0;
|
|
}
|
|
|
|
static DWORD WINAPI guest_main(void *p) {
|
|
void (*fn)(Cpu *) = (void (*)(Cpu *))p;
|
|
Cpu *c;
|
|
kern_main_thread(&c);
|
|
if (!setjmp(tcur->exit_jmp)) fn(c);
|
|
thread_finish(tcur);
|
|
return 0;
|
|
}
|
|
|
|
void kern_run_on_guest_main(void (*fn)(Cpu *c)) {
|
|
HANDLE h = CreateThread(NULL, HOST_STACK, guest_main, (void *)fn, STACK_SIZE_PARAM_IS_A_RESERVATION, NULL);
|
|
if (!h) rt_fatal("kernel: CreateThread for the guest main thread failed (error %lu)", GetLastError());
|
|
WaitForSingleObject(h, INFINITE);
|
|
CloseHandle(h);
|
|
}
|
|
|
|
// The high-resolution waitable timer ends a wait 0.3-0.5 ms late at best: the SDK's 20 us polling sleeps (main thread
|
|
// and job workers, about 1.5 per frame each) took 460-515 us. Sleeps up to SLEEP_SPIN_NS spin on the performance
|
|
// counter instead; the virtual core is already given up. Linux: clock_nanosleep wakes within about 50 us (timer slack),
|
|
// so a Linux host spins only below that, or not at all.
|
|
#define SLEEP_SPIN_NS 1000000
|
|
static void sleep_ns(RtThread *t, int64_t ns) {
|
|
if (ns <= SLEEP_SPIN_NS) {
|
|
uint64_t end = kern_now_ns() + (uint64_t)ns;
|
|
while (kern_now_ns() < end) YieldProcessor();
|
|
return;
|
|
}
|
|
if (!t->timer) {
|
|
t->timer = CreateWaitableTimerExW(NULL, NULL, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS);
|
|
if (!t->timer) t->timer = CreateWaitableTimerExW(NULL, NULL, 0, TIMER_ALL_ACCESS);
|
|
if (!t->timer) rt_fatal("kernel: CreateWaitableTimerExW failed (error %lu)", GetLastError());
|
|
}
|
|
LARGE_INTEGER due;
|
|
int64_t units = ns / 100 + (ns % 100 != 0); // 100 ns units, rounded up
|
|
due.QuadPart = -(units > 0 ? units : 1);
|
|
if (!SetWaitableTimer(t->timer, &due, 0, NULL, NULL, FALSE))
|
|
rt_fatal("kernel: SetWaitableTimer failed (error %lu)", GetLastError());
|
|
WaitForSingleObject(t->timer, INFINITE);
|
|
}
|
|
|
|
// ---- synchronization objects
|
|
static int sig_obj(uint32_t h, int *bad) { // 1 = signaled (holding G)
|
|
unsigned i = h - 0x1000;
|
|
if (h == H_CUR_THREAD) return 0;
|
|
if (i == 0 || i >= MAX_H) {
|
|
*bad = 1;
|
|
return 0;
|
|
}
|
|
switch (g_h[i].type) {
|
|
case O_THREAD: return ((RtThread *)g_h[i].obj)->exited;
|
|
case O_EVENT_R: return ((Event *)g_h[i].obj)->signaled;
|
|
default: *bad = 1; return 0;
|
|
}
|
|
}
|
|
|
|
int kern_thread_state(uint32_t h) {
|
|
lock();
|
|
RtThread *t = h_peek(h, O_THREAD);
|
|
int r = !t ? -1 : t->exited ? KERN_THREAD_EXITED : t->wkind;
|
|
unlock();
|
|
return r;
|
|
}
|
|
|
|
// CloseHandle: an event goes with its last end; a thread object once it has also exited.
|
|
static uint32_t handle_close(uint32_t h) {
|
|
unsigned i = h - 0x1000;
|
|
int ok = 0, kind = 0;
|
|
void *ipc = NULL;
|
|
lock();
|
|
if (i > 0 && i < MAX_H) {
|
|
int type = g_h[i].type;
|
|
void *o = g_h[i].obj;
|
|
if (type == O_EVENT_R || type == O_EVENT_W) {
|
|
Event *e = o;
|
|
if (--e->refs == 0) free(e);
|
|
ok = 1;
|
|
} else if (type == O_SESSION || type == O_TMEM) {
|
|
kind = type;
|
|
ipc = o;
|
|
ok = 1;
|
|
} else if (type == O_THREAD) {
|
|
RtThread *t = o;
|
|
t->closed = 1;
|
|
if (t->exited) thr_free(t);
|
|
ok = 1;
|
|
}
|
|
if (ok) {
|
|
g_h[i].type = O_FREE;
|
|
g_h[i].obj = NULL;
|
|
}
|
|
}
|
|
unlock();
|
|
if (ipc) ipc_handle_closed(kind, ipc);
|
|
return ok ? R_OK : R_BAD_HANDLE;
|
|
}
|
|
|
|
void kern_handle_close(uint32_t h) {
|
|
if (handle_close(h)) rt_fatal("kern_handle_close(%#x): bad handle", h);
|
|
}
|
|
|
|
// Hooks of the probe's condvar negative control.
|
|
volatile int kern_test_cv_old_order;
|
|
void (*volatile kern_test_cv_hook)(void);
|
|
|
|
static void cv_hook(void) {
|
|
void (*h)(void) = kern_test_cv_hook;
|
|
if (h) h();
|
|
}
|
|
|
|
int kern_svc(Cpu *c, uint32_t imm) {
|
|
uint64_t *x = c->x;
|
|
if (g_diag > 0 && tcur) tcur->dg_svc = imm;
|
|
switch (imm) {
|
|
case 0x08: { // CreateThread(-> w1 handle; x1 entry, x2 arg, x3 stack top, w4 priority, w5 core)
|
|
int prio = (int)x[4], core = (int)x[5];
|
|
if (!x[1] || !x[3] || prio < 28 || prio > 59 || core < -3 || core > 2) {
|
|
x[0] = R_BAD_STATE;
|
|
return 1;
|
|
}
|
|
RtThread *t = thr_new(prio, core >= 0 ? core : MAIN_CORE, core >= 0 ? 1ull << core : 7);
|
|
t->entry = x[1];
|
|
t->arg = x[2];
|
|
t->stack_top = x[3] & ~15ull;
|
|
lock();
|
|
thr_link(t);
|
|
unlock();
|
|
HANDLE h = CreateThread(NULL, HOST_STACK, thread_main, t, STACK_SIZE_PARAM_IS_A_RESERVATION, NULL);
|
|
if (!h) rt_fatal("kernel: CreateThread failed (error %lu)", GetLastError());
|
|
CloseHandle(h);
|
|
if (rt_trace & RT_TRACE_SVC) {
|
|
char d[128];
|
|
rt_log("CreateThread %#x entry %s stack %#llx prio %d core %d", t->handle,
|
|
ldr_describe(t->entry, d, sizeof d), (unsigned long long)t->stack_top, prio, core);
|
|
}
|
|
x[0] = R_OK;
|
|
x[1] = t->handle;
|
|
return 1;
|
|
}
|
|
case 0x09: { // StartThread(w0)
|
|
lock();
|
|
RtThread *t = h_get((uint32_t)x[0], O_THREAD);
|
|
if (t) {
|
|
t->started = 1;
|
|
WakeConditionVariable(&t->cv);
|
|
}
|
|
unlock();
|
|
x[0] = t ? R_OK : R_BAD_HANDLE;
|
|
return 1;
|
|
}
|
|
case 0x0A: { // ExitThread
|
|
RtThread *t = cur(imm);
|
|
if (t->fiber) { // on a guest fiber: unwind from the thread's own context (kern_fiber_switch resumes it)
|
|
t->exit_pending = 1;
|
|
t->leaving = t->fiber;
|
|
t->fiber = NULL;
|
|
t->cur = t->cpu;
|
|
SwitchToFiber(t->base_fiber);
|
|
rt_fatal("kernel: ExitThread on a fiber came back");
|
|
}
|
|
exit_now(t);
|
|
}
|
|
case 0x0B: { // SleepThread(x0 ns); 0, -1, -2 yield
|
|
int64_t ns = (int64_t)x[0];
|
|
RtThread *t = cur(imm);
|
|
if (ns > 0) {
|
|
lock();
|
|
core_put(t);
|
|
acct(t, S_SLEEP);
|
|
t->wobj = (uint64_t)ns;
|
|
t->nblk++;
|
|
unlock();
|
|
uint64_t t0 = kern_now_ns();
|
|
sleep_ns(t, ns);
|
|
uint64_t took = kern_now_ns() - t0;
|
|
lock();
|
|
t->sl_n++;
|
|
t->sl_req += (uint64_t)ns;
|
|
t->sl_act += took;
|
|
core_take(t);
|
|
unlock();
|
|
return 1;
|
|
}
|
|
// Yield: hand the core to a waiting thread of equal or better priority.
|
|
lock();
|
|
int give = 0;
|
|
int tp = prio_of(t);
|
|
for (RtThread *w = g_core_wait; w && !give; w = w->cnext)
|
|
give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && prio_of(w) <= tp;
|
|
if (give) {
|
|
core_put(t);
|
|
core_take(t);
|
|
}
|
|
unlock();
|
|
if (!give) SwitchToThread();
|
|
return 1;
|
|
}
|
|
case 0x0C: { // GetThreadPriority(w1 handle -> w1)
|
|
lock();
|
|
RtThread *t = h_get((uint32_t)x[1], O_THREAD);
|
|
x[0] = t ? R_OK : R_BAD_HANDLE;
|
|
if (t) x[1] = (uint64_t)t->prio;
|
|
unlock();
|
|
return 1;
|
|
}
|
|
case 0x0D: { // SetThreadPriority(w0 handle, w1 priority)
|
|
if ((uint32_t)x[1] < 28 || (uint32_t)x[1] > 59) {
|
|
x[0] = R_BAD_STATE;
|
|
return 1;
|
|
}
|
|
lock();
|
|
RtThread *t = h_get((uint32_t)x[0], O_THREAD);
|
|
if (t) t->prio = (int)x[1];
|
|
x[0] = t ? R_OK : R_BAD_HANDLE;
|
|
unlock();
|
|
return 1;
|
|
}
|
|
case 0x0E: { // GetThreadCoreMask(w2 handle -> w1 core, x2 mask)
|
|
lock();
|
|
RtThread *t = h_get((uint32_t)x[2], O_THREAD);
|
|
x[0] = t ? R_OK : R_BAD_HANDLE;
|
|
if (t) {
|
|
x[1] = (uint64_t)(uint32_t)t->core;
|
|
x[2] = t->mask;
|
|
}
|
|
unlock();
|
|
return 1;
|
|
}
|
|
case 0x0F: { // SetThreadCoreMask(w0 handle, w1 ideal core, x2 mask): -1 process default, -2/-3 keep the ideal core
|
|
lock();
|
|
RtThread *t = h_get((uint32_t)x[0], O_THREAD);
|
|
if (t) {
|
|
int core = (int)x[1];
|
|
if (core >= 0) t->core = core;
|
|
else if (core == -1) t->core = MAIN_CORE;
|
|
if (x[2]) t->mask = x[2];
|
|
if (t == tcur && t->cur_core >= 0 && !core_allowed(t, t->cur_core)) { // move off a forbidden core
|
|
core_put(t);
|
|
core_take(t);
|
|
}
|
|
}
|
|
x[0] = t ? R_OK : R_BAD_HANDLE;
|
|
unlock();
|
|
return 1;
|
|
}
|
|
case 0x10: { // GetCurrentProcessorNumber: the virtual core held
|
|
RtThread *t = cur(imm);
|
|
x[0] = (uint64_t)(uint32_t)(t->cur_core >= 0 ? t->cur_core : t->core);
|
|
return 1;
|
|
}
|
|
case 0x36: { // SynchronizePreemptionState: the SDK's spin locks call it after clearing TLR+0x100 (disable_count)
|
|
// when TLR+0x102 (interrupt_flag) is set; rt_svc's checkpoint has already honoured a pending preempt
|
|
RtThread *t = cur(imm);
|
|
volatile uint16_t *flag = (volatile uint16_t *)(t->tlr + 0x102);
|
|
__atomic_fetch_add(&g_sync36, 1, __ATOMIC_RELAXED);
|
|
if (*flag) __atomic_fetch_add(&g_sync36_flag, 1, __ATOMIC_RELAXED);
|
|
*flag = 0;
|
|
ap_sync(t);
|
|
x[0] = R_OK;
|
|
return 1;
|
|
}
|
|
case 0x25: { // GetThreadId(w1 handle -> x1)
|
|
lock();
|
|
RtThread *t = h_get((uint32_t)x[1], O_THREAD);
|
|
x[0] = t ? R_OK : R_BAD_HANDLE;
|
|
if (t) x[1] = t->id;
|
|
unlock();
|
|
return 1;
|
|
}
|
|
|
|
case 0x1A: { // ArbitrateLock(w0 owner handle, x1 address, w2 tag)
|
|
RtThread *t = cur(imm);
|
|
uint64_t addr = x[1];
|
|
uint32_t r = R_OK;
|
|
lock();
|
|
if (ld32(addr) == ((uint32_t)x[0] | WAITERS_BIT)) {
|
|
t->waddr = addr;
|
|
t->wtag = (uint32_t)x[2];
|
|
t->wobj = (uint32_t)x[0];
|
|
r = block(t, W_LOCK, -1);
|
|
}
|
|
unlock();
|
|
x[0] = r;
|
|
return 1;
|
|
}
|
|
case 0x1B: // ArbitrateUnlock(x0 address)
|
|
lock();
|
|
lock_handoff(x[0]);
|
|
unlock();
|
|
x[0] = R_OK;
|
|
return 1;
|
|
case 0x1C: { // WaitProcessWideKeyAtomic(x0 lock, x1 key, w2 tag, x3 timeout)
|
|
// The key is set before the lock is released. The SDK's Signal reads the key in user mode and skips svc 0x1D
|
|
// when it is 0, and user mode takes the lock without G: releasing first lets another thread take the lock,
|
|
// change the condition, read key 0 and wake nobody, after which this thread waits forever (MNG threads.c
|
|
// 439-446). Linking into the wait list happens in block, still holding G, so svc 0x1D cannot come between.
|
|
RtThread *t = cur(imm);
|
|
uint64_t addr = x[0], key = x[1];
|
|
lock();
|
|
if (!kern_test_cv_old_order) {
|
|
st32(key, 1);
|
|
t->wkey = key;
|
|
t->waddr = addr;
|
|
t->wtag = (uint32_t)x[2];
|
|
lock_handoff(addr);
|
|
cv_hook();
|
|
} else { // the probe's negative control: MNG's pre-fix order
|
|
lock_handoff(addr);
|
|
cv_hook();
|
|
st32(key, 1);
|
|
t->wkey = key;
|
|
t->waddr = addr;
|
|
t->wtag = (uint32_t)x[2];
|
|
}
|
|
t->wobj = key;
|
|
uint32_t r = block(t, W_CONDVAR, (int64_t)x[3]); // on timeout user mode takes the lock itself
|
|
unlock();
|
|
x[0] = r;
|
|
return 1;
|
|
}
|
|
case 0x1D: { // SignalProcessWideKey(x0 key, w1 count; <= 0 all)
|
|
uint64_t key = x[0];
|
|
int cnt = (int)x[1], n = 0;
|
|
lock();
|
|
for (RtThread *w; (cnt <= 0 || n < cnt) && (w = pick(W_CONDVAR, key)); n++) lock_acquire_for(w, w->waddr, w->wtag);
|
|
int left = 0;
|
|
for (RtThread *w = g_wait; w && !left; w = w->next) left = w->wkind == W_CONDVAR && w->wkey == key;
|
|
if (!left) st32(key, 0);
|
|
unlock();
|
|
x[0] = R_OK;
|
|
return 1;
|
|
}
|
|
|
|
case 0x34: { // WaitForAddress(x0 address, w1 type, w2 value, x3 timeout): 0 if less, 1 decrement if less, 2 if equal
|
|
RtThread *t = cur(imm);
|
|
uint64_t addr = x[0];
|
|
int type = (int)x[1];
|
|
int32_t val = (int32_t)x[2];
|
|
if (type < 0 || type > 2) {
|
|
x[0] = R_INVALID_ENUM;
|
|
return 1;
|
|
}
|
|
lock();
|
|
uint32_t r, v = ld32(addr);
|
|
int ok = type == 2 ? (int32_t)v == val : (int32_t)v < val;
|
|
if (ok && type == 1)
|
|
while (!cas32(addr, &v, v - 1))
|
|
if (!((int32_t)v < val)) {
|
|
ok = 0;
|
|
break;
|
|
}
|
|
if (!ok) r = R_BAD_STATE;
|
|
else {
|
|
t->waddr = addr;
|
|
t->wobj = addr;
|
|
r = block(t, W_ADDRESS, (int64_t)x[3]);
|
|
}
|
|
unlock();
|
|
x[0] = r;
|
|
return 1;
|
|
}
|
|
case 0x35: { // SignalToAddress(x0 address, w1 type, w2 value, w3 count): 0 wake, 1 increment if equal,
|
|
// 2 if equal modify by the number of waiters
|
|
uint64_t addr = x[0];
|
|
int type = (int)x[1], cnt = (int)x[3];
|
|
int32_t val = (int32_t)x[2];
|
|
if (type < 0 || type > 2) {
|
|
x[0] = R_INVALID_ENUM;
|
|
return 1;
|
|
}
|
|
lock();
|
|
int nw = 0;
|
|
for (RtThread *w = g_wait; w; w = w->next) nw += w->wkind == W_ADDRESS && w->waddr == addr;
|
|
uint32_t r = R_OK;
|
|
if (type != 0) {
|
|
int32_t nv = val + 1;
|
|
if (type == 2) {
|
|
if (cnt <= 0) nv = nw > 0 ? val - 1 : val + 1;
|
|
else nv = nw == 0 ? val + 1 : nw <= cnt ? val - 1 : val;
|
|
}
|
|
uint32_t e = (uint32_t)val;
|
|
if (!cas32(addr, &e, (uint32_t)nv)) r = R_BAD_STATE;
|
|
}
|
|
if (!r)
|
|
for (int n = 0; cnt <= 0 || n < cnt; n++) {
|
|
RtThread *w = pick(W_ADDRESS, addr);
|
|
if (!w) break;
|
|
wake(w, 0);
|
|
}
|
|
unlock();
|
|
x[0] = r;
|
|
return 1;
|
|
}
|
|
|
|
case 0x45: { // CreateEvent(-> w1 writable, w2 readable)
|
|
Event *e = calloc(1, sizeof *e);
|
|
if (!e) rt_fatal("kernel: event allocation");
|
|
e->refs = 2;
|
|
lock();
|
|
uint32_t hw = h_new(O_EVENT_W, e), hr = h_new(O_EVENT_R, e);
|
|
unlock();
|
|
x[0] = R_OK;
|
|
x[1] = hw;
|
|
x[2] = hr;
|
|
return 1;
|
|
}
|
|
case 0x11: case 0x12: case 0x17: { // SignalEvent(writable), ClearEvent(either end), ResetSignal(readable)
|
|
uint32_t h = (uint32_t)x[0];
|
|
uint32_t r = R_OK;
|
|
lock();
|
|
Event *e = imm == 0x11 ? h_peek(h, O_EVENT_W) : imm == 0x17 ? h_peek(h, O_EVENT_R)
|
|
: h_peek(h, O_EVENT_W) ? h_peek(h, O_EVENT_W) : h_peek(h, O_EVENT_R);
|
|
if (!e) {
|
|
r = imm == 0x17 && h_peek(h, O_THREAD) ? R_BAD_STATE : R_BAD_HANDLE;
|
|
if (r == R_BAD_HANDLE) rt_log("kernel: event svc %#x: invalid handle %#x", imm, h);
|
|
} else if (imm == 0x11) {
|
|
e->signaled = 1;
|
|
wake_sync_waiters();
|
|
} else if (imm == 0x17 && !e->signaled) {
|
|
r = R_BAD_STATE;
|
|
} else {
|
|
e->signaled = 0;
|
|
}
|
|
unlock();
|
|
x[0] = r;
|
|
return 1;
|
|
}
|
|
case 0x18: { // WaitSynchronization(-> w1 index; x1 handles, w2 count, x3 timeout)
|
|
RtThread *t = cur(imm);
|
|
const uint32_t *hs = (const uint32_t *)(uintptr_t)x[1];
|
|
int n = (int)x[2];
|
|
int64_t ns = (int64_t)x[3];
|
|
if (n < 0 || n > 64 || (!hs && n)) {
|
|
x[0] = R_BAD_STATE;
|
|
return 1;
|
|
}
|
|
uint64_t start = kern_now_ns();
|
|
uint32_t r;
|
|
int idx = -1;
|
|
lock();
|
|
g_sync_calls++;
|
|
for (;;) {
|
|
if (t->cancel) {
|
|
t->cancel = 0;
|
|
r = R_CANCELLED;
|
|
break;
|
|
}
|
|
int bad = 0;
|
|
for (int i = 0; i < n && idx < 0; i++)
|
|
if (sig_obj(hs[i], &bad)) idx = i;
|
|
if (bad) {
|
|
r = R_BAD_HANDLE;
|
|
break;
|
|
}
|
|
if (idx >= 0) {
|
|
r = R_OK;
|
|
break;
|
|
}
|
|
int64_t left = ns;
|
|
if (ns > 0) {
|
|
uint64_t spent = kern_now_ns() - start;
|
|
if (spent >= (uint64_t)ns) {
|
|
r = R_TIMEOUT;
|
|
break;
|
|
}
|
|
left = ns - (int64_t)spent;
|
|
}
|
|
t->wobj = n ? hs[0] | (uint64_t)n << 32 : 0;
|
|
g_sync_blocks++;
|
|
if (block(t, W_SYNC, left) == R_TIMEOUT) {
|
|
r = R_TIMEOUT;
|
|
break;
|
|
}
|
|
}
|
|
unlock();
|
|
x[0] = r;
|
|
x[1] = idx >= 0 ? (uint64_t)idx : 0;
|
|
return 1;
|
|
}
|
|
case 0x19: { // CancelSynchronization(w0 thread)
|
|
lock();
|
|
RtThread *t = h_get((uint32_t)x[0], O_THREAD);
|
|
if (t) {
|
|
t->cancel = 1;
|
|
if (t->wkind == W_SYNC) wake(t, (uint32_t)-1);
|
|
}
|
|
unlock();
|
|
x[0] = t ? R_OK : R_BAD_HANDLE;
|
|
return 1;
|
|
}
|
|
case 0x16: // CloseHandle(w0)
|
|
x[0] = handle_close((uint32_t)x[0]);
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// ---- handles and events for host services (ipc.c and the service overrides)
|
|
uint32_t kern_handle_new(int kind, void *obj) {
|
|
if (kind != KERN_SESSION && kind != KERN_TMEM) rt_fatal("kern_handle_new: kind %d", kind);
|
|
lock();
|
|
uint32_t h = h_new(kind, obj);
|
|
unlock();
|
|
return h;
|
|
}
|
|
|
|
void *kern_handle_get(uint32_t h, int kind) {
|
|
lock();
|
|
void *o = h_peek(h, kind);
|
|
unlock();
|
|
return o;
|
|
}
|
|
|
|
uint32_t kern_event_new(void) {
|
|
Event *e = calloc(1, sizeof *e);
|
|
if (!e) rt_fatal("kernel: event allocation");
|
|
e->refs = 1;
|
|
lock();
|
|
uint32_t hw = h_new(O_EVENT_W, e);
|
|
unlock();
|
|
return hw;
|
|
}
|
|
|
|
// Copy-handle transfer of the readable end: each call gives the client its own handle to the same event.
|
|
uint32_t kern_event_readable(uint32_t hw) {
|
|
lock();
|
|
Event *e = h_peek(hw, O_EVENT_W);
|
|
if (!e) rt_fatal("kern_event_readable: %#x is not a writable event handle", hw);
|
|
e->refs++;
|
|
uint32_t hr = h_new(O_EVENT_R, e);
|
|
unlock();
|
|
return hr;
|
|
}
|
|
|
|
void kern_event_signal(uint32_t hw) {
|
|
lock();
|
|
Event *e = h_peek(hw, O_EVENT_W);
|
|
if (!e) rt_fatal("kern_event_signal: %#x is not a writable event handle", hw);
|
|
e->signaled = 1;
|
|
wake_sync_waiters();
|
|
unlock();
|
|
}
|
|
|
|
// System events of host services: the SDK's own nn::os::CreateSystemEvent creates the kernel event (SVC 0x45) on a
|
|
// guest thread; the writable handle is kept, so any thread (host audio tick included) can signal it later.
|
|
static struct { uint64_t ev; uint32_t wh; } g_se[512];
|
|
static int g_nse;
|
|
static SRWLOCK g_se_lock = SRWLOCK_INIT;
|
|
|
|
static uint32_t se_find(uint64_t ev) {
|
|
AcquireSRWLockShared(&g_se_lock);
|
|
uint32_t wh = 0;
|
|
for (int i = 0; i < g_nse && !wh; i++)
|
|
if (g_se[i].ev == ev) wh = g_se[i].wh;
|
|
ReleaseSRWLockShared(&g_se_lock);
|
|
return wh;
|
|
}
|
|
|
|
void kern_bind_system_event(Cpu *c, uint64_t ev, int autoclear) {
|
|
if (se_find(ev)) return;
|
|
if (!tcur || !c) rt_fatal("kern_bind_system_event(%#llx) on a host thread", (unsigned long long)ev);
|
|
static uint64_t create, getw;
|
|
if (!create) {
|
|
create = ldr_symbol("_ZN2nn2os17CreateSystemEventEPNS0_15SystemEventTypeENS0_14EventClearModeEb");
|
|
getw = ldr_symbol("_ZN2nn2os30GetWritableHandleOfSystemEventEPKNS0_15SystemEventTypeE");
|
|
if (!create || !getw) rt_fatal("kern_bind_system_event: the SDK's system event functions are not loaded");
|
|
}
|
|
uint32_t r = (uint32_t)rt_call(c, create, 3, ev, (uint64_t)(autoclear ? 1 : 0), (uint64_t)1);
|
|
if (r) rt_fatal("kern_bind_system_event: CreateSystemEvent(%#llx) failed %#x", (unsigned long long)ev, r);
|
|
uint32_t wh = (uint32_t)rt_call(c, getw, 1, ev);
|
|
AcquireSRWLockExclusive(&g_se_lock);
|
|
if (g_nse == (int)(sizeof g_se / sizeof *g_se)) rt_fatal("kern_bind_system_event: table full");
|
|
g_se[g_nse].ev = ev;
|
|
g_se[g_nse++].wh = wh;
|
|
ReleaseSRWLockExclusive(&g_se_lock);
|
|
}
|
|
|
|
// An event not bound yet is bound on the spot with manual clear, which needs a guest thread.
|
|
void kern_signal_system_event(uint64_t ev) {
|
|
uint32_t wh = se_find(ev);
|
|
if (!wh) {
|
|
kern_bind_system_event(kern_cpu(), ev, 0);
|
|
wh = se_find(ev);
|
|
}
|
|
kern_event_signal(wh);
|
|
}
|
|
|
|
// ---- fibers
|
|
// The SDK switches fibers by swapping register context, which breaks "RET = C return". Each guest fiber gets a host
|
|
// fiber (own 256 MiB reserved host stack) and its own Cpu, so a suspended fiber's registers and host frames stay
|
|
// untouched while others run, and a fiber may resume on another thread. A switch carries the thread state (TLR, TLS
|
|
// pointer, FPCR/FPSR) over to the target's Cpu and clears the exclusive monitor.
|
|
enum { F_NEW, F_STARTED, F_DONE };
|
|
struct Fiber {
|
|
uint64_t addr, func, arg, stack, size;
|
|
Cpu *cpu;
|
|
void *host;
|
|
int state;
|
|
volatile int running;
|
|
};
|
|
static Fiber *g_fibers[FIBER_MAX];
|
|
static SRWLOCK g_fiber_lock = SRWLOCK_INIT;
|
|
|
|
static Fiber *fiber_find(uint64_t addr) {
|
|
Fiber *f = NULL;
|
|
AcquireSRWLockShared(&g_fiber_lock);
|
|
for (int i = 0; i < FIBER_MAX && !f; i++)
|
|
if (g_fibers[i] && g_fibers[i]->addr == addr) f = g_fibers[i];
|
|
ReleaseSRWLockShared(&g_fiber_lock);
|
|
return f;
|
|
}
|
|
|
|
static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to);
|
|
|
|
// After a host fiber switch: release the context left behind, and finish an ExitThread issued on a fiber.
|
|
static void fiber_resumed(void) {
|
|
RtThread *t = tcur;
|
|
rt_fpcr_sync(t->cur); // the host fiber resumes with its own MXCSR (FIBER_FLAG_FLOAT_SWITCH)
|
|
if (t->leaving) {
|
|
__atomic_store_n(&t->leaving->running, 0, __ATOMIC_RELEASE);
|
|
t->leaving = NULL;
|
|
}
|
|
if (t->exit_pending && !t->fiber) exit_now(t);
|
|
}
|
|
|
|
static void WINAPI fiber_main(void *p) {
|
|
Fiber *f = p;
|
|
fiber_resumed();
|
|
Cpu *c = f->cpu;
|
|
tcur->host_depth = 0; // a new fiber starts at the top level; each context restores its own depth on resume
|
|
uint64_t next = rt_call(c, f->func, 1, f->arg);
|
|
f->state = F_DONE;
|
|
Fiber *n = NULL;
|
|
if (next && !(n = fiber_find(next)))
|
|
rt_fatal("fiber %#llx returned %#llx, which is not an initialized fiber", (unsigned long long)f->addr,
|
|
(unsigned long long)next);
|
|
fiber_switch_to(tcur, c, n); // NULL: back to the thread's own context
|
|
rt_fatal("fiber %#llx resumed after it finished", (unsigned long long)f->addr);
|
|
}
|
|
|
|
static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) {
|
|
if (!t->base_fiber) {
|
|
t->base_fiber = ConvertThreadToFiberEx(NULL, FIBER_FLAG_FLOAT_SWITCH);
|
|
if (!t->base_fiber) rt_fatal("kernel: ConvertThreadToFiberEx failed (error %lu)", GetLastError());
|
|
}
|
|
Cpu *dst = to ? to->cpu : t->cpu;
|
|
if (to) {
|
|
if (to->state == F_DONE) rt_fatal("SwitchToFiber(%#llx): the fiber has finished", (unsigned long long)to->addr);
|
|
int idle = 0;
|
|
if (!__atomic_compare_exchange_n(&to->running, &idle, 1, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE))
|
|
rt_fatal("SwitchToFiber(%#llx): the fiber is running", (unsigned long long)to->addr);
|
|
if (to->state == F_NEW) {
|
|
dst->sp = (to->stack + to->size) & ~15ull;
|
|
to->state = F_STARTED;
|
|
}
|
|
}
|
|
dst->tpidr = from->tpidr;
|
|
dst->tpidrro = from->tpidrro;
|
|
dst->fpcr = from->fpcr;
|
|
dst->fpsr = from->fpsr;
|
|
dst->excl_addr = ~0ull;
|
|
dst->thread = t;
|
|
t->leaving = t->fiber;
|
|
t->fiber = to;
|
|
t->cur = dst;
|
|
int depth = t->host_depth;
|
|
SwitchToFiber(to ? to->host : t->base_fiber);
|
|
t->host_depth = depth;
|
|
fiber_resumed();
|
|
}
|
|
|
|
// InitializeFiber(x0 FiberType*, x1 function, x2 argument, x3 stack, x4 stack size, w5 flags)
|
|
void kern_fiber_initialize(Cpu *c) {
|
|
uint64_t addr = c->x[0];
|
|
Fiber *f = fiber_find(addr);
|
|
if (f && f->running) rt_fatal("InitializeFiber(%#llx): the fiber is running", (unsigned long long)addr);
|
|
if (!f) {
|
|
f = calloc(1, sizeof *f);
|
|
Cpu *fc = _aligned_malloc(sizeof(Cpu), 64);
|
|
if (!f || !fc) rt_fatal("kernel: fiber allocation");
|
|
f->cpu = fc;
|
|
AcquireSRWLockExclusive(&g_fiber_lock);
|
|
int i = 0;
|
|
while (i < FIBER_MAX && g_fibers[i]) i++;
|
|
if (i == FIBER_MAX) rt_fatal("kernel: more than %d fibers", FIBER_MAX);
|
|
g_fibers[i] = f;
|
|
ReleaseSRWLockExclusive(&g_fiber_lock);
|
|
} else if (f->host) {
|
|
DeleteFiber(f->host);
|
|
}
|
|
memset(f->cpu, 0, sizeof *f->cpu);
|
|
f->cpu->excl_addr = ~0ull;
|
|
f->addr = addr;
|
|
f->func = c->x[1];
|
|
f->arg = c->x[2];
|
|
f->stack = c->x[3];
|
|
f->size = c->x[4];
|
|
f->state = F_NEW;
|
|
f->running = 0;
|
|
f->host = CreateFiberEx(0x10000, HOST_STACK, FIBER_FLAG_FLOAT_SWITCH, fiber_main, f);
|
|
if (!f->host) rt_fatal("kernel: CreateFiberEx failed (error %lu)", GetLastError());
|
|
}
|
|
|
|
// FinalizeFiber(x0 FiberType*)
|
|
void kern_fiber_finalize(Cpu *c) {
|
|
uint64_t addr = c->x[0];
|
|
AcquireSRWLockExclusive(&g_fiber_lock);
|
|
Fiber *f = NULL;
|
|
for (int i = 0; i < FIBER_MAX && !f; i++)
|
|
if (g_fibers[i] && g_fibers[i]->addr == addr) {
|
|
f = g_fibers[i];
|
|
g_fibers[i] = NULL;
|
|
}
|
|
ReleaseSRWLockExclusive(&g_fiber_lock);
|
|
if (!f) rt_fatal("FinalizeFiber(%#llx): not an initialized fiber", (unsigned long long)addr);
|
|
if (f->running) rt_fatal("FinalizeFiber(%#llx): the fiber is running", (unsigned long long)addr);
|
|
DeleteFiber(f->host);
|
|
_aligned_free(f->cpu);
|
|
free(f);
|
|
}
|
|
|
|
// SwitchToFiber(x0 FiberType*)
|
|
void kern_fiber_switch(Cpu *c) {
|
|
RtThread *t = tcur;
|
|
if (!t) rt_fatal("SwitchToFiber on a host thread");
|
|
Fiber *to = fiber_find(c->x[0]);
|
|
if (!to) rt_fatal("SwitchToFiber(%#llx): not an initialized fiber", (unsigned long long)c->x[0]);
|
|
if (to == t->fiber) return;
|
|
fiber_switch_to(t, c, to);
|
|
}
|
|
|
|
// GetCurrentFiber() -> x0, NULL on the thread's own context
|
|
void kern_fiber_current(Cpu *c) {
|
|
RtThread *t = tcur;
|
|
c->x[0] = t && t->fiber ? t->fiber->addr : 0;
|
|
}
|
|
|
|
HOST_OVERRIDE(KERN_SYM_INITIALIZE_FIBER, kern_fiber_initialize);
|
|
HOST_OVERRIDE(KERN_SYM_FINALIZE_FIBER, kern_fiber_finalize);
|
|
HOST_OVERRIDE(KERN_SYM_SWITCH_TO_FIBER, kern_fiber_switch);
|
|
HOST_OVERRIDE(KERN_SYM_GET_CURRENT_FIBER, kern_fiber_current);
|