From d52cdfc6eeff48413dd7370ff0b9c95b28baf222 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 25 Oct 2026 23:23:48 +0300 Subject: [PATCH 1/6] Add TOTK_CORE_DIAG: classify every virtual core wait by what the owner was doing, and answer SVC 0x36 Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 +- runtime/host.h | 7 ++ runtime/hostfn.c | 16 +++ runtime/kernel.c | 270 ++++++++++++++++++++++++++++++++++++++++++++- runtime/prof_win.c | 1 + runtime/svc.c | 16 +++ 6 files changed, 308 insertions(+), 4 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index b2dbf1a0..6c0a385f 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -26,7 +26,7 @@ static const char *const known[] = { "APP_POOL", "ASSUME_FOCUS", "ASTC", "ASTC_TIME", "AUDIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", "BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", - "CLOTH_REALTIME", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", + "CLOTH_REALTIME", "CORE_DIAG", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "FREEZE_IDLE", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", diff --git a/runtime/host.h b/runtime/host.h index 58e59ea1..f7e9d34d 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -176,6 +176,13 @@ void kern_core_back(int *away); void kern_checkpoint(void); // preemption point, at every SVC and indirect call // A preempt request is pending for the thread (Cpu.thread): its RtThread starts with the flag (kernel.c). static inline int kern_preempt_pending(const void *thread) { return __atomic_load_n((const int *)thread, __ATOMIC_RELAXED); } +// Set once before guest threads start (TOTK_CORE_DIAG): host calls (hostfn.c) and SVCs (svc.c) count their depth in +// the thread's second int, so the kernel knows whether a core owner runs game code at the top level. +extern int kern_host_track; +static inline int *kern_host_depth(void *thread) { return (int *)thread + 1; } +void kern_host_left(void *thread); // back at depth 0 with a preempt request pending +int kern_core_diag_on(void); // TOTK_CORE_DIAG +void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns); // perf_present, every present void kern_sched_report(uint64_t frame); // TOTK_TRACE=sched: per-thread time split, on present uint32_t kern_handle_new(int kind, void *obj); // kind: KERN_SESSION or KERN_TMEM (ipc.c) void *kern_handle_get(uint32_t h, int kind); diff --git a/runtime/hostfn.c b/runtime/hostfn.c index 814ada50..0b885293 100644 --- a/runtime/hostfn.c +++ b/runtime/hostfn.c @@ -108,6 +108,18 @@ __attribute__((noinline, noreturn, cold)) static void bad_entry(Cpu *c) { ldr_describe(c->x[30], lr, sizeof lr)); } +// kern_host_track: the call counted in the thread's host depth. Not a tail call, so the host stack holds this frame +// while fn runs; longjmp out of fn (ExitThread) leaves the depth raised on a thread that ends. +__attribute__((noinline)) static void hostfn_tracked(Cpu *c, GuestFn fn) { + void *t = c->thread; + int *d = kern_host_depth(t); + __atomic_store_n(d, *d + 1, __ATOMIC_RELAXED); + fn(c); + int n = *d - 1; + __atomic_store_n(d, n, __ATOMIC_RELAXED); + if (!n && kern_preempt_pending(t)) kern_host_left(t); +} + static void hostfn_entry(Cpu *c) { uint64_t i = (c->pc - RT_HOSTFN_BASE) >> 4; if (__builtin_expect((c->pc & 15) || i >= (uint64_t)nslots, 0)) bad_entry(c); @@ -117,6 +129,10 @@ static void hostfn_entry(Cpu *c) { char lr[96]; rt_log("host call %s from %s", s->name, ldr_describe(c->x[30] - 4, lr, sizeof lr)); } + if (__builtin_expect(kern_host_track && c->thread, 0)) { + hostfn_tracked(c, s->fn); + return; + } __attribute__((musttail)) return s->fn(c); } diff --git a/runtime/kernel.c b/runtime/kernel.c index a3bdd79b..32a0f2a5 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -53,6 +53,7 @@ typedef struct Fiber Fiber; struct RtThread { int preempt; // first member: rt_dispatch reads it (host.h kern_preempt_pending) + int host_depth; // second member: host calls and SVCs in progress, while kern_host_track (host.h) uint32_t handle; uint64_t id; Cpu *cpu; // the thread's own context @@ -86,12 +87,23 @@ struct RtThread { int st; // S_* state since st_since uint64_t st_since, st_ns[S_N], nblk, npre, pre_since, cpu_last; uint64_t wobj; // what the current wait is on: owner handle, key, address or first handle + // TOTK_CORE_DIAG (holding G, except dg_host_end, which only the thread itself writes) + uint64_t dg_since; // as a waiter: core wait start + int dg_cls; // as a waiter: DG_* class at the start of the wait + RtThread *dg_target; // as a waiter: the owner asked to give its core up for it + uint64_t dg_kick; // as an owner: the preempt request + uint64_t dg_host_end; // as an owner: the return from the host call or SVC it was in at the request + uint64_t dg_pc, dg_lr; // as an owner: guest pc and lr at the checkpoint that gave the core up + int dg_host_kick; // as an owner: in a host call or SVC at the request + struct DgRel { uint64_t kick, host_end, pc, lr; int host, pre, prio; uint32_t h; } dg_rel; // as a waiter: the + // owner that released }; static SRWLOCK G = SRWLOCK_INIT; static struct { int type; void *obj; } g_h[MAX_H]; static _Thread_local RtThread *tcur; _Static_assert(offsetof(RtThread, preempt) == 0, "host.h kern_preempt_pending reads the first int"); +_Static_assert(offsetof(RtThread, host_depth) == sizeof(int), "host.h kern_host_depth reads the second int"); static RtThread *g_all, *g_wait, *g_main; static uint64_t g_next_id = 0x51, g_wseq, g_created; static uint64_t g_main_stack; @@ -103,7 +115,7 @@ uint64_t kern_now_ns(void) { return plat_now_ns(); } // ---- TOTK_TRACE=sched accounting (holding G) #define RW_N 32 -static RtThread *g_render; // the thread that presents +static RtThread *g_render; // the thread that presents (sched trace, core diag) static struct { int st; uint64_t obj, ns; } g_rw[RW_N]; // the render thread's waits by object static uint64_t g_pre_req, g_pre_n, g_pre_ns, g_pre_max, g_pre_stale; // preempt requests; releases after one and their latency static uint64_t g_sync_calls, g_sync_blocks; // WaitSynchronization calls and the blocks inside them @@ -243,19 +255,218 @@ static uint32_t block_inner(RtThread *t, int kind, int64_t ns) { static RtThread *g_core_owner[NCORE], *g_core_wait; static int g_cores_on = -1, g_held_max; +static void diag_init(void); + static int cores_enabled(void) { - if (g_cores_on < 0) g_cores_on = cfg_get("TOTK_FREE_CORES") ? 0 : 1; + if (g_cores_on < 0) { + g_cores_on = cfg_get("TOTK_FREE_CORES") ? 0 : 1; + diag_init(); + } return g_cores_on; } static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) & 1); } +// ---- TOTK_CORE_DIAG=1: why ready threads wait for a virtual core, and for how long +// Each core wait is split into classes: the owner asked to give the core up ran game code (DG_GAME, the part an +// asynchronous preemption would remove), was in a host call or SVC that kept the core (DG_HOST), every owner the waiter +// may displace was of better or equal priority (DG_EQ: Horizon waits too), or the worse owners were claimed by better +// waiters (DG_NOCORE). Per 60 presents: time per frame and a wait-length histogram per class, the owners that cost +// most; a line per present over 25 ms; a line per wait with over 2 ms of DG_GAME (where the owner reached its +// checkpoint), and one host RIP sample of the owner per such wait (exe RVA, for totk.map). +enum { DG_GAME, DG_HOST, DG_EQ, DG_NOCORE, DG_N }; +static const char *const dg_name[DG_N] = {"game", "host", "eq", "nocore"}; +#define DG_NB 8 // wait lengths: < 0.05, 0.25, 1, 2, 4, 8, 16 ms, longer +static const uint64_t dg_edge[DG_NB - 1] = {50000, 250000, 1000000, 2000000, 4000000, 8000000, 16000000}; +#define DG_LONG_LINES 400 +#define DG_SAMPLES 600 +#define DG_OWN 32 +int kern_host_track; +static int g_diag = -1; +typedef struct { + uint64_t ns[DG_N], n[DG_N], hist[DG_N][DG_NB]; // this window + struct { uint32_t h; int prio, cls; uint64_t ns; } own[DG_OWN]; // this window: owners by class + uint64_t long_frames, long_game_frames; +} DgWindow; +static DgWindow g_dgw; +static uint64_t g_dgf[DG_N], g_dgfr[DG_N]; // this frame: all threads, the render thread +static uint64_t g_dg_samples, g_dg_lines, g_sync36, g_sync36_flag; + +static void diag_init(void) { + if (g_diag >= 0) return; + const char *e = cfg_get("TOTK_CORE_DIAG"); + g_diag = e && *e && strcmp(e, "0"); + if (g_diag) { + kern_host_track = 1; + rt_log("core-diag: on (TOTK_CORE_DIAG)"); + } +} + +int kern_core_diag_on(void) { return g_diag > 0; } + +// A tracked host call or SVC returned to depth 0 with a preempt request pending (host.h); on the thread itself. +void kern_host_left(void *thread) { + RtThread *t = thread; + if (g_diag > 0 && t->dg_kick && t->dg_host_kick && !t->dg_host_end) t->dg_host_end = kern_now_ns(); +} + +static int dg_in_host(const RtThread *o) { return __atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0; } + +static void dg_add(const RtThread *w, int cls, uint64_t ns, uint32_t oh, int oprio) { + if (!ns) return; + g_dgw.ns[cls] += ns; + g_dgf[cls] += ns; + if (w == g_render) g_dgfr[cls] += ns; + if (!oh) return; + int i = 0; + for (; i < DG_OWN && g_dgw.own[i].ns; i++) + if (g_dgw.own[i].h == oh && g_dgw.own[i].cls == cls) break; + if (i == DG_OWN) return; + g_dgw.own[i].h = oh; + g_dgw.own[i].prio = oprio; + g_dgw.own[i].cls = cls; + g_dgw.own[i].ns += ns; +} + +// The waiter's class at the start of its wait, after core_kick claimed (or did not claim) an owner for it. +static int dg_class0(const RtThread *t) { + if (t->dg_target) return dg_in_host(t->dg_target) ? DG_HOST : DG_GAME; + for (int i = 0; i < NCORE; i++) + if (core_allowed(t, i) && g_core_owner[i] && g_core_owner[i]->prio > t->prio) return DG_NOCORE; + return DG_EQ; +} + +// The waiter got a core: split [dg_since, now] by what the owner that released it did after its preempt request. +static void dg_done(RtThread *t) { + uint64_t now = kern_now_ns(), t0 = t->dg_since, w = now - t0; + t->dg_since = 0; + if (!t0) return; + const struct DgRel *r = &t->dg_rel; + uint64_t part[DG_N] = {0}; + uint32_t oh = t->dg_target ? t->dg_target->handle : r->h; + int oprio = t->dg_target ? t->dg_target->prio : r->prio; + if (r->pre && r->kick) { + uint64_t k = r->kick > t0 ? (r->kick < now ? r->kick : now) : t0; + part[t->dg_cls] += k - t0; + if (r->host) { + uint64_t he = r->host_end > k && r->host_end < now ? r->host_end : now; + part[DG_HOST] += he - k; + part[DG_GAME] += now - he; + } else + part[DG_GAME] += now - k; + oh = r->h; + oprio = r->prio; + } else + part[t->dg_cls] += w; + int top = 0; + for (int c = 0; c < DG_N; c++) { + dg_add(t, c, part[c], oh, oprio); + if (part[c] > part[top]) top = c; + } + int b = 0; + while (b < DG_NB - 1 && w >= dg_edge[b]) b++; + g_dgw.n[top]++; + g_dgw.hist[top][b]++; + if (part[DG_GAME] >= 2000000 && g_dg_lines < DG_LONG_LINES) { + g_dg_lines++; + char a[128], l[128]; + rt_log("core-diag: wait %.2f ms (game %.2f host %.2f): %#x p%d mask %llx; released by %#x p%d%s at %s lr %s", + w / 1e6, part[DG_GAME] / 1e6, part[DG_HOST] / 1e6, t->handle, t->prio, (unsigned long long)t->mask, r->h, + r->prio, r->pre ? " (preempted)" : "", r->pc ? ldr_describe(r->pc, a, sizeof a) : "-", + r->lr ? ldr_describe(r->lr, l, sizeof l) : "-"); + } + t->dg_target = NULL; + memset(&t->dg_rel, 0, sizeof t->dg_rel); +} + +// Once per long DG_GAME wait: where the owner's host thread runs (exe RVA). Holding G; the owner is resumed before +// anything that could take a lock it holds. +static void dg_sample(RtThread *t) { + RtThread *o = t->dg_target; + if (!o || o->cur_core < 0 || !o->tid || g_dg_samples >= DG_SAMPLES) return; + HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, o->tid); + if (!h) return; + CONTEXT ctx; + memset(&ctx, 0, sizeof ctx); + ctx.ContextFlags = CONTEXT_CONTROL; + int depth = 0, ok = 0; + if (SuspendThread(h) != (DWORD)-1) { + ok = GetThreadContext(h, &ctx); + depth = __atomic_load_n(&o->host_depth, __ATOMIC_RELAXED); + ResumeThread(h); + } + CloseHandle(h); + if (!ok) return; + g_dg_samples++; + uintptr_t base = (uintptr_t)GetModuleHandleW(NULL); + rt_log("core-diag: sample owner %#x p%d depth %d rip exe+%#llx (waiter %#x p%d, %.2f ms)", o->handle, o->prio, depth, + (unsigned long long)(ctx.Rip - base), t->handle, t->prio, (kern_now_ns() - t->dg_since) / 1e6); +} + + +// perf_present, every present: a line for a present over 25 ms, the 60-present summary. +void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns) { + if (g_diag <= 0) return; + DgWindow w; + uint64_t f[DG_N], fr[DG_N], s36, s36f; + lock(); + g_render = tcur; + memcpy(f, g_dgf, sizeof f); + memcpy(fr, g_dgfr, sizeof fr); + memset(g_dgf, 0, sizeof g_dgf); + memset(g_dgfr, 0, sizeof g_dgfr); + int lng = interval_ns > 25000000ull; + if (lng) { + g_dgw.long_frames++; + if (f[DG_GAME] > 1000000) g_dgw.long_game_frames++; + } + int rep = frame && frame % 60 == 0; + if (rep) { + w = g_dgw; + memset(&g_dgw, 0, sizeof g_dgw); + } + s36 = g_sync36; + s36f = g_sync36_flag; + unlock(); + if (lng) + rt_log("core-diag: frame %llu: %.1f ms; core wait game %.2f host %.2f eq %.2f nocore %.2f ms; render thread game " + "%.2f host %.2f eq %.2f nocore %.2f ms", + (unsigned long long)frame, interval_ns / 1e6, f[0] / 1e6, f[1] / 1e6, f[2] / 1e6, f[3] / 1e6, + fr[0] / 1e6, fr[1] / 1e6, fr[2] / 1e6, fr[3] / 1e6); + if (!rep) return; + char s[1024]; + int o = 0; + for (int c = 0; c < DG_N; c++) { + o += snprintf(s + o, sizeof s - o, "%s %.3f ms/frame %llu waits [", dg_name[c], w.ns[c] / 60e6, + (unsigned long long)w.n[c]); + for (int b = 0; b < DG_NB; b++) + o += snprintf(s + o, sizeof s - o, "%s%llu", b ? " " : "", (unsigned long long)w.hist[c][b]); + o += snprintf(s + o, sizeof s - o, "]%s", c < DG_N - 1 ? "; " : ""); + } + rt_log("core-diag: frames %llu-%llu: %s; long frames %llu (game wait > 1 ms in %llu); SVC 0x36 %llu (flag set %llu)", + (unsigned long long)frame - 59, (unsigned long long)frame, s, (unsigned long long)w.long_frames, + (unsigned long long)w.long_game_frames, (unsigned long long)s36, (unsigned long long)s36f); + o = 0; + for (int k = 0; k < 6; k++) { + int bi = -1; + for (int i = 0; i < DG_OWN; i++) + if (w.own[i].ns && (bi < 0 || w.own[i].ns > w.own[bi].ns)) bi = i; + if (bi < 0) break; + o += snprintf(s + o, sizeof s - o, " %#x p%d %s %.3f;", w.own[bi].h, w.own[bi].prio, dg_name[w.own[bi].cls], + w.own[bi].ns / 60e6); + w.own[bi].ns = 0; + } + if (o) rt_log("core-diag: owners (ms/frame):%s", s); +} + static void core_count(void) { int n = 0; for (int i = 0; i < NCORE; i++) n += g_core_owner[i] != NULL; if (n > g_held_max) g_held_max = n; } +static struct DgRel g_dg_rel; // TOTK_CORE_DIAG: the owner core_put is releasing, for core_grant + static void core_grant(int c) { RtThread *b = NULL, **bp = NULL; for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext) @@ -269,6 +480,7 @@ static void core_grant(int c) { b->cur_core = c; g_core_owner[c] = b; core_count(); + if (g_diag > 0) b->dg_rel = g_dg_rel; WakeConditionVariable(&b->cv); } @@ -283,6 +495,12 @@ static void core_put(RtThread *t) { t->npre++; } t->pre_since = 0; + if (g_diag > 0) { + g_dg_rel = (struct DgRel){.kick = t->dg_kick, .host_end = t->dg_host_end, .pc = t->dg_pc, .lr = t->dg_lr, + .host = t->dg_host_kick, .pre = t->preempt, .prio = t->prio, .h = t->handle}; + t->dg_kick = t->dg_host_end = t->dg_pc = t->dg_lr = 0; + t->dg_host_kick = 0; + } t->cur_core = -1; t->preempt = 0; g_core_owner[c] = NULL; @@ -320,12 +538,18 @@ static void core_kick(void) { if (v < 0) continue; claimed |= 1 << v; RtThread *o = g_core_owner[v]; + w[k]->dg_target = o; if (o->preempt) continue; if (rt_trace & RT_TRACE_SCHED) { o->pre_since = kern_now_ns(); g_pre_req++; } - __atomic_store_n(&o->preempt, 1, __ATOMIC_RELAXED); + if (g_diag > 0) { + o->dg_kick = kern_now_ns(); + o->dg_host_kick = dg_in_host(o); + o->dg_host_end = 0; + } + __atomic_store_n(&o->preempt, 1, __ATOMIC_RELEASE); } } @@ -351,7 +575,28 @@ static void core_take(RtThread *t) { t->cseq = ++g_wseq; t->cnext = g_core_wait; g_core_wait = t; + if (g_diag > 0) { + t->dg_since = kern_now_ns(); + t->dg_target = NULL; + memset(&t->dg_rel, 0, sizeof t->dg_rel); + } core_kick(); + if (g_diag > 0) { + t->dg_cls = dg_class0(t); + int sampled = 0; + while (t->cur_core < 0) { + if (sampled || t->dg_cls != DG_GAME) { + SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0); + continue; + } + SleepConditionVariableSRW(&t->cv, &G, 2, 0); + if (t->cur_core < 0 && kern_now_ns() - t->dg_since >= 2000000) { + sampled = 1; + dg_sample(t); + } + } + dg_done(t); + } while (t->cur_core < 0) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0); acct(t, S_RUN); } @@ -393,12 +638,18 @@ void kern_checkpoint(void) { RtThread *t = tcur; if (!t || !__atomic_load_n(&t->preempt, __ATOMIC_RELAXED)) return; lock(); + if (g_diag > 0 && t->cur) { + t->dg_pc = t->cur->pc; + t->dg_lr = t->cur->x[30]; + } if (t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) { core_put(t); // clears preempt (after the sched trace reads it) core_take(t); } else { t->preempt = 0; t->pre_since = 0; + t->dg_kick = t->dg_host_end = 0; + t->dg_host_kick = 0; g_pre_stale++; } unlock(); @@ -932,6 +1183,16 @@ int kern_svc(Cpu *c, uint32_t imm) { x[0] = (uint64_t)(uint32_t)(t->cur_core >= 0 ? t->cur_core : t->core); return 1; } + case 0x36: { // SynchronizePreemptionState: the SDK's spin locks call it after clearing TLR+0x100 (disable_count) + // when TLR+0x102 (interrupt_flag) is set; rt_svc's checkpoint has already honoured a pending preempt + RtThread *t = cur(imm); + volatile uint16_t *flag = (volatile uint16_t *)(t->tlr + 0x102); + __atomic_fetch_add(&g_sync36, 1, __ATOMIC_RELAXED); + if (*flag) __atomic_fetch_add(&g_sync36_flag, 1, __ATOMIC_RELAXED); + *flag = 0; + x[0] = R_OK; + return 1; + } case 0x25: { // GetThreadId(w1 handle -> x1) lock(); RtThread *t = h_get((uint32_t)x[1], O_THREAD); @@ -1299,6 +1560,7 @@ static void WINAPI fiber_main(void *p) { Fiber *f = p; fiber_resumed(); Cpu *c = f->cpu; + tcur->host_depth = 0; // a new fiber starts at the top level; each context restores its own depth on resume uint64_t next = rt_call(c, f->func, 1, f->arg); f->state = F_DONE; Fiber *n = NULL; @@ -1335,7 +1597,9 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) { t->leaving = t->fiber; t->fiber = to; t->cur = dst; + int depth = t->host_depth; SwitchToFiber(to ? to->host : t->base_fiber); + t->host_depth = depth; fiber_resumed(); } diff --git a/runtime/prof_win.c b/runtime/prof_win.c index d81954ba..1468b9f9 100644 --- a/runtime/prof_win.c +++ b/runtime/prof_win.c @@ -409,6 +409,7 @@ void perf_present(uint64_t frame) { if (rt_trace & RT_TRACE_SCHED) kern_sched_report(frame); uint64_t now = kern_now_ns(), d = last ? now - last : 0; ft_frame(frame, now, d); // d is 0 on the first present: it only starts the period + kern_core_diag_frame(frame, d); if (flight_on) flight_frame(frame, now, d, frame_ns, counts); if (!(rt_trace & RT_TRACE_PERF)) { memset(frame_ns, 0, sizeof frame_ns); diff --git a/runtime/svc.c b/runtime/svc.c index 8454885a..4dc9b451 100644 --- a/runtime/svc.c +++ b/runtime/svc.c @@ -138,8 +138,24 @@ static void get_info(Cpu *c, uint64_t pc) { c->x[1] = v; } +static void svc_body(Cpu *c, uint32_t imm, uint64_t pc); + void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) { kern_checkpoint(); + if (__builtin_expect(!kern_host_track || !c->thread, 1)) { + svc_body(c, imm, pc); + return; + } + void *t = c->thread; // kern_host_track: the SVC counted in the thread's host depth (hostfn.c hostfn_tracked) + int *d = kern_host_depth(t); + __atomic_store_n(d, *d + 1, __ATOMIC_RELAXED); + svc_body(c, imm, pc); + int n = *d - 1; + __atomic_store_n(d, n, __ATOMIC_RELAXED); + if (!n && kern_preempt_pending(t)) kern_host_left(t); +} + +static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) { __atomic_fetch_add(&g_svcs, 1, __ATOMIC_RELAXED); if (rt_trace & RT_TRACE_SVC) { char at[160]; From c66ac84925878ef45d571ab48749661e19a77b69 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 25 Oct 2026 23:37:11 +0300 Subject: [PATCH 2/6] Preempt a virtual core owner in top-level game code from the waiting thread, with priority inheritance and the SDK's preemption disable, behind TOTK_ASYNC_PREEMPT Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 +- runtime/host.h | 5 +- runtime/kernel.c | 376 ++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 346 insertions(+), 37 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index 6c0a385f..f5c2be89 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -23,7 +23,7 @@ static int n_entries; // Every name the runtime reads (and the patch sets in generated/), without the prefix. A file key outside this list is // logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line. static const char *const known[] = { - "APP_POOL", "ASSUME_FOCUS", "ASTC", "ASTC_TIME", "AUDIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", + "APP_POOL", "ASSUME_FOCUS", "ASTC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", "BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "CLOTH_REALTIME", "CORE_DIAG", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", diff --git a/runtime/host.h b/runtime/host.h index f7e9d34d..43aef4be 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -176,13 +176,14 @@ void kern_core_back(int *away); void kern_checkpoint(void); // preemption point, at every SVC and indirect call // A preempt request is pending for the thread (Cpu.thread): its RtThread starts with the flag (kernel.c). static inline int kern_preempt_pending(const void *thread) { return __atomic_load_n((const int *)thread, __ATOMIC_RELAXED); } -// Set once before guest threads start (TOTK_CORE_DIAG): host calls (hostfn.c) and SVCs (svc.c) count their depth in +// Set once before guest threads start (TOTK_CORE_DIAG, TOTK_ASYNC_PREEMPT): host calls (hostfn.c) and SVCs (svc.c) count their depth in // the thread's second int, so the kernel knows whether a core owner runs game code at the top level. extern int kern_host_track; static inline int *kern_host_depth(void *thread) { return (int *)thread + 1; } void kern_host_left(void *thread); // back at depth 0 with a preempt request pending int kern_core_diag_on(void); // TOTK_CORE_DIAG -void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns); // perf_present, every present +void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns); // perf_present, every present (also logs the + // asynchronous preemption counters) void kern_sched_report(uint64_t frame); // TOTK_TRACE=sched: per-thread time split, on present uint32_t kern_handle_new(int kind, void *obj); // kind: KERN_SESSION or KERN_TMEM (ipc.c) void *kern_handle_get(uint32_t h, int kind); diff --git a/runtime/kernel.c b/runtime/kernel.c index 32a0f2a5..e2a9fc6d 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -87,10 +87,15 @@ struct RtThread { int st; // S_* state since st_since uint64_t st_since, st_ns[S_N], nblk, npre, pre_since, cpu_last; uint64_t wobj; // what the current wait is on: owner handle, key, address or first handle + RtThread *target; // as a core waiter: the owner core_kick asked to give its core up for it (holding G) + // TOTK_ASYNC_PREEMPT (holding G) + HANDLE hthr; // the host thread, for SuspendThread / Get/SetThreadContext; open while it may own a core + uint64_t ap_rip; // the interrupted host rip kern_ap_stub returns to + uint64_t ap_last; // the last redirect attempt on this owner + int ap_pending; // redirected to kern_ap_stub, which has not reached kern_ap_yield yet // TOTK_CORE_DIAG (holding G, except dg_host_end, which only the thread itself writes) uint64_t dg_since; // as a waiter: core wait start int dg_cls; // as a waiter: DG_* class at the start of the wait - RtThread *dg_target; // as a waiter: the owner asked to give its core up for it uint64_t dg_kick; // as an owner: the preempt request uint64_t dg_host_end; // as an owner: the return from the host call or SVC it was in at the request uint64_t dg_pc, dg_lr; // as an owner: guest pc and lr at the checkpoint that gave the core up @@ -256,17 +261,243 @@ static RtThread *g_core_owner[NCORE], *g_core_wait; static int g_cores_on = -1, g_held_max; static void diag_init(void); +static void ap_init(void); static int cores_enabled(void) { if (g_cores_on < 0) { g_cores_on = cfg_get("TOTK_FREE_CORES") ? 0 : 1; diag_init(); + ap_init(); } return g_cores_on; } static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) & 1); } +// ---- asynchronous preemption (TOTK_ASYNC_PREEMPT, on unless 0) +// Adapted from MacNativeGames games/totk/runtime/threads.c @ 33e8fa8 (8bf0893, PR #488; MIT, Copyright (c) 2026 +// MacNativeGames contributors): Horizon gives a core to a strictly better thread the moment that thread is ready. +// Here a better waiter only marks the owner, which gives the core up at its next SVC or indirect call; an owner in a +// stretch of game code with neither keeps it (TOTK_CORE_DIAG: 0.5-1.1 ms of thread-time per frame, waits to 16 ms and +// more, build/v121-preempt/NOTES.md). With it on, three parts, all of which MacNativeGames found necessary: +// - The waiter interrupts the owner (SuspendThread, GetThreadContext) when it has waited AP_FIRST_NS, then every +// millisecond: if the owner's host rip is in the lifted modules' code (g_gen_lo..g_gen_hi) and its host depth is 0 +// (not in a host call or SVC, nor in game code a host call made), it holds no host lock, so its rip is moved to +// kern_ap_stub, which saves every volatile register, gives the core up like a checkpoint (kern_ap_yield) and jumps +// back. Otherwise it is only marked, as before. +// - Priority inheritance (eff_prio): a thread owning a guest mutex counts at the priority of the best thread blocked on +// it, for granting, kicking, preempting and yielding. Without it MacNativeGames' game stopped at its first +// "Continue" (a better thread took the core of the low-priority owner of a mutex it then waited for). +// - Horizon's preemption disable: the SDK's spin locks (1.2.1 sdk f_70051242a8, f_7005124360 and siblings) count +// TLR+0x100 (disable_count) while they try or hold the lock, and after clearing it call SVC 0x36 when TLR+0x102 +// (interrupt_flag) is set. Such an owner is not preempted, at a checkpoint either: interrupt_flag is set and the +// switch happens at that SVC. Preempting it anyway hung MacNativeGames' game at "Continue" (the lock holder lost its +// core to a thread that then spun on that lock). +// TOTK_ASYNC_PREEMPT=0 keeps the old scheduling exactly: raw priorities, no interruption, no disable_count. +#define AP_FIRST_NS 250000ull // the waiter spins this long before the first interruption (most waits end sooner) +#define AP_GAP_NS 1000000ull // and interrupts the same owner at most this often +static int g_ap = -1; +static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start +static uint64_t g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok; +void kern_ap_stub(void); + +static void ap_init(void) { + if (g_ap >= 0) return; + const char *e = cfg_get("TOTK_ASYNC_PREEMPT"); + g_ap = !(e && !strcmp(e, "0")); + if (g_ap) { + RtModule m[16]; + int n = rt_linked_modules(m, 16); + uintptr_t lo = UINTPTR_MAX, hi = 0; + for (int i = 0; i < n; i++) + for (uint64_t k = 0; k < m[i].count; k++) { + uintptr_t f = (uintptr_t)m[i].fns[k]; + if (f < lo) lo = f; + if (f > hi) hi = f; + } + const uintptr_t host[] = {(uintptr_t)kern_ap_stub, (uintptr_t)kern_checkpoint, (uintptr_t)rt_svc, + (uintptr_t)rt_dispatch, (uintptr_t)rt_call}; + int bad = !n || lo >= hi; + for (unsigned i = 0; i < sizeof host / sizeof *host; i++) bad |= host[i] >= lo && host[i] <= hi; + if (bad) { + rt_log("kernel: asynchronous preemption off: the lifted code range %#llx-%#llx holds host code", + (unsigned long long)lo, (unsigned long long)hi); + g_ap = 0; + } else { + g_gen_lo = lo; + g_gen_hi = hi; + kern_host_track = 1; + } + } + rt_log("kernel: asynchronous preemption of virtual cores, priority inheritance, preemption disable: %s", + g_ap ? "on (TOTK_ASYNC_PREEMPT=0 turns them off)" : "off"); +} + +// Horizon's priority inheritance: o counts at the priority of the best thread blocked on a guest mutex it owns (the +// owner is read from the mutex word, so it follows hand-offs). Holding G. +static int eff_prio(const RtThread *o) { + int p = o->prio; + for (const RtThread *w = g_wait; w; w = w->next) + if (w->wkind == W_LOCK && w->prio < p && w->waddr && + (__atomic_load_n((const uint32_t *)(uintptr_t)w->waddr, __ATOMIC_RELAXED) & ~WAITERS_BIT) == o->handle) + p = w->prio; + return p; +} + +// The priority the virtual cores go by: raw with TOTK_ASYNC_PREEMPT=0, as before. +static int prio_of(const RtThread *t) { return g_ap > 0 ? eff_prio(t) : t->prio; } + +// TLR+0x100 disable_count, +0x102 interrupt_flag (any thread may read them; the flag is set from outside as the +// kernel does). +static volatile uint16_t *ap_tlr(const RtThread *t) { return (volatile uint16_t *)(t->tlr + 0x100); } +static int ap_disabled(const RtThread *t) { return g_ap > 0 && ap_tlr(t)[0]; } + +// kern_ap_stub: entered in place of the interrupted instruction of top-level game code. It keeps every register the +// game code may have live (volatile GPRs, flags, YMM0-15, MXCSR; the C calls preserve the rest), gives the core up +// in kern_ap_yield and jumps back to kern_ap_rip(), whose value it first stores in the return slot it reserved at +// entry, so a stack walk sees the interrupted function as its caller. No ret: the jump keeps call/return pairs +// balanced. The waiter never writes the owner's stack. +__asm__(".text\n" + ".globl kern_ap_stub\n" + ".def kern_ap_stub; .scl 2; .type 32; .endef\n" + ".p2align 4\n" + "kern_ap_stub:\n" + ".seh_proc kern_ap_stub\n" + " subq $8, %rsp\n" + ".seh_stackalloc 8\n" + " pushfq\n" + ".seh_stackalloc 8\n" + " pushq %rax\n" + ".seh_pushreg %rax\n" + " pushq %rcx\n" + ".seh_pushreg %rcx\n" + " pushq %rdx\n" + ".seh_pushreg %rdx\n" + " pushq %r8\n" + ".seh_pushreg %r8\n" + " pushq %r9\n" + ".seh_pushreg %r9\n" + " pushq %r10\n" + ".seh_pushreg %r10\n" + " pushq %r11\n" + ".seh_pushreg %r11\n" + " pushq %rbp\n" + ".seh_pushreg %rbp\n" + " movq %rsp, %rbp\n" + ".seh_setframe %rbp, 0\n" + ".seh_endprologue\n" + " andq $-32, %rsp\n" + " subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 + " vmovdqa %ymm0, 64(%rsp)\n" + " vmovdqa %ymm1, 96(%rsp)\n" + " vmovdqa %ymm2, 128(%rsp)\n" + " vmovdqa %ymm3, 160(%rsp)\n" + " vmovdqa %ymm4, 192(%rsp)\n" + " vmovdqa %ymm5, 224(%rsp)\n" + " vmovdqa %ymm6, 256(%rsp)\n" + " vmovdqa %ymm7, 288(%rsp)\n" + " vmovdqa %ymm8, 320(%rsp)\n" + " vmovdqa %ymm9, 352(%rsp)\n" + " vmovdqa %ymm10, 384(%rsp)\n" + " vmovdqa %ymm11, 416(%rsp)\n" + " vmovdqa %ymm12, 448(%rsp)\n" + " vmovdqa %ymm13, 480(%rsp)\n" + " vmovdqa %ymm14, 512(%rsp)\n" + " vmovdqa %ymm15, 544(%rsp)\n" + " stmxcsr 32(%rsp)\n" + " movl $0x1f80, 36(%rsp)\n" + " ldmxcsr 36(%rsp)\n" + " cld\n" + " vzeroupper\n" + " call kern_ap_rip\n" + " movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq + " call kern_ap_yield\n" + " ldmxcsr 32(%rsp)\n" + " vmovdqa 64(%rsp), %ymm0\n" + " vmovdqa 96(%rsp), %ymm1\n" + " vmovdqa 128(%rsp), %ymm2\n" + " vmovdqa 160(%rsp), %ymm3\n" + " vmovdqa 192(%rsp), %ymm4\n" + " vmovdqa 224(%rsp), %ymm5\n" + " vmovdqa 256(%rsp), %ymm6\n" + " vmovdqa 288(%rsp), %ymm7\n" + " vmovdqa 320(%rsp), %ymm8\n" + " vmovdqa 352(%rsp), %ymm9\n" + " vmovdqa 384(%rsp), %ymm10\n" + " vmovdqa 416(%rsp), %ymm11\n" + " vmovdqa 448(%rsp), %ymm12\n" + " vmovdqa 480(%rsp), %ymm13\n" + " vmovdqa 512(%rsp), %ymm14\n" + " vmovdqa 544(%rsp), %ymm15\n" + " movq %rbp, %rsp\n" + " popq %rbp\n" + " popq %r11\n" + " popq %r10\n" + " popq %r9\n" + " popq %r8\n" + " popq %rdx\n" + " popq %rcx\n" + " popq %rax\n" + " popfq\n" + " leaq 8(%rsp), %rsp\n" + " jmpq *-8(%rsp)\n" + ".seh_endproc\n"); + +enum { AP_NONE, AP_RETRY, AP_SENT }; + +// The waiter w interrupts its target owner if that runs top-level game code (holding G; the owner is suspended only +// between SuspendThread and ResumeThread, during which nothing here takes a lock). +static int ap_try(RtThread *w) { + RtThread *o = w->target; + if (!o || o->cur_core < 0 || !o->preempt) return AP_NONE; + if (!o->hthr || o->ap_pending) return AP_RETRY; + volatile uint16_t *tl = ap_tlr(o); + if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0) { + g_ap_host++; + return AP_RETRY; + } + if (tl[0]) { // preemption disabled: SVC 0x36 switches + tl[1] = 1; + g_ap_deferred++; + return AP_RETRY; + } + uint64_t now = kern_now_ns(); + if (o->ap_last && now - o->ap_last < AP_GAP_NS) return AP_RETRY; + o->ap_last = now; + if (SuspendThread(o->hthr) == (DWORD)-1) { + g_ap_fail++; + return AP_RETRY; + } + CONTEXT ctx; + memset(&ctx, 0, sizeof ctx); + ctx.ContextFlags = CONTEXT_CONTROL; + int r = AP_RETRY; + if (!GetThreadContext(o->hthr, &ctx)) + g_ap_fail++; + else if (ctx.Rip < g_gen_lo || ctx.Rip > g_gen_hi) + g_ap_pc++; + else if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0) + g_ap_host++; + else if (tl[0]) { + tl[1] = 1; + g_ap_deferred++; + } else { + o->ap_rip = ctx.Rip; + o->ap_pending = 1; + ctx.Rip = (DWORD64)(uintptr_t)kern_ap_stub; + ctx.ContextFlags = CONTEXT_CONTROL; + if (SetThreadContext(o->hthr, &ctx)) { + g_ap_sent++; + r = AP_SENT; + } else { + o->ap_pending = 0; + g_ap_fail++; + } + } + ResumeThread(o->hthr); + return r; +} + // ---- TOTK_CORE_DIAG=1: why ready threads wait for a virtual core, and for how long // Each core wait is split into classes: the owner asked to give the core up ran game code (DG_GAME, the part an // asynchronous preemption would remove), was in a host call or SVC that kept the core (DG_HOST), every owner the waiter @@ -330,7 +561,7 @@ static void dg_add(const RtThread *w, int cls, uint64_t ns, uint32_t oh, int opr // The waiter's class at the start of its wait, after core_kick claimed (or did not claim) an owner for it. static int dg_class0(const RtThread *t) { - if (t->dg_target) return dg_in_host(t->dg_target) ? DG_HOST : DG_GAME; + if (t->target) return dg_in_host(t->target) ? DG_HOST : DG_GAME; for (int i = 0; i < NCORE; i++) if (core_allowed(t, i) && g_core_owner[i] && g_core_owner[i]->prio > t->prio) return DG_NOCORE; return DG_EQ; @@ -343,8 +574,8 @@ static void dg_done(RtThread *t) { if (!t0) return; const struct DgRel *r = &t->dg_rel; uint64_t part[DG_N] = {0}; - uint32_t oh = t->dg_target ? t->dg_target->handle : r->h; - int oprio = t->dg_target ? t->dg_target->prio : r->prio; + uint32_t oh = t->target ? t->target->handle : r->h; + int oprio = t->target ? t->target->prio : r->prio; if (r->pre && r->kick) { uint64_t k = r->kick > t0 ? (r->kick < now ? r->kick : now) : t0; part[t->dg_cls] += k - t0; @@ -375,14 +606,14 @@ static void dg_done(RtThread *t) { r->prio, r->pre ? " (preempted)" : "", r->pc ? ldr_describe(r->pc, a, sizeof a) : "-", r->lr ? ldr_describe(r->lr, l, sizeof l) : "-"); } - t->dg_target = NULL; + t->target = NULL; memset(&t->dg_rel, 0, sizeof t->dg_rel); } // Once per long DG_GAME wait: where the owner's host thread runs (exe RVA). Holding G; the owner is resumed before // anything that could take a lock it holds. static void dg_sample(RtThread *t) { - RtThread *o = t->dg_target; + RtThread *o = t->target; if (!o || o->cur_core < 0 || !o->tid || g_dg_samples >= DG_SAMPLES) return; HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, o->tid); if (!h) return; @@ -406,6 +637,17 @@ static void dg_sample(RtThread *t) { // perf_present, every present: a line for a present over 25 ms, the 60-present summary. void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns) { + if (g_ap > 0 && frame && frame % 600 == 0) { // TOTK_ASYNC_PREEMPT counters since the start + lock(); + uint64_t v[8] = {g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok, g_sync36}; + unlock(); + rt_log("kernel: frame %llu: asynchronous preemption: %llu interrupted, %llu cores given up there; refused: " + "%llu in a host call, %llu outside game code, %llu preemption disabled, %llu failed; %llu waits ended " + "in the first spin; SVC 0x36 %llu", + (unsigned long long)frame, (unsigned long long)v[0], (unsigned long long)v[1], (unsigned long long)v[2], + (unsigned long long)v[3], (unsigned long long)v[4], (unsigned long long)v[5], (unsigned long long)v[6], + (unsigned long long)v[7]); + } if (g_diag <= 0) return; DgWindow w; uint64_t f[DG_N], fr[DG_N], s36, s36f; @@ -469,11 +711,16 @@ static struct DgRel g_dg_rel; // TOTK_CORE_DIAG: the owner core_put is releasi static void core_grant(int c) { RtThread *b = NULL, **bp = NULL; - for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext) - if (core_allowed(*p, c) && (!b || (*p)->prio < b->prio || ((*p)->prio == b->prio && (*p)->cseq < b->cseq))) { + int bpri = 0; + for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext) { + if (!core_allowed(*p, c)) continue; + int pr = prio_of(*p); + if (!b || pr < bpri || (pr == bpri && (*p)->cseq < b->cseq)) { b = *p; bp = p; + bpri = pr; } + } if (!b) return; *bp = b->cnext; b->cnext = NULL; @@ -509,8 +756,9 @@ static void core_put(RtThread *t) { // Whether a waiter better than t may run on core c. static int core_better_waiter(const RtThread *t, int c) { + int tp = prio_of(t); for (const RtThread *w = g_core_wait; w; w = w->cnext) - if (w->prio < t->prio && core_allowed(w, c)) return 1; + if (core_allowed(w, c) && prio_of(w) < tp) return 1; return 0; } @@ -520,25 +768,29 @@ static int core_better_waiter(const RtThread *t, int c) { // again (build/v121-dpre/NOTES.md). static void core_kick(void) { RtThread *w[64]; - int n = 0, claimed = 0; + int wp[64], op[NCORE], n = 0, claimed = 0; + for (int i = 0; i < NCORE; i++) op[i] = g_core_owner[i] ? prio_of(g_core_owner[i]) : 0; for (RtThread *p = g_core_wait; p && n < 64; p = p->cnext) { - int i = n++; - for (; i > 0 && (w[i - 1]->prio > p->prio || (w[i - 1]->prio == p->prio && w[i - 1]->cseq > p->cseq)); i--) + int i = n++, pp = prio_of(p); + for (; i > 0 && (wp[i - 1] > pp || (wp[i - 1] == pp && w[i - 1]->cseq > p->cseq)); i--) { w[i] = w[i - 1]; + wp[i] = wp[i - 1]; + } w[i] = p; + wp[i] = pp; } for (int k = 0; k < n && claimed != (1 << NCORE) - 1; k++) { int v = -1; for (int i = 0; i < NCORE; i++) { RtThread *o = g_core_owner[i]; - if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || o->prio <= w[k]->prio) continue; + if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || op[i] <= wp[k]) continue; RtThread *b = v < 0 ? NULL : g_core_owner[v]; - if (!b || o->prio > b->prio || (o->prio == b->prio && o->preempt && !b->preempt)) v = i; + if (!b || op[i] > op[v] || (op[i] == op[v] && o->preempt && !b->preempt)) v = i; } if (v < 0) continue; claimed |= 1 << v; RtThread *o = g_core_owner[v]; - w[k]->dg_target = o; + w[k]->target = o; if (o->preempt) continue; if (rt_trace & RT_TRACE_SCHED) { o->pre_since = kern_now_ns(); @@ -575,29 +827,34 @@ static void core_take(RtThread *t) { t->cseq = ++g_wseq; t->cnext = g_core_wait; g_core_wait = t; + t->target = NULL; if (g_diag > 0) { t->dg_since = kern_now_ns(); - t->dg_target = NULL; memset(&t->dg_rel, 0, sizeof t->dg_rel); } core_kick(); - if (g_diag > 0) { - t->dg_cls = dg_class0(t); - int sampled = 0; - while (t->cur_core < 0) { - if (sampled || t->dg_cls != DG_GAME) { - SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0); - continue; - } - SleepConditionVariableSRW(&t->cv, &G, 2, 0); - if (t->cur_core < 0 && kern_now_ns() - t->dg_since >= 2000000) { - sampled = 1; - dg_sample(t); - } - } - dg_done(t); + if (g_diag > 0) t->dg_cls = dg_class0(t); + if (g_ap > 0 && t->target && t->cur_core < 0) { // most waits end within microseconds: spin before interrupting + uint64_t end = kern_now_ns() + AP_FIRST_NS; + unlock(); + while (__atomic_load_n(&t->cur_core, __ATOMIC_RELAXED) < 0 && kern_now_ns() < end) YieldProcessor(); + lock(); + if (t->cur_core >= 0) g_ap_spin_ok++; } - while (t->cur_core < 0) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0); + int sampled = 0; + while (t->cur_core < 0) { + DWORD ms = INFINITE; + if (g_ap > 0 && ap_try(t) != AP_NONE) ms = 1; // the owner keeps its core: again in a millisecond + int sample = g_diag > 0 && !sampled && t->dg_cls == DG_GAME; + if (sample && ms == INFINITE) ms = 2; + SleepConditionVariableSRW(&t->cv, &G, ms, 0); + if (sample && t->cur_core < 0 && kern_now_ns() - t->dg_since >= 2000000) { + sampled = 1; + dg_sample(t); + } + } + if (g_diag > 0) dg_done(t); + t->target = NULL; acct(t, S_RUN); } @@ -642,7 +899,12 @@ void kern_checkpoint(void) { t->dg_pc = t->cur->pc; t->dg_lr = t->cur->x[30]; } - if (t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) { + int better = t->cur_core >= 0 && core_better_waiter(t, t->cur_core); + if (better && ap_disabled(t)) { // preemption disabled: the switch comes at SVC 0x36 + ap_tlr(t)[1] = 1; + t->preempt = 0; + g_ap_deferred++; + } else if (better) { core_put(t); // clears preempt (after the sched trace reads it) core_take(t); } else { @@ -655,6 +917,35 @@ void kern_checkpoint(void) { unlock(); } +// kern_ap_stub, on the interrupted owner: where to jump back to, then the core given up as at a checkpoint. +uint64_t kern_ap_rip(void) { return tcur->ap_rip; } + +void kern_ap_yield(void) { + RtThread *t = tcur; + lock(); + t->ap_pending = 0; + if (t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) { + if (g_diag > 0) { + t->dg_pc = t->cur->pc; + t->dg_lr = t->cur->x[30]; + } + g_ap_taken++; + core_put(t); + core_take(t); + } + unlock(); +} + +// SVC 0x36 SynchronizePreemptionState: a preemption deferred by disable_count happens here. +static void ap_sync(RtThread *t) { + lock(); + if (g_ap > 0 && t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) { + core_put(t); + core_take(t); + } + unlock(); +} + void kern_core_stats(int *held_now, int *held_max) { lock(); int n = 0; @@ -876,6 +1167,8 @@ static void thread_finish(RtThread *t) { lock(); core_put(t); acct(t, S_OFF); + if (t->hthr) CloseHandle(t->hthr); // no waiter interrupts a thread without a core + t->hthr = NULL; t->exited = 1; wake_sync_waiters(); if (t->closed) thr_free(t); @@ -887,11 +1180,22 @@ static void thread_finish(RtThread *t) { // is offset, and the landing code then reads its locals through the wrong base.) static __attribute__((noreturn)) void exit_now(RtThread *t) { longjmp(t->exit_jmp, 1); } +// The calling host thread, for ap_try (TOTK_ASYNC_PREEMPT only). +static HANDLE ap_open_self(void) { + if (!cores_enabled() || g_ap <= 0) return NULL; + HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_SET_CONTEXT | THREAD_QUERY_INFORMATION, + FALSE, GetCurrentThreadId()); + if (!h) rt_log("kernel: OpenThread for asynchronous preemption failed (error %lu)", GetLastError()); + return h; +} + static DWORD WINAPI thread_main(void *p) { RtThread *t = p; tcur = t; t->tid = GetCurrentThreadId(); + HANDLE self = ap_open_self(); lock(); + t->hthr = self; while (!t->started) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0); core_take(t); unlock(); @@ -934,7 +1238,9 @@ uint32_t kern_main_thread(Cpu **cpu) { t->stack_top = stack + MAIN_STACK; t->cpu->sp = t->stack_top; t->tid = GetCurrentThreadId(); + HANDLE self = ap_open_self(); lock(); + t->hthr = self; thr_link(t); g_main = t; g_main_stack = stack; @@ -1120,8 +1426,9 @@ int kern_svc(Cpu *c, uint32_t imm) { // Yield: hand the core to a waiting thread of equal or better priority. lock(); int give = 0; + int tp = prio_of(t); for (RtThread *w = g_core_wait; w && !give; w = w->cnext) - give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && w->prio <= t->prio; + give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && prio_of(w) <= tp; if (give) { core_put(t); core_take(t); @@ -1190,6 +1497,7 @@ int kern_svc(Cpu *c, uint32_t imm) { __atomic_fetch_add(&g_sync36, 1, __ATOMIC_RELAXED); if (*flag) __atomic_fetch_add(&g_sync36_flag, 1, __ATOMIC_RELAXED); *flag = 0; + ap_sync(t); x[0] = R_OK; return 1; } From 9fea5cc37b61a4abe10fe629463e3be59ef63f0c Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Sun, 25 Oct 2026 23:49:11 +0300 Subject: [PATCH 3/6] Credit MacNativeGames for the virtual-core preemption in THIRD_PARTY_NOTICES.md and the kernel.c header Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 1 + runtime/kernel.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 1275b3ff..7ec88bda 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -9,6 +9,7 @@ Code copied or adapted from another project is listed here with its source, the | LAS Recomp host runtime: NSO loader, host-function table, guest-space reservation and re-execution, guest thread start | https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f`, Copyright (c) 2026 Peiikai | MIT | `runtime/loader.c`, `runtime/hostfn.c`, `runtime/memory.c`; M3 tasks T2 and T6: `runtime/kernel.c`, `runtime/boot.c`; M4 T2: `runtime/vk_device.c`, `runtime/gfx_window.c`, M4 T5a: `runtime/tex_astc.c` (`runtime/nvn/astc.c`; its decode ported to GLSL in `runtime/shaders/astc_decode.comp`, compiled into `runtime/astc_decode_spv.h`, and its tables copied into `runtime/vk_astc.c`), `runtime/vk_formats.c` (format table of `runtime/nvn/vulkan/textures.c`), M4 T5b: `runtime/vk_memory.c` (page write generations of `runtime/nvn/vulkan/exec.c`, deferred destruction of `textures.c`), `runtime/vk_texture.c` (image creation, uploads and samplers of `runtime/nvn/vulkan/textures.c`), M4 T6a: `runtime/vk_sync.c` (timeline submissions and waits of `runtime/nvn/vulkan/exec.c`), M4 T6b: `runtime/vk_pipeline.c` (blend, compare and stencil mappings, vertex format table with its 3-component fallback and pipeline cache file of `runtime/nvn/vulkan/exec.c` and `device.c`), M4 T6c1: `runtime/vk_exec.c` (pass begin and end, scissor clip and attachment clears of `runtime/nvn/vulkan/exec.c`), M4 T6c3: `runtime/vk_exec_xfer.c` (copies of `runtime/nvn/vulkan/exec.c`), M4 T6c2: `runtime/vk_exec_draw.c` (uniform buffer spans, texture and sampler descriptors and dispatch flow of `runtime/nvn/vulkan/exec.c`) | | MacNativeGames TotK host runtime: dynamic linking, identity check, module permissions, guest memory and SVCs, threads, IPC, audio renderer, system services, RomFS and save storage, boot sequence | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/runtime/`, `games/totk/scripts/prepare_runtime.py`), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/loader.c`, `runtime/memory.c`; M3 tasks T2-T6: `runtime/kernel.c`, `runtime/svc.c`, `runtime/ipc.c`, `runtime/ipc.h`, `runtime/audren.c`, `runtime/audren.h`, `runtime/audio_mixer.c`, `runtime/audio_effects.c`, `runtime/audio_effects.h`, `runtime/audio_reverb.c`, `runtime/audio_reverb.h` (M5; SDL3 output replaces CoreAudio), `runtime/ov_system.c` (also `shared_font.c`: the BFTTF layout behind nn::pl), `runtime/ov_input.c` (also `hid_feedback.c`: vibration and six-axis handles and calls), `runtime/motion_fusion.c` (six-axis fusion of `motion_fusion.c`), `runtime/ov_media.c` (M5: hardware Opus decoder over libopus), `runtime/swkbd_win.c` (`swkbd_host.c`: ShowKeyboardArg offsets, UTF-8/UTF-16 conversion, disabled-key filter, result codes; Win32 window replaces the macOS text bar), `runtime/ov_gfx.c`, `runtime/romfs.c`, `runtime/romfs_path.h`, `runtime/fs_host.c`, `runtime/fs.h`, `runtime/boot.c`; M4 T2: `runtime/gfx_window.c` | | MacNativeGames TotK frame-rate patches: 60 fps present-interval controller, real-time frame step and its present sampling, hair and cloth clock, trampoline slow-motion check (instruction words and 1.2.1 sites), player position chain; quick-menu capture background RomFS overlay | https://github.com/ly05010419/MacNativeGames @ `ebe621de99bfc48f3c42f2d2aa73e04fbcc6eae2` (`games/totk/runtime/game_patches.c`, `game_version.h`, `world_survey.c`, `games/totk/scripts/make_romfs_patches.py`), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/frame_rate.c`, `recomp/patches-v121.txt`, `tools/make_romfs_patches.py` | +| MacNativeGames TotK virtual-core preemption: asynchronous preemption of an owner in top-level game code, priority inheritance through guest mutexes, the SDK's preemption-disable protocol (TLR+0x100/+0x102, SVC 0x36), the core-wait diagnostics | https://github.com/ly05010419/MacNativeGames @ `33e8fa83052ea1ae09a3583137008e9797e12950` (`games/totk/runtime/threads.c`, commits 8bf0893 and 19730cd), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/kernel.c` (SuspendThread and a register-saving stub replace SIGUSR2), `runtime/hostfn.c`, `runtime/svc.c` (host depth) | | MacNativeGames RomFS index builder (`TOTKRFS1` layout) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/scripts/prepare_romfs_backend.py`), Copyright (c) 2026 MacNativeGames contributors | MIT | `tools/romfs_index.py` | | d2r-mobile host thread environment (TLR, TLS, thread object), the origin of MacNativeGames' thread layer; d2r-mobile NVN host object layouts, command record format and NVN front end, as adapted for TotK in MacNativeGames' `d2r_gfx` copy | https://github.com/jhaizhou-ops/d2r-mobile @ `53e7c5bb9ca3ea53e2160005be8fbe920cd9cb53` (`dm/host/thread.c`, `dm/host/hle_thread.c`); the NVN layouts and front end via https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/third_party/d2r_gfx/gfx/nvn/nvn_obj.h`, the two types of `nvn_tex.h`, `hle_nvn.c`, `totk_nvn_ext.c`, `nvn_impl.h`, `nvn_hmap.h`, `nvn_pool_index.h`, the request checks and mapping semantics of `totk_vpool.c`, the generation stamps of `nvn_dirty.c`, texture mirroring and texel copies of `nvn_mtl.m`; licence in `games/totk/third_party/d2r_gfx/LICENSE`), Copyright (c) 2026 jhaizhou | MIT | M3 task T2: `runtime/kernel.c`; M4 task T1: `runtime/nvn_obj.h`; M4 task T3c: `runtime/nvn_api.h`, `runtime/nvn_api_core.c`, `runtime/nvn_hmap.h`, `runtime/nvn_pool_index.h`; M4 task T3d: `runtime/nvn_api_obj.c` (also `nvn_pace.h`); M4 task T3e: `runtime/nvn_api_cmd.c`, `runtime/nvn_ext.c`; M4 task T5b: `runtime/vk_memory.c`, `runtime/vk_texture.c`; M4 task T6a: `runtime/vk_sync.c` (side effects written after GPU completion, `hle_nvn.c`); M4 task T6b: `runtime/vk_pipeline.c` (pipeline key and state mapping of `nvn_mtl.m` `pso_key_fill` / `pso_desc`, its `vfmt` vertex formats and the missing-varying fix-up of `msl_varyings`); M4 task T6c1: `runtime/vk_exec.c`, `runtime/vk_exec.h` (op loop, pure-state ops and state defaults of `nvn_mtl.m` `mtl_exec` / `state_op` / `state_reset`); M4 task T6c3: `runtime/vk_exec_xfer.c` (copy, clear, event, counter and conditional-render semantics of `nvn_mtl.m` `copy_t2b` / `copy_b2t` / `copy_t2t` / `clear_tex` / `cond_eval`); M4 task T6c2: `runtime/vk_exec_draw.c` (bind ops, draw and dispatch flow, vertex streams of `nvn_mtl.m` `state_op` / `bind_vertex_streams` / `draw_inner` / `dispatch`) | | nxnative NVN front end (d2r-mobile's successor): texture view depth-stencil mode bits, stencil-view rule, `Downsample` and indirect draw and dispatch records | https://github.com/seedds/nxnative @ `d5b380b3a80495609c2e751bb994911a638935ad` (`dm/gfx/nvn/hle_nvn.c`, `nvn_obj.h`, `nvn_mtl.m`), Copyright (c) 2026 jhaizhou | MIT | M5: `runtime/nvn_obj.h`, `runtime/nvn_ext.c`, `runtime/nvn_api_cmd.c`, `runtime/vk_exec_draw.c` | diff --git a/runtime/kernel.c b/runtime/kernel.c index e2a9fc6d..315ea52d 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -1,6 +1,7 @@ // Adapted from MacNativeGames games/totk/runtime/threads.c @ b75be8c (MIT, Copyright (c) 2026 MacNativeGames // contributors; itself from d2r-mobile dm/host/hle_thread.c), d2r-mobile dm/host/thread.c @ 53e7c5b (MIT, Copyright -// (c) 2026 jhaizhou) and LAS Recomp runtime/kernel.c @ da35b83 (MIT, Copyright (c) 2026 Peiikai). +// (c) 2026 jhaizhou) and LAS Recomp runtime/kernel.c @ da35b83 (MIT, Copyright (c) 2026 Peiikai); the preemption of virtual cores from +// MacNativeGames threads.c @ 33e8fa8 (MIT, as above). // Threads, synchronization, virtual cores and fibers of the Horizon kernel. // // One guest thread = one Windows thread with a 256 MiB reserved host stack (every guest BL is a host call) and its own From 8f9716527a48f52f07edc117ef46d5c18496d7da Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 26 Oct 2026 00:03:10 +0300 Subject: [PATCH 4/6] Keep the interrupted code's flags in kern_ap_stub, take the whole integer context, log the first interrupts and add test modes 2 to 5 of TOTK_ASYNC_PREEMPT Co-Authored-By: Claude Opus 5.5 --- runtime/kernel.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/runtime/kernel.c b/runtime/kernel.c index 315ea52d..3216051e 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -1,7 +1,7 @@ // Adapted from MacNativeGames games/totk/runtime/threads.c @ b75be8c (MIT, Copyright (c) 2026 MacNativeGames // contributors; itself from d2r-mobile dm/host/hle_thread.c), d2r-mobile dm/host/thread.c @ 53e7c5b (MIT, Copyright -// (c) 2026 jhaizhou) and LAS Recomp runtime/kernel.c @ da35b83 (MIT, Copyright (c) 2026 Peiikai); the preemption of virtual cores from -// MacNativeGames threads.c @ 33e8fa8 (MIT, as above). +// (c) 2026 jhaizhou) and LAS Recomp runtime/kernel.c @ da35b83 (MIT, Copyright (c) 2026 Peiikai); the preemption of +// virtual cores from MacNativeGames threads.c @ 33e8fa8 (MIT, as above). // Threads, synchronization, virtual cores and fibers of the Horizon kernel. // // One guest thread = one Windows thread with a 256 MiB reserved host stack (every guest BL is a host call) and its own @@ -297,7 +297,9 @@ static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) // TOTK_ASYNC_PREEMPT=0 keeps the old scheduling exactly: raw priorities, no interruption, no disable_count. #define AP_FIRST_NS 250000ull // the waiter spins this long before the first interruption (most waits end sooner) #define AP_GAP_NS 1000000ull // and interrupts the same owner at most this often -static int g_ap = -1; +static int g_ap = -1, g_ap_int, g_ap_noyield, g_ap_test; // g_ap_test: TOTK_ASYNC_PREEMPT=4/5 (tests) // g_ap_noyield: TOTK_ASYNC_PREEMPT=3, the stub only (test) // g_ap_int: interrupt owners (TOTK_ASYNC_PREEMPT=2: inheritance and disable only) +static int g_ap_log; // redirects logged so far (AP_LOG_N) +#define AP_LOG_N 40 static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start static uint64_t g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok; void kern_ap_stub(void); @@ -306,6 +308,9 @@ static void ap_init(void) { if (g_ap >= 0) return; const char *e = cfg_get("TOTK_ASYNC_PREEMPT"); g_ap = !(e && !strcmp(e, "0")); + g_ap_int = g_ap && !(e && !strcmp(e, "2")); + g_ap_noyield = e && !strcmp(e, "3"); + g_ap_test = e && (*e == '4' || *e == '5') ? *e - '0' : 0; // 4: set the context back unchanged; 5: only suspend if (g_ap) { RtModule m[16]; int n = rt_linked_modules(m, 16); @@ -330,8 +335,10 @@ static void ap_init(void) { kern_host_track = 1; } } - rt_log("kernel: asynchronous preemption of virtual cores, priority inheritance, preemption disable: %s", - g_ap ? "on (TOTK_ASYNC_PREEMPT=0 turns them off)" : "off"); + rt_log("kernel: asynchronous preemption of virtual cores %s, priority inheritance and preemption disable %s; lifted " + "code exe+%#llx-exe+%#llx", g_ap_int ? "on" : "off", g_ap ? "on (TOTK_ASYNC_PREEMPT=0 turns all off)" : "off", + (unsigned long long)(g_gen_lo ? g_gen_lo - (uintptr_t)GetModuleHandleW(NULL) : 0), + (unsigned long long)(g_gen_hi ? g_gen_hi - (uintptr_t)GetModuleHandleW(NULL) : 0)); } // Horizon's priority inheritance: o counts at the priority of the best thread blocked on a guest mutex it owns (the @@ -364,7 +371,7 @@ __asm__(".text\n" ".p2align 4\n" "kern_ap_stub:\n" ".seh_proc kern_ap_stub\n" - " subq $8, %rsp\n" + " leaq -8(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq ".seh_stackalloc 8\n" " pushfq\n" ".seh_stackalloc 8\n" @@ -471,7 +478,7 @@ static int ap_try(RtThread *w) { } CONTEXT ctx; memset(&ctx, 0, sizeof ctx); - ctx.ContextFlags = CONTEXT_CONTROL; + ctx.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; // set back whole, so the kernel restores every GPR int r = AP_RETRY; if (!GetThreadContext(o->hthr, &ctx)) g_ap_fail++; @@ -482,14 +489,23 @@ static int ap_try(RtThread *w) { else if (tl[0]) { tl[1] = 1; g_ap_deferred++; + } else if (g_ap_test == 4) { + if (SetThreadContext(o->hthr, &ctx)) g_ap_sent++; + } else if (g_ap_test == 5) { + g_ap_sent++; } else { o->ap_rip = ctx.Rip; o->ap_pending = 1; ctx.Rip = (DWORD64)(uintptr_t)kern_ap_stub; - ctx.ContextFlags = CONTEXT_CONTROL; if (SetThreadContext(o->hthr, &ctx)) { g_ap_sent++; r = AP_SENT; + if (g_ap_log < AP_LOG_N) { + g_ap_log++; + rt_log("kernel: interrupt %#x p%d at exe+%#llx rsp %#llx for %#x p%d", o->handle, o->prio, + (unsigned long long)(o->ap_rip - (uintptr_t)GetModuleHandleW(NULL)), (unsigned long long)ctx.Rsp, + w->handle, w->prio); + } } else { o->ap_pending = 0; g_ap_fail++; @@ -835,7 +851,7 @@ static void core_take(RtThread *t) { } core_kick(); if (g_diag > 0) t->dg_cls = dg_class0(t); - if (g_ap > 0 && t->target && t->cur_core < 0) { // most waits end within microseconds: spin before interrupting + if (g_ap_int && t->target && t->cur_core < 0) { // most waits end within microseconds: spin before interrupting uint64_t end = kern_now_ns() + AP_FIRST_NS; unlock(); while (__atomic_load_n(&t->cur_core, __ATOMIC_RELAXED) < 0 && kern_now_ns() < end) YieldProcessor(); @@ -845,7 +861,7 @@ static void core_take(RtThread *t) { int sampled = 0; while (t->cur_core < 0) { DWORD ms = INFINITE; - if (g_ap > 0 && ap_try(t) != AP_NONE) ms = 1; // the owner keeps its core: again in a millisecond + if (g_ap_int && ap_try(t) != AP_NONE) ms = 1; // the owner keeps its core: again in a millisecond int sample = g_diag > 0 && !sampled && t->dg_cls == DG_GAME; if (sample && ms == INFINITE) ms = 2; SleepConditionVariableSRW(&t->cv, &G, ms, 0); @@ -925,7 +941,7 @@ void kern_ap_yield(void) { RtThread *t = tcur; lock(); t->ap_pending = 0; - if (t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) { + if (!g_ap_noyield && t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) { if (g_diag > 0) { t->dg_pc = t->cur->pc; t->dg_lr = t->cur->x[30]; @@ -1183,7 +1199,7 @@ static __attribute__((noreturn)) void exit_now(RtThread *t) { longjmp(t->exit_jm // The calling host thread, for ap_try (TOTK_ASYNC_PREEMPT only). static HANDLE ap_open_self(void) { - if (!cores_enabled() || g_ap <= 0) return NULL; + if (!cores_enabled() || !g_ap_int) return NULL; HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_SET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, GetCurrentThreadId()); if (!h) rt_log("kernel: OpenThread for asynchronous preemption failed (error %lu)", GetLastError()); From 8553b00c9b82486aba3d28d78f5659691c3ebf98 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 26 Oct 2026 07:44:47 +0300 Subject: [PATCH 5/6] Drop the bisect modes 3 to 5 of TOTK_ASYNC_PREEMPT and log interrupts after the owner is resumed Co-Authored-By: Claude Opus 5.5 --- runtime/kernel.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/runtime/kernel.c b/runtime/kernel.c index 3216051e..aab2a986 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -294,10 +294,11 @@ static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) // (interrupt_flag) is set. Such an owner is not preempted, at a checkpoint either: interrupt_flag is set and the // switch happens at that SVC. Preempting it anyway hung MacNativeGames' game at "Continue" (the lock holder lost its // core to a thread that then spun on that lock). -// TOTK_ASYNC_PREEMPT=0 keeps the old scheduling exactly: raw priorities, no interruption, no disable_count. +// TOTK_ASYNC_PREEMPT=0 keeps the old scheduling exactly: raw priorities, no interruption, no disable_count; =2 keeps +// inheritance and the preemption disable without interrupting owners. #define AP_FIRST_NS 250000ull // the waiter spins this long before the first interruption (most waits end sooner) #define AP_GAP_NS 1000000ull // and interrupts the same owner at most this often -static int g_ap = -1, g_ap_int, g_ap_noyield, g_ap_test; // g_ap_test: TOTK_ASYNC_PREEMPT=4/5 (tests) // g_ap_noyield: TOTK_ASYNC_PREEMPT=3, the stub only (test) // g_ap_int: interrupt owners (TOTK_ASYNC_PREEMPT=2: inheritance and disable only) +static int g_ap = -1, g_ap_int; // g_ap_int: interrupt owners (TOTK_ASYNC_PREEMPT=2: inheritance and disable only) static int g_ap_log; // redirects logged so far (AP_LOG_N) #define AP_LOG_N 40 static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start @@ -309,8 +310,6 @@ static void ap_init(void) { const char *e = cfg_get("TOTK_ASYNC_PREEMPT"); g_ap = !(e && !strcmp(e, "0")); g_ap_int = g_ap && !(e && !strcmp(e, "2")); - g_ap_noyield = e && !strcmp(e, "3"); - g_ap_test = e && (*e == '4' || *e == '5') ? *e - '0' : 0; // 4: set the context back unchanged; 5: only suspend if (g_ap) { RtModule m[16]; int n = rt_linked_modules(m, 16); @@ -489,10 +488,6 @@ static int ap_try(RtThread *w) { else if (tl[0]) { tl[1] = 1; g_ap_deferred++; - } else if (g_ap_test == 4) { - if (SetThreadContext(o->hthr, &ctx)) g_ap_sent++; - } else if (g_ap_test == 5) { - g_ap_sent++; } else { o->ap_rip = ctx.Rip; o->ap_pending = 1; @@ -500,18 +495,18 @@ static int ap_try(RtThread *w) { if (SetThreadContext(o->hthr, &ctx)) { g_ap_sent++; r = AP_SENT; - if (g_ap_log < AP_LOG_N) { - g_ap_log++; - rt_log("kernel: interrupt %#x p%d at exe+%#llx rsp %#llx for %#x p%d", o->handle, o->prio, - (unsigned long long)(o->ap_rip - (uintptr_t)GetModuleHandleW(NULL)), (unsigned long long)ctx.Rsp, - w->handle, w->prio); - } } else { o->ap_pending = 0; g_ap_fail++; } } ResumeThread(o->hthr); + if (r == AP_SENT && g_ap_log < AP_LOG_N) { + g_ap_log++; + rt_log("kernel: interrupt %#x p%d at exe+%#llx rsp %#llx for %#x p%d", o->handle, o->prio, + (unsigned long long)(o->ap_rip - (uintptr_t)GetModuleHandleW(NULL)), (unsigned long long)ctx.Rsp, + w->handle, w->prio); + } return r; } @@ -941,7 +936,7 @@ void kern_ap_yield(void) { RtThread *t = tcur; lock(); t->ap_pending = 0; - if (!g_ap_noyield && t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) { + if (t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) { if (g_diag > 0) { t->dg_pc = t->cur->pc; t->dg_lr = t->cur->x[30]; From 5eb60404a234e3f34fd25a660ec6b972c2180139 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 26 Oct 2026 08:01:21 +0300 Subject: [PATCH 6/6] Describe the asynchronous preemption of virtual cores in docs/architecture.md Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/architecture.md b/docs/architecture.md index bbae8326..24c4a6f7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -55,6 +55,13 @@ Every guest thread is a host thread. TotK assumes Horizon's scheduler: three app - Three virtual cores. A guest thread runs guest code only while it holds a core; the others wait in a priority-ordered list. - Preemption. When a waiting thread has a better priority than a holder on a core it may use, exactly one holder is flagged. The holder checks the flag at its next SVC and at every indirect call (`rt_dispatch`: two loads when nobody waits), checks again under the kernel lock that a better thread still waits, and only then hands its core over. The indirect-call check comes from SavageWindRecomp's BotW port; checking only at SVCs left a better thread waiting up to 26 ms. +- Asynchronous preemption (from MacNativeGames 8bf0893, on unless `TOTK_ASYNC_PREEMPT=0`). Game code can run for + milliseconds with neither (`TOTK_CORE_DIAG=1` measures it): a waiter that has spun 250 us suspends the holder and, + if its host rip is in lifted code at host depth 0 (not inside a host call or SVC), moves it to `kern_ap_stub`, which + saves the volatile registers and gives the core up there. A holder of a guest mutex counts at the priority of its + best waiter, and one inside an SDK spin lock (TLR+0x100 `disable_count`) is not preempted: its interrupt flag is set + and the switch happens at SVC 0x36. `TOTK_ASYNC_PREEMPT=2` keeps those two without interrupting. The interruption is + Win32 (SuspendThread, SetThreadContext); another host needs a signal handler there, as MacNativeGames has. - Host work gives the core back. Queue submits, presents, GPU waits and shader translation run without a core (`NVN_BLOCKING`, `kern_core_away`), so a thread blocked in the driver does not hold up guest threads. - `TOTK_FREE_CORES=1` turns the gate off, for experiments.