Merge asynchronous preemption of virtual cores with priority inheritance and the SDK's preemption disable, after MacNativeGames PR #488, behind TOTK_ASYNC_PREEMPT (on by default)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-26 08:22:25 +03:00
commit 0ecfe91063
8 files changed
+650 -17

No files matched your search

+1
View File
@@ -9,6 +9,7 @@ Code copied or adapted from another project is listed here with its source, the
| LAS Recomp host runtime: NSO loader, host-function table, guest-space reservation and re-execution, guest thread start | https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f`, Copyright (c) 2026 Peiikai | MIT | `runtime/loader.c`, `runtime/hostfn.c`, `runtime/memory.c`; M3 tasks T2 and T6: `runtime/kernel.c`, `runtime/boot.c`; M4 T2: `runtime/vk_device.c`, `runtime/gfx_window.c`, M4 T5a: `runtime/tex_astc.c` (`runtime/nvn/astc.c`; its decode ported to GLSL in `runtime/shaders/astc_decode.comp`, compiled into `runtime/astc_decode_spv.h`, and its tables copied into `runtime/vk_astc.c`), `runtime/vk_formats.c` (format table of `runtime/nvn/vulkan/textures.c`), M4 T5b: `runtime/vk_memory.c` (page write generations of `runtime/nvn/vulkan/exec.c`, deferred destruction of `textures.c`), `runtime/vk_texture.c` (image creation, uploads and samplers of `runtime/nvn/vulkan/textures.c`), M4 T6a: `runtime/vk_sync.c` (timeline submissions and waits of `runtime/nvn/vulkan/exec.c`), M4 T6b: `runtime/vk_pipeline.c` (blend, compare and stencil mappings, vertex format table with its 3-component fallback and pipeline cache file of `runtime/nvn/vulkan/exec.c` and `device.c`), M4 T6c1: `runtime/vk_exec.c` (pass begin and end, scissor clip and attachment clears of `runtime/nvn/vulkan/exec.c`), M4 T6c3: `runtime/vk_exec_xfer.c` (copies of `runtime/nvn/vulkan/exec.c`), M4 T6c2: `runtime/vk_exec_draw.c` (uniform buffer spans, texture and sampler descriptors and dispatch flow of `runtime/nvn/vulkan/exec.c`) |
| MacNativeGames TotK host runtime: dynamic linking, identity check, module permissions, guest memory and SVCs, threads, IPC, audio renderer, system services, RomFS and save storage, boot sequence | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/runtime/`, `games/totk/scripts/prepare_runtime.py`), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/loader.c`, `runtime/memory.c`; M3 tasks T2-T6: `runtime/kernel.c`, `runtime/svc.c`, `runtime/ipc.c`, `runtime/ipc.h`, `runtime/audren.c`, `runtime/audren.h`, `runtime/audio_mixer.c`, `runtime/audio_effects.c`, `runtime/audio_effects.h`, `runtime/audio_reverb.c`, `runtime/audio_reverb.h` (M5; SDL3 output replaces CoreAudio), `runtime/ov_system.c` (also `shared_font.c`: the BFTTF layout behind nn::pl), `runtime/ov_input.c` (also `hid_feedback.c`: vibration and six-axis handles and calls), `runtime/motion_fusion.c` (six-axis fusion of `motion_fusion.c`), `runtime/ov_media.c` (M5: hardware Opus decoder over libopus), `runtime/swkbd_win.c` (`swkbd_host.c`: ShowKeyboardArg offsets, UTF-8/UTF-16 conversion, disabled-key filter, result codes; Win32 window replaces the macOS text bar), `runtime/ov_gfx.c`, `runtime/romfs.c`, `runtime/romfs_path.h`, `runtime/fs_host.c`, `runtime/fs.h`, `runtime/boot.c`; M4 T2: `runtime/gfx_window.c` |
| MacNativeGames TotK frame-rate patches: 60 fps present-interval controller, real-time frame step and its present sampling, hair and cloth clock, trampoline slow-motion check (instruction words and 1.2.1 sites), player position chain; quick-menu capture background RomFS overlay | https://github.com/ly05010419/MacNativeGames @ `ebe621de99bfc48f3c42f2d2aa73e04fbcc6eae2` (`games/totk/runtime/game_patches.c`, `game_version.h`, `world_survey.c`, `games/totk/scripts/make_romfs_patches.py`), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/frame_rate.c`, `recomp/patches-v121.txt`, `tools/make_romfs_patches.py` |
| MacNativeGames TotK virtual-core preemption: asynchronous preemption of an owner in top-level game code, priority inheritance through guest mutexes, the SDK's preemption-disable protocol (TLR+0x100/+0x102, SVC 0x36), the core-wait diagnostics | https://github.com/ly05010419/MacNativeGames @ `33e8fa83052ea1ae09a3583137008e9797e12950` (`games/totk/runtime/threads.c`, commits 8bf0893 and 19730cd), Copyright (c) 2026 MacNativeGames contributors | MIT | `runtime/kernel.c` (SuspendThread and a register-saving stub replace SIGUSR2), `runtime/hostfn.c`, `runtime/svc.c` (host depth) |
| MacNativeGames RomFS index builder (`TOTKRFS1` layout) | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/scripts/prepare_romfs_backend.py`), Copyright (c) 2026 MacNativeGames contributors | MIT | `tools/romfs_index.py` |
| d2r-mobile host thread environment (TLR, TLS, thread object), the origin of MacNativeGames' thread layer; d2r-mobile NVN host object layouts, command record format and NVN front end, as adapted for TotK in MacNativeGames' `d2r_gfx` copy | https://github.com/jhaizhou-ops/d2r-mobile @ `53e7c5bb9ca3ea53e2160005be8fbe920cd9cb53` (`dm/host/thread.c`, `dm/host/hle_thread.c`); the NVN layouts and front end via https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/third_party/d2r_gfx/gfx/nvn/nvn_obj.h`, the two types of `nvn_tex.h`, `hle_nvn.c`, `totk_nvn_ext.c`, `nvn_impl.h`, `nvn_hmap.h`, `nvn_pool_index.h`, the request checks and mapping semantics of `totk_vpool.c`, the generation stamps of `nvn_dirty.c`, texture mirroring and texel copies of `nvn_mtl.m`; licence in `games/totk/third_party/d2r_gfx/LICENSE`), Copyright (c) 2026 jhaizhou | MIT | M3 task T2: `runtime/kernel.c`; M4 task T1: `runtime/nvn_obj.h`; M4 task T3c: `runtime/nvn_api.h`, `runtime/nvn_api_core.c`, `runtime/nvn_hmap.h`, `runtime/nvn_pool_index.h`; M4 task T3d: `runtime/nvn_api_obj.c` (also `nvn_pace.h`); M4 task T3e: `runtime/nvn_api_cmd.c`, `runtime/nvn_ext.c`; M4 task T5b: `runtime/vk_memory.c`, `runtime/vk_texture.c`; M4 task T6a: `runtime/vk_sync.c` (side effects written after GPU completion, `hle_nvn.c`); M4 task T6b: `runtime/vk_pipeline.c` (pipeline key and state mapping of `nvn_mtl.m` `pso_key_fill` / `pso_desc`, its `vfmt` vertex formats and the missing-varying fix-up of `msl_varyings`); M4 task T6c1: `runtime/vk_exec.c`, `runtime/vk_exec.h` (op loop, pure-state ops and state defaults of `nvn_mtl.m` `mtl_exec` / `state_op` / `state_reset`); M4 task T6c3: `runtime/vk_exec_xfer.c` (copy, clear, event, counter and conditional-render semantics of `nvn_mtl.m` `copy_t2b` / `copy_b2t` / `copy_t2t` / `clear_tex` / `cond_eval`); M4 task T6c2: `runtime/vk_exec_draw.c` (bind ops, draw and dispatch flow, vertex streams of `nvn_mtl.m` `state_op` / `bind_vertex_streams` / `draw_inner` / `dispatch`) |
| nxnative NVN front end (d2r-mobile's successor): texture view depth-stencil mode bits, stencil-view rule, `Downsample` and indirect draw and dispatch records | https://github.com/seedds/nxnative @ `d5b380b3a80495609c2e751bb994911a638935ad` (`dm/gfx/nvn/hle_nvn.c`, `nvn_obj.h`, `nvn_mtl.m`), Copyright (c) 2026 jhaizhou | MIT | M5: `runtime/nvn_obj.h`, `runtime/nvn_ext.c`, `runtime/nvn_api_cmd.c`, `runtime/vk_exec_draw.c` |
+7
View File
@@ -55,6 +55,13 @@ Every guest thread is a host thread. TotK assumes Horizon's scheduler: three app
- Three virtual cores. A guest thread runs guest code only while it holds a core; the others wait in a priority-ordered list.
- Preemption. When a waiting thread has a better priority than a holder on a core it may use, exactly one holder is flagged. The holder checks the flag at its next SVC and at every indirect call (`rt_dispatch`: two loads when nobody waits), checks again under the kernel lock that a better thread still waits, and only then hands its core over. The indirect-call check comes from SavageWindRecomp's BotW port; checking only at SVCs left a better thread waiting up to 26 ms.
- Asynchronous preemption (from MacNativeGames 8bf0893, on unless `TOTK_ASYNC_PREEMPT=0`). Game code can run for
milliseconds with neither (`TOTK_CORE_DIAG=1` measures it): a waiter that has spun 250 us suspends the holder and,
if its host rip is in lifted code at host depth 0 (not inside a host call or SVC), moves it to `kern_ap_stub`, which
saves the volatile registers and gives the core up there. A holder of a guest mutex counts at the priority of its
best waiter, and one inside an SDK spin lock (TLR+0x100 `disable_count`) is not preempted: its interrupt flag is set
and the switch happens at SVC 0x36. `TOTK_ASYNC_PREEMPT=2` keeps those two without interrupting. The interruption is
Win32 (SuspendThread, SetThreadContext); another host needs a signal handler there, as MacNativeGames has.
- Host work gives the core back. Queue submits, presents, GPU waits and shader translation run without a core (`NVN_BLOCKING`, `kern_core_away`), so a thread blocked in the driver does not hold up guest threads.
- `TOTK_FREE_CORES=1` turns the gate off, for experiments.
+2 -2
View File
@@ -23,10 +23,10 @@ static int n_entries;
// Every name the runtime reads (and the patch sets in generated/), without the prefix. A file key outside this list is
// logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line.
static const char *const known[] = {
"APP_POOL", "ASSUME_FOCUS", "ASTC", "ASTC_TIME", "AUDIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT",
"APP_POOL", "ASSUME_FOCUS", "ASTC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT",
"BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL",
"BIND_ZR",
"CLOTH_REALTIME", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT",
"CLOTH_REALTIME", "CORE_DIAG", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT",
"FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FPS60", "FREE_CORES", "FREEZE_IDLE", "GPL",
"GPL_DRAW_BUILD",
"GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
+8
View File
@@ -176,6 +176,14 @@ void kern_core_back(int *away);
void kern_checkpoint(void); // preemption point, at every SVC and indirect call
// A preempt request is pending for the thread (Cpu.thread): its RtThread starts with the flag (kernel.c).
static inline int kern_preempt_pending(const void *thread) { return __atomic_load_n((const int *)thread, __ATOMIC_RELAXED); }
// Set once before guest threads start (TOTK_CORE_DIAG, TOTK_ASYNC_PREEMPT): host calls (hostfn.c) and SVCs (svc.c) count their depth in
// the thread's second int, so the kernel knows whether a core owner runs game code at the top level.
extern int kern_host_track;
static inline int *kern_host_depth(void *thread) { return (int *)thread + 1; }
void kern_host_left(void *thread); // back at depth 0 with a preempt request pending
int kern_core_diag_on(void); // TOTK_CORE_DIAG
void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns); // perf_present, every present (also logs the
// asynchronous preemption counters)
void kern_sched_report(uint64_t frame); // TOTK_TRACE=sched: per-thread time split, on present
uint32_t kern_handle_new(int kind, void *obj); // kind: KERN_SESSION or KERN_TMEM (ipc.c)
void *kern_handle_get(uint32_t h, int kind);
+16
View File
@@ -108,6 +108,18 @@ __attribute__((noinline, noreturn, cold)) static void bad_entry(Cpu *c) {
ldr_describe(c->x[30], lr, sizeof lr));
}
// kern_host_track: the call counted in the thread's host depth. Not a tail call, so the host stack holds this frame
// while fn runs; longjmp out of fn (ExitThread) leaves the depth raised on a thread that ends.
__attribute__((noinline)) static void hostfn_tracked(Cpu *c, GuestFn fn) {
void *t = c->thread;
int *d = kern_host_depth(t);
__atomic_store_n(d, *d + 1, __ATOMIC_RELAXED);
fn(c);
int n = *d - 1;
__atomic_store_n(d, n, __ATOMIC_RELAXED);
if (!n && kern_preempt_pending(t)) kern_host_left(t);
}
static void hostfn_entry(Cpu *c) {
uint64_t i = (c->pc - RT_HOSTFN_BASE) >> 4;
if (__builtin_expect((c->pc & 15) || i >= (uint64_t)nslots, 0)) bad_entry(c);
@@ -117,6 +129,10 @@ static void hostfn_entry(Cpu *c) {
char lr[96];
rt_log("host call %s from %s", s->name, ldr_describe(c->x[30] - 4, lr, sizeof lr));
}
if (__builtin_expect(kern_host_track && c->thread, 0)) {
hostfn_tracked(c, s->fn);
return;
}
__attribute__((musttail)) return s->fn(c);
}
+599 -15
View File
@@ -1,6 +1,7 @@
// Adapted from MacNativeGames games/totk/runtime/threads.c @ b75be8c (MIT, Copyright (c) 2026 MacNativeGames
// contributors; itself from d2r-mobile dm/host/hle_thread.c), d2r-mobile dm/host/thread.c @ 53e7c5b (MIT, Copyright
// (c) 2026 jhaizhou) and LAS Recomp runtime/kernel.c @ da35b83 (MIT, Copyright (c) 2026 Peiikai).
// (c) 2026 jhaizhou) and LAS Recomp runtime/kernel.c @ da35b83 (MIT, Copyright (c) 2026 Peiikai); the preemption of
// virtual cores from MacNativeGames threads.c @ 33e8fa8 (MIT, as above).
// Threads, synchronization, virtual cores and fibers of the Horizon kernel.
//
// One guest thread = one Windows thread with a 256 MiB reserved host stack (every guest BL is a host call) and its own
@@ -53,6 +54,7 @@ typedef struct Fiber Fiber;
struct RtThread {
int preempt; // first member: rt_dispatch reads it (host.h kern_preempt_pending)
int host_depth; // second member: host calls and SVCs in progress, while kern_host_track (host.h)
uint32_t handle;
uint64_t id;
Cpu *cpu; // the thread's own context
@@ -86,12 +88,28 @@ struct RtThread {
int st; // S_* state since st_since
uint64_t st_since, st_ns[S_N], nblk, npre, pre_since, cpu_last;
uint64_t wobj; // what the current wait is on: owner handle, key, address or first handle
RtThread *target; // as a core waiter: the owner core_kick asked to give its core up for it (holding G)
// TOTK_ASYNC_PREEMPT (holding G)
HANDLE hthr; // the host thread, for SuspendThread / Get/SetThreadContext; open while it may own a core
uint64_t ap_rip; // the interrupted host rip kern_ap_stub returns to
uint64_t ap_last; // the last redirect attempt on this owner
int ap_pending; // redirected to kern_ap_stub, which has not reached kern_ap_yield yet
// TOTK_CORE_DIAG (holding G, except dg_host_end, which only the thread itself writes)
uint64_t dg_since; // as a waiter: core wait start
int dg_cls; // as a waiter: DG_* class at the start of the wait
uint64_t dg_kick; // as an owner: the preempt request
uint64_t dg_host_end; // as an owner: the return from the host call or SVC it was in at the request
uint64_t dg_pc, dg_lr; // as an owner: guest pc and lr at the checkpoint that gave the core up
int dg_host_kick; // as an owner: in a host call or SVC at the request
struct DgRel { uint64_t kick, host_end, pc, lr; int host, pre, prio; uint32_t h; } dg_rel; // as a waiter: the
// owner that released
};
static SRWLOCK G = SRWLOCK_INIT;
static struct { int type; void *obj; } g_h[MAX_H];
static _Thread_local RtThread *tcur;
_Static_assert(offsetof(RtThread, preempt) == 0, "host.h kern_preempt_pending reads the first int");
_Static_assert(offsetof(RtThread, host_depth) == sizeof(int), "host.h kern_host_depth reads the second int");
static RtThread *g_all, *g_wait, *g_main;
static uint64_t g_next_id = 0x51, g_wseq, g_created;
static uint64_t g_main_stack;
@@ -103,7 +121,7 @@ uint64_t kern_now_ns(void) { return plat_now_ns(); }
// ---- TOTK_TRACE=sched accounting (holding G)
#define RW_N 32
static RtThread *g_render; // the thread that presents
static RtThread *g_render; // the thread that presents (sched trace, core diag)
static struct { int st; uint64_t obj, ns; } g_rw[RW_N]; // the render thread's waits by object
static uint64_t g_pre_req, g_pre_n, g_pre_ns, g_pre_max, g_pre_stale; // preempt requests; releases after one and their latency
static uint64_t g_sync_calls, g_sync_blocks; // WaitSynchronization calls and the blocks inside them
@@ -243,32 +261,485 @@ static uint32_t block_inner(RtThread *t, int kind, int64_t ns) {
static RtThread *g_core_owner[NCORE], *g_core_wait;
static int g_cores_on = -1, g_held_max;
static void diag_init(void);
static void ap_init(void);
static int cores_enabled(void) {
if (g_cores_on < 0) g_cores_on = cfg_get("TOTK_FREE_CORES") ? 0 : 1;
if (g_cores_on < 0) {
g_cores_on = cfg_get("TOTK_FREE_CORES") ? 0 : 1;
diag_init();
ap_init();
}
return g_cores_on;
}
static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) & 1); }
// ---- asynchronous preemption (TOTK_ASYNC_PREEMPT, on unless 0)
// Adapted from MacNativeGames games/totk/runtime/threads.c @ 33e8fa8 (8bf0893, PR #488; MIT, Copyright (c) 2026
// MacNativeGames contributors): Horizon gives a core to a strictly better thread the moment that thread is ready.
// Here a better waiter only marks the owner, which gives the core up at its next SVC or indirect call; an owner in a
// stretch of game code with neither keeps it (TOTK_CORE_DIAG: 0.5-1.1 ms of thread-time per frame, waits to 16 ms and
// more, build/v121-preempt/NOTES.md). With it on, three parts, all of which MacNativeGames found necessary:
// - The waiter interrupts the owner (SuspendThread, GetThreadContext) when it has waited AP_FIRST_NS, then every
// millisecond: if the owner's host rip is in the lifted modules' code (g_gen_lo..g_gen_hi) and its host depth is 0
// (not in a host call or SVC, nor in game code a host call made), it holds no host lock, so its rip is moved to
// kern_ap_stub, which saves every volatile register, gives the core up like a checkpoint (kern_ap_yield) and jumps
// back. Otherwise it is only marked, as before.
// - Priority inheritance (eff_prio): a thread owning a guest mutex counts at the priority of the best thread blocked on
// it, for granting, kicking, preempting and yielding. Without it MacNativeGames' game stopped at its first
// "Continue" (a better thread took the core of the low-priority owner of a mutex it then waited for).
// - Horizon's preemption disable: the SDK's spin locks (1.2.1 sdk f_70051242a8, f_7005124360 and siblings) count
// TLR+0x100 (disable_count) while they try or hold the lock, and after clearing it call SVC 0x36 when TLR+0x102
// (interrupt_flag) is set. Such an owner is not preempted, at a checkpoint either: interrupt_flag is set and the
// switch happens at that SVC. Preempting it anyway hung MacNativeGames' game at "Continue" (the lock holder lost its
// core to a thread that then spun on that lock).
// TOTK_ASYNC_PREEMPT=0 keeps the old scheduling exactly: raw priorities, no interruption, no disable_count; =2 keeps
// inheritance and the preemption disable without interrupting owners.
#define AP_FIRST_NS 250000ull // the waiter spins this long before the first interruption (most waits end sooner)
#define AP_GAP_NS 1000000ull // and interrupts the same owner at most this often
static int g_ap = -1, g_ap_int; // g_ap_int: interrupt owners (TOTK_ASYNC_PREEMPT=2: inheritance and disable only)
static int g_ap_log; // redirects logged so far (AP_LOG_N)
#define AP_LOG_N 40
static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start
static uint64_t g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok;
void kern_ap_stub(void);
static void ap_init(void) {
if (g_ap >= 0) return;
const char *e = cfg_get("TOTK_ASYNC_PREEMPT");
g_ap = !(e && !strcmp(e, "0"));
g_ap_int = g_ap && !(e && !strcmp(e, "2"));
if (g_ap) {
RtModule m[16];
int n = rt_linked_modules(m, 16);
uintptr_t lo = UINTPTR_MAX, hi = 0;
for (int i = 0; i < n; i++)
for (uint64_t k = 0; k < m[i].count; k++) {
uintptr_t f = (uintptr_t)m[i].fns[k];
if (f < lo) lo = f;
if (f > hi) hi = f;
}
const uintptr_t host[] = {(uintptr_t)kern_ap_stub, (uintptr_t)kern_checkpoint, (uintptr_t)rt_svc,
(uintptr_t)rt_dispatch, (uintptr_t)rt_call};
int bad = !n || lo >= hi;
for (unsigned i = 0; i < sizeof host / sizeof *host; i++) bad |= host[i] >= lo && host[i] <= hi;
if (bad) {
rt_log("kernel: asynchronous preemption off: the lifted code range %#llx-%#llx holds host code",
(unsigned long long)lo, (unsigned long long)hi);
g_ap = 0;
} else {
g_gen_lo = lo;
g_gen_hi = hi;
kern_host_track = 1;
}
}
rt_log("kernel: asynchronous preemption of virtual cores %s, priority inheritance and preemption disable %s; lifted "
"code exe+%#llx-exe+%#llx", g_ap_int ? "on" : "off", g_ap ? "on (TOTK_ASYNC_PREEMPT=0 turns all off)" : "off",
(unsigned long long)(g_gen_lo ? g_gen_lo - (uintptr_t)GetModuleHandleW(NULL) : 0),
(unsigned long long)(g_gen_hi ? g_gen_hi - (uintptr_t)GetModuleHandleW(NULL) : 0));
}
// Horizon's priority inheritance: o counts at the priority of the best thread blocked on a guest mutex it owns (the
// owner is read from the mutex word, so it follows hand-offs). Holding G.
static int eff_prio(const RtThread *o) {
int p = o->prio;
for (const RtThread *w = g_wait; w; w = w->next)
if (w->wkind == W_LOCK && w->prio < p && w->waddr &&
(__atomic_load_n((const uint32_t *)(uintptr_t)w->waddr, __ATOMIC_RELAXED) & ~WAITERS_BIT) == o->handle)
p = w->prio;
return p;
}
// The priority the virtual cores go by: raw with TOTK_ASYNC_PREEMPT=0, as before.
static int prio_of(const RtThread *t) { return g_ap > 0 ? eff_prio(t) : t->prio; }
// TLR+0x100 disable_count, +0x102 interrupt_flag (any thread may read them; the flag is set from outside as the
// kernel does).
static volatile uint16_t *ap_tlr(const RtThread *t) { return (volatile uint16_t *)(t->tlr + 0x100); }
static int ap_disabled(const RtThread *t) { return g_ap > 0 && ap_tlr(t)[0]; }
// kern_ap_stub: entered in place of the interrupted instruction of top-level game code. It keeps every register the
// game code may have live (volatile GPRs, flags, YMM0-15, MXCSR; the C calls preserve the rest), gives the core up
// in kern_ap_yield and jumps back to kern_ap_rip(), whose value it first stores in the return slot it reserved at
// entry, so a stack walk sees the interrupted function as its caller. No ret: the jump keeps call/return pairs
// balanced. The waiter never writes the owner's stack.
__asm__(".text\n"
".globl kern_ap_stub\n"
".def kern_ap_stub; .scl 2; .type 32; .endef\n"
".p2align 4\n"
"kern_ap_stub:\n"
".seh_proc kern_ap_stub\n"
" leaq -8(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq
".seh_stackalloc 8\n"
" pushfq\n"
".seh_stackalloc 8\n"
" pushq %rax\n"
".seh_pushreg %rax\n"
" pushq %rcx\n"
".seh_pushreg %rcx\n"
" pushq %rdx\n"
".seh_pushreg %rdx\n"
" pushq %r8\n"
".seh_pushreg %r8\n"
" pushq %r9\n"
".seh_pushreg %r9\n"
" pushq %r10\n"
".seh_pushreg %r10\n"
" pushq %r11\n"
".seh_pushreg %r11\n"
" pushq %rbp\n"
".seh_pushreg %rbp\n"
" movq %rsp, %rbp\n"
".seh_setframe %rbp, 0\n"
".seh_endprologue\n"
" andq $-32, %rsp\n"
" subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15
" vmovdqa %ymm0, 64(%rsp)\n"
" vmovdqa %ymm1, 96(%rsp)\n"
" vmovdqa %ymm2, 128(%rsp)\n"
" vmovdqa %ymm3, 160(%rsp)\n"
" vmovdqa %ymm4, 192(%rsp)\n"
" vmovdqa %ymm5, 224(%rsp)\n"
" vmovdqa %ymm6, 256(%rsp)\n"
" vmovdqa %ymm7, 288(%rsp)\n"
" vmovdqa %ymm8, 320(%rsp)\n"
" vmovdqa %ymm9, 352(%rsp)\n"
" vmovdqa %ymm10, 384(%rsp)\n"
" vmovdqa %ymm11, 416(%rsp)\n"
" vmovdqa %ymm12, 448(%rsp)\n"
" vmovdqa %ymm13, 480(%rsp)\n"
" vmovdqa %ymm14, 512(%rsp)\n"
" vmovdqa %ymm15, 544(%rsp)\n"
" stmxcsr 32(%rsp)\n"
" movl $0x1f80, 36(%rsp)\n"
" ldmxcsr 36(%rsp)\n"
" cld\n"
" vzeroupper\n"
" call kern_ap_rip\n"
" movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq
" call kern_ap_yield\n"
" ldmxcsr 32(%rsp)\n"
" vmovdqa 64(%rsp), %ymm0\n"
" vmovdqa 96(%rsp), %ymm1\n"
" vmovdqa 128(%rsp), %ymm2\n"
" vmovdqa 160(%rsp), %ymm3\n"
" vmovdqa 192(%rsp), %ymm4\n"
" vmovdqa 224(%rsp), %ymm5\n"
" vmovdqa 256(%rsp), %ymm6\n"
" vmovdqa 288(%rsp), %ymm7\n"
" vmovdqa 320(%rsp), %ymm8\n"
" vmovdqa 352(%rsp), %ymm9\n"
" vmovdqa 384(%rsp), %ymm10\n"
" vmovdqa 416(%rsp), %ymm11\n"
" vmovdqa 448(%rsp), %ymm12\n"
" vmovdqa 480(%rsp), %ymm13\n"
" vmovdqa 512(%rsp), %ymm14\n"
" vmovdqa 544(%rsp), %ymm15\n"
" movq %rbp, %rsp\n"
" popq %rbp\n"
" popq %r11\n"
" popq %r10\n"
" popq %r9\n"
" popq %r8\n"
" popq %rdx\n"
" popq %rcx\n"
" popq %rax\n"
" popfq\n"
" leaq 8(%rsp), %rsp\n"
" jmpq *-8(%rsp)\n"
".seh_endproc\n");
enum { AP_NONE, AP_RETRY, AP_SENT };
// The waiter w interrupts its target owner if that runs top-level game code (holding G; the owner is suspended only
// between SuspendThread and ResumeThread, during which nothing here takes a lock).
static int ap_try(RtThread *w) {
RtThread *o = w->target;
if (!o || o->cur_core < 0 || !o->preempt) return AP_NONE;
if (!o->hthr || o->ap_pending) return AP_RETRY;
volatile uint16_t *tl = ap_tlr(o);
if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0) {
g_ap_host++;
return AP_RETRY;
}
if (tl[0]) { // preemption disabled: SVC 0x36 switches
tl[1] = 1;
g_ap_deferred++;
return AP_RETRY;
}
uint64_t now = kern_now_ns();
if (o->ap_last && now - o->ap_last < AP_GAP_NS) return AP_RETRY;
o->ap_last = now;
if (SuspendThread(o->hthr) == (DWORD)-1) {
g_ap_fail++;
return AP_RETRY;
}
CONTEXT ctx;
memset(&ctx, 0, sizeof ctx);
ctx.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; // set back whole, so the kernel restores every GPR
int r = AP_RETRY;
if (!GetThreadContext(o->hthr, &ctx))
g_ap_fail++;
else if (ctx.Rip < g_gen_lo || ctx.Rip > g_gen_hi)
g_ap_pc++;
else if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0)
g_ap_host++;
else if (tl[0]) {
tl[1] = 1;
g_ap_deferred++;
} else {
o->ap_rip = ctx.Rip;
o->ap_pending = 1;
ctx.Rip = (DWORD64)(uintptr_t)kern_ap_stub;
if (SetThreadContext(o->hthr, &ctx)) {
g_ap_sent++;
r = AP_SENT;
} else {
o->ap_pending = 0;
g_ap_fail++;
}
}
ResumeThread(o->hthr);
if (r == AP_SENT && g_ap_log < AP_LOG_N) {
g_ap_log++;
rt_log("kernel: interrupt %#x p%d at exe+%#llx rsp %#llx for %#x p%d", o->handle, o->prio,
(unsigned long long)(o->ap_rip - (uintptr_t)GetModuleHandleW(NULL)), (unsigned long long)ctx.Rsp,
w->handle, w->prio);
}
return r;
}
// ---- TOTK_CORE_DIAG=1: why ready threads wait for a virtual core, and for how long
// Each core wait is split into classes: the owner asked to give the core up ran game code (DG_GAME, the part an
// asynchronous preemption would remove), was in a host call or SVC that kept the core (DG_HOST), every owner the waiter
// may displace was of better or equal priority (DG_EQ: Horizon waits too), or the worse owners were claimed by better
// waiters (DG_NOCORE). Per 60 presents: time per frame and a wait-length histogram per class, the owners that cost
// most; a line per present over 25 ms; a line per wait with over 2 ms of DG_GAME (where the owner reached its
// checkpoint), and one host RIP sample of the owner per such wait (exe RVA, for totk.map).
enum { DG_GAME, DG_HOST, DG_EQ, DG_NOCORE, DG_N };
static const char *const dg_name[DG_N] = {"game", "host", "eq", "nocore"};
#define DG_NB 8 // wait lengths: < 0.05, 0.25, 1, 2, 4, 8, 16 ms, longer
static const uint64_t dg_edge[DG_NB - 1] = {50000, 250000, 1000000, 2000000, 4000000, 8000000, 16000000};
#define DG_LONG_LINES 400
#define DG_SAMPLES 600
#define DG_OWN 32
int kern_host_track;
static int g_diag = -1;
typedef struct {
uint64_t ns[DG_N], n[DG_N], hist[DG_N][DG_NB]; // this window
struct { uint32_t h; int prio, cls; uint64_t ns; } own[DG_OWN]; // this window: owners by class
uint64_t long_frames, long_game_frames;
} DgWindow;
static DgWindow g_dgw;
static uint64_t g_dgf[DG_N], g_dgfr[DG_N]; // this frame: all threads, the render thread
static uint64_t g_dg_samples, g_dg_lines, g_sync36, g_sync36_flag;
static void diag_init(void) {
if (g_diag >= 0) return;
const char *e = cfg_get("TOTK_CORE_DIAG");
g_diag = e && *e && strcmp(e, "0");
if (g_diag) {
kern_host_track = 1;
rt_log("core-diag: on (TOTK_CORE_DIAG)");
}
}
int kern_core_diag_on(void) { return g_diag > 0; }
// A tracked host call or SVC returned to depth 0 with a preempt request pending (host.h); on the thread itself.
void kern_host_left(void *thread) {
RtThread *t = thread;
if (g_diag > 0 && t->dg_kick && t->dg_host_kick && !t->dg_host_end) t->dg_host_end = kern_now_ns();
}
static int dg_in_host(const RtThread *o) { return __atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0; }
static void dg_add(const RtThread *w, int cls, uint64_t ns, uint32_t oh, int oprio) {
if (!ns) return;
g_dgw.ns[cls] += ns;
g_dgf[cls] += ns;
if (w == g_render) g_dgfr[cls] += ns;
if (!oh) return;
int i = 0;
for (; i < DG_OWN && g_dgw.own[i].ns; i++)
if (g_dgw.own[i].h == oh && g_dgw.own[i].cls == cls) break;
if (i == DG_OWN) return;
g_dgw.own[i].h = oh;
g_dgw.own[i].prio = oprio;
g_dgw.own[i].cls = cls;
g_dgw.own[i].ns += ns;
}
// The waiter's class at the start of its wait, after core_kick claimed (or did not claim) an owner for it.
static int dg_class0(const RtThread *t) {
if (t->target) return dg_in_host(t->target) ? DG_HOST : DG_GAME;
for (int i = 0; i < NCORE; i++)
if (core_allowed(t, i) && g_core_owner[i] && g_core_owner[i]->prio > t->prio) return DG_NOCORE;
return DG_EQ;
}
// The waiter got a core: split [dg_since, now] by what the owner that released it did after its preempt request.
static void dg_done(RtThread *t) {
uint64_t now = kern_now_ns(), t0 = t->dg_since, w = now - t0;
t->dg_since = 0;
if (!t0) return;
const struct DgRel *r = &t->dg_rel;
uint64_t part[DG_N] = {0};
uint32_t oh = t->target ? t->target->handle : r->h;
int oprio = t->target ? t->target->prio : r->prio;
if (r->pre && r->kick) {
uint64_t k = r->kick > t0 ? (r->kick < now ? r->kick : now) : t0;
part[t->dg_cls] += k - t0;
if (r->host) {
uint64_t he = r->host_end > k && r->host_end < now ? r->host_end : now;
part[DG_HOST] += he - k;
part[DG_GAME] += now - he;
} else
part[DG_GAME] += now - k;
oh = r->h;
oprio = r->prio;
} else
part[t->dg_cls] += w;
int top = 0;
for (int c = 0; c < DG_N; c++) {
dg_add(t, c, part[c], oh, oprio);
if (part[c] > part[top]) top = c;
}
int b = 0;
while (b < DG_NB - 1 && w >= dg_edge[b]) b++;
g_dgw.n[top]++;
g_dgw.hist[top][b]++;
if (part[DG_GAME] >= 2000000 && g_dg_lines < DG_LONG_LINES) {
g_dg_lines++;
char a[128], l[128];
rt_log("core-diag: wait %.2f ms (game %.2f host %.2f): %#x p%d mask %llx; released by %#x p%d%s at %s lr %s",
w / 1e6, part[DG_GAME] / 1e6, part[DG_HOST] / 1e6, t->handle, t->prio, (unsigned long long)t->mask, r->h,
r->prio, r->pre ? " (preempted)" : "", r->pc ? ldr_describe(r->pc, a, sizeof a) : "-",
r->lr ? ldr_describe(r->lr, l, sizeof l) : "-");
}
t->target = NULL;
memset(&t->dg_rel, 0, sizeof t->dg_rel);
}
// Once per long DG_GAME wait: where the owner's host thread runs (exe RVA). Holding G; the owner is resumed before
// anything that could take a lock it holds.
static void dg_sample(RtThread *t) {
RtThread *o = t->target;
if (!o || o->cur_core < 0 || !o->tid || g_dg_samples >= DG_SAMPLES) return;
HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, o->tid);
if (!h) return;
CONTEXT ctx;
memset(&ctx, 0, sizeof ctx);
ctx.ContextFlags = CONTEXT_CONTROL;
int depth = 0, ok = 0;
if (SuspendThread(h) != (DWORD)-1) {
ok = GetThreadContext(h, &ctx);
depth = __atomic_load_n(&o->host_depth, __ATOMIC_RELAXED);
ResumeThread(h);
}
CloseHandle(h);
if (!ok) return;
g_dg_samples++;
uintptr_t base = (uintptr_t)GetModuleHandleW(NULL);
rt_log("core-diag: sample owner %#x p%d depth %d rip exe+%#llx (waiter %#x p%d, %.2f ms)", o->handle, o->prio, depth,
(unsigned long long)(ctx.Rip - base), t->handle, t->prio, (kern_now_ns() - t->dg_since) / 1e6);
}
// perf_present, every present: a line for a present over 25 ms, the 60-present summary.
void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns) {
if (g_ap > 0 && frame && frame % 600 == 0) { // TOTK_ASYNC_PREEMPT counters since the start
lock();
uint64_t v[8] = {g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok, g_sync36};
unlock();
rt_log("kernel: frame %llu: asynchronous preemption: %llu interrupted, %llu cores given up there; refused: "
"%llu in a host call, %llu outside game code, %llu preemption disabled, %llu failed; %llu waits ended "
"in the first spin; SVC 0x36 %llu",
(unsigned long long)frame, (unsigned long long)v[0], (unsigned long long)v[1], (unsigned long long)v[2],
(unsigned long long)v[3], (unsigned long long)v[4], (unsigned long long)v[5], (unsigned long long)v[6],
(unsigned long long)v[7]);
}
if (g_diag <= 0) return;
DgWindow w;
uint64_t f[DG_N], fr[DG_N], s36, s36f;
lock();
g_render = tcur;
memcpy(f, g_dgf, sizeof f);
memcpy(fr, g_dgfr, sizeof fr);
memset(g_dgf, 0, sizeof g_dgf);
memset(g_dgfr, 0, sizeof g_dgfr);
int lng = interval_ns > 25000000ull;
if (lng) {
g_dgw.long_frames++;
if (f[DG_GAME] > 1000000) g_dgw.long_game_frames++;
}
int rep = frame && frame % 60 == 0;
if (rep) {
w = g_dgw;
memset(&g_dgw, 0, sizeof g_dgw);
}
s36 = g_sync36;
s36f = g_sync36_flag;
unlock();
if (lng)
rt_log("core-diag: frame %llu: %.1f ms; core wait game %.2f host %.2f eq %.2f nocore %.2f ms; render thread game "
"%.2f host %.2f eq %.2f nocore %.2f ms",
(unsigned long long)frame, interval_ns / 1e6, f[0] / 1e6, f[1] / 1e6, f[2] / 1e6, f[3] / 1e6,
fr[0] / 1e6, fr[1] / 1e6, fr[2] / 1e6, fr[3] / 1e6);
if (!rep) return;
char s[1024];
int o = 0;
for (int c = 0; c < DG_N; c++) {
o += snprintf(s + o, sizeof s - o, "%s %.3f ms/frame %llu waits [", dg_name[c], w.ns[c] / 60e6,
(unsigned long long)w.n[c]);
for (int b = 0; b < DG_NB; b++)
o += snprintf(s + o, sizeof s - o, "%s%llu", b ? " " : "", (unsigned long long)w.hist[c][b]);
o += snprintf(s + o, sizeof s - o, "]%s", c < DG_N - 1 ? "; " : "");
}
rt_log("core-diag: frames %llu-%llu: %s; long frames %llu (game wait > 1 ms in %llu); SVC 0x36 %llu (flag set %llu)",
(unsigned long long)frame - 59, (unsigned long long)frame, s, (unsigned long long)w.long_frames,
(unsigned long long)w.long_game_frames, (unsigned long long)s36, (unsigned long long)s36f);
o = 0;
for (int k = 0; k < 6; k++) {
int bi = -1;
for (int i = 0; i < DG_OWN; i++)
if (w.own[i].ns && (bi < 0 || w.own[i].ns > w.own[bi].ns)) bi = i;
if (bi < 0) break;
o += snprintf(s + o, sizeof s - o, " %#x p%d %s %.3f;", w.own[bi].h, w.own[bi].prio, dg_name[w.own[bi].cls],
w.own[bi].ns / 60e6);
w.own[bi].ns = 0;
}
if (o) rt_log("core-diag: owners (ms/frame):%s", s);
}
static void core_count(void) {
int n = 0;
for (int i = 0; i < NCORE; i++) n += g_core_owner[i] != NULL;
if (n > g_held_max) g_held_max = n;
}
static struct DgRel g_dg_rel; // TOTK_CORE_DIAG: the owner core_put is releasing, for core_grant
static void core_grant(int c) {
RtThread *b = NULL, **bp = NULL;
for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext)
if (core_allowed(*p, c) && (!b || (*p)->prio < b->prio || ((*p)->prio == b->prio && (*p)->cseq < b->cseq))) {
int bpri = 0;
for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext) {
if (!core_allowed(*p, c)) continue;
int pr = prio_of(*p);
if (!b || pr < bpri || (pr == bpri && (*p)->cseq < b->cseq)) {
b = *p;
bp = p;
bpri = pr;
}
}
if (!b) return;
*bp = b->cnext;
b->cnext = NULL;
b->cur_core = c;
g_core_owner[c] = b;
core_count();
if (g_diag > 0) b->dg_rel = g_dg_rel;
WakeConditionVariable(&b->cv);
}
@@ -283,6 +754,12 @@ static void core_put(RtThread *t) {
t->npre++;
}
t->pre_since = 0;
if (g_diag > 0) {
g_dg_rel = (struct DgRel){.kick = t->dg_kick, .host_end = t->dg_host_end, .pc = t->dg_pc, .lr = t->dg_lr,
.host = t->dg_host_kick, .pre = t->preempt, .prio = t->prio, .h = t->handle};
t->dg_kick = t->dg_host_end = t->dg_pc = t->dg_lr = 0;
t->dg_host_kick = 0;
}
t->cur_core = -1;
t->preempt = 0;
g_core_owner[c] = NULL;
@@ -291,8 +768,9 @@ static void core_put(RtThread *t) {
// Whether a waiter better than t may run on core c.
static int core_better_waiter(const RtThread *t, int c) {
int tp = prio_of(t);
for (const RtThread *w = g_core_wait; w; w = w->cnext)
if (w->prio < t->prio && core_allowed(w, c)) return 1;
if (core_allowed(w, c) && prio_of(w) < tp) return 1;
return 0;
}
@@ -302,30 +780,40 @@ static int core_better_waiter(const RtThread *t, int c) {
// again (build/v121-dpre/NOTES.md).
static void core_kick(void) {
RtThread *w[64];
int n = 0, claimed = 0;
int wp[64], op[NCORE], n = 0, claimed = 0;
for (int i = 0; i < NCORE; i++) op[i] = g_core_owner[i] ? prio_of(g_core_owner[i]) : 0;
for (RtThread *p = g_core_wait; p && n < 64; p = p->cnext) {
int i = n++;
for (; i > 0 && (w[i - 1]->prio > p->prio || (w[i - 1]->prio == p->prio && w[i - 1]->cseq > p->cseq)); i--)
int i = n++, pp = prio_of(p);
for (; i > 0 && (wp[i - 1] > pp || (wp[i - 1] == pp && w[i - 1]->cseq > p->cseq)); i--) {
w[i] = w[i - 1];
wp[i] = wp[i - 1];
}
w[i] = p;
wp[i] = pp;
}
for (int k = 0; k < n && claimed != (1 << NCORE) - 1; k++) {
int v = -1;
for (int i = 0; i < NCORE; i++) {
RtThread *o = g_core_owner[i];
if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || o->prio <= w[k]->prio) continue;
if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || op[i] <= wp[k]) continue;
RtThread *b = v < 0 ? NULL : g_core_owner[v];
if (!b || o->prio > b->prio || (o->prio == b->prio && o->preempt && !b->preempt)) v = i;
if (!b || op[i] > op[v] || (op[i] == op[v] && o->preempt && !b->preempt)) v = i;
}
if (v < 0) continue;
claimed |= 1 << v;
RtThread *o = g_core_owner[v];
w[k]->target = o;
if (o->preempt) continue;
if (rt_trace & RT_TRACE_SCHED) {
o->pre_since = kern_now_ns();
g_pre_req++;
}
__atomic_store_n(&o->preempt, 1, __ATOMIC_RELAXED);
if (g_diag > 0) {
o->dg_kick = kern_now_ns();
o->dg_host_kick = dg_in_host(o);
o->dg_host_end = 0;
}
__atomic_store_n(&o->preempt, 1, __ATOMIC_RELEASE);
}
}
@@ -351,8 +839,34 @@ static void core_take(RtThread *t) {
t->cseq = ++g_wseq;
t->cnext = g_core_wait;
g_core_wait = t;
t->target = NULL;
if (g_diag > 0) {
t->dg_since = kern_now_ns();
memset(&t->dg_rel, 0, sizeof t->dg_rel);
}
core_kick();
while (t->cur_core < 0) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
if (g_diag > 0) t->dg_cls = dg_class0(t);
if (g_ap_int && t->target && t->cur_core < 0) { // most waits end within microseconds: spin before interrupting
uint64_t end = kern_now_ns() + AP_FIRST_NS;
unlock();
while (__atomic_load_n(&t->cur_core, __ATOMIC_RELAXED) < 0 && kern_now_ns() < end) YieldProcessor();
lock();
if (t->cur_core >= 0) g_ap_spin_ok++;
}
int sampled = 0;
while (t->cur_core < 0) {
DWORD ms = INFINITE;
if (g_ap_int && ap_try(t) != AP_NONE) ms = 1; // the owner keeps its core: again in a millisecond
int sample = g_diag > 0 && !sampled && t->dg_cls == DG_GAME;
if (sample && ms == INFINITE) ms = 2;
SleepConditionVariableSRW(&t->cv, &G, ms, 0);
if (sample && t->cur_core < 0 && kern_now_ns() - t->dg_since >= 2000000) {
sampled = 1;
dg_sample(t);
}
}
if (g_diag > 0) dg_done(t);
t->target = NULL;
acct(t, S_RUN);
}
@@ -393,17 +907,57 @@ void kern_checkpoint(void) {
RtThread *t = tcur;
if (!t || !__atomic_load_n(&t->preempt, __ATOMIC_RELAXED)) return;
lock();
if (t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) {
if (g_diag > 0 && t->cur) {
t->dg_pc = t->cur->pc;
t->dg_lr = t->cur->x[30];
}
int better = t->cur_core >= 0 && core_better_waiter(t, t->cur_core);
if (better && ap_disabled(t)) { // preemption disabled: the switch comes at SVC 0x36
ap_tlr(t)[1] = 1;
t->preempt = 0;
g_ap_deferred++;
} else if (better) {
core_put(t); // clears preempt (after the sched trace reads it)
core_take(t);
} else {
t->preempt = 0;
t->pre_since = 0;
t->dg_kick = t->dg_host_end = 0;
t->dg_host_kick = 0;
g_pre_stale++;
}
unlock();
}
// kern_ap_stub, on the interrupted owner: where to jump back to, then the core given up as at a checkpoint.
uint64_t kern_ap_rip(void) { return tcur->ap_rip; }
void kern_ap_yield(void) {
RtThread *t = tcur;
lock();
t->ap_pending = 0;
if (t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) {
if (g_diag > 0) {
t->dg_pc = t->cur->pc;
t->dg_lr = t->cur->x[30];
}
g_ap_taken++;
core_put(t);
core_take(t);
}
unlock();
}
// SVC 0x36 SynchronizePreemptionState: a preemption deferred by disable_count happens here.
static void ap_sync(RtThread *t) {
lock();
if (g_ap > 0 && t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) {
core_put(t);
core_take(t);
}
unlock();
}
void kern_core_stats(int *held_now, int *held_max) {
lock();
int n = 0;
@@ -625,6 +1179,8 @@ static void thread_finish(RtThread *t) {
lock();
core_put(t);
acct(t, S_OFF);
if (t->hthr) CloseHandle(t->hthr); // no waiter interrupts a thread without a core
t->hthr = NULL;
t->exited = 1;
wake_sync_waiters();
if (t->closed) thr_free(t);
@@ -636,11 +1192,22 @@ static void thread_finish(RtThread *t) {
// is offset, and the landing code then reads its locals through the wrong base.)
static __attribute__((noreturn)) void exit_now(RtThread *t) { longjmp(t->exit_jmp, 1); }
// The calling host thread, for ap_try (TOTK_ASYNC_PREEMPT only).
static HANDLE ap_open_self(void) {
if (!cores_enabled() || !g_ap_int) return NULL;
HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_SET_CONTEXT | THREAD_QUERY_INFORMATION,
FALSE, GetCurrentThreadId());
if (!h) rt_log("kernel: OpenThread for asynchronous preemption failed (error %lu)", GetLastError());
return h;
}
static DWORD WINAPI thread_main(void *p) {
RtThread *t = p;
tcur = t;
t->tid = GetCurrentThreadId();
HANDLE self = ap_open_self();
lock();
t->hthr = self;
while (!t->started) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
core_take(t);
unlock();
@@ -683,7 +1250,9 @@ uint32_t kern_main_thread(Cpu **cpu) {
t->stack_top = stack + MAIN_STACK;
t->cpu->sp = t->stack_top;
t->tid = GetCurrentThreadId();
HANDLE self = ap_open_self();
lock();
t->hthr = self;
thr_link(t);
g_main = t;
g_main_stack = stack;
@@ -869,8 +1438,9 @@ int kern_svc(Cpu *c, uint32_t imm) {
// Yield: hand the core to a waiting thread of equal or better priority.
lock();
int give = 0;
int tp = prio_of(t);
for (RtThread *w = g_core_wait; w && !give; w = w->cnext)
give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && w->prio <= t->prio;
give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && prio_of(w) <= tp;
if (give) {
core_put(t);
core_take(t);
@@ -932,6 +1502,17 @@ int kern_svc(Cpu *c, uint32_t imm) {
x[0] = (uint64_t)(uint32_t)(t->cur_core >= 0 ? t->cur_core : t->core);
return 1;
}
case 0x36: { // SynchronizePreemptionState: the SDK's spin locks call it after clearing TLR+0x100 (disable_count)
// when TLR+0x102 (interrupt_flag) is set; rt_svc's checkpoint has already honoured a pending preempt
RtThread *t = cur(imm);
volatile uint16_t *flag = (volatile uint16_t *)(t->tlr + 0x102);
__atomic_fetch_add(&g_sync36, 1, __ATOMIC_RELAXED);
if (*flag) __atomic_fetch_add(&g_sync36_flag, 1, __ATOMIC_RELAXED);
*flag = 0;
ap_sync(t);
x[0] = R_OK;
return 1;
}
case 0x25: { // GetThreadId(w1 handle -> x1)
lock();
RtThread *t = h_get((uint32_t)x[1], O_THREAD);
@@ -1299,6 +1880,7 @@ static void WINAPI fiber_main(void *p) {
Fiber *f = p;
fiber_resumed();
Cpu *c = f->cpu;
tcur->host_depth = 0; // a new fiber starts at the top level; each context restores its own depth on resume
uint64_t next = rt_call(c, f->func, 1, f->arg);
f->state = F_DONE;
Fiber *n = NULL;
@@ -1335,7 +1917,9 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) {
t->leaving = t->fiber;
t->fiber = to;
t->cur = dst;
int depth = t->host_depth;
SwitchToFiber(to ? to->host : t->base_fiber);
t->host_depth = depth;
fiber_resumed();
}
+1
View File
@@ -409,6 +409,7 @@ void perf_present(uint64_t frame) {
if (rt_trace & RT_TRACE_SCHED) kern_sched_report(frame);
uint64_t now = kern_now_ns(), d = last ? now - last : 0;
ft_frame(frame, now, d); // d is 0 on the first present: it only starts the period
kern_core_diag_frame(frame, d);
if (flight_on) flight_frame(frame, now, d, frame_ns, counts);
if (!(rt_trace & RT_TRACE_PERF)) {
memset(frame_ns, 0, sizeof frame_ns);
+16
View File
@@ -138,8 +138,24 @@ static void get_info(Cpu *c, uint64_t pc) {
c->x[1] = v;
}
static void svc_body(Cpu *c, uint32_t imm, uint64_t pc);
void rt_svc(Cpu *c, uint32_t imm, uint64_t pc) {
kern_checkpoint();
if (__builtin_expect(!kern_host_track || !c->thread, 1)) {
svc_body(c, imm, pc);
return;
}
void *t = c->thread; // kern_host_track: the SVC counted in the thread's host depth (hostfn.c hostfn_tracked)
int *d = kern_host_depth(t);
__atomic_store_n(d, *d + 1, __ATOMIC_RELAXED);
svc_body(c, imm, pc);
int n = *d - 1;
__atomic_store_n(d, n, __ATOMIC_RELAXED);
if (!n && kern_preempt_pending(t)) kern_host_left(t);
}
static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) {
__atomic_fetch_add(&g_svcs, 1, __ATOMIC_RELAXED);
if (rt_trace & RT_TRACE_SVC) {
char at[160];