mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-21 04:55:56 +00:00
Preempt a virtual core owner in top-level game code from the waiting thread, with priority inheritance and the SDK's preemption disable, behind TOTK_ASYNC_PREEMPT
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d52cdfc6ee
commit
c66ac84925
3 files changed
+346
-37
No files matched your search
+1
-1
@@ -23,7 +23,7 @@ static int n_entries;
|
||||
// Every name the runtime reads (and the patch sets in generated/), without the prefix. A file key outside this list is
|
||||
// logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line.
|
||||
static const char *const known[] = {
|
||||
"APP_POOL", "ASSUME_FOCUS", "ASTC", "ASTC_TIME", "AUDIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT",
|
||||
"APP_POOL", "ASSUME_FOCUS", "ASTC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT",
|
||||
"BIND_MINUS", "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL",
|
||||
"BIND_ZR",
|
||||
"CLOTH_REALTIME", "CORE_DIAG", "DESC_TEMPLATE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT",
|
||||
|
||||
+3
-2
@@ -176,13 +176,14 @@ void kern_core_back(int *away);
|
||||
void kern_checkpoint(void); // preemption point, at every SVC and indirect call
|
||||
// A preempt request is pending for the thread (Cpu.thread): its RtThread starts with the flag (kernel.c).
|
||||
static inline int kern_preempt_pending(const void *thread) { return __atomic_load_n((const int *)thread, __ATOMIC_RELAXED); }
|
||||
// Set once before guest threads start (TOTK_CORE_DIAG): host calls (hostfn.c) and SVCs (svc.c) count their depth in
|
||||
// Set once before guest threads start (TOTK_CORE_DIAG, TOTK_ASYNC_PREEMPT): host calls (hostfn.c) and SVCs (svc.c) count their depth in
|
||||
// the thread's second int, so the kernel knows whether a core owner runs game code at the top level.
|
||||
extern int kern_host_track;
|
||||
static inline int *kern_host_depth(void *thread) { return (int *)thread + 1; }
|
||||
void kern_host_left(void *thread); // back at depth 0 with a preempt request pending
|
||||
int kern_core_diag_on(void); // TOTK_CORE_DIAG
|
||||
void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns); // perf_present, every present
|
||||
void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns); // perf_present, every present (also logs the
|
||||
// asynchronous preemption counters)
|
||||
void kern_sched_report(uint64_t frame); // TOTK_TRACE=sched: per-thread time split, on present
|
||||
uint32_t kern_handle_new(int kind, void *obj); // kind: KERN_SESSION or KERN_TMEM (ipc.c)
|
||||
void *kern_handle_get(uint32_t h, int kind);
|
||||
|
||||
+342
-34
@@ -87,10 +87,15 @@ struct RtThread {
|
||||
int st; // S_* state since st_since
|
||||
uint64_t st_since, st_ns[S_N], nblk, npre, pre_since, cpu_last;
|
||||
uint64_t wobj; // what the current wait is on: owner handle, key, address or first handle
|
||||
RtThread *target; // as a core waiter: the owner core_kick asked to give its core up for it (holding G)
|
||||
// TOTK_ASYNC_PREEMPT (holding G)
|
||||
HANDLE hthr; // the host thread, for SuspendThread / Get/SetThreadContext; open while it may own a core
|
||||
uint64_t ap_rip; // the interrupted host rip kern_ap_stub returns to
|
||||
uint64_t ap_last; // the last redirect attempt on this owner
|
||||
int ap_pending; // redirected to kern_ap_stub, which has not reached kern_ap_yield yet
|
||||
// TOTK_CORE_DIAG (holding G, except dg_host_end, which only the thread itself writes)
|
||||
uint64_t dg_since; // as a waiter: core wait start
|
||||
int dg_cls; // as a waiter: DG_* class at the start of the wait
|
||||
RtThread *dg_target; // as a waiter: the owner asked to give its core up for it
|
||||
uint64_t dg_kick; // as an owner: the preempt request
|
||||
uint64_t dg_host_end; // as an owner: the return from the host call or SVC it was in at the request
|
||||
uint64_t dg_pc, dg_lr; // as an owner: guest pc and lr at the checkpoint that gave the core up
|
||||
@@ -256,17 +261,243 @@ static RtThread *g_core_owner[NCORE], *g_core_wait;
|
||||
static int g_cores_on = -1, g_held_max;
|
||||
|
||||
static void diag_init(void);
|
||||
static void ap_init(void);
|
||||
|
||||
static int cores_enabled(void) {
|
||||
if (g_cores_on < 0) {
|
||||
g_cores_on = cfg_get("TOTK_FREE_CORES") ? 0 : 1;
|
||||
diag_init();
|
||||
ap_init();
|
||||
}
|
||||
return g_cores_on;
|
||||
}
|
||||
|
||||
static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) & 1); }
|
||||
|
||||
// ---- asynchronous preemption (TOTK_ASYNC_PREEMPT, on unless 0)
|
||||
// Adapted from MacNativeGames games/totk/runtime/threads.c @ 33e8fa8 (8bf0893, PR #488; MIT, Copyright (c) 2026
|
||||
// MacNativeGames contributors): Horizon gives a core to a strictly better thread the moment that thread is ready.
|
||||
// Here a better waiter only marks the owner, which gives the core up at its next SVC or indirect call; an owner in a
|
||||
// stretch of game code with neither keeps it (TOTK_CORE_DIAG: 0.5-1.1 ms of thread-time per frame, waits to 16 ms and
|
||||
// more, build/v121-preempt/NOTES.md). With it on, three parts, all of which MacNativeGames found necessary:
|
||||
// - The waiter interrupts the owner (SuspendThread, GetThreadContext) when it has waited AP_FIRST_NS, then every
|
||||
// millisecond: if the owner's host rip is in the lifted modules' code (g_gen_lo..g_gen_hi) and its host depth is 0
|
||||
// (not in a host call or SVC, nor in game code a host call made), it holds no host lock, so its rip is moved to
|
||||
// kern_ap_stub, which saves every volatile register, gives the core up like a checkpoint (kern_ap_yield) and jumps
|
||||
// back. Otherwise it is only marked, as before.
|
||||
// - Priority inheritance (eff_prio): a thread owning a guest mutex counts at the priority of the best thread blocked on
|
||||
// it, for granting, kicking, preempting and yielding. Without it MacNativeGames' game stopped at its first
|
||||
// "Continue" (a better thread took the core of the low-priority owner of a mutex it then waited for).
|
||||
// - Horizon's preemption disable: the SDK's spin locks (1.2.1 sdk f_70051242a8, f_7005124360 and siblings) count
|
||||
// TLR+0x100 (disable_count) while they try or hold the lock, and after clearing it call SVC 0x36 when TLR+0x102
|
||||
// (interrupt_flag) is set. Such an owner is not preempted, at a checkpoint either: interrupt_flag is set and the
|
||||
// switch happens at that SVC. Preempting it anyway hung MacNativeGames' game at "Continue" (the lock holder lost its
|
||||
// core to a thread that then spun on that lock).
|
||||
// TOTK_ASYNC_PREEMPT=0 keeps the old scheduling exactly: raw priorities, no interruption, no disable_count.
|
||||
#define AP_FIRST_NS 250000ull // the waiter spins this long before the first interruption (most waits end sooner)
|
||||
#define AP_GAP_NS 1000000ull // and interrupts the same owner at most this often
|
||||
static int g_ap = -1;
|
||||
static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start
|
||||
static uint64_t g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok;
|
||||
void kern_ap_stub(void);
|
||||
|
||||
static void ap_init(void) {
|
||||
if (g_ap >= 0) return;
|
||||
const char *e = cfg_get("TOTK_ASYNC_PREEMPT");
|
||||
g_ap = !(e && !strcmp(e, "0"));
|
||||
if (g_ap) {
|
||||
RtModule m[16];
|
||||
int n = rt_linked_modules(m, 16);
|
||||
uintptr_t lo = UINTPTR_MAX, hi = 0;
|
||||
for (int i = 0; i < n; i++)
|
||||
for (uint64_t k = 0; k < m[i].count; k++) {
|
||||
uintptr_t f = (uintptr_t)m[i].fns[k];
|
||||
if (f < lo) lo = f;
|
||||
if (f > hi) hi = f;
|
||||
}
|
||||
const uintptr_t host[] = {(uintptr_t)kern_ap_stub, (uintptr_t)kern_checkpoint, (uintptr_t)rt_svc,
|
||||
(uintptr_t)rt_dispatch, (uintptr_t)rt_call};
|
||||
int bad = !n || lo >= hi;
|
||||
for (unsigned i = 0; i < sizeof host / sizeof *host; i++) bad |= host[i] >= lo && host[i] <= hi;
|
||||
if (bad) {
|
||||
rt_log("kernel: asynchronous preemption off: the lifted code range %#llx-%#llx holds host code",
|
||||
(unsigned long long)lo, (unsigned long long)hi);
|
||||
g_ap = 0;
|
||||
} else {
|
||||
g_gen_lo = lo;
|
||||
g_gen_hi = hi;
|
||||
kern_host_track = 1;
|
||||
}
|
||||
}
|
||||
rt_log("kernel: asynchronous preemption of virtual cores, priority inheritance, preemption disable: %s",
|
||||
g_ap ? "on (TOTK_ASYNC_PREEMPT=0 turns them off)" : "off");
|
||||
}
|
||||
|
||||
// Horizon's priority inheritance: o counts at the priority of the best thread blocked on a guest mutex it owns (the
|
||||
// owner is read from the mutex word, so it follows hand-offs). Holding G.
|
||||
static int eff_prio(const RtThread *o) {
|
||||
int p = o->prio;
|
||||
for (const RtThread *w = g_wait; w; w = w->next)
|
||||
if (w->wkind == W_LOCK && w->prio < p && w->waddr &&
|
||||
(__atomic_load_n((const uint32_t *)(uintptr_t)w->waddr, __ATOMIC_RELAXED) & ~WAITERS_BIT) == o->handle)
|
||||
p = w->prio;
|
||||
return p;
|
||||
}
|
||||
|
||||
// The priority the virtual cores go by: raw with TOTK_ASYNC_PREEMPT=0, as before.
|
||||
static int prio_of(const RtThread *t) { return g_ap > 0 ? eff_prio(t) : t->prio; }
|
||||
|
||||
// TLR+0x100 disable_count, +0x102 interrupt_flag (any thread may read them; the flag is set from outside as the
|
||||
// kernel does).
|
||||
static volatile uint16_t *ap_tlr(const RtThread *t) { return (volatile uint16_t *)(t->tlr + 0x100); }
|
||||
static int ap_disabled(const RtThread *t) { return g_ap > 0 && ap_tlr(t)[0]; }
|
||||
|
||||
// kern_ap_stub: entered in place of the interrupted instruction of top-level game code. It keeps every register the
|
||||
// game code may have live (volatile GPRs, flags, YMM0-15, MXCSR; the C calls preserve the rest), gives the core up
|
||||
// in kern_ap_yield and jumps back to kern_ap_rip(), whose value it first stores in the return slot it reserved at
|
||||
// entry, so a stack walk sees the interrupted function as its caller. No ret: the jump keeps call/return pairs
|
||||
// balanced. The waiter never writes the owner's stack.
|
||||
__asm__(".text\n"
|
||||
".globl kern_ap_stub\n"
|
||||
".def kern_ap_stub; .scl 2; .type 32; .endef\n"
|
||||
".p2align 4\n"
|
||||
"kern_ap_stub:\n"
|
||||
".seh_proc kern_ap_stub\n"
|
||||
" subq $8, %rsp\n"
|
||||
".seh_stackalloc 8\n"
|
||||
" pushfq\n"
|
||||
".seh_stackalloc 8\n"
|
||||
" pushq %rax\n"
|
||||
".seh_pushreg %rax\n"
|
||||
" pushq %rcx\n"
|
||||
".seh_pushreg %rcx\n"
|
||||
" pushq %rdx\n"
|
||||
".seh_pushreg %rdx\n"
|
||||
" pushq %r8\n"
|
||||
".seh_pushreg %r8\n"
|
||||
" pushq %r9\n"
|
||||
".seh_pushreg %r9\n"
|
||||
" pushq %r10\n"
|
||||
".seh_pushreg %r10\n"
|
||||
" pushq %r11\n"
|
||||
".seh_pushreg %r11\n"
|
||||
" pushq %rbp\n"
|
||||
".seh_pushreg %rbp\n"
|
||||
" movq %rsp, %rbp\n"
|
||||
".seh_setframe %rbp, 0\n"
|
||||
".seh_endprologue\n"
|
||||
" andq $-32, %rsp\n"
|
||||
" subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15
|
||||
" vmovdqa %ymm0, 64(%rsp)\n"
|
||||
" vmovdqa %ymm1, 96(%rsp)\n"
|
||||
" vmovdqa %ymm2, 128(%rsp)\n"
|
||||
" vmovdqa %ymm3, 160(%rsp)\n"
|
||||
" vmovdqa %ymm4, 192(%rsp)\n"
|
||||
" vmovdqa %ymm5, 224(%rsp)\n"
|
||||
" vmovdqa %ymm6, 256(%rsp)\n"
|
||||
" vmovdqa %ymm7, 288(%rsp)\n"
|
||||
" vmovdqa %ymm8, 320(%rsp)\n"
|
||||
" vmovdqa %ymm9, 352(%rsp)\n"
|
||||
" vmovdqa %ymm10, 384(%rsp)\n"
|
||||
" vmovdqa %ymm11, 416(%rsp)\n"
|
||||
" vmovdqa %ymm12, 448(%rsp)\n"
|
||||
" vmovdqa %ymm13, 480(%rsp)\n"
|
||||
" vmovdqa %ymm14, 512(%rsp)\n"
|
||||
" vmovdqa %ymm15, 544(%rsp)\n"
|
||||
" stmxcsr 32(%rsp)\n"
|
||||
" movl $0x1f80, 36(%rsp)\n"
|
||||
" ldmxcsr 36(%rsp)\n"
|
||||
" cld\n"
|
||||
" vzeroupper\n"
|
||||
" call kern_ap_rip\n"
|
||||
" movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq
|
||||
" call kern_ap_yield\n"
|
||||
" ldmxcsr 32(%rsp)\n"
|
||||
" vmovdqa 64(%rsp), %ymm0\n"
|
||||
" vmovdqa 96(%rsp), %ymm1\n"
|
||||
" vmovdqa 128(%rsp), %ymm2\n"
|
||||
" vmovdqa 160(%rsp), %ymm3\n"
|
||||
" vmovdqa 192(%rsp), %ymm4\n"
|
||||
" vmovdqa 224(%rsp), %ymm5\n"
|
||||
" vmovdqa 256(%rsp), %ymm6\n"
|
||||
" vmovdqa 288(%rsp), %ymm7\n"
|
||||
" vmovdqa 320(%rsp), %ymm8\n"
|
||||
" vmovdqa 352(%rsp), %ymm9\n"
|
||||
" vmovdqa 384(%rsp), %ymm10\n"
|
||||
" vmovdqa 416(%rsp), %ymm11\n"
|
||||
" vmovdqa 448(%rsp), %ymm12\n"
|
||||
" vmovdqa 480(%rsp), %ymm13\n"
|
||||
" vmovdqa 512(%rsp), %ymm14\n"
|
||||
" vmovdqa 544(%rsp), %ymm15\n"
|
||||
" movq %rbp, %rsp\n"
|
||||
" popq %rbp\n"
|
||||
" popq %r11\n"
|
||||
" popq %r10\n"
|
||||
" popq %r9\n"
|
||||
" popq %r8\n"
|
||||
" popq %rdx\n"
|
||||
" popq %rcx\n"
|
||||
" popq %rax\n"
|
||||
" popfq\n"
|
||||
" leaq 8(%rsp), %rsp\n"
|
||||
" jmpq *-8(%rsp)\n"
|
||||
".seh_endproc\n");
|
||||
|
||||
enum { AP_NONE, AP_RETRY, AP_SENT };
|
||||
|
||||
// The waiter w interrupts its target owner if that runs top-level game code (holding G; the owner is suspended only
|
||||
// between SuspendThread and ResumeThread, during which nothing here takes a lock).
|
||||
static int ap_try(RtThread *w) {
|
||||
RtThread *o = w->target;
|
||||
if (!o || o->cur_core < 0 || !o->preempt) return AP_NONE;
|
||||
if (!o->hthr || o->ap_pending) return AP_RETRY;
|
||||
volatile uint16_t *tl = ap_tlr(o);
|
||||
if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0) {
|
||||
g_ap_host++;
|
||||
return AP_RETRY;
|
||||
}
|
||||
if (tl[0]) { // preemption disabled: SVC 0x36 switches
|
||||
tl[1] = 1;
|
||||
g_ap_deferred++;
|
||||
return AP_RETRY;
|
||||
}
|
||||
uint64_t now = kern_now_ns();
|
||||
if (o->ap_last && now - o->ap_last < AP_GAP_NS) return AP_RETRY;
|
||||
o->ap_last = now;
|
||||
if (SuspendThread(o->hthr) == (DWORD)-1) {
|
||||
g_ap_fail++;
|
||||
return AP_RETRY;
|
||||
}
|
||||
CONTEXT ctx;
|
||||
memset(&ctx, 0, sizeof ctx);
|
||||
ctx.ContextFlags = CONTEXT_CONTROL;
|
||||
int r = AP_RETRY;
|
||||
if (!GetThreadContext(o->hthr, &ctx))
|
||||
g_ap_fail++;
|
||||
else if (ctx.Rip < g_gen_lo || ctx.Rip > g_gen_hi)
|
||||
g_ap_pc++;
|
||||
else if (__atomic_load_n(&o->host_depth, __ATOMIC_RELAXED) > 0)
|
||||
g_ap_host++;
|
||||
else if (tl[0]) {
|
||||
tl[1] = 1;
|
||||
g_ap_deferred++;
|
||||
} else {
|
||||
o->ap_rip = ctx.Rip;
|
||||
o->ap_pending = 1;
|
||||
ctx.Rip = (DWORD64)(uintptr_t)kern_ap_stub;
|
||||
ctx.ContextFlags = CONTEXT_CONTROL;
|
||||
if (SetThreadContext(o->hthr, &ctx)) {
|
||||
g_ap_sent++;
|
||||
r = AP_SENT;
|
||||
} else {
|
||||
o->ap_pending = 0;
|
||||
g_ap_fail++;
|
||||
}
|
||||
}
|
||||
ResumeThread(o->hthr);
|
||||
return r;
|
||||
}
|
||||
|
||||
// ---- TOTK_CORE_DIAG=1: why ready threads wait for a virtual core, and for how long
|
||||
// Each core wait is split into classes: the owner asked to give the core up ran game code (DG_GAME, the part an
|
||||
// asynchronous preemption would remove), was in a host call or SVC that kept the core (DG_HOST), every owner the waiter
|
||||
@@ -330,7 +561,7 @@ static void dg_add(const RtThread *w, int cls, uint64_t ns, uint32_t oh, int opr
|
||||
|
||||
// The waiter's class at the start of its wait, after core_kick claimed (or did not claim) an owner for it.
|
||||
static int dg_class0(const RtThread *t) {
|
||||
if (t->dg_target) return dg_in_host(t->dg_target) ? DG_HOST : DG_GAME;
|
||||
if (t->target) return dg_in_host(t->target) ? DG_HOST : DG_GAME;
|
||||
for (int i = 0; i < NCORE; i++)
|
||||
if (core_allowed(t, i) && g_core_owner[i] && g_core_owner[i]->prio > t->prio) return DG_NOCORE;
|
||||
return DG_EQ;
|
||||
@@ -343,8 +574,8 @@ static void dg_done(RtThread *t) {
|
||||
if (!t0) return;
|
||||
const struct DgRel *r = &t->dg_rel;
|
||||
uint64_t part[DG_N] = {0};
|
||||
uint32_t oh = t->dg_target ? t->dg_target->handle : r->h;
|
||||
int oprio = t->dg_target ? t->dg_target->prio : r->prio;
|
||||
uint32_t oh = t->target ? t->target->handle : r->h;
|
||||
int oprio = t->target ? t->target->prio : r->prio;
|
||||
if (r->pre && r->kick) {
|
||||
uint64_t k = r->kick > t0 ? (r->kick < now ? r->kick : now) : t0;
|
||||
part[t->dg_cls] += k - t0;
|
||||
@@ -375,14 +606,14 @@ static void dg_done(RtThread *t) {
|
||||
r->prio, r->pre ? " (preempted)" : "", r->pc ? ldr_describe(r->pc, a, sizeof a) : "-",
|
||||
r->lr ? ldr_describe(r->lr, l, sizeof l) : "-");
|
||||
}
|
||||
t->dg_target = NULL;
|
||||
t->target = NULL;
|
||||
memset(&t->dg_rel, 0, sizeof t->dg_rel);
|
||||
}
|
||||
|
||||
// Once per long DG_GAME wait: where the owner's host thread runs (exe RVA). Holding G; the owner is resumed before
|
||||
// anything that could take a lock it holds.
|
||||
static void dg_sample(RtThread *t) {
|
||||
RtThread *o = t->dg_target;
|
||||
RtThread *o = t->target;
|
||||
if (!o || o->cur_core < 0 || !o->tid || g_dg_samples >= DG_SAMPLES) return;
|
||||
HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, o->tid);
|
||||
if (!h) return;
|
||||
@@ -406,6 +637,17 @@ static void dg_sample(RtThread *t) {
|
||||
|
||||
// perf_present, every present: a line for a present over 25 ms, the 60-present summary.
|
||||
void kern_core_diag_frame(uint64_t frame, uint64_t interval_ns) {
|
||||
if (g_ap > 0 && frame && frame % 600 == 0) { // TOTK_ASYNC_PREEMPT counters since the start
|
||||
lock();
|
||||
uint64_t v[8] = {g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok, g_sync36};
|
||||
unlock();
|
||||
rt_log("kernel: frame %llu: asynchronous preemption: %llu interrupted, %llu cores given up there; refused: "
|
||||
"%llu in a host call, %llu outside game code, %llu preemption disabled, %llu failed; %llu waits ended "
|
||||
"in the first spin; SVC 0x36 %llu",
|
||||
(unsigned long long)frame, (unsigned long long)v[0], (unsigned long long)v[1], (unsigned long long)v[2],
|
||||
(unsigned long long)v[3], (unsigned long long)v[4], (unsigned long long)v[5], (unsigned long long)v[6],
|
||||
(unsigned long long)v[7]);
|
||||
}
|
||||
if (g_diag <= 0) return;
|
||||
DgWindow w;
|
||||
uint64_t f[DG_N], fr[DG_N], s36, s36f;
|
||||
@@ -469,11 +711,16 @@ static struct DgRel g_dg_rel; // TOTK_CORE_DIAG: the owner core_put is releasi
|
||||
|
||||
static void core_grant(int c) {
|
||||
RtThread *b = NULL, **bp = NULL;
|
||||
for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext)
|
||||
if (core_allowed(*p, c) && (!b || (*p)->prio < b->prio || ((*p)->prio == b->prio && (*p)->cseq < b->cseq))) {
|
||||
int bpri = 0;
|
||||
for (RtThread **p = &g_core_wait; *p; p = &(*p)->cnext) {
|
||||
if (!core_allowed(*p, c)) continue;
|
||||
int pr = prio_of(*p);
|
||||
if (!b || pr < bpri || (pr == bpri && (*p)->cseq < b->cseq)) {
|
||||
b = *p;
|
||||
bp = p;
|
||||
bpri = pr;
|
||||
}
|
||||
}
|
||||
if (!b) return;
|
||||
*bp = b->cnext;
|
||||
b->cnext = NULL;
|
||||
@@ -509,8 +756,9 @@ static void core_put(RtThread *t) {
|
||||
|
||||
// Whether a waiter better than t may run on core c.
|
||||
static int core_better_waiter(const RtThread *t, int c) {
|
||||
int tp = prio_of(t);
|
||||
for (const RtThread *w = g_core_wait; w; w = w->cnext)
|
||||
if (w->prio < t->prio && core_allowed(w, c)) return 1;
|
||||
if (core_allowed(w, c) && prio_of(w) < tp) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -520,25 +768,29 @@ static int core_better_waiter(const RtThread *t, int c) {
|
||||
// again (build/v121-dpre/NOTES.md).
|
||||
static void core_kick(void) {
|
||||
RtThread *w[64];
|
||||
int n = 0, claimed = 0;
|
||||
int wp[64], op[NCORE], n = 0, claimed = 0;
|
||||
for (int i = 0; i < NCORE; i++) op[i] = g_core_owner[i] ? prio_of(g_core_owner[i]) : 0;
|
||||
for (RtThread *p = g_core_wait; p && n < 64; p = p->cnext) {
|
||||
int i = n++;
|
||||
for (; i > 0 && (w[i - 1]->prio > p->prio || (w[i - 1]->prio == p->prio && w[i - 1]->cseq > p->cseq)); i--)
|
||||
int i = n++, pp = prio_of(p);
|
||||
for (; i > 0 && (wp[i - 1] > pp || (wp[i - 1] == pp && w[i - 1]->cseq > p->cseq)); i--) {
|
||||
w[i] = w[i - 1];
|
||||
wp[i] = wp[i - 1];
|
||||
}
|
||||
w[i] = p;
|
||||
wp[i] = pp;
|
||||
}
|
||||
for (int k = 0; k < n && claimed != (1 << NCORE) - 1; k++) {
|
||||
int v = -1;
|
||||
for (int i = 0; i < NCORE; i++) {
|
||||
RtThread *o = g_core_owner[i];
|
||||
if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || o->prio <= w[k]->prio) continue;
|
||||
if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || op[i] <= wp[k]) continue;
|
||||
RtThread *b = v < 0 ? NULL : g_core_owner[v];
|
||||
if (!b || o->prio > b->prio || (o->prio == b->prio && o->preempt && !b->preempt)) v = i;
|
||||
if (!b || op[i] > op[v] || (op[i] == op[v] && o->preempt && !b->preempt)) v = i;
|
||||
}
|
||||
if (v < 0) continue;
|
||||
claimed |= 1 << v;
|
||||
RtThread *o = g_core_owner[v];
|
||||
w[k]->dg_target = o;
|
||||
w[k]->target = o;
|
||||
if (o->preempt) continue;
|
||||
if (rt_trace & RT_TRACE_SCHED) {
|
||||
o->pre_since = kern_now_ns();
|
||||
@@ -575,29 +827,34 @@ static void core_take(RtThread *t) {
|
||||
t->cseq = ++g_wseq;
|
||||
t->cnext = g_core_wait;
|
||||
g_core_wait = t;
|
||||
t->target = NULL;
|
||||
if (g_diag > 0) {
|
||||
t->dg_since = kern_now_ns();
|
||||
t->dg_target = NULL;
|
||||
memset(&t->dg_rel, 0, sizeof t->dg_rel);
|
||||
}
|
||||
core_kick();
|
||||
if (g_diag > 0) {
|
||||
t->dg_cls = dg_class0(t);
|
||||
int sampled = 0;
|
||||
while (t->cur_core < 0) {
|
||||
if (sampled || t->dg_cls != DG_GAME) {
|
||||
SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
|
||||
continue;
|
||||
}
|
||||
SleepConditionVariableSRW(&t->cv, &G, 2, 0);
|
||||
if (t->cur_core < 0 && kern_now_ns() - t->dg_since >= 2000000) {
|
||||
sampled = 1;
|
||||
dg_sample(t);
|
||||
}
|
||||
}
|
||||
dg_done(t);
|
||||
if (g_diag > 0) t->dg_cls = dg_class0(t);
|
||||
if (g_ap > 0 && t->target && t->cur_core < 0) { // most waits end within microseconds: spin before interrupting
|
||||
uint64_t end = kern_now_ns() + AP_FIRST_NS;
|
||||
unlock();
|
||||
while (__atomic_load_n(&t->cur_core, __ATOMIC_RELAXED) < 0 && kern_now_ns() < end) YieldProcessor();
|
||||
lock();
|
||||
if (t->cur_core >= 0) g_ap_spin_ok++;
|
||||
}
|
||||
while (t->cur_core < 0) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
|
||||
int sampled = 0;
|
||||
while (t->cur_core < 0) {
|
||||
DWORD ms = INFINITE;
|
||||
if (g_ap > 0 && ap_try(t) != AP_NONE) ms = 1; // the owner keeps its core: again in a millisecond
|
||||
int sample = g_diag > 0 && !sampled && t->dg_cls == DG_GAME;
|
||||
if (sample && ms == INFINITE) ms = 2;
|
||||
SleepConditionVariableSRW(&t->cv, &G, ms, 0);
|
||||
if (sample && t->cur_core < 0 && kern_now_ns() - t->dg_since >= 2000000) {
|
||||
sampled = 1;
|
||||
dg_sample(t);
|
||||
}
|
||||
}
|
||||
if (g_diag > 0) dg_done(t);
|
||||
t->target = NULL;
|
||||
acct(t, S_RUN);
|
||||
}
|
||||
|
||||
@@ -642,7 +899,12 @@ void kern_checkpoint(void) {
|
||||
t->dg_pc = t->cur->pc;
|
||||
t->dg_lr = t->cur->x[30];
|
||||
}
|
||||
if (t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) {
|
||||
int better = t->cur_core >= 0 && core_better_waiter(t, t->cur_core);
|
||||
if (better && ap_disabled(t)) { // preemption disabled: the switch comes at SVC 0x36
|
||||
ap_tlr(t)[1] = 1;
|
||||
t->preempt = 0;
|
||||
g_ap_deferred++;
|
||||
} else if (better) {
|
||||
core_put(t); // clears preempt (after the sched trace reads it)
|
||||
core_take(t);
|
||||
} else {
|
||||
@@ -655,6 +917,35 @@ void kern_checkpoint(void) {
|
||||
unlock();
|
||||
}
|
||||
|
||||
// kern_ap_stub, on the interrupted owner: where to jump back to, then the core given up as at a checkpoint.
|
||||
uint64_t kern_ap_rip(void) { return tcur->ap_rip; }
|
||||
|
||||
void kern_ap_yield(void) {
|
||||
RtThread *t = tcur;
|
||||
lock();
|
||||
t->ap_pending = 0;
|
||||
if (t->preempt && t->cur_core >= 0 && !ap_disabled(t) && core_better_waiter(t, t->cur_core)) {
|
||||
if (g_diag > 0) {
|
||||
t->dg_pc = t->cur->pc;
|
||||
t->dg_lr = t->cur->x[30];
|
||||
}
|
||||
g_ap_taken++;
|
||||
core_put(t);
|
||||
core_take(t);
|
||||
}
|
||||
unlock();
|
||||
}
|
||||
|
||||
// SVC 0x36 SynchronizePreemptionState: a preemption deferred by disable_count happens here.
|
||||
static void ap_sync(RtThread *t) {
|
||||
lock();
|
||||
if (g_ap > 0 && t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) {
|
||||
core_put(t);
|
||||
core_take(t);
|
||||
}
|
||||
unlock();
|
||||
}
|
||||
|
||||
void kern_core_stats(int *held_now, int *held_max) {
|
||||
lock();
|
||||
int n = 0;
|
||||
@@ -876,6 +1167,8 @@ static void thread_finish(RtThread *t) {
|
||||
lock();
|
||||
core_put(t);
|
||||
acct(t, S_OFF);
|
||||
if (t->hthr) CloseHandle(t->hthr); // no waiter interrupts a thread without a core
|
||||
t->hthr = NULL;
|
||||
t->exited = 1;
|
||||
wake_sync_waiters();
|
||||
if (t->closed) thr_free(t);
|
||||
@@ -887,11 +1180,22 @@ static void thread_finish(RtThread *t) {
|
||||
// is offset, and the landing code then reads its locals through the wrong base.)
|
||||
static __attribute__((noreturn)) void exit_now(RtThread *t) { longjmp(t->exit_jmp, 1); }
|
||||
|
||||
// The calling host thread, for ap_try (TOTK_ASYNC_PREEMPT only).
|
||||
static HANDLE ap_open_self(void) {
|
||||
if (!cores_enabled() || g_ap <= 0) return NULL;
|
||||
HANDLE h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_SET_CONTEXT | THREAD_QUERY_INFORMATION,
|
||||
FALSE, GetCurrentThreadId());
|
||||
if (!h) rt_log("kernel: OpenThread for asynchronous preemption failed (error %lu)", GetLastError());
|
||||
return h;
|
||||
}
|
||||
|
||||
static DWORD WINAPI thread_main(void *p) {
|
||||
RtThread *t = p;
|
||||
tcur = t;
|
||||
t->tid = GetCurrentThreadId();
|
||||
HANDLE self = ap_open_self();
|
||||
lock();
|
||||
t->hthr = self;
|
||||
while (!t->started) SleepConditionVariableSRW(&t->cv, &G, INFINITE, 0);
|
||||
core_take(t);
|
||||
unlock();
|
||||
@@ -934,7 +1238,9 @@ uint32_t kern_main_thread(Cpu **cpu) {
|
||||
t->stack_top = stack + MAIN_STACK;
|
||||
t->cpu->sp = t->stack_top;
|
||||
t->tid = GetCurrentThreadId();
|
||||
HANDLE self = ap_open_self();
|
||||
lock();
|
||||
t->hthr = self;
|
||||
thr_link(t);
|
||||
g_main = t;
|
||||
g_main_stack = stack;
|
||||
@@ -1120,8 +1426,9 @@ int kern_svc(Cpu *c, uint32_t imm) {
|
||||
// Yield: hand the core to a waiting thread of equal or better priority.
|
||||
lock();
|
||||
int give = 0;
|
||||
int tp = prio_of(t);
|
||||
for (RtThread *w = g_core_wait; w && !give; w = w->cnext)
|
||||
give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && w->prio <= t->prio;
|
||||
give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && prio_of(w) <= tp;
|
||||
if (give) {
|
||||
core_put(t);
|
||||
core_take(t);
|
||||
@@ -1190,6 +1497,7 @@ int kern_svc(Cpu *c, uint32_t imm) {
|
||||
__atomic_fetch_add(&g_sync36, 1, __ATOMIC_RELAXED);
|
||||
if (*flag) __atomic_fetch_add(&g_sync36_flag, 1, __ATOMIC_RELAXED);
|
||||
*flag = 0;
|
||||
ap_sync(t);
|
||||
x[0] = R_OK;
|
||||
return 1;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user