From 06d70d0dfafb7a8a684877d0f5854f62c0930fce Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Mon, 9 Nov 2026 08:19:31 +0300 Subject: [PATCH] Report host registers, code bytes and memory at a crash The crash report named the lifted function and printed the guest registers, but the lifted code keeps guest registers in host locals and writes them back only at calls, so the dump showed stale values and the faulting instruction stayed unknown. Logs from #1005, #967 and #998 could not be traced further than the function. The report now adds the x86 registers of the faulting thread (Windows and Linux), 64 bytes of machine code around rip with the offset into the lifted function, so the instruction can be found in a local build of the same version, 64 bytes of memory around every register value that points at readable memory, and the first 128 bytes of the guest stack. --probe-kernel checks the new lines with a child that faults with known values in rbx and x19. Co-Authored-By: Claude Opus 5.5 --- runtime/kernel_probe.c | 48 ++++++++++++++++++++ runtime/main.c | 88 ++++++++++++++++++++++++++++++++++--- runtime/platform/linux.c | 4 ++ runtime/platform/platform.h | 4 ++ runtime/platform/win32.c | 10 +++++ 5 files changed, 149 insertions(+), 5 deletions(-) diff --git a/runtime/kernel_probe.c b/runtime/kernel_probe.c index 191a1c28..6ee32e2b 100644 --- a/runtime/kernel_probe.c +++ b/runtime/kernel_probe.c @@ -719,6 +719,47 @@ static void test_platform(void) { CHECK(over[N / 2] < 500000, "plat_sleep_ns(1 ms) median overshoot %.3f ms (want < 0.5)", over[N / 2] / 1e6); } +// ---- crash report (main.c crash_handler): a child faults on a guest thread with known values in rbx and x19; the +// report must hold the host registers, the faulting instruction's bytes and the memory both values point at. +#define CRASH_CHILD_ENV "TOTK_PROBE_KERNEL_CRASH_CHILD" +static uint64_t crash_host_words[6] = {0, 0, 0x5ca1ab1e0c0de001ull, 0, 0, 0}; +static uint64_t crash_guest_words[6] = {0, 0, 0x5ca1ab1e0c0de002ull, 0, 0, 0}; + +static void crash_child_main(Cpu *c) { + c->x[19] = (uint64_t)(uintptr_t)&crash_guest_words[2]; + c->sp -= 0x100; // a frame on the guest stack, as a crashing function has + __asm__ volatile("movq %0, %%rbx\n\tmovq 0x10, %%rax" : : "r"(&crash_host_words[2]) : "rax", "rbx", "memory"); +} + +// The report line that holds needle, NULL when none does +static const char *report_line(const char *out, const char *needle, char *line, size_t cap) { + const char *p = strstr(out, needle); + if (!p) return NULL; + while (p > out && p[-1] != '\n') p--; + size_t n = strcspn(p, "\r\n"); + snprintf(line, cap, "%.*s", (int)(n < cap ? n : cap - 1), p); + return line; +} + +static void test_crash_report(void) { + static char out[65536]; + char line[512], want[64]; + int code = plat_run_self(CRASH_CHILD_ENV, "1", out, sizeof out, 30000); + CHECK(code == 71 && strstr(out, "FATAL: access violation") && strstr(out, "reading address 0x10"), + "crash child: exit code %d, no access violation at 0x10 reported", code); + snprintf(want, sizeof want, "rbx %016llx", (unsigned long long)(uintptr_t)&crash_host_words[2]); + CHECK(strstr(out, want), "crash report: no host register line with %s", want); + CHECK(report_line(out, "host code at rip-", line, sizeof line) && strstr(line, "| 48 8b 04 25 10 00 00 00"), + "crash report: the faulting instruction's bytes are missing"); + const char *h = report_line(out, "*5ca1ab1e0c0de001", line, sizeof line); + CHECK(h && strstr(h, "rbx"), "crash report: no memory line for rbx (%s)", h ? h : "none"); + const char *g = report_line(out, "*5ca1ab1e0c0de002", line, sizeof line); + CHECK(g && strstr(g, "x19"), "crash report: no memory line for x19 (%s)", g ? g : "none"); + CHECK(strstr(out, "guest stack sp+0x00:"), "crash report: no guest stack words"); + printf("crash report: exit code %d, host registers, code bytes, memory at rbx and x19, guest stack %s\n", code, + failures ? "checked with failures" : "present"); +} + int probe_kernel(void) { a_lock_waiter = hostfn_register("probe-kernel lock waiter", lock_waiter); a_cv_waiter = hostfn_register("probe-kernel condvar waiter", cv_waiter); @@ -742,7 +783,14 @@ int probe_kernel(void) { printf("probe-kernel: FAILED (no PLT stub for %s in main)\n", TICK_SYMBOL); return 1; } + const char *child = getenv(CRASH_CHILD_ENV); + if (child && *child) { + kern_run_on_guest_main(crash_child_main); + printf("crash child: returned without a fault\n"); + return 1; + } test_platform(); + test_crash_report(); kern_run_on_guest_main(probe_main); RtStats s = rt_stats(); printf("stats: %llu SVCs, %llu threads, %llu host calls\n", (unsigned long long)s.svcs, diff --git a/runtime/main.c b/runtime/main.c index cba26ccf..b8ff1132 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -21,7 +21,8 @@ // // Also here: the runtime services the generated code calls besides rt_dispatch and rt_svc (rt_brk, rt_udf, // rt_ticks, the function profiler hook), logging, SHA-256, and the crash reporter (a vectored exception handler that -// names the lifted function, the guest registers and the guest frame-pointer chain). +// names the lifted function, the host and guest registers, the machine code around the fault, the guest frame-pointer +// chain and the memory the registers point at). #include #include #include @@ -188,8 +189,8 @@ static int by_host(const void *a, const void *b) { } // The lifted function that contains a host code address, by the nearest function start below it; 0 outside the lifted -// code. -static uint64_t lifted_function(uintptr_t rip) { +// code. *host, when given, gets the function's host start. +static uint64_t lifted_function_at(uintptr_t rip, uintptr_t *host) { if (!fn_rev) { static RtModule mods[MAX_MODULES]; int n = rt_linked_modules(mods, MAX_MODULES); @@ -211,11 +212,78 @@ static uint64_t lifted_function(uintptr_t rip) { if (fn_rev[mid].host <= rip) lo = mid + 1; else hi = mid; } + if (lo && host) *host = fn_rev[lo - 1].host; return lo ? fn_rev[lo - 1].guest : 0; } +static uint64_t lifted_function(uintptr_t rip) { return lifted_function_at(rip, NULL); } + static int readable(uint64_t a, uint64_t n) { return plat_mem_readable(a, n); } +// The machine code around the faulting instruction: 32 bytes before rip, then 32 from rip. The user's totk.exe is built +// on their machine, so its offsets need not match ours; these bytes and the offset into the lifted function find the +// same instruction in a local build of that version, and with it the guest instruction and the operands' registers. +static void crash_code_bytes(uintptr_t rip) { + uint8_t b[64]; + int before = 32; + if (!plat_mem_read(rip - 32, b, sizeof b)) { + before = 0; // rip near the start of a mapping: the bytes from rip only + if (!plat_mem_read(rip, b + 32, 32)) return; + } + rt_logf(" host code at rip-%d:", before); + for (int i = 32 - before; i < 64; i++) rt_logf("%s%02x", i == 32 ? " | " : " ", b[i]); + rt_log_raw("\n", 1); +} + +// 64 bytes around each register value that points at readable memory, as 8-byte words from 16 bytes below the value, +// the word holding the value marked with *. Host and guest registers that land on the same window share a line. A +// guest pointer is a host address here (the guest address space is mapped one to one), so both kinds are read alike. +// The guest registers come from the Cpu, which the lifted code writes back only at calls and returns, so the host +// registers carry the faulting function's live values. +typedef struct { uint64_t start; char names[56]; uint64_t w[8]; } CrashWin; + +static void crash_win_add(CrashWin *wins, int *n, int cap, const char *name, uint64_t v) { + if (v < 0x10000 || v > 0x00007fffffffffffull) return; // small integers and non-canonical values + uint64_t start = (v & ~7ull) - 16; + for (int i = 0; i < *n; i++) + if (wins[i].start == start) { + size_t len = strlen(wins[i].names); + if (len + strlen(name) + 2 < sizeof wins[i].names) + snprintf(wins[i].names + len, sizeof wins[i].names - len, " %s", name); + return; + } + if (*n == cap || !plat_mem_read(start, wins[*n].w, sizeof wins[*n].w)) return; + wins[*n].start = start; + snprintf(wins[*n].names, sizeof wins[*n].names, "%s", name); + (*n)++; +} + +static void crash_memory(const PlatCrash *x, const Cpu *c) { + static CrashWin wins[64]; // the crash handler runs once (busy flag), so static storage needs no allocation + static const char *const gpr_names[16] = PLAT_GPR_NAMES; + int n = 0; + char name[8]; + if (x->has_regs) + for (int i = 0; i < 16; i++) crash_win_add(wins, &n, 64, gpr_names[i], x->gpr[i]); + if (c) { + for (int i = 0; i < 31; i++) snprintf(name, sizeof name, "x%d", i), crash_win_add(wins, &n, 64, name, c->x[i]); + crash_win_add(wins, &n, 64, "sp", c->sp); + } + if (!n) return; + rt_logf(" memory at the register values (8-byte words from value-16, * the value's word):\n"); + for (int i = 0; i < n; i++) { + rt_logf(" %016llx %-12s", (unsigned long long)wins[i].start, wins[i].names); + for (int k = 0; k < 8; k++) rt_logf(" %s%016llx", k == 2 ? "*" : "", (unsigned long long)wins[i].w[k]); + rt_log_raw("\n", 1); + } + // The guest stack above sp: the crashing function's spills and locals, and the callers' saved registers + uint64_t s[16]; + if (c && plat_mem_read(c->sp, s, sizeof s)) + for (int i = 0; i < 16; i += 4) + rt_logf(" guest stack sp+0x%02x: %016llx %016llx %016llx %016llx\n", i * 8, (unsigned long long)s[i], + (unsigned long long)s[i + 1], (unsigned long long)s[i + 2], (unsigned long long)s[i + 3]); +} + static void crash_handler(const PlatCrash *x) { static volatile int32_t busy; if (plat_xchg32(&busy, 1)) plat_exit(71); @@ -230,8 +298,17 @@ static void crash_handler(const PlatCrash *x) { rt_log_raw("\n", 1); if (x->cxx_type) rt_logf(" C++ type %s%s%s\n", x->cxx_type, x->cxx_what ? ", what(): " : "", x->cxx_what ? x->cxx_what : ""); - uint64_t f = x->module ? 0 : lifted_function(rip); - if (f) rt_logf(" in lifted function f_%llx = %s\n", (unsigned long long)f, ldr_describe(f, d, sizeof d)); + uintptr_t fstart = 0; + uint64_t f = x->module ? 0 : lifted_function_at(rip, &fstart); + if (f) + rt_logf(" in lifted function f_%llx = %s, host +%#llx into it\n", (unsigned long long)f, + ldr_describe(f, d, sizeof d), (unsigned long long)(rip - fstart)); + if (x->has_regs) { + static const char *const gpr_names[16] = PLAT_GPR_NAMES; + for (int i = 0; i < 16; i++) + rt_logf(" %-3s %016llx%s", gpr_names[i], (unsigned long long)x->gpr[i], i % 4 == 3 ? "\n" : ""); + crash_code_bytes(rip); + } // abort and invalid parameter: where the host code called it; a fault in another module: the driver code and the // host calls that led into it uintptr_t exe = plat_exe_base(); @@ -266,6 +343,7 @@ static void crash_handler(const PlatCrash *x) { } else { rt_logf(" not on a guest thread\n"); } + crash_memory(x, c); kern_free_cores_note(); fflush(stderr); plat_exit(71); diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index fcb88d77..24512abb 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -957,6 +957,10 @@ static void fault_handler(int sig, siginfo_t *si, void *uc) { } if (sig == SIGSEGV || sig == SIGBUS) c.access = (err & 16) ? 8 : (err & 2) ? 1 : 0, c.addr = (uint64_t)(uintptr_t)si->si_addr; + static const int gr[16] = {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP, REG_RSI, REG_RDI, + REG_R8, REG_R9, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15}; + for (int i = 0; i < 16; i++) c.gpr[i] = (uint64_t)u->uc_mcontext.gregs[gr[i]]; + c.has_regs = 1; crash_fn(&c); } signal(sig, SIG_DFL); // returns into the faulting instruction, which faults again with the default action diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 78873ced..0ddeaa78 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -249,7 +249,11 @@ typedef struct { const char *module; // the module holding rip: NULL for this executable, "" for none (a jump to address 0) const char *cxx_type; // a C++ exception from MSVC-built code: the thrown type's decorated name (.?AV...@@) const char *cxx_what; // and what() when the type derives from std::exception; NULL when unknown + int has_regs; // gpr holds the faulting thread's registers (a fault; not abort or invalid parameter) + uint64_t gpr[16]; // rax rcx rdx rbx rsp rbp rsi rdi r8..r15, in the x86 encoding order } PlatCrash; +#define PLAT_GPR_NAMES {"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", \ + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"} void plat_crash_hook(void (*fn)(const PlatCrash *c)); int plat_mem_read(uint64_t addr, void *out, size_t n); // copies n bytes, 0 instead of faulting on unmapped memory int plat_screen_size(int *w, int *h); // the primary display's current mode in pixels; 0 unknown diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index 83518517..e753b313 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -736,6 +736,14 @@ void plat_mem_fault_hook(int (*fn)(uint64_t addr)) { static void (*crash_fn)(const PlatCrash *c); +// The faulting thread's general registers, for the report to tie the faulting instruction to its operands. +static void regs_of(const CONTEXT *x, PlatCrash *c) { + const DWORD64 r[16] = {x->Rax, x->Rcx, x->Rdx, x->Rbx, x->Rsp, x->Rbp, x->Rsi, x->Rdi, + x->R8, x->R9, x->R10, x->R11, x->R12, x->R13, x->R14, x->R15}; + for (int i = 0; i < 16; i++) c->gpr[i] = r[i]; + c->has_regs = 1; +} + static LONG WINAPI crash_filter(EXCEPTION_POINTERS *ep); static LONG CALLBACK crash_veh(EXCEPTION_POINTERS *ep) { @@ -758,6 +766,7 @@ static LONG CALLBACK crash_veh(EXCEPTION_POINTERS *ep) { PlatCrash c = {what, (uint32_t)code, rip, rip - (uintptr_t)mi.lpBaseOfDll, -1, 0}; if (code == EXCEPTION_ACCESS_VIOLATION && ep->ExceptionRecord->NumberParameters >= 2) c.access = (int)ep->ExceptionRecord->ExceptionInformation[0], c.addr = ep->ExceptionRecord->ExceptionInformation[1]; + regs_of(ep->ContextRecord, &c); crash_fn(&c); return EXCEPTION_CONTINUE_SEARCH; } @@ -895,6 +904,7 @@ static LONG WINAPI crash_filter(EXCEPTION_POINTERS *ep) { (uint32_t)code, rip, 0, -1, 0}; if (code == EXCEPTION_ACCESS_VIOLATION && er->NumberParameters >= 2) c.access = (int)er->ExceptionInformation[0], c.addr = er->ExceptionInformation[1]; + regs_of(ep->ContextRecord, &c); void *base = NULL; RtlPcToFileHeader((void *)rip, &base); uintptr_t exe = (uintptr_t)GetModuleHandleW(NULL);