diff --git a/runtime/kernel_probe.c b/runtime/kernel_probe.c index 191a1c28..6ee32e2b 100644 --- a/runtime/kernel_probe.c +++ b/runtime/kernel_probe.c @@ -719,6 +719,47 @@ static void test_platform(void) { CHECK(over[N / 2] < 500000, "plat_sleep_ns(1 ms) median overshoot %.3f ms (want < 0.5)", over[N / 2] / 1e6); } +// ---- crash report (main.c crash_handler): a child faults on a guest thread with known values in rbx and x19; the +// report must hold the host registers, the faulting instruction's bytes and the memory both values point at. +#define CRASH_CHILD_ENV "TOTK_PROBE_KERNEL_CRASH_CHILD" +static uint64_t crash_host_words[6] = {0, 0, 0x5ca1ab1e0c0de001ull, 0, 0, 0}; +static uint64_t crash_guest_words[6] = {0, 0, 0x5ca1ab1e0c0de002ull, 0, 0, 0}; + +static void crash_child_main(Cpu *c) { + c->x[19] = (uint64_t)(uintptr_t)&crash_guest_words[2]; + c->sp -= 0x100; // a frame on the guest stack, as a crashing function has + __asm__ volatile("movq %0, %%rbx\n\tmovq 0x10, %%rax" : : "r"(&crash_host_words[2]) : "rax", "rbx", "memory"); +} + +// The report line that holds needle, NULL when none does +static const char *report_line(const char *out, const char *needle, char *line, size_t cap) { + const char *p = strstr(out, needle); + if (!p) return NULL; + while (p > out && p[-1] != '\n') p--; + size_t n = strcspn(p, "\r\n"); + snprintf(line, cap, "%.*s", (int)(n < cap ? n : cap - 1), p); + return line; +} + +static void test_crash_report(void) { + static char out[65536]; + char line[512], want[64]; + int code = plat_run_self(CRASH_CHILD_ENV, "1", out, sizeof out, 30000); + CHECK(code == 71 && strstr(out, "FATAL: access violation") && strstr(out, "reading address 0x10"), + "crash child: exit code %d, no access violation at 0x10 reported", code); + snprintf(want, sizeof want, "rbx %016llx", (unsigned long long)(uintptr_t)&crash_host_words[2]); + CHECK(strstr(out, want), "crash report: no host register line with %s", want); + CHECK(report_line(out, "host code at rip-", line, sizeof line) && strstr(line, "| 48 8b 04 25 10 00 00 00"), + "crash report: the faulting instruction's bytes are missing"); + const char *h = report_line(out, "*5ca1ab1e0c0de001", line, sizeof line); + CHECK(h && strstr(h, "rbx"), "crash report: no memory line for rbx (%s)", h ? h : "none"); + const char *g = report_line(out, "*5ca1ab1e0c0de002", line, sizeof line); + CHECK(g && strstr(g, "x19"), "crash report: no memory line for x19 (%s)", g ? g : "none"); + CHECK(strstr(out, "guest stack sp+0x00:"), "crash report: no guest stack words"); + printf("crash report: exit code %d, host registers, code bytes, memory at rbx and x19, guest stack %s\n", code, + failures ? "checked with failures" : "present"); +} + int probe_kernel(void) { a_lock_waiter = hostfn_register("probe-kernel lock waiter", lock_waiter); a_cv_waiter = hostfn_register("probe-kernel condvar waiter", cv_waiter); @@ -742,7 +783,14 @@ int probe_kernel(void) { printf("probe-kernel: FAILED (no PLT stub for %s in main)\n", TICK_SYMBOL); return 1; } + const char *child = getenv(CRASH_CHILD_ENV); + if (child && *child) { + kern_run_on_guest_main(crash_child_main); + printf("crash child: returned without a fault\n"); + return 1; + } test_platform(); + test_crash_report(); kern_run_on_guest_main(probe_main); RtStats s = rt_stats(); printf("stats: %llu SVCs, %llu threads, %llu host calls\n", (unsigned long long)s.svcs, diff --git a/runtime/main.c b/runtime/main.c index cba26ccf..b8ff1132 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -21,7 +21,8 @@ // // Also here: the runtime services the generated code calls besides rt_dispatch and rt_svc (rt_brk, rt_udf, // rt_ticks, the function profiler hook), logging, SHA-256, and the crash reporter (a vectored exception handler that -// names the lifted function, the guest registers and the guest frame-pointer chain). +// names the lifted function, the host and guest registers, the machine code around the fault, the guest frame-pointer +// chain and the memory the registers point at). #include #include #include @@ -188,8 +189,8 @@ static int by_host(const void *a, const void *b) { } // The lifted function that contains a host code address, by the nearest function start below it; 0 outside the lifted -// code. -static uint64_t lifted_function(uintptr_t rip) { +// code. *host, when given, gets the function's host start. +static uint64_t lifted_function_at(uintptr_t rip, uintptr_t *host) { if (!fn_rev) { static RtModule mods[MAX_MODULES]; int n = rt_linked_modules(mods, MAX_MODULES); @@ -211,11 +212,78 @@ static uint64_t lifted_function(uintptr_t rip) { if (fn_rev[mid].host <= rip) lo = mid + 1; else hi = mid; } + if (lo && host) *host = fn_rev[lo - 1].host; return lo ? fn_rev[lo - 1].guest : 0; } +static uint64_t lifted_function(uintptr_t rip) { return lifted_function_at(rip, NULL); } + static int readable(uint64_t a, uint64_t n) { return plat_mem_readable(a, n); } +// The machine code around the faulting instruction: 32 bytes before rip, then 32 from rip. The user's totk.exe is built +// on their machine, so its offsets need not match ours; these bytes and the offset into the lifted function find the +// same instruction in a local build of that version, and with it the guest instruction and the operands' registers. +static void crash_code_bytes(uintptr_t rip) { + uint8_t b[64]; + int before = 32; + if (!plat_mem_read(rip - 32, b, sizeof b)) { + before = 0; // rip near the start of a mapping: the bytes from rip only + if (!plat_mem_read(rip, b + 32, 32)) return; + } + rt_logf(" host code at rip-%d:", before); + for (int i = 32 - before; i < 64; i++) rt_logf("%s%02x", i == 32 ? " | " : " ", b[i]); + rt_log_raw("\n", 1); +} + +// 64 bytes around each register value that points at readable memory, as 8-byte words from 16 bytes below the value, +// the word holding the value marked with *. Host and guest registers that land on the same window share a line. A +// guest pointer is a host address here (the guest address space is mapped one to one), so both kinds are read alike. +// The guest registers come from the Cpu, which the lifted code writes back only at calls and returns, so the host +// registers carry the faulting function's live values. +typedef struct { uint64_t start; char names[56]; uint64_t w[8]; } CrashWin; + +static void crash_win_add(CrashWin *wins, int *n, int cap, const char *name, uint64_t v) { + if (v < 0x10000 || v > 0x00007fffffffffffull) return; // small integers and non-canonical values + uint64_t start = (v & ~7ull) - 16; + for (int i = 0; i < *n; i++) + if (wins[i].start == start) { + size_t len = strlen(wins[i].names); + if (len + strlen(name) + 2 < sizeof wins[i].names) + snprintf(wins[i].names + len, sizeof wins[i].names - len, " %s", name); + return; + } + if (*n == cap || !plat_mem_read(start, wins[*n].w, sizeof wins[*n].w)) return; + wins[*n].start = start; + snprintf(wins[*n].names, sizeof wins[*n].names, "%s", name); + (*n)++; +} + +static void crash_memory(const PlatCrash *x, const Cpu *c) { + static CrashWin wins[64]; // the crash handler runs once (busy flag), so static storage needs no allocation + static const char *const gpr_names[16] = PLAT_GPR_NAMES; + int n = 0; + char name[8]; + if (x->has_regs) + for (int i = 0; i < 16; i++) crash_win_add(wins, &n, 64, gpr_names[i], x->gpr[i]); + if (c) { + for (int i = 0; i < 31; i++) snprintf(name, sizeof name, "x%d", i), crash_win_add(wins, &n, 64, name, c->x[i]); + crash_win_add(wins, &n, 64, "sp", c->sp); + } + if (!n) return; + rt_logf(" memory at the register values (8-byte words from value-16, * the value's word):\n"); + for (int i = 0; i < n; i++) { + rt_logf(" %016llx %-12s", (unsigned long long)wins[i].start, wins[i].names); + for (int k = 0; k < 8; k++) rt_logf(" %s%016llx", k == 2 ? "*" : "", (unsigned long long)wins[i].w[k]); + rt_log_raw("\n", 1); + } + // The guest stack above sp: the crashing function's spills and locals, and the callers' saved registers + uint64_t s[16]; + if (c && plat_mem_read(c->sp, s, sizeof s)) + for (int i = 0; i < 16; i += 4) + rt_logf(" guest stack sp+0x%02x: %016llx %016llx %016llx %016llx\n", i * 8, (unsigned long long)s[i], + (unsigned long long)s[i + 1], (unsigned long long)s[i + 2], (unsigned long long)s[i + 3]); +} + static void crash_handler(const PlatCrash *x) { static volatile int32_t busy; if (plat_xchg32(&busy, 1)) plat_exit(71); @@ -230,8 +298,17 @@ static void crash_handler(const PlatCrash *x) { rt_log_raw("\n", 1); if (x->cxx_type) rt_logf(" C++ type %s%s%s\n", x->cxx_type, x->cxx_what ? ", what(): " : "", x->cxx_what ? x->cxx_what : ""); - uint64_t f = x->module ? 0 : lifted_function(rip); - if (f) rt_logf(" in lifted function f_%llx = %s\n", (unsigned long long)f, ldr_describe(f, d, sizeof d)); + uintptr_t fstart = 0; + uint64_t f = x->module ? 0 : lifted_function_at(rip, &fstart); + if (f) + rt_logf(" in lifted function f_%llx = %s, host +%#llx into it\n", (unsigned long long)f, + ldr_describe(f, d, sizeof d), (unsigned long long)(rip - fstart)); + if (x->has_regs) { + static const char *const gpr_names[16] = PLAT_GPR_NAMES; + for (int i = 0; i < 16; i++) + rt_logf(" %-3s %016llx%s", gpr_names[i], (unsigned long long)x->gpr[i], i % 4 == 3 ? "\n" : ""); + crash_code_bytes(rip); + } // abort and invalid parameter: where the host code called it; a fault in another module: the driver code and the // host calls that led into it uintptr_t exe = plat_exe_base(); @@ -266,6 +343,7 @@ static void crash_handler(const PlatCrash *x) { } else { rt_logf(" not on a guest thread\n"); } + crash_memory(x, c); kern_free_cores_note(); fflush(stderr); plat_exit(71); diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index fcb88d77..24512abb 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -957,6 +957,10 @@ static void fault_handler(int sig, siginfo_t *si, void *uc) { } if (sig == SIGSEGV || sig == SIGBUS) c.access = (err & 16) ? 8 : (err & 2) ? 1 : 0, c.addr = (uint64_t)(uintptr_t)si->si_addr; + static const int gr[16] = {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP, REG_RSI, REG_RDI, + REG_R8, REG_R9, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15}; + for (int i = 0; i < 16; i++) c.gpr[i] = (uint64_t)u->uc_mcontext.gregs[gr[i]]; + c.has_regs = 1; crash_fn(&c); } signal(sig, SIG_DFL); // returns into the faulting instruction, which faults again with the default action diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 78873ced..0ddeaa78 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -249,7 +249,11 @@ typedef struct { const char *module; // the module holding rip: NULL for this executable, "" for none (a jump to address 0) const char *cxx_type; // a C++ exception from MSVC-built code: the thrown type's decorated name (.?AV...@@) const char *cxx_what; // and what() when the type derives from std::exception; NULL when unknown + int has_regs; // gpr holds the faulting thread's registers (a fault; not abort or invalid parameter) + uint64_t gpr[16]; // rax rcx rdx rbx rsp rbp rsi rdi r8..r15, in the x86 encoding order } PlatCrash; +#define PLAT_GPR_NAMES {"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", \ + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"} void plat_crash_hook(void (*fn)(const PlatCrash *c)); int plat_mem_read(uint64_t addr, void *out, size_t n); // copies n bytes, 0 instead of faulting on unmapped memory int plat_screen_size(int *w, int *h); // the primary display's current mode in pixels; 0 unknown diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index 83518517..e753b313 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -736,6 +736,14 @@ void plat_mem_fault_hook(int (*fn)(uint64_t addr)) { static void (*crash_fn)(const PlatCrash *c); +// The faulting thread's general registers, for the report to tie the faulting instruction to its operands. +static void regs_of(const CONTEXT *x, PlatCrash *c) { + const DWORD64 r[16] = {x->Rax, x->Rcx, x->Rdx, x->Rbx, x->Rsp, x->Rbp, x->Rsi, x->Rdi, + x->R8, x->R9, x->R10, x->R11, x->R12, x->R13, x->R14, x->R15}; + for (int i = 0; i < 16; i++) c->gpr[i] = r[i]; + c->has_regs = 1; +} + static LONG WINAPI crash_filter(EXCEPTION_POINTERS *ep); static LONG CALLBACK crash_veh(EXCEPTION_POINTERS *ep) { @@ -758,6 +766,7 @@ static LONG CALLBACK crash_veh(EXCEPTION_POINTERS *ep) { PlatCrash c = {what, (uint32_t)code, rip, rip - (uintptr_t)mi.lpBaseOfDll, -1, 0}; if (code == EXCEPTION_ACCESS_VIOLATION && ep->ExceptionRecord->NumberParameters >= 2) c.access = (int)ep->ExceptionRecord->ExceptionInformation[0], c.addr = ep->ExceptionRecord->ExceptionInformation[1]; + regs_of(ep->ContextRecord, &c); crash_fn(&c); return EXCEPTION_CONTINUE_SEARCH; } @@ -895,6 +904,7 @@ static LONG WINAPI crash_filter(EXCEPTION_POINTERS *ep) { (uint32_t)code, rip, 0, -1, 0}; if (code == EXCEPTION_ACCESS_VIOLATION && er->NumberParameters >= 2) c.access = (int)er->ExceptionInformation[0], c.addr = er->ExceptionInformation[1]; + regs_of(ep->ContextRecord, &c); void *base = NULL; RtlPcToFileHeader((void *)rip, &base); uintptr_t exe = (uintptr_t)GetModuleHandleW(NULL);