Files
totk-pc-recomp/runtime/memory.c
T

568 lines
24 KiB
C

// Adapted from MacNativeGames games/totk/runtime/services.c and loader.c @ b75be8c (MIT, Copyright (c) 2026
// MacNativeGames contributors), and LAS Recomp runtime/main.c and platform/windows.c @ da35b83 (MIT, Copyright (c)
// 2026 Peiikai).
// Guest address space and the memory SVCs: heap, module data permissions, stack moves,
// QueryMemory, physical memory in the alias region, GetInfo memory ids, and the kernel-region allocator.
//
// Every region is a plain reservation; pages are committed and decommitted at 4 KiB granularity. Decommitted pages
// come back zeroed, so no stale-page bookkeeping is needed. The stack and heap block and the alias region are
// reserved with MEM_WRITE_WATCH: mem_written_pages reports the pages the CPU wrote, which
// the Vulkan back end uploads again.
#include <windows.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cfg.h"
#include "host.h"
#include "layout.h"
#define PAGE 0x1000ull
#define GiB (1ull << 30)
#define LOW_SEARCH_FROM 0x100000000ull
#define LOW_SIZE (6 * GiB)
#define STACK_OFF 0x10000000ull
#define STACK_SIZE 0x10000000ull
#define HEAP_OFF (STACK_OFF + STACK_SIZE + 0x10000000ull)
#define HEAP_CAP (4 * GiB)
#define ALIAS_SIZE (64 * GiB)
#define ALIAS_ALIGN (4 * GiB)
#define ALIAS_PAGES (ALIAS_SIZE / PAGE)
#define DEFAULT_POOL 0xCD500000ull // retail application pool
#define SYSTEM_RESOURCE_BUDGET (32ull << 20)
#define MAX_STACK_MAPS 4096
#define REEXEC_LIMIT 20
// Horizon memory states, permissions, attributes and results used here.
enum { ST_FREE = 0, ST_CODE = 3, ST_CODEDATA = 4, ST_NORMAL = 5, ST_STACK = 0xB, ST_THREADLOCAL = 0xC,
ST_INACCESSIBLE = 0x10 };
enum { PERM_NONE = 0, PERM_R = 1, PERM_RW = 3, PERM_RX = 5 };
enum { ATTR_PERMISSION_LOCKED = 16 };
#define R_OK 0u
#define R_INVALID_ADDRESS 0xcc01u // misaligned address
#define R_INVALID_SIZE 0xca01u
#define R_INVALID_NEW_PERM 0xd801u
#define R_INVALID_STATE 0xd401u // InvalidCurrentMemory
#define R_INVALID_REGION 0xdc01u
#define R_LIMIT_REACHED 0x10801u
static RtGuestLayout L;
static uint64_t mods_lo, mods_hi; // fixed module window
static SRWLOCK lock = SRWLOCK_INIT;
static uint64_t heap_size, alias_bytes;
static uint8_t *alias_pages; // 1 per mapped 4 KiB page of the alias region
typedef struct { uint64_t dst, src, size; } StackMap;
static StackMap stack_maps[MAX_STACK_MAPS];
static unsigned nstack_maps;
static uint8_t *module_perm[MOD_COUNT]; // per data page of each module: perm | ATTR_PERMISSION_LOCKED
static uint64_t align_up(uint64_t v, uint64_t a) { return (v + a - 1) & ~(a - 1); }
static void *reserve_at(uint64_t at, uint64_t size, DWORD extra) {
void *p = VirtualAlloc((void *)(uintptr_t)at, size, MEM_RESERVE | extra, PAGE_NOACCESS);
if (p && p != (void *)(uintptr_t)at) {
VirtualFree(p, 0, MEM_RELEASE);
p = NULL;
}
return p;
}
static void commit(uint64_t a, uint64_t n, const char *what) {
if (n && !VirtualAlloc((void *)(uintptr_t)a, n, MEM_COMMIT, PAGE_READWRITE))
rt_fatal("memory: commit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
(unsigned long long)n, GetLastError());
}
static void decommit(uint64_t a, uint64_t n, const char *what) {
if (n && !VirtualFree((void *)(uintptr_t)a, n, MEM_DECOMMIT))
rt_fatal("memory: decommit %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
(unsigned long long)n, GetLastError());
}
static void protect(uint64_t a, uint64_t n, DWORD prot, const char *what) {
DWORD old;
if (n && !VirtualProtect((void *)(uintptr_t)a, n, prot, &old))
rt_fatal("memory: protect %s %#llx+%#llx failed (error %lu)", what, (unsigned long long)a,
(unsigned long long)n, GetLastError());
}
// The process is re-run when a region is taken: Windows places DLLs, heaps and thread stacks at random addresses,
// and a fresh process gets a fresh layout. The parent waits and exits with the child's code.
static void reexec_or_fail(const char *what) {
const char *e = cfg_get("TOTK_REEXEC");
int n = e ? atoi(e) : 0;
if (n >= REEXEC_LIMIT) rt_fatal("memory: %s is not free after %d attempts", what, n);
char buf[16];
snprintf(buf, sizeof buf, "%d", n + 1);
SetEnvironmentVariableA("TOTK_REEXEC", buf);
fflush(stdout);
fflush(stderr);
STARTUPINFOW si = {.cb = sizeof si};
PROCESS_INFORMATION pi;
wchar_t *cmd = _wcsdup(GetCommandLineW()); // CreateProcessW may write its command line
if (!cmd || !CreateProcessW(NULL, cmd, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi))
rt_fatal("memory: %s is not free and re-running the process failed (error %lu)", what, GetLastError());
CloseHandle(pi.hThread);
WaitForSingleObject(pi.hProcess, INFINITE);
DWORD code = 70;
GetExitCodeProcess(pi.hProcess, &code);
ExitProcess(code);
}
void mem_init(void) {
if (L.pool) return;
// Fixed window: modules, kernel region, host-function addresses.
mods_lo = MOD_MAIN_BASE;
mods_hi = align_up(rt_layout[rt_layout_count - 1].base + rt_layout[rt_layout_count - 1].size, 0x10000);
if (mods_hi > RT_KERNEL_BASE) rt_fatal("memory: modules end at %#llx, inside the kernel region",
(unsigned long long)mods_hi);
if (!reserve_at(mods_lo, mods_hi - mods_lo, 0)) reexec_or_fail("the module window");
if (!reserve_at(RT_KERNEL_BASE, RT_KERNEL_SIZE, 0)) reexec_or_fail("the kernel region");
if (!reserve_at(RT_HOSTFN_BASE, RT_HOSTFN_SIZE, 0)) reexec_or_fail("the host-function window");
// Stack and heap block, then the alias region above it, both searched below the module window.
uint64_t low = 0, alias = 0;
for (uint64_t at = LOW_SEARCH_FROM; at + LOW_SIZE <= mods_lo && !low; at += GiB)
if (reserve_at(at, LOW_SIZE, MEM_WRITE_WATCH)) low = at;
if (!low) reexec_or_fail("a 6 GiB stack and heap block below 2^39");
for (uint64_t at = align_up(low + LOW_SIZE, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN)
if (reserve_at(at, ALIAS_SIZE, MEM_WRITE_WATCH)) alias = at;
if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39");
alias_pages = VirtualAlloc(NULL, ALIAS_PAGES, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
if (!alias_pages) rt_fatal("memory: cannot allocate the alias page map");
const char *pool = cfg_get("TOTK_APP_POOL");
L = (RtGuestLayout){
.aslr = {low, RT_ADDRESS_LIMIT - low},
.stack = {low + STACK_OFF, STACK_SIZE},
.heap = {low + HEAP_OFF, HEAP_CAP},
.alias = {alias, ALIAS_SIZE},
.kernel = {RT_KERNEL_BASE, RT_KERNEL_SIZE},
.hostfn = {RT_HOSTFN_BASE, RT_HOSTFN_SIZE},
.pool = pool ? strtoull(pool, NULL, 0) : DEFAULT_POOL,
};
if (!L.pool) rt_fatal("memory: TOTK_APP_POOL is 0");
}
const RtGuestLayout *mem_layout(void) { return &L; }
// Adapted from LAS Recomp runtime/platform/windows.c:40-63 @ da35b83 (MIT, Copyright (c) 2026 Peiikai): GetWriteWatch
// with the reset flag. A range inside one watched reservation (stack and heap block, or alias region) only.
uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
if (!max || !size) return 0;
const RtGuestLayout *g = &L;
int low_block = addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size;
int alias = addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size;
if (!low_block && !alias) return UINT64_MAX;
ULONG_PTR n = (ULONG_PTR)max;
DWORD gran;
_Static_assert(sizeof(PVOID) == sizeof(uint64_t), "page addresses are written straight into a uint64_t array");
if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, (void *)(uintptr_t)addr, (SIZE_T)size, (PVOID *)pages, &n, &gran))
return UINT64_MAX;
return n;
}
static uint64_t image_bytes(void) {
uint64_t n = 0;
for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size;
return align_up(n, PAGE);
}
static int overlaps(uint64_t a, uint64_t n, uint64_t b, uint64_t m) { return a < b + m && b < a + n; }
static int inside(uint64_t a, uint64_t n, RtRange r) { return a >= r.base && n <= r.size && a - r.base <= r.size - n; }
// ---- 0x01 SetHeapSize
uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) {
AcquireSRWLockExclusive(&lock);
if (size > HEAP_CAP || size % 0x200000 || size + image_bytes() > L.pool) {
ReleaseSRWLockExclusive(&lock);
return R_INVALID_STATE;
}
if (size > heap_size) commit(L.heap.base + heap_size, size - heap_size, "heap");
if (size < heap_size) decommit(L.heap.base + size, heap_size - size, "heap");
heap_size = size;
ReleaseSRWLockExclusive(&lock);
if (addr) *addr = L.heap.base;
return R_OK;
}
// ---- module data permissions (0x02, 0x03) and module pages for QueryMemory
static int module_index(uint64_t a) {
for (int i = 0; i < ldr_count(); i++) {
const RtLoaded *m = ldr_module(i);
if (a - m->base < m->size) return i;
}
return -1;
}
static uint8_t *perm_table(int i) { // under lock
if (!module_perm[i]) {
const RtLoaded *m = ldr_module(i);
size_t n = (m->size - m->data_off) / PAGE;
module_perm[i] = malloc(n);
if (!module_perm[i]) rt_fatal("memory: module permission table");
memset(module_perm[i], PERM_RW, n);
}
return module_perm[i];
}
static unsigned module_page_perm(int i, uint64_t off) { // under lock
const RtLoaded *m = ldr_module(i);
if (off < m->data_off) return off < align_up(m->text_end, PAGE) ? PERM_RX : PERM_R;
return module_perm[i] ? module_perm[i][(off - m->data_off) / PAGE] : PERM_RW;
}
// The module whose data segment holds [a, a+n), or -1.
static int module_data(uint64_t a, uint64_t n) {
int i = module_index(a);
if (i < 0) return -1;
const RtLoaded *m = ldr_module(i);
uint64_t off = a - m->base;
return off >= m->data_off && n <= m->size - off ? i : -1;
}
uint32_t mem_set_permission(uint64_t a, uint64_t n, uint32_t perm) {
if (a & (PAGE - 1)) return R_INVALID_ADDRESS;
if (!n || (n & (PAGE - 1))) return R_INVALID_SIZE;
if (perm != PERM_NONE && perm != PERM_R && perm != PERM_RW) return R_INVALID_NEW_PERM;
int i = module_data(a, n);
if (i < 0) return R_INVALID_STATE;
const RtLoaded *m = ldr_module(i);
AcquireSRWLockExclusive(&lock);
uint8_t *t = perm_table(i) + (a - m->base - m->data_off) / PAGE;
for (uint64_t p = 0; p < n / PAGE; p++)
if (t[p] & ATTR_PERMISSION_LOCKED) {
ReleaseSRWLockExclusive(&lock);
return R_INVALID_STATE;
}
memset(t, (int)perm, n / PAGE);
protect(a, n, perm == PERM_RW ? PAGE_READWRITE : perm == PERM_R ? PAGE_READONLY : PAGE_NOACCESS, "module data");
ReleaseSRWLockExclusive(&lock);
rt_log("%s data permission +%#llx size %#llx perm %u", m->name, (unsigned long long)(a - m->base),
(unsigned long long)n, perm);
return R_OK;
}
// The only module attribute the SDK sets is the irreversible RELRO lock.
uint32_t mem_set_attribute(uint64_t a, uint64_t n, uint32_t mask, uint32_t at) {
if (a & (PAGE - 1)) return R_INVALID_ADDRESS;
if (!n || (n & (PAGE - 1))) return R_INVALID_SIZE;
if (mask != ATTR_PERMISSION_LOCKED || at != ATTR_PERMISSION_LOCKED) return R_INVALID_NEW_PERM;
int i = module_data(a, n);
if (i < 0) return R_INVALID_STATE;
const RtLoaded *m = ldr_module(i);
AcquireSRWLockExclusive(&lock);
uint8_t *t = perm_table(i) + (a - m->base - m->data_off) / PAGE;
for (uint64_t p = 0; p < n / PAGE; p++) t[p] |= ATTR_PERMISSION_LOCKED;
ReleaseSRWLockExclusive(&lock);
return R_OK;
}
// ---- 0x04/0x05 MapMemory, UnmapMemory: thread stacks move from heap or alias into the stack region by copy; the
// source is inaccessible while moved and gets the contents back on unmap.
static int alias_all_mapped(uint64_t a, uint64_t n) { // under lock
uint64_t first = (a - L.alias.base) / PAGE;
for (uint64_t i = 0; i < n / PAGE; i++)
if (!alias_pages[first + i]) return 0;
return 1;
}
static uint32_t stack_move(int unmap, uint64_t dst, uint64_t src, uint64_t n) {
if (!n || ((dst | src | n) & (PAGE - 1))) return R_INVALID_ADDRESS;
if (!inside(dst, n, L.stack)) return R_INVALID_REGION;
int in_heap = inside(src, n, (RtRange){L.heap.base, heap_size});
int in_alias = inside(src, n, L.alias);
if (!in_heap && !in_alias) return R_INVALID_REGION;
AcquireSRWLockExclusive(&lock);
uint32_t r = R_OK;
if (in_alias && !alias_all_mapped(src, n)) {
r = R_INVALID_REGION;
goto out;
}
unsigned found = nstack_maps;
for (unsigned i = 0; i < nstack_maps; i++)
if (stack_maps[i].dst == dst && stack_maps[i].src == src && stack_maps[i].size == n) found = i;
if (unmap) {
if (found == nstack_maps) {
r = R_INVALID_STATE;
goto out;
}
protect(src, n, PAGE_READWRITE, "moved stack source");
memcpy((void *)(uintptr_t)src, (const void *)(uintptr_t)dst, n);
decommit(dst, n, "moved stack");
stack_maps[found] = stack_maps[--nstack_maps];
} else {
if (nstack_maps == MAX_STACK_MAPS) {
r = R_INVALID_STATE;
goto out;
}
for (unsigned i = 0; i < nstack_maps; i++)
if (overlaps(dst, n, stack_maps[i].dst, stack_maps[i].size) ||
overlaps(src, n, stack_maps[i].src, stack_maps[i].size)) {
r = R_INVALID_REGION;
goto out;
}
commit(dst, n, "moved stack");
memcpy((void *)(uintptr_t)dst, (const void *)(uintptr_t)src, n);
protect(src, n, PAGE_NOACCESS, "moved stack source");
stack_maps[nstack_maps++] = (StackMap){dst, src, n};
}
out:
ReleaseSRWLockExclusive(&lock);
return r;
}
uint32_t mem_map(uint64_t dst, uint64_t src, uint64_t n) { return stack_move(0, dst, src, n); }
uint32_t mem_unmap(uint64_t dst, uint64_t src, uint64_t n) { return stack_move(1, dst, src, n); }
// ---- 0x2C/0x2D MapPhysicalMemory, UnmapPhysicalMemory. Horizon maps only the still-unmapped pages of the range and
// leaves mapped pages and their contents alone; TotK's heaps grow by re-mapping ranges they partly own.
static void alias_runs(uint64_t first, uint64_t count, uint8_t want, int do_commit) { // under lock
uint64_t i = 0;
while (i < count) {
if (alias_pages[first + i] != want) { i++; continue; }
uint64_t j = i;
while (j < count && alias_pages[first + j] == want) j++;
uint64_t a = L.alias.base + (first + i) * PAGE, n = (j - i) * PAGE;
if (do_commit) commit(a, n, "physical memory");
else decommit(a, n, "physical memory");
memset(alias_pages + first + i, !want, j - i);
i = j;
}
}
static uint32_t alias_map(int unmap, uint64_t a, uint64_t n) {
if (!n || ((a | n) & (PAGE - 1))) return R_INVALID_ADDRESS;
if (!inside(a, n, L.alias)) return R_INVALID_REGION;
uint64_t first = (a - L.alias.base) / PAGE, count = n / PAGE;
AcquireSRWLockExclusive(&lock);
uint32_t r = R_OK;
if (unmap) {
for (unsigned i = 0; i < nstack_maps; i++)
if (overlaps(a, n, stack_maps[i].src, stack_maps[i].size)) {
r = R_INVALID_REGION;
goto out;
}
uint64_t released = 0;
for (uint64_t i = 0; i < count; i++) released += alias_pages[first + i];
alias_runs(first, count, 1, 0);
alias_bytes -= released * PAGE;
} else {
uint64_t fresh = 0;
for (uint64_t i = 0; i < count; i++) fresh += !alias_pages[first + i];
if (fresh * PAGE + alias_bytes + heap_size + image_bytes() > L.pool) {
r = R_LIMIT_REACHED; // the SDK maps LimitReached to its recoverable out-of-memory result
goto out;
}
alias_runs(first, count, 0, 1);
alias_bytes += fresh * PAGE;
}
out:
ReleaseSRWLockExclusive(&lock);
if (r) rt_log("%sMapPhysicalMemory %#llx+%#llx failed %#x (alias %#llx used, heap %#llx)", unmap ? "Un" : "",
(unsigned long long)a, (unsigned long long)n, r, (unsigned long long)alias_bytes,
(unsigned long long)heap_size);
return r;
}
uint32_t mem_map_physical(uint64_t a, uint64_t n) { return alias_map(0, a, n); }
uint32_t mem_unmap_physical(uint64_t a, uint64_t n) { return alias_map(1, a, n); }
// ---- 0x06 QueryMemory
// The region around a that no known region covers: everything else below 2^39 belongs to the host and is reported
// as taken (Code, RX), so the SDK never picks it.
static void taken_gap(uint64_t a, uint64_t *base, uint64_t *size) {
uint64_t lo = 0, hi = RT_ADDRESS_LIMIT;
RtRange known[3 + MOD_COUNT + 2];
int n = 0;
known[n++] = (RtRange){L.aslr.base, LOW_SIZE};
known[n++] = L.alias;
known[n++] = L.kernel;
known[n++] = L.hostfn;
for (int i = 0; i < ldr_count(); i++) known[n++] = (RtRange){ldr_module(i)->base, ldr_module(i)->size};
for (int i = 0; i < n; i++) {
uint64_t e = known[i].base + known[i].size;
if (e <= a && e > lo) lo = e;
if (known[i].base > a && known[i].base < hi) hi = known[i].base;
}
*base = lo;
*size = hi - lo;
}
static uint64_t run_end(const uint8_t *p, uint64_t i, uint64_t n, uint8_t v) {
while (i < n && (i & 7)) { if (p[i] != v) return i; i++; }
uint64_t w = v ? 0x0101010101010101ull : 0;
while (i + 8 <= n && *(const uint64_t *)(p + i) == w) i += 8;
while (i < n && p[i] == v) i++;
return i;
}
static uint64_t run_start(const uint8_t *p, uint64_t i, uint8_t v) { // first index of the run that ends at i
while (i > 0 && (i & 7)) { if (p[i - 1] != v) return i; i--; }
uint64_t w = v ? 0x0101010101010101ull : 0;
while (i >= 8 && *(const uint64_t *)(p + i - 8) == w) i -= 8;
while (i > 0 && p[i - 1] == v) i--;
return i;
}
uint32_t mem_query(uint64_t a, RtMemoryInfo *out) {
RtMemoryInfo r = {.base = a & ~(PAGE - 1), .size = PAGE};
uint64_t low = L.aslr.base;
AcquireSRWLockShared(&lock);
int mi;
if (a >= RT_ADDRESS_LIMIT) {
r.base = RT_ADDRESS_LIMIT;
r.size = 0 - RT_ADDRESS_LIMIT;
r.state = ST_INACCESSIBLE;
} else if (a < low) {
r.base = 0;
r.size = low;
r.state = ST_FREE;
} else if (a < low + LOW_SIZE) {
uint64_t heap = L.heap.base;
if (a >= heap && a < heap + heap_size) {
r = (RtMemoryInfo){.base = heap, .size = heap_size, .state = ST_NORMAL, .perm = PERM_RW};
} else if (a < heap) {
r = (RtMemoryInfo){.base = low, .size = heap - low, .state = ST_FREE};
} else {
r = (RtMemoryInfo){.base = heap + heap_size, .size = low + LOW_SIZE - heap - heap_size, .state = ST_FREE};
}
} else if (a - L.alias.base < L.alias.size) {
uint64_t p = (a - L.alias.base) / PAGE;
uint8_t v = alias_pages[p];
uint64_t s = run_start(alias_pages, p, v), e = run_end(alias_pages, p, ALIAS_PAGES, v);
r = (RtMemoryInfo){.base = L.alias.base + s * PAGE, .size = (e - s) * PAGE, .state = v ? ST_NORMAL : ST_FREE,
.perm = v ? PERM_RW : PERM_NONE};
} else if ((mi = module_index(a)) >= 0) {
const RtLoaded *m = ldr_module(mi);
uint64_t off = (a - m->base) & ~(PAGE - 1), lo = off, hi = off + PAGE;
unsigned perm = module_page_perm(mi, off);
int data = off >= m->data_off;
uint64_t limit = data ? m->data_off : 0, end = data ? m->size : m->data_off;
while (lo > limit && module_page_perm(mi, lo - PAGE) == perm) lo -= PAGE;
while (hi < end && module_page_perm(mi, hi) == perm) hi += PAGE;
r = (RtMemoryInfo){.base = m->base + lo, .size = hi - lo, .state = data ? ST_CODEDATA : ST_CODE,
.perm = perm & 7, .attr = perm & ATTR_PERMISSION_LOCKED};
} else if (a - L.kernel.base < L.kernel.size) {
r = (RtMemoryInfo){.base = L.kernel.base, .size = L.kernel.size, .state = ST_THREADLOCAL, .perm = PERM_RW};
} else if (a - L.hostfn.base < L.hostfn.size) {
r = (RtMemoryInfo){.base = L.hostfn.base, .size = L.hostfn.size, .state = ST_INACCESSIBLE};
} else {
taken_gap(a, &r.base, &r.size);
r.state = ST_CODE;
r.perm = PERM_RX;
}
// Moved stacks: the destination is Stack, the source Normal without access; free and normal regions around them
// are clipped (MNG services.c:357-363).
for (unsigned i = 0; i < nstack_maps; i++) {
const StackMap *m = &stack_maps[i];
uint64_t end = m->dst + m->size, send = m->src + m->size;
if (a >= m->dst && a < end) {
r = (RtMemoryInfo){.base = m->dst, .size = m->size, .state = ST_STACK, .perm = PERM_RW};
break;
}
if (a >= m->src && a < send) {
r = (RtMemoryInfo){.base = m->src, .size = m->size, .state = ST_NORMAL, .perm = PERM_NONE, .attr = 4};
break;
}
uint64_t lo = r.base, hi = r.base + r.size;
uint64_t b = r.state == ST_FREE ? m->dst : r.state == ST_NORMAL ? m->src : 0;
uint64_t e = b + m->size;
if (!b) continue;
if (e <= a && e > lo) lo = e;
if (b > a && b < hi) hi = b;
r.base = lo;
r.size = hi - lo;
}
ReleaseSRWLockShared(&lock);
if (out) *out = r;
return R_OK;
}
// ---- GetInfo memory ids
int mem_info(uint32_t id, uint64_t *out) {
uint64_t v;
switch (id) {
case 2: v = L.alias.base; break;
case 3: v = L.alias.size; break;
case 4: v = L.heap.base; break;
case 5: v = L.heap.size; break;
case 6: case 21: v = L.pool; break;
case 7: case 22:
AcquireSRWLockShared(&lock);
v = image_bytes() + heap_size + alias_bytes;
ReleaseSRWLockShared(&lock);
break;
case 12: v = L.aslr.base; break;
case 13: v = L.aslr.size; break;
case 14: v = L.stack.base; break;
case 15: v = L.stack.size; break;
case 16: v = SYSTEM_RESOURCE_BUDGET; break;
case 17: v = sizeof stack_maps + ALIAS_PAGES; break; // system resource in use: our bookkeeping
case 28: v = 0; break; // AliasRegionExtraSize: no kernel-owned extension
default: return 0;
}
*out = v;
return 1;
}
// ---- kernel region: power-of-two blocks, a free list per size, bump allocation with lazy commit.
#define KCLASSES 29 // 64 B .. 256 MiB
#define KMIN_SHIFT 6
#define KCOMMIT 0x10000ull
typedef struct KFree { struct KFree *next; } KFree;
static KFree *kfree_list[KCLASSES];
static uint64_t kbump, kcommitted;
static SRWLOCK klock = SRWLOCK_INIT;
static int kclass(uint64_t size) {
int s = KMIN_SHIFT;
while ((1ull << s) < size) s++;
return s - KMIN_SHIFT;
}
void *mem_kernel_alloc(uint64_t size, uint64_t align) {
if (!L.pool) rt_fatal("mem_kernel_alloc before mem_init");
if (align > KCOMMIT || (align & (align - 1))) rt_fatal("mem_kernel_alloc: alignment %#llx", (unsigned long long)align);
uint64_t want = size > align ? size : align;
int k = kclass(want ? want : 1);
if (k >= KCLASSES) rt_fatal("mem_kernel_alloc: %#llx bytes", (unsigned long long)size);
uint64_t bytes = 1ull << (k + KMIN_SHIFT), al = bytes < KCOMMIT ? bytes : KCOMMIT;
AcquireSRWLockExclusive(&klock);
KFree *f = kfree_list[k];
void *p;
if (f) {
kfree_list[k] = f->next;
p = f;
} else {
uint64_t a = align_up(kbump ? kbump : RT_KERNEL_BASE, al);
if (a + bytes > RT_KERNEL_BASE + RT_KERNEL_SIZE) rt_fatal("kernel region exhausted (%#llx bytes)",
(unsigned long long)bytes);
uint64_t need = align_up(a + bytes, KCOMMIT);
if (!kcommitted) kcommitted = RT_KERNEL_BASE;
if (need > kcommitted) {
commit(kcommitted, need - kcommitted, "kernel region");
kcommitted = need;
}
kbump = a + bytes;
p = (void *)(uintptr_t)a;
}
ReleaseSRWLockExclusive(&klock);
memset(p, 0, size ? size : 1);
return p;
}
void mem_kernel_free(void *p, uint64_t size) {
if (!p) return;
uint64_t a = (uintptr_t)p;
if (a - RT_KERNEL_BASE >= RT_KERNEL_SIZE) rt_fatal("mem_kernel_free(%p): not in the kernel region", p);
int k = kclass(size ? size : 1);
// A block allocated with a larger alignment sits in a larger class; freeing it into the smaller class only
// wastes the rest.
AcquireSRWLockExclusive(&klock);
KFree *f = p;
f->next = kfree_list[k];
kfree_list[k] = f;
ReleaseSRWLockExclusive(&klock);
}