diff --git a/docs/settings.md b/docs/settings.md index 20a51ef2..9dcf5748 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -77,6 +77,7 @@ Many rows below are developer tools that only act together with another setting | `probe_movie_bench` | `1` makes the movie probe also time the resampler that fits a movie to a game resolution of another shape, and print the time per frame. | Only used by the movie decoder probe. | Safe | | `swkbd_answer` | ``, `` or ``: answers the on-screen keyboard (for example a horse name) without opening a window. | The game gets that answer every time it asks for text. | Caution | | `swkbd_test` | `ok` or `cancel` opens the keyboard window and presses OK or Cancel on the first timer tick. | The keyboard window closes by itself. | Caution | +| `test_oom` | `:[,...]` fails the first n allocations at a site: `commit` (guest memory), `staging` (upload buffers), `image` (texture images), `chunk` (draw data buffers). | Each forced failure takes the out-of-memory path: a memory release, a wait of 10 ms per retry and log lines; a large n stalls up to 30 s and then ends the game. | Caution | | `test_pad` | `,,,` attaches a virtual gamepad whose sticks rest at these raw SDL axis values (-32768 to 32767). | The game sees a controller with that drift or deflection; a malformed value is logged and ignored. | Caution | | `test_prepare_shaders` | `1` translates every recorded shader on all usable CPUs during start. | Start takes longer and fills the shader cache folder; the CPU is fully used while it runs. | Caution | diff --git a/runtime/cfg.c b/runtime/cfg.c index 2d79b5fc..08b57739 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -59,7 +59,7 @@ static const char *const known[] = { "SHADER_MEMO", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHADOW_SNAP", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UB_OVERLAY_OUT", "UPSCALER", "VB_USED", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", + "SWKBD_TEST", "TEST_OOM", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_HEAP_EXTRA_MB", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UB_OVERLAY", "UB_OVERLAY_OUT", "UPSCALER", "VB_USED", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VRAM_LIMIT_MIB", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", diff --git a/runtime/host.h b/runtime/host.h index 4bb5b1d5..72e0a489 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -175,6 +175,23 @@ int mem_guest_readable(uint64_t a, uint64_t n); // [a, a+n) can be re // module or the committed heap answered from the layout, // anything else by plat_mem_readable (an OS query) +// ---- short out-of-memory spells (oom.c) +enum { RT_OOM_COMMIT, RT_OOM_STAGING, RT_OOM_IMAGE, RT_OOM_CHUNK, RT_OOM_SITES }; +extern volatile int32_t rt_oom_req; // set by rt_oom_request; vk_texture_new_submit takes it +void rt_oom_request(void); // asks the Vulkan back end to give memory back at its next + // submit (released textures, headroom blocks); any thread, + // any lock held +int rt_oom_wait(const char *what, uint64_t bytes, int (*retry)(void *arg), void *arg); + // after a failed allocation: requests a release, then calls + // retry every 10 ms for up to 30 s; 1 once retry returned 1, + // 0 after the wait (the caller ends the game). Logs the + // first 8 with the commit numbers +void *rt_realloc_wait(void *old, size_t n, const char *what); // realloc, NULL only after rt_oom_wait gave up +int rt_oom_forced(int site); // 1: fail this attempt at site (TOTK_TEST_OOM=:, + // sites commit, staging, image, chunk; or rt_oom_force) +void rt_oom_force(int site, int n); // probes: fail the next n attempts at site +int rt_oom_forced_left(int site); // forced failures still to come at site + // ---- modules (loader.c) typedef struct { const char *name; uint64_t base, size, text_end, data_off; // text_end, data_off: offsets from base diff --git a/runtime/kernel_probe.c b/runtime/kernel_probe.c index cbbc9297..83f11101 100644 --- a/runtime/kernel_probe.c +++ b/runtime/kernel_probe.c @@ -682,6 +682,25 @@ static int cmp_u64(const void *a, const void *b) { return x < y ? -1 : x > y; } +// A refused commit waits and retries (memory.c commit, oom.c rt_oom_wait): three forced refusals, then the page maps. +static void test_oom_commit(void) { + const RtGuestLayout *g = mem_layout(); + uint64_t a = g->alias.base + g->alias.size - 0x10000; + rt_oom_force(RT_OOM_COMMIT, 3); + uint64_t t0 = plat_now_ms(); + uint32_t r = mem_map_physical(a, 0x1000); + uint64_t ms = plat_now_ms() - t0; + CHECK(r == 0 && rt_oom_forced_left(RT_OOM_COMMIT) == 0 && ms >= 20, + "MapPhysicalMemory after 3 refused commits: result %#x, %d refusals left, %llu ms", r, + rt_oom_forced_left(RT_OOM_COMMIT), (unsigned long long)ms); + rt_oom_force(RT_OOM_COMMIT, 0); + if (r == 0) { + *(volatile uint64_t *)(uintptr_t)a = 1; + CHECK(mem_unmap_physical(a, 0x1000) == 0, "UnmapPhysicalMemory after the refused commits"); + } + printf("oom: commit waited %llu ms through 3 refusals\n", (unsigned long long)ms); +} + static void test_platform(void) { PlatThread *t = plat_thread_new(plat_worker, (void *)(intptr_t)1, "probe-kernel plat"); CHECK(t, "plat_thread_new"); @@ -792,6 +811,7 @@ int probe_kernel(void) { return 1; } test_platform(); + test_oom_commit(); test_crash_report(); kern_run_on_guest_main(probe_main); RtStats s = rt_stats(); diff --git a/runtime/memory.c b/runtime/memory.c index 019c0fab..bfe11290 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -175,12 +175,29 @@ static int thaw_fault(uint64_t a) { } // Mapping changes thaw the regions they touch and run under frz_lock, so no freeze sees them half done. -static void commit(uint64_t a, uint64_t n, const char *what) { +typedef struct { uint64_t a, n; } CommitTry; +static int commit_try(void *arg) { + const CommitTry *t = arg; + if (rt_oom_forced(RT_OOM_COMMIT)) return 0; plat_mutex_lock(&frz_lock); - thaw_range_locked(a, n, 3); - int ok = !n || plat_mem_commit(a, n); + thaw_range_locked(t->a, t->n, 3); + int ok = !t->n || plat_mem_commit(t->a, t->n); plat_mutex_unlock(&frz_lock); - if (!ok) rt_fatal("memory: commit %s %#llx+%#llx failed", what, (unsigned long long)a, (unsigned long long)n); + return ok; +} + +// A refused commit waits for memory (rt_oom_wait): the system's commit ran out for a moment (a page file growing, the +// driver's device memory charged to this process), and the Vulkan back end gives memory back at its next submit. The +// wait holds the caller's lock (lock or klock: only memory SVCs and kernel-region allocations wait behind it) but not +// frz_lock, which write faults on frozen regions need. Every caller waits, the boot's ones too: there a refusal has the +// same cause, and the wait only delays an end the game would reach anyway. +static void commit(uint64_t a, uint64_t n, const char *what) { + CommitTry t = {a, n}; + if (commit_try(&t)) return; + char w[64]; + snprintf(w, sizeof w, "memory: commit of %s at %#llx", what, (unsigned long long)a); + if (!rt_oom_wait(w, n, commit_try, &t)) + rt_fatal("memory: commit %s %#llx+%#llx failed", what, (unsigned long long)a, (unsigned long long)n); } static void decommit(uint64_t a, uint64_t n, const char *what) { diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index 874002f7..8141c61b 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -148,6 +148,7 @@ static void *realloc_wait(void *old, size_t n, const char *what) { if (p) return p; pool_drain(); big_drain(); + rt_oom_request(); // the Vulkan back end gives memory back at its next submit (oom.c) static uint32_t reports; int report = __atomic_fetch_add(&reports, 1, __ATOMIC_RELAXED) < 8; if (report) { diff --git a/runtime/oom.c b/runtime/oom.c new file mode 100644 index 00000000..77f891c0 --- /dev/null +++ b/runtime/oom.c @@ -0,0 +1,106 @@ +// Short out-of-memory spells: a failed allocation waits for memory instead of ending the game. +// +// Windows refuses commit beyond the current limit while it grows a system-managed page file, and on some drivers device +// memory counts as process commit (vk_memory.c vk_mem_host_probe), so for a moment every allocation of the process can +// fail and the same request succeeds a little later. The failure sites (memory.c commit, vk_memory.c vk_staging, +// vk_texture.c create_tex, vk_exec_draw.c transient chunks, nvn_api_cmd.c realloc_wait) ask the Vulkan back end to give +// memory back (rt_oom_request: released textures and headroom blocks, at its next submit, where its locks are free) and +// retry for up to OOM_WAIT_MS before they end the game. +// +// TOTK_TEST_OOM=:[,:...] (sites commit, staging, image, chunk) fails the first n attempts at a site, so +// the wait can be tested without running out of memory; the probes arm it with rt_oom_force. +#include +#include + +#include "cfg.h" +#include "host.h" +#include "platform/platform.h" + +#define OOM_WAIT_MS 30000u +#define OOM_STEP_MS 10u +#define OOM_REPORTS 8 +#define OOM_REQUEST_EVERY 100 // retries between two release requests: a second, since a release only frees what is old enough + +volatile int32_t rt_oom_req; + +static const char *const site_name[RT_OOM_SITES] = {"commit", "staging", "image", "chunk"}; +static volatile int32_t force_left[RT_OOM_SITES]; +static volatile int32_t forced_parsed; +static PlatOnce force_once = PLAT_ONCE_INIT; + +static void force_parse(void *arg) { + (void)arg; + const char *s = cfg_get("TOTK_TEST_OOM"); + while (s && *s) { + const char *colon = strchr(s, ':'), *end = strchr(s, ','); + if (!end) end = s + strlen(s); + if (colon && colon < end) { + for (int i = 0; i < RT_OOM_SITES; i++) + if ((size_t)(colon - s) == strlen(site_name[i]) && !strncmp(s, site_name[i], (size_t)(colon - s))) { + int n = atoi(colon + 1); + __atomic_store_n(&force_left[i], n > 0 ? n : 0, __ATOMIC_RELAXED); + rt_log("oom: TOTK_TEST_OOM fails the first %d %s attempts", n, site_name[i]); + } + } + s = *end ? end + 1 : end; + } + __atomic_store_n(&forced_parsed, 1, __ATOMIC_RELEASE); +} + +int rt_oom_forced(int site) { + if (!__atomic_load_n(&forced_parsed, __ATOMIC_ACQUIRE)) plat_once(&force_once, force_parse, NULL); + if (__atomic_load_n(&force_left[site], __ATOMIC_RELAXED) <= 0) return 0; + return plat_dec32(&force_left[site]) >= 0; +} + +void rt_oom_force(int site, int n) { + if (!__atomic_load_n(&forced_parsed, __ATOMIC_ACQUIRE)) plat_once(&force_once, force_parse, NULL); + __atomic_store_n(&force_left[site], n, __ATOMIC_RELAXED); +} + +int rt_oom_forced_left(int site) { + int n = __atomic_load_n(&force_left[site], __ATOMIC_RELAXED); + return n > 0 ? n : 0; +} + +void rt_oom_request(void) { __atomic_store_n(&rt_oom_req, 1, __ATOMIC_RELEASE); } + +int rt_oom_wait(const char *what, uint64_t bytes, int (*retry)(void *arg), void *arg) { + rt_oom_request(); + static volatile int32_t reports; + int report = plat_inc32(&reports) <= OOM_REPORTS; + double proc, used, limit; + if (report) { + plat_commit_info(&proc, &used, &limit); + rt_log("%s of %llu bytes failed (memory commit: this process %.0f MB, the system %.0f of %.0f MB); waiting up " + "to %u s for memory", what, (unsigned long long)bytes, proc, used, limit, OOM_WAIT_MS / 1000); + } + uint64_t t0 = plat_now_ms(); + for (int n = 1; plat_now_ms() - t0 < OOM_WAIT_MS; n++) { + plat_sleep_ms(OOM_STEP_MS); + if (retry(arg)) { + if (report) + rt_log("%s of %llu bytes succeeded after %llu ms", what, (unsigned long long)bytes, + (unsigned long long)(plat_now_ms() - t0)); + return 1; + } + if (n % OOM_REQUEST_EVERY == 0) rt_oom_request(); + } + plat_commit_info(&proc, &used, &limit); + rt_log("%s of %llu bytes still failed after %u s (memory commit: this process %.0f MB, the system %.0f of %.0f MB)", + what, (unsigned long long)bytes, OOM_WAIT_MS / 1000, proc, used, limit); + return 0; +} + +typedef struct { void *old, *p; size_t n; } Realloc; +static int realloc_try(void *arg) { + Realloc *r = arg; + return (r->p = realloc(r->old, r->n)) != NULL; +} + +void *rt_realloc_wait(void *old, size_t n, const char *what) { + void *p = realloc(old, n); + if (p || !n) return p; + Realloc r = {old, NULL, n}; + return rt_oom_wait(what, n, realloc_try, &r) ? r.p : NULL; +} diff --git a/runtime/vk.h b/runtime/vk.h index 9cac3e54..9ca1051d 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -314,6 +314,8 @@ int vk_mem_uma(void); // 1: the device class is uma (vk_ const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void); void vk_headroom_start(void); void vk_headroom_stop(void); // before vmaDestroyAllocator +void vk_headroom_release(void); // after a failed allocation: frees the spare blocks and keeps none for a while; + // any thread that holds no VMA lock (takes only the headroom lock) int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed // Changes whenever vk_mem_resolve plus VK_MEM_USE may give another span for an address: a pool added, finalized or // mapped, a mirror created, refused or forgotten, a submission started. @@ -402,6 +404,10 @@ int vk_mem_span_live(const VkMemSpan *sp); void vk_mem_note_fence(uint64_t timeline_value); void vk_mem_fault_log(void); // vk_device_fault_report: the deferred-free state, lock-free void vk_mem_collect(void); // frees what has completed +// After a failed allocation, on a thread that may hold the texture lock and the upload lock (cmd_lock) but no VMA +// lock: frees the headroom blocks and the completed upload batches and deferred objects, and requests the release +// of released textures at the next submit (rt_oom_request). +void vk_oom_release(void); typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats; void vk_mem_stats(VkMemStats *s); void vk_mem_shutdown(void); // before vk_shutdown: waits idle, frees every memory and texture object diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 51a2e829..f7ed6d10 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -363,6 +363,16 @@ static CbRes *cb_res(VkExec *ex) { return D->cur; } +typedef struct { const VkBufferCreateInfo *bi; const VmaAllocationCreateInfo *ai; Chunk *c; VmaAllocationInfo info; + VkResult r; } ChunkTry; +static int chunk_try(void *arg) { + ChunkTry *t = arg; + vk_mem_collect(); // the fence thread's collect waits while the caller holds cmd_lock + t->r = rt_oom_forced(RT_OOM_CHUNK) ? VK_ERROR_OUT_OF_DEVICE_MEMORY + : vmaCreateBuffer(vk_ctx()->vma, t->bi, t->ai, &t->c->buf, &t->c->alloc, &t->info); + return t->r == VK_SUCCESS; +} + static uint8_t *scratch(VkExec *ex, uint64_t size, VkBuffer *buf, VkDeviceSize *off) { CbRes *r = cb_res(ex); size = (size + 255) & ~255ull; @@ -396,10 +406,17 @@ static uint8_t *scratch(VkExec *ex, uint64_t size, VkBuffer *buf, VkDeviceSize * VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, .requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT}; - VmaAllocationInfo info; - VkResult res = vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &c->buf, &c->alloc, &info); - if (res != VK_SUCCESS) rt_fatal("vk: transient buffer of %llu bytes failed (%d)", (unsigned long long)n, (int)res); - c->map = info.pMappedData, c->size = n, c->used = 0; + // Out of memory: the recording waits for memory (rt_oom_wait) with what it holds (the texture lock at most, + // which vk_oom_release takes before cmd_lock); new chunks are made only when the spare ones ran out. + ChunkTry ct = {.bi = &bi, .ai = &ai, .c = c}; + ct.r = rt_oom_forced(RT_OOM_CHUNK) ? VK_ERROR_OUT_OF_DEVICE_MEMORY + : vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &c->buf, &c->alloc, &ct.info); + if (ct.r == VK_ERROR_OUT_OF_HOST_MEMORY || ct.r == VK_ERROR_OUT_OF_DEVICE_MEMORY) { + vk_oom_release(); + rt_oom_wait("vk: transient buffer", n, chunk_try, &ct); + } + if (ct.r != VK_SUCCESS) rt_fatal("vk: transient buffer of %llu bytes failed (%d)", (unsigned long long)n, (int)ct.r); + c->map = ct.info.pMappedData, c->size = n, c->used = 0; r->nchunks++; } *buf = c->buf; diff --git a/runtime/vk_exec_probe.c b/runtime/vk_exec_probe.c index 9d244cdf..825b47a6 100644 --- a/runtime/vk_exec_probe.c +++ b/runtime/vk_exec_probe.c @@ -968,6 +968,9 @@ int probe_vkexec(void) { const GfxBackend *b = &gfx_vulkan_backend; uint64_t heap = 0; CHECK(mem_set_heap_size(16ull << 20, &heap) == 0 && heap, "SetHeapSize 16 MiB"); + // Out-of-memory waits (oom.c): the first two staging buffers, the first texture image and the first draw-data chunk + // fail; every part below passes through the wait or the retry that follows. + rt_oom_force(RT_OOM_STAGING, 2), rt_oom_force(RT_OOM_IMAGE, 1), rt_oom_force(RT_OOM_CHUNK, 1); uint8_t *guest = (uint8_t *)(uintptr_t)heap; uint64_t first = 0, last = 0; int w = 0, h = 0, inside = 0, xfer_bytes = 0; @@ -989,6 +992,12 @@ int probe_vkexec(void) { vk_exec_xfer_shutdown(); vk_backend_shutdown(); long errors = vk_validation_errors_total(); + int oom_left[3] = {rt_oom_forced_left(RT_OOM_STAGING), rt_oom_forced_left(RT_OOM_IMAGE), rt_oom_forced_left(RT_OOM_CHUNK)}; + if (!bad && !errors && (oom_left[0] || oom_left[1] || oom_left[2])) { + printf("probe-vkexec: FAILED (forced allocation failures never reached: staging %d, image %d, chunk %d)\n", + oom_left[0], oom_left[1], oom_left[2]); + return 1; + } if (bad || errors) { printf("probe-vkexec: FAILED (%s, %ld validation errors)\n", sync_bad ? "a sync check failed" : draw_bad ? "a draw check failed" : bad ? "a clear or xfer check failed" : "checks passed", errors); @@ -1034,5 +1043,7 @@ int probe_vkexec(void) { "finished mark is the last pass, ring records exact); the report reads:\n%s\n", marks_text); printf("probe-vkexec: empty pool part passed (a zero-byte pool at the start of another: no Vulkan objects, lookups " "there find the other pool before and after its Finalize)\n"); + printf("probe-vkexec: out-of-memory part passed (2 failed staging buffers waited for, 1 failed texture image retried, " + "1 failed draw-data chunk waited for)\n"); return 0; } diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c index 88083025..ccbacb06 100644 --- a/runtime/vk_headroom.c +++ b/runtime/vk_headroom.c @@ -12,7 +12,8 @@ // a block per memory type is held back. // Under VRAM pressure no headroom is kept: the spare block counts against the budget that decides whether a pool gets // its VRAM mirror (vk_memory.c), so while use plus two blocks passes 85% of the budget the anchor goes and nothing new -// is allocated. The same holds when the system runs short of memory (room). +// is allocated. The same holds when the system runs short of memory (room), and for HR_HOLD_MS after an allocation +// failed anywhere (vk_headroom_release). // TOTK_VRAM_HEADROOM=0 turns it off. #include "cfg.h" @@ -22,6 +23,7 @@ #define HR_PERIOD_MS 500 #define HR_SMALL_HEAP (1ull << 30) // VMA_SMALL_HEAP_MAX_SIZE: smaller heaps use blocks of a heap eighth +#define HR_HOLD_MS 30000u static struct { int on, stop; @@ -33,6 +35,7 @@ static struct { volatile int64_t made; // blocks this thread brought in volatile int64_t other; // device memory of 32 MiB or more other threads allocated (VMA blocks, large dedicated) VmaAllocation anchor[64]; // per memory type: the half block that keeps the newest spare block from being deleted + volatile uint64_t hold_until; // plat_now_ms before which no spare block is kept (vk_headroom_release) } H; static void VKAPI_PTR on_allocate(VmaAllocator a, uint32_t type, VkDeviceMemory m, VkDeviceSize size, void *user) { @@ -79,7 +82,7 @@ static void top_up(const VkCtx *c, uint32_t type) { // blocks (vk_memory.c vk_staging), while their default pool held 4 MiB: on the RX 9070 the spare blocks of types 1 // and 3 held 512 MiB of commit for nothing. if (!(c->mem_props.memoryTypes[type].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) || - !room(c, type, block)) { + plat_now_ms() < __atomic_load_n(&H.hold_until, __ATOMIC_RELAXED) || !room(c, type, block)) { drop_anchor(c, type); return; } @@ -132,6 +135,22 @@ void vk_headroom_stop(void) { plat_event_set(&H.ev); } +// After a failed allocation: the spare blocks go at once (an empty VMA block takes the next allocation of its type or +// is given back), and none is kept for HR_HOLD_MS. H.lock is held only around VMA calls, so the caller may hold any lock +// of ours, just no VMA lock. +void vk_headroom_release(void) { + __atomic_store_n(&H.hold_until, plat_now_ms() + HR_HOLD_MS, __ATOMIC_RELAXED); + if (!H.started) return; + int n = 0; + plat_mutex_lock(&H.lock); + const VkCtx *c = H.stop ? NULL : vk_ctx(); + for (uint32_t t = 0; c && t < 64; t++) + if (H.anchor[t]) drop_anchor(c, t), n++; + plat_mutex_unlock(&H.lock); + static volatile int32_t logged; + if (n && plat_inc32(&logged) <= 8) rt_log("vk: headroom: %d spare blocks freed after a failed allocation", n); +} + void vk_headroom_start(void) { if (!H.on || H.started) return; plat_mutex_init(&H.lock); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index c887a560..9300e9cd 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2482,6 +2482,20 @@ void vk_upload_end(void) { plat_rmutex_unlock(&cmd_lock); } static VmaPool stg_pool; static int stg_pool_tried; +typedef struct { const VkBufferCreateInfo *bi; const VmaAllocationCreateInfo *aci; VkBuffer b; VmaAllocation a; + VmaAllocationInfo info; VkResult r; } StgTry; +static int staging_try(void *arg) { + StgTry *t = arg; + vk_mem_collect(); // upload batches and deferred objects the GPU finished meanwhile + t->r = rt_oom_forced(RT_OOM_STAGING) ? VK_ERROR_OUT_OF_DEVICE_MEMORY + : vmaCreateBuffer(vk_ctx()->vma, t->bi, t->aci, &t->b, &t->a, &t->info); + return t->r == VK_SUCCESS; +} + +// A failed staging buffer waits for memory (rt_oom_wait) before it ends the game: the headroom blocks go and the GPU's +// finished batches are freed at once, released textures at the next submit (vk_oom_release). The caller may hold the +// texture lock and cmd_lock (vk_upload_begin), both of which vk_oom_release may take after them, and no VMA lock. The open +// batch is not submitted: its caller records into it after this returns. void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { VkCtx *c = vk_ctx(); VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16, @@ -2505,11 +2519,22 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { plat_rmutex_unlock(&cmd_lock); } if (stg_pool && bi.size <= STG_BLOCK / 2) aci.pool = stg_pool; - VmaAllocationInfo info; - VkBuffer b; - VmaAllocation a; - VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info); - if (r != VK_SUCCESS) rt_fatal("vk: staging buffer of %llu bytes failed (%d)", (unsigned long long)size, (int)r); + StgTry st = {.bi = &bi, .aci = &aci}; + st.r = rt_oom_forced(RT_OOM_STAGING) ? VK_ERROR_OUT_OF_DEVICE_MEMORY + : vmaCreateBuffer(c->vma, &bi, &aci, &st.b, &st.a, &st.info); + if (st.r != VK_SUCCESS) { + VkResult first = st.r; + if (first == VK_ERROR_OUT_OF_HOST_MEMORY || first == VK_ERROR_OUT_OF_DEVICE_MEMORY) { + vk_oom_release(); + rt_oom_wait("vk: staging buffer", bi.size, staging_try, &st); + } + if (st.r != VK_SUCCESS) + rt_fatal("vk: staging buffer of %llu bytes failed (%d, first %d)", (unsigned long long)size, (int)st.r, + (int)first); + } + VkBuffer b = st.b; + VmaAllocation a = st.a; + VmaAllocationInfo info = st.info; *buf = b; if (alloc) *alloc = a; // the caller destroys it (vmaDestroyBuffer) after vk_upload_flush(1) else { @@ -2688,6 +2713,15 @@ void vk_mem_collect(void) { plat_rmutex_unlock(&cmd_lock); } +// Lock order: the texture lock, then cmd_lock (create_tex records under the first), then the headroom lock, which is +// held only around VMA calls. A caller holding cmd_lock takes it again (recursive) and keeps the fence thread's own +// collect (vk_sync.c) out meanwhile, so this collects for it. +void vk_oom_release(void) { + rt_oom_request(); + vk_headroom_release(); + vk_mem_collect(); +} + void vk_mem_stats(VkMemStats *s) { memset(s, 0, sizeof *s); plat_mutex_lock_shared(&pool_lock); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index a2a9ea41..9ebd7a0b 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -458,7 +458,19 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { } if (!img) { VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; - VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); + VkResult r = rt_oom_forced(RT_OOM_IMAGE) ? VK_ERROR_OUT_OF_DEVICE_MEMORY + : vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); + // Out of memory: one more try after the headroom blocks and the finished deferred objects are freed + // (vk_oom_release, which takes cmd_lock after the texture lock held here); released textures go at the next + // submit. No wait: the texture is asked for again at its next use, while a wait would hold the texture lock. + if (r == VK_ERROR_OUT_OF_HOST_MEMORY || r == VK_ERROR_OUT_OF_DEVICE_MEMORY) { + static volatile int32_t n_retried; + vk_oom_release(); + VkResult again = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); + log_limited(&n_retried, "image creation failed (%d), tried again after a memory release: %s (%ux%ux%u fmt %#x)", + (int)r, again == VK_SUCCESS ? "created" : "failed again", w, h, depth, d->format); + r = again; + } if (r != VK_SUCCESS) { log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r, d->format, w, h, depth, d->target, levels, layers); @@ -703,16 +715,19 @@ static int dead_candidate(const Tex *e) { // Drops overwritten entries until `need` bytes are free or the step is used up. The sort runs only under pressure, while // an old enough entry exists, and rests for a while when it finds nothing to drop. Returns the bytes freed. static uint64_t pressure_search_ns; // longest search since the last mem: released line -static uint64_t drop_overwritten(uint64_t need, int *dropped) { +// step: PRESSURE_STEP, or OOM_STEP from oom_release, which neither waits out nor starts the rest. +#define OOM_STEP 256 +static uint64_t drop_overwritten(uint64_t need, int *dropped, int step) { static int rest; - if (rest > 0) { rest--; return 0; } + int urgent = step > PRESSURE_STEP; + if (rest > 0 && !urgent) { rest--; return 0; } if (!rel_head || coarse_ms - rel_head->rel_ms < DEAD_AGE_MS) return 0; uint64_t t0 = plat_now_ns(), freed = 0; AddrIndex x; if (!addr_index(&x)) return 0; - Tex *cand[PRESSURE_STEP]; + Tex *cand[OOM_STEP]; int nc = 0; - for (Tex *e = rel_head; e && nc < PRESSURE_STEP && coarse_ms - e->rel_ms >= DEAD_AGE_MS; e = e->rel_next) { + for (Tex *e = rel_head; e && nc < step && coarse_ms - e->rel_ms >= DEAD_AGE_MS; e = e->rel_next) { if (!dead_candidate(e)) continue; int lo = 0, hi = x.n; // the first entry at e's address, then e among those while (lo < hi) { @@ -730,12 +745,25 @@ static uint64_t drop_overwritten(uint64_t need, int *dropped) { destroy_tex(cand[i]); (*dropped)++, n_pressure_drops++, n_pressure_overwritten++; } - if (!nc) rest = DEAD_RETRY_QUERIES; + if (!nc && !urgent) rest = DEAD_RETRY_QUERIES; uint64_t ns = plat_now_ns() - t0; if (ns > pressure_search_ns) pressure_search_ns = ns; return freed; } +// Clean released entries, oldest first: mirrored ones the GPU never wrote, which come back from their storage. +static uint64_t drop_clean(uint64_t need, uint64_t freed, int *dropped, int step) { + for (Tex *e = rel_head, *next; e && freed < need && *dropped < step && coarse_ms - e->rel_ms >= CLEAN_AGE_MS; e = next) { + next = e->rel_next; + if (!e->pub.mirrored || e->pub.gpu_owned || e->gpu_dirty) continue; + freed += own_bytes(e); + dropped_note(e); + destroy_tex(e); + (*dropped)++, n_pressure_drops++; + } + return freed; +} + static void sweep_pressure(void) { // tex_lock held if (vram_pressure < 0) { const char *s = cfg_get("TOTK_TEX_VRAM_PRESSURE"); @@ -750,15 +778,22 @@ static void sweep_pressure(void) { // tex_lock held uint64_t line = budget / 100 * (uint64_t)vram_pressure; if (used <= line) return; int dropped = 0; - uint64_t need = used - line, freed = drop_overwritten(need, &dropped); - for (Tex *e = rel_head, *next; e && freed < need && dropped < PRESSURE_STEP && coarse_ms - e->rel_ms >= CLEAN_AGE_MS; e = next) { - next = e->rel_next; - if (!e->pub.mirrored || e->pub.gpu_owned || e->gpu_dirty) continue; - freed += own_bytes(e); - dropped_note(e); - destroy_tex(e); - dropped++, n_pressure_drops++; - } + uint64_t need = used - line, freed = drop_overwritten(need, &dropped, PRESSURE_STEP); + drop_clean(need, freed, &dropped, PRESSURE_STEP); +} + +// rt_oom_request (an allocation failed somewhere, oom.c): the same two kinds as sweep_pressure, as many as OOM_STEP +// whatever the budget says, and the headroom blocks. Never GPU-written content that lives only in its image. The memory +// comes back once the submission being recorded is done (destroy_tex defers the images). Texture lock held. +static void oom_release(void) { + int dropped = 0; + uint64_t freed = drop_overwritten(UINT64_MAX, &dropped, OOM_STEP); + int overwritten = dropped; + freed = drop_clean(UINT64_MAX, freed, &dropped, OOM_STEP); + vk_headroom_release(); + static volatile int32_t n_logged; + log_limited(&n_logged, "out of memory: %d released textures dropped (%d overwritten, %d clean), %.0f MiB, %d kept", + dropped, overwritten, dropped - overwritten, freed / 1048576.0, nreleased); } // ---------------------------------------------------------------- uploads @@ -1714,6 +1749,8 @@ void vk_texture_new_submit(void) { if (!++submit_epoch) submit_epoch = 1; coarse_ms = plat_now_ms(); sweep_pressure(); + if (__atomic_load_n(&rt_oom_req, __ATOMIC_RELAXED) && __atomic_exchange_n(&rt_oom_req, 0, __ATOMIC_ACQUIRE)) + oom_release(); static uint64_t log_at; uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) / 60; if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) { @@ -2037,7 +2074,12 @@ static void write_back_n(Tex **v, int n) { if (np) vk_upload_flush(1); for (int i = 0; i < np; i++) { WbPart *q = &p[i]; - uint8_t *buf = malloc(q->g.img * (size_t)q->g.nz); + size_t bytes = q->g.img * (size_t)q->g.nz; + uint8_t *buf = malloc(bytes); + if (!buf) { // waits for memory with the texture lock held, after the release that needs no submit + vk_oom_release(); + buf = rt_realloc_wait(NULL, bytes, "vk: texture write-back buffer"); + } if (!buf) rt_fatal("vk: out of memory for a texture write-back"); texels_read_take(q->e, &q->g, &q->t, buf); storage_texels(&q->e->t, &q->g, &q->r, buf, 1);