diff --git a/docs/agent-playbook.md b/docs/agent-playbook.md index b18f0ffe..9fdc6481 100644 --- a/docs/agent-playbook.md +++ b/docs/agent-playbook.md @@ -96,8 +96,9 @@ Check these classes before you re-investigate a symptom that looks like one of t waits for a fence, targets up to 64 KiB rendered before it go back too (`vk_texture.c` `vk_texture_write_back_small`): the 128x128 cut map is read with no invalidate. Without it a cut left the grass standing, with no blades and no insects. -- A one-shot draw (a target not rendered in the previous frame) waits for its shader and pipeline instead of being - skipped: `vk_exec_draw.c` `prepare_draw`. Skipping left the cloud and fog volumes empty for the session. +- A one-shot draw (a pass none of whose targets the previous frame rendered) waits for its shader and pipeline instead + of being skipped: `vk_exec_draw.c` `prepare_draw`, `TOTK_PIPE_SKIP`. Skipping left the cloud and fog volumes empty + for the session. Its stage libraries build on idle workers in parallel (`vk_pipeline.c` `lib_wait`). - Storage buffers bound by Ryujinx first-use order instead of the NVN c0 slot: `vk_exec_draw.c` `ssbo_slot`. It showed as tile rectangles. - 3D colour targets rejected: `vk_exec.c` `resolve_target`. The grading LUT was never written, so the dialog diff --git a/runtime/cfg.c b/runtime/cfg.c index 0d1fb86f..b2dbf1a0 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -31,7 +31,8 @@ static const char *const known[] = { "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "MEMORY", "MINIMAL_LAYOUT", "MIRROR_GAP", "MOTION", "MOUSE_SENS", "MOVIE_HOST", - "MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SYNC", "PREWARM_XLATE", + "MOVIE_PACE", "OPERATION_MODE", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", + "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_RENDER", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REEXEC", "RESOLVE_CHECK", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index ae6cd861..ece91c69 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -308,14 +308,38 @@ static int tfr_recent(const VkExecTarget *t, uint64_t stamp) { return tfr_match(e, t) && (e->last == stamp ? e->prev : e->last) == stamp - 1; } -int vk_exec_targets_recent(VkExec *ex) { +int vk_exec_targets_recent(VkExec *ex, int any) { resolve_rt(ex); uint64_t stamp = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1; - int n = 0; - for (int i = 0; i < 8; i++) - if (ex->color[i].image && (n++, !tfr_recent(&ex->color[i], stamp))) return 0; - if (ex->depth.image && (n++, !tfr_recent(&ex->depth, stamp))) return 0; - return n > 0; + int n = 0, recent = 0; + for (int i = 0; i < 9; i++) { + const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth; + if (!t->image) continue; + n++; + if (tfr_recent(t, stamp)) recent++; + else if (!any) return 0; + } + return any ? recent > 0 : n > 0; +} + +// One line naming each target of the current pass: format, extent, layer and the two frames it was last rendered in +// (prepare_draw's pipeline wait log). +void vk_exec_targets_describe(VkExec *ex, char *buf, size_t cap) { + resolve_rt(ex); + size_t n = 0; + buf[0] = 0; + for (int i = 0; i < 9 && n < cap; i++) { + const VkExecTarget *t = i < 8 ? &ex->color[i] : &ex->depth; + if (!t->image) continue; + const TargetFrames *e = tfr_slot(t); + int m = tfr_match(e, t); + int w = snprintf(buf + n, cap - n, "%s%c%d fmt %d %ux%u L%u seen %llu/%llu", n ? "; " : "", i < 8 ? 'c' : 'd', + i < 8 ? i : 0, (int)t->format, t->width, t->height, t->base_layer, + (unsigned long long)(m ? e->last : 0), (unsigned long long)(m ? e->prev : 0)); + if (w < 0) break; + n += (size_t)w; + } + if (!n) snprintf(buf, cap, "no targets"); } // ---- passes and barriers diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index ca022f0d..69f3314e 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -123,9 +123,11 @@ void vk_exec_draw_new_cb(void); VkBuffer vk_exec_inline_buffer(VkDeviceSize size); // Scissor 0 clipped to the render area of the open pass; extent 0: nothing to draw. VkRect2D vk_exec_scissor(const VkExec *ex); -// 1: every target of the current pass was rendered in the previous presented frame (per-frame targets); 0 when one -// was not, or the pass has none. -int vk_exec_targets_recent(VkExec *ex); +// 1: every target of the current pass (any 1: at least one) was rendered in the previous presented frame (per-frame +// targets); 0 otherwise, or when the pass has none. +int vk_exec_targets_recent(VkExec *ex, int any); +// The current pass's targets with their formats, extents and the frames they were last rendered in (diagnostics). +void vk_exec_targets_describe(VkExec *ex, char *buf, size_t cap); // TOTK_TRACE=gfx: replay counters of the current frame, logged and reset at each present (vk_exec_frame_log). typedef struct { diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a751ea01..c2b19645 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1447,6 +1447,26 @@ static void bind_vertex_buffers(VkExec *ex) { B.vb_valid = 1; } +// TOTK_PIPE_SKIP: which draws are skipped, rather than waited for, while their shaders or pipeline build. +// strict (0): only draws whose pass targets were all rendered in the previous presented frame; a pass with a target +// that was not counts as one-shot and waits. +// any (1, default): also draws in a pass with at least one target rendered in the previous frame (a per-frame depth +// buffer or g-buffer with a colour target the frame before did not use: half-resolution effect buffers in fights). +// Passes with none (volumes, LUTs, shadow layers seen for the first time) still wait. +// fullscreen (2): as any, and full-screen draws into such passes are skipped instead of waiting from their budget. +static int pipe_skip_mode(void) { + static int mode = -1; + if (mode < 0) { + const char *e = cfg_get("TOTK_PIPE_SKIP"); + mode = !e || !*e ? 1 : !strcmp(e, "strict") || !strcmp(e, "0") ? 0 : !strcmp(e, "fullscreen") || !strcmp(e, "2") ? 2 : 1; + static const char *const what[3] = {"only draws into per-frame passes (TOTK_PIPE_SKIP=strict)", + "draws into passes with a target the previous frame rendered", + "those and full-screen draws (TOTK_PIPE_SKIP=fullscreen)"}; + rt_log("vk_exec: draws skipped while their pipeline builds: %s", what[mode]); + } + return mode; +} + // The previous draw's shaders, kept while its program, bound pipeline state, the vertex and fragment stages' first 32 // texture and image handles and every slot's resolution (vk_texture_slots_gen, read unlocked: a registration racing // with it is ordered after this draw, as answers_for's own check would order it) stay the same: answers_for reads @@ -1516,9 +1536,9 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { // A draw into a target the previous frame did not render (or into none) waits for a pending shader or pipeline: // one-shot draws (volumes and LUTs generated once, later only sampled) would leave their target empty for the // session. A draw into per-frame targets is skipped until they are ready and comes back next frame. - int oneshot = -1; // -1: not looked up + int oneshot = -1, skip = pipe_skip_mode(); // -1: not looked up if (vs_wait || fs_wait) { - if (!(oneshot = !vk_exec_targets_recent(ex))) return perf_count(PERF_CNT_SHADER_SKIP), 0; // skipped until ready + if (!(oneshot = !vk_exec_targets_recent(ex, skip >= 1))) return perf_count(PERF_CNT_SHADER_SKIP), 0; // skipped until ready perf_count(PERF_CNT_ONESHOT_WAIT); if (vs_wait) vs = shader_for(ex, NVN_STAGE_VS, &has_vs, NULL, 1); if (fs_wait) fs = shader_for(ex, NVN_STAGE_FS, &has_fs, NULL, 1); @@ -1535,29 +1555,43 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { const VkPipeInfo *info = NULL; int pending; // A draw of at most 6 vertices over half the render area or more (a full-screen triangle or quad, a tile) waits for - // its pipeline: skipping it can leave a black tile. The waits share 50 ms per presented frame. A one-shot draw - // waits without that budget. Everything else is skipped while its pipeline is built. + // its pipeline: skipping it can leave a black tile. The waits share 50 ms per presented frame + // (TOTK_PIPE_SKIP=fullscreen skips them instead). A one-shot draw waits without that budget. Everything else is + // skipped while its pipeline is built. static uint64_t wait_frame; static uint64_t wait_left_ns; uint32_t wait_ms = 0; uint64_t t0 = 0; if (oneshot > 0) { wait_ms = ONESHOT_WAIT_MS; - } else if (count <= 6 && (uint64_t)sc.extent.width * sc.extent.height * 2 >= (uint64_t)ex->pass_w * ex->pass_h) { + } else if (skip < 2 && count <= 6 && (uint64_t)sc.extent.width * sc.extent.height * 2 >= (uint64_t)ex->pass_w * ex->pass_h) { uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (f != wait_frame) wait_frame = f, wait_left_ns = 50000000; wait_ms = (uint32_t)(wait_left_ns / 1000000); if (wait_ms) t0 = kern_now_ns(); } + uint64_t w0 = kern_now_ns(); VkPipeline p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); if (t0) { uint64_t spent = kern_now_ns() - t0; wait_left_ns = spent < wait_left_ns ? wait_left_ns - spent : 0; } - if (pending && oneshot < 0 && !vk_exec_targets_recent(ex)) { + const char *why = oneshot > 0 ? "one-shot, shader" : wait_ms ? "full-screen" : NULL; + if (pending && oneshot < 0 && !vk_exec_targets_recent(ex, skip >= 1)) { perf_count(PERF_CNT_ONESHOT_WAIT); + why = "one-shot"; p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS); } + uint64_t waited = kern_now_ns() - w0; + static int wait_logs; + if (why && waited >= 2000000 && wait_logs < 200) { + char tg[512]; + wait_logs++; + vk_exec_targets_describe(ex, tg, sizeof tg); + rt_log("vk_exec: frame %llu: a draw waited %.1f ms for its pipeline (%s%s): %u vertices, pass %ux%u, %s", + (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1, waited / 1e6, why, + pending ? ", skipped after all" : "", count, ex->pass_w, ex->pass_h, tg); + } if (pending) return 0; // a background worker is building it: the draw is skipped until it is ready if (!p || !info) return warn_once(W_PIPELINE, "a graphics pipeline could not be created: draws with it are skipped"), 0; bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 7ea6c2d5..ffa82aed 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -1456,10 +1456,15 @@ out: // thread of the draw that needs it (no link-time optimisation); the optimised pipeline is the job pipe_miss queued // (a monolithic build with the same modules), which replaces the fast one in the Pipe when done. Libraries and fast // pipelines live until vk_pipeline_shutdown. +// Background build signalling, both with P.lock: work_cv wakes the pipeline workers, done_cv wakes threads waiting for a +// pipeline or a stage library to be published. +static CONDITION_VARIABLE work_cv = CONDITION_VARIABLE_INIT, done_cv = CONDITION_VARIABLE_INIT; #define LIB_BUCKETS 1024 enum { LIB_VI, LIB_PR, LIB_FS, LIB_FO }; -typedef struct Lib { struct Lib *next; uint64_t hash; VkPipeline pipe; uint32_t size; uint8_t pre, used, pad[2]; uint8_t key[]; } Lib; -// pre: built by the boot prebuild ("stage library prebuild" below); used: a draw's link found it +typedef struct Lib { struct Lib *next; uint64_t hash; VkPipeline pipe; uint32_t size; uint8_t pre, used, building, pad; + struct LibJob *job; uint8_t key[]; } Lib; +// pre: built by the boot prebuild ("stage library prebuild" below); used: a draw's link found it; building: a thread +// builds it or job holds it queued (pipe not set yet); entries live until libs_destroy static Lib *libs[LIB_BUCKETS]; typedef struct { uint32_t kind, topo_class, attr_fmt[32], attr_off[32]; uint8_t attr_bind[32], rate[17], pad[3]; } ViKey; @@ -1603,6 +1608,10 @@ static Lib *lib_find(uint64_t h, const void *key, uint32_t size) { // Returns the entered handle. Caller holds P.lock. static VkPipeline lib_put(uint64_t h, const void *key, uint32_t size, VkPipeline pipe) { Lib *b = lib_find(h, key, size); + if (b && b->building) { // another thread's build publishes the entry + if (pipe) vkDestroyPipeline(P.device, pipe, NULL); + return VK_NULL_HANDLE; + } if (b) { if (pipe) vkDestroyPipeline(P.device, pipe, NULL); return b->pipe; @@ -1615,10 +1624,72 @@ static VkPipeline lib_put(uint64_t h, const void *key, uint32_t size, VkPipeline return pipe; } +// Shader libraries a waiting draw needs (lib_wait): queued here ahead of every pipeline job, so an idle worker builds +// each at once and the vertex and fragment libraries of one draw build in parallel. A job no worker took is built by +// the first thread that needs its library (lib_get, lib_wait). Under P.lock. +typedef struct LibJob { struct LibJob *next; Lib *b; int kind; Layout *l; VkPipeState s; GKey k; } LibJob; +static struct { + LibJob *head, *tail; + int idle; // workers asleep on work_cv + int running; // jobs workers are building + int on; // TOTK_PIPE_WAIT: 1 workers (default), 0 the drawing thread builds them (draw); + // set by vk_pipeline_async_enable, before the replay runs + uint64_t waits, queued, by_worker, stolen, timeouts; +} U; + +// A building entry for key, NULL when out of memory. Caller holds P.lock and found none. +static Lib *lib_new(uint64_t h, const void *key, uint32_t size) { + Lib *b = calloc(1, sizeof *b + size); + if (!b) return NULL; + b->hash = h, b->size = size, b->building = 1; + memcpy(b->key, key, size); + b->next = libs[h % LIB_BUCKETS]; + libs[h % LIB_BUCKETS] = b; + return b; +} + +// Builds b (building, not queued) with P.lock released and publishes it; a failed build is entered as VK_NULL_HANDLE. +static VkPipeline lib_build(Lib *b, int kind, const VkPipeState *s, const GKey *k, Layout *l) { + VkPipelineLayout lay = kind == LIB_PR ? half_layout(l, 0) : kind == LIB_FS ? half_layout(l, 1) : VK_NULL_HANDLE; + VkPipeline pipe = VK_NULL_HANDLE; + if (lay || kind == LIB_VI || kind == LIB_FO) { + ReleaseSRWLockExclusive(&P.lock); + uint64_t t0 = qpc(); + pipe = lib_create(kind, s, k, lay, b->key); + double ms = ms_since(t0); + AcquireSRWLockExclusive(&P.lock); + P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms; + if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms; + } + if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind); + b->pipe = pipe, b->building = 0; + WakeAllConditionVariable(&done_cv); + return pipe; +} + +// Takes b's queued job out of the queue and builds it on this thread. Caller holds P.lock. +static void lib_steal(Lib *b) { + LibJob *j = b->job, **pp = &U.head, *prev = NULL; + while (*pp && *pp != j) prev = *pp, pp = &(*pp)->next; + if (*pp) *pp = j->next; + if (U.tail == j) U.tail = prev; + b->job = NULL; + lib_build(b, j->kind, &j->s, &j->k, j->l); + free(j); +} + static VkPipeline lib_get(int kind, const VkPipeState *s, const GKey *k, Layout *l, const void *key, uint32_t size, int build, int *miss) { uint64_t h = key_hash(key, size); Lib *have = lib_find(h, key, size); + while (have && have->building) { // being built (wait) or queued (build it here) + if (!build) { + *miss = 1; + return VK_NULL_HANDLE; + } + if (have->job) U.stolen++, lib_steal(have); + else SleepConditionVariableSRW(&done_cv, &P.lock, INFINITE, 0); + } if (have) { if (have->pre && !have->used) have->used = 1, P.g.pre_used[kind]++; return have->pipe; @@ -1628,12 +1699,14 @@ static VkPipeline lib_get(int kind, const VkPipeState *s, const GKey *k, Layout *miss = 1; return VK_NULL_HANDLE; } + Lib *b = lib_new(h, key, size); // entered first, so other threads wait for this build instead of repeating it + if (b) return lib_build(b, kind, s, k, l); ReleaseSRWLockExclusive(&P.lock); uint64_t t0 = qpc(); VkPipeline pipe = lib_create(kind, s, k, lay, key); double ms = ms_since(t0); AcquireSRWLockExclusive(&P.lock); - if ((have = lib_find(h, key, size))) { // another thread built it meanwhile + if ((have = lib_find(h, key, size)) && !have->building) { // another thread built it meanwhile if (pipe) vkDestroyPipeline(P.device, pipe, NULL); return have->pipe; } @@ -1654,6 +1727,26 @@ static void libs_destroy(void) { } } +// The four library keys of pipeline s (key k). +static void lib_keys(const VkPipeState *s, const GKey *k, ViKey *v, PrKey *pk, FsKey *fk, FoKey *ok) { + memset(v, 0, sizeof *v); + v->kind = LIB_VI, v->topo_class = k->topo_class; + memcpy(v->attr_fmt, k->attr_fmt, sizeof v->attr_fmt), memcpy(v->attr_off, k->attr_off, sizeof v->attr_off); + memcpy(v->attr_bind, k->attr_bind, sizeof v->attr_bind), memcpy(v->rate, k->rate, sizeof v->rate); + memset(pk, 0, sizeof *pk); + GfxShaderIo fin[32]; + pk->kind = LIB_PR, pk->vs = s->vs, pk->clamp = k->clamp, pk->poly_mode = k->poly_mode, pk->minus_w = k->minus_w; + pk->missing = vs_union_missing(s, fin); + for (int i = 0; i < 32; i++) + if (pk->missing >> i & 1) pk->fin[i] = fin[i]; + memset(fk, 0, sizeof *fk); + fk->kind = LIB_FS, fk->fs = s->fs, fk->dsb = k->dsb, fk->samples = k->samples, fk->a2c = k->a2c; + memset(ok, 0, sizeof *ok); + ok->kind = LIB_FO, ok->n_color = k->n_color, ok->samples = k->samples, ok->a2c = k->a2c, ok->logic_op = k->logic_op; + ok->blend_en = k->blend_en, ok->depth = k->depth, ok->stencil = k->stencil, ok->wmask = k->wmask; + memcpy(ok->color, k->color, sizeof ok->color), memcpy(ok->blend, k->blend, sizeof ok->blend); +} + // A fast-linked pipeline for s from its stage libraries. Caller holds P.lock. // build 0 (drawing thread): links only when the shader libraries exist (vertex input and fragment output libraries, // which hold no shader, are built here); *final 0 = not yet, a worker builds them. build 1 (worker): builds the @@ -1665,25 +1758,10 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc if (!l->set1 || k->fbl) return VK_NULL_HANDLE; uint64_t t0 = qpc(); ViKey vk_; - memset(&vk_, 0, sizeof vk_); - vk_.kind = LIB_VI, vk_.topo_class = k->topo_class; - memcpy(vk_.attr_fmt, k->attr_fmt, sizeof vk_.attr_fmt), memcpy(vk_.attr_off, k->attr_off, sizeof vk_.attr_off); - memcpy(vk_.attr_bind, k->attr_bind, sizeof vk_.attr_bind), memcpy(vk_.rate, k->rate, sizeof vk_.rate); PrKey pk; - memset(&pk, 0, sizeof pk); - GfxShaderIo fin[32]; - pk.kind = LIB_PR, pk.vs = s->vs, pk.clamp = k->clamp, pk.poly_mode = k->poly_mode, pk.minus_w = k->minus_w; - pk.missing = vs_union_missing(s, fin); - for (int i = 0; i < 32; i++) - if (pk.missing >> i & 1) pk.fin[i] = fin[i]; FsKey fk; - memset(&fk, 0, sizeof fk); - fk.kind = LIB_FS, fk.fs = s->fs, fk.dsb = k->dsb, fk.samples = k->samples, fk.a2c = k->a2c; FoKey ok; - memset(&ok, 0, sizeof ok); - ok.kind = LIB_FO, ok.n_color = k->n_color, ok.samples = k->samples, ok.a2c = k->a2c, ok.logic_op = k->logic_op; - ok.blend_en = k->blend_en, ok.depth = k->depth, ok.stencil = k->stencil, ok.wmask = k->wmask; - memcpy(ok.color, k->color, sizeof ok.color), memcpy(ok.blend, k->blend, sizeof ok.blend); + lib_keys(s, k, &vk_, &pk, &fk, &ok); int miss = 0; VkPipeline lib[4]; lib[1] = lib_get(LIB_PR, s, k, l, &pk, sizeof pk, build, &miss); // shader libraries first: no work when absent @@ -1750,8 +1828,9 @@ static struct { Job *head, *tail; // draws' misses Job *low, *low_tail; // prebuild uint32_t inflight; // queued or running, both queues - CONDITION_VARIABLE work, idle, done; // all with P.lock; done: a job finished -} A = {.work = CONDITION_VARIABLE_INIT, .idle = CONDITION_VARIABLE_INIT, .done = CONDITION_VARIABLE_INIT}; + int low_running; // pool workers running a job taken from the low queue + CONDITION_VARIABLE idle; // with P.lock: no job queued or running +} A = {.idle = CONDITION_VARIABLE_INIT}; // ---- pipeline keys (build/v121/pipeline-keys.bin next to the pipeline cache) // Every graphics pipeline built in a game run appends a KeyRec: the VkPipeState it was built from with the shader @@ -1920,14 +1999,33 @@ static DWORD WINAPI pipe_worker(void *arg) { int me = (int)(intptr_t)arg; AcquireSRWLockExclusive(&P.lock); for (;;) { - while (!A.head && !A.low) SleepConditionVariableSRW(&A.work, &P.lock, INFINITE, 0); + // With 3 or more workers one never takes low-queue work, so a waiting draw's library starts at once + // (not with TOTK_PIPE_WAIT=draw). + while (!U.head && !A.head && !(A.low && (A.threads < 3 || !U.on || A.low_running < A.threads - 1))) { + U.idle++; + SleepConditionVariableSRW(&work_cv, &P.lock, INFINITE, 0); + U.idle--; + } + if (U.head) { // a library a draw waits for + LibJob *lj = U.head; + if (!(U.head = lj->next)) U.tail = NULL; + lj->b->job = NULL; + if (A.prio_low[me]) A.prio_low[me] = 0, SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_NORMAL); + U.by_worker++, U.running++; + lib_build(lj->b, lj->kind, &lj->s, &lj->k, lj->l); // releases P.lock while building + U.running--; + free(lj); + continue; + } Job *j = A.head; + int lowrun = !j; if (j) { if (!(A.head = j->next)) A.tail = NULL; } else { j = A.low; if ((A.low = j->next)) A.low->prev = NULL; else A.low_tail = NULL; + A.low_running++; } j->next = j->prev = NULL; j->worker = me; @@ -1943,12 +2041,13 @@ static DWORD WINAPI pipe_worker(void *arg) { j->p->fast_tried = 1; if (f && !j->p->info.pipeline) { j->p->fast = j->p->info.pipeline = f, P.g.wlinks++; - WakeAllConditionVariable(&A.done); + WakeAllConditionVariable(&done_cv); j->low = 1, j->worker = -1, P.g.requeued++; j->prev = NULL, j->next = A.low; if (A.low) A.low->prev = j; else A.low_tail = j; A.low = j; + A.low_running -= lowrun; continue; } else if (f) { vkDestroyPipeline(P.device, f, NULL); @@ -1974,8 +2073,9 @@ static DWORD WINAPI pipe_worker(void *arg) { K.promoted += j->demanded; if (!--K.left && !K.loading) prebuild_report(); } + A.low_running -= lowrun; if (!--A.inflight) WakeAllConditionVariable(&A.idle); - WakeAllConditionVariable(&A.done); + WakeAllConditionVariable(&done_cv); if (pipe && (!j->prebuild || j->demanded)) perf_count(PERF_CNT_PIPE_ASYNC); // a draw waited for it free(j); } @@ -2007,11 +2107,20 @@ static void workers_drain(void) { free(j); } A.low = A.low_tail = NULL; + for (LibJob *j = U.head, *n; j; j = n) { // queued library jobs too: their entries read as failed builds + n = j->next; + j->b->job = NULL, j->b->building = 0; + free(j); + } + U.head = U.tail = NULL; while (A.inflight) SleepConditionVariableSRW(&A.idle, &P.lock, INFINITE, 0); + while (U.running) SleepConditionVariableSRW(&done_cv, &P.lock, INFINITE, 0); ReleaseSRWLockExclusive(&P.lock); } void vk_pipeline_async_enable(void) { + const char *w = cfg_get("TOTK_PIPE_WAIT"); + U.on = !(w && !strcmp(w, "draw")); const char *e = cfg_get("TOTK_PIPE_SYNC"); if (e && *e && strcmp(e, "0")) return; A.on = 1; @@ -2034,7 +2143,7 @@ static void promote(Pipe *p) { if (A.tail) A.tail->next = j; else A.head = j; A.tail = j; - WakeConditionVariable(&A.work); + WakeConditionVariable(&work_cv); } else if (A.prio_low[j->worker]) { A.prio_low[j->worker] = 0; SetThreadPriority(A.thread[j->worker], THREAD_PRIORITY_NORMAL); @@ -2093,11 +2202,80 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode A.tail = j; } A.inflight++; - WakeConditionVariable(&A.work); + WakeConditionVariable(&work_cv); *queued = 1; return p; } +// A job still in the draw queue goes to the head of the low queue: its pipeline has (or is about to have) a fast-linked +// one, so the optimised build waits behind other draws' misses. Caller holds P.lock. +static void job_lower(Job *j) { + if (!j || j->worker >= 0 || j->low) return; + if (A.head == j) { + if (!(A.head = j->next)) A.tail = NULL; + } else { + Job *q = A.head; + while (q && q->next != j) q = q->next; + if (!q) return; + if (!(q->next = j->next)) A.tail = q; + } + j->low = 1, P.g.requeued++; + j->prev = NULL, j->next = A.low; + if (A.low) A.low->prev = j; + else A.low_tail = j; + A.low = j; +} + +// A draw waits for pipeline s: its missing pre-rasterization and fragment shader libraries go to the head of the +// worker queues (U), so they build in parallel on idle workers, and this thread sleeps until both are published or +// end (now_ms) passes. A job no worker took within 1 ms (all busy) is built here. Caller holds P.lock. +static void lib_wait(const VkPipeState *s, const GKey *k, Layout *l, double end) { + ViKey vk_; + PrKey pk; + FsKey fk; + FoKey ok; + lib_keys(s, k, &vk_, &pk, &fk, &ok); + U.waits++; + const void *key[2] = {&pk, &fk}; + uint32_t size[2] = {sizeof pk, sizeof fk}; + Lib *b[2]; + int queued = 0; + for (int i = 0; i < 2; i++) { + uint64_t h = key_hash(key[i], size[i]); + if ((b[i] = lib_find(h, key[i], size[i]))) continue; + if (!half_layout(l, i)) return; // gpl_link reports it + LibJob *j = calloc(1, sizeof *j); + if (!j || !(b[i] = lib_new(h, key[i], size[i]))) { + free(j); + return; + } + *j = (LibJob){.b = b[i], .kind = i ? LIB_FS : LIB_PR, .l = l, .s = *s, .k = *k}; + b[i]->job = j; + if (U.tail) U.tail->next = j; + else U.head = j; + U.tail = j; + U.queued++, queued++; + } + if (queued) WakeAllConditionVariable(&work_cv); + double steal_at = now_ms() + 1.0; + for (;;) { + if (!b[0]->building && !b[1]->building) return; + double now = now_ms(); + if (now >= end) { + U.timeouts++; + return; + } + Lib *q = b[0]->job ? b[0] : b[1]->job ? b[1] : NULL; + if (q && (now >= steal_at || !U.idle)) { + U.stolen++; + lib_steal(q); // releases P.lock while building + continue; + } + double until = q && steal_at < end ? steal_at : end; + SleepConditionVariableSRW(&done_cv, &P.lock, (DWORD)(until - now) + 1, 0); + } +} + static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int async, int *pending, uint32_t wait_ms) { if (info) *info = NULL; if (pending) *pending = 0; @@ -2113,46 +2291,45 @@ static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int as } else { p = pipe_miss(s, &k, h, async, &queued); } - // A draw that would wait (wait_ms) builds the missing shader libraries itself and links (about 0.12 ms per library - // from the driver's disk cache): the workers may all be in optimised builds of earlier pipelines, and its job would - // queue behind them. Its job then only does the optimised build, from the low queue. Other draws leave the - // libraries to the job and are skipped meanwhile. TOTK_GPL_DRAW_BUILD=0 leaves them to the job too. + // A draw that would wait (wait_ms) gets its missing shader libraries built at once, the vertex and fragment ones in + // parallel on idle workers (lib_wait; about 0.12 ms per library from the driver's disk cache, 5-180 ms cold), and + // links them here: its own job may queue behind optimised builds of earlier pipelines. The job then only does the + // optimised build, from the low queue. Other draws leave the libraries to the job and are skipped meanwhile. + // TOTK_PIPE_WAIT=draw builds them on this thread one after the other instead, as before; TOTK_GPL_DRAW_BUILD=0 + // leaves them to the job. static int draw_build = -1; if (draw_build < 0) { const char *e = cfg_get("TOTK_GPL_DRAW_BUILD"); draw_build = !(e && !strcmp(e, "0")); + rt_log("vk_pipeline: a draw that waits for its pipeline %s", !draw_build ? "leaves its libraries to its job" + : U.on ? "has its libraries built in parallel on the workers, ahead of other jobs" + : "builds its libraries itself (TOTK_PIPE_WAIT=draw)"); } + double end = wait_ms ? now_ms() + wait_ms : 0; if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && p->lay) { - int final, build = wait_ms != 0 && draw_build; - if (build) p->fast_tried = 1; // the job, if a worker takes it meanwhile, goes straight to the optimised build + int final = 0, build = wait_ms != 0 && draw_build; uint64_t p0 = perf_begin(), t0 = qpc(); - VkPipeline f = gpl_link(s, &k, p->lay, build, &final); // build 1 releases P.lock while building + if (build && U.on) { + job_lower(p->job); // its worker would only wait for the same libraries; it links them if this draw cannot + lib_wait(s, &k, (Layout *)p->lay, end); // releases P.lock while waiting + build = 0; // links here when both libraries exist (gpl_link build 0) + } else if (build) { + p->fast_tried = 1; // the job, if a worker takes it meanwhile, goes straight to the optimised build + } + VkPipeline f = p->info.pipeline ? VK_NULL_HANDLE : gpl_link(s, &k, p->lay, build, &final); // build 1 releases P.lock perf_end(PERF_PIPE, p0); - if (build) { + if (wait_ms && draw_build) { double ms = ms_since(t0); P.g.dlinks++, P.g.dlink_ms += ms; if (ms > P.g.dlink_max) P.g.dlink_max = ms; - if (ms > 5.0 && P.g.slow++ < 16) rt_log("vk_pipeline: gpl: a waiting draw built its libraries in %.1f ms", ms); + if (ms > 5.0 && P.g.slow++ < 16) + rt_log("vk_pipeline: gpl: a waiting draw %s its libraries in %.1f ms", U.on ? "got" : "built", ms); } if (final) p->fast_tried = 1; if (f && !p->info.pipeline) { p->fast = p->info.pipeline = f; - Job *j = p->job; - if (j && j->worker < 0 && !j->low) { // still in the draw queue: to the head of the low queue - if (A.head == j) { - if (!(A.head = j->next)) A.tail = NULL; - } else { - Job *q = A.head; - while (q && q->next != j) q = q->next; - if (q && !(q->next = j->next)) A.tail = q; - } - j->low = 1, P.g.requeued++; - j->prev = NULL, j->next = A.low; - if (A.low) A.low->prev = j; - else A.low_tail = j; - A.low = j; - } - WakeAllConditionVariable(&A.done); + job_lower(p->job); + WakeAllConditionVariable(&done_cv); } else if (f) { vkDestroyPipeline(P.device, f, NULL); // a worker published one while the lock was free } @@ -2160,9 +2337,9 @@ static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int as if (p && p->building && !p->info.pipeline) { promote(p); uint64_t p0 = wait_ms ? perf_begin() : 0; // the wait counts as the render thread's pipeline time - double end = wait_ms ? now_ms() + wait_ms : 0, now; + double now; while (p->building && !p->info.pipeline && wait_ms && (now = now_ms()) < end) - SleepConditionVariableSRW(&A.done, &P.lock, (DWORD)(end - now) + 1, 0); + SleepConditionVariableSRW(&done_cv, &P.lock, (DWORD)(end - now) + 1, 0); if (wait_ms) perf_end(PERF_PIPE, p0); if (wait_ms && !p->building) perf_count(PERF_CNT_PIPE_WAIT); } @@ -2218,7 +2395,7 @@ static DWORD WINAPI warm_builder(void *arg) { K.promoted += j->demanded; K.left--; if (!--A.inflight) WakeAllConditionVariable(&A.idle); - WakeAllConditionVariable(&A.done); + WakeAllConditionVariable(&done_cv); free(j); } ReleaseSRWLockExclusive(&P.lock); @@ -2317,7 +2494,7 @@ void vk_pipeline_prewarm(void) { } for (int i = 0; i < nt; i++) CloseHandle(t[i]); AcquireSRWLockExclusive(&P.lock); - while (K.left) SleepConditionVariableSRW(&A.done, &P.lock, INFINITE, 0); // jobs a pool worker took + while (K.left) SleepConditionVariableSRW(&done_cv, &P.lock, INFINITE, 0); // jobs a pool worker took rt_log("vk_pipeline: prewarm: %zu records in %s: %u pipelines built, %u failed, %u already built, %u dropped " "(shader neither cached nor stored); shaders %.0f ms, pipelines %.0f ms on %d threads, total %.0f ms", n, K.path, K.built, K.failed, known, dropped, t1 - t0, now_ms() - t1, nt, now_ms() - t0); @@ -2822,6 +2999,11 @@ void vk_pipeline_save(void) { P.g.link_max, (unsigned long long)P.g.link_fail, (unsigned long long)P.g.defer, (unsigned long long)P.g.swaps, (unsigned long long)P.g.requeued, (unsigned long long)P.g.slow, (unsigned long long)P.g.dlinks, P.g.dlinks ? P.g.dlink_ms / P.g.dlinks : 0.0, P.g.dlink_max); + if (U.waits || U.queued) + rt_log("vk_pipeline: gpl: library jobs for waiting draws: %llu queued, %llu built by a worker, %llu by the " + "thread that needed them (no worker free), %llu waits ended at their limit", + (unsigned long long)U.queued, (unsigned long long)U.by_worker, (unsigned long long)U.stolen, + (unsigned long long)U.timeouts); static const char *const kn[4] = {"vertex input", "pre-rasterization", "fragment shader", "fragment output"}; for (int i = 0; i < 4; i++) rt_log("vk_pipeline: gpl: %s libraries %llu (avg %.2f ms, max %.2f ms)", kn[i], (unsigned long long)P.g.lib_n[i],