diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 03448928..3ff91b2d 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -90,12 +90,12 @@ void vk_exec_frame_log(const tex_t *p, uint64_t frame) { fw[i].h, fw[i].fmt, fw[i].n); } rt_log("frame %llu: submits %llu passes %llu draws %llu/%llu clears %llu copies %llu dispatches %llu; present %p " - "%ux%u/%#x written %d; pass ends: targets %llu barrier %llu (%llu merged) bits %llu/%llu/%llu/%llu/%llu/%llu/%llu/%llu; feedback draws %llu barriers %llu restarts %llu layouts %llu; targets%s", + "%ux%u/%#x written %d; pass ends: targets %llu barrier %llu (%llu merged, %llu undone) bits %llu/%llu/%llu/%llu/%llu/%llu/%llu/%llu; feedback draws %llu barriers %llu restarts %llu layouts %llu; targets%s", (unsigned long long)frame, (unsigned long long)f->submits, (unsigned long long)f->passes, (unsigned long long)f->draws, (unsigned long long)f->draws_seen, (unsigned long long)f->clears, (unsigned long long)f->copies, (unsigned long long)f->dispatches, p ? (const void *)p->mem : NULL, p ? p->d.width : 0, p ? p->d.height : 0, p ? (uint32_t)p->d.format : 0, - hit, (unsigned long long)f->pass_end_rt, (unsigned long long)f->pass_end_barrier, (unsigned long long)f->barrier_merged, (unsigned long long)f->barrier_bit[0], (unsigned long long)f->barrier_bit[1], (unsigned long long)f->barrier_bit[2], (unsigned long long)f->barrier_bit[3], (unsigned long long)f->barrier_bit[4], (unsigned long long)f->barrier_bit[5], (unsigned long long)f->barrier_bit[6], (unsigned long long)f->barrier_bit[7], (unsigned long long)f->fb_draws, (unsigned long long)f->fb_barriers, (unsigned long long)f->fb_restarts, + hit, (unsigned long long)f->pass_end_rt, (unsigned long long)f->pass_end_barrier, (unsigned long long)f->barrier_merged, (unsigned long long)f->barrier_undone, (unsigned long long)f->barrier_bit[0], (unsigned long long)f->barrier_bit[1], (unsigned long long)f->barrier_bit[2], (unsigned long long)f->barrier_bit[3], (unsigned long long)f->barrier_bit[4], (unsigned long long)f->barrier_bit[5], (unsigned long long)f->barrier_bit[6], (unsigned long long)f->barrier_bit[7], (unsigned long long)f->fb_draws, (unsigned long long)f->fb_barriers, (unsigned long long)f->fb_restarts, (unsigned long long)f->fb_layouts, list); memset(f, 0, sizeof *f); nfw = 0; @@ -136,6 +136,7 @@ static void state_reset(VkExec *ex) { vk_pipe_state_defaults(&ex->pipe); ex->barrier_at = -1; ex->oq_open = -1; + ex->bmerge = (uint8_t)vk_exec_barrier_merge_on(); vk_exec_dirty_all(ex); } @@ -644,7 +645,7 @@ static VkImageMemoryBarrier2 layout_barrier(VkImage img, VkImageAspectFlags aspe // go back to GENERAL and the new ones leave it, in the same command. static void barrier_to(VkExec *ex, const VkImage *want, const VkImageAspectFlags *wasp, int nwant) { vk_exec_pass_end(ex); - ex->pass_storage = 0; + ex->pass_storage = ex->pass_merged = 0; // barrier_merge: every pass begins behind this barrier vk_exec_ub_flush(ex); // uniform updates overlaid in the pass reach their buffers before the barrier below VkImageMemoryBarrier2 ib[18]; uint32_t n = 0; @@ -896,11 +897,25 @@ static void clear_ds(VkExec *ex, const p_clear_ds *c) { // made visible to a later fragment shader read of the open pass, feedback_prepare / feedback_barrier already do before the // draw that samples it, with a by-region barrier inside the pass. So the pass keeps going, unless a draw of it has a shader // that may write a storage buffer or image (pass_storage), which only a full barrier makes visible, or the barrier asks for -// more than those two bits, or the feedback layout is not in use. -static int barrier_merges(const VkExec *ex, int bits) { +// more than those two bits, or the feedback layout is not in use, or a target is a 3D BC stand-in (its texture gets the +// pass's writes only at the pass end, bc_copy_back). The reads the by-region barrier does not cover end the pass at the +// draw that makes them (vk_exec_draw.c merge_guard). The by-region barrier orders only reads of the pixel a fragment +// writes; NVN's ORDER_FRAGMENTS orders every pixel (ORDER_FRAGMENTS_TILED is the per-region one), so a draw that reads a +// neighbouring pixel of a target written in the pass relies on the driver running the barrier for the whole target, as +// RADV does; hence experimental. +int vk_exec_barrier_merge_on(void) { static int on = -1; if (on < 0) on = cfg_bool("TOTK_BARRIER_MERGE"); - return on && ex->in_pass && !ex->pass_storage && (bits & 0x10) && !(bits & ~0x12) && vk_ctx()->feedback == VK_FEEDBACK_LAYOUT; + return on; +} + +static int barrier_merges(const VkExec *ex, int bits) { + if (!ex->bmerge || !ex->in_pass || ex->pass_storage || !(bits & 0x10) || (bits & ~0x12) || + vk_ctx()->feedback != VK_FEEDBACK_LAYOUT) + return 0; + for (int i = 0; i < ex->pass_ncolor && i < 8; i++) + if (ex->color[i].image && ex->color[i].img && ex->color[i].img->image != ex->color[i].image) return 0; + return 1; } static void set_rt(VkExec *ex, const p_set_rt *r) { @@ -1071,11 +1086,13 @@ static int walk(VkExec *ex, const op_hdr *h, int depth, const op_hdr *stop) { case OP_CLEAR_DS: clear_ds(ex, a); break; case OP_BARRIER: if (barrier_merges(ex, ((const p_barrier *)a)->bits)) { // barrier_merge: the pass goes on (see barrier_merges) + ex->pass_merged = 1; ex->fr.barrier_merged++; break; } if (ex->in_pass) ex->fr.pass_end_barrier++; - for (int i = 0; i < 8; i++) ex->fr.barrier_bit[i] += (uint32_t)((const p_barrier *)a)->bits >> i & 1; + if (rt_trace & RT_TRACE_GFX) + for (int i = 0; i < 8; i++) ex->fr.barrier_bit[i] += (uint32_t)((const p_barrier *)a)->bits >> i & 1; vk_exec_barrier(ex); break; case OP_DRAW: case OP_DRAW_IDX: case OP_DISPATCH: case OP_UPDATE_UB: case OP_DRAW_IDX_IND: case OP_DISPATCH_IND: diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 839d586b..ff3e5907 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -56,6 +56,7 @@ typedef struct { uint64_t passes, draws_seen, draws, clears, copies, dispatches, submits; uint64_t barrier_bit[8]; // the game's Barrier calls that carry NVN barrier bit 0..7 (TOTK_TRACE=gfx) uint64_t barrier_merged; // the game's Barrier calls the open pass went on through (barrier_merge) + uint64_t barrier_undone; // passes a draw then ended for a read the merge cannot order (merge_guard) uint64_t pass_end_rt, pass_end_barrier; // passes ended by other render targets, by the game's own Barrier (TOTK_TRACE=gfx) uint64_t fb_draws, fb_barriers, fb_restarts, fb_layouts; // feedback loops: draws sampling a written target of // their pass, in-pass barriers, pass restarts, layout @@ -158,10 +159,15 @@ typedef struct VkExec { int nfbl; uint32_t fbl_mask; // attachments of the open pass in that layout: bit i colour i, bit 8 depth uint32_t fb_dirty; // attachments of the open pass written by a draw or clear since the pass began - uint8_t pass_storage; // a draw of the open pass has a shader that may write a storage buffer or image (barrier_merge) // or the last in-pass barrier const void *fb_fs; // feedback_scan's memo: fragment shader, texture bind serial, pass serial uint32_t fb_serial, fb_pass, fb_sampled; + // barrier_merge (vk_exec.c barrier_merges): bmerge is TOTK_BARRIER_MERGE, read once when the context is reset; + // pass_storage: a draw of the open pass may write a storage buffer or image; pass_merged: the open pass went on + // through a game Barrier. Both are cleared by the full barrier of every pass begin (barrier_to). + uint8_t bmerge, pass_storage, pass_merged; + const void *bm_vs; // merge_guard's memo: vertex shader, texture bind serial, pass serial, answer + uint32_t bm_serial, bm_pass, bm_hit; uint32_t tex_serial, pass_serial; // bumped by every texture or image bind and pass begin // TOTK_BIND_INC (vk_exec_draw.c): the slots the bind ops changed since this context's last prepared draw, cleared // by that draw; vk_exec_draw_new_cb and copy_state set every bit. pipe_gen: bumped by every write that changes @@ -203,6 +209,7 @@ void vk_exec_pass_end(VkExec *ex); // Full memory barrier (all commands, all memory); ends the open pass first. Copies and dispatches order themselves with // it. Skipped when nothing was recorded since the last one. void vk_exec_barrier(VkExec *ex); +int vk_exec_barrier_merge_on(void); // TOTK_BARRIER_MERGE, read once (vk_exec.c barrier_merges) // UpdateUniformBuffer overlays (vk_exec_draw.c): the copies into the real buffers that updates inside a pass left for // the next barrier (barrier_to records them first), and a GPU write of another kind into [addr, addr+n) of this // submission (a copy, a fill, a storage write), which retires the overlays it touches. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index c61f07a9..42513cc2 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -2992,7 +2992,8 @@ static int pipe_memo_on(void) { return on; } -// A shader that may write a storage buffer or image: a written SSBO (flags bit 0) or any storage image. +// barrier_merge (vk_exec.c barrier_merges). A shader that may write a storage buffer or image: a written SSBO (flags +// bit 0) or any storage image. gfx_shader_resources only reads the shader's parsed arrays. static int shader_may_write_storage(const GfxShader *sh) { const GfxShaderRes *r; if (!sh) return 0; @@ -3003,6 +3004,32 @@ static int shader_may_write_storage(const GfxShader *sh) { return 0; } +// barrier_merge: a draw in a pass that went on through a game Barrier whose reads the in-pass by-region barrier +// (feedback_barrier: fragment shader sampling only, found by feedback_scan) would not order. Those are a storage image +// of either stage (an imageLoad of a target) and a vertex shader texture that is an attachment of the pass. Such a draw +// ends the pass, and the next one begins behind the full barrier, as the game's Barrier would have done. The vertex +// scan is kept while the shader, the texture bindings and the pass stay the same, as feedback_scan's. +static int merge_guard(VkExec *ex, const GfxShader *vs, const GfxShader *fs) { + const GfxShaderRes *res; + if (gfx_shader_resources(vs, GFX_RES_IMG, &res) || (fs && gfx_shader_resources(fs, GFX_RES_IMG, &res))) return 1; + int n = gfx_shader_resources(vs, GFX_RES_TEX, &res); + if (!n) return 0; + if (ex->bm_vs == vs && ex->bm_serial == ex->tex_serial && ex->bm_pass == ex->pass_serial) return (int)ex->bm_hit; + uint32_t m = 0; + vk_texture_lock(); + for (int r = 0; r < n && !m; r++) + for (int k = 0; k < (res[r].count ? res[r].count : 1) && !m; k++) { + int slot = res[r].slot + k; + uint64_t h = slot >= 0 && slot < 128 ? ex->tex[NVN_STAGE_VS][slot] : 0; + if (!h) continue; + SubRange sr = sampled_sub(h); + m = attached_as(ex, &sr); + } + vk_texture_unlock(); + ex->bm_vs = vs, ex->bm_serial = ex->tex_serial, ex->bm_pass = ex->pass_serial, ex->bm_hit = m != 0; + return m != 0; +} + // ---- incremental draw preparation (TOTK_BIND_INC) // A draw is prepared from what changed since the context's last prepared draw in the same command buffer: the bind // ops mark the slots whose bytes they changed (VkExec.dirt, vk_exec_bind), every write that changes the pipeline state @@ -3276,8 +3303,12 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { } if (!vs || (has_fs && !fs)) return 0; if (vk_dlaa_skip_draw(ex, fs)) return 0; // the game's FXAA off, or its scaling replaced by DLSS (vk_dlaa.c) + if (ex->pass_merged && merge_guard(ex, vs, fs)) { // barrier_merge: the full barrier the merged Barrier skipped + vk_exec_pass_end(ex); + vk_exec_pass_begin(ex); + ex->fr.barrier_undone++; + } uint32_t fbl = feedback_prepare(ex, fs); - if (!ex->pass_storage && (shader_may_write_storage(vs) || shader_may_write_storage(fs))) ex->pass_storage = 1; uint8_t fb = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0)), dmw = (uint8_t)(nvn_depth_minus_w() != 0); // TOTK_BIND_INC: the pipeline memo's state is this draw's when pipe was not written since the memo's state was built // from it and the fields a draw sets are the same (y_flip is always 1): its pipeline is taken without building the @@ -3412,6 +3443,8 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (vk_hdr_track) vk_hdr_draw(ex, info); vk_exec_oq_draw(ex); if (bim) bi_done(ex, info, sp, bgt, bgb, bsg); + // barrier_merge: noted last, in the pass the draw is recorded in (bind_resources may restart the pass, ub_written) + if (ex->bmerge && !ex->pass_storage && (shader_may_write_storage(vs) || shader_may_write_storage(fs))) ex->pass_storage = 1; return 1; }