diff --git a/runtime/bl_deswizzle_spv.h b/runtime/bl_deswizzle_spv.h new file mode 100644 index 00000000..4bb18500 --- /dev/null +++ b/runtime/bl_deswizzle_spv.h @@ -0,0 +1,111 @@ +// Generated by tools/build_shaders.py from runtime/shaders/bl_deswizzle.comp; do not edit. +// source sha256 bd2065290e315e85d2fcdb609d812c15ba9a109d7506b3105850a5860677b0ef +#pragma once +#include +static const uint32_t bl_deswizzle_spv[] = { + 0x07230203, 0x00010300, 0x000d000b, 0x000000d0, 0x00000000, 0x00020011, 0x00000001, 0x0006000b, + 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001, + 0x0007000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000b, 0x0000001f, 0x00060010, + 0x00000004, 0x00000011, 0x00000040, 0x00000001, 0x00000001, 0x00040047, 0x0000000b, 0x0000000b, + 0x0000001a, 0x00030047, 0x00000010, 0x00000002, 0x00050048, 0x00000010, 0x00000000, 0x00000023, + 0x00000000, 0x00050048, 0x00000010, 0x00000001, 0x00000023, 0x00000004, 0x00050048, 0x00000010, + 0x00000002, 0x00000023, 0x00000008, 0x00050048, 0x00000010, 0x00000003, 0x00000023, 0x0000000c, + 0x00050048, 0x00000010, 0x00000004, 0x00000023, 0x00000010, 0x00050048, 0x00000010, 0x00000005, + 0x00000023, 0x00000014, 0x00050048, 0x00000010, 0x00000006, 0x00000023, 0x00000018, 0x00050048, + 0x00000010, 0x00000007, 0x00000023, 0x0000001c, 0x00050048, 0x00000010, 0x00000008, 0x00000023, + 0x00000020, 0x00050048, 0x00000010, 0x00000009, 0x00000023, 0x00000024, 0x00050048, 0x00000010, + 0x0000000a, 0x00000023, 0x00000028, 0x00040047, 0x0000001f, 0x0000000b, 0x0000001b, 0x00040047, + 0x000000ad, 0x00000006, 0x00000010, 0x00030047, 0x000000ae, 0x00000002, 0x00040048, 0x000000ae, + 0x00000000, 0x00000019, 0x00050048, 0x000000ae, 0x00000000, 0x00000023, 0x00000000, 0x00030047, + 0x000000b0, 0x00000019, 0x00040047, 0x000000b0, 0x00000021, 0x00000001, 0x00040047, 0x000000b0, + 0x00000022, 0x00000000, 0x00040047, 0x000000c1, 0x00000006, 0x00000010, 0x00030047, 0x000000c2, + 0x00000002, 0x00040048, 0x000000c2, 0x00000000, 0x00000018, 0x00050048, 0x000000c2, 0x00000000, + 0x00000023, 0x00000000, 0x00030047, 0x000000c4, 0x00000018, 0x00040047, 0x000000c4, 0x00000021, + 0x00000000, 0x00040047, 0x000000c4, 0x00000022, 0x00000000, 0x00040047, 0x000000cd, 0x0000000b, + 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006, + 0x00000020, 0x00000000, 0x00040017, 0x00000009, 0x00000006, 0x00000003, 0x00040020, 0x0000000a, + 0x00000001, 0x00000009, 0x0004003b, 0x0000000a, 0x0000000b, 0x00000001, 0x0004002b, 0x00000006, + 0x0000000c, 0x00000001, 0x00040020, 0x0000000d, 0x00000001, 0x00000006, 0x000d001e, 0x00000010, + 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, + 0x00000006, 0x00000006, 0x00000006, 0x00040020, 0x00000011, 0x00000009, 0x00000010, 0x0004003b, + 0x00000011, 0x00000012, 0x00000009, 0x00040015, 0x00000013, 0x00000020, 0x00000001, 0x0004002b, + 0x00000013, 0x00000014, 0x0000000a, 0x00040020, 0x00000015, 0x00000009, 0x00000006, 0x0004002b, + 0x00000006, 0x00000019, 0x00000000, 0x0004002b, 0x00000006, 0x0000001d, 0x00000040, 0x0004003b, + 0x0000000a, 0x0000001f, 0x00000001, 0x0004002b, 0x00000013, 0x00000024, 0x00000002, 0x00020014, + 0x00000027, 0x0004002b, 0x00000013, 0x0000002d, 0x00000006, 0x0004002b, 0x00000013, 0x00000032, + 0x00000007, 0x0004002b, 0x00000006, 0x00000037, 0x00000020, 0x0004002b, 0x00000006, 0x00000048, + 0x00000005, 0x0004002b, 0x00000006, 0x0000004c, 0x0000001f, 0x0004002b, 0x00000013, 0x0000004f, + 0x00000008, 0x0004002b, 0x00000013, 0x00000052, 0x00000009, 0x0004002b, 0x00000006, 0x0000006a, + 0x00000004, 0x0004002b, 0x00000006, 0x0000006f, 0x00000002, 0x0004002b, 0x00000006, 0x00000078, + 0x00000008, 0x0004002b, 0x00000006, 0x00000083, 0x00000003, 0x0004002b, 0x00000013, 0x00000093, + 0x00000003, 0x0004002b, 0x00000013, 0x0000009b, 0x00000004, 0x0004002b, 0x00000013, 0x000000a4, + 0x00000005, 0x00040017, 0x000000ac, 0x00000006, 0x00000004, 0x0003001d, 0x000000ad, 0x000000ac, + 0x0003001e, 0x000000ae, 0x000000ad, 0x00040020, 0x000000af, 0x0000000c, 0x000000ae, 0x0004003b, + 0x000000af, 0x000000b0, 0x0000000c, 0x0004002b, 0x00000013, 0x000000b1, 0x00000000, 0x0004002b, + 0x00000013, 0x000000b2, 0x00000001, 0x0003001d, 0x000000c1, 0x000000ac, 0x0003001e, 0x000000c2, + 0x000000c1, 0x00040020, 0x000000c3, 0x0000000c, 0x000000c2, 0x0004003b, 0x000000c3, 0x000000c4, + 0x0000000c, 0x00040020, 0x000000c9, 0x0000000c, 0x000000ac, 0x0006002c, 0x00000009, 0x000000cd, + 0x0000001d, 0x0000000c, 0x0000000c, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, + 0x000200f8, 0x00000005, 0x000300f7, 0x000000ce, 0x00000000, 0x000300fb, 0x00000019, 0x000000cf, + 0x000200f8, 0x000000cf, 0x00050041, 0x0000000d, 0x0000000e, 0x0000000b, 0x0000000c, 0x0004003d, + 0x00000006, 0x0000000f, 0x0000000e, 0x00050041, 0x00000015, 0x00000016, 0x00000012, 0x00000014, + 0x0004003d, 0x00000006, 0x00000017, 0x00000016, 0x00050084, 0x00000006, 0x00000018, 0x0000000f, + 0x00000017, 0x00050041, 0x0000000d, 0x0000001a, 0x0000000b, 0x00000019, 0x0004003d, 0x00000006, + 0x0000001b, 0x0000001a, 0x00050080, 0x00000006, 0x0000001c, 0x00000018, 0x0000001b, 0x00050084, + 0x00000006, 0x0000001e, 0x0000001c, 0x0000001d, 0x00050041, 0x0000000d, 0x00000020, 0x0000001f, + 0x00000019, 0x0004003d, 0x00000006, 0x00000021, 0x00000020, 0x00050080, 0x00000006, 0x00000022, + 0x0000001e, 0x00000021, 0x00050041, 0x00000015, 0x00000025, 0x00000012, 0x00000024, 0x0004003d, + 0x00000006, 0x00000026, 0x00000025, 0x000500ae, 0x00000027, 0x00000028, 0x00000022, 0x00000026, + 0x000300f7, 0x0000002a, 0x00000000, 0x000400fa, 0x00000028, 0x00000029, 0x0000002a, 0x000200f8, + 0x00000029, 0x000200f9, 0x000000ce, 0x000200f8, 0x0000002a, 0x00050041, 0x00000015, 0x0000002e, + 0x00000012, 0x0000002d, 0x0004003d, 0x00000006, 0x0000002f, 0x0000002e, 0x000500c4, 0x00000006, + 0x00000030, 0x0000000c, 0x0000002f, 0x00050041, 0x00000015, 0x00000033, 0x00000012, 0x00000032, + 0x0004003d, 0x00000006, 0x00000034, 0x00000033, 0x000500c4, 0x00000006, 0x00000035, 0x0000000c, + 0x00000034, 0x00050080, 0x00000006, 0x0000003c, 0x0000002f, 0x00000034, 0x000500c4, 0x00000006, + 0x0000003d, 0x00000037, 0x0000003c, 0x00050086, 0x00000006, 0x00000041, 0x00000022, 0x0000003d, + 0x00050089, 0x00000006, 0x00000045, 0x00000022, 0x0000003d, 0x000500c2, 0x00000006, 0x00000049, + 0x00000045, 0x00000048, 0x000500c7, 0x00000006, 0x0000004d, 0x00000045, 0x0000004c, 0x00050041, + 0x00000015, 0x00000050, 0x00000012, 0x0000004f, 0x0004003d, 0x00000006, 0x00000051, 0x00000050, + 0x00050041, 0x00000015, 0x00000053, 0x00000012, 0x00000052, 0x0004003d, 0x00000006, 0x00000054, + 0x00000053, 0x00050084, 0x00000006, 0x00000055, 0x00000051, 0x00000054, 0x00050086, 0x00000006, + 0x00000059, 0x00000041, 0x00000055, 0x00050089, 0x00000006, 0x0000005d, 0x00000041, 0x00000055, + 0x00050086, 0x00000006, 0x00000062, 0x0000005d, 0x00000051, 0x00050089, 0x00000006, 0x00000067, + 0x0000005d, 0x00000051, 0x00050084, 0x00000006, 0x0000006b, 0x00000067, 0x0000006a, 0x000500c2, + 0x00000006, 0x0000006d, 0x0000004d, 0x0000006a, 0x000500c7, 0x00000006, 0x0000006e, 0x0000006d, + 0x0000000c, 0x00050084, 0x00000006, 0x00000070, 0x0000006e, 0x0000006f, 0x00050080, 0x00000006, + 0x00000071, 0x0000006b, 0x00000070, 0x000500c2, 0x00000006, 0x00000073, 0x0000004d, 0x0000000c, + 0x000500c7, 0x00000006, 0x00000074, 0x00000073, 0x0000000c, 0x00050080, 0x00000006, 0x00000075, + 0x00000071, 0x00000074, 0x00050084, 0x00000006, 0x00000079, 0x00000062, 0x00000078, 0x00050084, + 0x00000006, 0x0000007b, 0x00000079, 0x00000030, 0x00050089, 0x00000006, 0x0000007e, 0x00000049, + 0x00000030, 0x00050084, 0x00000006, 0x0000007f, 0x0000007e, 0x00000078, 0x00050080, 0x00000006, + 0x00000080, 0x0000007b, 0x0000007f, 0x000500c2, 0x00000006, 0x00000082, 0x0000004d, 0x0000006f, + 0x000500c7, 0x00000006, 0x00000084, 0x00000082, 0x00000083, 0x00050084, 0x00000006, 0x00000085, + 0x00000084, 0x0000006f, 0x00050080, 0x00000006, 0x00000086, 0x00000080, 0x00000085, 0x000500c7, + 0x00000006, 0x00000088, 0x00000045, 0x0000000c, 0x00050080, 0x00000006, 0x00000089, 0x00000086, + 0x00000088, 0x00050084, 0x00000006, 0x0000008d, 0x00000059, 0x00000035, 0x00050086, 0x00000006, + 0x00000090, 0x00000049, 0x00000030, 0x00050080, 0x00000006, 0x00000091, 0x0000008d, 0x00000090, + 0x00050041, 0x00000015, 0x00000094, 0x00000012, 0x00000093, 0x0004003d, 0x00000006, 0x00000095, + 0x00000094, 0x000500ae, 0x00000027, 0x00000096, 0x00000075, 0x00000095, 0x000400a8, 0x00000027, + 0x00000097, 0x00000096, 0x000300f7, 0x00000099, 0x00000000, 0x000400fa, 0x00000097, 0x00000098, + 0x00000099, 0x000200f8, 0x00000098, 0x00050041, 0x00000015, 0x0000009c, 0x00000012, 0x0000009b, + 0x0004003d, 0x00000006, 0x0000009d, 0x0000009c, 0x000500ae, 0x00000027, 0x0000009e, 0x00000089, + 0x0000009d, 0x000200f9, 0x00000099, 0x000200f8, 0x00000099, 0x000700f5, 0x00000027, 0x0000009f, + 0x00000096, 0x0000002a, 0x0000009e, 0x00000098, 0x000400a8, 0x00000027, 0x000000a0, 0x0000009f, + 0x000300f7, 0x000000a2, 0x00000000, 0x000400fa, 0x000000a0, 0x000000a1, 0x000000a2, 0x000200f8, + 0x000000a1, 0x00050041, 0x00000015, 0x000000a5, 0x00000012, 0x000000a4, 0x0004003d, 0x00000006, + 0x000000a6, 0x000000a5, 0x000500ae, 0x00000027, 0x000000a7, 0x00000091, 0x000000a6, 0x000200f9, + 0x000000a2, 0x000200f8, 0x000000a2, 0x000700f5, 0x00000027, 0x000000a8, 0x0000009f, 0x00000099, + 0x000000a7, 0x000000a1, 0x000300f7, 0x000000aa, 0x00000000, 0x000400fa, 0x000000a8, 0x000000a9, + 0x000000aa, 0x000200f8, 0x000000a9, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000aa, 0x00050041, + 0x00000015, 0x000000b3, 0x00000012, 0x000000b2, 0x0004003d, 0x00000006, 0x000000b4, 0x000000b3, + 0x00050041, 0x00000015, 0x000000b6, 0x00000012, 0x0000009b, 0x0004003d, 0x00000006, 0x000000b7, + 0x000000b6, 0x00050084, 0x00000006, 0x000000b8, 0x00000091, 0x000000b7, 0x00050080, 0x00000006, + 0x000000ba, 0x000000b8, 0x00000089, 0x00050084, 0x00000006, 0x000000bd, 0x000000ba, 0x00000095, + 0x00050080, 0x00000006, 0x000000be, 0x000000b4, 0x000000bd, 0x00050080, 0x00000006, 0x000000c0, + 0x000000be, 0x00000075, 0x00050041, 0x00000015, 0x000000c5, 0x00000012, 0x000000b1, 0x0004003d, + 0x00000006, 0x000000c6, 0x000000c5, 0x00050080, 0x00000006, 0x000000c8, 0x000000c6, 0x00000022, + 0x00060041, 0x000000c9, 0x000000ca, 0x000000c4, 0x000000b1, 0x000000c8, 0x0004003d, 0x000000ac, + 0x000000cb, 0x000000ca, 0x00060041, 0x000000c9, 0x000000cc, 0x000000b0, 0x000000b1, 0x000000c0, + 0x0003003e, 0x000000cc, 0x000000cb, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000ce, 0x000100fd, + 0x00010038, +}; diff --git a/runtime/cfg.c b/runtime/cfg.c index 23d1756d..6136f26d 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -40,7 +40,7 @@ static const char *const known[] = { "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", - "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", + "DESC_TEMPLATE", "DESWIZZLE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DYNAMIC_RES", "DYN_DELTA", "EXEFS", "FEEDBACK", "FIXED_RES_PATCH", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FRAME_QUEUE", "FREEZE_IDLE", "FREE_CORES", diff --git a/runtime/shaders/bl_deswizzle.comp b/runtime/shaders/bl_deswizzle.comp new file mode 100644 index 00000000..f7a56e72 --- /dev/null +++ b/runtime/shaders/bl_deswizzle.comp @@ -0,0 +1,37 @@ +#version 450 +// Block-linear to linear copy of one texture level and layer on the GPU (vk_deswizzle.c), the same layout as +// tex_deswizzle.c nvn_bl_offset: GOB = 64 bytes x 8 rows; a block is 1 GOB wide, 2^th GOBs high and 2^td deep, its +// GOBs ordered y first, then z; blocks fill the level x first, then y, then z. +// One invocation per 16-byte segment of the block-linear storage, in storage order, so the reads (from the pool's +// imported buffer: guest memory, over PCIe on a discrete GPU) are contiguous; the writes go to a device-local buffer +// with rows of pitch16 segments. Segments outside the level's rows, slices or row width are padding and skipped. +layout(local_size_x = 64) in; +layout(std430, set = 0, binding = 0) readonly buffer Src { uvec4 src[]; }; +layout(std430, set = 0, binding = 1) writeonly buffer Dst { uvec4 dst[]; }; +layout(push_constant) uniform Push { + uint src_base; // first segment of the level and layer in the src binding + uint dst_base; // first segment of its linear copy in the dst binding + uint segs; // segments to visit: whole blocks covering the level + uint pitch16; // segments per linear row (row bytes rounded up to 16) + uint rows, slices, th, td; + uint gobs_x, blocks_y; + uint nx; // workgroups along x (the dispatch is 2D when one dimension is not enough) +} pc; + +void main() { + uint s = (gl_WorkGroupID.y * pc.nx + gl_WorkGroupID.x) * 64u + gl_LocalInvocationID.x; + if (s >= pc.segs) return; + uint bh = 1u << pc.th, bd = 1u << pc.td; + uint per_block = 32u << (pc.th + pc.td); // 512-byte GOBs of 32 segments each + uint block = s / per_block, inb = s % per_block; + uint gob = inb >> 5u, ing = inb & 31u; + uint per_layer = pc.gobs_x * pc.blocks_y; + uint bz = block / per_layer, rem = block % per_layer; + uint by = rem / pc.gobs_x, bx = rem % pc.gobs_x; + // Inside a GOB, segment bits: 4 = x % 64 / 32, 3..2 = y % 8 / 2, 1 = x % 32 / 16, 0 = y % 2. + uint xseg = bx * 4u + ((ing >> 4u) & 1u) * 2u + ((ing >> 1u) & 1u); + uint y = by * 8u * bh + (gob % bh) * 8u + ((ing >> 2u) & 3u) * 2u + (ing & 1u); + uint z = bz * bd + gob / bh; + if (xseg >= pc.pitch16 || y >= pc.rows || z >= pc.slices) return; + dst[pc.dst_base + (z * pc.rows + y) * pc.pitch16 + xseg] = src[pc.src_base + s]; +} diff --git a/runtime/vk.h b/runtime/vk.h index abe40f57..91a67370 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -329,6 +329,15 @@ void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestro void vk_defer_view(VkImageView view); void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc); +// free(p) once the GPU is past the submission being recorded, like the deferrals above: host records that a recording +// may still read through a pointer taken before another thread dropped them (vk_texture.c destroy_tex). +void vk_defer_free(void *p); +// Device-local scratch for the upload batch (storage, transfer src and dst), from a pool of its own (vk_memory.c). +VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc); +// Staging and scratch bytes the open upload batch holds until it completes. Large texture loads submit the batch early +// (vk_texture.c UPLOAD_HELD_MAX) so the GPU starts on it and its buffers come back before the pools need new blocks. +VkDeviceSize vk_upload_held(void); +void vk_upload_pools_init(void); // vk_device.c, after the allocator: the staging and scratch pools and their blocks // ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c. enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK }; int vk_astc_mode(void); // decided once per device (logged); VK_ASTC_CPU before vk_init @@ -352,7 +361,31 @@ void vk_astc_time_enable(int on); int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us); // totals so far void vk_astc_time_summary(void); // logs the totals void vk_astc_release(void); // vk_device.c, before the device goes: frees the decoder's objects -void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value +// ---- Block-linear deswizzle on the GPU (vk_deswizzle.c). TOTK_DESWIZZLE = auto | gpu | cpu | check. +enum { VK_DESWZ_CPU, VK_DESWZ_GPU, VK_DESWZ_CHECK }; +int vk_deswz_mode(void); // decided once per device (logged); VK_DESWZ_CPU before vk_init +// One level and layer: its storage starts at byte src of the bound source range and covers segs 16-byte segments +// (whole blocks, cut at the storage end); its linear copy goes to byte dst of the destination, rows of pitch16 segments, +// rows x slices of them. th, td, gobs_x, blocks_y: as tex_deswizzle.c nvn_bl_offset. +typedef struct { + VkDeviceSize src, dst; + uint32_t segs, pitch16, rows, slices, th, td, gobs_x, blocks_y; +} VkDeswzRegion; +// Records into cb a dispatch per region reading [src_off, src_off + src_size) of src (a storage buffer) and writing +// dst (device-local, dst_size bytes). The caller records the barriers around it. 0 when nothing was recorded: no +// pipeline, src_off not 16-byte aligned, a range over the device's storage-buffer limit, a region outside the ranges. +int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst, + VkDeviceSize dst_size, const VkDeswzRegion *rg, int n); +// 1 when vk_deswz_record will take these regions (their dst offsets aside) with destinations of up to max_dst bytes. +int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst); +void vk_deswz_release(void); // vk_device.c, before the device goes +// 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission +// sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch). +int vk_mem_span_live(const VkMemSpan *sp); +// vk_sync.c, when it starts recording the submission that will signal ctx->timeline with this value: objects destroyed +// from then on are freed once the timeline reaches it. +void vk_mem_note_fence(uint64_t timeline_value); +void vk_mem_fault_log(void); // vk_device_fault_report: the deferred-free state, lock-free void vk_mem_collect(void); // frees what has completed typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats; void vk_mem_stats(VkMemStats *s); @@ -384,6 +417,9 @@ typedef struct { uint8_t scale; // 1, or TOTK_SHADOW_SCALE for a shadow map made larger than the game's (vk_texture.c // shadow_scale_of): width and height above are the larger ones, the game's regions // are multiplied by it + uint8_t gone; // the entry was dropped (vk_texture.c destroy_tex, under the texture lock): image and + // record live until the submission being recorded is done, and nothing may be cached + // by its image handle, which the driver can hand out again after that } VkTexImage; // The window texture the backend presents (vk_backend.c): *image and *format become the float picture the game copied // into it, when the last draw into it was that copy and the swapchain is HDR10. diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index 0b91e70f..ccfa4235 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -463,13 +463,9 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons cap_in = (cap_in + al - 1) / al * al; cap_out = (cap_out + al - 1) / al * al; if (!bc) cap_bc = 0; - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = cap_in + cap_out + cap_bc, - .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | - VK_BUFFER_USAGE_TRANSFER_DST_BIT}; - VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; VkBuffer sb; VmaAllocation sa; - if (vmaCreateBuffer(c->vma, &bi, &aci, &sb, &sa, NULL) != VK_SUCCESS) { + if (vk_scratch_buffer(cap_in + cap_out + cap_bc, &sb, &sa) != VK_SUCCESS) { if (plat_inc32(&n_fail) <= 16) rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded", (unsigned long long)(cap_in + cap_out)); diff --git a/runtime/vk_deswizzle.c b/runtime/vk_deswizzle.c new file mode 100644 index 00000000..d70f0886 --- /dev/null +++ b/runtime/vk_deswizzle.c @@ -0,0 +1,176 @@ +// Block-linear textures deswizzled by the GPU (runtime/shaders/bl_deswizzle.comp): vk_texture.c upload binds the +// texture's storage in its pool's imported buffer and lets a compute pass write the linear copy that a buffer-to-image +// copy (or the ASTC decoder) then takes, instead of tex_deswizzle.c walking every 16 bytes on the replay thread +// (50-236 ms frames on entering new areas, 2026-11-07 play sessions). The same design as MacNativeGames' TotK port +// (d2r_gfx nvn_mtl.m: deswizzle from the pool at GPU execution time), written for Vulkan from our CPU layout code. +// +// TOTK_DESWIZZLE = auto (default) / gpu: the compute pass where the storage is an imported pool (vk_mem_span_live), +// the CPU otherwise; cpu: always the CPU; check: gpu, and every upload is also deswizzled on the CPU and compared +// (waits for the GPU; for testing only). +#include +#include + +#include "bl_deswizzle_spv.h" +#include "cfg.h" +#include "host.h" +#include "vk.h" +#include "vk_rec.h" +#include "platform/platform.h" + +typedef struct { + uint32_t src_base, dst_base, segs, pitch16, rows, slices, th, td, gobs_x, blocks_y, nx; +} Push; + +static PlatMutex g_lock = PLAT_MUTEX_INIT; +static struct { + VkDevice dev; + int mode; // VK_DESWZ_*; -1 before the first vk_deswz_mode with a device + VkDescriptorSetLayout dsl; + VkPipelineLayout layout; + VkPipeline pipe; + PFN_vkCmdPushDescriptorSetKHR push; +} G = {.mode = -1}; +static volatile int64_t n_regions, n_bytes; + +static int create_pipeline(VkCtx *c) { + G.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR"); + if (!G.push) return 0; + VkDescriptorSetLayoutBinding b[2]; + for (int i = 0; i < 2; i++) + b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}; + VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, + .bindingCount = 2, .pBindings = b}; + if (vkCreateDescriptorSetLayout(c->device, &dci, NULL, &G.dsl) != VK_SUCCESS) return 0; + VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push)}; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, + .pSetLayouts = &G.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr}; + if (vkCreatePipelineLayout(c->device, &li, NULL, &G.layout) != VK_SUCCESS) return 0; + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bl_deswizzle_spv, + .pCode = bl_deswizzle_spv}; + VkShaderModule m = VK_NULL_HANDLE; + if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = G.layout}; + VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.pipe); + vkDestroyShaderModule(c->device, m, NULL); + if (r != VK_SUCCESS) G.pipe = VK_NULL_HANDLE; + return r == VK_SUCCESS; +} + +int vk_deswz_mode(void) { + VkCtx *c = vk_ctx(); + if (!c) return VK_DESWZ_CPU; + plat_mutex_lock_shared(&g_lock); + int m = G.dev == c->device ? G.mode : -1; + plat_mutex_unlock_shared(&g_lock); + if (m >= 0) return m; + plat_mutex_lock(&g_lock); + if (G.dev != c->device) { // the old device's objects went with it (probes recreate the device) + memset(&G, 0, sizeof G); + G.dev = c->device; + G.mode = -1; + } + if (G.mode < 0) { + const char *e = cfg_get("TOTK_DESWIZZLE"); + if (!e || !*e) e = "auto"; + int want = !strcmp(e, "cpu") || !strcmp(e, "0") ? VK_DESWZ_CPU : !strcmp(e, "check") ? VK_DESWZ_CHECK : VK_DESWZ_GPU; + if (strcmp(e, "auto") && strcmp(e, "gpu") && strcmp(e, "1") && strcmp(e, "cpu") && strcmp(e, "0") && + strcmp(e, "check")) + rt_log("vk: TOTK_DESWIZZLE=%s is not auto, gpu, cpu or check: auto", e); + if (want != VK_DESWZ_CPU && !create_pipeline(c)) { + rt_log("vk: the deswizzle compute pass is unavailable on %s: textures deswizzle on the CPU", c->props.deviceName); + want = VK_DESWZ_CPU; + } + G.mode = want; + rt_log("vk: block-linear textures deswizzle on the %s%s", want == VK_DESWZ_CPU ? "CPU" : "GPU where the storage is " + "an imported pool", want == VK_DESWZ_CHECK ? ", checked against the CPU (TOTK_DESWIZZLE=check)" : ""); + } + m = G.mode; + plat_mutex_unlock(&g_lock); + return m; +} + +static uint32_t max_groups_x(const VkCtx *c) { + uint32_t m = c->props.limits.maxComputeWorkGroupCount[0]; + return m > 65535 ? 65535 : m; +} + +// The bound source range starts at the alignment below src_off. +static VkDeviceSize src_lo(const VkCtx *c, VkDeviceSize src_off) { + VkDeviceSize al = c->props.limits.minStorageBufferOffsetAlignment; + if (al < 16) al = 16; + return src_off / al * al; +} + +int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst) { + VkCtx *c = vk_ctx(); + if (!c || !n || (src_off & 15) || vk_deswz_mode() == VK_DESWZ_CPU) return 0; + VkDeviceSize range = src_off + src_size - src_lo(c, src_off), lim = c->props.limits.maxStorageBufferRange; + if (range > lim || max_dst > lim) return 0; + uint32_t maxg = max_groups_x(c); + for (int i = 0; i < n; i++) { // the 32-bit segment indices of the shader stay inside the bindings + uint64_t groups = ((uint64_t)rg[i].segs + 63) / 64, out = (uint64_t)rg[i].pitch16 * rg[i].rows * rg[i].slices; + if ((rg[i].src & 15) || rg[i].src + (uint64_t)rg[i].segs * 16 > src_size || !rg[i].segs || + (src_off - src_lo(c, src_off) + rg[i].src) / 16 + rg[i].segs > UINT32_MAX || out > UINT32_MAX || + out * 16 > max_dst || (groups + maxg - 1) / maxg > c->props.limits.maxComputeWorkGroupCount[1]) + return 0; + } + return 1; +} + +int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst, + VkDeviceSize dst_size, const VkDeswzRegion *rg, int n) { + VkCtx *c = vk_ctx(); + if (!vk_deswz_fits(src_off, src_size, rg, n, dst_size)) return 0; + for (int i = 0; i < n; i++) + if ((rg[i].dst & 15) || rg[i].dst + (uint64_t)rg[i].pitch16 * 16 * rg[i].rows * rg[i].slices > dst_size) return 0; + plat_mutex_lock_shared(&g_lock); + VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE; + plat_mutex_unlock_shared(&g_lock); + if (!pipe) return 0; + VkDeviceSize lo = src_lo(c, src_off), range = src_off + src_size - lo; + uint32_t maxg = max_groups_x(c); + VkDescriptorBufferInfo dbi[2] = {{src, lo, range}, {dst, 0, dst_size}}; + VkWriteDescriptorSet w[2]; + for (int i = 0; i < 2; i++) + w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i, + .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .pBufferInfo = &dbi[i]}; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); + rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 2, w); + uint64_t bytes = 0; + for (int i = 0; i < n; i++) { + const VkDeswzRegion *r = &rg[i]; + uint64_t groups = ((uint64_t)r->segs + 63) / 64; + uint32_t nx = groups < maxg ? (uint32_t)groups : maxg, ny = (uint32_t)((groups + nx - 1) / nx); + Push pc = {.src_base = (uint32_t)((src_off - lo + r->src) / 16), .dst_base = (uint32_t)(r->dst / 16), + .segs = r->segs, .pitch16 = r->pitch16, .rows = r->rows, .slices = r->slices, .th = r->th, .td = r->td, + .gobs_x = r->gobs_x, .blocks_y = r->blocks_y, .nx = nx}; + vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); + vkCmdDispatch(cb, nx, ny, 1); + bytes += (uint64_t)r->segs * 16; + } + int64_t before = plat_add64(&n_regions, n) - n; + plat_add64(&n_bytes, (int64_t)bytes); + if (!before || (before + n) / 4096 != before / 4096) + rt_log("vk: deswizzle: %lld regions, %.1f MiB of block-linear storage deswizzled on the GPU so far", + (long long)(before + n), (double)n_bytes / 1048576.0); + return 1; +} + +void vk_deswz_release(void) { + VkCtx *c = vk_ctx(); + plat_mutex_lock(&g_lock); + if (c && G.dev == c->device) { + if (G.pipe) vkDestroyPipeline(c->device, G.pipe, NULL); + if (G.layout) vkDestroyPipelineLayout(c->device, G.layout, NULL); + if (G.dsl) vkDestroyDescriptorSetLayout(c->device, G.dsl, NULL); + } + memset(&G, 0, sizeof G); + G.mode = -1; + plat_mutex_unlock(&g_lock); +} diff --git a/runtime/vk_device.c b/runtime/vk_device.c index a063955e..31fbc1a4 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -16,9 +16,69 @@ static VkCtx g_vk; static int g_ready; static PlatMutex g_lock = PLAT_MUTEX_INIT; +// ---- GPU address map for device faults +// VK_EXT_device_address_binding_report: the driver reports each bind and unbind of an object's GPU virtual range through +// the debug messenger. The newest AB_N events stay in a ring written without a lock (one atomic add each), so a device +// fault names the object behind its address and whether it was unbound before the fault, which is memory freed while +// a submission still used it. One event per image, buffer or memory block made or freed, none per draw. +// TOTK_ADDR_REPORT=0 turns it off. +#define AB_N (1u << 15) +typedef struct { uint64_t seq, base, size, handle, issued, done; int32_t type; uint32_t unbind; } AbEvent; +static AbEvent *ab_ring; +static uint64_t ab_head; +static int ab_want; // the instance has the messenger for it (create_instance) + +static void ab_record(const VkDebugUtilsMessengerCallbackDataEXT *d) { + const VkDeviceAddressBindingCallbackDataEXT *b = NULL; + for (const VkBaseInStructure *p = d->pNext; p; p = p->pNext) + if (p->sType == VK_STRUCTURE_TYPE_DEVICE_ADDRESS_BINDING_CALLBACK_DATA_EXT) b = (const void *)p; + if (!b || !ab_ring) return; + uint64_t i = __atomic_fetch_add(&ab_head, 1, __ATOMIC_RELAXED); + AbEvent *e = &ab_ring[i & (AB_N - 1)]; + __atomic_store_n(&e->seq, 0, __ATOMIC_RELAXED); + e->base = b->baseAddress, e->size = b->size; + e->handle = d->objectCount ? d->pObjects[0].objectHandle : 0; + e->type = d->objectCount ? (int32_t)d->pObjects[0].objectType : 0; + e->unbind = b->bindingType == VK_DEVICE_ADDRESS_BINDING_TYPE_UNBIND_EXT; + e->issued = vk_sync_issued(), e->done = vk_sync_fence_done(); + __atomic_store_n(&e->seq, i + 1, __ATOMIC_RELEASE); +} + +static const char *ab_type_name(int32_t t, char *buf, size_t n) { + switch (t) { + case VK_OBJECT_TYPE_IMAGE: return "image"; + case VK_OBJECT_TYPE_BUFFER: return "buffer"; + case VK_OBJECT_TYPE_DEVICE_MEMORY: return "memory"; + default: snprintf(buf, n, "object type %d", (int)t); return buf; + } +} + +// The newest events whose range meets the precision-aligned range around a fault address, newest first. +static void ab_report(uint64_t addr, uint64_t precision) { + if (!ab_ring) return; + uint64_t lo = precision > 1 ? addr & ~(precision - 1) : addr, hi = lo + (precision > 1 ? precision : 1); + uint64_t head = __atomic_load_n(&ab_head, __ATOMIC_ACQUIRE), n = head < AB_N ? head : AB_N; + int found = 0; + for (uint64_t k = 1; k <= n && found < 6; k++) { + const AbEvent *e = &ab_ring[(head - k) & (AB_N - 1)]; + if (__atomic_load_n(&e->seq, __ATOMIC_ACQUIRE) != head - k + 1) continue; + if (e->base >= hi || e->base + e->size <= lo) continue; + found++; + char tn[32]; + rt_log("vk: fault address %#llx: %s %#llx [%#llx, +%#llx] %s %llu address events before the fault, " + "submitted %llu, done %llu", (unsigned long long)addr, ab_type_name(e->type, tn, sizeof tn),(unsigned long long)e->handle, + (unsigned long long)e->base, (unsigned long long)e->size, e->unbind ? "unbound" : "bound", + (unsigned long long)k, (unsigned long long)e->issued, (unsigned long long)e->done); + } + if (!found) rt_log("vk: fault address %#llx: no binding in the last %llu address events", (unsigned long long)addr, + (unsigned long long)n); +} + static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type, const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) { - (void)type, (void)user; + (void)user; + if (type & VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT) { ab_record(d); return VK_FALSE; } + if (sev < VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) return VK_FALSE; // info, asked for the address events int err = (sev & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) != 0; if (err) plat_inc32(&g_vk.validation_errors); static volatile int32_t logged; @@ -76,7 +136,10 @@ static int create_instance(void) { rt_log("vk: TOTK_VK_VALIDATION set but %s is not installed; continuing without", layers[0]); validate = 0; } - if (validate) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; + const char *ar = cfg_get("TOTK_ADDR_REPORT"); + ab_want = !(ar && !strcmp(ar, "0")) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + if (ab_want && !ab_ring && !(ab_ring = calloc(AB_N, sizeof *ab_ring))) ab_want = 0; + if (validate || ab_want) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; if (vk_hdr_wanted() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats ext[n_ext++] = VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME; vk_dlaa_instance_exts(ext, &n_ext, 12); @@ -89,16 +152,18 @@ static int create_instance(void) { VkResult r = vkCreateInstance(&ci, NULL, &g_vk.instance); if (r != VK_SUCCESS) { rt_log("vk: vkCreateInstance failed (%d); is a Vulkan 1.3 driver installed?", (int)r); return 0; } g_vk.validation = validate; - if (validate) { + if (validate || ab_want) { PFN_vkCreateDebugUtilsMessengerEXT f = (PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkCreateDebugUtilsMessengerEXT"); VkDebugUtilsMessengerCreateInfoEXT mi = { .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT, - .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT, - .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | - VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT, + .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT | + (ab_want ? VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT : 0), + .messageType = (validate ? VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT : 0) | + (ab_want ? VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT : 0), .pfnUserCallback = debug_cb}; - if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger); + if (!f || f(g_vk.instance, &mi, NULL, &g_vk.messenger) != VK_SUCCESS) ab_want = 0; } return 1; } @@ -217,6 +282,7 @@ typedef struct { VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl; VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbl; VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr; + VkPhysicalDeviceAddressBindingReportFeaturesEXT abr; } FeatureChain; static int create_device(void) { @@ -251,6 +317,13 @@ static int create_device(void) { // Ryujinx's SPIR-V declares fragment shader interlock whenever the host reports it (vk_pipeline.c spirv_fixup). OPTIONAL_EXT(has_fragment_shader_interlock, VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME, fsi, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT); + // The GPU address map for device faults (ab_record), with the messenger made for it + int has_abr = 0; + if (ab_want && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME)) { + has_abr = 1; + ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME; + CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT); + } // Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering int has_fbl = 0, has_lr = 0; if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) && @@ -314,6 +387,10 @@ static int create_device(void) { else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension + rt_log("vk: GPU address map for device faults: %s", + has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report; TOTK_ADDR_REPORT=0 turns it off)" + : !ab_want ? "off (TOTK_ADDR_REPORT=0 or no VK_EXT_debug_utils)" + : "unavailable (no VK_EXT_device_address_binding_report)"); g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { @@ -376,6 +453,7 @@ static void destroy_all(void) { gfx_window_release_vk(); if (g_vk.device) vkDeviceWaitIdle(g_vk.device); if (g_vk.device) vk_astc_release(); + if (g_vk.device) vk_deswz_release(); if (g_vk.device) vk_crumbs_release(); if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL); vk_headroom_stop(); @@ -397,7 +475,7 @@ int vk_init(void) { plat_mutex_lock(&g_lock); if (!g_ready) { memset(&g_vk, 0, sizeof g_vk); - if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_headroom_start(), vk_dlaa_init(); + if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_upload_pools_init(), vk_headroom_start(), vk_dlaa_init(); else destroy_all(); } int ok = g_ready; @@ -437,6 +515,7 @@ void vk_device_fault_report(VkResult r) { done = 1; vk_crumbs_report(); vk_pass_marks_report(); + vk_mem_fault_log(); VkCtx *c = vk_ctx(); PFN_vkGetDeviceFaultInfoEXT get = c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT") @@ -461,9 +540,11 @@ void vk_device_fault_report(VkResult r) { rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description, n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary); // The faulting access lies in the precision-aligned range around the reported address. - for (uint32_t i = 0; i < n.addressInfoCount; i++) + for (uint32_t i = 0; i < n.addressInfoCount; i++) { rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType), (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision); + ab_report(ai[i].reportedAddress, ai[i].addressPrecision); + } for (uint32_t i = 0; i < n.vendorInfoCount; i++) rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description, (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData); diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index b35831c8..6408d2be 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -183,6 +183,7 @@ static void memory_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAcce // be made (logged). Replay thread. static int bc_standin(const VkTexImage *img, uint32_t level, VkFormat fmt, BcStandIn *out) { static int warned; + if (img->gone) return 0; // dropped since the lookup: a stand-in kept by its handle would outlive it plat_mutex_lock(&bcs_lock); int full = nbcs == BC_STANDINS; for (int i = 0; i < nbcs; i++) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index eee270c9..7bede297 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -593,6 +593,10 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c) VkImageView view; if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE; + if (img->gone) { // dropped since the lookup: its views were purged and a new one would outlive the image + vk_defer_view(view); + return view; + } ViewNode *n = malloc(sizeof *n); if (!n) rt_fatal("vk: out of memory"); *n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view}; @@ -619,11 +623,15 @@ VkImageView vk_exec_target_view(const VkTexImage *img, VkFormat format, VkImageV if (blocks) ci.pNext = &use; // not the BC image's sampling and transfer usages if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) rt_fatal("vk: render target view (format %d) could not be created", (int)format); - ViewNode *n = malloc(sizeof *n); - if (!n) rt_fatal("vk: out of memory"); - *n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view}; - nvn_hm_put(&views_by_image, &img->image, n); - nvn_hm_put(&view_of, &k, (void *)view); + if (img->gone) { // dropped since resolve_target (get_view) + vk_defer_view(view); + } else { + ViewNode *n = malloc(sizeof *n); + if (!n) rt_fatal("vk: out of memory"); + *n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view}; + nvn_hm_put(&views_by_image, &img->image, n); + nvn_hm_put(&view_of, &k, (void *)view); + } } vk_texture_unlock(); return view; diff --git a/runtime/vk_hdr.c b/runtime/vk_hdr.c index e8480bb2..806ffd1b 100644 --- a/runtime/vk_hdr.c +++ b/runtime/vk_hdr.c @@ -272,7 +272,7 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) { } if (ntex == 1 && slot >= 0 && slot < 128 && ex->tex[1][slot]) { const VkTexImage *s = vk_texture_by_id(NVN_HANDLE_TEX(ex->tex[1][slot]), NULL, NULL); - if (s && is_float(s->format) && s->width == t->width && s->height == t->height && s->samples == 1) + if (s && !s->gone && is_float(s->format) && s->width == t->width && s->height == t->height && s->samples == 1) src = s->image, fmt = s->format; } plat_mutex_lock(&rec_lock); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 6c6b1a8b..3e30682f 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -78,8 +78,9 @@ static int shadow_on; // shadow mode: imported pools get shad static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size); static void shadow_touch(PoolRec *r); static void shadow_forget(PoolRec *r); -static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown) -static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record +static void pool_retiring(PoolRec *r); // vk_pool_fini: kept until pool_retire (or vk_mem_shutdown) +static void pool_retire(void *arg); // completion thread: destroys a finalized pool's record +static uint64_t retire_value(void); // the timeline value after which a finalized pool's record may go static void grow(void **arr, int *cap, size_t elem); static PlatMutex pool_lock = PLAT_MUTEX_INIT; @@ -333,10 +334,14 @@ void vk_pool_fini(pool_t *p) { vk_queue_unlock(); } vk_mem_collect(); - // A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may - // name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued). - if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r); - else destroy_rec(c, r); + // The record, its buffers and its memory outlive the submission being recorded, of any pool kind. The wait above + // covers submitted work only. With the synchronous replay another game thread may be recording right now, and that + // recording may already name this pool: a mirrored pool's sync copies and write-backs, an import pool's buffer + // through a texture upload or a released cache entry (vk_texture.c slot_resolve), a physical pool's memory under + // the images vk_texture_range_gone just deferred to that submission. The game frees a pool only once its own GPU + // work is done, so only the port's work can name it. Freed here, the submission ran with freed memory. + pool_retiring(r); + vk_completion_add(retire_value(), pool_retire, r); } // Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c); apply them in order. @@ -416,6 +421,10 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { return resolve_locked(addr, size, out); } +int vk_mem_span_live(const VkMemSpan *sp) { + return sp->buffer && sp->pool && !sp->alloc && !shadow_on && ((const PoolRec *)sp->pool)->kind == POOL_IMPORT; +} + // The pool index lookup under the lock; a non-overlapped import or physical pool goes into the address's cache slot. static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) { memset(out, 0, sizeof *out); @@ -507,6 +516,9 @@ static void dirty_frame(uint64_t frame); static void mirror_log(uint64_t frame); static void shadow_log(uint64_t frame); static int dead_pending(void); +// Deferrals made off the recording thread while another thread recorded a submission (cmd_lock, defer), the newest +// one's stamp: the case the stamp from the recording start covers. In the census line and the device fault report. +static uint64_t n_cross_defer, cross_defer_tl; static void vram_log(uint64_t frame); @@ -542,10 +554,11 @@ void vk_mem_census(uint64_t frame) { } plat_mutex_unlock_shared(&pool_lock); rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%d without a GPU buffer), physical %llu pools %.1f MiB, " - "virtual %llu pools %.1f MiB of address space", + "virtual %llu pools %.1f MiB of address space; %llu objects deferred off the recording thread during a recording", (unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0, n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0, - (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0); + (unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0, + (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED)); static const char *const kinds[] = {"", "import", "physical", "virtual"}; for (int j = 0; j < ng; j++) { char f[160]; @@ -1390,19 +1403,19 @@ static void mirror_forget(PoolRec *r) { } } -// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran. +// Finalized pools waiting for pool_retire; vk_mem_shutdown destroys those whose callback never ran. static PoolRec **retiring; static int nretiring, cretiring; static PlatMutex retire_lock = PLAT_MUTEX_INIT; -static void mirror_retiring(PoolRec *r) { +static void pool_retiring(PoolRec *r) { plat_mutex_lock(&retire_lock); if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring); retiring[nretiring++] = r; plat_mutex_unlock(&retire_lock); } -static void mirror_retire(void *arg) { +static void pool_retire(void *arg) { plat_mutex_lock(&retire_lock); for (int i = 0; i < nretiring; i++) if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; } @@ -2152,9 +2165,10 @@ typedef struct { uint64_t seq; Staging *stg; int nstg, cap; + VkDeviceSize held; // staging and scratch bytes freed when the batch completes (vk_upload_held) } Batch; -enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA }; +enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST }; // DEAD_HOST: free(handle) typedef struct { int kind; uint64_t handle; VmaAllocation alloc; uint64_t seq, tl; } Dead; static PlatRMutex cmd_lock; @@ -2169,7 +2183,8 @@ static int nspare, cap_spare; static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq) static Dead *dead; static int ndead, cap_dead; -static uint64_t tl_noted; // newest timeline value vk_sync.c reported submitted +static uint64_t tl_noted; // value of the submission being recorded, else of the last one (vk_mem_note_fence) +static _Thread_local uint64_t tl_noted_here; // the value this thread noted: it records that submission static void cmd_init(void *p) { (void)p; @@ -2212,6 +2227,10 @@ static void collect_locked(VkCtx *c) { for (int i = 0; i < ndead; i++) { Dead *d = &dead[i]; if (d->seq > done_seq || d->tl > tl) { dead[k++] = *d; continue; } + if (d->kind == DEAD_HOST) { + free((void *)(uintptr_t)d->handle); + continue; + } rec_check_handle(d->kind == DEAD_VIEW ? REC_VIEW : d->kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, d->handle, 1, d->tl, tl); switch (d->kind) { @@ -2305,10 +2324,68 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { else { if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg); open_b.stg[open_b.nstg++] = (Staging){b, a}; + open_b.held += bi.size; } return info.pMappedData; } +// Device-local scratch of the upload batch (the GPU deswizzle's linear copies, the ASTC decoder's blocks and texels): +// a pool of its own, like staging, so the replay thread neither waits on the default pool's lock while another thread +// makes a block there (the headroom thread: about 8 ms for 256 MiB) nor makes one itself for a few MiB. One block +// of SCR_BLOCK is kept from the first use; larger buffers than half a block go to the default pool. +#define SCR_BLOCK (64ull << 20) +static VmaPool scr_pool; +static int scr_pool_tried; + +VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { + VkCtx *c = vk_ctx(); + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (!__atomic_load_n(&scr_pool_tried, __ATOMIC_ACQUIRE)) { + plat_once(&cmd_once, cmd_init, NULL); + plat_rmutex_lock(&cmd_lock); + if (!scr_pool_tried) { + // two blocks with 8 GiB or more of device-local memory: fight2's first world frame deswizzles about + // 280 MiB, more than the GPU finishes while the replay records, and the pool grew by four blocks there + VkDeviceSize vram = 0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT && c->mem_props.memoryHeaps[h].size > vram) + vram = c->mem_props.memoryHeaps[h].size; + uint32_t type; + VmaPoolCreateInfo pi = {.blockSize = SCR_BLOCK, .minBlockCount = vram >= (8ull << 30) ? 2 : 1}; + if (vmaFindMemoryTypeIndexForBufferInfo(c->vma, &bi, &aci, &type) == VK_SUCCESS) { + pi.memoryTypeIndex = type; + if (vmaCreatePool(c->vma, &pi, &scr_pool) != VK_SUCCESS) scr_pool = NULL; + } + rt_log("vk: upload scratch pool %s (%u blocks of %llu MiB kept)", scr_pool ? "created" : "unavailable: default pool", + (unsigned)pi.minBlockCount, (unsigned long long)(SCR_BLOCK >> 20)); + __atomic_store_n(&scr_pool_tried, 1, __ATOMIC_RELEASE); + } + plat_rmutex_unlock(&cmd_lock); + } + if (scr_pool && bi.size <= SCR_BLOCK / 2) aci.pool = scr_pool; + VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, buf, alloc, NULL); + if (r == VK_SUCCESS && open_active) open_b.held += bi.size; // the caller defers its free to the open batch + return r; +} + +VkDeviceSize vk_upload_held(void) { return open_active ? open_b.held : 0; } + +// At device creation: the staging and scratch pools with their kept blocks, made through a first small buffer each +// (freed at once), so the first world frame does not allocate them (staging was first used there once the GPU took +// the early texture uploads). +void vk_upload_pools_init(void) { + VkCtx *c = vk_ctx(); + if (!c) return; + VkBuffer b; + VmaAllocation a; + vk_staging(16, &b, &a); + vmaDestroyBuffer(c->vma, b, a); + if (vk_scratch_buffer(16, &b, &a) == VK_SUCCESS) vmaDestroyBuffer(c->vma, b, a); +} + uint64_t vk_upload_flush(int wait) { plat_once(&cmd_once, cmd_init, NULL); plat_rmutex_lock(&cmd_lock); @@ -2346,20 +2423,21 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) { plat_rmutex_lock(&cmd_lock); if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead); // The open batch may record into the object: it counts as the newest user. The GPU must also be past every - // submission that names it. tl_noted covers the submitted ones; an object the replay lets go of while it records - // a submission (vk_exec_recording_fence) may be named by that one too, not yet submitted: vk_texture.c - // slot_resolve hands a draw a released entry, and entry_of destroys released entries at that address later in - // the same submission. Stamped with the last submitted value, the object went as soon as the GPU reached it, - // before the submission that names it ran, and with the recorder thread (vk_rec.c) behind, before the recorder - // made the call that names it. LAS Recomp and Savage Wind Recomp stamp every deferral with the submission being - // recorded (runtime/nvn/vulkan/textures.c defer: vk_pending_seq). TOTK_DEFER_STAMP=0 brings back the old stamp. - static int stamp = -1; - if (stamp < 0) stamp = rt_knob("DEFER_STAMP", "1") != 0; - uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0; - if (rf > tl) tl = rf; + // submission that names it, including the one being recorded, which may name it and is not submitted yet: + // vk_texture.c slot_resolve hands a draw a released entry, and entry_of, sweep_released and the VRAM pressure drop + // destroy released entries. tl_noted is that submission's value from the start of its recording (vk_sync.c + // vk_sync_submit), whichever thread destroys the object. With the synchronous replay a game thread finalizing a + // texture (vk_backend.c vk_be_texture_release) destroys entries while another game thread records; stamped with + // the last submitted value, as the recording thread's own fence was the only one seen, the object was freed once + // the GPU reached that value, before the submission naming it ran: "write invalid" device faults on NVIDIA. LAS + // Recomp and Savage Wind Recomp stamp every deferral with the submission being recorded (runtime/nvn/vulkan/ + // textures.c defer: vk_pending_seq). + uint64_t tl = tl_noted; + if (kind != DEAD_HOST && tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl; dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl}; - rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl, - vk_sync_fence_done()); + if (kind != DEAD_HOST) + rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl, + vk_sync_fence_done()); plat_rmutex_unlock(&cmd_lock); } @@ -2378,14 +2456,38 @@ void vk_defer_view(VkImageView view) { if (view) defer(DEAD_VIEW, (uint64_t)(uin void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) { if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc); } +void vk_defer_free(void *p) { if (p) defer(DEAD_HOST, (uint64_t)(uintptr_t)p, NULL); } -void vk_mem_note_submit(uint64_t timeline_value) { +// The device fault report: no lock, since a thread that holds cmd_lock may be waiting for the report's own lock +// (vk_upload_flush reports a failed submit with cmd_lock held). +void vk_mem_fault_log(void) { + rt_log("vk: deferred frees at the fault: %d pending, stamp %llu, submitted %llu, done %llu; %llu deferred off the " + "recording thread during a recording (newest stamp %llu)", + __atomic_load_n(&ndead, __ATOMIC_RELAXED), (unsigned long long)__atomic_load_n(&tl_noted, __ATOMIC_RELAXED), + (unsigned long long)vk_sync_issued(), (unsigned long long)vk_sync_fence_done(), + (unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED), + (unsigned long long)__atomic_load_n(&cross_defer_tl, __ATOMIC_RELAXED)); +} + +void vk_mem_note_fence(uint64_t timeline_value) { plat_once(&cmd_once, cmd_init, NULL); plat_rmutex_lock(&cmd_lock); if (timeline_value > tl_noted) tl_noted = timeline_value; + tl_noted_here = timeline_value; plat_rmutex_unlock(&cmd_lock); } +// The later of the submission being recorded (tl_noted, noted from its start: vk_sync.c vk_sync_submit) and the next +// one to be submitted. +static uint64_t retire_value(void) { + plat_once(&cmd_once, cmd_init, NULL); + plat_rmutex_lock(&cmd_lock); + uint64_t v = vk_sync_issued() + 1; + if (tl_noted > v) v = tl_noted; + plat_rmutex_unlock(&cmd_lock); + return v; +} + void vk_mem_collect(void) { VkCtx *c = vk_ctx(); if (!c) return; @@ -2434,6 +2536,9 @@ void vk_mem_shutdown(void) { if (stg_pool) vmaDestroyPool(c->vma, stg_pool); // after retire: no staging buffer is left in it stg_pool = NULL; stg_pool_tried = 0; + if (scr_pool) vmaDestroyPool(c->vma, scr_pool); // deferred scratch buffers went with collect_locked above + scr_pool = NULL; + scr_pool_tried = 0; plat_rmutex_unlock(&cmd_lock); plat_mutex_lock(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index dedf7a37..57142b21 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -101,6 +101,12 @@ static struct { VkPipeStats stats; } P = {.lock = PLAT_MUTEX_INIT}; +// Builds into P.cache take it shared, vk_pipeline_save takes it to copy the cache out. Vulkan lets +// vkGetPipelineCacheData run alongside builds into a cache created without the EXTERNALLY_SYNCHRONIZED bit, but +// Intel's Windows driver (101.7085, Iris Xe) faults in it (igvk64.dll, reading 0x170) when the first autosave meets the +// boot's pipeline builds (#681). Nothing is taken while it is held. +static PlatMutex cache_use = PLAT_MUTEX_INIT; + typedef struct Layout { struct Layout *next; uint64_t hash; @@ -1440,7 +1446,9 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay if (k->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; if (k->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; uint64_t p0 = perf_begin(); + plat_mutex_lock_shared(&cache_use); VkResult r = vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe); + plat_mutex_unlock_shared(&cache_use); perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it if (r != VK_SUCCESS) { rt_log("vk_pipeline: vkCreateGraphicsPipelines failed (%d)", (int)r); @@ -1591,7 +1599,9 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil; pi.pColorBlendState = &cb, pi.pMultisampleState = &ms; } + plat_mutex_lock_shared(&cache_use); if (vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + plat_mutex_unlock_shared(&cache_use); out: if (m) vkDestroyShaderModule(P.device, m, NULL); free(patched); @@ -1803,7 +1813,8 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc // shaders (gfx_shader_get results) live for the process (gfx_shader_drop_memory is the shader probe's, which builds no // pipelines), layouts until vk_pipeline_shutdown, which drains the queue first. Workers call vkCreateGraphicsPipelines // without P.lock: P.cache is created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT, so the driver -// synchronises it, vkGetPipelineCacheData in vk_pipeline_save included. Compute pipelines stay synchronous: a skipped +// synchronises the builds among themselves. vk_pipeline_save still keeps them apart from vkGetPipelineCacheData (see +// cache_use). Compute pipelines stay synchronous: a skipped // dispatch can change what the game computes. // // Two queues: draws' misses (A.head) and the boot prebuild (A.low, see "pipeline keys" below). Workers take the low @@ -2512,7 +2523,7 @@ static void prebuild_start(void) { plat_mutex_unlock(&P.lock); } -// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised). +// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. Takes cache_use shared, like the graphics builds. static VkPipeline compute_create(const GfxShader *cs, const Layout *l) { uint64_t words; const uint32_t *code = gfx_shader_spirv(cs, &words); @@ -2525,7 +2536,9 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l) { .layout = l->layout}; pin_subgroup32(&ci.stage, &rqc, code, words); VkPipeline pipe = VK_NULL_HANDLE; + plat_mutex_lock_shared(&cache_use); if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + plat_mutex_unlock_shared(&cache_use); vkDestroyShaderModule(P.device, m, NULL); return pipe; } @@ -3290,7 +3303,8 @@ void vk_pipeline_stats(VkPipeStats *s) { static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache -void vk_pipeline_save(void) { +// One cache save; wait: queue for cache_use (the exit saves) rather than try for it (the autosave). +static void cache_save(int wait) { if (P.ready) keys_flush(); if (P.ready && P.gpl) { rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws " @@ -3317,23 +3331,46 @@ void vk_pipeline_save(void) { } gfx_shader_record_flush(1); if (!P.ready || !P.cache || !P.cache_path[0]) return; - // Not under P.lock: the driver synchronises P.cache, and holding P.lock through the copy and the file write (about - // 60 ms on the 1.4.3 fight) stopped every pipeline lookup of the replay, a freeze at each 20 s autosave. + // Not under P.lock: holding it through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped every + // pipeline lookup of the replay, a freeze at each 20 s autosave. Only the copy takes cache_use, so no pipeline is + // being built into P.cache while the driver reads it. plat_mutex_lock(&save_lock); if (!P.cache) { // vk_pipeline_shutdown destroyed it plat_mutex_unlock(&save_lock); return; } + uint64_t t0 = qpc(); + if (wait) { + plat_mutex_lock(&cache_use); + } else { + // The autosave never queues for cache_use: a queued exclusive request on an SRW lock holds back new shared + // ones, so the drawing thread's next build would wait for a worker's build to end. It tries for about 2 s + // and otherwise saves on its next round. + int got = 0; + for (int i = 0; i < 1000 && !(got = plat_mutex_trylock(&cache_use)); i++) plat_sleep_ms(2); + if (!got) { + rt_log("vk_pipeline: cache save put off, pipelines were being built for %.0f ms", ms_since(t0)); + plat_mutex_unlock(&save_lock); + return; + } + } + uint64_t t1 = qpc(); size_t size = 0; void *data = NULL; - if (vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) && - vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS) { - if (!plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path); - } + int ok = vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) && + vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS; + plat_mutex_unlock(&cache_use); + // The copy time is how long a pipeline build on the drawing thread (a compute miss, under P.lock) can wait for + // this save; the wait is how long the save waited for builds in flight. + rt_log("vk_pipeline: cache %s: %.1f MB copied in %.1f ms, after %.1f ms waiting for builds", wait ? "save" : "autosave", + (double)size / 1e6, ms_since(t1), (double)(t1 - t0) / 1e6); + if (ok && !plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path); free(data); plat_mutex_unlock(&save_lock); } +void vk_pipeline_save(void) { cache_save(1); } + // The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it // runs: at most every 20 s, and only after new pipelines were built. New pipeline keys are appended every 2 s. static void autosave_once(void) { @@ -3347,7 +3384,7 @@ static void autosave_once(void) { vk_pipeline_stats(&s); if (s.graphics + s.compute == last_n) return; last_n = s.graphics + s.compute, last_ms = now; - vk_pipeline_save(); + cache_save(0); } // On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index e3162d12..3b90c026 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -309,7 +309,7 @@ void vk_sync_start(void) { if (reserved < now) reserved = now; stop = 0; plat_mutex_unlock(&st_lock); - vk_mem_note_submit(now); + vk_mem_note_fence(now); completion_thread = plat_thread_new(completion_main, c, "vk completion"); if (!completion_thread) rt_fatal("vk: cannot start the completion thread"); __atomic_store_n(&started, 1, __ATOMIC_RELEASE); @@ -379,7 +379,6 @@ VkCommandBuffer vk_sync_split(void) { vk_device_fault_report(r); rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r); } - vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence); ts_begin(s->cb); @@ -404,6 +403,9 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1; cur_fence = fence; + // Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c + // defer), since the recording may already name it. + vk_mem_note_fence(fence); CmdSlot *s = take_slot(c, fence); uint64_t t0 = perf_begin(); rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence); // begun by the recorder when it records the part @@ -434,7 +436,6 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r); } s->fence = recorded ? fence : 0; - vk_mem_note_submit(fence); plat_mutex_lock(&st_lock); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index aafe5ff7..d461f561 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -597,10 +597,16 @@ static void destroy_tex(Tex *e) { if (p->next_at == e) { p->next_at = e->next_at; break; } } if (e->released) nreleased--, rel_unlink(e); + // Another thread may hold &e->pub, taken under the lock and used after it (vk_texture_get, vk_texture_by_id: render + // targets for a whole pass, vk_exec.c resolve_target, vk_texture_storage_bound_locked writes through it). Freed + // here, that thread read a stale VkImage from freed memory, made views of it and cached them by that handle. The + // record goes with the image, once the submission being recorded is done; gone keeps views of it out of the caches. + e->pub.gone = 1; vk_exec_image_gone(e->pub.image); vk_defer_image(e->pub.image, e->alloc); free(e->snap); - free(e); + e->snap = NULL; + vk_defer_free(e); } // Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries the @@ -819,15 +825,20 @@ static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar, // large ASTC array decodes to RGBA8 in one go: a staging buffer of 676682336 bytes failed (-2) on an Intel UHD with // 8 GB of shared memory and the game closed before the title screen. #define UPLOAD_PART (32ull << 20) // fits the staging pool (vk_staging takes up to half of its 64 MiB block) +// The open upload batch is submitted between chunks (and parts of the CPU path) once its staging and scratch pass this: +// on fight2's first world frame one batch took hundreds of MiB, and the staging and scratch pools grew by 64 MiB blocks +// on the replay thread. Submitting earlier keeps the queue order (the batch still precedes the part that samples the +// images) and lets the GPU finish and return the buffers sooner. +#define UPLOAD_HELD_MAX (48ull << 20) -// Copies the whole storage into the image (every level and layer the image has). -static void upload(Tex *e) { +static int upload_gpu(Tex *e); + +// The CPU side of upload: deswizzles (and decodes or expands) into staging buffers, then copies them into the image. +static void upload_cpu(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; const tex_t *t = &e->t; const nvn_texdesc *d = &t->d; int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); - if (!bpb || !t->mem) return; - uint64_t p0 = perf_begin(), u0 = kern_now_ns(); int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D; uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers; int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb; @@ -908,6 +919,7 @@ static void upload(Tex *e) { } barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_upload_end(); + if (!acheck && vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0); } if (parts > 1) log_limited(&n_split, "texture %#llx fmt %#x %ux%u, %u levels, %u layers: %.1f MiB staged in %d parts", @@ -924,7 +936,201 @@ static void upload(Tex *e) { free(raw_copy); free(ar); free(rg); +} + +// Block-linear storage in an imported pool, deswizzled by the GPU (vk_deswizzle.c): the replay thread only records +// the dispatches, the image copies and (ASTC) the decode into the upload batch the CPU path also uses, which runs before +// the part that samples the image. The GPU reads the pool when the batch runs, as the Switch's GPU reads it at +// execution; a CPU write after make_fresh's write-watch reset still marks the pages and brings another upload. Each +// chunk of regions gets a device-local scratch buffer of at most DESWZ_CHUNK bytes (one region, when larger), freed +// once the batch completes. TOTK_DESWIZZLE=check compares every chunk with the CPU deswizzle (waits for the GPU). +// 0 when not handled, before anything is recorded: the CPU path takes the texture. +#define DESWZ_CHUNK (32ull << 20) + +// Uploads by path since the last perf summary (TOTK_TRACE=perf): 0 GPU, else why the CPU took it. +enum { UP_GPU, UP_MODE, UP_LINEAR, UP_RGB, UP_ASTC_CPU, UP_BPB, UP_NOT_LIVE, UP_GEOM, UP_LIMITS, UP_SCRATCH, UP_WHY }; +static const char *const up_why_name[UP_WHY] = {"gpu", "cpu mode", "linear", "rgb", "astc on cpu", "bytes per block", + "not an imported pool", "layout", "device limits", "scratch"}; +static uint64_t up_n[UP_WHY], up_bytes[UP_WHY], up_ns[UP_WHY]; +static int up_why; // upload_gpu's reason for returning 0 (texture lock held) + +static int upload_gpu(Tex *e) { + static volatile int32_t n_scratch_fail, n_undecoded, n_bad; + static volatile int64_t n_checked, n_differ; + const tex_t *t = &e->t; + const nvn_texdesc *d = &t->d; + int mode = vk_deswz_mode(); + up_why = mode == VK_DESWZ_CPU ? UP_MODE : nvn_tex_is_linear(d) ? UP_LINEAR : e->f.expand_rgb ? UP_RGB : UP_GPU; + if (up_why) return 0; + int astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu(); + if (astc && (!agpu || vk_astc_mode() == VK_ASTC_CHECK)) return up_why = UP_ASTC_CPU, 0; // or ASTC's check mode + int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); + if (!bpb || 16 % bpb) return up_why = UP_BPB, 0; // the rows of the linear copy are padded to 16 bytes + VkMemSpan sp; + if (!vk_mem_resolve(e->pub.addr, t->L.size, &sp) || !vk_mem_span_live(&sp)) return up_why = UP_NOT_LIVE, 0; + int one_d = e->pub.type == VK_IMAGE_TYPE_1D, is3d = d->target == NVN_TEXTURE_TARGET_3D; + uint32_t levels = e->pub.levels, layers = e->pub.layers, n = levels * layers, k = 0; + VkDeswzRegion *dr = calloc(n, sizeof *dr); + VkBufferImageCopy *rg = calloc(n, sizeof *rg); + VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL; + if (!dr || !rg || (agpu && !ar)) rt_fatal("vk: out of memory (texture upload regions)"); + int ok = 1; + VkDeviceSize max_dst = DESWZ_CHUNK; + for (uint32_t lv = 0; ok && lv < levels; lv++) + for (uint32_t ly = 0; ok && ly < layers; ly++) { + uint32_t row_bytes, rows, slices, gbpb; + uint64_t base; + unsigned th, td; + uint32_t w = mip(d->width, lv), h = one_d ? 1 : mip(d->height, lv), sl = is3d ? mip(d->depth, lv) : 1; + if (nvn_deswizzle_geom(d, lv, ly, &row_bytes, &rows, &slices, &base, &th, &td, &gbpb) || gbpb != (uint32_t)bpb || + base >= t->L.size || rows != (h + bh - 1) / bh || slices != sl) { + ok = 0; + break; + } + uint64_t bhg = 1ull << th, bdg = 1ull << td, gobs_x = (row_bytes + 63) / 64; + uint64_t blocks_y = (rows + 8 * bhg - 1) / (8 * bhg), blocks_z = (slices + bdg - 1) / bdg; + uint64_t segs = gobs_x * blocks_y * blocks_z * (32ull << (th + td)), room = (t->L.size - base) / 16; + if (segs > room) segs = room; + uint32_t pitch16 = (row_bytes + 15) / 16; + VkDeviceSize out = (VkDeviceSize)pitch16 * 16 * rows * slices; + if (segs > UINT32_MAX || gobs_x * blocks_y > UINT32_MAX) { ok = 0; break; } + if (out > max_dst) max_dst = out; + dr[k] = (VkDeswzRegion){.src = base, .segs = (uint32_t)segs, .pitch16 = pitch16, .rows = rows, .slices = slices, + .th = th, .td = td, .gobs_x = (uint32_t)gobs_x, .blocks_y = (uint32_t)blocks_y}; + rg[k] = (VkBufferImageCopy){.bufferRowLength = pitch16 * 16 / (uint32_t)bpb * (uint32_t)bw, + .bufferImageHeight = rows * (uint32_t)bh, + .imageSubresource = {e->pub.aspect, lv, ly, 1}, .imageExtent = {w, h, sl}}; + if (agpu) ar[k] = (VkAstcRegion){.w = w, .h = h, .slices = sl, .level = lv, .layer = ly}; + k++; + } + if (!ok || k != n || !vk_deswz_fits(sp.offset, t->L.size, dr, (int)n, max_dst)) { + free(dr), free(rg), free(ar); + return up_why = !ok || k != n ? UP_GEOM : UP_LIMITS, 0; + } + VkCtx *c = vk_ctx(); + VkCommandBuffer cb = vk_upload_begin(); + // Earlier GPU writes into the pool (copies, shaders) before the reads; the image's earlier uses before its copy. + barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) { + VkDeviceSize size = 0; + for (k1 = k0; k1 < n; k1++) { + VkDeviceSize out = (VkDeviceSize)dr[k1].pitch16 * 16 * dr[k1].rows * dr[k1].slices; + if (k1 > k0 && size + out > DESWZ_CHUNK) break; + dr[k1].dst = size, rg[k1].bufferOffset = size; + if (agpu) ar[k1].src = size; + size += out; + } + VkBuffer sb; + VmaAllocation sa; + if (vk_scratch_buffer(size, &sb, &sa) != VK_SUCCESS) { + log_limited(&n_scratch_fail, "texture %#llx: deswizzle scratch of %llu bytes failed%s", + (unsigned long long)e->pub.addr, (unsigned long long)size, + k0 ? "; its later levels left as they were" : "; deswizzled on the CPU"); + if (!k0) { + vk_upload_end(); + free(dr), free(rg), free(ar); + return up_why = UP_SCRATCH, 0; + } + break; + } + if (!vk_deswz_record(cb, sp.buffer, sp.offset, t->L.size, sb, size, dr + k0, (int)(k1 - k0))) { + log_limited(&n_bad, "texture %#llx: deswizzle of levels %u-%u not recorded; left as they were", + (unsigned long long)e->pub.addr, k0, k1 - 1); // vk_deswz_fits passed: not expected + vk_defer_buffer(sb, sa); + break; + } + barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, + VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); + if (agpu) { + uint64_t a0 = perf_begin(); + if (!vk_astc_record(cb, sb, size, ar + k0, (int)(k1 - k0), e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, + e->f.astc_bc, e->pub.image, e->pub.aspect, VK_NULL_HANDLE)) + log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was", + (unsigned long long)e->pub.addr, d->format, d->width, d->height); + perf_end(PERF_ASTC, a0); + } else { + vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, k1 - k0, rg + k0); + } + if (mode == VK_DESWZ_CHECK) { // the linear copy back to the host, compared with the CPU deswizzle + VkBuffer rb; + VmaAllocation ra; + const uint8_t *rmap = vk_staging(size, &rb, &ra); + VkBufferCopy cp = {0, 0, size}; + vkCmdCopyBuffer(cb, sb, rb, 1, &cp); + barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT, + VK_ACCESS_2_HOST_READ_BIT); + vk_defer_buffer(sb, sa); + vk_upload_end(); + vk_upload_flush(1); + vmaInvalidateAllocation(c->vma, ra, 0, VK_WHOLE_SIZE); + for (uint32_t i = k0; i < k1; i++) { + const VkDeswzRegion *r = &dr[i]; + size_t pitch = (size_t)r->pitch16 * 16, bytes = pitch * r->rows * r->slices; + uint8_t *cpu = malloc(bytes); + if (!cpu) rt_fatal("vk: out of memory (deswizzle check)"); + uint32_t row_bytes = 0, rows, slices, gbpb; + uint64_t base; + unsigned th, td; + nvn_deswizzle_geom(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, &row_bytes, + &rows, &slices, &base, &th, &td, &gbpb); + nvn_deswizzle(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, t->mem, cpu, pitch); + uint64_t bad = 0, first = UINT64_MAX; + for (uint64_t row = 0; row < (uint64_t)r->rows * r->slices; row++) + if (memcmp(cpu + row * pitch, rmap + r->dst + row * pitch, row_bytes)) { + bad++; + if (first == UINT64_MAX) first = row; + } + free(cpu); + plat_add64(&n_checked, 1); + if (bad) { + plat_add64(&n_differ, 1); + rt_log("vk: deswizzle check: texture %#llx fmt %#x %ux%u level %u layer %u: %llu of %u rows differ, the " + "first %llu (th %u td %u, %u rows of %u bytes, %u slices)", (unsigned long long)e->pub.addr, + d->format, d->width, d->height, rg[i].imageSubresource.mipLevel, + rg[i].imageSubresource.baseArrayLayer, (unsigned long long)bad, r->rows * r->slices, + (unsigned long long)first, r->th, r->td, r->rows, row_bytes, r->slices); + } + } + vmaDestroyBuffer(c->vma, rb, ra); + if ((n_checked & 1023) == 0 || n_checked < 2) + rt_log("vk: deswizzle check: %lld regions compared, %lld differ", (long long)n_checked, (long long)n_differ); + cb = vk_upload_begin(); + barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + continue; + } + vk_defer_buffer(sb, sa); + if (k1 < n && vk_upload_held() > UPLOAD_HELD_MAX) { // submit what is recorded, go on in a new batch + barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW); + vk_upload_end(); + vk_upload_flush(0); + cb = vk_upload_begin(); + barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + } + } + barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW); + vk_upload_end(); + if (vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0); + free(dr), free(rg), free(ar); + return 1; +} + +// Copies the whole storage into the image (every level and layer the image has): on the GPU where it can +// (upload_gpu), else on the CPU. +static void upload(Tex *e) { + const tex_t *t = &e->t; + const nvn_texdesc *d = &t->d; + int bw, bh; + if (!nvn_fmt_bpb(d->format, &bw, &bh) || !t->mem) return; + uint64_t p0 = perf_begin(), u0 = kern_now_ns(); + int why = UP_GPU; + if (!upload_gpu(e)) why = up_why, upload_cpu(e); uint64_t dt = kern_now_ns() - u0; + up_n[why]++, up_bytes[why] += t->L.size, up_ns[why] += dt; if (e->uploaded) { n_reuploads++, perf_count(PERF_CNT_TEX_REUP); ru_n++, ru_bytes += t->L.size, ru_ns += dt; @@ -1404,6 +1610,14 @@ void vk_texture_new_submit(void) { (unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6, (unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6); ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = 0; + char ub[512]; + size_t uo = 0; + for (int i = 0; i < UP_WHY; i++) + if (up_n[i] && uo < sizeof ub) + uo += (size_t)snprintf(ub + uo, sizeof ub - uo, "%s %s %llu (%.1f MiB, %.2f ms)", uo ? "," : "", + up_why_name[i], (unsigned long long)up_n[i], up_bytes[i] / 1048576.0, up_ns[i] / 1e6); + if (uo) rt_log("perf: frame %llu: texture uploads by path:%s", (unsigned long long)f * 60, ub); + memset(up_n, 0, sizeof up_n), memset(up_bytes, 0, sizeof up_bytes), memset(up_ns, 0, sizeof up_ns); uint64_t ck, calls, untr; vk_mem_snap_stats(&ck, &calls, &untr); rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu",