mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 12:25:56 +00:00
Merge dev into carry/dynamic-res-100
cfg.c: the key list takes dev's DESWIZZLE and keeps this branch's DYNAMIC_RES and FIXED_RES_PATCH. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
91f6fc9838
14 files changed
+870
-67
No files matched your search
@@ -0,0 +1,111 @@
|
||||
// Generated by tools/build_shaders.py from runtime/shaders/bl_deswizzle.comp; do not edit.
|
||||
// source sha256 bd2065290e315e85d2fcdb609d812c15ba9a109d7506b3105850a5860677b0ef
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
static const uint32_t bl_deswizzle_spv[] = {
|
||||
0x07230203, 0x00010300, 0x000d000b, 0x000000d0, 0x00000000, 0x00020011, 0x00000001, 0x0006000b,
|
||||
0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001,
|
||||
0x0007000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000b, 0x0000001f, 0x00060010,
|
||||
0x00000004, 0x00000011, 0x00000040, 0x00000001, 0x00000001, 0x00040047, 0x0000000b, 0x0000000b,
|
||||
0x0000001a, 0x00030047, 0x00000010, 0x00000002, 0x00050048, 0x00000010, 0x00000000, 0x00000023,
|
||||
0x00000000, 0x00050048, 0x00000010, 0x00000001, 0x00000023, 0x00000004, 0x00050048, 0x00000010,
|
||||
0x00000002, 0x00000023, 0x00000008, 0x00050048, 0x00000010, 0x00000003, 0x00000023, 0x0000000c,
|
||||
0x00050048, 0x00000010, 0x00000004, 0x00000023, 0x00000010, 0x00050048, 0x00000010, 0x00000005,
|
||||
0x00000023, 0x00000014, 0x00050048, 0x00000010, 0x00000006, 0x00000023, 0x00000018, 0x00050048,
|
||||
0x00000010, 0x00000007, 0x00000023, 0x0000001c, 0x00050048, 0x00000010, 0x00000008, 0x00000023,
|
||||
0x00000020, 0x00050048, 0x00000010, 0x00000009, 0x00000023, 0x00000024, 0x00050048, 0x00000010,
|
||||
0x0000000a, 0x00000023, 0x00000028, 0x00040047, 0x0000001f, 0x0000000b, 0x0000001b, 0x00040047,
|
||||
0x000000ad, 0x00000006, 0x00000010, 0x00030047, 0x000000ae, 0x00000002, 0x00040048, 0x000000ae,
|
||||
0x00000000, 0x00000019, 0x00050048, 0x000000ae, 0x00000000, 0x00000023, 0x00000000, 0x00030047,
|
||||
0x000000b0, 0x00000019, 0x00040047, 0x000000b0, 0x00000021, 0x00000001, 0x00040047, 0x000000b0,
|
||||
0x00000022, 0x00000000, 0x00040047, 0x000000c1, 0x00000006, 0x00000010, 0x00030047, 0x000000c2,
|
||||
0x00000002, 0x00040048, 0x000000c2, 0x00000000, 0x00000018, 0x00050048, 0x000000c2, 0x00000000,
|
||||
0x00000023, 0x00000000, 0x00030047, 0x000000c4, 0x00000018, 0x00040047, 0x000000c4, 0x00000021,
|
||||
0x00000000, 0x00040047, 0x000000c4, 0x00000022, 0x00000000, 0x00040047, 0x000000cd, 0x0000000b,
|
||||
0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006,
|
||||
0x00000020, 0x00000000, 0x00040017, 0x00000009, 0x00000006, 0x00000003, 0x00040020, 0x0000000a,
|
||||
0x00000001, 0x00000009, 0x0004003b, 0x0000000a, 0x0000000b, 0x00000001, 0x0004002b, 0x00000006,
|
||||
0x0000000c, 0x00000001, 0x00040020, 0x0000000d, 0x00000001, 0x00000006, 0x000d001e, 0x00000010,
|
||||
0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006,
|
||||
0x00000006, 0x00000006, 0x00000006, 0x00040020, 0x00000011, 0x00000009, 0x00000010, 0x0004003b,
|
||||
0x00000011, 0x00000012, 0x00000009, 0x00040015, 0x00000013, 0x00000020, 0x00000001, 0x0004002b,
|
||||
0x00000013, 0x00000014, 0x0000000a, 0x00040020, 0x00000015, 0x00000009, 0x00000006, 0x0004002b,
|
||||
0x00000006, 0x00000019, 0x00000000, 0x0004002b, 0x00000006, 0x0000001d, 0x00000040, 0x0004003b,
|
||||
0x0000000a, 0x0000001f, 0x00000001, 0x0004002b, 0x00000013, 0x00000024, 0x00000002, 0x00020014,
|
||||
0x00000027, 0x0004002b, 0x00000013, 0x0000002d, 0x00000006, 0x0004002b, 0x00000013, 0x00000032,
|
||||
0x00000007, 0x0004002b, 0x00000006, 0x00000037, 0x00000020, 0x0004002b, 0x00000006, 0x00000048,
|
||||
0x00000005, 0x0004002b, 0x00000006, 0x0000004c, 0x0000001f, 0x0004002b, 0x00000013, 0x0000004f,
|
||||
0x00000008, 0x0004002b, 0x00000013, 0x00000052, 0x00000009, 0x0004002b, 0x00000006, 0x0000006a,
|
||||
0x00000004, 0x0004002b, 0x00000006, 0x0000006f, 0x00000002, 0x0004002b, 0x00000006, 0x00000078,
|
||||
0x00000008, 0x0004002b, 0x00000006, 0x00000083, 0x00000003, 0x0004002b, 0x00000013, 0x00000093,
|
||||
0x00000003, 0x0004002b, 0x00000013, 0x0000009b, 0x00000004, 0x0004002b, 0x00000013, 0x000000a4,
|
||||
0x00000005, 0x00040017, 0x000000ac, 0x00000006, 0x00000004, 0x0003001d, 0x000000ad, 0x000000ac,
|
||||
0x0003001e, 0x000000ae, 0x000000ad, 0x00040020, 0x000000af, 0x0000000c, 0x000000ae, 0x0004003b,
|
||||
0x000000af, 0x000000b0, 0x0000000c, 0x0004002b, 0x00000013, 0x000000b1, 0x00000000, 0x0004002b,
|
||||
0x00000013, 0x000000b2, 0x00000001, 0x0003001d, 0x000000c1, 0x000000ac, 0x0003001e, 0x000000c2,
|
||||
0x000000c1, 0x00040020, 0x000000c3, 0x0000000c, 0x000000c2, 0x0004003b, 0x000000c3, 0x000000c4,
|
||||
0x0000000c, 0x00040020, 0x000000c9, 0x0000000c, 0x000000ac, 0x0006002c, 0x00000009, 0x000000cd,
|
||||
0x0000001d, 0x0000000c, 0x0000000c, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003,
|
||||
0x000200f8, 0x00000005, 0x000300f7, 0x000000ce, 0x00000000, 0x000300fb, 0x00000019, 0x000000cf,
|
||||
0x000200f8, 0x000000cf, 0x00050041, 0x0000000d, 0x0000000e, 0x0000000b, 0x0000000c, 0x0004003d,
|
||||
0x00000006, 0x0000000f, 0x0000000e, 0x00050041, 0x00000015, 0x00000016, 0x00000012, 0x00000014,
|
||||
0x0004003d, 0x00000006, 0x00000017, 0x00000016, 0x00050084, 0x00000006, 0x00000018, 0x0000000f,
|
||||
0x00000017, 0x00050041, 0x0000000d, 0x0000001a, 0x0000000b, 0x00000019, 0x0004003d, 0x00000006,
|
||||
0x0000001b, 0x0000001a, 0x00050080, 0x00000006, 0x0000001c, 0x00000018, 0x0000001b, 0x00050084,
|
||||
0x00000006, 0x0000001e, 0x0000001c, 0x0000001d, 0x00050041, 0x0000000d, 0x00000020, 0x0000001f,
|
||||
0x00000019, 0x0004003d, 0x00000006, 0x00000021, 0x00000020, 0x00050080, 0x00000006, 0x00000022,
|
||||
0x0000001e, 0x00000021, 0x00050041, 0x00000015, 0x00000025, 0x00000012, 0x00000024, 0x0004003d,
|
||||
0x00000006, 0x00000026, 0x00000025, 0x000500ae, 0x00000027, 0x00000028, 0x00000022, 0x00000026,
|
||||
0x000300f7, 0x0000002a, 0x00000000, 0x000400fa, 0x00000028, 0x00000029, 0x0000002a, 0x000200f8,
|
||||
0x00000029, 0x000200f9, 0x000000ce, 0x000200f8, 0x0000002a, 0x00050041, 0x00000015, 0x0000002e,
|
||||
0x00000012, 0x0000002d, 0x0004003d, 0x00000006, 0x0000002f, 0x0000002e, 0x000500c4, 0x00000006,
|
||||
0x00000030, 0x0000000c, 0x0000002f, 0x00050041, 0x00000015, 0x00000033, 0x00000012, 0x00000032,
|
||||
0x0004003d, 0x00000006, 0x00000034, 0x00000033, 0x000500c4, 0x00000006, 0x00000035, 0x0000000c,
|
||||
0x00000034, 0x00050080, 0x00000006, 0x0000003c, 0x0000002f, 0x00000034, 0x000500c4, 0x00000006,
|
||||
0x0000003d, 0x00000037, 0x0000003c, 0x00050086, 0x00000006, 0x00000041, 0x00000022, 0x0000003d,
|
||||
0x00050089, 0x00000006, 0x00000045, 0x00000022, 0x0000003d, 0x000500c2, 0x00000006, 0x00000049,
|
||||
0x00000045, 0x00000048, 0x000500c7, 0x00000006, 0x0000004d, 0x00000045, 0x0000004c, 0x00050041,
|
||||
0x00000015, 0x00000050, 0x00000012, 0x0000004f, 0x0004003d, 0x00000006, 0x00000051, 0x00000050,
|
||||
0x00050041, 0x00000015, 0x00000053, 0x00000012, 0x00000052, 0x0004003d, 0x00000006, 0x00000054,
|
||||
0x00000053, 0x00050084, 0x00000006, 0x00000055, 0x00000051, 0x00000054, 0x00050086, 0x00000006,
|
||||
0x00000059, 0x00000041, 0x00000055, 0x00050089, 0x00000006, 0x0000005d, 0x00000041, 0x00000055,
|
||||
0x00050086, 0x00000006, 0x00000062, 0x0000005d, 0x00000051, 0x00050089, 0x00000006, 0x00000067,
|
||||
0x0000005d, 0x00000051, 0x00050084, 0x00000006, 0x0000006b, 0x00000067, 0x0000006a, 0x000500c2,
|
||||
0x00000006, 0x0000006d, 0x0000004d, 0x0000006a, 0x000500c7, 0x00000006, 0x0000006e, 0x0000006d,
|
||||
0x0000000c, 0x00050084, 0x00000006, 0x00000070, 0x0000006e, 0x0000006f, 0x00050080, 0x00000006,
|
||||
0x00000071, 0x0000006b, 0x00000070, 0x000500c2, 0x00000006, 0x00000073, 0x0000004d, 0x0000000c,
|
||||
0x000500c7, 0x00000006, 0x00000074, 0x00000073, 0x0000000c, 0x00050080, 0x00000006, 0x00000075,
|
||||
0x00000071, 0x00000074, 0x00050084, 0x00000006, 0x00000079, 0x00000062, 0x00000078, 0x00050084,
|
||||
0x00000006, 0x0000007b, 0x00000079, 0x00000030, 0x00050089, 0x00000006, 0x0000007e, 0x00000049,
|
||||
0x00000030, 0x00050084, 0x00000006, 0x0000007f, 0x0000007e, 0x00000078, 0x00050080, 0x00000006,
|
||||
0x00000080, 0x0000007b, 0x0000007f, 0x000500c2, 0x00000006, 0x00000082, 0x0000004d, 0x0000006f,
|
||||
0x000500c7, 0x00000006, 0x00000084, 0x00000082, 0x00000083, 0x00050084, 0x00000006, 0x00000085,
|
||||
0x00000084, 0x0000006f, 0x00050080, 0x00000006, 0x00000086, 0x00000080, 0x00000085, 0x000500c7,
|
||||
0x00000006, 0x00000088, 0x00000045, 0x0000000c, 0x00050080, 0x00000006, 0x00000089, 0x00000086,
|
||||
0x00000088, 0x00050084, 0x00000006, 0x0000008d, 0x00000059, 0x00000035, 0x00050086, 0x00000006,
|
||||
0x00000090, 0x00000049, 0x00000030, 0x00050080, 0x00000006, 0x00000091, 0x0000008d, 0x00000090,
|
||||
0x00050041, 0x00000015, 0x00000094, 0x00000012, 0x00000093, 0x0004003d, 0x00000006, 0x00000095,
|
||||
0x00000094, 0x000500ae, 0x00000027, 0x00000096, 0x00000075, 0x00000095, 0x000400a8, 0x00000027,
|
||||
0x00000097, 0x00000096, 0x000300f7, 0x00000099, 0x00000000, 0x000400fa, 0x00000097, 0x00000098,
|
||||
0x00000099, 0x000200f8, 0x00000098, 0x00050041, 0x00000015, 0x0000009c, 0x00000012, 0x0000009b,
|
||||
0x0004003d, 0x00000006, 0x0000009d, 0x0000009c, 0x000500ae, 0x00000027, 0x0000009e, 0x00000089,
|
||||
0x0000009d, 0x000200f9, 0x00000099, 0x000200f8, 0x00000099, 0x000700f5, 0x00000027, 0x0000009f,
|
||||
0x00000096, 0x0000002a, 0x0000009e, 0x00000098, 0x000400a8, 0x00000027, 0x000000a0, 0x0000009f,
|
||||
0x000300f7, 0x000000a2, 0x00000000, 0x000400fa, 0x000000a0, 0x000000a1, 0x000000a2, 0x000200f8,
|
||||
0x000000a1, 0x00050041, 0x00000015, 0x000000a5, 0x00000012, 0x000000a4, 0x0004003d, 0x00000006,
|
||||
0x000000a6, 0x000000a5, 0x000500ae, 0x00000027, 0x000000a7, 0x00000091, 0x000000a6, 0x000200f9,
|
||||
0x000000a2, 0x000200f8, 0x000000a2, 0x000700f5, 0x00000027, 0x000000a8, 0x0000009f, 0x00000099,
|
||||
0x000000a7, 0x000000a1, 0x000300f7, 0x000000aa, 0x00000000, 0x000400fa, 0x000000a8, 0x000000a9,
|
||||
0x000000aa, 0x000200f8, 0x000000a9, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000aa, 0x00050041,
|
||||
0x00000015, 0x000000b3, 0x00000012, 0x000000b2, 0x0004003d, 0x00000006, 0x000000b4, 0x000000b3,
|
||||
0x00050041, 0x00000015, 0x000000b6, 0x00000012, 0x0000009b, 0x0004003d, 0x00000006, 0x000000b7,
|
||||
0x000000b6, 0x00050084, 0x00000006, 0x000000b8, 0x00000091, 0x000000b7, 0x00050080, 0x00000006,
|
||||
0x000000ba, 0x000000b8, 0x00000089, 0x00050084, 0x00000006, 0x000000bd, 0x000000ba, 0x00000095,
|
||||
0x00050080, 0x00000006, 0x000000be, 0x000000b4, 0x000000bd, 0x00050080, 0x00000006, 0x000000c0,
|
||||
0x000000be, 0x00000075, 0x00050041, 0x00000015, 0x000000c5, 0x00000012, 0x000000b1, 0x0004003d,
|
||||
0x00000006, 0x000000c6, 0x000000c5, 0x00050080, 0x00000006, 0x000000c8, 0x000000c6, 0x00000022,
|
||||
0x00060041, 0x000000c9, 0x000000ca, 0x000000c4, 0x000000b1, 0x000000c8, 0x0004003d, 0x000000ac,
|
||||
0x000000cb, 0x000000ca, 0x00060041, 0x000000c9, 0x000000cc, 0x000000b0, 0x000000b1, 0x000000c0,
|
||||
0x0003003e, 0x000000cc, 0x000000cb, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000ce, 0x000100fd,
|
||||
0x00010038,
|
||||
};
|
||||
+1
-1
@@ -40,7 +40,7 @@ static const char *const known[] = {
|
||||
"BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y",
|
||||
"BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA",
|
||||
"COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT",
|
||||
"DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
|
||||
"DESC_TEMPLATE", "DESWIZZLE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
|
||||
"DUMP_LOG", "DYNAMIC_RES", "DYN_DELTA", "EXEFS", "FEEDBACK", "FIXED_RES_PATCH", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES",
|
||||
"FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FRAME_QUEUE",
|
||||
"FREEZE_IDLE", "FREE_CORES",
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#version 450
|
||||
// Block-linear to linear copy of one texture level and layer on the GPU (vk_deswizzle.c), the same layout as
|
||||
// tex_deswizzle.c nvn_bl_offset: GOB = 64 bytes x 8 rows; a block is 1 GOB wide, 2^th GOBs high and 2^td deep, its
|
||||
// GOBs ordered y first, then z; blocks fill the level x first, then y, then z.
|
||||
// One invocation per 16-byte segment of the block-linear storage, in storage order, so the reads (from the pool's
|
||||
// imported buffer: guest memory, over PCIe on a discrete GPU) are contiguous; the writes go to a device-local buffer
|
||||
// with rows of pitch16 segments. Segments outside the level's rows, slices or row width are padding and skipped.
|
||||
layout(local_size_x = 64) in;
|
||||
layout(std430, set = 0, binding = 0) readonly buffer Src { uvec4 src[]; };
|
||||
layout(std430, set = 0, binding = 1) writeonly buffer Dst { uvec4 dst[]; };
|
||||
layout(push_constant) uniform Push {
|
||||
uint src_base; // first segment of the level and layer in the src binding
|
||||
uint dst_base; // first segment of its linear copy in the dst binding
|
||||
uint segs; // segments to visit: whole blocks covering the level
|
||||
uint pitch16; // segments per linear row (row bytes rounded up to 16)
|
||||
uint rows, slices, th, td;
|
||||
uint gobs_x, blocks_y;
|
||||
uint nx; // workgroups along x (the dispatch is 2D when one dimension is not enough)
|
||||
} pc;
|
||||
|
||||
void main() {
|
||||
uint s = (gl_WorkGroupID.y * pc.nx + gl_WorkGroupID.x) * 64u + gl_LocalInvocationID.x;
|
||||
if (s >= pc.segs) return;
|
||||
uint bh = 1u << pc.th, bd = 1u << pc.td;
|
||||
uint per_block = 32u << (pc.th + pc.td); // 512-byte GOBs of 32 segments each
|
||||
uint block = s / per_block, inb = s % per_block;
|
||||
uint gob = inb >> 5u, ing = inb & 31u;
|
||||
uint per_layer = pc.gobs_x * pc.blocks_y;
|
||||
uint bz = block / per_layer, rem = block % per_layer;
|
||||
uint by = rem / pc.gobs_x, bx = rem % pc.gobs_x;
|
||||
// Inside a GOB, segment bits: 4 = x % 64 / 32, 3..2 = y % 8 / 2, 1 = x % 32 / 16, 0 = y % 2.
|
||||
uint xseg = bx * 4u + ((ing >> 4u) & 1u) * 2u + ((ing >> 1u) & 1u);
|
||||
uint y = by * 8u * bh + (gob % bh) * 8u + ((ing >> 2u) & 3u) * 2u + (ing & 1u);
|
||||
uint z = bz * bd + gob / bh;
|
||||
if (xseg >= pc.pitch16 || y >= pc.rows || z >= pc.slices) return;
|
||||
dst[pc.dst_base + (z * pc.rows + y) * pc.pitch16 + xseg] = src[pc.src_base + s];
|
||||
}
|
||||
+37
-1
@@ -329,6 +329,15 @@ void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestro
|
||||
void vk_defer_view(VkImageView view);
|
||||
void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it
|
||||
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc);
|
||||
// free(p) once the GPU is past the submission being recorded, like the deferrals above: host records that a recording
|
||||
// may still read through a pointer taken before another thread dropped them (vk_texture.c destroy_tex).
|
||||
void vk_defer_free(void *p);
|
||||
// Device-local scratch for the upload batch (storage, transfer src and dst), from a pool of its own (vk_memory.c).
|
||||
VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc);
|
||||
// Staging and scratch bytes the open upload batch holds until it completes. Large texture loads submit the batch early
|
||||
// (vk_texture.c UPLOAD_HELD_MAX) so the GPU starts on it and its buffers come back before the pools need new blocks.
|
||||
VkDeviceSize vk_upload_held(void);
|
||||
void vk_upload_pools_init(void); // vk_device.c, after the allocator: the staging and scratch pools and their blocks
|
||||
// ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c.
|
||||
enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK };
|
||||
int vk_astc_mode(void); // decided once per device (logged); VK_ASTC_CPU before vk_init
|
||||
@@ -352,7 +361,31 @@ void vk_astc_time_enable(int on);
|
||||
int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us); // totals so far
|
||||
void vk_astc_time_summary(void); // logs the totals
|
||||
void vk_astc_release(void); // vk_device.c, before the device goes: frees the decoder's objects
|
||||
void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value
|
||||
// ---- Block-linear deswizzle on the GPU (vk_deswizzle.c). TOTK_DESWIZZLE = auto | gpu | cpu | check.
|
||||
enum { VK_DESWZ_CPU, VK_DESWZ_GPU, VK_DESWZ_CHECK };
|
||||
int vk_deswz_mode(void); // decided once per device (logged); VK_DESWZ_CPU before vk_init
|
||||
// One level and layer: its storage starts at byte src of the bound source range and covers segs 16-byte segments
|
||||
// (whole blocks, cut at the storage end); its linear copy goes to byte dst of the destination, rows of pitch16 segments,
|
||||
// rows x slices of them. th, td, gobs_x, blocks_y: as tex_deswizzle.c nvn_bl_offset.
|
||||
typedef struct {
|
||||
VkDeviceSize src, dst;
|
||||
uint32_t segs, pitch16, rows, slices, th, td, gobs_x, blocks_y;
|
||||
} VkDeswzRegion;
|
||||
// Records into cb a dispatch per region reading [src_off, src_off + src_size) of src (a storage buffer) and writing
|
||||
// dst (device-local, dst_size bytes). The caller records the barriers around it. 0 when nothing was recorded: no
|
||||
// pipeline, src_off not 16-byte aligned, a range over the device's storage-buffer limit, a region outside the ranges.
|
||||
int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst,
|
||||
VkDeviceSize dst_size, const VkDeswzRegion *rg, int n);
|
||||
// 1 when vk_deswz_record will take these regions (their dst offsets aside) with destinations of up to max_dst bytes.
|
||||
int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst);
|
||||
void vk_deswz_release(void); // vk_device.c, before the device goes
|
||||
// 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission
|
||||
// sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch).
|
||||
int vk_mem_span_live(const VkMemSpan *sp);
|
||||
// vk_sync.c, when it starts recording the submission that will signal ctx->timeline with this value: objects destroyed
|
||||
// from then on are freed once the timeline reaches it.
|
||||
void vk_mem_note_fence(uint64_t timeline_value);
|
||||
void vk_mem_fault_log(void); // vk_device_fault_report: the deferred-free state, lock-free
|
||||
void vk_mem_collect(void); // frees what has completed
|
||||
typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats;
|
||||
void vk_mem_stats(VkMemStats *s);
|
||||
@@ -384,6 +417,9 @@ typedef struct {
|
||||
uint8_t scale; // 1, or TOTK_SHADOW_SCALE for a shadow map made larger than the game's (vk_texture.c
|
||||
// shadow_scale_of): width and height above are the larger ones, the game's regions
|
||||
// are multiplied by it
|
||||
uint8_t gone; // the entry was dropped (vk_texture.c destroy_tex, under the texture lock): image and
|
||||
// record live until the submission being recorded is done, and nothing may be cached
|
||||
// by its image handle, which the driver can hand out again after that
|
||||
} VkTexImage;
|
||||
// The window texture the backend presents (vk_backend.c): *image and *format become the float picture the game copied
|
||||
// into it, when the last draw into it was that copy and the swapchain is HDR10.
|
||||
|
||||
+1
-5
@@ -463,13 +463,9 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons
|
||||
cap_in = (cap_in + al - 1) / al * al;
|
||||
cap_out = (cap_out + al - 1) / al * al;
|
||||
if (!bc) cap_bc = 0;
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = cap_in + cap_out + cap_bc,
|
||||
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
|
||||
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
||||
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
||||
VkBuffer sb;
|
||||
VmaAllocation sa;
|
||||
if (vmaCreateBuffer(c->vma, &bi, &aci, &sb, &sa, NULL) != VK_SUCCESS) {
|
||||
if (vk_scratch_buffer(cap_in + cap_out + cap_bc, &sb, &sa) != VK_SUCCESS) {
|
||||
if (plat_inc32(&n_fail) <= 16)
|
||||
rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded",
|
||||
(unsigned long long)(cap_in + cap_out));
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
// Block-linear textures deswizzled by the GPU (runtime/shaders/bl_deswizzle.comp): vk_texture.c upload binds the
|
||||
// texture's storage in its pool's imported buffer and lets a compute pass write the linear copy that a buffer-to-image
|
||||
// copy (or the ASTC decoder) then takes, instead of tex_deswizzle.c walking every 16 bytes on the replay thread
|
||||
// (50-236 ms frames on entering new areas, 2026-11-07 play sessions). The same design as MacNativeGames' TotK port
|
||||
// (d2r_gfx nvn_mtl.m: deswizzle from the pool at GPU execution time), written for Vulkan from our CPU layout code.
|
||||
//
|
||||
// TOTK_DESWIZZLE = auto (default) / gpu: the compute pass where the storage is an imported pool (vk_mem_span_live),
|
||||
// the CPU otherwise; cpu: always the CPU; check: gpu, and every upload is also deswizzled on the CPU and compared
|
||||
// (waits for the GPU; for testing only).
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "bl_deswizzle_spv.h"
|
||||
#include "cfg.h"
|
||||
#include "host.h"
|
||||
#include "vk.h"
|
||||
#include "vk_rec.h"
|
||||
#include "platform/platform.h"
|
||||
|
||||
typedef struct {
|
||||
uint32_t src_base, dst_base, segs, pitch16, rows, slices, th, td, gobs_x, blocks_y, nx;
|
||||
} Push;
|
||||
|
||||
static PlatMutex g_lock = PLAT_MUTEX_INIT;
|
||||
static struct {
|
||||
VkDevice dev;
|
||||
int mode; // VK_DESWZ_*; -1 before the first vk_deswz_mode with a device
|
||||
VkDescriptorSetLayout dsl;
|
||||
VkPipelineLayout layout;
|
||||
VkPipeline pipe;
|
||||
PFN_vkCmdPushDescriptorSetKHR push;
|
||||
} G = {.mode = -1};
|
||||
static volatile int64_t n_regions, n_bytes;
|
||||
|
||||
static int create_pipeline(VkCtx *c) {
|
||||
G.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR");
|
||||
if (!G.push) return 0;
|
||||
VkDescriptorSetLayoutBinding b[2];
|
||||
for (int i = 0; i < 2; i++)
|
||||
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
||||
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
|
||||
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
||||
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
|
||||
.bindingCount = 2, .pBindings = b};
|
||||
if (vkCreateDescriptorSetLayout(c->device, &dci, NULL, &G.dsl) != VK_SUCCESS) return 0;
|
||||
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push)};
|
||||
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
|
||||
.pSetLayouts = &G.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
|
||||
if (vkCreatePipelineLayout(c->device, &li, NULL, &G.layout) != VK_SUCCESS) return 0;
|
||||
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bl_deswizzle_spv,
|
||||
.pCode = bl_deswizzle_spv};
|
||||
VkShaderModule m = VK_NULL_HANDLE;
|
||||
if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0;
|
||||
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
||||
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
||||
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
||||
.layout = G.layout};
|
||||
VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.pipe);
|
||||
vkDestroyShaderModule(c->device, m, NULL);
|
||||
if (r != VK_SUCCESS) G.pipe = VK_NULL_HANDLE;
|
||||
return r == VK_SUCCESS;
|
||||
}
|
||||
|
||||
int vk_deswz_mode(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!c) return VK_DESWZ_CPU;
|
||||
plat_mutex_lock_shared(&g_lock);
|
||||
int m = G.dev == c->device ? G.mode : -1;
|
||||
plat_mutex_unlock_shared(&g_lock);
|
||||
if (m >= 0) return m;
|
||||
plat_mutex_lock(&g_lock);
|
||||
if (G.dev != c->device) { // the old device's objects went with it (probes recreate the device)
|
||||
memset(&G, 0, sizeof G);
|
||||
G.dev = c->device;
|
||||
G.mode = -1;
|
||||
}
|
||||
if (G.mode < 0) {
|
||||
const char *e = cfg_get("TOTK_DESWIZZLE");
|
||||
if (!e || !*e) e = "auto";
|
||||
int want = !strcmp(e, "cpu") || !strcmp(e, "0") ? VK_DESWZ_CPU : !strcmp(e, "check") ? VK_DESWZ_CHECK : VK_DESWZ_GPU;
|
||||
if (strcmp(e, "auto") && strcmp(e, "gpu") && strcmp(e, "1") && strcmp(e, "cpu") && strcmp(e, "0") &&
|
||||
strcmp(e, "check"))
|
||||
rt_log("vk: TOTK_DESWIZZLE=%s is not auto, gpu, cpu or check: auto", e);
|
||||
if (want != VK_DESWZ_CPU && !create_pipeline(c)) {
|
||||
rt_log("vk: the deswizzle compute pass is unavailable on %s: textures deswizzle on the CPU", c->props.deviceName);
|
||||
want = VK_DESWZ_CPU;
|
||||
}
|
||||
G.mode = want;
|
||||
rt_log("vk: block-linear textures deswizzle on the %s%s", want == VK_DESWZ_CPU ? "CPU" : "GPU where the storage is "
|
||||
"an imported pool", want == VK_DESWZ_CHECK ? ", checked against the CPU (TOTK_DESWIZZLE=check)" : "");
|
||||
}
|
||||
m = G.mode;
|
||||
plat_mutex_unlock(&g_lock);
|
||||
return m;
|
||||
}
|
||||
|
||||
static uint32_t max_groups_x(const VkCtx *c) {
|
||||
uint32_t m = c->props.limits.maxComputeWorkGroupCount[0];
|
||||
return m > 65535 ? 65535 : m;
|
||||
}
|
||||
|
||||
// The bound source range starts at the alignment below src_off.
|
||||
static VkDeviceSize src_lo(const VkCtx *c, VkDeviceSize src_off) {
|
||||
VkDeviceSize al = c->props.limits.minStorageBufferOffsetAlignment;
|
||||
if (al < 16) al = 16;
|
||||
return src_off / al * al;
|
||||
}
|
||||
|
||||
int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!c || !n || (src_off & 15) || vk_deswz_mode() == VK_DESWZ_CPU) return 0;
|
||||
VkDeviceSize range = src_off + src_size - src_lo(c, src_off), lim = c->props.limits.maxStorageBufferRange;
|
||||
if (range > lim || max_dst > lim) return 0;
|
||||
uint32_t maxg = max_groups_x(c);
|
||||
for (int i = 0; i < n; i++) { // the 32-bit segment indices of the shader stay inside the bindings
|
||||
uint64_t groups = ((uint64_t)rg[i].segs + 63) / 64, out = (uint64_t)rg[i].pitch16 * rg[i].rows * rg[i].slices;
|
||||
if ((rg[i].src & 15) || rg[i].src + (uint64_t)rg[i].segs * 16 > src_size || !rg[i].segs ||
|
||||
(src_off - src_lo(c, src_off) + rg[i].src) / 16 + rg[i].segs > UINT32_MAX || out > UINT32_MAX ||
|
||||
out * 16 > max_dst || (groups + maxg - 1) / maxg > c->props.limits.maxComputeWorkGroupCount[1])
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst,
|
||||
VkDeviceSize dst_size, const VkDeswzRegion *rg, int n) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!vk_deswz_fits(src_off, src_size, rg, n, dst_size)) return 0;
|
||||
for (int i = 0; i < n; i++)
|
||||
if ((rg[i].dst & 15) || rg[i].dst + (uint64_t)rg[i].pitch16 * 16 * rg[i].rows * rg[i].slices > dst_size) return 0;
|
||||
plat_mutex_lock_shared(&g_lock);
|
||||
VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE;
|
||||
plat_mutex_unlock_shared(&g_lock);
|
||||
if (!pipe) return 0;
|
||||
VkDeviceSize lo = src_lo(c, src_off), range = src_off + src_size - lo;
|
||||
uint32_t maxg = max_groups_x(c);
|
||||
VkDescriptorBufferInfo dbi[2] = {{src, lo, range}, {dst, 0, dst_size}};
|
||||
VkWriteDescriptorSet w[2];
|
||||
for (int i = 0; i < 2; i++)
|
||||
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
|
||||
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
||||
.pBufferInfo = &dbi[i]};
|
||||
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe);
|
||||
rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 2, w);
|
||||
uint64_t bytes = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
const VkDeswzRegion *r = &rg[i];
|
||||
uint64_t groups = ((uint64_t)r->segs + 63) / 64;
|
||||
uint32_t nx = groups < maxg ? (uint32_t)groups : maxg, ny = (uint32_t)((groups + nx - 1) / nx);
|
||||
Push pc = {.src_base = (uint32_t)((src_off - lo + r->src) / 16), .dst_base = (uint32_t)(r->dst / 16),
|
||||
.segs = r->segs, .pitch16 = r->pitch16, .rows = r->rows, .slices = r->slices, .th = r->th, .td = r->td,
|
||||
.gobs_x = r->gobs_x, .blocks_y = r->blocks_y, .nx = nx};
|
||||
vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
|
||||
vkCmdDispatch(cb, nx, ny, 1);
|
||||
bytes += (uint64_t)r->segs * 16;
|
||||
}
|
||||
int64_t before = plat_add64(&n_regions, n) - n;
|
||||
plat_add64(&n_bytes, (int64_t)bytes);
|
||||
if (!before || (before + n) / 4096 != before / 4096)
|
||||
rt_log("vk: deswizzle: %lld regions, %.1f MiB of block-linear storage deswizzled on the GPU so far",
|
||||
(long long)(before + n), (double)n_bytes / 1048576.0);
|
||||
return 1;
|
||||
}
|
||||
|
||||
void vk_deswz_release(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
plat_mutex_lock(&g_lock);
|
||||
if (c && G.dev == c->device) {
|
||||
if (G.pipe) vkDestroyPipeline(c->device, G.pipe, NULL);
|
||||
if (G.layout) vkDestroyPipelineLayout(c->device, G.layout, NULL);
|
||||
if (G.dsl) vkDestroyDescriptorSetLayout(c->device, G.dsl, NULL);
|
||||
}
|
||||
memset(&G, 0, sizeof G);
|
||||
G.mode = -1;
|
||||
plat_mutex_unlock(&g_lock);
|
||||
}
|
||||
+90
-9
@@ -16,9 +16,69 @@ static VkCtx g_vk;
|
||||
static int g_ready;
|
||||
static PlatMutex g_lock = PLAT_MUTEX_INIT;
|
||||
|
||||
// ---- GPU address map for device faults
|
||||
// VK_EXT_device_address_binding_report: the driver reports each bind and unbind of an object's GPU virtual range through
|
||||
// the debug messenger. The newest AB_N events stay in a ring written without a lock (one atomic add each), so a device
|
||||
// fault names the object behind its address and whether it was unbound before the fault, which is memory freed while
|
||||
// a submission still used it. One event per image, buffer or memory block made or freed, none per draw.
|
||||
// TOTK_ADDR_REPORT=0 turns it off.
|
||||
#define AB_N (1u << 15)
|
||||
typedef struct { uint64_t seq, base, size, handle, issued, done; int32_t type; uint32_t unbind; } AbEvent;
|
||||
static AbEvent *ab_ring;
|
||||
static uint64_t ab_head;
|
||||
static int ab_want; // the instance has the messenger for it (create_instance)
|
||||
|
||||
static void ab_record(const VkDebugUtilsMessengerCallbackDataEXT *d) {
|
||||
const VkDeviceAddressBindingCallbackDataEXT *b = NULL;
|
||||
for (const VkBaseInStructure *p = d->pNext; p; p = p->pNext)
|
||||
if (p->sType == VK_STRUCTURE_TYPE_DEVICE_ADDRESS_BINDING_CALLBACK_DATA_EXT) b = (const void *)p;
|
||||
if (!b || !ab_ring) return;
|
||||
uint64_t i = __atomic_fetch_add(&ab_head, 1, __ATOMIC_RELAXED);
|
||||
AbEvent *e = &ab_ring[i & (AB_N - 1)];
|
||||
__atomic_store_n(&e->seq, 0, __ATOMIC_RELAXED);
|
||||
e->base = b->baseAddress, e->size = b->size;
|
||||
e->handle = d->objectCount ? d->pObjects[0].objectHandle : 0;
|
||||
e->type = d->objectCount ? (int32_t)d->pObjects[0].objectType : 0;
|
||||
e->unbind = b->bindingType == VK_DEVICE_ADDRESS_BINDING_TYPE_UNBIND_EXT;
|
||||
e->issued = vk_sync_issued(), e->done = vk_sync_fence_done();
|
||||
__atomic_store_n(&e->seq, i + 1, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
static const char *ab_type_name(int32_t t, char *buf, size_t n) {
|
||||
switch (t) {
|
||||
case VK_OBJECT_TYPE_IMAGE: return "image";
|
||||
case VK_OBJECT_TYPE_BUFFER: return "buffer";
|
||||
case VK_OBJECT_TYPE_DEVICE_MEMORY: return "memory";
|
||||
default: snprintf(buf, n, "object type %d", (int)t); return buf;
|
||||
}
|
||||
}
|
||||
|
||||
// The newest events whose range meets the precision-aligned range around a fault address, newest first.
|
||||
static void ab_report(uint64_t addr, uint64_t precision) {
|
||||
if (!ab_ring) return;
|
||||
uint64_t lo = precision > 1 ? addr & ~(precision - 1) : addr, hi = lo + (precision > 1 ? precision : 1);
|
||||
uint64_t head = __atomic_load_n(&ab_head, __ATOMIC_ACQUIRE), n = head < AB_N ? head : AB_N;
|
||||
int found = 0;
|
||||
for (uint64_t k = 1; k <= n && found < 6; k++) {
|
||||
const AbEvent *e = &ab_ring[(head - k) & (AB_N - 1)];
|
||||
if (__atomic_load_n(&e->seq, __ATOMIC_ACQUIRE) != head - k + 1) continue;
|
||||
if (e->base >= hi || e->base + e->size <= lo) continue;
|
||||
found++;
|
||||
char tn[32];
|
||||
rt_log("vk: fault address %#llx: %s %#llx [%#llx, +%#llx] %s %llu address events before the fault, "
|
||||
"submitted %llu, done %llu", (unsigned long long)addr, ab_type_name(e->type, tn, sizeof tn),(unsigned long long)e->handle,
|
||||
(unsigned long long)e->base, (unsigned long long)e->size, e->unbind ? "unbound" : "bound",
|
||||
(unsigned long long)k, (unsigned long long)e->issued, (unsigned long long)e->done);
|
||||
}
|
||||
if (!found) rt_log("vk: fault address %#llx: no binding in the last %llu address events", (unsigned long long)addr,
|
||||
(unsigned long long)n);
|
||||
}
|
||||
|
||||
static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type,
|
||||
const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) {
|
||||
(void)type, (void)user;
|
||||
(void)user;
|
||||
if (type & VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT) { ab_record(d); return VK_FALSE; }
|
||||
if (sev < VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) return VK_FALSE; // info, asked for the address events
|
||||
int err = (sev & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) != 0;
|
||||
if (err) plat_inc32(&g_vk.validation_errors);
|
||||
static volatile int32_t logged;
|
||||
@@ -76,7 +136,10 @@ static int create_instance(void) {
|
||||
rt_log("vk: TOTK_VK_VALIDATION set but %s is not installed; continuing without", layers[0]);
|
||||
validate = 0;
|
||||
}
|
||||
if (validate) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
|
||||
const char *ar = cfg_get("TOTK_ADDR_REPORT");
|
||||
ab_want = !(ar && !strcmp(ar, "0")) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
|
||||
if (ab_want && !ab_ring && !(ab_ring = calloc(AB_N, sizeof *ab_ring))) ab_want = 0;
|
||||
if (validate || ab_want) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
|
||||
if (vk_hdr_wanted() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats
|
||||
ext[n_ext++] = VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME;
|
||||
vk_dlaa_instance_exts(ext, &n_ext, 12);
|
||||
@@ -89,16 +152,18 @@ static int create_instance(void) {
|
||||
VkResult r = vkCreateInstance(&ci, NULL, &g_vk.instance);
|
||||
if (r != VK_SUCCESS) { rt_log("vk: vkCreateInstance failed (%d); is a Vulkan 1.3 driver installed?", (int)r); return 0; }
|
||||
g_vk.validation = validate;
|
||||
if (validate) {
|
||||
if (validate || ab_want) {
|
||||
PFN_vkCreateDebugUtilsMessengerEXT f =
|
||||
(PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkCreateDebugUtilsMessengerEXT");
|
||||
VkDebugUtilsMessengerCreateInfoEXT mi = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
|
||||
.messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
|
||||
.messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
|
||||
VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
|
||||
.messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT |
|
||||
(ab_want ? VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT : 0),
|
||||
.messageType = (validate ? VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
|
||||
VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT : 0) |
|
||||
(ab_want ? VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT : 0),
|
||||
.pfnUserCallback = debug_cb};
|
||||
if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger);
|
||||
if (!f || f(g_vk.instance, &mi, NULL, &g_vk.messenger) != VK_SUCCESS) ab_want = 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
@@ -217,6 +282,7 @@ typedef struct {
|
||||
VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl;
|
||||
VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbl;
|
||||
VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr;
|
||||
VkPhysicalDeviceAddressBindingReportFeaturesEXT abr;
|
||||
} FeatureChain;
|
||||
|
||||
static int create_device(void) {
|
||||
@@ -251,6 +317,13 @@ static int create_device(void) {
|
||||
// Ryujinx's SPIR-V declares fragment shader interlock whenever the host reports it (vk_pipeline.c spirv_fixup).
|
||||
OPTIONAL_EXT(has_fragment_shader_interlock, VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME, fsi,
|
||||
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT);
|
||||
// The GPU address map for device faults (ab_record), with the messenger made for it
|
||||
int has_abr = 0;
|
||||
if (ab_want && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME)) {
|
||||
has_abr = 1;
|
||||
ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME;
|
||||
CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT);
|
||||
}
|
||||
// Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering
|
||||
int has_fbl = 0, has_lr = 0;
|
||||
if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) &&
|
||||
@@ -314,6 +387,10 @@ static int create_device(void) {
|
||||
else if (!h->textureCompressionBC) miss = "textureCompressionBC";
|
||||
if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; }
|
||||
if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension
|
||||
rt_log("vk: GPU address map for device faults: %s",
|
||||
has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report; TOTK_ADDR_REPORT=0 turns it off)"
|
||||
: !ab_want ? "off (TOTK_ADDR_REPORT=0 or no VK_EXT_debug_utils)"
|
||||
: "unavailable (no VK_EXT_device_address_binding_report)");
|
||||
g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress;
|
||||
g_vk.has_astc_ldr = h->textureCompressionASTC_LDR;
|
||||
{
|
||||
@@ -376,6 +453,7 @@ static void destroy_all(void) {
|
||||
gfx_window_release_vk();
|
||||
if (g_vk.device) vkDeviceWaitIdle(g_vk.device);
|
||||
if (g_vk.device) vk_astc_release();
|
||||
if (g_vk.device) vk_deswz_release();
|
||||
if (g_vk.device) vk_crumbs_release();
|
||||
if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL);
|
||||
vk_headroom_stop();
|
||||
@@ -397,7 +475,7 @@ int vk_init(void) {
|
||||
plat_mutex_lock(&g_lock);
|
||||
if (!g_ready) {
|
||||
memset(&g_vk, 0, sizeof g_vk);
|
||||
if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_headroom_start(), vk_dlaa_init();
|
||||
if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_upload_pools_init(), vk_headroom_start(), vk_dlaa_init();
|
||||
else destroy_all();
|
||||
}
|
||||
int ok = g_ready;
|
||||
@@ -437,6 +515,7 @@ void vk_device_fault_report(VkResult r) {
|
||||
done = 1;
|
||||
vk_crumbs_report();
|
||||
vk_pass_marks_report();
|
||||
vk_mem_fault_log();
|
||||
VkCtx *c = vk_ctx();
|
||||
PFN_vkGetDeviceFaultInfoEXT get =
|
||||
c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT")
|
||||
@@ -461,9 +540,11 @@ void vk_device_fault_report(VkResult r) {
|
||||
rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description,
|
||||
n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary);
|
||||
// The faulting access lies in the precision-aligned range around the reported address.
|
||||
for (uint32_t i = 0; i < n.addressInfoCount; i++)
|
||||
for (uint32_t i = 0; i < n.addressInfoCount; i++) {
|
||||
rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType),
|
||||
(unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision);
|
||||
ab_report(ai[i].reportedAddress, ai[i].addressPrecision);
|
||||
}
|
||||
for (uint32_t i = 0; i < n.vendorInfoCount; i++)
|
||||
rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description,
|
||||
(unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData);
|
||||
|
||||
@@ -183,6 +183,7 @@ static void memory_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAcce
|
||||
// be made (logged). Replay thread.
|
||||
static int bc_standin(const VkTexImage *img, uint32_t level, VkFormat fmt, BcStandIn *out) {
|
||||
static int warned;
|
||||
if (img->gone) return 0; // dropped since the lookup: a stand-in kept by its handle would outlive it
|
||||
plat_mutex_lock(&bcs_lock);
|
||||
int full = nbcs == BC_STANDINS;
|
||||
for (int i = 0; i < nbcs; i++)
|
||||
|
||||
+13
-5
@@ -593,6 +593,10 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v,
|
||||
if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c)
|
||||
VkImageView view;
|
||||
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE;
|
||||
if (img->gone) { // dropped since the lookup: its views were purged and a new one would outlive the image
|
||||
vk_defer_view(view);
|
||||
return view;
|
||||
}
|
||||
ViewNode *n = malloc(sizeof *n);
|
||||
if (!n) rt_fatal("vk: out of memory");
|
||||
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
|
||||
@@ -619,11 +623,15 @@ VkImageView vk_exec_target_view(const VkTexImage *img, VkFormat format, VkImageV
|
||||
if (blocks) ci.pNext = &use; // not the BC image's sampling and transfer usages
|
||||
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS)
|
||||
rt_fatal("vk: render target view (format %d) could not be created", (int)format);
|
||||
ViewNode *n = malloc(sizeof *n);
|
||||
if (!n) rt_fatal("vk: out of memory");
|
||||
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
|
||||
nvn_hm_put(&views_by_image, &img->image, n);
|
||||
nvn_hm_put(&view_of, &k, (void *)view);
|
||||
if (img->gone) { // dropped since resolve_target (get_view)
|
||||
vk_defer_view(view);
|
||||
} else {
|
||||
ViewNode *n = malloc(sizeof *n);
|
||||
if (!n) rt_fatal("vk: out of memory");
|
||||
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
|
||||
nvn_hm_put(&views_by_image, &img->image, n);
|
||||
nvn_hm_put(&view_of, &k, (void *)view);
|
||||
}
|
||||
}
|
||||
vk_texture_unlock();
|
||||
return view;
|
||||
|
||||
+1
-1
@@ -272,7 +272,7 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) {
|
||||
}
|
||||
if (ntex == 1 && slot >= 0 && slot < 128 && ex->tex[1][slot]) {
|
||||
const VkTexImage *s = vk_texture_by_id(NVN_HANDLE_TEX(ex->tex[1][slot]), NULL, NULL);
|
||||
if (s && is_float(s->format) && s->width == t->width && s->height == t->height && s->samples == 1)
|
||||
if (s && !s->gone && is_float(s->format) && s->width == t->width && s->height == t->height && s->samples == 1)
|
||||
src = s->image, fmt = s->format;
|
||||
}
|
||||
plat_mutex_lock(&rec_lock);
|
||||
|
||||
+132
-27
@@ -78,8 +78,9 @@ static int shadow_on; // shadow mode: imported pools get shad
|
||||
static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size);
|
||||
static void shadow_touch(PoolRec *r);
|
||||
static void shadow_forget(PoolRec *r);
|
||||
static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown)
|
||||
static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record
|
||||
static void pool_retiring(PoolRec *r); // vk_pool_fini: kept until pool_retire (or vk_mem_shutdown)
|
||||
static void pool_retire(void *arg); // completion thread: destroys a finalized pool's record
|
||||
static uint64_t retire_value(void); // the timeline value after which a finalized pool's record may go
|
||||
static void grow(void **arr, int *cap, size_t elem);
|
||||
|
||||
static PlatMutex pool_lock = PLAT_MUTEX_INIT;
|
||||
@@ -333,10 +334,14 @@ void vk_pool_fini(pool_t *p) {
|
||||
vk_queue_unlock();
|
||||
}
|
||||
vk_mem_collect();
|
||||
// A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may
|
||||
// name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued).
|
||||
if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r);
|
||||
else destroy_rec(c, r);
|
||||
// The record, its buffers and its memory outlive the submission being recorded, of any pool kind. The wait above
|
||||
// covers submitted work only. With the synchronous replay another game thread may be recording right now, and that
|
||||
// recording may already name this pool: a mirrored pool's sync copies and write-backs, an import pool's buffer
|
||||
// through a texture upload or a released cache entry (vk_texture.c slot_resolve), a physical pool's memory under
|
||||
// the images vk_texture_range_gone just deferred to that submission. The game frees a pool only once its own GPU
|
||||
// work is done, so only the port's work can name it. Freed here, the submission ran with freed memory.
|
||||
pool_retiring(r);
|
||||
vk_completion_add(retire_value(), pool_retire, r);
|
||||
}
|
||||
|
||||
// Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c); apply them in order.
|
||||
@@ -416,6 +421,10 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) {
|
||||
return resolve_locked(addr, size, out);
|
||||
}
|
||||
|
||||
int vk_mem_span_live(const VkMemSpan *sp) {
|
||||
return sp->buffer && sp->pool && !sp->alloc && !shadow_on && ((const PoolRec *)sp->pool)->kind == POOL_IMPORT;
|
||||
}
|
||||
|
||||
// The pool index lookup under the lock; a non-overlapped import or physical pool goes into the address's cache slot.
|
||||
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) {
|
||||
memset(out, 0, sizeof *out);
|
||||
@@ -507,6 +516,9 @@ static void dirty_frame(uint64_t frame);
|
||||
static void mirror_log(uint64_t frame);
|
||||
static void shadow_log(uint64_t frame);
|
||||
static int dead_pending(void);
|
||||
// Deferrals made off the recording thread while another thread recorded a submission (cmd_lock, defer), the newest
|
||||
// one's stamp: the case the stamp from the recording start covers. In the census line and the device fault report.
|
||||
static uint64_t n_cross_defer, cross_defer_tl;
|
||||
|
||||
static void vram_log(uint64_t frame);
|
||||
|
||||
@@ -542,10 +554,11 @@ void vk_mem_census(uint64_t frame) {
|
||||
}
|
||||
plat_mutex_unlock_shared(&pool_lock);
|
||||
rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%d without a GPU buffer), physical %llu pools %.1f MiB, "
|
||||
"virtual %llu pools %.1f MiB of address space",
|
||||
"virtual %llu pools %.1f MiB of address space; %llu objects deferred off the recording thread during a recording",
|
||||
(unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0,
|
||||
n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0,
|
||||
(unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0);
|
||||
(unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0,
|
||||
(unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED));
|
||||
static const char *const kinds[] = {"", "import", "physical", "virtual"};
|
||||
for (int j = 0; j < ng; j++) {
|
||||
char f[160];
|
||||
@@ -1390,19 +1403,19 @@ static void mirror_forget(PoolRec *r) {
|
||||
}
|
||||
}
|
||||
|
||||
// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran.
|
||||
// Finalized pools waiting for pool_retire; vk_mem_shutdown destroys those whose callback never ran.
|
||||
static PoolRec **retiring;
|
||||
static int nretiring, cretiring;
|
||||
static PlatMutex retire_lock = PLAT_MUTEX_INIT;
|
||||
|
||||
static void mirror_retiring(PoolRec *r) {
|
||||
static void pool_retiring(PoolRec *r) {
|
||||
plat_mutex_lock(&retire_lock);
|
||||
if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring);
|
||||
retiring[nretiring++] = r;
|
||||
plat_mutex_unlock(&retire_lock);
|
||||
}
|
||||
|
||||
static void mirror_retire(void *arg) {
|
||||
static void pool_retire(void *arg) {
|
||||
plat_mutex_lock(&retire_lock);
|
||||
for (int i = 0; i < nretiring; i++)
|
||||
if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; }
|
||||
@@ -2152,9 +2165,10 @@ typedef struct {
|
||||
uint64_t seq;
|
||||
Staging *stg;
|
||||
int nstg, cap;
|
||||
VkDeviceSize held; // staging and scratch bytes freed when the batch completes (vk_upload_held)
|
||||
} Batch;
|
||||
|
||||
enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA };
|
||||
enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST }; // DEAD_HOST: free(handle)
|
||||
typedef struct { int kind; uint64_t handle; VmaAllocation alloc; uint64_t seq, tl; } Dead;
|
||||
|
||||
static PlatRMutex cmd_lock;
|
||||
@@ -2169,7 +2183,8 @@ static int nspare, cap_spare;
|
||||
static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq)
|
||||
static Dead *dead;
|
||||
static int ndead, cap_dead;
|
||||
static uint64_t tl_noted; // newest timeline value vk_sync.c reported submitted
|
||||
static uint64_t tl_noted; // value of the submission being recorded, else of the last one (vk_mem_note_fence)
|
||||
static _Thread_local uint64_t tl_noted_here; // the value this thread noted: it records that submission
|
||||
|
||||
static void cmd_init(void *p) {
|
||||
(void)p;
|
||||
@@ -2212,6 +2227,10 @@ static void collect_locked(VkCtx *c) {
|
||||
for (int i = 0; i < ndead; i++) {
|
||||
Dead *d = &dead[i];
|
||||
if (d->seq > done_seq || d->tl > tl) { dead[k++] = *d; continue; }
|
||||
if (d->kind == DEAD_HOST) {
|
||||
free((void *)(uintptr_t)d->handle);
|
||||
continue;
|
||||
}
|
||||
rec_check_handle(d->kind == DEAD_VIEW ? REC_VIEW : d->kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, d->handle, 1,
|
||||
d->tl, tl);
|
||||
switch (d->kind) {
|
||||
@@ -2305,10 +2324,68 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
|
||||
else {
|
||||
if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg);
|
||||
open_b.stg[open_b.nstg++] = (Staging){b, a};
|
||||
open_b.held += bi.size;
|
||||
}
|
||||
return info.pMappedData;
|
||||
}
|
||||
|
||||
// Device-local scratch of the upload batch (the GPU deswizzle's linear copies, the ASTC decoder's blocks and texels):
|
||||
// a pool of its own, like staging, so the replay thread neither waits on the default pool's lock while another thread
|
||||
// makes a block there (the headroom thread: about 8 ms for 256 MiB) nor makes one itself for a few MiB. One block
|
||||
// of SCR_BLOCK is kept from the first use; larger buffers than half a block go to the default pool.
|
||||
#define SCR_BLOCK (64ull << 20)
|
||||
static VmaPool scr_pool;
|
||||
static int scr_pool_tried;
|
||||
|
||||
VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
|
||||
VkCtx *c = vk_ctx();
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16,
|
||||
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
|
||||
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
||||
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
||||
if (!__atomic_load_n(&scr_pool_tried, __ATOMIC_ACQUIRE)) {
|
||||
plat_once(&cmd_once, cmd_init, NULL);
|
||||
plat_rmutex_lock(&cmd_lock);
|
||||
if (!scr_pool_tried) {
|
||||
// two blocks with 8 GiB or more of device-local memory: fight2's first world frame deswizzles about
|
||||
// 280 MiB, more than the GPU finishes while the replay records, and the pool grew by four blocks there
|
||||
VkDeviceSize vram = 0;
|
||||
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
|
||||
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT && c->mem_props.memoryHeaps[h].size > vram)
|
||||
vram = c->mem_props.memoryHeaps[h].size;
|
||||
uint32_t type;
|
||||
VmaPoolCreateInfo pi = {.blockSize = SCR_BLOCK, .minBlockCount = vram >= (8ull << 30) ? 2 : 1};
|
||||
if (vmaFindMemoryTypeIndexForBufferInfo(c->vma, &bi, &aci, &type) == VK_SUCCESS) {
|
||||
pi.memoryTypeIndex = type;
|
||||
if (vmaCreatePool(c->vma, &pi, &scr_pool) != VK_SUCCESS) scr_pool = NULL;
|
||||
}
|
||||
rt_log("vk: upload scratch pool %s (%u blocks of %llu MiB kept)", scr_pool ? "created" : "unavailable: default pool",
|
||||
(unsigned)pi.minBlockCount, (unsigned long long)(SCR_BLOCK >> 20));
|
||||
__atomic_store_n(&scr_pool_tried, 1, __ATOMIC_RELEASE);
|
||||
}
|
||||
plat_rmutex_unlock(&cmd_lock);
|
||||
}
|
||||
if (scr_pool && bi.size <= SCR_BLOCK / 2) aci.pool = scr_pool;
|
||||
VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, buf, alloc, NULL);
|
||||
if (r == VK_SUCCESS && open_active) open_b.held += bi.size; // the caller defers its free to the open batch
|
||||
return r;
|
||||
}
|
||||
|
||||
VkDeviceSize vk_upload_held(void) { return open_active ? open_b.held : 0; }
|
||||
|
||||
// At device creation: the staging and scratch pools with their kept blocks, made through a first small buffer each
|
||||
// (freed at once), so the first world frame does not allocate them (staging was first used there once the GPU took
|
||||
// the early texture uploads).
|
||||
void vk_upload_pools_init(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!c) return;
|
||||
VkBuffer b;
|
||||
VmaAllocation a;
|
||||
vk_staging(16, &b, &a);
|
||||
vmaDestroyBuffer(c->vma, b, a);
|
||||
if (vk_scratch_buffer(16, &b, &a) == VK_SUCCESS) vmaDestroyBuffer(c->vma, b, a);
|
||||
}
|
||||
|
||||
uint64_t vk_upload_flush(int wait) {
|
||||
plat_once(&cmd_once, cmd_init, NULL);
|
||||
plat_rmutex_lock(&cmd_lock);
|
||||
@@ -2346,20 +2423,21 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) {
|
||||
plat_rmutex_lock(&cmd_lock);
|
||||
if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead);
|
||||
// The open batch may record into the object: it counts as the newest user. The GPU must also be past every
|
||||
// submission that names it. tl_noted covers the submitted ones; an object the replay lets go of while it records
|
||||
// a submission (vk_exec_recording_fence) may be named by that one too, not yet submitted: vk_texture.c
|
||||
// slot_resolve hands a draw a released entry, and entry_of destroys released entries at that address later in
|
||||
// the same submission. Stamped with the last submitted value, the object went as soon as the GPU reached it,
|
||||
// before the submission that names it ran, and with the recorder thread (vk_rec.c) behind, before the recorder
|
||||
// made the call that names it. LAS Recomp and Savage Wind Recomp stamp every deferral with the submission being
|
||||
// recorded (runtime/nvn/vulkan/textures.c defer: vk_pending_seq). TOTK_DEFER_STAMP=0 brings back the old stamp.
|
||||
static int stamp = -1;
|
||||
if (stamp < 0) stamp = rt_knob("DEFER_STAMP", "1") != 0;
|
||||
uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0;
|
||||
if (rf > tl) tl = rf;
|
||||
// submission that names it, including the one being recorded, which may name it and is not submitted yet:
|
||||
// vk_texture.c slot_resolve hands a draw a released entry, and entry_of, sweep_released and the VRAM pressure drop
|
||||
// destroy released entries. tl_noted is that submission's value from the start of its recording (vk_sync.c
|
||||
// vk_sync_submit), whichever thread destroys the object. With the synchronous replay a game thread finalizing a
|
||||
// texture (vk_backend.c vk_be_texture_release) destroys entries while another game thread records; stamped with
|
||||
// the last submitted value, as the recording thread's own fence was the only one seen, the object was freed once
|
||||
// the GPU reached that value, before the submission naming it ran: "write invalid" device faults on NVIDIA. LAS
|
||||
// Recomp and Savage Wind Recomp stamp every deferral with the submission being recorded (runtime/nvn/vulkan/
|
||||
// textures.c defer: vk_pending_seq).
|
||||
uint64_t tl = tl_noted;
|
||||
if (kind != DEAD_HOST && tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl;
|
||||
dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl};
|
||||
rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl,
|
||||
vk_sync_fence_done());
|
||||
if (kind != DEAD_HOST)
|
||||
rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl,
|
||||
vk_sync_fence_done());
|
||||
plat_rmutex_unlock(&cmd_lock);
|
||||
}
|
||||
|
||||
@@ -2378,14 +2456,38 @@ void vk_defer_view(VkImageView view) { if (view) defer(DEAD_VIEW, (uint64_t)(uin
|
||||
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) {
|
||||
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
|
||||
}
|
||||
void vk_defer_free(void *p) { if (p) defer(DEAD_HOST, (uint64_t)(uintptr_t)p, NULL); }
|
||||
|
||||
void vk_mem_note_submit(uint64_t timeline_value) {
|
||||
// The device fault report: no lock, since a thread that holds cmd_lock may be waiting for the report's own lock
|
||||
// (vk_upload_flush reports a failed submit with cmd_lock held).
|
||||
void vk_mem_fault_log(void) {
|
||||
rt_log("vk: deferred frees at the fault: %d pending, stamp %llu, submitted %llu, done %llu; %llu deferred off the "
|
||||
"recording thread during a recording (newest stamp %llu)",
|
||||
__atomic_load_n(&ndead, __ATOMIC_RELAXED), (unsigned long long)__atomic_load_n(&tl_noted, __ATOMIC_RELAXED),
|
||||
(unsigned long long)vk_sync_issued(), (unsigned long long)vk_sync_fence_done(),
|
||||
(unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED),
|
||||
(unsigned long long)__atomic_load_n(&cross_defer_tl, __ATOMIC_RELAXED));
|
||||
}
|
||||
|
||||
void vk_mem_note_fence(uint64_t timeline_value) {
|
||||
plat_once(&cmd_once, cmd_init, NULL);
|
||||
plat_rmutex_lock(&cmd_lock);
|
||||
if (timeline_value > tl_noted) tl_noted = timeline_value;
|
||||
tl_noted_here = timeline_value;
|
||||
plat_rmutex_unlock(&cmd_lock);
|
||||
}
|
||||
|
||||
// The later of the submission being recorded (tl_noted, noted from its start: vk_sync.c vk_sync_submit) and the next
|
||||
// one to be submitted.
|
||||
static uint64_t retire_value(void) {
|
||||
plat_once(&cmd_once, cmd_init, NULL);
|
||||
plat_rmutex_lock(&cmd_lock);
|
||||
uint64_t v = vk_sync_issued() + 1;
|
||||
if (tl_noted > v) v = tl_noted;
|
||||
plat_rmutex_unlock(&cmd_lock);
|
||||
return v;
|
||||
}
|
||||
|
||||
void vk_mem_collect(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!c) return;
|
||||
@@ -2434,6 +2536,9 @@ void vk_mem_shutdown(void) {
|
||||
if (stg_pool) vmaDestroyPool(c->vma, stg_pool); // after retire: no staging buffer is left in it
|
||||
stg_pool = NULL;
|
||||
stg_pool_tried = 0;
|
||||
if (scr_pool) vmaDestroyPool(c->vma, scr_pool); // deferred scratch buffers went with collect_locked above
|
||||
scr_pool = NULL;
|
||||
scr_pool_tried = 0;
|
||||
plat_rmutex_unlock(&cmd_lock);
|
||||
plat_mutex_lock(&pool_lock);
|
||||
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
|
||||
|
||||
+47
-10
@@ -101,6 +101,12 @@ static struct {
|
||||
VkPipeStats stats;
|
||||
} P = {.lock = PLAT_MUTEX_INIT};
|
||||
|
||||
// Builds into P.cache take it shared, vk_pipeline_save takes it to copy the cache out. Vulkan lets
|
||||
// vkGetPipelineCacheData run alongside builds into a cache created without the EXTERNALLY_SYNCHRONIZED bit, but
|
||||
// Intel's Windows driver (101.7085, Iris Xe) faults in it (igvk64.dll, reading 0x170) when the first autosave meets the
|
||||
// boot's pipeline builds (#681). Nothing is taken while it is held.
|
||||
static PlatMutex cache_use = PLAT_MUTEX_INIT;
|
||||
|
||||
typedef struct Layout {
|
||||
struct Layout *next;
|
||||
uint64_t hash;
|
||||
@@ -1440,7 +1446,9 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay
|
||||
if (k->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
||||
if (k->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
||||
uint64_t p0 = perf_begin();
|
||||
plat_mutex_lock_shared(&cache_use);
|
||||
VkResult r = vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe);
|
||||
plat_mutex_unlock_shared(&cache_use);
|
||||
perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it
|
||||
if (r != VK_SUCCESS) {
|
||||
rt_log("vk_pipeline: vkCreateGraphicsPipelines failed (%d)", (int)r);
|
||||
@@ -1591,7 +1599,9 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi
|
||||
ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil;
|
||||
pi.pColorBlendState = &cb, pi.pMultisampleState = &ms;
|
||||
}
|
||||
plat_mutex_lock_shared(&cache_use);
|
||||
if (vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
||||
plat_mutex_unlock_shared(&cache_use);
|
||||
out:
|
||||
if (m) vkDestroyShaderModule(P.device, m, NULL);
|
||||
free(patched);
|
||||
@@ -1803,7 +1813,8 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc
|
||||
// shaders (gfx_shader_get results) live for the process (gfx_shader_drop_memory is the shader probe's, which builds no
|
||||
// pipelines), layouts until vk_pipeline_shutdown, which drains the queue first. Workers call vkCreateGraphicsPipelines
|
||||
// without P.lock: P.cache is created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT, so the driver
|
||||
// synchronises it, vkGetPipelineCacheData in vk_pipeline_save included. Compute pipelines stay synchronous: a skipped
|
||||
// synchronises the builds among themselves. vk_pipeline_save still keeps them apart from vkGetPipelineCacheData (see
|
||||
// cache_use). Compute pipelines stay synchronous: a skipped
|
||||
// dispatch can change what the game computes.
|
||||
//
|
||||
// Two queues: draws' misses (A.head) and the boot prebuild (A.low, see "pipeline keys" below). Workers take the low
|
||||
@@ -2512,7 +2523,7 @@ static void prebuild_start(void) {
|
||||
plat_mutex_unlock(&P.lock);
|
||||
}
|
||||
|
||||
// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised).
|
||||
// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. Takes cache_use shared, like the graphics builds.
|
||||
static VkPipeline compute_create(const GfxShader *cs, const Layout *l) {
|
||||
uint64_t words;
|
||||
const uint32_t *code = gfx_shader_spirv(cs, &words);
|
||||
@@ -2525,7 +2536,9 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l) {
|
||||
.layout = l->layout};
|
||||
pin_subgroup32(&ci.stage, &rqc, code, words);
|
||||
VkPipeline pipe = VK_NULL_HANDLE;
|
||||
plat_mutex_lock_shared(&cache_use);
|
||||
if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
||||
plat_mutex_unlock_shared(&cache_use);
|
||||
vkDestroyShaderModule(P.device, m, NULL);
|
||||
return pipe;
|
||||
}
|
||||
@@ -3290,7 +3303,8 @@ void vk_pipeline_stats(VkPipeStats *s) {
|
||||
|
||||
static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache
|
||||
|
||||
void vk_pipeline_save(void) {
|
||||
// One cache save; wait: queue for cache_use (the exit saves) rather than try for it (the autosave).
|
||||
static void cache_save(int wait) {
|
||||
if (P.ready) keys_flush();
|
||||
if (P.ready && P.gpl) {
|
||||
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws "
|
||||
@@ -3317,23 +3331,46 @@ void vk_pipeline_save(void) {
|
||||
}
|
||||
gfx_shader_record_flush(1);
|
||||
if (!P.ready || !P.cache || !P.cache_path[0]) return;
|
||||
// Not under P.lock: the driver synchronises P.cache, and holding P.lock through the copy and the file write (about
|
||||
// 60 ms on the 1.4.3 fight) stopped every pipeline lookup of the replay, a freeze at each 20 s autosave.
|
||||
// Not under P.lock: holding it through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped every
|
||||
// pipeline lookup of the replay, a freeze at each 20 s autosave. Only the copy takes cache_use, so no pipeline is
|
||||
// being built into P.cache while the driver reads it.
|
||||
plat_mutex_lock(&save_lock);
|
||||
if (!P.cache) { // vk_pipeline_shutdown destroyed it
|
||||
plat_mutex_unlock(&save_lock);
|
||||
return;
|
||||
}
|
||||
uint64_t t0 = qpc();
|
||||
if (wait) {
|
||||
plat_mutex_lock(&cache_use);
|
||||
} else {
|
||||
// The autosave never queues for cache_use: a queued exclusive request on an SRW lock holds back new shared
|
||||
// ones, so the drawing thread's next build would wait for a worker's build to end. It tries for about 2 s
|
||||
// and otherwise saves on its next round.
|
||||
int got = 0;
|
||||
for (int i = 0; i < 1000 && !(got = plat_mutex_trylock(&cache_use)); i++) plat_sleep_ms(2);
|
||||
if (!got) {
|
||||
rt_log("vk_pipeline: cache save put off, pipelines were being built for %.0f ms", ms_since(t0));
|
||||
plat_mutex_unlock(&save_lock);
|
||||
return;
|
||||
}
|
||||
}
|
||||
uint64_t t1 = qpc();
|
||||
size_t size = 0;
|
||||
void *data = NULL;
|
||||
if (vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) &&
|
||||
vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS) {
|
||||
if (!plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path);
|
||||
}
|
||||
int ok = vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) &&
|
||||
vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS;
|
||||
plat_mutex_unlock(&cache_use);
|
||||
// The copy time is how long a pipeline build on the drawing thread (a compute miss, under P.lock) can wait for
|
||||
// this save; the wait is how long the save waited for builds in flight.
|
||||
rt_log("vk_pipeline: cache %s: %.1f MB copied in %.1f ms, after %.1f ms waiting for builds", wait ? "save" : "autosave",
|
||||
(double)size / 1e6, ms_since(t1), (double)(t1 - t0) / 1e6);
|
||||
if (ok && !plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path);
|
||||
free(data);
|
||||
plat_mutex_unlock(&save_lock);
|
||||
}
|
||||
|
||||
void vk_pipeline_save(void) { cache_save(1); }
|
||||
|
||||
// The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it
|
||||
// runs: at most every 20 s, and only after new pipelines were built. New pipeline keys are appended every 2 s.
|
||||
static void autosave_once(void) {
|
||||
@@ -3347,7 +3384,7 @@ static void autosave_once(void) {
|
||||
vk_pipeline_stats(&s);
|
||||
if (s.graphics + s.compute == last_n) return;
|
||||
last_n = s.graphics + s.compute, last_ms = now;
|
||||
vk_pipeline_save();
|
||||
cache_save(0);
|
||||
}
|
||||
|
||||
// On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to
|
||||
|
||||
+4
-3
@@ -309,7 +309,7 @@ void vk_sync_start(void) {
|
||||
if (reserved < now) reserved = now;
|
||||
stop = 0;
|
||||
plat_mutex_unlock(&st_lock);
|
||||
vk_mem_note_submit(now);
|
||||
vk_mem_note_fence(now);
|
||||
completion_thread = plat_thread_new(completion_main, c, "vk completion");
|
||||
if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
|
||||
__atomic_store_n(&started, 1, __ATOMIC_RELEASE);
|
||||
@@ -379,7 +379,6 @@ VkCommandBuffer vk_sync_split(void) {
|
||||
vk_device_fault_report(r);
|
||||
rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r);
|
||||
}
|
||||
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
|
||||
CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached
|
||||
rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence);
|
||||
ts_begin(s->cb);
|
||||
@@ -404,6 +403,9 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
|
||||
vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit
|
||||
uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1;
|
||||
cur_fence = fence;
|
||||
// Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c
|
||||
// defer), since the recording may already name it.
|
||||
vk_mem_note_fence(fence);
|
||||
CmdSlot *s = take_slot(c, fence);
|
||||
uint64_t t0 = perf_begin();
|
||||
rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence); // begun by the recorder when it records the part
|
||||
@@ -434,7 +436,6 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
|
||||
rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r);
|
||||
}
|
||||
s->fence = recorded ? fence : 0;
|
||||
vk_mem_note_submit(fence);
|
||||
plat_mutex_lock(&st_lock);
|
||||
__atomic_store_n(&issued, fence, __ATOMIC_RELEASE);
|
||||
plat_cond_broadcast(&cv_work);
|
||||
|
||||
+219
-5
@@ -597,10 +597,16 @@ static void destroy_tex(Tex *e) {
|
||||
if (p->next_at == e) { p->next_at = e->next_at; break; }
|
||||
}
|
||||
if (e->released) nreleased--, rel_unlink(e);
|
||||
// Another thread may hold &e->pub, taken under the lock and used after it (vk_texture_get, vk_texture_by_id: render
|
||||
// targets for a whole pass, vk_exec.c resolve_target, vk_texture_storage_bound_locked writes through it). Freed
|
||||
// here, that thread read a stale VkImage from freed memory, made views of it and cached them by that handle. The
|
||||
// record goes with the image, once the submission being recorded is done; gone keeps views of it out of the caches.
|
||||
e->pub.gone = 1;
|
||||
vk_exec_image_gone(e->pub.image);
|
||||
vk_defer_image(e->pub.image, e->alloc);
|
||||
free(e->snap);
|
||||
free(e);
|
||||
e->snap = NULL;
|
||||
vk_defer_free(e);
|
||||
}
|
||||
|
||||
// Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries the
|
||||
@@ -819,15 +825,20 @@ static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar,
|
||||
// large ASTC array decodes to RGBA8 in one go: a staging buffer of 676682336 bytes failed (-2) on an Intel UHD with
|
||||
// 8 GB of shared memory and the game closed before the title screen.
|
||||
#define UPLOAD_PART (32ull << 20) // fits the staging pool (vk_staging takes up to half of its 64 MiB block)
|
||||
// The open upload batch is submitted between chunks (and parts of the CPU path) once its staging and scratch pass this:
|
||||
// on fight2's first world frame one batch took hundreds of MiB, and the staging and scratch pools grew by 64 MiB blocks
|
||||
// on the replay thread. Submitting earlier keeps the queue order (the batch still precedes the part that samples the
|
||||
// images) and lets the GPU finish and return the buffers sooner.
|
||||
#define UPLOAD_HELD_MAX (48ull << 20)
|
||||
|
||||
// Copies the whole storage into the image (every level and layer the image has).
|
||||
static void upload(Tex *e) {
|
||||
static int upload_gpu(Tex *e);
|
||||
|
||||
// The CPU side of upload: deswizzles (and decodes or expands) into staging buffers, then copies them into the image.
|
||||
static void upload_cpu(Tex *e) {
|
||||
static volatile int32_t n_bad, n_undecoded, n_split;
|
||||
const tex_t *t = &e->t;
|
||||
const nvn_texdesc *d = &t->d;
|
||||
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
|
||||
if (!bpb || !t->mem) return;
|
||||
uint64_t p0 = perf_begin(), u0 = kern_now_ns();
|
||||
int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D;
|
||||
uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers;
|
||||
int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb;
|
||||
@@ -908,6 +919,7 @@ static void upload(Tex *e) {
|
||||
}
|
||||
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
|
||||
vk_upload_end();
|
||||
if (!acheck && vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0);
|
||||
}
|
||||
if (parts > 1)
|
||||
log_limited(&n_split, "texture %#llx fmt %#x %ux%u, %u levels, %u layers: %.1f MiB staged in %d parts",
|
||||
@@ -924,7 +936,201 @@ static void upload(Tex *e) {
|
||||
free(raw_copy);
|
||||
free(ar);
|
||||
free(rg);
|
||||
}
|
||||
|
||||
// Block-linear storage in an imported pool, deswizzled by the GPU (vk_deswizzle.c): the replay thread only records
|
||||
// the dispatches, the image copies and (ASTC) the decode into the upload batch the CPU path also uses, which runs before
|
||||
// the part that samples the image. The GPU reads the pool when the batch runs, as the Switch's GPU reads it at
|
||||
// execution; a CPU write after make_fresh's write-watch reset still marks the pages and brings another upload. Each
|
||||
// chunk of regions gets a device-local scratch buffer of at most DESWZ_CHUNK bytes (one region, when larger), freed
|
||||
// once the batch completes. TOTK_DESWIZZLE=check compares every chunk with the CPU deswizzle (waits for the GPU).
|
||||
// 0 when not handled, before anything is recorded: the CPU path takes the texture.
|
||||
#define DESWZ_CHUNK (32ull << 20)
|
||||
|
||||
// Uploads by path since the last perf summary (TOTK_TRACE=perf): 0 GPU, else why the CPU took it.
|
||||
enum { UP_GPU, UP_MODE, UP_LINEAR, UP_RGB, UP_ASTC_CPU, UP_BPB, UP_NOT_LIVE, UP_GEOM, UP_LIMITS, UP_SCRATCH, UP_WHY };
|
||||
static const char *const up_why_name[UP_WHY] = {"gpu", "cpu mode", "linear", "rgb", "astc on cpu", "bytes per block",
|
||||
"not an imported pool", "layout", "device limits", "scratch"};
|
||||
static uint64_t up_n[UP_WHY], up_bytes[UP_WHY], up_ns[UP_WHY];
|
||||
static int up_why; // upload_gpu's reason for returning 0 (texture lock held)
|
||||
|
||||
static int upload_gpu(Tex *e) {
|
||||
static volatile int32_t n_scratch_fail, n_undecoded, n_bad;
|
||||
static volatile int64_t n_checked, n_differ;
|
||||
const tex_t *t = &e->t;
|
||||
const nvn_texdesc *d = &t->d;
|
||||
int mode = vk_deswz_mode();
|
||||
up_why = mode == VK_DESWZ_CPU ? UP_MODE : nvn_tex_is_linear(d) ? UP_LINEAR : e->f.expand_rgb ? UP_RGB : UP_GPU;
|
||||
if (up_why) return 0;
|
||||
int astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu();
|
||||
if (astc && (!agpu || vk_astc_mode() == VK_ASTC_CHECK)) return up_why = UP_ASTC_CPU, 0; // or ASTC's check mode
|
||||
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
|
||||
if (!bpb || 16 % bpb) return up_why = UP_BPB, 0; // the rows of the linear copy are padded to 16 bytes
|
||||
VkMemSpan sp;
|
||||
if (!vk_mem_resolve(e->pub.addr, t->L.size, &sp) || !vk_mem_span_live(&sp)) return up_why = UP_NOT_LIVE, 0;
|
||||
int one_d = e->pub.type == VK_IMAGE_TYPE_1D, is3d = d->target == NVN_TEXTURE_TARGET_3D;
|
||||
uint32_t levels = e->pub.levels, layers = e->pub.layers, n = levels * layers, k = 0;
|
||||
VkDeswzRegion *dr = calloc(n, sizeof *dr);
|
||||
VkBufferImageCopy *rg = calloc(n, sizeof *rg);
|
||||
VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL;
|
||||
if (!dr || !rg || (agpu && !ar)) rt_fatal("vk: out of memory (texture upload regions)");
|
||||
int ok = 1;
|
||||
VkDeviceSize max_dst = DESWZ_CHUNK;
|
||||
for (uint32_t lv = 0; ok && lv < levels; lv++)
|
||||
for (uint32_t ly = 0; ok && ly < layers; ly++) {
|
||||
uint32_t row_bytes, rows, slices, gbpb;
|
||||
uint64_t base;
|
||||
unsigned th, td;
|
||||
uint32_t w = mip(d->width, lv), h = one_d ? 1 : mip(d->height, lv), sl = is3d ? mip(d->depth, lv) : 1;
|
||||
if (nvn_deswizzle_geom(d, lv, ly, &row_bytes, &rows, &slices, &base, &th, &td, &gbpb) || gbpb != (uint32_t)bpb ||
|
||||
base >= t->L.size || rows != (h + bh - 1) / bh || slices != sl) {
|
||||
ok = 0;
|
||||
break;
|
||||
}
|
||||
uint64_t bhg = 1ull << th, bdg = 1ull << td, gobs_x = (row_bytes + 63) / 64;
|
||||
uint64_t blocks_y = (rows + 8 * bhg - 1) / (8 * bhg), blocks_z = (slices + bdg - 1) / bdg;
|
||||
uint64_t segs = gobs_x * blocks_y * blocks_z * (32ull << (th + td)), room = (t->L.size - base) / 16;
|
||||
if (segs > room) segs = room;
|
||||
uint32_t pitch16 = (row_bytes + 15) / 16;
|
||||
VkDeviceSize out = (VkDeviceSize)pitch16 * 16 * rows * slices;
|
||||
if (segs > UINT32_MAX || gobs_x * blocks_y > UINT32_MAX) { ok = 0; break; }
|
||||
if (out > max_dst) max_dst = out;
|
||||
dr[k] = (VkDeswzRegion){.src = base, .segs = (uint32_t)segs, .pitch16 = pitch16, .rows = rows, .slices = slices,
|
||||
.th = th, .td = td, .gobs_x = (uint32_t)gobs_x, .blocks_y = (uint32_t)blocks_y};
|
||||
rg[k] = (VkBufferImageCopy){.bufferRowLength = pitch16 * 16 / (uint32_t)bpb * (uint32_t)bw,
|
||||
.bufferImageHeight = rows * (uint32_t)bh,
|
||||
.imageSubresource = {e->pub.aspect, lv, ly, 1}, .imageExtent = {w, h, sl}};
|
||||
if (agpu) ar[k] = (VkAstcRegion){.w = w, .h = h, .slices = sl, .level = lv, .layer = ly};
|
||||
k++;
|
||||
}
|
||||
if (!ok || k != n || !vk_deswz_fits(sp.offset, t->L.size, dr, (int)n, max_dst)) {
|
||||
free(dr), free(rg), free(ar);
|
||||
return up_why = !ok || k != n ? UP_GEOM : UP_LIMITS, 0;
|
||||
}
|
||||
VkCtx *c = vk_ctx();
|
||||
VkCommandBuffer cb = vk_upload_begin();
|
||||
// Earlier GPU writes into the pool (copies, shaders) before the reads; the image's earlier uses before its copy.
|
||||
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
||||
for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) {
|
||||
VkDeviceSize size = 0;
|
||||
for (k1 = k0; k1 < n; k1++) {
|
||||
VkDeviceSize out = (VkDeviceSize)dr[k1].pitch16 * 16 * dr[k1].rows * dr[k1].slices;
|
||||
if (k1 > k0 && size + out > DESWZ_CHUNK) break;
|
||||
dr[k1].dst = size, rg[k1].bufferOffset = size;
|
||||
if (agpu) ar[k1].src = size;
|
||||
size += out;
|
||||
}
|
||||
VkBuffer sb;
|
||||
VmaAllocation sa;
|
||||
if (vk_scratch_buffer(size, &sb, &sa) != VK_SUCCESS) {
|
||||
log_limited(&n_scratch_fail, "texture %#llx: deswizzle scratch of %llu bytes failed%s",
|
||||
(unsigned long long)e->pub.addr, (unsigned long long)size,
|
||||
k0 ? "; its later levels left as they were" : "; deswizzled on the CPU");
|
||||
if (!k0) {
|
||||
vk_upload_end();
|
||||
free(dr), free(rg), free(ar);
|
||||
return up_why = UP_SCRATCH, 0;
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (!vk_deswz_record(cb, sp.buffer, sp.offset, t->L.size, sb, size, dr + k0, (int)(k1 - k0))) {
|
||||
log_limited(&n_bad, "texture %#llx: deswizzle of levels %u-%u not recorded; left as they were",
|
||||
(unsigned long long)e->pub.addr, k0, k1 - 1); // vk_deswz_fits passed: not expected
|
||||
vk_defer_buffer(sb, sa);
|
||||
break;
|
||||
}
|
||||
barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
|
||||
VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
|
||||
if (agpu) {
|
||||
uint64_t a0 = perf_begin();
|
||||
if (!vk_astc_record(cb, sb, size, ar + k0, (int)(k1 - k0), e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb,
|
||||
e->f.astc_bc, e->pub.image, e->pub.aspect, VK_NULL_HANDLE))
|
||||
log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was",
|
||||
(unsigned long long)e->pub.addr, d->format, d->width, d->height);
|
||||
perf_end(PERF_ASTC, a0);
|
||||
} else {
|
||||
vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, k1 - k0, rg + k0);
|
||||
}
|
||||
if (mode == VK_DESWZ_CHECK) { // the linear copy back to the host, compared with the CPU deswizzle
|
||||
VkBuffer rb;
|
||||
VmaAllocation ra;
|
||||
const uint8_t *rmap = vk_staging(size, &rb, &ra);
|
||||
VkBufferCopy cp = {0, 0, size};
|
||||
vkCmdCopyBuffer(cb, sb, rb, 1, &cp);
|
||||
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
|
||||
VK_ACCESS_2_HOST_READ_BIT);
|
||||
vk_defer_buffer(sb, sa);
|
||||
vk_upload_end();
|
||||
vk_upload_flush(1);
|
||||
vmaInvalidateAllocation(c->vma, ra, 0, VK_WHOLE_SIZE);
|
||||
for (uint32_t i = k0; i < k1; i++) {
|
||||
const VkDeswzRegion *r = &dr[i];
|
||||
size_t pitch = (size_t)r->pitch16 * 16, bytes = pitch * r->rows * r->slices;
|
||||
uint8_t *cpu = malloc(bytes);
|
||||
if (!cpu) rt_fatal("vk: out of memory (deswizzle check)");
|
||||
uint32_t row_bytes = 0, rows, slices, gbpb;
|
||||
uint64_t base;
|
||||
unsigned th, td;
|
||||
nvn_deswizzle_geom(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, &row_bytes,
|
||||
&rows, &slices, &base, &th, &td, &gbpb);
|
||||
nvn_deswizzle(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, t->mem, cpu, pitch);
|
||||
uint64_t bad = 0, first = UINT64_MAX;
|
||||
for (uint64_t row = 0; row < (uint64_t)r->rows * r->slices; row++)
|
||||
if (memcmp(cpu + row * pitch, rmap + r->dst + row * pitch, row_bytes)) {
|
||||
bad++;
|
||||
if (first == UINT64_MAX) first = row;
|
||||
}
|
||||
free(cpu);
|
||||
plat_add64(&n_checked, 1);
|
||||
if (bad) {
|
||||
plat_add64(&n_differ, 1);
|
||||
rt_log("vk: deswizzle check: texture %#llx fmt %#x %ux%u level %u layer %u: %llu of %u rows differ, the "
|
||||
"first %llu (th %u td %u, %u rows of %u bytes, %u slices)", (unsigned long long)e->pub.addr,
|
||||
d->format, d->width, d->height, rg[i].imageSubresource.mipLevel,
|
||||
rg[i].imageSubresource.baseArrayLayer, (unsigned long long)bad, r->rows * r->slices,
|
||||
(unsigned long long)first, r->th, r->td, r->rows, row_bytes, r->slices);
|
||||
}
|
||||
}
|
||||
vmaDestroyBuffer(c->vma, rb, ra);
|
||||
if ((n_checked & 1023) == 0 || n_checked < 2)
|
||||
rt_log("vk: deswizzle check: %lld regions compared, %lld differ", (long long)n_checked, (long long)n_differ);
|
||||
cb = vk_upload_begin();
|
||||
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
||||
continue;
|
||||
}
|
||||
vk_defer_buffer(sb, sa);
|
||||
if (k1 < n && vk_upload_held() > UPLOAD_HELD_MAX) { // submit what is recorded, go on in a new batch
|
||||
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
|
||||
VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW);
|
||||
vk_upload_end();
|
||||
vk_upload_flush(0);
|
||||
cb = vk_upload_begin();
|
||||
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
|
||||
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
||||
}
|
||||
}
|
||||
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
|
||||
VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW);
|
||||
vk_upload_end();
|
||||
if (vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0);
|
||||
free(dr), free(rg), free(ar);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Copies the whole storage into the image (every level and layer the image has): on the GPU where it can
|
||||
// (upload_gpu), else on the CPU.
|
||||
static void upload(Tex *e) {
|
||||
const tex_t *t = &e->t;
|
||||
const nvn_texdesc *d = &t->d;
|
||||
int bw, bh;
|
||||
if (!nvn_fmt_bpb(d->format, &bw, &bh) || !t->mem) return;
|
||||
uint64_t p0 = perf_begin(), u0 = kern_now_ns();
|
||||
int why = UP_GPU;
|
||||
if (!upload_gpu(e)) why = up_why, upload_cpu(e);
|
||||
uint64_t dt = kern_now_ns() - u0;
|
||||
up_n[why]++, up_bytes[why] += t->L.size, up_ns[why] += dt;
|
||||
if (e->uploaded) {
|
||||
n_reuploads++, perf_count(PERF_CNT_TEX_REUP);
|
||||
ru_n++, ru_bytes += t->L.size, ru_ns += dt;
|
||||
@@ -1404,6 +1610,14 @@ void vk_texture_new_submit(void) {
|
||||
(unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6,
|
||||
(unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6);
|
||||
ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = 0;
|
||||
char ub[512];
|
||||
size_t uo = 0;
|
||||
for (int i = 0; i < UP_WHY; i++)
|
||||
if (up_n[i] && uo < sizeof ub)
|
||||
uo += (size_t)snprintf(ub + uo, sizeof ub - uo, "%s %s %llu (%.1f MiB, %.2f ms)", uo ? "," : "",
|
||||
up_why_name[i], (unsigned long long)up_n[i], up_bytes[i] / 1048576.0, up_ns[i] / 1e6);
|
||||
if (uo) rt_log("perf: frame %llu: texture uploads by path:%s", (unsigned long long)f * 60, ub);
|
||||
memset(up_n, 0, sizeof up_n), memset(up_bytes, 0, sizeof up_bytes), memset(up_ns, 0, sizeof up_ns);
|
||||
uint64_t ck, calls, untr;
|
||||
vk_mem_snap_stats(&ck, &calls, &untr);
|
||||
rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu",
|
||||
|
||||
Reference in new issue
Block a user