Merge dev into carry/dynamic-res-100

cfg.c: the key list takes dev's DESWIZZLE and keeps this branch's
DYNAMIC_RES and FIXED_RES_PATCH.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
itaskaev2andClaude Opus 5.5 committed 2026-11-07 18:21:41 +01:00
commit 91f6fc9838
14 files changed
+870 -67

No files matched your search

+111
View File
@@ -0,0 +1,111 @@
// Generated by tools/build_shaders.py from runtime/shaders/bl_deswizzle.comp; do not edit.
// source sha256 bd2065290e315e85d2fcdb609d812c15ba9a109d7506b3105850a5860677b0ef
#pragma once
#include <stdint.h>
static const uint32_t bl_deswizzle_spv[] = {
0x07230203, 0x00010300, 0x000d000b, 0x000000d0, 0x00000000, 0x00020011, 0x00000001, 0x0006000b,
0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001,
0x0007000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000b, 0x0000001f, 0x00060010,
0x00000004, 0x00000011, 0x00000040, 0x00000001, 0x00000001, 0x00040047, 0x0000000b, 0x0000000b,
0x0000001a, 0x00030047, 0x00000010, 0x00000002, 0x00050048, 0x00000010, 0x00000000, 0x00000023,
0x00000000, 0x00050048, 0x00000010, 0x00000001, 0x00000023, 0x00000004, 0x00050048, 0x00000010,
0x00000002, 0x00000023, 0x00000008, 0x00050048, 0x00000010, 0x00000003, 0x00000023, 0x0000000c,
0x00050048, 0x00000010, 0x00000004, 0x00000023, 0x00000010, 0x00050048, 0x00000010, 0x00000005,
0x00000023, 0x00000014, 0x00050048, 0x00000010, 0x00000006, 0x00000023, 0x00000018, 0x00050048,
0x00000010, 0x00000007, 0x00000023, 0x0000001c, 0x00050048, 0x00000010, 0x00000008, 0x00000023,
0x00000020, 0x00050048, 0x00000010, 0x00000009, 0x00000023, 0x00000024, 0x00050048, 0x00000010,
0x0000000a, 0x00000023, 0x00000028, 0x00040047, 0x0000001f, 0x0000000b, 0x0000001b, 0x00040047,
0x000000ad, 0x00000006, 0x00000010, 0x00030047, 0x000000ae, 0x00000002, 0x00040048, 0x000000ae,
0x00000000, 0x00000019, 0x00050048, 0x000000ae, 0x00000000, 0x00000023, 0x00000000, 0x00030047,
0x000000b0, 0x00000019, 0x00040047, 0x000000b0, 0x00000021, 0x00000001, 0x00040047, 0x000000b0,
0x00000022, 0x00000000, 0x00040047, 0x000000c1, 0x00000006, 0x00000010, 0x00030047, 0x000000c2,
0x00000002, 0x00040048, 0x000000c2, 0x00000000, 0x00000018, 0x00050048, 0x000000c2, 0x00000000,
0x00000023, 0x00000000, 0x00030047, 0x000000c4, 0x00000018, 0x00040047, 0x000000c4, 0x00000021,
0x00000000, 0x00040047, 0x000000c4, 0x00000022, 0x00000000, 0x00040047, 0x000000cd, 0x0000000b,
0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006,
0x00000020, 0x00000000, 0x00040017, 0x00000009, 0x00000006, 0x00000003, 0x00040020, 0x0000000a,
0x00000001, 0x00000009, 0x0004003b, 0x0000000a, 0x0000000b, 0x00000001, 0x0004002b, 0x00000006,
0x0000000c, 0x00000001, 0x00040020, 0x0000000d, 0x00000001, 0x00000006, 0x000d001e, 0x00000010,
0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006,
0x00000006, 0x00000006, 0x00000006, 0x00040020, 0x00000011, 0x00000009, 0x00000010, 0x0004003b,
0x00000011, 0x00000012, 0x00000009, 0x00040015, 0x00000013, 0x00000020, 0x00000001, 0x0004002b,
0x00000013, 0x00000014, 0x0000000a, 0x00040020, 0x00000015, 0x00000009, 0x00000006, 0x0004002b,
0x00000006, 0x00000019, 0x00000000, 0x0004002b, 0x00000006, 0x0000001d, 0x00000040, 0x0004003b,
0x0000000a, 0x0000001f, 0x00000001, 0x0004002b, 0x00000013, 0x00000024, 0x00000002, 0x00020014,
0x00000027, 0x0004002b, 0x00000013, 0x0000002d, 0x00000006, 0x0004002b, 0x00000013, 0x00000032,
0x00000007, 0x0004002b, 0x00000006, 0x00000037, 0x00000020, 0x0004002b, 0x00000006, 0x00000048,
0x00000005, 0x0004002b, 0x00000006, 0x0000004c, 0x0000001f, 0x0004002b, 0x00000013, 0x0000004f,
0x00000008, 0x0004002b, 0x00000013, 0x00000052, 0x00000009, 0x0004002b, 0x00000006, 0x0000006a,
0x00000004, 0x0004002b, 0x00000006, 0x0000006f, 0x00000002, 0x0004002b, 0x00000006, 0x00000078,
0x00000008, 0x0004002b, 0x00000006, 0x00000083, 0x00000003, 0x0004002b, 0x00000013, 0x00000093,
0x00000003, 0x0004002b, 0x00000013, 0x0000009b, 0x00000004, 0x0004002b, 0x00000013, 0x000000a4,
0x00000005, 0x00040017, 0x000000ac, 0x00000006, 0x00000004, 0x0003001d, 0x000000ad, 0x000000ac,
0x0003001e, 0x000000ae, 0x000000ad, 0x00040020, 0x000000af, 0x0000000c, 0x000000ae, 0x0004003b,
0x000000af, 0x000000b0, 0x0000000c, 0x0004002b, 0x00000013, 0x000000b1, 0x00000000, 0x0004002b,
0x00000013, 0x000000b2, 0x00000001, 0x0003001d, 0x000000c1, 0x000000ac, 0x0003001e, 0x000000c2,
0x000000c1, 0x00040020, 0x000000c3, 0x0000000c, 0x000000c2, 0x0004003b, 0x000000c3, 0x000000c4,
0x0000000c, 0x00040020, 0x000000c9, 0x0000000c, 0x000000ac, 0x0006002c, 0x00000009, 0x000000cd,
0x0000001d, 0x0000000c, 0x0000000c, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003,
0x000200f8, 0x00000005, 0x000300f7, 0x000000ce, 0x00000000, 0x000300fb, 0x00000019, 0x000000cf,
0x000200f8, 0x000000cf, 0x00050041, 0x0000000d, 0x0000000e, 0x0000000b, 0x0000000c, 0x0004003d,
0x00000006, 0x0000000f, 0x0000000e, 0x00050041, 0x00000015, 0x00000016, 0x00000012, 0x00000014,
0x0004003d, 0x00000006, 0x00000017, 0x00000016, 0x00050084, 0x00000006, 0x00000018, 0x0000000f,
0x00000017, 0x00050041, 0x0000000d, 0x0000001a, 0x0000000b, 0x00000019, 0x0004003d, 0x00000006,
0x0000001b, 0x0000001a, 0x00050080, 0x00000006, 0x0000001c, 0x00000018, 0x0000001b, 0x00050084,
0x00000006, 0x0000001e, 0x0000001c, 0x0000001d, 0x00050041, 0x0000000d, 0x00000020, 0x0000001f,
0x00000019, 0x0004003d, 0x00000006, 0x00000021, 0x00000020, 0x00050080, 0x00000006, 0x00000022,
0x0000001e, 0x00000021, 0x00050041, 0x00000015, 0x00000025, 0x00000012, 0x00000024, 0x0004003d,
0x00000006, 0x00000026, 0x00000025, 0x000500ae, 0x00000027, 0x00000028, 0x00000022, 0x00000026,
0x000300f7, 0x0000002a, 0x00000000, 0x000400fa, 0x00000028, 0x00000029, 0x0000002a, 0x000200f8,
0x00000029, 0x000200f9, 0x000000ce, 0x000200f8, 0x0000002a, 0x00050041, 0x00000015, 0x0000002e,
0x00000012, 0x0000002d, 0x0004003d, 0x00000006, 0x0000002f, 0x0000002e, 0x000500c4, 0x00000006,
0x00000030, 0x0000000c, 0x0000002f, 0x00050041, 0x00000015, 0x00000033, 0x00000012, 0x00000032,
0x0004003d, 0x00000006, 0x00000034, 0x00000033, 0x000500c4, 0x00000006, 0x00000035, 0x0000000c,
0x00000034, 0x00050080, 0x00000006, 0x0000003c, 0x0000002f, 0x00000034, 0x000500c4, 0x00000006,
0x0000003d, 0x00000037, 0x0000003c, 0x00050086, 0x00000006, 0x00000041, 0x00000022, 0x0000003d,
0x00050089, 0x00000006, 0x00000045, 0x00000022, 0x0000003d, 0x000500c2, 0x00000006, 0x00000049,
0x00000045, 0x00000048, 0x000500c7, 0x00000006, 0x0000004d, 0x00000045, 0x0000004c, 0x00050041,
0x00000015, 0x00000050, 0x00000012, 0x0000004f, 0x0004003d, 0x00000006, 0x00000051, 0x00000050,
0x00050041, 0x00000015, 0x00000053, 0x00000012, 0x00000052, 0x0004003d, 0x00000006, 0x00000054,
0x00000053, 0x00050084, 0x00000006, 0x00000055, 0x00000051, 0x00000054, 0x00050086, 0x00000006,
0x00000059, 0x00000041, 0x00000055, 0x00050089, 0x00000006, 0x0000005d, 0x00000041, 0x00000055,
0x00050086, 0x00000006, 0x00000062, 0x0000005d, 0x00000051, 0x00050089, 0x00000006, 0x00000067,
0x0000005d, 0x00000051, 0x00050084, 0x00000006, 0x0000006b, 0x00000067, 0x0000006a, 0x000500c2,
0x00000006, 0x0000006d, 0x0000004d, 0x0000006a, 0x000500c7, 0x00000006, 0x0000006e, 0x0000006d,
0x0000000c, 0x00050084, 0x00000006, 0x00000070, 0x0000006e, 0x0000006f, 0x00050080, 0x00000006,
0x00000071, 0x0000006b, 0x00000070, 0x000500c2, 0x00000006, 0x00000073, 0x0000004d, 0x0000000c,
0x000500c7, 0x00000006, 0x00000074, 0x00000073, 0x0000000c, 0x00050080, 0x00000006, 0x00000075,
0x00000071, 0x00000074, 0x00050084, 0x00000006, 0x00000079, 0x00000062, 0x00000078, 0x00050084,
0x00000006, 0x0000007b, 0x00000079, 0x00000030, 0x00050089, 0x00000006, 0x0000007e, 0x00000049,
0x00000030, 0x00050084, 0x00000006, 0x0000007f, 0x0000007e, 0x00000078, 0x00050080, 0x00000006,
0x00000080, 0x0000007b, 0x0000007f, 0x000500c2, 0x00000006, 0x00000082, 0x0000004d, 0x0000006f,
0x000500c7, 0x00000006, 0x00000084, 0x00000082, 0x00000083, 0x00050084, 0x00000006, 0x00000085,
0x00000084, 0x0000006f, 0x00050080, 0x00000006, 0x00000086, 0x00000080, 0x00000085, 0x000500c7,
0x00000006, 0x00000088, 0x00000045, 0x0000000c, 0x00050080, 0x00000006, 0x00000089, 0x00000086,
0x00000088, 0x00050084, 0x00000006, 0x0000008d, 0x00000059, 0x00000035, 0x00050086, 0x00000006,
0x00000090, 0x00000049, 0x00000030, 0x00050080, 0x00000006, 0x00000091, 0x0000008d, 0x00000090,
0x00050041, 0x00000015, 0x00000094, 0x00000012, 0x00000093, 0x0004003d, 0x00000006, 0x00000095,
0x00000094, 0x000500ae, 0x00000027, 0x00000096, 0x00000075, 0x00000095, 0x000400a8, 0x00000027,
0x00000097, 0x00000096, 0x000300f7, 0x00000099, 0x00000000, 0x000400fa, 0x00000097, 0x00000098,
0x00000099, 0x000200f8, 0x00000098, 0x00050041, 0x00000015, 0x0000009c, 0x00000012, 0x0000009b,
0x0004003d, 0x00000006, 0x0000009d, 0x0000009c, 0x000500ae, 0x00000027, 0x0000009e, 0x00000089,
0x0000009d, 0x000200f9, 0x00000099, 0x000200f8, 0x00000099, 0x000700f5, 0x00000027, 0x0000009f,
0x00000096, 0x0000002a, 0x0000009e, 0x00000098, 0x000400a8, 0x00000027, 0x000000a0, 0x0000009f,
0x000300f7, 0x000000a2, 0x00000000, 0x000400fa, 0x000000a0, 0x000000a1, 0x000000a2, 0x000200f8,
0x000000a1, 0x00050041, 0x00000015, 0x000000a5, 0x00000012, 0x000000a4, 0x0004003d, 0x00000006,
0x000000a6, 0x000000a5, 0x000500ae, 0x00000027, 0x000000a7, 0x00000091, 0x000000a6, 0x000200f9,
0x000000a2, 0x000200f8, 0x000000a2, 0x000700f5, 0x00000027, 0x000000a8, 0x0000009f, 0x00000099,
0x000000a7, 0x000000a1, 0x000300f7, 0x000000aa, 0x00000000, 0x000400fa, 0x000000a8, 0x000000a9,
0x000000aa, 0x000200f8, 0x000000a9, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000aa, 0x00050041,
0x00000015, 0x000000b3, 0x00000012, 0x000000b2, 0x0004003d, 0x00000006, 0x000000b4, 0x000000b3,
0x00050041, 0x00000015, 0x000000b6, 0x00000012, 0x0000009b, 0x0004003d, 0x00000006, 0x000000b7,
0x000000b6, 0x00050084, 0x00000006, 0x000000b8, 0x00000091, 0x000000b7, 0x00050080, 0x00000006,
0x000000ba, 0x000000b8, 0x00000089, 0x00050084, 0x00000006, 0x000000bd, 0x000000ba, 0x00000095,
0x00050080, 0x00000006, 0x000000be, 0x000000b4, 0x000000bd, 0x00050080, 0x00000006, 0x000000c0,
0x000000be, 0x00000075, 0x00050041, 0x00000015, 0x000000c5, 0x00000012, 0x000000b1, 0x0004003d,
0x00000006, 0x000000c6, 0x000000c5, 0x00050080, 0x00000006, 0x000000c8, 0x000000c6, 0x00000022,
0x00060041, 0x000000c9, 0x000000ca, 0x000000c4, 0x000000b1, 0x000000c8, 0x0004003d, 0x000000ac,
0x000000cb, 0x000000ca, 0x00060041, 0x000000c9, 0x000000cc, 0x000000b0, 0x000000b1, 0x000000c0,
0x0003003e, 0x000000cc, 0x000000cb, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000ce, 0x000100fd,
0x00010038,
};
+1 -1
View File
@@ -40,7 +40,7 @@ static const char *const known[] = {
"BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y",
"BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA",
"COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT",
"DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
"DESC_TEMPLATE", "DESWIZZLE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
"DUMP_LOG", "DYNAMIC_RES", "DYN_DELTA", "EXEFS", "FEEDBACK", "FIXED_RES_PATCH", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES",
"FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FRAME_QUEUE",
"FREEZE_IDLE", "FREE_CORES",
+37
View File
@@ -0,0 +1,37 @@
#version 450
// Block-linear to linear copy of one texture level and layer on the GPU (vk_deswizzle.c), the same layout as
// tex_deswizzle.c nvn_bl_offset: GOB = 64 bytes x 8 rows; a block is 1 GOB wide, 2^th GOBs high and 2^td deep, its
// GOBs ordered y first, then z; blocks fill the level x first, then y, then z.
// One invocation per 16-byte segment of the block-linear storage, in storage order, so the reads (from the pool's
// imported buffer: guest memory, over PCIe on a discrete GPU) are contiguous; the writes go to a device-local buffer
// with rows of pitch16 segments. Segments outside the level's rows, slices or row width are padding and skipped.
layout(local_size_x = 64) in;
layout(std430, set = 0, binding = 0) readonly buffer Src { uvec4 src[]; };
layout(std430, set = 0, binding = 1) writeonly buffer Dst { uvec4 dst[]; };
layout(push_constant) uniform Push {
uint src_base; // first segment of the level and layer in the src binding
uint dst_base; // first segment of its linear copy in the dst binding
uint segs; // segments to visit: whole blocks covering the level
uint pitch16; // segments per linear row (row bytes rounded up to 16)
uint rows, slices, th, td;
uint gobs_x, blocks_y;
uint nx; // workgroups along x (the dispatch is 2D when one dimension is not enough)
} pc;
void main() {
uint s = (gl_WorkGroupID.y * pc.nx + gl_WorkGroupID.x) * 64u + gl_LocalInvocationID.x;
if (s >= pc.segs) return;
uint bh = 1u << pc.th, bd = 1u << pc.td;
uint per_block = 32u << (pc.th + pc.td); // 512-byte GOBs of 32 segments each
uint block = s / per_block, inb = s % per_block;
uint gob = inb >> 5u, ing = inb & 31u;
uint per_layer = pc.gobs_x * pc.blocks_y;
uint bz = block / per_layer, rem = block % per_layer;
uint by = rem / pc.gobs_x, bx = rem % pc.gobs_x;
// Inside a GOB, segment bits: 4 = x % 64 / 32, 3..2 = y % 8 / 2, 1 = x % 32 / 16, 0 = y % 2.
uint xseg = bx * 4u + ((ing >> 4u) & 1u) * 2u + ((ing >> 1u) & 1u);
uint y = by * 8u * bh + (gob % bh) * 8u + ((ing >> 2u) & 3u) * 2u + (ing & 1u);
uint z = bz * bd + gob / bh;
if (xseg >= pc.pitch16 || y >= pc.rows || z >= pc.slices) return;
dst[pc.dst_base + (z * pc.rows + y) * pc.pitch16 + xseg] = src[pc.src_base + s];
}
+37 -1
View File
@@ -329,6 +329,15 @@ void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestro
void vk_defer_view(VkImageView view);
void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc);
// free(p) once the GPU is past the submission being recorded, like the deferrals above: host records that a recording
// may still read through a pointer taken before another thread dropped them (vk_texture.c destroy_tex).
void vk_defer_free(void *p);
// Device-local scratch for the upload batch (storage, transfer src and dst), from a pool of its own (vk_memory.c).
VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc);
// Staging and scratch bytes the open upload batch holds until it completes. Large texture loads submit the batch early
// (vk_texture.c UPLOAD_HELD_MAX) so the GPU starts on it and its buffers come back before the pools need new blocks.
VkDeviceSize vk_upload_held(void);
void vk_upload_pools_init(void); // vk_device.c, after the allocator: the staging and scratch pools and their blocks
// ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c.
enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK };
int vk_astc_mode(void); // decided once per device (logged); VK_ASTC_CPU before vk_init
@@ -352,7 +361,31 @@ void vk_astc_time_enable(int on);
int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us); // totals so far
void vk_astc_time_summary(void); // logs the totals
void vk_astc_release(void); // vk_device.c, before the device goes: frees the decoder's objects
void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value
// ---- Block-linear deswizzle on the GPU (vk_deswizzle.c). TOTK_DESWIZZLE = auto | gpu | cpu | check.
enum { VK_DESWZ_CPU, VK_DESWZ_GPU, VK_DESWZ_CHECK };
int vk_deswz_mode(void); // decided once per device (logged); VK_DESWZ_CPU before vk_init
// One level and layer: its storage starts at byte src of the bound source range and covers segs 16-byte segments
// (whole blocks, cut at the storage end); its linear copy goes to byte dst of the destination, rows of pitch16 segments,
// rows x slices of them. th, td, gobs_x, blocks_y: as tex_deswizzle.c nvn_bl_offset.
typedef struct {
VkDeviceSize src, dst;
uint32_t segs, pitch16, rows, slices, th, td, gobs_x, blocks_y;
} VkDeswzRegion;
// Records into cb a dispatch per region reading [src_off, src_off + src_size) of src (a storage buffer) and writing
// dst (device-local, dst_size bytes). The caller records the barriers around it. 0 when nothing was recorded: no
// pipeline, src_off not 16-byte aligned, a range over the device's storage-buffer limit, a region outside the ranges.
int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst,
VkDeviceSize dst_size, const VkDeswzRegion *rg, int n);
// 1 when vk_deswz_record will take these regions (their dst offsets aside) with destinations of up to max_dst bytes.
int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst);
void vk_deswz_release(void); // vk_device.c, before the device goes
// 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission
// sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch).
int vk_mem_span_live(const VkMemSpan *sp);
// vk_sync.c, when it starts recording the submission that will signal ctx->timeline with this value: objects destroyed
// from then on are freed once the timeline reaches it.
void vk_mem_note_fence(uint64_t timeline_value);
void vk_mem_fault_log(void); // vk_device_fault_report: the deferred-free state, lock-free
void vk_mem_collect(void); // frees what has completed
typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats;
void vk_mem_stats(VkMemStats *s);
@@ -384,6 +417,9 @@ typedef struct {
uint8_t scale; // 1, or TOTK_SHADOW_SCALE for a shadow map made larger than the game's (vk_texture.c
// shadow_scale_of): width and height above are the larger ones, the game's regions
// are multiplied by it
uint8_t gone; // the entry was dropped (vk_texture.c destroy_tex, under the texture lock): image and
// record live until the submission being recorded is done, and nothing may be cached
// by its image handle, which the driver can hand out again after that
} VkTexImage;
// The window texture the backend presents (vk_backend.c): *image and *format become the float picture the game copied
// into it, when the last draw into it was that copy and the swapchain is HDR10.
+1 -5
View File
@@ -463,13 +463,9 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons
cap_in = (cap_in + al - 1) / al * al;
cap_out = (cap_out + al - 1) / al * al;
if (!bc) cap_bc = 0;
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = cap_in + cap_out + cap_bc,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
VkBuffer sb;
VmaAllocation sa;
if (vmaCreateBuffer(c->vma, &bi, &aci, &sb, &sa, NULL) != VK_SUCCESS) {
if (vk_scratch_buffer(cap_in + cap_out + cap_bc, &sb, &sa) != VK_SUCCESS) {
if (plat_inc32(&n_fail) <= 16)
rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded",
(unsigned long long)(cap_in + cap_out));
+176
View File
@@ -0,0 +1,176 @@
// Block-linear textures deswizzled by the GPU (runtime/shaders/bl_deswizzle.comp): vk_texture.c upload binds the
// texture's storage in its pool's imported buffer and lets a compute pass write the linear copy that a buffer-to-image
// copy (or the ASTC decoder) then takes, instead of tex_deswizzle.c walking every 16 bytes on the replay thread
// (50-236 ms frames on entering new areas, 2026-11-07 play sessions). The same design as MacNativeGames' TotK port
// (d2r_gfx nvn_mtl.m: deswizzle from the pool at GPU execution time), written for Vulkan from our CPU layout code.
//
// TOTK_DESWIZZLE = auto (default) / gpu: the compute pass where the storage is an imported pool (vk_mem_span_live),
// the CPU otherwise; cpu: always the CPU; check: gpu, and every upload is also deswizzled on the CPU and compared
// (waits for the GPU; for testing only).
#include <stdlib.h>
#include <string.h>
#include "bl_deswizzle_spv.h"
#include "cfg.h"
#include "host.h"
#include "vk.h"
#include "vk_rec.h"
#include "platform/platform.h"
typedef struct {
uint32_t src_base, dst_base, segs, pitch16, rows, slices, th, td, gobs_x, blocks_y, nx;
} Push;
static PlatMutex g_lock = PLAT_MUTEX_INIT;
static struct {
VkDevice dev;
int mode; // VK_DESWZ_*; -1 before the first vk_deswz_mode with a device
VkDescriptorSetLayout dsl;
VkPipelineLayout layout;
VkPipeline pipe;
PFN_vkCmdPushDescriptorSetKHR push;
} G = {.mode = -1};
static volatile int64_t n_regions, n_bytes;
static int create_pipeline(VkCtx *c) {
G.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR");
if (!G.push) return 0;
VkDescriptorSetLayoutBinding b[2];
for (int i = 0; i < 2; i++)
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
.bindingCount = 2, .pBindings = b};
if (vkCreateDescriptorSetLayout(c->device, &dci, NULL, &G.dsl) != VK_SUCCESS) return 0;
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push)};
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
.pSetLayouts = &G.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
if (vkCreatePipelineLayout(c->device, &li, NULL, &G.layout) != VK_SUCCESS) return 0;
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bl_deswizzle_spv,
.pCode = bl_deswizzle_spv};
VkShaderModule m = VK_NULL_HANDLE;
if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0;
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
.layout = G.layout};
VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.pipe);
vkDestroyShaderModule(c->device, m, NULL);
if (r != VK_SUCCESS) G.pipe = VK_NULL_HANDLE;
return r == VK_SUCCESS;
}
int vk_deswz_mode(void) {
VkCtx *c = vk_ctx();
if (!c) return VK_DESWZ_CPU;
plat_mutex_lock_shared(&g_lock);
int m = G.dev == c->device ? G.mode : -1;
plat_mutex_unlock_shared(&g_lock);
if (m >= 0) return m;
plat_mutex_lock(&g_lock);
if (G.dev != c->device) { // the old device's objects went with it (probes recreate the device)
memset(&G, 0, sizeof G);
G.dev = c->device;
G.mode = -1;
}
if (G.mode < 0) {
const char *e = cfg_get("TOTK_DESWIZZLE");
if (!e || !*e) e = "auto";
int want = !strcmp(e, "cpu") || !strcmp(e, "0") ? VK_DESWZ_CPU : !strcmp(e, "check") ? VK_DESWZ_CHECK : VK_DESWZ_GPU;
if (strcmp(e, "auto") && strcmp(e, "gpu") && strcmp(e, "1") && strcmp(e, "cpu") && strcmp(e, "0") &&
strcmp(e, "check"))
rt_log("vk: TOTK_DESWIZZLE=%s is not auto, gpu, cpu or check: auto", e);
if (want != VK_DESWZ_CPU && !create_pipeline(c)) {
rt_log("vk: the deswizzle compute pass is unavailable on %s: textures deswizzle on the CPU", c->props.deviceName);
want = VK_DESWZ_CPU;
}
G.mode = want;
rt_log("vk: block-linear textures deswizzle on the %s%s", want == VK_DESWZ_CPU ? "CPU" : "GPU where the storage is "
"an imported pool", want == VK_DESWZ_CHECK ? ", checked against the CPU (TOTK_DESWIZZLE=check)" : "");
}
m = G.mode;
plat_mutex_unlock(&g_lock);
return m;
}
static uint32_t max_groups_x(const VkCtx *c) {
uint32_t m = c->props.limits.maxComputeWorkGroupCount[0];
return m > 65535 ? 65535 : m;
}
// The bound source range starts at the alignment below src_off.
static VkDeviceSize src_lo(const VkCtx *c, VkDeviceSize src_off) {
VkDeviceSize al = c->props.limits.minStorageBufferOffsetAlignment;
if (al < 16) al = 16;
return src_off / al * al;
}
int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst) {
VkCtx *c = vk_ctx();
if (!c || !n || (src_off & 15) || vk_deswz_mode() == VK_DESWZ_CPU) return 0;
VkDeviceSize range = src_off + src_size - src_lo(c, src_off), lim = c->props.limits.maxStorageBufferRange;
if (range > lim || max_dst > lim) return 0;
uint32_t maxg = max_groups_x(c);
for (int i = 0; i < n; i++) { // the 32-bit segment indices of the shader stay inside the bindings
uint64_t groups = ((uint64_t)rg[i].segs + 63) / 64, out = (uint64_t)rg[i].pitch16 * rg[i].rows * rg[i].slices;
if ((rg[i].src & 15) || rg[i].src + (uint64_t)rg[i].segs * 16 > src_size || !rg[i].segs ||
(src_off - src_lo(c, src_off) + rg[i].src) / 16 + rg[i].segs > UINT32_MAX || out > UINT32_MAX ||
out * 16 > max_dst || (groups + maxg - 1) / maxg > c->props.limits.maxComputeWorkGroupCount[1])
return 0;
}
return 1;
}
int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst,
VkDeviceSize dst_size, const VkDeswzRegion *rg, int n) {
VkCtx *c = vk_ctx();
if (!vk_deswz_fits(src_off, src_size, rg, n, dst_size)) return 0;
for (int i = 0; i < n; i++)
if ((rg[i].dst & 15) || rg[i].dst + (uint64_t)rg[i].pitch16 * 16 * rg[i].rows * rg[i].slices > dst_size) return 0;
plat_mutex_lock_shared(&g_lock);
VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE;
plat_mutex_unlock_shared(&g_lock);
if (!pipe) return 0;
VkDeviceSize lo = src_lo(c, src_off), range = src_off + src_size - lo;
uint32_t maxg = max_groups_x(c);
VkDescriptorBufferInfo dbi[2] = {{src, lo, range}, {dst, 0, dst_size}};
VkWriteDescriptorSet w[2];
for (int i = 0; i < 2; i++)
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &dbi[i]};
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe);
rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 2, w);
uint64_t bytes = 0;
for (int i = 0; i < n; i++) {
const VkDeswzRegion *r = &rg[i];
uint64_t groups = ((uint64_t)r->segs + 63) / 64;
uint32_t nx = groups < maxg ? (uint32_t)groups : maxg, ny = (uint32_t)((groups + nx - 1) / nx);
Push pc = {.src_base = (uint32_t)((src_off - lo + r->src) / 16), .dst_base = (uint32_t)(r->dst / 16),
.segs = r->segs, .pitch16 = r->pitch16, .rows = r->rows, .slices = r->slices, .th = r->th, .td = r->td,
.gobs_x = r->gobs_x, .blocks_y = r->blocks_y, .nx = nx};
vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
vkCmdDispatch(cb, nx, ny, 1);
bytes += (uint64_t)r->segs * 16;
}
int64_t before = plat_add64(&n_regions, n) - n;
plat_add64(&n_bytes, (int64_t)bytes);
if (!before || (before + n) / 4096 != before / 4096)
rt_log("vk: deswizzle: %lld regions, %.1f MiB of block-linear storage deswizzled on the GPU so far",
(long long)(before + n), (double)n_bytes / 1048576.0);
return 1;
}
void vk_deswz_release(void) {
VkCtx *c = vk_ctx();
plat_mutex_lock(&g_lock);
if (c && G.dev == c->device) {
if (G.pipe) vkDestroyPipeline(c->device, G.pipe, NULL);
if (G.layout) vkDestroyPipelineLayout(c->device, G.layout, NULL);
if (G.dsl) vkDestroyDescriptorSetLayout(c->device, G.dsl, NULL);
}
memset(&G, 0, sizeof G);
G.mode = -1;
plat_mutex_unlock(&g_lock);
}
+90 -9
View File
@@ -16,9 +16,69 @@ static VkCtx g_vk;
static int g_ready;
static PlatMutex g_lock = PLAT_MUTEX_INIT;
// ---- GPU address map for device faults
// VK_EXT_device_address_binding_report: the driver reports each bind and unbind of an object's GPU virtual range through
// the debug messenger. The newest AB_N events stay in a ring written without a lock (one atomic add each), so a device
// fault names the object behind its address and whether it was unbound before the fault, which is memory freed while
// a submission still used it. One event per image, buffer or memory block made or freed, none per draw.
// TOTK_ADDR_REPORT=0 turns it off.
#define AB_N (1u << 15)
typedef struct { uint64_t seq, base, size, handle, issued, done; int32_t type; uint32_t unbind; } AbEvent;
static AbEvent *ab_ring;
static uint64_t ab_head;
static int ab_want; // the instance has the messenger for it (create_instance)
static void ab_record(const VkDebugUtilsMessengerCallbackDataEXT *d) {
const VkDeviceAddressBindingCallbackDataEXT *b = NULL;
for (const VkBaseInStructure *p = d->pNext; p; p = p->pNext)
if (p->sType == VK_STRUCTURE_TYPE_DEVICE_ADDRESS_BINDING_CALLBACK_DATA_EXT) b = (const void *)p;
if (!b || !ab_ring) return;
uint64_t i = __atomic_fetch_add(&ab_head, 1, __ATOMIC_RELAXED);
AbEvent *e = &ab_ring[i & (AB_N - 1)];
__atomic_store_n(&e->seq, 0, __ATOMIC_RELAXED);
e->base = b->baseAddress, e->size = b->size;
e->handle = d->objectCount ? d->pObjects[0].objectHandle : 0;
e->type = d->objectCount ? (int32_t)d->pObjects[0].objectType : 0;
e->unbind = b->bindingType == VK_DEVICE_ADDRESS_BINDING_TYPE_UNBIND_EXT;
e->issued = vk_sync_issued(), e->done = vk_sync_fence_done();
__atomic_store_n(&e->seq, i + 1, __ATOMIC_RELEASE);
}
static const char *ab_type_name(int32_t t, char *buf, size_t n) {
switch (t) {
case VK_OBJECT_TYPE_IMAGE: return "image";
case VK_OBJECT_TYPE_BUFFER: return "buffer";
case VK_OBJECT_TYPE_DEVICE_MEMORY: return "memory";
default: snprintf(buf, n, "object type %d", (int)t); return buf;
}
}
// The newest events whose range meets the precision-aligned range around a fault address, newest first.
static void ab_report(uint64_t addr, uint64_t precision) {
if (!ab_ring) return;
uint64_t lo = precision > 1 ? addr & ~(precision - 1) : addr, hi = lo + (precision > 1 ? precision : 1);
uint64_t head = __atomic_load_n(&ab_head, __ATOMIC_ACQUIRE), n = head < AB_N ? head : AB_N;
int found = 0;
for (uint64_t k = 1; k <= n && found < 6; k++) {
const AbEvent *e = &ab_ring[(head - k) & (AB_N - 1)];
if (__atomic_load_n(&e->seq, __ATOMIC_ACQUIRE) != head - k + 1) continue;
if (e->base >= hi || e->base + e->size <= lo) continue;
found++;
char tn[32];
rt_log("vk: fault address %#llx: %s %#llx [%#llx, +%#llx] %s %llu address events before the fault, "
"submitted %llu, done %llu", (unsigned long long)addr, ab_type_name(e->type, tn, sizeof tn),(unsigned long long)e->handle,
(unsigned long long)e->base, (unsigned long long)e->size, e->unbind ? "unbound" : "bound",
(unsigned long long)k, (unsigned long long)e->issued, (unsigned long long)e->done);
}
if (!found) rt_log("vk: fault address %#llx: no binding in the last %llu address events", (unsigned long long)addr,
(unsigned long long)n);
}
static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type,
const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) {
(void)type, (void)user;
(void)user;
if (type & VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT) { ab_record(d); return VK_FALSE; }
if (sev < VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) return VK_FALSE; // info, asked for the address events
int err = (sev & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) != 0;
if (err) plat_inc32(&g_vk.validation_errors);
static volatile int32_t logged;
@@ -76,7 +136,10 @@ static int create_instance(void) {
rt_log("vk: TOTK_VK_VALIDATION set but %s is not installed; continuing without", layers[0]);
validate = 0;
}
if (validate) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
const char *ar = cfg_get("TOTK_ADDR_REPORT");
ab_want = !(ar && !strcmp(ar, "0")) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
if (ab_want && !ab_ring && !(ab_ring = calloc(AB_N, sizeof *ab_ring))) ab_want = 0;
if (validate || ab_want) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
if (vk_hdr_wanted() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats
ext[n_ext++] = VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME;
vk_dlaa_instance_exts(ext, &n_ext, 12);
@@ -89,16 +152,18 @@ static int create_instance(void) {
VkResult r = vkCreateInstance(&ci, NULL, &g_vk.instance);
if (r != VK_SUCCESS) { rt_log("vk: vkCreateInstance failed (%d); is a Vulkan 1.3 driver installed?", (int)r); return 0; }
g_vk.validation = validate;
if (validate) {
if (validate || ab_want) {
PFN_vkCreateDebugUtilsMessengerEXT f =
(PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkCreateDebugUtilsMessengerEXT");
VkDebugUtilsMessengerCreateInfoEXT mi = {
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
.messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
.messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
.messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT |
(ab_want ? VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT : 0),
.messageType = (validate ? VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT : 0) |
(ab_want ? VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT : 0),
.pfnUserCallback = debug_cb};
if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger);
if (!f || f(g_vk.instance, &mi, NULL, &g_vk.messenger) != VK_SUCCESS) ab_want = 0;
}
return 1;
}
@@ -217,6 +282,7 @@ typedef struct {
VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl;
VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbl;
VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr;
VkPhysicalDeviceAddressBindingReportFeaturesEXT abr;
} FeatureChain;
static int create_device(void) {
@@ -251,6 +317,13 @@ static int create_device(void) {
// Ryujinx's SPIR-V declares fragment shader interlock whenever the host reports it (vk_pipeline.c spirv_fixup).
OPTIONAL_EXT(has_fragment_shader_interlock, VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME, fsi,
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT);
// The GPU address map for device faults (ab_record), with the messenger made for it
int has_abr = 0;
if (ab_want && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME)) {
has_abr = 1;
ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME;
CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT);
}
// Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering
int has_fbl = 0, has_lr = 0;
if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) &&
@@ -314,6 +387,10 @@ static int create_device(void) {
else if (!h->textureCompressionBC) miss = "textureCompressionBC";
if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; }
if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension
rt_log("vk: GPU address map for device faults: %s",
has_abr && c->abr.reportAddressBinding ? "on (VK_EXT_device_address_binding_report; TOTK_ADDR_REPORT=0 turns it off)"
: !ab_want ? "off (TOTK_ADDR_REPORT=0 or no VK_EXT_debug_utils)"
: "unavailable (no VK_EXT_device_address_binding_report)");
g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress;
g_vk.has_astc_ldr = h->textureCompressionASTC_LDR;
{
@@ -376,6 +453,7 @@ static void destroy_all(void) {
gfx_window_release_vk();
if (g_vk.device) vkDeviceWaitIdle(g_vk.device);
if (g_vk.device) vk_astc_release();
if (g_vk.device) vk_deswz_release();
if (g_vk.device) vk_crumbs_release();
if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL);
vk_headroom_stop();
@@ -397,7 +475,7 @@ int vk_init(void) {
plat_mutex_lock(&g_lock);
if (!g_ready) {
memset(&g_vk, 0, sizeof g_vk);
if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_headroom_start(), vk_dlaa_init();
if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_upload_pools_init(), vk_headroom_start(), vk_dlaa_init();
else destroy_all();
}
int ok = g_ready;
@@ -437,6 +515,7 @@ void vk_device_fault_report(VkResult r) {
done = 1;
vk_crumbs_report();
vk_pass_marks_report();
vk_mem_fault_log();
VkCtx *c = vk_ctx();
PFN_vkGetDeviceFaultInfoEXT get =
c && c->has_device_fault ? (PFN_vkGetDeviceFaultInfoEXT)vkGetDeviceProcAddr(c->device, "vkGetDeviceFaultInfoEXT")
@@ -461,9 +540,11 @@ void vk_device_fault_report(VkResult r) {
rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description,
n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary);
// The faulting access lies in the precision-aligned range around the reported address.
for (uint32_t i = 0; i < n.addressInfoCount; i++)
for (uint32_t i = 0; i < n.addressInfoCount; i++) {
rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType),
(unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision);
ab_report(ai[i].reportedAddress, ai[i].addressPrecision);
}
for (uint32_t i = 0; i < n.vendorInfoCount; i++)
rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description,
(unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData);
+1
View File
@@ -183,6 +183,7 @@ static void memory_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAcce
// be made (logged). Replay thread.
static int bc_standin(const VkTexImage *img, uint32_t level, VkFormat fmt, BcStandIn *out) {
static int warned;
if (img->gone) return 0; // dropped since the lookup: a stand-in kept by its handle would outlive it
plat_mutex_lock(&bcs_lock);
int full = nbcs == BC_STANDINS;
for (int i = 0; i < nbcs; i++)
+13 -5
View File
@@ -593,6 +593,10 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v,
if (img->block_views) ci.pNext = &use; // the image's COLOR_ATTACHMENT usage is its block views' (vk_texture.c)
VkImageView view;
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS) return VK_NULL_HANDLE;
if (img->gone) { // dropped since the lookup: its views were purged and a new one would outlive the image
vk_defer_view(view);
return view;
}
ViewNode *n = malloc(sizeof *n);
if (!n) rt_fatal("vk: out of memory");
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
@@ -619,11 +623,15 @@ VkImageView vk_exec_target_view(const VkTexImage *img, VkFormat format, VkImageV
if (blocks) ci.pNext = &use; // not the BC image's sampling and transfer usages
if (vkCreateImageView(vk_ctx()->device, &ci, NULL, &view) != VK_SUCCESS)
rt_fatal("vk: render target view (format %d) could not be created", (int)format);
ViewNode *n = malloc(sizeof *n);
if (!n) rt_fatal("vk: out of memory");
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
nvn_hm_put(&views_by_image, &img->image, n);
nvn_hm_put(&view_of, &k, (void *)view);
if (img->gone) { // dropped since resolve_target (get_view)
vk_defer_view(view);
} else {
ViewNode *n = malloc(sizeof *n);
if (!n) rt_fatal("vk: out of memory");
*n = (ViewNode){nvn_hm_get(&views_by_image, &img->image), k, view};
nvn_hm_put(&views_by_image, &img->image, n);
nvn_hm_put(&view_of, &k, (void *)view);
}
}
vk_texture_unlock();
return view;
+1 -1
View File
@@ -272,7 +272,7 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) {
}
if (ntex == 1 && slot >= 0 && slot < 128 && ex->tex[1][slot]) {
const VkTexImage *s = vk_texture_by_id(NVN_HANDLE_TEX(ex->tex[1][slot]), NULL, NULL);
if (s && is_float(s->format) && s->width == t->width && s->height == t->height && s->samples == 1)
if (s && !s->gone && is_float(s->format) && s->width == t->width && s->height == t->height && s->samples == 1)
src = s->image, fmt = s->format;
}
plat_mutex_lock(&rec_lock);
+132 -27
View File
@@ -78,8 +78,9 @@ static int shadow_on; // shadow mode: imported pools get shad
static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size);
static void shadow_touch(PoolRec *r);
static void shadow_forget(PoolRec *r);
static void mirror_retiring(PoolRec *r); // vk_pool_fini: kept until mirror_retire (or vk_mem_shutdown)
static void mirror_retire(void *arg); // completion thread: destroys a finalized mirrored pool's record
static void pool_retiring(PoolRec *r); // vk_pool_fini: kept until pool_retire (or vk_mem_shutdown)
static void pool_retire(void *arg); // completion thread: destroys a finalized pool's record
static uint64_t retire_value(void); // the timeline value after which a finalized pool's record may go
static void grow(void **arr, int *cap, size_t elem);
static PlatMutex pool_lock = PLAT_MUTEX_INIT;
@@ -333,10 +334,14 @@ void vk_pool_fini(pool_t *p) {
vk_queue_unlock();
}
vk_mem_collect();
// A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may
// name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued).
if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r);
else destroy_rec(c, r);
// The record, its buffers and its memory outlive the submission being recorded, of any pool kind. The wait above
// covers submitted work only. With the synchronous replay another game thread may be recording right now, and that
// recording may already name this pool: a mirrored pool's sync copies and write-backs, an import pool's buffer
// through a texture upload or a released cache entry (vk_texture.c slot_resolve), a physical pool's memory under
// the images vk_texture_range_gone just deferred to that submission. The game frees a pool only once its own GPU
// work is done, so only the port's work can name it. Freed here, the submission ran with freed memory.
pool_retiring(r);
vk_completion_add(retire_value(), pool_retire, r);
}
// Requests were checked by the front end (nvn_api_core.c, MNG totk_vpool.c); apply them in order.
@@ -416,6 +421,10 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) {
return resolve_locked(addr, size, out);
}
int vk_mem_span_live(const VkMemSpan *sp) {
return sp->buffer && sp->pool && !sp->alloc && !shadow_on && ((const PoolRec *)sp->pool)->kind == POOL_IMPORT;
}
// The pool index lookup under the lock; a non-overlapped import or physical pool goes into the address's cache slot.
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) {
memset(out, 0, sizeof *out);
@@ -507,6 +516,9 @@ static void dirty_frame(uint64_t frame);
static void mirror_log(uint64_t frame);
static void shadow_log(uint64_t frame);
static int dead_pending(void);
// Deferrals made off the recording thread while another thread recorded a submission (cmd_lock, defer), the newest
// one's stamp: the case the stamp from the recording start covers. In the census line and the device fault report.
static uint64_t n_cross_defer, cross_defer_tl;
static void vram_log(uint64_t frame);
@@ -542,10 +554,11 @@ void vk_mem_census(uint64_t frame) {
}
plat_mutex_unlock_shared(&pool_lock);
rt_log("mem: census at frame %llu: import %llu pools %.1f MiB (%d without a GPU buffer), physical %llu pools %.1f MiB, "
"virtual %llu pools %.1f MiB of address space",
"virtual %llu pools %.1f MiB of address space; %llu objects deferred off the recording thread during a recording",
(unsigned long long)frame, (unsigned long long)count[POOL_IMPORT], total[POOL_IMPORT] / 1048576.0,
n_import_failures, (unsigned long long)count[POOL_PHYS], total[POOL_PHYS] / 1048576.0,
(unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0);
(unsigned long long)count[POOL_VIRT], total[POOL_VIRT] / 1048576.0,
(unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED));
static const char *const kinds[] = {"", "import", "physical", "virtual"};
for (int j = 0; j < ng; j++) {
char f[160];
@@ -1390,19 +1403,19 @@ static void mirror_forget(PoolRec *r) {
}
}
// Finalized mirrored pools waiting for mirror_retire; vk_mem_shutdown destroys those whose callback never ran.
// Finalized pools waiting for pool_retire; vk_mem_shutdown destroys those whose callback never ran.
static PoolRec **retiring;
static int nretiring, cretiring;
static PlatMutex retire_lock = PLAT_MUTEX_INIT;
static void mirror_retiring(PoolRec *r) {
static void pool_retiring(PoolRec *r) {
plat_mutex_lock(&retire_lock);
if (nretiring == cretiring) grow((void **)&retiring, &cretiring, sizeof *retiring);
retiring[nretiring++] = r;
plat_mutex_unlock(&retire_lock);
}
static void mirror_retire(void *arg) {
static void pool_retire(void *arg) {
plat_mutex_lock(&retire_lock);
for (int i = 0; i < nretiring; i++)
if (retiring[i] == arg) { retiring[i] = retiring[--nretiring]; break; }
@@ -2152,9 +2165,10 @@ typedef struct {
uint64_t seq;
Staging *stg;
int nstg, cap;
VkDeviceSize held; // staging and scratch bytes freed when the batch completes (vk_upload_held)
} Batch;
enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA };
enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST }; // DEAD_HOST: free(handle)
typedef struct { int kind; uint64_t handle; VmaAllocation alloc; uint64_t seq, tl; } Dead;
static PlatRMutex cmd_lock;
@@ -2169,7 +2183,8 @@ static int nspare, cap_spare;
static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq)
static Dead *dead;
static int ndead, cap_dead;
static uint64_t tl_noted; // newest timeline value vk_sync.c reported submitted
static uint64_t tl_noted; // value of the submission being recorded, else of the last one (vk_mem_note_fence)
static _Thread_local uint64_t tl_noted_here; // the value this thread noted: it records that submission
static void cmd_init(void *p) {
(void)p;
@@ -2212,6 +2227,10 @@ static void collect_locked(VkCtx *c) {
for (int i = 0; i < ndead; i++) {
Dead *d = &dead[i];
if (d->seq > done_seq || d->tl > tl) { dead[k++] = *d; continue; }
if (d->kind == DEAD_HOST) {
free((void *)(uintptr_t)d->handle);
continue;
}
rec_check_handle(d->kind == DEAD_VIEW ? REC_VIEW : d->kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, d->handle, 1,
d->tl, tl);
switch (d->kind) {
@@ -2305,10 +2324,68 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
else {
if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg);
open_b.stg[open_b.nstg++] = (Staging){b, a};
open_b.held += bi.size;
}
return info.pMappedData;
}
// Device-local scratch of the upload batch (the GPU deswizzle's linear copies, the ASTC decoder's blocks and texels):
// a pool of its own, like staging, so the replay thread neither waits on the default pool's lock while another thread
// makes a block there (the headroom thread: about 8 ms for 256 MiB) nor makes one itself for a few MiB. One block
// of SCR_BLOCK is kept from the first use; larger buffers than half a block go to the default pool.
#define SCR_BLOCK (64ull << 20)
static VmaPool scr_pool;
static int scr_pool_tried;
VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
VkCtx *c = vk_ctx();
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
if (!__atomic_load_n(&scr_pool_tried, __ATOMIC_ACQUIRE)) {
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
if (!scr_pool_tried) {
// two blocks with 8 GiB or more of device-local memory: fight2's first world frame deswizzles about
// 280 MiB, more than the GPU finishes while the replay records, and the pool grew by four blocks there
VkDeviceSize vram = 0;
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT && c->mem_props.memoryHeaps[h].size > vram)
vram = c->mem_props.memoryHeaps[h].size;
uint32_t type;
VmaPoolCreateInfo pi = {.blockSize = SCR_BLOCK, .minBlockCount = vram >= (8ull << 30) ? 2 : 1};
if (vmaFindMemoryTypeIndexForBufferInfo(c->vma, &bi, &aci, &type) == VK_SUCCESS) {
pi.memoryTypeIndex = type;
if (vmaCreatePool(c->vma, &pi, &scr_pool) != VK_SUCCESS) scr_pool = NULL;
}
rt_log("vk: upload scratch pool %s (%u blocks of %llu MiB kept)", scr_pool ? "created" : "unavailable: default pool",
(unsigned)pi.minBlockCount, (unsigned long long)(SCR_BLOCK >> 20));
__atomic_store_n(&scr_pool_tried, 1, __ATOMIC_RELEASE);
}
plat_rmutex_unlock(&cmd_lock);
}
if (scr_pool && bi.size <= SCR_BLOCK / 2) aci.pool = scr_pool;
VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, buf, alloc, NULL);
if (r == VK_SUCCESS && open_active) open_b.held += bi.size; // the caller defers its free to the open batch
return r;
}
VkDeviceSize vk_upload_held(void) { return open_active ? open_b.held : 0; }
// At device creation: the staging and scratch pools with their kept blocks, made through a first small buffer each
// (freed at once), so the first world frame does not allocate them (staging was first used there once the GPU took
// the early texture uploads).
void vk_upload_pools_init(void) {
VkCtx *c = vk_ctx();
if (!c) return;
VkBuffer b;
VmaAllocation a;
vk_staging(16, &b, &a);
vmaDestroyBuffer(c->vma, b, a);
if (vk_scratch_buffer(16, &b, &a) == VK_SUCCESS) vmaDestroyBuffer(c->vma, b, a);
}
uint64_t vk_upload_flush(int wait) {
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
@@ -2346,20 +2423,21 @@ static void defer(int kind, uint64_t handle, VmaAllocation alloc) {
plat_rmutex_lock(&cmd_lock);
if (ndead == cap_dead) grow((void **)&dead, &cap_dead, sizeof *dead);
// The open batch may record into the object: it counts as the newest user. The GPU must also be past every
// submission that names it. tl_noted covers the submitted ones; an object the replay lets go of while it records
// a submission (vk_exec_recording_fence) may be named by that one too, not yet submitted: vk_texture.c
// slot_resolve hands a draw a released entry, and entry_of destroys released entries at that address later in
// the same submission. Stamped with the last submitted value, the object went as soon as the GPU reached it,
// before the submission that names it ran, and with the recorder thread (vk_rec.c) behind, before the recorder
// made the call that names it. LAS Recomp and Savage Wind Recomp stamp every deferral with the submission being
// recorded (runtime/nvn/vulkan/textures.c defer: vk_pending_seq). TOTK_DEFER_STAMP=0 brings back the old stamp.
static int stamp = -1;
if (stamp < 0) stamp = rt_knob("DEFER_STAMP", "1") != 0;
uint64_t tl = tl_noted, rf = stamp ? vk_exec_recording_fence() : 0;
if (rf > tl) tl = rf;
// submission that names it, including the one being recorded, which may name it and is not submitted yet:
// vk_texture.c slot_resolve hands a draw a released entry, and entry_of, sweep_released and the VRAM pressure drop
// destroy released entries. tl_noted is that submission's value from the start of its recording (vk_sync.c
// vk_sync_submit), whichever thread destroys the object. With the synchronous replay a game thread finalizing a
// texture (vk_backend.c vk_be_texture_release) destroys entries while another game thread records; stamped with
// the last submitted value, as the recording thread's own fence was the only one seen, the object was freed once
// the GPU reached that value, before the submission naming it ran: "write invalid" device faults on NVIDIA. LAS
// Recomp and Savage Wind Recomp stamp every deferral with the submission being recorded (runtime/nvn/vulkan/
// textures.c defer: vk_pending_seq).
uint64_t tl = tl_noted;
if (kind != DEAD_HOST && tl > vk_sync_issued() && tl_noted_here != tl) n_cross_defer++, cross_defer_tl = tl;
dead[ndead++] = (Dead){kind, handle, alloc, open_active ? open_b.seq : next_seq, tl};
rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl,
vk_sync_fence_done());
if (kind != DEAD_HOST)
rec_check_handle(kind == DEAD_VIEW ? REC_VIEW : kind == DEAD_BUFFER_VMA ? REC_BUFFER : REC_IMAGE, handle, 0, tl,
vk_sync_fence_done());
plat_rmutex_unlock(&cmd_lock);
}
@@ -2378,14 +2456,38 @@ void vk_defer_view(VkImageView view) { if (view) defer(DEAD_VIEW, (uint64_t)(uin
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) {
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
}
void vk_defer_free(void *p) { if (p) defer(DEAD_HOST, (uint64_t)(uintptr_t)p, NULL); }
void vk_mem_note_submit(uint64_t timeline_value) {
// The device fault report: no lock, since a thread that holds cmd_lock may be waiting for the report's own lock
// (vk_upload_flush reports a failed submit with cmd_lock held).
void vk_mem_fault_log(void) {
rt_log("vk: deferred frees at the fault: %d pending, stamp %llu, submitted %llu, done %llu; %llu deferred off the "
"recording thread during a recording (newest stamp %llu)",
__atomic_load_n(&ndead, __ATOMIC_RELAXED), (unsigned long long)__atomic_load_n(&tl_noted, __ATOMIC_RELAXED),
(unsigned long long)vk_sync_issued(), (unsigned long long)vk_sync_fence_done(),
(unsigned long long)__atomic_load_n(&n_cross_defer, __ATOMIC_RELAXED),
(unsigned long long)__atomic_load_n(&cross_defer_tl, __ATOMIC_RELAXED));
}
void vk_mem_note_fence(uint64_t timeline_value) {
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
if (timeline_value > tl_noted) tl_noted = timeline_value;
tl_noted_here = timeline_value;
plat_rmutex_unlock(&cmd_lock);
}
// The later of the submission being recorded (tl_noted, noted from its start: vk_sync.c vk_sync_submit) and the next
// one to be submitted.
static uint64_t retire_value(void) {
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
uint64_t v = vk_sync_issued() + 1;
if (tl_noted > v) v = tl_noted;
plat_rmutex_unlock(&cmd_lock);
return v;
}
void vk_mem_collect(void) {
VkCtx *c = vk_ctx();
if (!c) return;
@@ -2434,6 +2536,9 @@ void vk_mem_shutdown(void) {
if (stg_pool) vmaDestroyPool(c->vma, stg_pool); // after retire: no staging buffer is left in it
stg_pool = NULL;
stg_pool_tried = 0;
if (scr_pool) vmaDestroyPool(c->vma, scr_pool); // deferred scratch buffers went with collect_locked above
scr_pool = NULL;
scr_pool_tried = 0;
plat_rmutex_unlock(&cmd_lock);
plat_mutex_lock(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
+47 -10
View File
@@ -101,6 +101,12 @@ static struct {
VkPipeStats stats;
} P = {.lock = PLAT_MUTEX_INIT};
// Builds into P.cache take it shared, vk_pipeline_save takes it to copy the cache out. Vulkan lets
// vkGetPipelineCacheData run alongside builds into a cache created without the EXTERNALLY_SYNCHRONIZED bit, but
// Intel's Windows driver (101.7085, Iris Xe) faults in it (igvk64.dll, reading 0x170) when the first autosave meets the
// boot's pipeline builds (#681). Nothing is taken while it is held.
static PlatMutex cache_use = PLAT_MUTEX_INIT;
typedef struct Layout {
struct Layout *next;
uint64_t hash;
@@ -1440,7 +1446,9 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay
if (k->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
if (k->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
uint64_t p0 = perf_begin();
plat_mutex_lock_shared(&cache_use);
VkResult r = vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe);
plat_mutex_unlock_shared(&cache_use);
perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it
if (r != VK_SUCCESS) {
rt_log("vk_pipeline: vkCreateGraphicsPipelines failed (%d)", (int)r);
@@ -1591,7 +1599,9 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi
ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil;
pi.pColorBlendState = &cb, pi.pMultisampleState = &ms;
}
plat_mutex_lock_shared(&cache_use);
if (vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
plat_mutex_unlock_shared(&cache_use);
out:
if (m) vkDestroyShaderModule(P.device, m, NULL);
free(patched);
@@ -1803,7 +1813,8 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc
// shaders (gfx_shader_get results) live for the process (gfx_shader_drop_memory is the shader probe's, which builds no
// pipelines), layouts until vk_pipeline_shutdown, which drains the queue first. Workers call vkCreateGraphicsPipelines
// without P.lock: P.cache is created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT, so the driver
// synchronises it, vkGetPipelineCacheData in vk_pipeline_save included. Compute pipelines stay synchronous: a skipped
// synchronises the builds among themselves. vk_pipeline_save still keeps them apart from vkGetPipelineCacheData (see
// cache_use). Compute pipelines stay synchronous: a skipped
// dispatch can change what the game computes.
//
// Two queues: draws' misses (A.head) and the boot prebuild (A.low, see "pipeline keys" below). Workers take the low
@@ -2512,7 +2523,7 @@ static void prebuild_start(void) {
plat_mutex_unlock(&P.lock);
}
// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised).
// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. Takes cache_use shared, like the graphics builds.
static VkPipeline compute_create(const GfxShader *cs, const Layout *l) {
uint64_t words;
const uint32_t *code = gfx_shader_spirv(cs, &words);
@@ -2525,7 +2536,9 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l) {
.layout = l->layout};
pin_subgroup32(&ci.stage, &rqc, code, words);
VkPipeline pipe = VK_NULL_HANDLE;
plat_mutex_lock_shared(&cache_use);
if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
plat_mutex_unlock_shared(&cache_use);
vkDestroyShaderModule(P.device, m, NULL);
return pipe;
}
@@ -3290,7 +3303,8 @@ void vk_pipeline_stats(VkPipeStats *s) {
static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache
void vk_pipeline_save(void) {
// One cache save; wait: queue for cache_use (the exit saves) rather than try for it (the autosave).
static void cache_save(int wait) {
if (P.ready) keys_flush();
if (P.ready && P.gpl) {
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws "
@@ -3317,23 +3331,46 @@ void vk_pipeline_save(void) {
}
gfx_shader_record_flush(1);
if (!P.ready || !P.cache || !P.cache_path[0]) return;
// Not under P.lock: the driver synchronises P.cache, and holding P.lock through the copy and the file write (about
// 60 ms on the 1.4.3 fight) stopped every pipeline lookup of the replay, a freeze at each 20 s autosave.
// Not under P.lock: holding it through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped every
// pipeline lookup of the replay, a freeze at each 20 s autosave. Only the copy takes cache_use, so no pipeline is
// being built into P.cache while the driver reads it.
plat_mutex_lock(&save_lock);
if (!P.cache) { // vk_pipeline_shutdown destroyed it
plat_mutex_unlock(&save_lock);
return;
}
uint64_t t0 = qpc();
if (wait) {
plat_mutex_lock(&cache_use);
} else {
// The autosave never queues for cache_use: a queued exclusive request on an SRW lock holds back new shared
// ones, so the drawing thread's next build would wait for a worker's build to end. It tries for about 2 s
// and otherwise saves on its next round.
int got = 0;
for (int i = 0; i < 1000 && !(got = plat_mutex_trylock(&cache_use)); i++) plat_sleep_ms(2);
if (!got) {
rt_log("vk_pipeline: cache save put off, pipelines were being built for %.0f ms", ms_since(t0));
plat_mutex_unlock(&save_lock);
return;
}
}
uint64_t t1 = qpc();
size_t size = 0;
void *data = NULL;
if (vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) &&
vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS) {
if (!plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path);
}
int ok = vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) &&
vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS;
plat_mutex_unlock(&cache_use);
// The copy time is how long a pipeline build on the drawing thread (a compute miss, under P.lock) can wait for
// this save; the wait is how long the save waited for builds in flight.
rt_log("vk_pipeline: cache %s: %.1f MB copied in %.1f ms, after %.1f ms waiting for builds", wait ? "save" : "autosave",
(double)size / 1e6, ms_since(t1), (double)(t1 - t0) / 1e6);
if (ok && !plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path);
free(data);
plat_mutex_unlock(&save_lock);
}
void vk_pipeline_save(void) { cache_save(1); }
// The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it
// runs: at most every 20 s, and only after new pipelines were built. New pipeline keys are appended every 2 s.
static void autosave_once(void) {
@@ -3347,7 +3384,7 @@ static void autosave_once(void) {
vk_pipeline_stats(&s);
if (s.graphics + s.compute == last_n) return;
last_n = s.graphics + s.compute, last_ms = now;
vk_pipeline_save();
cache_save(0);
}
// On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to
+4 -3
View File
@@ -309,7 +309,7 @@ void vk_sync_start(void) {
if (reserved < now) reserved = now;
stop = 0;
plat_mutex_unlock(&st_lock);
vk_mem_note_submit(now);
vk_mem_note_fence(now);
completion_thread = plat_thread_new(completion_main, c, "vk completion");
if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
__atomic_store_n(&started, 1, __ATOMIC_RELEASE);
@@ -379,7 +379,6 @@ VkCommandBuffer vk_sync_split(void) {
vk_device_fault_report(r);
rt_fatal("vk: vkQueueSubmit2 of a part of value %llu failed (%d)", (unsigned long long)cur_fence, (int)r);
}
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached
rec_begin(s->cb, cur_fence, VK_CRUMB_SUBMIT, cur_fence);
ts_begin(s->cb);
@@ -404,6 +403,9 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
vk_mem_dirty_submit(); // TOTK_TRACE=perf: what a mirror would upload before this submit
uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1;
cur_fence = fence;
// Before the first object is resolved: what any thread destroys from here on waits for this value (vk_memory.c
// defer), since the recording may already name it.
vk_mem_note_fence(fence);
CmdSlot *s = take_slot(c, fence);
uint64_t t0 = perf_begin();
rec_begin(s->cb, fence, VK_CRUMB_SUBMIT, fence); // begun by the recorder when it records the part
@@ -434,7 +436,6 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
rt_fatal("vk: vkQueueSubmit2 of value %llu failed (%d)", (unsigned long long)fence, (int)r);
}
s->fence = recorded ? fence : 0;
vk_mem_note_submit(fence);
plat_mutex_lock(&st_lock);
__atomic_store_n(&issued, fence, __ATOMIC_RELEASE);
plat_cond_broadcast(&cv_work);
+219 -5
View File
@@ -597,10 +597,16 @@ static void destroy_tex(Tex *e) {
if (p->next_at == e) { p->next_at = e->next_at; break; }
}
if (e->released) nreleased--, rel_unlink(e);
// Another thread may hold &e->pub, taken under the lock and used after it (vk_texture_get, vk_texture_by_id: render
// targets for a whole pass, vk_exec.c resolve_target, vk_texture_storage_bound_locked writes through it). Freed
// here, that thread read a stale VkImage from freed memory, made views of it and cached them by that handle. The
// record goes with the image, once the submission being recorded is done; gone keeps views of it out of the caches.
e->pub.gone = 1;
vk_exec_image_gone(e->pub.image);
vk_defer_image(e->pub.image, e->alloc);
free(e->snap);
free(e);
e->snap = NULL;
vk_defer_free(e);
}
// Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries the
@@ -819,15 +825,20 @@ static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar,
// large ASTC array decodes to RGBA8 in one go: a staging buffer of 676682336 bytes failed (-2) on an Intel UHD with
// 8 GB of shared memory and the game closed before the title screen.
#define UPLOAD_PART (32ull << 20) // fits the staging pool (vk_staging takes up to half of its 64 MiB block)
// The open upload batch is submitted between chunks (and parts of the CPU path) once its staging and scratch pass this:
// on fight2's first world frame one batch took hundreds of MiB, and the staging and scratch pools grew by 64 MiB blocks
// on the replay thread. Submitting earlier keeps the queue order (the batch still precedes the part that samples the
// images) and lets the GPU finish and return the buffers sooner.
#define UPLOAD_HELD_MAX (48ull << 20)
// Copies the whole storage into the image (every level and layer the image has).
static void upload(Tex *e) {
static int upload_gpu(Tex *e);
// The CPU side of upload: deswizzles (and decodes or expands) into staging buffers, then copies them into the image.
static void upload_cpu(Tex *e) {
static volatile int32_t n_bad, n_undecoded, n_split;
const tex_t *t = &e->t;
const nvn_texdesc *d = &t->d;
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
if (!bpb || !t->mem) return;
uint64_t p0 = perf_begin(), u0 = kern_now_ns();
int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D;
uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers;
int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb;
@@ -908,6 +919,7 @@ static void upload(Tex *e) {
}
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
vk_upload_end();
if (!acheck && vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0);
}
if (parts > 1)
log_limited(&n_split, "texture %#llx fmt %#x %ux%u, %u levels, %u layers: %.1f MiB staged in %d parts",
@@ -924,7 +936,201 @@ static void upload(Tex *e) {
free(raw_copy);
free(ar);
free(rg);
}
// Block-linear storage in an imported pool, deswizzled by the GPU (vk_deswizzle.c): the replay thread only records
// the dispatches, the image copies and (ASTC) the decode into the upload batch the CPU path also uses, which runs before
// the part that samples the image. The GPU reads the pool when the batch runs, as the Switch's GPU reads it at
// execution; a CPU write after make_fresh's write-watch reset still marks the pages and brings another upload. Each
// chunk of regions gets a device-local scratch buffer of at most DESWZ_CHUNK bytes (one region, when larger), freed
// once the batch completes. TOTK_DESWIZZLE=check compares every chunk with the CPU deswizzle (waits for the GPU).
// 0 when not handled, before anything is recorded: the CPU path takes the texture.
#define DESWZ_CHUNK (32ull << 20)
// Uploads by path since the last perf summary (TOTK_TRACE=perf): 0 GPU, else why the CPU took it.
enum { UP_GPU, UP_MODE, UP_LINEAR, UP_RGB, UP_ASTC_CPU, UP_BPB, UP_NOT_LIVE, UP_GEOM, UP_LIMITS, UP_SCRATCH, UP_WHY };
static const char *const up_why_name[UP_WHY] = {"gpu", "cpu mode", "linear", "rgb", "astc on cpu", "bytes per block",
"not an imported pool", "layout", "device limits", "scratch"};
static uint64_t up_n[UP_WHY], up_bytes[UP_WHY], up_ns[UP_WHY];
static int up_why; // upload_gpu's reason for returning 0 (texture lock held)
static int upload_gpu(Tex *e) {
static volatile int32_t n_scratch_fail, n_undecoded, n_bad;
static volatile int64_t n_checked, n_differ;
const tex_t *t = &e->t;
const nvn_texdesc *d = &t->d;
int mode = vk_deswz_mode();
up_why = mode == VK_DESWZ_CPU ? UP_MODE : nvn_tex_is_linear(d) ? UP_LINEAR : e->f.expand_rgb ? UP_RGB : UP_GPU;
if (up_why) return 0;
int astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu();
if (astc && (!agpu || vk_astc_mode() == VK_ASTC_CHECK)) return up_why = UP_ASTC_CPU, 0; // or ASTC's check mode
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
if (!bpb || 16 % bpb) return up_why = UP_BPB, 0; // the rows of the linear copy are padded to 16 bytes
VkMemSpan sp;
if (!vk_mem_resolve(e->pub.addr, t->L.size, &sp) || !vk_mem_span_live(&sp)) return up_why = UP_NOT_LIVE, 0;
int one_d = e->pub.type == VK_IMAGE_TYPE_1D, is3d = d->target == NVN_TEXTURE_TARGET_3D;
uint32_t levels = e->pub.levels, layers = e->pub.layers, n = levels * layers, k = 0;
VkDeswzRegion *dr = calloc(n, sizeof *dr);
VkBufferImageCopy *rg = calloc(n, sizeof *rg);
VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL;
if (!dr || !rg || (agpu && !ar)) rt_fatal("vk: out of memory (texture upload regions)");
int ok = 1;
VkDeviceSize max_dst = DESWZ_CHUNK;
for (uint32_t lv = 0; ok && lv < levels; lv++)
for (uint32_t ly = 0; ok && ly < layers; ly++) {
uint32_t row_bytes, rows, slices, gbpb;
uint64_t base;
unsigned th, td;
uint32_t w = mip(d->width, lv), h = one_d ? 1 : mip(d->height, lv), sl = is3d ? mip(d->depth, lv) : 1;
if (nvn_deswizzle_geom(d, lv, ly, &row_bytes, &rows, &slices, &base, &th, &td, &gbpb) || gbpb != (uint32_t)bpb ||
base >= t->L.size || rows != (h + bh - 1) / bh || slices != sl) {
ok = 0;
break;
}
uint64_t bhg = 1ull << th, bdg = 1ull << td, gobs_x = (row_bytes + 63) / 64;
uint64_t blocks_y = (rows + 8 * bhg - 1) / (8 * bhg), blocks_z = (slices + bdg - 1) / bdg;
uint64_t segs = gobs_x * blocks_y * blocks_z * (32ull << (th + td)), room = (t->L.size - base) / 16;
if (segs > room) segs = room;
uint32_t pitch16 = (row_bytes + 15) / 16;
VkDeviceSize out = (VkDeviceSize)pitch16 * 16 * rows * slices;
if (segs > UINT32_MAX || gobs_x * blocks_y > UINT32_MAX) { ok = 0; break; }
if (out > max_dst) max_dst = out;
dr[k] = (VkDeswzRegion){.src = base, .segs = (uint32_t)segs, .pitch16 = pitch16, .rows = rows, .slices = slices,
.th = th, .td = td, .gobs_x = (uint32_t)gobs_x, .blocks_y = (uint32_t)blocks_y};
rg[k] = (VkBufferImageCopy){.bufferRowLength = pitch16 * 16 / (uint32_t)bpb * (uint32_t)bw,
.bufferImageHeight = rows * (uint32_t)bh,
.imageSubresource = {e->pub.aspect, lv, ly, 1}, .imageExtent = {w, h, sl}};
if (agpu) ar[k] = (VkAstcRegion){.w = w, .h = h, .slices = sl, .level = lv, .layer = ly};
k++;
}
if (!ok || k != n || !vk_deswz_fits(sp.offset, t->L.size, dr, (int)n, max_dst)) {
free(dr), free(rg), free(ar);
return up_why = !ok || k != n ? UP_GEOM : UP_LIMITS, 0;
}
VkCtx *c = vk_ctx();
VkCommandBuffer cb = vk_upload_begin();
// Earlier GPU writes into the pool (copies, shaders) before the reads; the image's earlier uses before its copy.
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) {
VkDeviceSize size = 0;
for (k1 = k0; k1 < n; k1++) {
VkDeviceSize out = (VkDeviceSize)dr[k1].pitch16 * 16 * dr[k1].rows * dr[k1].slices;
if (k1 > k0 && size + out > DESWZ_CHUNK) break;
dr[k1].dst = size, rg[k1].bufferOffset = size;
if (agpu) ar[k1].src = size;
size += out;
}
VkBuffer sb;
VmaAllocation sa;
if (vk_scratch_buffer(size, &sb, &sa) != VK_SUCCESS) {
log_limited(&n_scratch_fail, "texture %#llx: deswizzle scratch of %llu bytes failed%s",
(unsigned long long)e->pub.addr, (unsigned long long)size,
k0 ? "; its later levels left as they were" : "; deswizzled on the CPU");
if (!k0) {
vk_upload_end();
free(dr), free(rg), free(ar);
return up_why = UP_SCRATCH, 0;
}
break;
}
if (!vk_deswz_record(cb, sp.buffer, sp.offset, t->L.size, sb, size, dr + k0, (int)(k1 - k0))) {
log_limited(&n_bad, "texture %#llx: deswizzle of levels %u-%u not recorded; left as they were",
(unsigned long long)e->pub.addr, k0, k1 - 1); // vk_deswz_fits passed: not expected
vk_defer_buffer(sb, sa);
break;
}
barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
if (agpu) {
uint64_t a0 = perf_begin();
if (!vk_astc_record(cb, sb, size, ar + k0, (int)(k1 - k0), e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb,
e->f.astc_bc, e->pub.image, e->pub.aspect, VK_NULL_HANDLE))
log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was",
(unsigned long long)e->pub.addr, d->format, d->width, d->height);
perf_end(PERF_ASTC, a0);
} else {
vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, k1 - k0, rg + k0);
}
if (mode == VK_DESWZ_CHECK) { // the linear copy back to the host, compared with the CPU deswizzle
VkBuffer rb;
VmaAllocation ra;
const uint8_t *rmap = vk_staging(size, &rb, &ra);
VkBufferCopy cp = {0, 0, size};
vkCmdCopyBuffer(cb, sb, rb, 1, &cp);
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
VK_ACCESS_2_HOST_READ_BIT);
vk_defer_buffer(sb, sa);
vk_upload_end();
vk_upload_flush(1);
vmaInvalidateAllocation(c->vma, ra, 0, VK_WHOLE_SIZE);
for (uint32_t i = k0; i < k1; i++) {
const VkDeswzRegion *r = &dr[i];
size_t pitch = (size_t)r->pitch16 * 16, bytes = pitch * r->rows * r->slices;
uint8_t *cpu = malloc(bytes);
if (!cpu) rt_fatal("vk: out of memory (deswizzle check)");
uint32_t row_bytes = 0, rows, slices, gbpb;
uint64_t base;
unsigned th, td;
nvn_deswizzle_geom(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, &row_bytes,
&rows, &slices, &base, &th, &td, &gbpb);
nvn_deswizzle(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, t->mem, cpu, pitch);
uint64_t bad = 0, first = UINT64_MAX;
for (uint64_t row = 0; row < (uint64_t)r->rows * r->slices; row++)
if (memcmp(cpu + row * pitch, rmap + r->dst + row * pitch, row_bytes)) {
bad++;
if (first == UINT64_MAX) first = row;
}
free(cpu);
plat_add64(&n_checked, 1);
if (bad) {
plat_add64(&n_differ, 1);
rt_log("vk: deswizzle check: texture %#llx fmt %#x %ux%u level %u layer %u: %llu of %u rows differ, the "
"first %llu (th %u td %u, %u rows of %u bytes, %u slices)", (unsigned long long)e->pub.addr,
d->format, d->width, d->height, rg[i].imageSubresource.mipLevel,
rg[i].imageSubresource.baseArrayLayer, (unsigned long long)bad, r->rows * r->slices,
(unsigned long long)first, r->th, r->td, r->rows, row_bytes, r->slices);
}
}
vmaDestroyBuffer(c->vma, rb, ra);
if ((n_checked & 1023) == 0 || n_checked < 2)
rt_log("vk: deswizzle check: %lld regions compared, %lld differ", (long long)n_checked, (long long)n_differ);
cb = vk_upload_begin();
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
continue;
}
vk_defer_buffer(sb, sa);
if (k1 < n && vk_upload_held() > UPLOAD_HELD_MAX) { // submit what is recorded, go on in a new batch
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW);
vk_upload_end();
vk_upload_flush(0);
cb = vk_upload_begin();
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
}
}
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW);
vk_upload_end();
if (vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0);
free(dr), free(rg), free(ar);
return 1;
}
// Copies the whole storage into the image (every level and layer the image has): on the GPU where it can
// (upload_gpu), else on the CPU.
static void upload(Tex *e) {
const tex_t *t = &e->t;
const nvn_texdesc *d = &t->d;
int bw, bh;
if (!nvn_fmt_bpb(d->format, &bw, &bh) || !t->mem) return;
uint64_t p0 = perf_begin(), u0 = kern_now_ns();
int why = UP_GPU;
if (!upload_gpu(e)) why = up_why, upload_cpu(e);
uint64_t dt = kern_now_ns() - u0;
up_n[why]++, up_bytes[why] += t->L.size, up_ns[why] += dt;
if (e->uploaded) {
n_reuploads++, perf_count(PERF_CNT_TEX_REUP);
ru_n++, ru_bytes += t->L.size, ru_ns += dt;
@@ -1404,6 +1610,14 @@ void vk_texture_new_submit(void) {
(unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6,
(unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6);
ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = 0;
char ub[512];
size_t uo = 0;
for (int i = 0; i < UP_WHY; i++)
if (up_n[i] && uo < sizeof ub)
uo += (size_t)snprintf(ub + uo, sizeof ub - uo, "%s %s %llu (%.1f MiB, %.2f ms)", uo ? "," : "",
up_why_name[i], (unsigned long long)up_n[i], up_bytes[i] / 1048576.0, up_ns[i] / 1e6);
if (uo) rt_log("perf: frame %llu: texture uploads by path:%s", (unsigned long long)f * 60, ub);
memset(up_n, 0, sizeof up_n), memset(up_bytes, 0, sizeof up_bytes), memset(up_ns, 0, sizeof up_ns);
uint64_t ck, calls, untr;
vk_mem_snap_stats(&ck, &calls, &untr);
rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu",