Mirrors/totk-pc-recomp
1
0 Fork 0
mirror of https://github.com/valentderah/totk-pc-recomp.git synced 2026-11-11 01:41:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deswizzle block-linear textures on the GPU when their storage is an imported pool

Browse Source 
The replay thread deswizzled every block-linear texture on the CPU, and entering a new area stalled it for 50-236 ms. Now a compute pass reads the texture straight from the pool's imported buffer and writes a linear copy into a device-local scratch pool; the upload batch copies it into the image, or the ASTC decoder takes it. Large texture loads submit the batch early so the staging and scratch pools do not grow on the replay thread, and both pools are made at device creation.

On fight2 the render thread waits about 270 ms on textures per run instead of 900, and the load frame drops from 145 to about 80 ms. Mean fps is unchanged. TOTK_DESWIZZLE=cpu turns it off and check compares every upload with the CPU result (0 of 196k regions differ). Shadow mode (Linux/Wine) and textures outside the device's limits stay on the CPU.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-07 16:38:26 +03:00
1 parent 71d8b0cc61
commit 57037ccc29
9 files changed
+633 -11
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

runtime/bl_deswizzle_spv.h
+111
View File
@@ -0,0 +1,111 @@
// Generated by tools/build_shaders.py from runtime/shaders/bl_deswizzle.comp; do not edit.
// source sha256 bd2065290e315e85d2fcdb609d812c15ba9a109d7506b3105850a5860677b0ef
#pragma once
#include <stdint.h>
static const uint32_t bl_deswizzle_spv[] = {
0x07230203, 0x00010300, 0x000d000b, 0x000000d0, 0x00000000, 0x00020011, 0x00000001, 0x0006000b,
0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001,
0x0007000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000b, 0x0000001f, 0x00060010,
0x00000004, 0x00000011, 0x00000040, 0x00000001, 0x00000001, 0x00040047, 0x0000000b, 0x0000000b,
0x0000001a, 0x00030047, 0x00000010, 0x00000002, 0x00050048, 0x00000010, 0x00000000, 0x00000023,
0x00000000, 0x00050048, 0x00000010, 0x00000001, 0x00000023, 0x00000004, 0x00050048, 0x00000010,
0x00000002, 0x00000023, 0x00000008, 0x00050048, 0x00000010, 0x00000003, 0x00000023, 0x0000000c,
0x00050048, 0x00000010, 0x00000004, 0x00000023, 0x00000010, 0x00050048, 0x00000010, 0x00000005,
0x00000023, 0x00000014, 0x00050048, 0x00000010, 0x00000006, 0x00000023, 0x00000018, 0x00050048,
0x00000010, 0x00000007, 0x00000023, 0x0000001c, 0x00050048, 0x00000010, 0x00000008, 0x00000023,
0x00000020, 0x00050048, 0x00000010, 0x00000009, 0x00000023, 0x00000024, 0x00050048, 0x00000010,
0x0000000a, 0x00000023, 0x00000028, 0x00040047, 0x0000001f, 0x0000000b, 0x0000001b, 0x00040047,
0x000000ad, 0x00000006, 0x00000010, 0x00030047, 0x000000ae, 0x00000002, 0x00040048, 0x000000ae,
0x00000000, 0x00000019, 0x00050048, 0x000000ae, 0x00000000, 0x00000023, 0x00000000, 0x00030047,
0x000000b0, 0x00000019, 0x00040047, 0x000000b0, 0x00000021, 0x00000001, 0x00040047, 0x000000b0,
0x00000022, 0x00000000, 0x00040047, 0x000000c1, 0x00000006, 0x00000010, 0x00030047, 0x000000c2,
0x00000002, 0x00040048, 0x000000c2, 0x00000000, 0x00000018, 0x00050048, 0x000000c2, 0x00000000,
0x00000023, 0x00000000, 0x00030047, 0x000000c4, 0x00000018, 0x00040047, 0x000000c4, 0x00000021,
0x00000000, 0x00040047, 0x000000c4, 0x00000022, 0x00000000, 0x00040047, 0x000000cd, 0x0000000b,
0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006,
0x00000020, 0x00000000, 0x00040017, 0x00000009, 0x00000006, 0x00000003, 0x00040020, 0x0000000a,
0x00000001, 0x00000009, 0x0004003b, 0x0000000a, 0x0000000b, 0x00000001, 0x0004002b, 0x00000006,
0x0000000c, 0x00000001, 0x00040020, 0x0000000d, 0x00000001, 0x00000006, 0x000d001e, 0x00000010,
0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006, 0x00000006,
0x00000006, 0x00000006, 0x00000006, 0x00040020, 0x00000011, 0x00000009, 0x00000010, 0x0004003b,
0x00000011, 0x00000012, 0x00000009, 0x00040015, 0x00000013, 0x00000020, 0x00000001, 0x0004002b,
0x00000013, 0x00000014, 0x0000000a, 0x00040020, 0x00000015, 0x00000009, 0x00000006, 0x0004002b,
0x00000006, 0x00000019, 0x00000000, 0x0004002b, 0x00000006, 0x0000001d, 0x00000040, 0x0004003b,
0x0000000a, 0x0000001f, 0x00000001, 0x0004002b, 0x00000013, 0x00000024, 0x00000002, 0x00020014,
0x00000027, 0x0004002b, 0x00000013, 0x0000002d, 0x00000006, 0x0004002b, 0x00000013, 0x00000032,
0x00000007, 0x0004002b, 0x00000006, 0x00000037, 0x00000020, 0x0004002b, 0x00000006, 0x00000048,
0x00000005, 0x0004002b, 0x00000006, 0x0000004c, 0x0000001f, 0x0004002b, 0x00000013, 0x0000004f,
0x00000008, 0x0004002b, 0x00000013, 0x00000052, 0x00000009, 0x0004002b, 0x00000006, 0x0000006a,
0x00000004, 0x0004002b, 0x00000006, 0x0000006f, 0x00000002, 0x0004002b, 0x00000006, 0x00000078,
0x00000008, 0x0004002b, 0x00000006, 0x00000083, 0x00000003, 0x0004002b, 0x00000013, 0x00000093,
0x00000003, 0x0004002b, 0x00000013, 0x0000009b, 0x00000004, 0x0004002b, 0x00000013, 0x000000a4,
0x00000005, 0x00040017, 0x000000ac, 0x00000006, 0x00000004, 0x0003001d, 0x000000ad, 0x000000ac,
0x0003001e, 0x000000ae, 0x000000ad, 0x00040020, 0x000000af, 0x0000000c, 0x000000ae, 0x0004003b,
0x000000af, 0x000000b0, 0x0000000c, 0x0004002b, 0x00000013, 0x000000b1, 0x00000000, 0x0004002b,
0x00000013, 0x000000b2, 0x00000001, 0x0003001d, 0x000000c1, 0x000000ac, 0x0003001e, 0x000000c2,
0x000000c1, 0x00040020, 0x000000c3, 0x0000000c, 0x000000c2, 0x0004003b, 0x000000c3, 0x000000c4,
0x0000000c, 0x00040020, 0x000000c9, 0x0000000c, 0x000000ac, 0x0006002c, 0x00000009, 0x000000cd,
0x0000001d, 0x0000000c, 0x0000000c, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003,
0x000200f8, 0x00000005, 0x000300f7, 0x000000ce, 0x00000000, 0x000300fb, 0x00000019, 0x000000cf,
0x000200f8, 0x000000cf, 0x00050041, 0x0000000d, 0x0000000e, 0x0000000b, 0x0000000c, 0x0004003d,
0x00000006, 0x0000000f, 0x0000000e, 0x00050041, 0x00000015, 0x00000016, 0x00000012, 0x00000014,
0x0004003d, 0x00000006, 0x00000017, 0x00000016, 0x00050084, 0x00000006, 0x00000018, 0x0000000f,
0x00000017, 0x00050041, 0x0000000d, 0x0000001a, 0x0000000b, 0x00000019, 0x0004003d, 0x00000006,
0x0000001b, 0x0000001a, 0x00050080, 0x00000006, 0x0000001c, 0x00000018, 0x0000001b, 0x00050084,
0x00000006, 0x0000001e, 0x0000001c, 0x0000001d, 0x00050041, 0x0000000d, 0x00000020, 0x0000001f,
0x00000019, 0x0004003d, 0x00000006, 0x00000021, 0x00000020, 0x00050080, 0x00000006, 0x00000022,
0x0000001e, 0x00000021, 0x00050041, 0x00000015, 0x00000025, 0x00000012, 0x00000024, 0x0004003d,
0x00000006, 0x00000026, 0x00000025, 0x000500ae, 0x00000027, 0x00000028, 0x00000022, 0x00000026,
0x000300f7, 0x0000002a, 0x00000000, 0x000400fa, 0x00000028, 0x00000029, 0x0000002a, 0x000200f8,
0x00000029, 0x000200f9, 0x000000ce, 0x000200f8, 0x0000002a, 0x00050041, 0x00000015, 0x0000002e,
0x00000012, 0x0000002d, 0x0004003d, 0x00000006, 0x0000002f, 0x0000002e, 0x000500c4, 0x00000006,
0x00000030, 0x0000000c, 0x0000002f, 0x00050041, 0x00000015, 0x00000033, 0x00000012, 0x00000032,
0x0004003d, 0x00000006, 0x00000034, 0x00000033, 0x000500c4, 0x00000006, 0x00000035, 0x0000000c,
0x00000034, 0x00050080, 0x00000006, 0x0000003c, 0x0000002f, 0x00000034, 0x000500c4, 0x00000006,
0x0000003d, 0x00000037, 0x0000003c, 0x00050086, 0x00000006, 0x00000041, 0x00000022, 0x0000003d,
0x00050089, 0x00000006, 0x00000045, 0x00000022, 0x0000003d, 0x000500c2, 0x00000006, 0x00000049,
0x00000045, 0x00000048, 0x000500c7, 0x00000006, 0x0000004d, 0x00000045, 0x0000004c, 0x00050041,
0x00000015, 0x00000050, 0x00000012, 0x0000004f, 0x0004003d, 0x00000006, 0x00000051, 0x00000050,
0x00050041, 0x00000015, 0x00000053, 0x00000012, 0x00000052, 0x0004003d, 0x00000006, 0x00000054,
0x00000053, 0x00050084, 0x00000006, 0x00000055, 0x00000051, 0x00000054, 0x00050086, 0x00000006,
0x00000059, 0x00000041, 0x00000055, 0x00050089, 0x00000006, 0x0000005d, 0x00000041, 0x00000055,
0x00050086, 0x00000006, 0x00000062, 0x0000005d, 0x00000051, 0x00050089, 0x00000006, 0x00000067,
0x0000005d, 0x00000051, 0x00050084, 0x00000006, 0x0000006b, 0x00000067, 0x0000006a, 0x000500c2,
0x00000006, 0x0000006d, 0x0000004d, 0x0000006a, 0x000500c7, 0x00000006, 0x0000006e, 0x0000006d,
0x0000000c, 0x00050084, 0x00000006, 0x00000070, 0x0000006e, 0x0000006f, 0x00050080, 0x00000006,
0x00000071, 0x0000006b, 0x00000070, 0x000500c2, 0x00000006, 0x00000073, 0x0000004d, 0x0000000c,
0x000500c7, 0x00000006, 0x00000074, 0x00000073, 0x0000000c, 0x00050080, 0x00000006, 0x00000075,
0x00000071, 0x00000074, 0x00050084, 0x00000006, 0x00000079, 0x00000062, 0x00000078, 0x00050084,
0x00000006, 0x0000007b, 0x00000079, 0x00000030, 0x00050089, 0x00000006, 0x0000007e, 0x00000049,
0x00000030, 0x00050084, 0x00000006, 0x0000007f, 0x0000007e, 0x00000078, 0x00050080, 0x00000006,
0x00000080, 0x0000007b, 0x0000007f, 0x000500c2, 0x00000006, 0x00000082, 0x0000004d, 0x0000006f,
0x000500c7, 0x00000006, 0x00000084, 0x00000082, 0x00000083, 0x00050084, 0x00000006, 0x00000085,
0x00000084, 0x0000006f, 0x00050080, 0x00000006, 0x00000086, 0x00000080, 0x00000085, 0x000500c7,
0x00000006, 0x00000088, 0x00000045, 0x0000000c, 0x00050080, 0x00000006, 0x00000089, 0x00000086,
0x00000088, 0x00050084, 0x00000006, 0x0000008d, 0x00000059, 0x00000035, 0x00050086, 0x00000006,
0x00000090, 0x00000049, 0x00000030, 0x00050080, 0x00000006, 0x00000091, 0x0000008d, 0x00000090,
0x00050041, 0x00000015, 0x00000094, 0x00000012, 0x00000093, 0x0004003d, 0x00000006, 0x00000095,
0x00000094, 0x000500ae, 0x00000027, 0x00000096, 0x00000075, 0x00000095, 0x000400a8, 0x00000027,
0x00000097, 0x00000096, 0x000300f7, 0x00000099, 0x00000000, 0x000400fa, 0x00000097, 0x00000098,
0x00000099, 0x000200f8, 0x00000098, 0x00050041, 0x00000015, 0x0000009c, 0x00000012, 0x0000009b,
0x0004003d, 0x00000006, 0x0000009d, 0x0000009c, 0x000500ae, 0x00000027, 0x0000009e, 0x00000089,
0x0000009d, 0x000200f9, 0x00000099, 0x000200f8, 0x00000099, 0x000700f5, 0x00000027, 0x0000009f,
0x00000096, 0x0000002a, 0x0000009e, 0x00000098, 0x000400a8, 0x00000027, 0x000000a0, 0x0000009f,
0x000300f7, 0x000000a2, 0x00000000, 0x000400fa, 0x000000a0, 0x000000a1, 0x000000a2, 0x000200f8,
0x000000a1, 0x00050041, 0x00000015, 0x000000a5, 0x00000012, 0x000000a4, 0x0004003d, 0x00000006,
0x000000a6, 0x000000a5, 0x000500ae, 0x00000027, 0x000000a7, 0x00000091, 0x000000a6, 0x000200f9,
0x000000a2, 0x000200f8, 0x000000a2, 0x000700f5, 0x00000027, 0x000000a8, 0x0000009f, 0x00000099,
0x000000a7, 0x000000a1, 0x000300f7, 0x000000aa, 0x00000000, 0x000400fa, 0x000000a8, 0x000000a9,
0x000000aa, 0x000200f8, 0x000000a9, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000aa, 0x00050041,
0x00000015, 0x000000b3, 0x00000012, 0x000000b2, 0x0004003d, 0x00000006, 0x000000b4, 0x000000b3,
0x00050041, 0x00000015, 0x000000b6, 0x00000012, 0x0000009b, 0x0004003d, 0x00000006, 0x000000b7,
0x000000b6, 0x00050084, 0x00000006, 0x000000b8, 0x00000091, 0x000000b7, 0x00050080, 0x00000006,
0x000000ba, 0x000000b8, 0x00000089, 0x00050084, 0x00000006, 0x000000bd, 0x000000ba, 0x00000095,
0x00050080, 0x00000006, 0x000000be, 0x000000b4, 0x000000bd, 0x00050080, 0x00000006, 0x000000c0,
0x000000be, 0x00000075, 0x00050041, 0x00000015, 0x000000c5, 0x00000012, 0x000000b1, 0x0004003d,
0x00000006, 0x000000c6, 0x000000c5, 0x00050080, 0x00000006, 0x000000c8, 0x000000c6, 0x00000022,
0x00060041, 0x000000c9, 0x000000ca, 0x000000c4, 0x000000b1, 0x000000c8, 0x0004003d, 0x000000ac,
0x000000cb, 0x000000ca, 0x00060041, 0x000000c9, 0x000000cc, 0x000000b0, 0x000000b1, 0x000000c0,
0x0003003e, 0x000000cc, 0x000000cb, 0x000200f9, 0x000000ce, 0x000200f8, 0x000000ce, 0x000100fd,
0x00010038,
};
runtime/cfg.c
+1 -1
View File
@@ -40,7 +40,7 @@ static const char *const known[] = {
"BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y",
"BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA",
"COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT",
"DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
"DESC_TEMPLATE", "DESWIZZLE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
"DUMP_LOG", "DYN_DELTA", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES",
"FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FRAME_QUEUE",
"FREEZE_IDLE", "FREE_CORES",
runtime/shaders/bl_deswizzle.comp
+37
View File
@@ -0,0 +1,37 @@
#version 450
// Block-linear to linear copy of one texture level and layer on the GPU (vk_deswizzle.c), the same layout as
// tex_deswizzle.c nvn_bl_offset: GOB = 64 bytes x 8 rows; a block is 1 GOB wide, 2^th GOBs high and 2^td deep, its
// GOBs ordered y first, then z; blocks fill the level x first, then y, then z.
// One invocation per 16-byte segment of the block-linear storage, in storage order, so the reads (from the pool's
// imported buffer: guest memory, over PCIe on a discrete GPU) are contiguous; the writes go to a device-local buffer
// with rows of pitch16 segments. Segments outside the level's rows, slices or row width are padding and skipped.
layout(local_size_x = 64) in;
layout(std430, set = 0, binding = 0) readonly buffer Src { uvec4 src[]; };
layout(std430, set = 0, binding = 1) writeonly buffer Dst { uvec4 dst[]; };
layout(push_constant) uniform Push {
uint src_base; // first segment of the level and layer in the src binding
uint dst_base; // first segment of its linear copy in the dst binding
uint segs; // segments to visit: whole blocks covering the level
uint pitch16; // segments per linear row (row bytes rounded up to 16)
uint rows, slices, th, td;
uint gobs_x, blocks_y;
uint nx; // workgroups along x (the dispatch is 2D when one dimension is not enough)
} pc;
void main() {
uint s = (gl_WorkGroupID.y * pc.nx + gl_WorkGroupID.x) * 64u + gl_LocalInvocationID.x;
if (s >= pc.segs) return;
uint bh = 1u << pc.th, bd = 1u << pc.td;
uint per_block = 32u << (pc.th + pc.td); // 512-byte GOBs of 32 segments each
uint block = s / per_block, inb = s % per_block;
uint gob = inb >> 5u, ing = inb & 31u;
uint per_layer = pc.gobs_x * pc.blocks_y;
uint bz = block / per_layer, rem = block % per_layer;
uint by = rem / pc.gobs_x, bx = rem % pc.gobs_x;
// Inside a GOB, segment bits: 4 = x % 64 / 32, 3..2 = y % 8 / 2, 1 = x % 32 / 16, 0 = y % 2.
uint xseg = bx * 4u + ((ing >> 4u) & 1u) * 2u + ((ing >> 1u) & 1u);
uint y = by * 8u * bh + (gob % bh) * 8u + ((ing >> 2u) & 3u) * 2u + (ing & 1u);
uint z = bz * bd + gob / bh;
if (xseg >= pc.pitch16 || y >= pc.rows || z >= pc.slices) return;
dst[pc.dst_base + (z * pc.rows + y) * pc.pitch16 + xseg] = src[pc.src_base + s];
}
runtime/vk.h
+27
View File
@@ -329,6 +329,12 @@ void vk_defer_image(VkImage img, VmaAllocation alloc); // alloc NULL: vkDestro
void vk_defer_view(VkImageView view);
void vk_exec_image_gone(VkImage image); // destroy_tex, texture lock held: the replay's views of image go with it
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc);
// Device-local scratch for the upload batch (storage, transfer src and dst), from a pool of its own (vk_memory.c).
VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc);
// Staging and scratch bytes the open upload batch holds until it completes. Large texture loads submit the batch early
// (vk_texture.c UPLOAD_HELD_MAX) so the GPU starts on it and its buffers come back before the pools need new blocks.
VkDeviceSize vk_upload_held(void);
void vk_upload_pools_init(void); // vk_device.c, after the allocator: the staging and scratch pools and their blocks
// ---- ASTC (vk_astc.c). TOTK_ASTC = auto | gpu | native | cpu | check; see vk_astc.c.
enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK };
int vk_astc_mode(void); // decided once per device (logged); VK_ASTC_CPU before vk_init
@@ -352,6 +358,27 @@ void vk_astc_time_enable(int on);
int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us); // totals so far
void vk_astc_time_summary(void); // logs the totals
void vk_astc_release(void); // vk_device.c, before the device goes: frees the decoder's objects
// ---- Block-linear deswizzle on the GPU (vk_deswizzle.c). TOTK_DESWIZZLE = auto | gpu | cpu | check.
enum { VK_DESWZ_CPU, VK_DESWZ_GPU, VK_DESWZ_CHECK };
int vk_deswz_mode(void); // decided once per device (logged); VK_DESWZ_CPU before vk_init
// One level and layer: its storage starts at byte src of the bound source range and covers segs 16-byte segments
// (whole blocks, cut at the storage end); its linear copy goes to byte dst of the destination, rows of pitch16 segments,
// rows x slices of them. th, td, gobs_x, blocks_y: as tex_deswizzle.c nvn_bl_offset.
typedef struct {
VkDeviceSize src, dst;
uint32_t segs, pitch16, rows, slices, th, td, gobs_x, blocks_y;
} VkDeswzRegion;
// Records into cb a dispatch per region reading [src_off, src_off + src_size) of src (a storage buffer) and writing
// dst (device-local, dst_size bytes). The caller records the barriers around it. 0 when nothing was recorded: no
// pipeline, src_off not 16-byte aligned, a range over the device's storage-buffer limit, a region outside the ranges.
int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst,
VkDeviceSize dst_size, const VkDeswzRegion *rg, int n);
// 1 when vk_deswz_record will take these regions (their dst offsets aside) with destinations of up to max_dst bytes.
int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst);
void vk_deswz_release(void); // vk_device.c, before the device goes
// 1 when sp (from vk_mem_resolve) is an imported pool's buffer over guest memory itself, so a GPU read in a submission
// sees every CPU write made before the submit; 0 for device memory and shadow storage (synced after the upload batch).
int vk_mem_span_live(const VkMemSpan *sp);
void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value
void vk_mem_collect(void); // frees what has completed
typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats;
runtime/vk_astc.c
+1 -5
View File
@@ -463,13 +463,9 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons
cap_in = (cap_in + al - 1) / al * al;
cap_out = (cap_out + al - 1) / al * al;
if (!bc) cap_bc = 0;
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = cap_in + cap_out + cap_bc,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
VkBuffer sb;
VmaAllocation sa;
if (vmaCreateBuffer(c->vma, &bi, &aci, &sb, &sa, NULL) != VK_SUCCESS) {
if (vk_scratch_buffer(cap_in + cap_out + cap_bc, &sb, &sa) != VK_SUCCESS) {
if (plat_inc32(&n_fail) <= 16)
rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded",
(unsigned long long)(cap_in + cap_out));
runtime/vk_deswizzle.c
+176
View File
@@ -0,0 +1,176 @@
// Block-linear textures deswizzled by the GPU (runtime/shaders/bl_deswizzle.comp): vk_texture.c upload binds the
// texture's storage in its pool's imported buffer and lets a compute pass write the linear copy that a buffer-to-image
// copy (or the ASTC decoder) then takes, instead of tex_deswizzle.c walking every 16 bytes on the replay thread
// (50-236 ms frames on entering new areas, 2026-11-07 play sessions). The same design as MacNativeGames' TotK port
// (d2r_gfx nvn_mtl.m: deswizzle from the pool at GPU execution time), written for Vulkan from our CPU layout code.
//
// TOTK_DESWIZZLE = auto (default) / gpu: the compute pass where the storage is an imported pool (vk_mem_span_live),
// the CPU otherwise; cpu: always the CPU; check: gpu, and every upload is also deswizzled on the CPU and compared
// (waits for the GPU; for testing only).
#include <stdlib.h>
#include <string.h>
#include "bl_deswizzle_spv.h"
#include "cfg.h"
#include "host.h"
#include "vk.h"
#include "vk_rec.h"
#include "platform/platform.h"
typedef struct {
uint32_t src_base, dst_base, segs, pitch16, rows, slices, th, td, gobs_x, blocks_y, nx;
} Push;
static PlatMutex g_lock = PLAT_MUTEX_INIT;
static struct {
VkDevice dev;
int mode; // VK_DESWZ_*; -1 before the first vk_deswz_mode with a device
VkDescriptorSetLayout dsl;
VkPipelineLayout layout;
VkPipeline pipe;
PFN_vkCmdPushDescriptorSetKHR push;
} G = {.mode = -1};
static volatile int64_t n_regions, n_bytes;
static int create_pipeline(VkCtx *c) {
G.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR");
if (!G.push) return 0;
VkDescriptorSetLayoutBinding b[2];
for (int i = 0; i < 2; i++)
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
.bindingCount = 2, .pBindings = b};
if (vkCreateDescriptorSetLayout(c->device, &dci, NULL, &G.dsl) != VK_SUCCESS) return 0;
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push)};
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
.pSetLayouts = &G.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
if (vkCreatePipelineLayout(c->device, &li, NULL, &G.layout) != VK_SUCCESS) return 0;
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof bl_deswizzle_spv,
.pCode = bl_deswizzle_spv};
VkShaderModule m = VK_NULL_HANDLE;
if (vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) return 0;
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
.layout = G.layout};
VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &G.pipe);
vkDestroyShaderModule(c->device, m, NULL);
if (r != VK_SUCCESS) G.pipe = VK_NULL_HANDLE;
return r == VK_SUCCESS;
}
int vk_deswz_mode(void) {
VkCtx *c = vk_ctx();
if (!c) return VK_DESWZ_CPU;
plat_mutex_lock_shared(&g_lock);
int m = G.dev == c->device ? G.mode : -1;
plat_mutex_unlock_shared(&g_lock);
if (m >= 0) return m;
plat_mutex_lock(&g_lock);
if (G.dev != c->device) { // the old device's objects went with it (probes recreate the device)
memset(&G, 0, sizeof G);
G.dev = c->device;
G.mode = -1;
}
if (G.mode < 0) {
const char *e = cfg_get("TOTK_DESWIZZLE");
if (!e || !*e) e = "auto";
int want = !strcmp(e, "cpu") || !strcmp(e, "0") ? VK_DESWZ_CPU : !strcmp(e, "check") ? VK_DESWZ_CHECK : VK_DESWZ_GPU;
if (strcmp(e, "auto") && strcmp(e, "gpu") && strcmp(e, "1") && strcmp(e, "cpu") && strcmp(e, "0") &&
strcmp(e, "check"))
rt_log("vk: TOTK_DESWIZZLE=%s is not auto, gpu, cpu or check: auto", e);
if (want != VK_DESWZ_CPU && !create_pipeline(c)) {
rt_log("vk: the deswizzle compute pass is unavailable on %s: textures deswizzle on the CPU", c->props.deviceName);
want = VK_DESWZ_CPU;
}
G.mode = want;
rt_log("vk: block-linear textures deswizzle on the %s%s", want == VK_DESWZ_CPU ? "CPU" : "GPU where the storage is "
"an imported pool", want == VK_DESWZ_CHECK ? ", checked against the CPU (TOTK_DESWIZZLE=check)" : "");
}
m = G.mode;
plat_mutex_unlock(&g_lock);
return m;
}
static uint32_t max_groups_x(const VkCtx *c) {
uint32_t m = c->props.limits.maxComputeWorkGroupCount[0];
return m > 65535 ? 65535 : m;
}
// The bound source range starts at the alignment below src_off.
static VkDeviceSize src_lo(const VkCtx *c, VkDeviceSize src_off) {
VkDeviceSize al = c->props.limits.minStorageBufferOffsetAlignment;
if (al < 16) al = 16;
return src_off / al * al;
}
int vk_deswz_fits(VkDeviceSize src_off, VkDeviceSize src_size, const VkDeswzRegion *rg, int n, VkDeviceSize max_dst) {
VkCtx *c = vk_ctx();
if (!c || !n || (src_off & 15) || vk_deswz_mode() == VK_DESWZ_CPU) return 0;
VkDeviceSize range = src_off + src_size - src_lo(c, src_off), lim = c->props.limits.maxStorageBufferRange;
if (range > lim || max_dst > lim) return 0;
uint32_t maxg = max_groups_x(c);
for (int i = 0; i < n; i++) { // the 32-bit segment indices of the shader stay inside the bindings
uint64_t groups = ((uint64_t)rg[i].segs + 63) / 64, out = (uint64_t)rg[i].pitch16 * rg[i].rows * rg[i].slices;
if ((rg[i].src & 15) || rg[i].src + (uint64_t)rg[i].segs * 16 > src_size || !rg[i].segs ||
(src_off - src_lo(c, src_off) + rg[i].src) / 16 + rg[i].segs > UINT32_MAX || out > UINT32_MAX ||
out * 16 > max_dst || (groups + maxg - 1) / maxg > c->props.limits.maxComputeWorkGroupCount[1])
return 0;
}
return 1;
}
int vk_deswz_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, VkDeviceSize src_size, VkBuffer dst,
VkDeviceSize dst_size, const VkDeswzRegion *rg, int n) {
VkCtx *c = vk_ctx();
if (!vk_deswz_fits(src_off, src_size, rg, n, dst_size)) return 0;
for (int i = 0; i < n; i++)
if ((rg[i].dst & 15) || rg[i].dst + (uint64_t)rg[i].pitch16 * 16 * rg[i].rows * rg[i].slices > dst_size) return 0;
plat_mutex_lock_shared(&g_lock);
VkPipeline pipe = G.dev == c->device ? G.pipe : VK_NULL_HANDLE;
plat_mutex_unlock_shared(&g_lock);
if (!pipe) return 0;
VkDeviceSize lo = src_lo(c, src_off), range = src_off + src_size - lo;
uint32_t maxg = max_groups_x(c);
VkDescriptorBufferInfo dbi[2] = {{src, lo, range}, {dst, 0, dst_size}};
VkWriteDescriptorSet w[2];
for (int i = 0; i < 2; i++)
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &dbi[i]};
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe);
rec_push(G.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, G.layout, 0, 2, w);
uint64_t bytes = 0;
for (int i = 0; i < n; i++) {
const VkDeswzRegion *r = &rg[i];
uint64_t groups = ((uint64_t)r->segs + 63) / 64;
uint32_t nx = groups < maxg ? (uint32_t)groups : maxg, ny = (uint32_t)((groups + nx - 1) / nx);
Push pc = {.src_base = (uint32_t)((src_off - lo + r->src) / 16), .dst_base = (uint32_t)(r->dst / 16),
.segs = r->segs, .pitch16 = r->pitch16, .rows = r->rows, .slices = r->slices, .th = r->th, .td = r->td,
.gobs_x = r->gobs_x, .blocks_y = r->blocks_y, .nx = nx};
vkCmdPushConstants(cb, G.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
vkCmdDispatch(cb, nx, ny, 1);
bytes += (uint64_t)r->segs * 16;
}
int64_t before = plat_add64(&n_regions, n) - n;
plat_add64(&n_bytes, (int64_t)bytes);
if (!before || (before + n) / 4096 != before / 4096)
rt_log("vk: deswizzle: %lld regions, %.1f MiB of block-linear storage deswizzled on the GPU so far",
(long long)(before + n), (double)n_bytes / 1048576.0);
return 1;
}
void vk_deswz_release(void) {
VkCtx *c = vk_ctx();
plat_mutex_lock(&g_lock);
if (c && G.dev == c->device) {
if (G.pipe) vkDestroyPipeline(c->device, G.pipe, NULL);
if (G.layout) vkDestroyPipelineLayout(c->device, G.layout, NULL);
if (G.dsl) vkDestroyDescriptorSetLayout(c->device, G.dsl, NULL);
}
memset(&G, 0, sizeof G);
G.mode = -1;
plat_mutex_unlock(&g_lock);
}
runtime/vk_device.c
+2 -1
View File
@@ -376,6 +376,7 @@ static void destroy_all(void) {
gfx_window_release_vk();
if (g_vk.device) vkDeviceWaitIdle(g_vk.device);
if (g_vk.device) vk_astc_release();
if (g_vk.device) vk_deswz_release();
if (g_vk.device) vk_crumbs_release();
if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL);
vk_headroom_stop();
@@ -397,7 +398,7 @@ int vk_init(void) {
plat_mutex_lock(&g_lock);
if (!g_ready) {
memset(&g_vk, 0, sizeof g_vk);
if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_headroom_start(), vk_dlaa_init();
if (create_instance() && pick_device() && create_device()) g_ready = 1, vk_upload_pools_init(), vk_headroom_start(), vk_dlaa_init();
else destroy_all();
}
int ok = g_ready;
runtime/vk_memory.c
+66
View File
@@ -416,6 +416,10 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) {
return resolve_locked(addr, size, out);
}
int vk_mem_span_live(const VkMemSpan *sp) {
return sp->buffer && sp->pool && !sp->alloc && !shadow_on && ((const PoolRec *)sp->pool)->kind == POOL_IMPORT;
}
// The pool index lookup under the lock; a non-overlapped import or physical pool goes into the address's cache slot.
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) {
memset(out, 0, sizeof *out);
@@ -2152,6 +2156,7 @@ typedef struct {
uint64_t seq;
Staging *stg;
int nstg, cap;
VkDeviceSize held; // staging and scratch bytes freed when the batch completes (vk_upload_held)
} Batch;
enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA };
@@ -2305,10 +2310,68 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
else {
if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg);
open_b.stg[open_b.nstg++] = (Staging){b, a};
open_b.held += bi.size;
}
return info.pMappedData;
}
// Device-local scratch of the upload batch (the GPU deswizzle's linear copies, the ASTC decoder's blocks and texels):
// a pool of its own, like staging, so the replay thread neither waits on the default pool's lock while another thread
// makes a block there (the headroom thread: about 8 ms for 256 MiB) nor makes one itself for a few MiB. One block
// of SCR_BLOCK is kept from the first use; larger buffers than half a block go to the default pool.
#define SCR_BLOCK (64ull << 20)
static VmaPool scr_pool;
static int scr_pool_tried;
VkResult vk_scratch_buffer(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) {
VkCtx *c = vk_ctx();
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size ? size : 16,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
if (!__atomic_load_n(&scr_pool_tried, __ATOMIC_ACQUIRE)) {
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
if (!scr_pool_tried) {
// two blocks with 8 GiB or more of device-local memory: fight2's first world frame deswizzles about
// 280 MiB, more than the GPU finishes while the replay records, and the pool grew by four blocks there
VkDeviceSize vram = 0;
for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++)
if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT && c->mem_props.memoryHeaps[h].size > vram)
vram = c->mem_props.memoryHeaps[h].size;
uint32_t type;
VmaPoolCreateInfo pi = {.blockSize = SCR_BLOCK, .minBlockCount = vram >= (8ull << 30) ? 2 : 1};
if (vmaFindMemoryTypeIndexForBufferInfo(c->vma, &bi, &aci, &type) == VK_SUCCESS) {
pi.memoryTypeIndex = type;
if (vmaCreatePool(c->vma, &pi, &scr_pool) != VK_SUCCESS) scr_pool = NULL;
}
rt_log("vk: upload scratch pool %s (%u blocks of %llu MiB kept)", scr_pool ? "created" : "unavailable: default pool",
(unsigned)pi.minBlockCount, (unsigned long long)(SCR_BLOCK >> 20));
__atomic_store_n(&scr_pool_tried, 1, __ATOMIC_RELEASE);
}
plat_rmutex_unlock(&cmd_lock);
}
if (scr_pool && bi.size <= SCR_BLOCK / 2) aci.pool = scr_pool;
VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, buf, alloc, NULL);
if (r == VK_SUCCESS && open_active) open_b.held += bi.size; // the caller defers its free to the open batch
return r;
}
VkDeviceSize vk_upload_held(void) { return open_active ? open_b.held : 0; }
// At device creation: the staging and scratch pools with their kept blocks, made through a first small buffer each
// (freed at once), so the first world frame does not allocate them (staging was first used there once the GPU took
// the early texture uploads).
void vk_upload_pools_init(void) {
VkCtx *c = vk_ctx();
if (!c) return;
VkBuffer b;
VmaAllocation a;
vk_staging(16, &b, &a);
vmaDestroyBuffer(c->vma, b, a);
if (vk_scratch_buffer(16, &b, &a) == VK_SUCCESS) vmaDestroyBuffer(c->vma, b, a);
}
uint64_t vk_upload_flush(int wait) {
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
@@ -2434,6 +2497,9 @@ void vk_mem_shutdown(void) {
if (stg_pool) vmaDestroyPool(c->vma, stg_pool); // after retire: no staging buffer is left in it
stg_pool = NULL;
stg_pool_tried = 0;
if (scr_pool) vmaDestroyPool(c->vma, scr_pool); // deferred scratch buffers went with collect_locked above
scr_pool = NULL;
scr_pool_tried = 0;
plat_rmutex_unlock(&cmd_lock);
plat_mutex_lock(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
runtime/vk_texture.c
+212 -4
View File
@@ -819,15 +819,20 @@ static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar,
// large ASTC array decodes to RGBA8 in one go: a staging buffer of 676682336 bytes failed (-2) on an Intel UHD with
// 8 GB of shared memory and the game closed before the title screen.
#define UPLOAD_PART (32ull << 20) // fits the staging pool (vk_staging takes up to half of its 64 MiB block)
// The open upload batch is submitted between chunks (and parts of the CPU path) once its staging and scratch pass this:
// on fight2's first world frame one batch took hundreds of MiB, and the staging and scratch pools grew by 64 MiB blocks
// on the replay thread. Submitting earlier keeps the queue order (the batch still precedes the part that samples the
// images) and lets the GPU finish and return the buffers sooner.
#define UPLOAD_HELD_MAX (48ull << 20)
// Copies the whole storage into the image (every level and layer the image has).
static void upload(Tex *e) {
static int upload_gpu(Tex *e);
// The CPU side of upload: deswizzles (and decodes or expands) into staging buffers, then copies them into the image.
static void upload_cpu(Tex *e) {
static volatile int32_t n_bad, n_undecoded, n_split;
const tex_t *t = &e->t;
const nvn_texdesc *d = &t->d;
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
if (!bpb || !t->mem) return;
uint64_t p0 = perf_begin(), u0 = kern_now_ns();
int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D;
uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers;
int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb;
@@ -908,6 +913,7 @@ static void upload(Tex *e) {
}
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
vk_upload_end();
if (!acheck && vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0);
}
if (parts > 1)
log_limited(&n_split, "texture %#llx fmt %#x %ux%u, %u levels, %u layers: %.1f MiB staged in %d parts",
@@ -924,7 +930,201 @@ static void upload(Tex *e) {
free(raw_copy);
free(ar);
free(rg);
}
// Block-linear storage in an imported pool, deswizzled by the GPU (vk_deswizzle.c): the replay thread only records
// the dispatches, the image copies and (ASTC) the decode into the upload batch the CPU path also uses, which runs before
// the part that samples the image. The GPU reads the pool when the batch runs, as the Switch's GPU reads it at
// execution; a CPU write after make_fresh's write-watch reset still marks the pages and brings another upload. Each
// chunk of regions gets a device-local scratch buffer of at most DESWZ_CHUNK bytes (one region, when larger), freed
// once the batch completes. TOTK_DESWIZZLE=check compares every chunk with the CPU deswizzle (waits for the GPU).
// 0 when not handled, before anything is recorded: the CPU path takes the texture.
#define DESWZ_CHUNK (32ull << 20)
// Uploads by path since the last perf summary (TOTK_TRACE=perf): 0 GPU, else why the CPU took it.
enum { UP_GPU, UP_MODE, UP_LINEAR, UP_RGB, UP_ASTC_CPU, UP_BPB, UP_NOT_LIVE, UP_GEOM, UP_LIMITS, UP_SCRATCH, UP_WHY };
static const char *const up_why_name[UP_WHY] = {"gpu", "cpu mode", "linear", "rgb", "astc on cpu", "bytes per block",
"not an imported pool", "layout", "device limits", "scratch"};
static uint64_t up_n[UP_WHY], up_bytes[UP_WHY], up_ns[UP_WHY];
static int up_why; // upload_gpu's reason for returning 0 (texture lock held)
static int upload_gpu(Tex *e) {
static volatile int32_t n_scratch_fail, n_undecoded, n_bad;
static volatile int64_t n_checked, n_differ;
const tex_t *t = &e->t;
const nvn_texdesc *d = &t->d;
int mode = vk_deswz_mode();
up_why = mode == VK_DESWZ_CPU ? UP_MODE : nvn_tex_is_linear(d) ? UP_LINEAR : e->f.expand_rgb ? UP_RGB : UP_GPU;
if (up_why) return 0;
int astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu();
if (astc && (!agpu || vk_astc_mode() == VK_ASTC_CHECK)) return up_why = UP_ASTC_CPU, 0; // or ASTC's check mode
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
if (!bpb || 16 % bpb) return up_why = UP_BPB, 0; // the rows of the linear copy are padded to 16 bytes
VkMemSpan sp;
if (!vk_mem_resolve(e->pub.addr, t->L.size, &sp) || !vk_mem_span_live(&sp)) return up_why = UP_NOT_LIVE, 0;
int one_d = e->pub.type == VK_IMAGE_TYPE_1D, is3d = d->target == NVN_TEXTURE_TARGET_3D;
uint32_t levels = e->pub.levels, layers = e->pub.layers, n = levels * layers, k = 0;
VkDeswzRegion *dr = calloc(n, sizeof *dr);
VkBufferImageCopy *rg = calloc(n, sizeof *rg);
VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL;
if (!dr || !rg || (agpu && !ar)) rt_fatal("vk: out of memory (texture upload regions)");
int ok = 1;
VkDeviceSize max_dst = DESWZ_CHUNK;
for (uint32_t lv = 0; ok && lv < levels; lv++)
for (uint32_t ly = 0; ok && ly < layers; ly++) {
uint32_t row_bytes, rows, slices, gbpb;
uint64_t base;
unsigned th, td;
uint32_t w = mip(d->width, lv), h = one_d ? 1 : mip(d->height, lv), sl = is3d ? mip(d->depth, lv) : 1;
if (nvn_deswizzle_geom(d, lv, ly, &row_bytes, &rows, &slices, &base, &th, &td, &gbpb) || gbpb != (uint32_t)bpb ||
base >= t->L.size || rows != (h + bh - 1) / bh || slices != sl) {
ok = 0;
break;
}
uint64_t bhg = 1ull << th, bdg = 1ull << td, gobs_x = (row_bytes + 63) / 64;
uint64_t blocks_y = (rows + 8 * bhg - 1) / (8 * bhg), blocks_z = (slices + bdg - 1) / bdg;
uint64_t segs = gobs_x * blocks_y * blocks_z * (32ull << (th + td)), room = (t->L.size - base) / 16;
if (segs > room) segs = room;
uint32_t pitch16 = (row_bytes + 15) / 16;
VkDeviceSize out = (VkDeviceSize)pitch16 * 16 * rows * slices;
if (segs > UINT32_MAX || gobs_x * blocks_y > UINT32_MAX) { ok = 0; break; }
if (out > max_dst) max_dst = out;
dr[k] = (VkDeswzRegion){.src = base, .segs = (uint32_t)segs, .pitch16 = pitch16, .rows = rows, .slices = slices,
.th = th, .td = td, .gobs_x = (uint32_t)gobs_x, .blocks_y = (uint32_t)blocks_y};
rg[k] = (VkBufferImageCopy){.bufferRowLength = pitch16 * 16 / (uint32_t)bpb * (uint32_t)bw,
.bufferImageHeight = rows * (uint32_t)bh,
.imageSubresource = {e->pub.aspect, lv, ly, 1}, .imageExtent = {w, h, sl}};
if (agpu) ar[k] = (VkAstcRegion){.w = w, .h = h, .slices = sl, .level = lv, .layer = ly};
k++;
}
if (!ok || k != n || !vk_deswz_fits(sp.offset, t->L.size, dr, (int)n, max_dst)) {
free(dr), free(rg), free(ar);
return up_why = !ok || k != n ? UP_GEOM : UP_LIMITS, 0;
}
VkCtx *c = vk_ctx();
VkCommandBuffer cb = vk_upload_begin();
// Earlier GPU writes into the pool (copies, shaders) before the reads; the image's earlier uses before its copy.
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) {
VkDeviceSize size = 0;
for (k1 = k0; k1 < n; k1++) {
VkDeviceSize out = (VkDeviceSize)dr[k1].pitch16 * 16 * dr[k1].rows * dr[k1].slices;
if (k1 > k0 && size + out > DESWZ_CHUNK) break;
dr[k1].dst = size, rg[k1].bufferOffset = size;
if (agpu) ar[k1].src = size;
size += out;
}
VkBuffer sb;
VmaAllocation sa;
if (vk_scratch_buffer(size, &sb, &sa) != VK_SUCCESS) {
log_limited(&n_scratch_fail, "texture %#llx: deswizzle scratch of %llu bytes failed%s",
(unsigned long long)e->pub.addr, (unsigned long long)size,
k0 ? "; its later levels left as they were" : "; deswizzled on the CPU");
if (!k0) {
vk_upload_end();
free(dr), free(rg), free(ar);
return up_why = UP_SCRATCH, 0;
}
break;
}
if (!vk_deswz_record(cb, sp.buffer, sp.offset, t->L.size, sb, size, dr + k0, (int)(k1 - k0))) {
log_limited(&n_bad, "texture %#llx: deswizzle of levels %u-%u not recorded; left as they were",
(unsigned long long)e->pub.addr, k0, k1 - 1); // vk_deswz_fits passed: not expected
vk_defer_buffer(sb, sa);
break;
}
barrier(cb, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
if (agpu) {
uint64_t a0 = perf_begin();
if (!vk_astc_record(cb, sb, size, ar + k0, (int)(k1 - k0), e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb,
e->f.astc_bc, e->pub.image, e->pub.aspect, VK_NULL_HANDLE))
log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was",
(unsigned long long)e->pub.addr, d->format, d->width, d->height);
perf_end(PERF_ASTC, a0);
} else {
vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, k1 - k0, rg + k0);
}
if (mode == VK_DESWZ_CHECK) { // the linear copy back to the host, compared with the CPU deswizzle
VkBuffer rb;
VmaAllocation ra;
const uint8_t *rmap = vk_staging(size, &rb, &ra);
VkBufferCopy cp = {0, 0, size};
vkCmdCopyBuffer(cb, sb, rb, 1, &cp);
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
VK_ACCESS_2_HOST_READ_BIT);
vk_defer_buffer(sb, sa);
vk_upload_end();
vk_upload_flush(1);
vmaInvalidateAllocation(c->vma, ra, 0, VK_WHOLE_SIZE);
for (uint32_t i = k0; i < k1; i++) {
const VkDeswzRegion *r = &dr[i];
size_t pitch = (size_t)r->pitch16 * 16, bytes = pitch * r->rows * r->slices;
uint8_t *cpu = malloc(bytes);
if (!cpu) rt_fatal("vk: out of memory (deswizzle check)");
uint32_t row_bytes = 0, rows, slices, gbpb;
uint64_t base;
unsigned th, td;
nvn_deswizzle_geom(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, &row_bytes,
&rows, &slices, &base, &th, &td, &gbpb);
nvn_deswizzle(d, rg[i].imageSubresource.mipLevel, rg[i].imageSubresource.baseArrayLayer, t->mem, cpu, pitch);
uint64_t bad = 0, first = UINT64_MAX;
for (uint64_t row = 0; row < (uint64_t)r->rows * r->slices; row++)
if (memcmp(cpu + row * pitch, rmap + r->dst + row * pitch, row_bytes)) {
bad++;
if (first == UINT64_MAX) first = row;
}
free(cpu);
plat_add64(&n_checked, 1);
if (bad) {
plat_add64(&n_differ, 1);
rt_log("vk: deswizzle check: texture %#llx fmt %#x %ux%u level %u layer %u: %llu of %u rows differ, the "
"first %llu (th %u td %u, %u rows of %u bytes, %u slices)", (unsigned long long)e->pub.addr,
d->format, d->width, d->height, rg[i].imageSubresource.mipLevel,
rg[i].imageSubresource.baseArrayLayer, (unsigned long long)bad, r->rows * r->slices,
(unsigned long long)first, r->th, r->td, r->rows, row_bytes, r->slices);
}
}
vmaDestroyBuffer(c->vma, rb, ra);
if ((n_checked & 1023) == 0 || n_checked < 2)
rt_log("vk: deswizzle check: %lld regions compared, %lld differ", (long long)n_checked, (long long)n_differ);
cb = vk_upload_begin();
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
continue;
}
vk_defer_buffer(sb, sa);
if (k1 < n && vk_upload_held() > UPLOAD_HELD_MAX) { // submit what is recorded, go on in a new batch
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW);
vk_upload_end();
vk_upload_flush(0);
cb = vk_upload_begin();
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
}
}
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
VK_ACCESS_2_TRANSFER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, ALL_RW);
vk_upload_end();
if (vk_upload_held() > UPLOAD_HELD_MAX) vk_upload_flush(0);
free(dr), free(rg), free(ar);
return 1;
}
// Copies the whole storage into the image (every level and layer the image has): on the GPU where it can
// (upload_gpu), else on the CPU.
static void upload(Tex *e) {
const tex_t *t = &e->t;
const nvn_texdesc *d = &t->d;
int bw, bh;
if (!nvn_fmt_bpb(d->format, &bw, &bh) || !t->mem) return;
uint64_t p0 = perf_begin(), u0 = kern_now_ns();
int why = UP_GPU;
if (!upload_gpu(e)) why = up_why, upload_cpu(e);
uint64_t dt = kern_now_ns() - u0;
up_n[why]++, up_bytes[why] += t->L.size, up_ns[why] += dt;
if (e->uploaded) {
n_reuploads++, perf_count(PERF_CNT_TEX_REUP);
ru_n++, ru_bytes += t->L.size, ru_ns += dt;
@@ -1404,6 +1604,14 @@ void vk_texture_new_submit(void) {
(unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6,
(unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6);
ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = 0;
char ub[512];
size_t uo = 0;
for (int i = 0; i < UP_WHY; i++)
if (up_n[i] && uo < sizeof ub)
uo += (size_t)snprintf(ub + uo, sizeof ub - uo, "%s %s %llu (%.1f MiB, %.2f ms)", uo ? "," : "",
up_why_name[i], (unsigned long long)up_n[i], up_bytes[i] / 1048576.0, up_ns[i] / 1e6);
if (uo) rt_log("perf: frame %llu: texture uploads by path:%s", (unsigned long long)f * 60, ub);
memset(up_n, 0, sizeof up_n), memset(up_bytes, 0, sizeof up_bytes), memset(up_ns, 0, sizeof up_ns);
uint64_t ck, calls, untr;
vk_mem_snap_stats(&ck, &calls, &untr);
rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu",