diff --git a/docs/architecture.md b/docs/architecture.md index 949aff73..49a977aa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -118,8 +118,13 @@ Every guest thread is a host thread. TotK assumes Horizon's scheduler: three app - DLAA (`vk_dlaa.c`, `dlaa`, off by default). NVIDIA DLSS at the scene's own size, through the driver's NGX core loaded at run time (no SDK code compiled in). It runs on the R11G11B10F light target right before the bloom chain's first pass, so post effects and the HUD read the anti-aliased picture. The scene's draws get a sub-pixel Halton viewport - jitter; motion vectors are camera-only, reprojected from the scene depth through this and last frame's - view-projection of the scene camera block (constant bank c4). `fxaa` switches the game's own FXAA pass. + jitter; camera motion vectors are reprojected from the scene depth through this and last frame's view-projection of + the scene camera block (constant bank c4), in double precision on the CPU. Per-object motion (`dlaa_objmv`, + `vk_mvprev.c`, the per-object section of `vk_exec_draw.c`, translator patch 0015): solid scene draws whose position + reads an object's own uniform blocks (bones c5, place c7, c6) are drawn a second time with a vertex shader variant + that computes last frame's position from last frame's blocks (the game writes them into two copies that take turns; + the uniform snapshot keeps both) and writes the difference into the motion image. `TOTK_MV_CHECK` measures the + vectors against the picture, `TOTK_DLAA_TIMES` times the passes on the GPU. `fxaa` switches the game's own FXAA pass. - Occlusion queries never block the CPU; a result that is not ready counts as visible. - ASTC textures are uploaded as they are where the device reports `textureCompressionASTC_LDR`, and decoded by a compute shader everywhere else (`vk_astc.c`, `TOTK_ASTC`). The CPU decoder (`tex_astc*.c`) is the fallback and the test oracle. - Draw state. A draw whose program, bound state and textures repeat reuses the previous draw's shaders and skips rebuilding its descriptor sets; descriptor sets are written through update templates (`vk_exec_draw.c`, `vk_pipeline.c`). diff --git a/runtime/cfg.c b/runtime/cfg.c index 7fd33b50..f977aa92 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -37,13 +37,15 @@ static const char *const known[] = { "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", - "DESC_TEMPLATE", "DEVICE_PROFILE", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLSS", "DUMP", "DUMP_BUF", "DUMP_DIR", + "DESC_TEMPLATE", "DEVICE_PROFILE", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FREEZE_IDLE", "FREE_CORES", "FULLSCREEN", "FXAA", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MENU_LANG", "MENU_TEST", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", + "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", + "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", @@ -55,7 +57,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SWKBD_TEST", "TEST_PAD", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", }; @@ -97,6 +99,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_EXACT", "diagnostics"}, + {"MV_SURVEY", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -111,7 +114,7 @@ static const CfgMeta metas[] = { {"PERF_OVERLAY", "off", "off|fps|full", LIVE}, {"OUTPUT_RES", "1920x1080", NULL, RESTART}, {"RENDER_SCALE", "game", "game|50|67|75|100|125|150|200", RESTART}, {"UPSCALER", "fsr", "fsr|bilinear", LIVE}, {"SHARPNESS", "80", "0..100", LIVE}, - {"DLAA", "0", "0|1", RESTART}, {"DLSS", "dlaa", "dlaa|quality|balanced|performance", RESTART}, {"FXAA", "1", "0|1", LIVE}, + {"DLAA", "0", "0|1", RESTART}, {"DLSS", "dlaa", "dlaa|quality|balanced|performance|ultra_performance", LIVE}, {"FXAA", "1", "0|1", LIVE}, {"ASPECT", "fit", "fit|stretch", LIVE}, {"FOV", "100", "70..140", LIVE}, {"ANISO", "game", "game|2|4|8|16", RESTART}, // vk_texture.c sampler_features // colour adjustment (vk_color.c); the preset is the menu's, the four values are what the present uses diff --git a/runtime/dlaa_mask_spv.h b/runtime/dlaa_mask_spv.h new file mode 100644 index 00000000..1a21adbf --- /dev/null +++ b/runtime/dlaa_mask_spv.h @@ -0,0 +1,66 @@ +// Generated by tools/make_dlaa_mask.py: DLSS's reactive mask pass (vk_dlaa.c mask_pass). Do not edit. +#pragma once +#include +static const uint32_t dlaa_mask_spv[] = { + 0x07230203, 0x00010000, 0x00000000, 0x00000055, 0x00000000, 0x00020011, 0x00000001, 0x00020011, + 0x00000031, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, + 0x00000000, 0x00000001, 0x0006000f, 0x00000005, 0x00000002, 0x6e69616d, 0x00000000, 0x0000001b, + 0x00060010, 0x00000002, 0x00000011, 0x00000008, 0x00000008, 0x00000001, 0x00040047, 0x0000001b, + 0x0000000b, 0x0000001c, 0x00040047, 0x00000018, 0x00000022, 0x00000000, 0x00040047, 0x00000018, + 0x00000021, 0x00000000, 0x00040047, 0x00000019, 0x00000022, 0x00000000, 0x00040047, 0x00000019, + 0x00000021, 0x00000001, 0x00040047, 0x0000001a, 0x00000022, 0x00000000, 0x00040047, 0x0000001a, + 0x00000021, 0x00000002, 0x00030047, 0x0000001a, 0x00000019, 0x00030047, 0x00000015, 0x00000002, + 0x00050048, 0x00000015, 0x00000000, 0x00000023, 0x00000000, 0x00020013, 0x00000003, 0x00030021, + 0x00000004, 0x00000003, 0x00020014, 0x00000005, 0x00030016, 0x00000006, 0x00000020, 0x00040015, + 0x00000007, 0x00000020, 0x00000001, 0x00040015, 0x00000008, 0x00000020, 0x00000000, 0x00040017, + 0x00000009, 0x00000006, 0x00000002, 0x00040017, 0x0000000a, 0x00000006, 0x00000003, 0x00040017, + 0x0000000b, 0x00000006, 0x00000004, 0x00040017, 0x0000000c, 0x00000007, 0x00000002, 0x00040017, + 0x0000000d, 0x00000008, 0x00000002, 0x00040017, 0x0000000e, 0x00000008, 0x00000003, 0x00090019, + 0x0000000f, 0x00000006, 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x00000000, + 0x0003001b, 0x00000010, 0x0000000f, 0x00090019, 0x00000011, 0x00000006, 0x00000001, 0x00000000, + 0x00000000, 0x00000000, 0x00000002, 0x0000000f, 0x00040020, 0x00000012, 0x00000000, 0x00000010, + 0x00040020, 0x00000013, 0x00000000, 0x00000011, 0x00040020, 0x00000014, 0x00000001, 0x0000000e, + 0x0003001e, 0x00000015, 0x0000000b, 0x00040020, 0x00000016, 0x00000009, 0x00000015, 0x00040020, + 0x00000017, 0x00000009, 0x0000000b, 0x0004002b, 0x00000007, 0x0000001d, 0x00000000, 0x0004002b, + 0x00000006, 0x0000001e, 0x00000000, 0x0004002b, 0x00000006, 0x0000001f, 0x3f800000, 0x0004002b, + 0x00000006, 0x00000020, 0x38d1b717, 0x0004003b, 0x00000012, 0x00000018, 0x00000000, 0x0004003b, + 0x00000012, 0x00000019, 0x00000000, 0x0004003b, 0x00000013, 0x0000001a, 0x00000000, 0x0004003b, + 0x00000014, 0x0000001b, 0x00000001, 0x0004003b, 0x00000016, 0x0000001c, 0x00000009, 0x00050036, + 0x00000003, 0x00000002, 0x00000000, 0x00000004, 0x000200f8, 0x00000021, 0x00050041, 0x00000017, + 0x00000024, 0x0000001c, 0x0000001d, 0x0004003d, 0x0000000b, 0x00000025, 0x00000024, 0x0004003d, + 0x0000000e, 0x00000026, 0x0000001b, 0x0007004f, 0x0000000d, 0x00000027, 0x00000026, 0x00000026, + 0x00000000, 0x00000001, 0x00050051, 0x00000008, 0x00000028, 0x00000026, 0x00000000, 0x00040070, + 0x00000006, 0x00000029, 0x00000028, 0x00050051, 0x00000008, 0x0000002a, 0x00000026, 0x00000001, + 0x00040070, 0x00000006, 0x0000002b, 0x0000002a, 0x00050051, 0x00000006, 0x0000002c, 0x00000025, + 0x00000000, 0x00050051, 0x00000006, 0x0000002d, 0x00000025, 0x00000001, 0x000500b8, 0x00000005, + 0x0000002e, 0x00000029, 0x0000002c, 0x000500b8, 0x00000005, 0x0000002f, 0x0000002b, 0x0000002d, + 0x000500a7, 0x00000005, 0x00000030, 0x0000002e, 0x0000002f, 0x000300f7, 0x00000023, 0x00000000, + 0x000400fa, 0x00000030, 0x00000022, 0x00000023, 0x000200f8, 0x00000022, 0x0004007c, 0x0000000c, + 0x00000031, 0x00000027, 0x0004003d, 0x00000010, 0x00000032, 0x00000018, 0x00040064, 0x0000000f, + 0x00000033, 0x00000032, 0x0007005f, 0x0000000b, 0x00000034, 0x00000033, 0x00000031, 0x00000002, + 0x0000001d, 0x0008004f, 0x0000000a, 0x00000035, 0x00000034, 0x00000034, 0x00000000, 0x00000001, + 0x00000002, 0x0004003d, 0x00000010, 0x00000036, 0x00000019, 0x00040064, 0x0000000f, 0x00000037, + 0x00000036, 0x0007005f, 0x0000000b, 0x00000038, 0x00000037, 0x00000031, 0x00000002, 0x0000001d, + 0x0008004f, 0x0000000a, 0x00000039, 0x00000038, 0x00000038, 0x00000000, 0x00000001, 0x00000002, + 0x00050083, 0x0000000a, 0x0000003a, 0x00000039, 0x00000035, 0x0006000c, 0x0000000a, 0x0000003b, + 0x00000001, 0x00000004, 0x0000003a, 0x00050051, 0x00000006, 0x0000003c, 0x0000003b, 0x00000000, + 0x00050051, 0x00000006, 0x0000003d, 0x0000003b, 0x00000001, 0x00050051, 0x00000006, 0x0000003e, + 0x0000003b, 0x00000002, 0x0007000c, 0x00000006, 0x0000003f, 0x00000001, 0x00000028, 0x0000003c, + 0x0000003d, 0x0007000c, 0x00000006, 0x00000040, 0x00000001, 0x00000028, 0x0000003f, 0x0000003e, + 0x00050051, 0x00000006, 0x00000041, 0x00000039, 0x00000000, 0x00050051, 0x00000006, 0x00000042, + 0x00000039, 0x00000001, 0x00050051, 0x00000006, 0x00000043, 0x00000039, 0x00000002, 0x0007000c, + 0x00000006, 0x00000044, 0x00000001, 0x00000028, 0x00000041, 0x00000042, 0x0007000c, 0x00000006, + 0x00000045, 0x00000001, 0x00000028, 0x00000044, 0x00000043, 0x00050051, 0x00000006, 0x00000046, + 0x00000035, 0x00000000, 0x00050051, 0x00000006, 0x00000047, 0x00000035, 0x00000001, 0x00050051, + 0x00000006, 0x00000048, 0x00000035, 0x00000002, 0x0007000c, 0x00000006, 0x00000049, 0x00000001, + 0x00000028, 0x00000046, 0x00000047, 0x0007000c, 0x00000006, 0x0000004a, 0x00000001, 0x00000028, + 0x00000049, 0x00000048, 0x0007000c, 0x00000006, 0x0000004b, 0x00000001, 0x00000028, 0x00000045, + 0x0000004a, 0x0007000c, 0x00000006, 0x0000004c, 0x00000001, 0x00000028, 0x0000004b, 0x00000020, + 0x00050088, 0x00000006, 0x0000004d, 0x00000040, 0x0000004c, 0x00050051, 0x00000006, 0x0000004e, + 0x00000025, 0x00000002, 0x00050051, 0x00000006, 0x0000004f, 0x00000025, 0x00000003, 0x00050083, + 0x00000006, 0x00000050, 0x0000004d, 0x0000004e, 0x00050085, 0x00000006, 0x00000051, 0x00000050, + 0x0000004f, 0x0008000c, 0x00000006, 0x00000052, 0x00000001, 0x0000002b, 0x00000051, 0x0000001e, + 0x0000001f, 0x00070050, 0x0000000b, 0x00000053, 0x00000052, 0x00000052, 0x00000052, 0x00000052, + 0x0004003d, 0x00000011, 0x00000054, 0x0000001a, 0x00040063, 0x00000054, 0x00000031, 0x00000053, + 0x000200f9, 0x00000023, 0x000200f8, 0x00000023, 0x000100fd, 0x00010038 +}; diff --git a/runtime/dlaa_mv_spv.h b/runtime/dlaa_mv_spv.h index 9aec1064..e1b31e6c 100644 --- a/runtime/dlaa_mv_spv.h +++ b/runtime/dlaa_mv_spv.h @@ -1,84 +1,119 @@ // Generated by tools/build_shaders.py from runtime/shaders/dlaa_mv.comp; do not edit. -// source sha256 f5ddb0e90c78900c1bdabb45d7c2fc44febae07e39dc0931ed1b47beefa2414e +// source sha256 bfa9b6d9e0f9d5cfd4382fa5080fa46f007a1a4e2b886b8fecd9adba11767686 #pragma once #include static const uint32_t dlaa_mv_spv[] = { - 0x07230203, 0x00010300, 0x000d000b, 0x000000a5, 0x00000000, 0x00020011, 0x00000001, 0x00020011, + 0x07230203, 0x00010600, 0x0008000b, 0x00000131, 0x00000000, 0x00020011, 0x00000001, 0x00020011, 0x00000031, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, - 0x00000000, 0x00000001, 0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000d, - 0x00060010, 0x00000004, 0x00000011, 0x00000008, 0x00000008, 0x00000001, 0x00040047, 0x0000000d, - 0x0000000b, 0x0000001c, 0x00030047, 0x0000001b, 0x00000002, 0x00040048, 0x0000001b, 0x00000000, - 0x00000005, 0x00050048, 0x0000001b, 0x00000000, 0x00000007, 0x00000010, 0x00050048, 0x0000001b, - 0x00000000, 0x00000023, 0x00000000, 0x00050048, 0x0000001b, 0x00000001, 0x00000023, 0x00000040, - 0x00050048, 0x0000001b, 0x00000002, 0x00000023, 0x00000048, 0x00040047, 0x00000037, 0x00000021, - 0x00000000, 0x00040047, 0x00000037, 0x00000022, 0x00000000, 0x00030047, 0x0000008c, 0x00000019, - 0x00040047, 0x0000008c, 0x00000021, 0x00000001, 0x00040047, 0x0000008c, 0x00000022, 0x00000000, - 0x00040047, 0x00000094, 0x0000000b, 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, - 0x00000002, 0x00040015, 0x00000006, 0x00000020, 0x00000001, 0x00040017, 0x00000007, 0x00000006, - 0x00000002, 0x00040015, 0x0000000a, 0x00000020, 0x00000000, 0x00040017, 0x0000000b, 0x0000000a, - 0x00000003, 0x00040020, 0x0000000c, 0x00000001, 0x0000000b, 0x0004003b, 0x0000000c, 0x0000000d, - 0x00000001, 0x00040017, 0x0000000e, 0x0000000a, 0x00000002, 0x00020014, 0x00000012, 0x0004002b, - 0x0000000a, 0x00000013, 0x00000000, 0x00030016, 0x00000017, 0x00000020, 0x00040017, 0x00000018, - 0x00000017, 0x00000004, 0x00040018, 0x00000019, 0x00000018, 0x00000004, 0x00040017, 0x0000001a, - 0x00000017, 0x00000002, 0x0005001e, 0x0000001b, 0x00000019, 0x0000001a, 0x0000001a, 0x00040020, - 0x0000001c, 0x00000009, 0x0000001b, 0x0004003b, 0x0000001c, 0x0000001d, 0x00000009, 0x0004002b, - 0x00000006, 0x0000001e, 0x00000001, 0x00040020, 0x0000001f, 0x00000009, 0x00000017, 0x0004002b, - 0x0000000a, 0x00000027, 0x00000001, 0x00090019, 0x00000034, 0x00000017, 0x00000001, 0x00000000, - 0x00000000, 0x00000000, 0x00000001, 0x00000000, 0x0003001b, 0x00000035, 0x00000034, 0x00040020, - 0x00000036, 0x00000000, 0x00000035, 0x0004003b, 0x00000036, 0x00000037, 0x00000000, 0x0004002b, - 0x00000006, 0x0000003a, 0x00000000, 0x0004002b, 0x00000017, 0x00000042, 0x3f000000, 0x0004002b, - 0x00000006, 0x00000045, 0x00000002, 0x00040020, 0x00000046, 0x00000009, 0x0000001a, 0x0004002b, - 0x00000017, 0x00000051, 0x40000000, 0x0004002b, 0x00000017, 0x00000053, 0x3f800000, 0x00040020, - 0x00000061, 0x00000009, 0x00000019, 0x0004002b, 0x00000017, 0x00000067, 0x00000000, 0x0005002c, - 0x0000001a, 0x00000068, 0x00000067, 0x00000067, 0x0004002b, 0x00000017, 0x0000006c, 0x358637bd, - 0x00090019, 0x0000008a, 0x00000017, 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0x00000002, - 0x00000007, 0x00040020, 0x0000008b, 0x00000000, 0x0000008a, 0x0004003b, 0x0000008b, 0x0000008c, - 0x00000000, 0x0004002b, 0x0000000a, 0x00000093, 0x00000008, 0x0006002c, 0x0000000b, 0x00000094, - 0x00000093, 0x00000093, 0x00000027, 0x0005002c, 0x0000001a, 0x000000a4, 0x00000042, 0x00000042, + 0x00000000, 0x00000001, 0x000b000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x00000036, + 0x00000043, 0x0000005d, 0x00000089, 0x000000a0, 0x000000e7, 0x00060010, 0x00000004, 0x00000011, + 0x00000008, 0x00000008, 0x00000001, 0x00030003, 0x00000002, 0x000001c2, 0x00040005, 0x00000004, + 0x6e69616d, 0x00000000, 0x00080005, 0x00000036, 0x475f6c67, 0x61626f6c, 0x766e496c, 0x7461636f, + 0x496e6f69, 0x00000044, 0x00030005, 0x00000041, 0x00000050, 0x00050006, 0x00000041, 0x00000000, + 0x657a6973, 0x00000000, 0x00050006, 0x00000041, 0x00000001, 0x7474696a, 0x00007265, 0x00030005, + 0x00000043, 0x00006370, 0x00050005, 0x0000005d, 0x74706564, 0x65745f68, 0x00000078, 0x00030005, + 0x00000087, 0x00727543, 0x00040006, 0x00000087, 0x00000000, 0x00000072, 0x00030005, 0x00000089, + 0x00727563, 0x00040005, 0x0000009e, 0x76657250, 0x00000000, 0x00040006, 0x0000009e, 0x00000000, + 0x00000072, 0x00040005, 0x000000a0, 0x76657270, 0x00000000, 0x00040005, 0x000000e7, 0x6f5f766d, + 0x00007475, 0x00040047, 0x00000036, 0x0000000b, 0x0000001c, 0x00030047, 0x00000041, 0x00000002, + 0x00050048, 0x00000041, 0x00000000, 0x00000023, 0x00000000, 0x00050048, 0x00000041, 0x00000001, + 0x00000023, 0x00000008, 0x00040047, 0x0000005d, 0x00000021, 0x00000000, 0x00040047, 0x0000005d, + 0x00000022, 0x00000000, 0x00040047, 0x00000086, 0x00000006, 0x00000010, 0x00030047, 0x00000087, + 0x00000002, 0x00050048, 0x00000087, 0x00000000, 0x00000023, 0x00000000, 0x00040047, 0x00000089, + 0x00000021, 0x00000002, 0x00040047, 0x00000089, 0x00000022, 0x00000000, 0x00040047, 0x0000009d, + 0x00000006, 0x00000010, 0x00030047, 0x0000009e, 0x00000002, 0x00050048, 0x0000009e, 0x00000000, + 0x00000023, 0x00000000, 0x00040047, 0x000000a0, 0x00000021, 0x00000003, 0x00040047, 0x000000a0, + 0x00000022, 0x00000000, 0x00030047, 0x000000e7, 0x00000019, 0x00040047, 0x000000e7, 0x00000021, + 0x00000001, 0x00040047, 0x000000e7, 0x00000022, 0x00000000, 0x00020013, 0x00000002, 0x00030021, + 0x00000003, 0x00000002, 0x00030016, 0x00000006, 0x00000020, 0x00040017, 0x00000007, 0x00000006, + 0x00000004, 0x00040018, 0x00000009, 0x00000007, 0x00000004, 0x0004002b, 0x00000006, 0x00000015, + 0x3f800000, 0x0004002b, 0x00000006, 0x00000016, 0x00000000, 0x00040015, 0x0000002f, 0x00000020, + 0x00000001, 0x00040017, 0x00000030, 0x0000002f, 0x00000002, 0x00040015, 0x00000033, 0x00000020, + 0x00000000, 0x00040017, 0x00000034, 0x00000033, 0x00000003, 0x00040020, 0x00000035, 0x00000001, + 0x00000034, 0x0004003b, 0x00000035, 0x00000036, 0x00000001, 0x00040017, 0x00000037, 0x00000033, + 0x00000002, 0x00020014, 0x0000003b, 0x0004002b, 0x00000033, 0x0000003c, 0x00000000, 0x00040017, + 0x00000040, 0x00000006, 0x00000002, 0x0004001e, 0x00000041, 0x00000040, 0x00000040, 0x00040020, + 0x00000042, 0x00000009, 0x00000041, 0x0004003b, 0x00000042, 0x00000043, 0x00000009, 0x0004002b, + 0x0000002f, 0x00000044, 0x00000000, 0x00040020, 0x00000045, 0x00000009, 0x00000006, 0x0004002b, + 0x00000033, 0x0000004d, 0x00000001, 0x00090019, 0x0000005a, 0x00000006, 0x00000001, 0x00000000, + 0x00000000, 0x00000000, 0x00000001, 0x00000000, 0x0003001b, 0x0000005b, 0x0000005a, 0x00040020, + 0x0000005c, 0x00000000, 0x0000005b, 0x0004003b, 0x0000005c, 0x0000005d, 0x00000000, 0x0004002b, + 0x00000006, 0x00000067, 0x3f000000, 0x0004002b, 0x0000002f, 0x0000006a, 0x00000001, 0x00040020, + 0x0000006b, 0x00000009, 0x00000040, 0x0004002b, 0x00000006, 0x00000075, 0x40000000, 0x0004002b, + 0x00000033, 0x00000085, 0x00000010, 0x0004001c, 0x00000086, 0x00000007, 0x00000085, 0x0003001e, + 0x00000087, 0x00000086, 0x00040020, 0x00000088, 0x00000002, 0x00000087, 0x0004003b, 0x00000088, + 0x00000089, 0x00000002, 0x0004002b, 0x0000002f, 0x0000008a, 0x00000003, 0x0004002b, 0x0000002f, + 0x0000008b, 0x00000004, 0x0004002b, 0x0000002f, 0x0000008c, 0x00000005, 0x0004002b, 0x0000002f, + 0x0000008d, 0x00000006, 0x00040020, 0x0000008f, 0x00000002, 0x00000007, 0x0004001c, 0x0000009d, + 0x00000007, 0x00000085, 0x0003001e, 0x0000009e, 0x0000009d, 0x00040020, 0x0000009f, 0x00000002, + 0x0000009e, 0x0004003b, 0x0000009f, 0x000000a0, 0x00000002, 0x0005002c, 0x00000040, 0x000000b4, + 0x00000016, 0x00000016, 0x0004002b, 0x00000006, 0x000000b9, 0x2b8cbccc, 0x0004002b, 0x00000006, + 0x000000c7, 0x358637bd, 0x00090019, 0x000000e5, 0x00000006, 0x00000001, 0x00000000, 0x00000000, + 0x00000000, 0x00000002, 0x00000007, 0x00040020, 0x000000e6, 0x00000000, 0x000000e5, 0x0004003b, + 0x000000e6, 0x000000e7, 0x00000000, 0x0005002c, 0x00000040, 0x00000130, 0x00000067, 0x00000067, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, 0x000200f8, 0x00000005, 0x000300f7, - 0x00000095, 0x00000000, 0x000300fb, 0x00000013, 0x00000096, 0x000200f8, 0x00000096, 0x0004003d, - 0x0000000b, 0x0000000f, 0x0000000d, 0x0007004f, 0x0000000e, 0x00000010, 0x0000000f, 0x0000000f, - 0x00000000, 0x00000001, 0x0004007c, 0x00000007, 0x00000011, 0x00000010, 0x00050051, 0x00000006, - 0x00000016, 0x00000011, 0x00000000, 0x00060041, 0x0000001f, 0x00000020, 0x0000001d, 0x0000001e, - 0x00000013, 0x0004003d, 0x00000017, 0x00000021, 0x00000020, 0x0004006e, 0x00000006, 0x00000022, - 0x00000021, 0x000500af, 0x00000012, 0x00000023, 0x00000016, 0x00000022, 0x000400a8, 0x00000012, - 0x00000024, 0x00000023, 0x000300f7, 0x00000026, 0x00000000, 0x000400fa, 0x00000024, 0x00000025, - 0x00000026, 0x000200f8, 0x00000025, 0x00050051, 0x00000006, 0x00000029, 0x00000011, 0x00000001, - 0x00060041, 0x0000001f, 0x0000002a, 0x0000001d, 0x0000001e, 0x00000027, 0x0004003d, 0x00000017, - 0x0000002b, 0x0000002a, 0x0004006e, 0x00000006, 0x0000002c, 0x0000002b, 0x000500af, 0x00000012, - 0x0000002d, 0x00000029, 0x0000002c, 0x000200f9, 0x00000026, 0x000200f8, 0x00000026, 0x000700f5, - 0x00000012, 0x0000002e, 0x00000023, 0x00000096, 0x0000002d, 0x00000025, 0x000300f7, 0x00000030, - 0x00000000, 0x000400fa, 0x0000002e, 0x0000002f, 0x00000030, 0x000200f8, 0x0000002f, 0x000200f9, - 0x00000095, 0x000200f8, 0x00000030, 0x0004003d, 0x00000035, 0x00000038, 0x00000037, 0x00040064, - 0x00000034, 0x0000003b, 0x00000038, 0x0007005f, 0x00000018, 0x0000003c, 0x0000003b, 0x00000011, - 0x00000002, 0x0000003a, 0x00050051, 0x00000017, 0x0000003d, 0x0000003c, 0x00000000, 0x0004006f, - 0x0000001a, 0x00000041, 0x00000011, 0x00050081, 0x0000001a, 0x00000044, 0x00000041, 0x000000a4, - 0x00050041, 0x00000046, 0x00000047, 0x0000001d, 0x00000045, 0x0004003d, 0x0000001a, 0x00000048, - 0x00000047, 0x00050083, 0x0000001a, 0x00000049, 0x00000044, 0x00000048, 0x00050051, 0x00000017, - 0x0000004d, 0x00000049, 0x00000000, 0x00050088, 0x00000017, 0x00000050, 0x0000004d, 0x00000021, - 0x00050085, 0x00000017, 0x00000052, 0x00000050, 0x00000051, 0x00050083, 0x00000017, 0x00000054, - 0x00000052, 0x00000053, 0x00050051, 0x00000017, 0x00000056, 0x00000049, 0x00000001, 0x00060041, - 0x0000001f, 0x00000057, 0x0000001d, 0x0000001e, 0x00000027, 0x0004003d, 0x00000017, 0x00000058, - 0x00000057, 0x00050088, 0x00000017, 0x00000059, 0x00000056, 0x00000058, 0x00050085, 0x00000017, - 0x0000005a, 0x00000059, 0x00000051, 0x00050083, 0x00000017, 0x0000005b, 0x00000053, 0x0000005a, - 0x00050085, 0x00000017, 0x0000005d, 0x0000003d, 0x00000051, 0x00050083, 0x00000017, 0x0000005e, - 0x0000005d, 0x00000053, 0x00070050, 0x00000018, 0x0000005f, 0x00000054, 0x0000005b, 0x0000005e, - 0x00000053, 0x00050041, 0x00000061, 0x00000062, 0x0000001d, 0x0000003a, 0x0004003d, 0x00000019, - 0x00000063, 0x00000062, 0x00050091, 0x00000018, 0x00000065, 0x00000063, 0x0000005f, 0x00050051, - 0x00000017, 0x0000006b, 0x00000065, 0x00000003, 0x000500ba, 0x00000012, 0x0000006d, 0x0000006b, - 0x0000006c, 0x000300f7, 0x0000006f, 0x00000000, 0x000400fa, 0x0000006d, 0x0000006e, 0x0000006f, - 0x000200f8, 0x0000006e, 0x0007004f, 0x0000001a, 0x00000072, 0x00000065, 0x00000065, 0x00000000, - 0x00000001, 0x00050050, 0x0000001a, 0x00000075, 0x0000006b, 0x0000006b, 0x00050088, 0x0000001a, - 0x00000076, 0x00000072, 0x00000075, 0x00050051, 0x00000017, 0x00000079, 0x00000076, 0x00000000, - 0x00050085, 0x00000017, 0x0000007a, 0x00000079, 0x00000042, 0x00050081, 0x00000017, 0x0000007b, - 0x0000007a, 0x00000042, 0x00050085, 0x00000017, 0x0000007e, 0x0000007b, 0x00000021, 0x00050051, - 0x00000017, 0x00000080, 0x00000076, 0x00000001, 0x00050085, 0x00000017, 0x00000081, 0x00000080, - 0x00000042, 0x00050083, 0x00000017, 0x00000082, 0x00000042, 0x00000081, 0x00050085, 0x00000017, - 0x00000085, 0x00000082, 0x00000058, 0x00050050, 0x0000001a, 0x00000086, 0x0000007e, 0x00000085, - 0x00050083, 0x0000001a, 0x00000089, 0x00000086, 0x00000049, 0x000200f9, 0x0000006f, 0x000200f8, - 0x0000006f, 0x000700f5, 0x0000001a, 0x000000a3, 0x00000068, 0x00000030, 0x00000089, 0x0000006e, - 0x0004003d, 0x0000008a, 0x0000008d, 0x0000008c, 0x00050051, 0x00000017, 0x00000090, 0x000000a3, - 0x00000000, 0x00050051, 0x00000017, 0x00000091, 0x000000a3, 0x00000001, 0x00070050, 0x00000018, - 0x00000092, 0x00000090, 0x00000091, 0x00000067, 0x00000067, 0x00040063, 0x0000008d, 0x00000011, - 0x00000092, 0x000200f9, 0x00000095, 0x000200f8, 0x00000095, 0x000100fd, 0x00010038, + 0x000000f0, 0x00000000, 0x000300fb, 0x0000003c, 0x000000f1, 0x000200f8, 0x000000f1, 0x0004003d, + 0x00000034, 0x00000038, 0x00000036, 0x0007004f, 0x00000037, 0x00000039, 0x00000038, 0x00000038, + 0x00000000, 0x00000001, 0x0004007c, 0x00000030, 0x0000003a, 0x00000039, 0x00050051, 0x0000002f, + 0x0000003f, 0x0000003a, 0x00000000, 0x00060041, 0x00000045, 0x00000046, 0x00000043, 0x00000044, + 0x0000003c, 0x0004003d, 0x00000006, 0x00000047, 0x00000046, 0x0004006e, 0x0000002f, 0x00000048, + 0x00000047, 0x000500af, 0x0000003b, 0x00000049, 0x0000003f, 0x00000048, 0x000400a8, 0x0000003b, + 0x0000004a, 0x00000049, 0x000300f7, 0x0000004c, 0x00000000, 0x000400fa, 0x0000004a, 0x0000004b, + 0x0000004c, 0x000200f8, 0x0000004b, 0x00050051, 0x0000002f, 0x0000004f, 0x0000003a, 0x00000001, + 0x00060041, 0x00000045, 0x00000050, 0x00000043, 0x00000044, 0x0000004d, 0x0004003d, 0x00000006, + 0x00000051, 0x00000050, 0x0004006e, 0x0000002f, 0x00000052, 0x00000051, 0x000500af, 0x0000003b, + 0x00000053, 0x0000004f, 0x00000052, 0x000200f9, 0x0000004c, 0x000200f8, 0x0000004c, 0x000700f5, + 0x0000003b, 0x00000054, 0x00000049, 0x000000f1, 0x00000053, 0x0000004b, 0x000300f7, 0x00000056, + 0x00000000, 0x000400fa, 0x00000054, 0x00000055, 0x00000056, 0x000200f8, 0x00000055, 0x000200f9, + 0x000000f0, 0x000200f8, 0x00000056, 0x0004003d, 0x0000005b, 0x0000005e, 0x0000005d, 0x00040064, + 0x0000005a, 0x00000060, 0x0000005e, 0x0007005f, 0x00000007, 0x00000061, 0x00000060, 0x0000003a, + 0x00000002, 0x00000044, 0x00050051, 0x00000006, 0x00000062, 0x00000061, 0x00000000, 0x0004006f, + 0x00000040, 0x00000066, 0x0000003a, 0x00050081, 0x00000040, 0x00000069, 0x00000066, 0x00000130, + 0x00050041, 0x0000006b, 0x0000006c, 0x00000043, 0x0000006a, 0x0004003d, 0x00000040, 0x0000006d, + 0x0000006c, 0x00050083, 0x00000040, 0x0000006e, 0x00000069, 0x0000006d, 0x00050051, 0x00000006, + 0x00000071, 0x0000006e, 0x00000000, 0x00050088, 0x00000006, 0x00000074, 0x00000071, 0x00000047, + 0x00050085, 0x00000006, 0x00000076, 0x00000074, 0x00000075, 0x00050083, 0x00000006, 0x00000077, + 0x00000076, 0x00000015, 0x00050051, 0x00000006, 0x00000079, 0x0000006e, 0x00000001, 0x00060041, + 0x00000045, 0x0000007a, 0x00000043, 0x00000044, 0x0000004d, 0x0004003d, 0x00000006, 0x0000007b, + 0x0000007a, 0x00050088, 0x00000006, 0x0000007c, 0x00000079, 0x0000007b, 0x00050085, 0x00000006, + 0x0000007d, 0x0000007c, 0x00000075, 0x00050083, 0x00000006, 0x0000007e, 0x00000015, 0x0000007d, + 0x00050085, 0x00000006, 0x00000080, 0x00000062, 0x00000075, 0x00050083, 0x00000006, 0x00000081, + 0x00000080, 0x00000015, 0x00070050, 0x00000007, 0x00000082, 0x00000077, 0x0000007e, 0x00000081, + 0x00000015, 0x00060041, 0x0000008f, 0x00000090, 0x00000089, 0x00000044, 0x0000008a, 0x0004003d, + 0x00000007, 0x00000091, 0x00000090, 0x00060041, 0x0000008f, 0x00000093, 0x00000089, 0x00000044, + 0x0000008b, 0x0004003d, 0x00000007, 0x00000094, 0x00000093, 0x00060041, 0x0000008f, 0x00000096, + 0x00000089, 0x00000044, 0x0000008c, 0x0004003d, 0x00000007, 0x00000097, 0x00000096, 0x00060041, + 0x0000008f, 0x00000099, 0x00000089, 0x00000044, 0x0000008d, 0x0004003d, 0x00000007, 0x0000009a, + 0x00000099, 0x00070050, 0x00000009, 0x00000110, 0x00000091, 0x00000094, 0x00000097, 0x0000009a, + 0x00040054, 0x00000009, 0x00000111, 0x00000110, 0x00060041, 0x0000008f, 0x000000a2, 0x000000a0, + 0x00000044, 0x0000008a, 0x0004003d, 0x00000007, 0x000000a3, 0x000000a2, 0x00060041, 0x0000008f, + 0x000000a5, 0x000000a0, 0x00000044, 0x0000008b, 0x0004003d, 0x00000007, 0x000000a6, 0x000000a5, + 0x00060041, 0x0000008f, 0x000000a8, 0x000000a0, 0x00000044, 0x0000008c, 0x0004003d, 0x00000007, + 0x000000a9, 0x000000a8, 0x00060041, 0x0000008f, 0x000000ab, 0x000000a0, 0x00000044, 0x0000008d, + 0x0004003d, 0x00000007, 0x000000ac, 0x000000ab, 0x00070050, 0x00000009, 0x0000012c, 0x000000a3, + 0x000000a6, 0x000000a9, 0x000000ac, 0x00040054, 0x00000009, 0x0000012d, 0x0000012c, 0x0006000c, + 0x00000009, 0x000000b0, 0x00000001, 0x00000022, 0x00000111, 0x00050091, 0x00000007, 0x000000b2, + 0x000000b0, 0x00000082, 0x00050051, 0x00000006, 0x000000b7, 0x000000b2, 0x00000003, 0x0006000c, + 0x00000006, 0x000000b8, 0x00000001, 0x00000004, 0x000000b7, 0x000500ba, 0x0000003b, 0x000000ba, + 0x000000b8, 0x000000b9, 0x000300f7, 0x000000bc, 0x00000000, 0x000400fa, 0x000000ba, 0x000000bb, + 0x000000bc, 0x000200f8, 0x000000bb, 0x00070050, 0x00000007, 0x000000c2, 0x000000b7, 0x000000b7, + 0x000000b7, 0x000000b7, 0x00050088, 0x00000007, 0x000000c3, 0x000000b2, 0x000000c2, 0x00050091, + 0x00000007, 0x000000c4, 0x0000012d, 0x000000c3, 0x00050051, 0x00000006, 0x000000c6, 0x000000c4, + 0x00000003, 0x000500ba, 0x0000003b, 0x000000c8, 0x000000c6, 0x000000c7, 0x000300f7, 0x000000ca, + 0x00000000, 0x000400fa, 0x000000c8, 0x000000c9, 0x000000ca, 0x000200f8, 0x000000c9, 0x0007004f, + 0x00000040, 0x000000cd, 0x000000c4, 0x000000c4, 0x00000000, 0x00000001, 0x00050050, 0x00000040, + 0x000000d0, 0x000000c6, 0x000000c6, 0x00050088, 0x00000040, 0x000000d1, 0x000000cd, 0x000000d0, + 0x00050051, 0x00000006, 0x000000d4, 0x000000d1, 0x00000000, 0x00050085, 0x00000006, 0x000000d5, + 0x000000d4, 0x00000067, 0x00050081, 0x00000006, 0x000000d6, 0x000000d5, 0x00000067, 0x00050085, + 0x00000006, 0x000000d9, 0x000000d6, 0x00000047, 0x00050051, 0x00000006, 0x000000db, 0x000000d1, + 0x00000001, 0x00050085, 0x00000006, 0x000000dc, 0x000000db, 0x00000067, 0x00050083, 0x00000006, + 0x000000dd, 0x00000067, 0x000000dc, 0x00050085, 0x00000006, 0x000000e0, 0x000000dd, 0x0000007b, + 0x00050050, 0x00000040, 0x000000e1, 0x000000d9, 0x000000e0, 0x00050083, 0x00000040, 0x000000e4, + 0x000000e1, 0x0000006e, 0x000200f9, 0x000000ca, 0x000200f8, 0x000000ca, 0x000700f5, 0x00000040, + 0x0000012f, 0x000000b4, 0x000000bb, 0x000000e4, 0x000000c9, 0x000200f9, 0x000000bc, 0x000200f8, + 0x000000bc, 0x000700f5, 0x00000040, 0x0000012e, 0x000000b4, 0x00000056, 0x0000012f, 0x000000ca, + 0x0004003d, 0x000000e5, 0x000000e8, 0x000000e7, 0x00050051, 0x00000006, 0x000000eb, 0x0000012e, + 0x00000000, 0x00050051, 0x00000006, 0x000000ec, 0x0000012e, 0x00000001, 0x00070050, 0x00000007, + 0x000000ed, 0x000000eb, 0x000000ec, 0x00000016, 0x00000016, 0x00040063, 0x000000e8, 0x0000003a, + 0x000000ed, 0x000200f9, 0x000000f0, 0x000200f8, 0x000000f0, 0x000100fd, 0x00010038, }; diff --git a/runtime/gfx.h b/runtime/gfx.h index f21d0d51..614e8f65 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -35,6 +35,8 @@ typedef struct { // the translator's a uint8_t attr_type[32], frag_out_type[8]; // GFX_TYPE_* uint16_t local[3]; uint32_t shared_bytes; uint8_t depth_minus_w; + uint8_t mvprev; // VS: also write last frame's position (patch 0015, + // vk_mvprev.c); packed as bit 7 of the depth_minus_w byte } GfxShaderAnswers; typedef struct GfxShader GfxShader; // SPIR-V words + parsed reflection; owned by the cache const GfxShader *gfx_shader_get(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, @@ -73,6 +75,9 @@ size_t gfx_shader_warm_list(GfxWarmEnt **out); // malloc'd; recorde const GfxShader *gfx_shader_load_private(const uint8_t iid[32], int *owned); void gfx_shader_release_private(const GfxShader *s); const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words); +// A shader of the runtime's own (SPIR-V and a .res.json text as the translator writes them), outside the cache and +// never freed; its key is the sha256 of both. NULL when the SPIR-V is malformed. +const GfxShader *gfx_shader_own(const uint32_t *spirv, uint64_t words, const char *res_json); // vk_hdr.c: with TOTK_HDR=1, replaces a fragment shader's SPIR-V (malloc'd, *words long) by the HDR tonemap when it is // the game's tonemap; gfx_shader.c calls it for every fragment shader entering the cache. void vk_hdr_shader_swap(uint32_t **spirv, uint64_t *words); @@ -97,7 +102,8 @@ int gfx_shader_stage(const GfxShader *s); // GFX_STAGE_VS, GFX int gfx_shader_resources(const GfxShader *s, int kind, const GfxShaderRes **res); // count of GFX_RES_* kind; array in *res uint32_t gfx_shader_slot_mask(const GfxShader *s); // texture/image slots whose type its translation read uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]); // Input Locations (VS attributes, FS varyings) as a mask -uint32_t gfx_shader_outputs(const GfxShader *s, GfxShaderIo io[32]); // Output Locations (VS varyings, FS colour targets) +uint32_t gfx_shader_outputs(const GfxShader *s, GfxShaderIo io[32]); +uint32_t gfx_shader_mvprev_banks(const GfxShader *s); // a last-frame variant: the banks (bit n: cn) its position reads // Output Locations (VS varyings, FS colour targets) void gfx_shader_local_size(const GfxShader *s, uint32_t local[3]); // CS: the LocalSize execution mode; 1, 1, 1 otherwise // ---- back end (gfx_null.c, vk_*.c). Called by the NVN front end (nvn_api_*.c, nvn_ext.c) on guest threads. diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 09419be3..881fa86d 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -86,6 +86,7 @@ struct GfxShader { uint32_t in_mask, out_mask; // user Locations of Input / Output variables GfxShaderIo in[32], out[32]; uint32_t local[3]; + uint32_t mvprev_banks; // .res.json "mvprev_banks" (patch 0015), 0 none }; static PlatMutex cache_lock = PLAT_MUTEX_INIT; // memory cache @@ -593,6 +594,25 @@ static GfxShader *disk_load_from(const uint8_t key[32], const uint8_t *iid, int return s; } +const GfxShader *gfx_shader_own(const uint32_t *spirv, uint64_t words, const char *res_json) { + if (words < 5 || spirv[0] != 0x07230203 || !spirv_complete(spirv, words) || + !reflection_valid(res_json, strlen(res_json))) + return NULL; + GfxShader *s = calloc(1, sizeof *s); + if (!s) return NULL; + size_t rn = strlen(res_json); + uint8_t *buf = malloc(words * 4 + rn); + s->spirv = malloc(words * 4), s->reflection = malloc(rn + 1); + if (!buf || !s->spirv || !s->reflection) rt_fatal("gfx_shader: out of memory"); + memcpy(buf, spirv, words * 4), memcpy(buf + words * 4, res_json, rn); + rt_sha256(buf, words * 4 + rn, s->key); + free(buf); + memcpy(s->spirv, spirv, words * 4), memcpy(s->reflection, res_json, rn + 1); + s->words = words, s->reflection_size = rn; + reflect(s); + return s; +} + // The pack, then the loose files (a newer translation of a key the pack holds a bad copy of); NULL when neither has a // valid pair (the caller translates again and writes loose files). static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) { @@ -826,6 +846,7 @@ static void reflect(GfxShader *s) { char stage[8]; json_str(js, js + strlen(js), "stage", stage, sizeof stage); s->stage = !strcmp(stage, "cs") ? GFX_STAGE_CS : !strcmp(stage, "ps") ? GFX_STAGE_FS : GFX_STAGE_VS; + s->mvprev_banks = (uint32_t)json_int(js, js + strlen(js), "mvprev_banks", 0); for (int k = 0; k < GFX_RES_KINDS; k++) { int count = json_objects(js, kinds[k], NULL, s, k); s->res[k] = calloc(count ? count : 1, sizeof *s->res[k]); @@ -1037,7 +1058,7 @@ static size_t pack_answers(uint8_t *o, const GfxShaderAnswers *a) { memcpy(p, a->frag_out_type, 8), p += 8; for (int i = 0; i < 3; i++) *p++ = (uint8_t)a->local[i], *p++ = (uint8_t)(a->local[i] >> 8); put_u32(&p, a->shared_bytes); - *p++ = a->depth_minus_w; + *p++ = (uint8_t)(a->depth_minus_w | (a->mvprev ? 0x80 : 0)); return (size_t)(p - o); } @@ -1069,6 +1090,7 @@ static void mask_answers(const GfxShaderAnswers *a, const UseMask *m, GfxShaderA if (m->fouts >> i & 1) o->frag_out_type[i] = a->frag_out_type[i]; memcpy(o->local, a->local, sizeof o->local); o->shared_bytes = a->shared_bytes; + o->mvprev = a->mvprev; // another translation, not an answer the use mask covers } // The cache key of an input id under the current translator. @@ -1148,6 +1170,7 @@ static size_t build_request(char *o0, const char *hex, const char *stage, const if (!strcmp(stage, "ps") && (control[0x718] & 1)) o += sprintf(o, ",\"early_fragment_tests\":true"); int si = !strcmp(stage, "vs") ? 0 : !strcmp(stage, "cs") ? 2 : 1; // gfx_shader_set_host answers if (!host_f64) o += sprintf(o, ",\"softfloat64\":true"); + if (a && a->mvprev && !strcmp(stage, "vs")) o += sprintf(o, ",\"mvprev\":true"); if (host_sbuf) o += sprintf(o, ",\"sbuf_align\":%u", host_sbuf); if (host_sg[si] != 32) o += sprintf(o, ",\"subgroup\":%u", host_sg[si]); if (host_gather) o += sprintf(o, ",\"gather_bias\":%u", host_gather); @@ -1350,8 +1373,18 @@ static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *co if (!req) return plat_file_delete(code_path), fail("out of memory", hex); build_request(req, hex, r->stage_name, code_path, r->c1, r->c1_len, a, control); int ok = svc_request(req, reply, sizeof reply); + // TOTK_XLATE_KEEP=1: a last-frame variant's job stays as .job (one request line) with its code, to run the + // translator on by hand (totk_xlate --serve < .job) + static int keep = -1; + if (keep < 0) keep = rt_knob("XLATE_KEEP", "0"); + int kept = keep && a && a->mvprev; + if (kept) { + char job_path[PATH_CAP + 80]; + snprintf(job_path, sizeof job_path, "%s/%s.job", cache_dir, hex); + write_file(job_path, req, strlen(req)); + } free(req); - plat_file_delete(code_path); + if (!kept) plat_file_delete(code_path); if (!ok) return fail("translator unavailable", hex); if (strncmp(reply, "ok ", 3)) { plat_file_delete(spv_path), plat_file_delete(res_path); @@ -1706,6 +1739,8 @@ uint32_t gfx_shader_inputs(const GfxShader *s, GfxShaderIo io[32]) { return s->in_mask; } +uint32_t gfx_shader_mvprev_banks(const GfxShader *s) { return s->mvprev_banks; } + uint32_t gfx_shader_outputs(const GfxShader *s, GfxShaderIo io[32]) { if (io) memcpy(io, s->out, sizeof s->out); return s->out_mask; @@ -1795,7 +1830,7 @@ static void unpack_answers(const uint8_t *p, GfxShaderAnswers *a) { memcpy(a->frag_out_type, p, 8), p += 8; for (int i = 0; i < 3; i++) a->local[i] = (uint16_t)(p[0] | p[1] << 8), p += 2; memcpy(&a->shared_bytes, p, 4), p += 4; - a->depth_minus_w = *p; + a->depth_minus_w = *p & 0x7f, a->mvprev = *p >> 7; } // Reads a record file's body (after its header) into *body; 0 when absent, and a file with another header is removed. diff --git a/runtime/host.h b/runtime/host.h index 5766ee4c..5b3070f8 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -262,6 +262,9 @@ int res_heapdef_build(const void *in, size_t n, void **out, size_t *out_n); // void res_heapdef_served(void); // romfs.c serves the grown table void res_init(void); // after rt_patches_init: the patch page and the sets float res_dlss_lod_bias(void); // TOTK_DLSS upscaling: texture LOD bias (<= 0) +int res_dlss_mode(const char *mode); // TOTK_DLSS changed while running: new scene size +void res_fn_hit(uint64_t addr, uint64_t x0); // main.c fnprof_hit: catches the renderer at start +void res_present(Cpu *c); // nvnQueuePresentTexture: applies a pending scene size void res_sizes(unsigned out[2], unsigned scene[2]); // the output and scene sizes in effect (the game's own // until res_init switched the sets on) int res_is_16x9(unsigned w, unsigned h); // within 1% of 16:9 (the test res_init and romfs.c share) diff --git a/runtime/main.c b/runtime/main.c index e0a82263..2cb40da7 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -153,8 +153,8 @@ static RtConfig cfg; const RtConfig *rt_config(void) { return &cfg; } // ---- runtime services the generated code calls (cpu.h) -volatile int g_fnprof = 0; -void fnprof_hit(uint64_t addr, uint64_t lr, uint64_t x0) { (void)addr, (void)lr, (void)x0; } +volatile int g_fnprof = 0; // resolution.c switches it on until the game sets its render size (res_fn_hit) +void fnprof_hit(uint64_t addr, uint64_t lr, uint64_t x0) { (void)lr, res_fn_hit(addr, x0); } void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) { char at[128], lr[128]; diff --git a/runtime/nvn_api_obj.c b/runtime/nvn_api_obj.c index ffe0648a..bca459ac 100644 --- a/runtime/nvn_api_obj.c +++ b/runtime/nvn_api_obj.c @@ -699,6 +699,7 @@ NVN_API(nvnQueuePresentTexture) { win_t *o = arg_p(c, 1); int idx = (int)arg_x(c, 2); movie_tick(c); // ov_movie.c: hand over decoded movie frames that are due; needs the core, so before the wait + res_present(c); // resolution.c: a DLSS mode change sets the game's render size again, between frames NVN_BLOCKING(); if (o->magic != NVN_WIN_MAGIC || idx < 0 || idx >= o->ntex || idx >= 4 || !o->tex[idx]) rt_fatal("nvn: QueuePresentTexture of image %d on window %p", idx, (void *)o); diff --git a/runtime/resolution.c b/runtime/resolution.c index cd93a74c..0814ccbb 100644 --- a/runtime/resolution.c +++ b/runtime/resolution.c @@ -53,7 +53,9 @@ static struct { int heapdef_ok; // romfs.c serves the grown HeapDef int on; // res_init switched the patch sets on unsigned shadow; // fixed sun shadow map size (TOTK_SHADOW_RES), 0: the game's own - double dlss_k; // TOTK_DLSS quality/balanced/performance: the scene's share of the output, else 0 + double dlss_k; // TOTK_DLSS quality/balanced/performance/ultra_performance: the scene's share of the output, else 0 + double base_k; // the scene's share without a DLSS mode (render_scale, or the game's 5/6) + unsigned hw, hh; // the largest scene a DLSS mode change can ask for: the heap is sized for it } R; // The scene size is decided before the device exists, so NGX cannot be asked yet. NGX loads DLSS from nvngx_dlss.dll @@ -72,6 +74,36 @@ static int dlss_dll(void) { #endif } +// The scene size for a share k of the output, within SCENE_MAX per side. +static void scene_for(double k, unsigned *w, unsigned *h) { + double sw = R.ow * k, sh = R.oh * k; + double fit = fmin(1.0, fmin(SCENE_MAX / sw, SCENE_MAX / sh)); + *w = (unsigned)(sw * fit + 0.5); + *h = (unsigned)(sh * fit + 0.5); +} + +// The scene's share of the output for a TOTK_DLSS mode; 0 for dlaa (the scene keeps base_k), while DLSS/DLAA is off +// (the mode the menu still holds would shrink the scene for the plain upscaler) and where DLSS cannot run (dlss_dll). +static double dlss_share(const char *m) { + static const struct { const char *name; double k; } dlss[] = { + {"quality", 2.0 / 3.0}, {"balanced", 0.58}, {"performance", 0.5}, {"ultra_performance", 1.0 / 3.0}}; + static int dll = -1; + if (!cfg_bool("TOTK_DLAA")) return 0; + for (int i = 0; i < (int)(sizeof dlss / sizeof dlss[0]); i++) + if (!strcmp(m, dlss[i].name)) { + if (dll < 0 && !(dll = dlss_dll())) { +#ifdef _WIN32 + rt_log("resolution: DLSS %s needs nvngx_dlss.dll next to totk.exe, which is missing: the scene follows " + "render_scale", m); +#else + rt_log("resolution: DLSS %s runs on Windows only: the scene follows render_scale", m); +#endif + } + return dll ? dlss[i].k : 0; + } + return 0; +} + static void decide(void) { if (R.decided) return; R.decided = 1; @@ -88,26 +120,7 @@ static void decide(void) { } const char *s = cfg_value("TOTK_RENDER_SCALE"); double k = 5.0 / 6.0; - // a DLSS mode other than dlaa sets the scene size, only while DLSS/DLAA is on: switched off, the mode the menu - // still holds would shrink the scene for the plain upscaler - const char *m = cfg_value("TOTK_DLSS"); - static const struct { const char *name; double k; } dlss[] = { - {"quality", 2.0 / 3.0}, {"balanced", 0.58}, {"performance", 0.5}}; - for (int i = 0; cfg_bool("TOTK_DLAA") && i < 3; i++) - if (!strcmp(m, dlss[i].name)) R.dlss_k = dlss[i].k; - if (R.dlss_k && !dlss_dll()) { -#ifdef _WIN32 - rt_log("resolution: DLSS %s needs nvngx_dlss.dll next to totk.exe, which is missing: the scene follows render_scale", m); -#else - rt_log("resolution: DLSS %s runs on Windows only: the scene follows render_scale", m); -#endif - R.dlss_k = 0; - } - if (R.dlss_k) { - k = R.dlss_k; - R.scene_set = 1; - rt_log("resolution: DLSS %s: the scene at %.0f %% of the output (render_scale ignored)", m, k * 100); - } else if (strcmp(s, "game")) { + if (strcmp(s, "game")) { char *end; double pct = strtod(s, &end); if (end != s && !*end && pct >= 25 && pct <= 400) { @@ -117,11 +130,15 @@ static void decide(void) { rt_log("resolution: TOTK_RENDER_SCALE=%s is not game or a percentage 25..400; game", s); } } - double sw = R.ow * k, sh = R.oh * k; - double fit = fmin(1.0, fmin(SCENE_MAX / sw, SCENE_MAX / sh)); - if (fit < 1.0) R.scene_set = 1; - R.sw = (unsigned)(sw * fit + 0.5); - R.sh = (unsigned)(sh * fit + 0.5); + // A DLSS mode other than dlaa sets the scene size instead, and can change while the game runs (res_dlss_mode): the + // scene patch is always on, and the heap is sized for the largest scene the modes ask for. + R.base_k = k; + R.scene_set = 1; + const char *m = cfg_value("TOTK_DLSS"); + R.dlss_k = dlss_share(m); + if (R.dlss_k) rt_log("resolution: DLSS %s: the scene at %.0f %% of the output (render_scale ignored)", m, R.dlss_k * 100); + scene_for(R.dlss_k ? R.dlss_k : k, &R.sw, &R.sh); + scene_for(fmax(k, 2.0 / 3.0), &R.hw, &R.hh); const char *sr = cfg_value("TOTK_SHADOW_RES"); if (strcmp(sr, "game")) { unsigned n = (unsigned)atoi(sr); @@ -144,7 +161,7 @@ static int bigger(void) { uint64_t res_heap_extra_kib(void) { if (!bigger()) return 0; double grow = fmax(0.0, (double)R.ow * R.oh / (BASE_W * BASE_H) - 1.0) + - fmax(0.0, (double)R.sw * R.sh / (SCENE_W * SCENE_H) - 1.0); + fmax(0.0, (double)R.hw * R.hh / (SCENE_W * SCENE_H) - 1.0); uint64_t kib = grow > 0 ? (uint64_t)ceil(DTA_KIB * (1.0 + grow) / 1024.0) * 1024 - DTA_KIB : 0; return kib + ((shadow_kib() + 1023) & ~1023ull); // whole MiB } @@ -309,6 +326,7 @@ void res_init(void) { rt_patch_switch(output, R.ow != BASE_W || R.oh != BASE_H); rt_patch_switch(scene, scene_on); R.on = 1; + if (RT_V143) g_fnprof = 1; // until the game sets its render size: res_fn_hit // Another shape than 16:9 (more than 1% off): the camera takes the output's ratio (recomp/patches-v121.txt and // patches-v143.txt, set aspect), so a 21:9 output shows more of the world at the sides instead of a stretched 16:9 view. The HUD's layouts // for that shape come from romfs-patches-x (romfs.c overlay_init, tools/make_romfs_patches.py --aspect). @@ -379,6 +397,55 @@ void res_wanted(unsigned out[2]) { out[0] = R.ow, out[1] = R.oh; } +// A TOTK_DLSS change while the game runs (vk_dlaa.c, at a present): the new scene size goes to the patch page, where +// the game reads it when it sets its render size (main+0x585e48). 1.4.3 does that once, at start: its renderer setup +// (main+0x7000e67028) calls main+0x585bd0(renderer, force), which picks the console mode's sizes and flags the render +// targets for re-creation. The setup call is caught through the function-entry hook (res_fn_hit, on only until then), +// and the next nvnQueuePresentTexture on the game's thread calls it again with force set (res_present), between frames +// as at start. The heap was sized for the largest scene a mode asks for. 1 when the size changed. +#define SET_RENDER_SIZE 0x7000585bd0ull +static uint64_t renderer; // x0 of the game's own call of SET_RENDER_SIZE +static volatile int size_pending; // the page holds a size the game has not set yet + +void res_fn_hit(uint64_t addr, uint64_t x0) { + if (addr != SET_RENDER_SIZE) return; + renderer = x0; + g_fnprof = 0; + rt_log("resolution: renderer at %#llx; the scene size can change while the game runs", (unsigned long long)x0); +} + +void res_present(Cpu *c) { + if (!size_pending || !renderer) return; + size_pending = 0; + Cpu keep = *c; // the guest's registers as at the present call: the setup runs as if it never happened + rt_call(c, SET_RENDER_SIZE, 2, (uint64_t)renderer, (uint64_t)1); + memcpy(c->x, keep.x, sizeof c->x); + memcpy(c->v, keep.v, sizeof c->v); + c->nzcv = keep.nzcv, c->fpsr = keep.fpsr; + rt_log("resolution: the game set the scene size %ux%u", R.sw, R.sh); +} + +int res_dlss_mode(const char *m) { + decide(); + double k = dlss_share(m); + if (k == R.dlss_k || !R.on) return 0; + if (!RT_V143 || !renderer) { + rt_log("resolution: DLSS %s takes effect at the next start (the game set its render size already)", m); + return 0; + } + R.dlss_k = k; + unsigned w, h; + scene_for(k ? k : R.base_k, &w, &h); + if (w == R.sw && h == R.sh) return 0; + double s = RT_V143 ? (double)R.ow / BASE_W : 1.0; + float sc[2] = {(float)(w / s), (float)(h / s)}; + memcpy((uint8_t *)rt_patch_page() + 16, sc, sizeof sc); + R.sw = w, R.sh = h; + size_pending = 1; + rt_log("resolution: DLSS %s: scene %ux%u", m, w, h); + return 1; +} + void res_sizes(unsigned out[2], unsigned scene[2]) { decide(); out[0] = R.on ? R.ow : BASE_W, out[1] = R.on ? R.oh : BASE_H; diff --git a/runtime/shaders/dlaa_mv.comp b/runtime/shaders/dlaa_mv.comp index ea21288f..f7ced22c 100644 --- a/runtime/shaders/dlaa_mv.comp +++ b/runtime/shaders/dlaa_mv.comp @@ -1,32 +1,40 @@ #version 450 -// Camera motion vectors for DLAA (vk_dlaa.c mv_pass): each pixel's depth goes through reproj, last frame's -// view-projection times the inverse of this frame's (made on the CPU in double precision, so a still camera gives the -// identity and no motion); the vector is last frame's window position minus this frame's, in render pixels, without -// the jitter (DLSS's default convention). Objects that move on their own get only the camera's motion. -// The view-projections are TotK's scene uniform block (bound as c4 in the scene's vertex shaders), vec4 rows 3-6, with -// OpenGL clip depth (-w..w). The replay's viewports have negative heights, so window y grows downwards from ndc y = +1, -// and depth is z_ndc * 0.5 + 0.5. +// Camera motion vectors for DLAA (vk_dlaa.c mv_pass): each pixel's depth is taken back to the world through this +// frame's view-projection and forward through last frame's; the vector is last frame's window position minus this +// frame's, in render pixels, without the jitter (DLSS's default convention). Objects that move on their own get only +// the camera's motion. +// The camera block is TotK's scene uniform block (bound as c4 in the scene's vertex shaders): vec4 rows 3-6 are the +// view-projection, rows of a row-major 4x4 with OpenGL clip depth (-w..w). The replay's viewports have negative +// heights, so window y grows downwards from ndc y = +1, and depth is z_ndc * 0.5 + 0.5. layout(local_size_x = 8, local_size_y = 8) in; layout(binding = 0) uniform sampler2D depth_tex; layout(binding = 1, rg16f) uniform writeonly image2D mv_out; +layout(binding = 2, std140) uniform Cur { vec4 r[16]; } cur; +layout(binding = 3, std140) uniform Prev { vec4 r[16]; } prev; layout(push_constant) uniform P { - mat4 reproj; // this frame's clip space to last frame's vec2 size; // render size in pixels vec2 jitter; // this frame's jitter, pixels, as the scene was drawn (window x right, y down) } pc; +mat4 vp(vec4 a, vec4 b, vec4 c, vec4 d) { return transpose(mat4(a, b, c, d)); } + void main() { ivec2 p = ivec2(gl_GlobalInvocationID.xy); if (p.x >= int(pc.size.x) || p.y >= int(pc.size.y)) return; float d = texelFetch(depth_tex, p, 0).r; vec2 win = vec2(p) + 0.5 - pc.jitter; // where this pixel's surface sits without the jitter vec4 ndc = vec4(win.x / pc.size.x * 2.0 - 1.0, 1.0 - win.y / pc.size.y * 2.0, d * 2.0 - 1.0, 1.0); - vec4 clip = pc.reproj * ndc; + mat4 c = vp(cur.r[3], cur.r[4], cur.r[5], cur.r[6]); + mat4 q = vp(prev.r[3], prev.r[4], prev.r[5], prev.r[6]); + vec4 world = inverse(c) * ndc; vec2 mv = vec2(0.0); - if (clip.w > 1e-6) { - vec2 pn = clip.xy / clip.w; - vec2 pwin = vec2((pn.x * 0.5 + 0.5) * pc.size.x, (0.5 - pn.y * 0.5) * pc.size.y); - mv = pwin - win; + if (abs(world.w) > 1e-12) { + vec4 clip = q * (world / world.w); + if (clip.w > 1e-6) { + vec2 pn = clip.xy / clip.w; + vec2 pwin = vec2((pn.x * 0.5 + 0.5) * pc.size.x, (0.5 - pn.y * 0.5) * pc.size.y); + mv = pwin - win; + } } imageStore(mv_out, p, vec4(mv, 0.0, 0.0)); } diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index b1777f22..ddd097ac 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -680,8 +680,8 @@ static void tab_screen(float scale) { opt_slider("TOTK_CONTACT_SHADOWS", UI_MENU_CONTACT_SHADOWS, 0, 100, UI_MENU_CONTACT_SHADOWS_DESC, 10, "%.0f %%"); opt_toggle("TOTK_DLAA", UI_MENU_DLAA, UI_MENU_DLAA_DESC); static const UiTextId dlss[] = {UI_MENU_DLSS_DLAA, UI_MENU_DLSS_QUALITY, UI_MENU_DLSS_BALANCED, - UI_MENU_DLSS_PERFORMANCE}; - opt_choice("TOTK_DLSS", UI_MENU_DLSS, dlss, 4, UI_MENU_DLSS_DESC); + UI_MENU_DLSS_PERFORMANCE, UI_MENU_DLSS_ULTRA}; + opt_choice("TOTK_DLSS", UI_MENU_DLSS, dlss, 5, UI_MENU_DLSS_DESC); opt_toggle("TOTK_FXAA", UI_MENU_FXAA, UI_MENU_FXAA_DESC); rows_color(); opt_group(UI_MENU_HDR_GROUP); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index d023b66a..105b1ac1 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -130,11 +130,11 @@ static const char *const text_en[UI_TEXT_COUNT] = { "and less shimmer. The HUD is not touched. The mode below sets the resolution it works from. Takes effect " "after a restart.", [UI_MENU_DLSS] = "DLSS mode", - [UI_MENU_DLSS_DESC] = "DLAA draws the 3D scene at full resolution. Quality, Balanced and Performance draw it at 67, 58 " - "and 50 % and let DLSS scale it up, for more frames per second; they replace the 3D render " - "scale. Takes effect after a restart.", + [UI_MENU_DLSS_DESC] = "DLAA draws the 3D scene at full resolution. Quality, Balanced, Performance and Ultra " + "Performance draw it at 67, 58, 50 and 33 % and let DLSS scale it up, which saves GPU time; " + "they replace the 3D render scale. Takes effect at once.", [UI_MENU_DLSS_DLAA] = "DLAA (native)", [UI_MENU_DLSS_QUALITY] = "Quality", [UI_MENU_DLSS_BALANCED] = "Balanced", - [UI_MENU_DLSS_PERFORMANCE] = "Performance", + [UI_MENU_DLSS_PERFORMANCE] = "Performance", [UI_MENU_DLSS_ULTRA] = "Ultra Performance", [UI_MENU_FXAA] = "Game anti-aliasing (FXAA)", [UI_MENU_FXAA_DESC] = "The game's own FXAA edge smoothing. It blurs fine detail a little; with DLAA on it is not needed.", @@ -562,11 +562,11 @@ static const char *const text_ru[UI_TEXT_COUNT] = { "меньше мерцания. Интерфейс не затрагивается. Режим ниже задаёт разрешение, с которым он работает. " "Действует после перезапуска.", [UI_MENU_DLSS] = "Режим DLSS", - [UI_MENU_DLSS_DESC] = "DLAA рисует 3D-сцену в полном разрешении. «Качество», «Баланс» и «Производительность» рисуют её " - "в 67, 58 и 50 % и масштабируют через DLSS ради большего числа кадров; они заменяют масштаб " - "3D-отрисовки. Действует после перезапуска.", + [UI_MENU_DLSS_DESC] = "DLAA рисует 3D-сцену в полном разрешении. «Качество», «Баланс», «Производительность» и " + "«Макс. производительность» рисуют её в 67, 58, 50 и 33 % и масштабируют через DLSS, что " + "экономит время видеокарты; они заменяют масштаб 3D-отрисовки. Действует сразу.", [UI_MENU_DLSS_DLAA] = "DLAA (родное)", [UI_MENU_DLSS_QUALITY] = "Качество", [UI_MENU_DLSS_BALANCED] = "Баланс", - [UI_MENU_DLSS_PERFORMANCE] = "Производительность", + [UI_MENU_DLSS_PERFORMANCE] = "Производительность", [UI_MENU_DLSS_ULTRA] = "Макс. производительность", [UI_MENU_FXAA] = "Сглаживание игры (FXAA)", [UI_MENU_FXAA_DESC] = "Собственное сглаживание FXAA. Слегка размывает мелкие детали; с DLAA оно не нужно.", @@ -1309,13 +1309,13 @@ static const char *const text_pt_br[UI_TEXT_COUNT] = { "mais suaves e menos cintilação. O HUD não é afetado. O modo abaixo define a resolução de " "onde ele parte. Vale após reiniciar.", [UI_MENU_DLSS] = "Modo do DLSS", - [UI_MENU_DLSS_DESC] = "DLAA desenha a cena 3D na resolução cheia. Qualidade, Equilibrado e Desempenho a desenham " - "em 67, 58 e 50 % e deixam o DLSS ampliá-la, para mais quadros por segundo; eles substituem " - "a escala 3D. Vale após reiniciar.", + [UI_MENU_DLSS_DESC] = "DLAA desenha a cena 3D na resolução cheia. Qualidade, Equilibrado, Desempenho e Ultra " + "Desempenho a desenham em 67, 58, 50 e 33 % e deixam o DLSS ampliá-la, o que poupa tempo da " + "GPU; eles substituem a escala 3D. Vale na hora.", [UI_MENU_DLSS_DLAA] = "DLAA (nativo)", [UI_MENU_DLSS_QUALITY] = "Qualidade", [UI_MENU_DLSS_BALANCED] = "Equilibrado", - [UI_MENU_DLSS_PERFORMANCE] = "Desempenho", + [UI_MENU_DLSS_PERFORMANCE] = "Desempenho", [UI_MENU_DLSS_ULTRA] = "Ultra Desempenho", [UI_MENU_FXAA] = "Antisserrilhado do jogo (FXAA)", [UI_MENU_FXAA_DESC] = "A suavização de bordas FXAA do próprio jogo. Borra um pouco os detalhes finos; com o DLAA " "ligado não é necessária.", @@ -1873,11 +1873,12 @@ static const char *const text_zh_hans[UI_TEXT_COUNT] = { [UI_MENU_DLAA_DESC] = "用于 3D 场景的 NVIDIA DLSS(NVIDIA RTX 显卡;nvngx_dlss.dll 放在 totk.exe 旁):边缘更平滑,闪烁更少。HUD " "不受影响。下方的模式决定它的工作分辨率。重启后生效。", [UI_MENU_DLSS] = "DLSS 模式", - [UI_MENU_DLSS_DESC] = "DLAA 以全分辨率绘制 3D 场景。质量、平衡和性能分别以 67%、58% 和 50% 绘制,再由 DLSS 放大,以获得更高帧率;它们会取代 3D 渲染比例。重启后生效。", + [UI_MENU_DLSS_DESC] = "DLAA 以全分辨率绘制 3D 场景。质量、平衡、性能和超级性能分别以 67%、58%、50% 和 33% 绘制,再由 DLSS 放大,以节省 GPU 时间;它们会取代 3D 渲染比例。立即生效。", [UI_MENU_DLSS_DLAA] = "DLAA(原生)", [UI_MENU_DLSS_QUALITY] = "质量", [UI_MENU_DLSS_BALANCED] = "平衡", [UI_MENU_DLSS_PERFORMANCE] = "性能", + [UI_MENU_DLSS_ULTRA] = "超级性能", [UI_MENU_FXAA] = "游戏抗锯齿(FXAA)", [UI_MENU_FXAA_DESC] = "游戏自身的 FXAA 边缘平滑。会让细节略微模糊;开启 DLAA 时不需要。", [UI_MENU_TAB_CHEATS] = "金手指", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 1845d052..f4c2ebf3 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -54,7 +54,7 @@ typedef enum { UI_MENU_CONTACT_SHADOWS, UI_MENU_CONTACT_SHADOWS_DESC, UI_MENU_PCSS, UI_MENU_PCSS_DESC, UI_MENU_DLAA, UI_MENU_DLAA_DESC, UI_MENU_DLSS, UI_MENU_DLSS_DESC, UI_MENU_DLSS_DLAA, UI_MENU_DLSS_QUALITY, - UI_MENU_DLSS_BALANCED, UI_MENU_DLSS_PERFORMANCE, UI_MENU_FXAA, UI_MENU_FXAA_DESC, + UI_MENU_DLSS_BALANCED, UI_MENU_DLSS_PERFORMANCE, UI_MENU_DLSS_ULTRA, UI_MENU_FXAA, UI_MENU_FXAA_DESC, // Game UI_MENU_FPS_LIMIT, UI_MENU_FPS_LIMIT_DESC, UI_MENU_FPS_30, UI_MENU_FPS_40, UI_MENU_FPS_45, UI_MENU_FPS_60, UI_MENU_FPS_90, diff --git a/runtime/vk.h b/runtime/vk.h index 722bb260..3bc6f7ef 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -255,10 +255,20 @@ void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size); void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence); void vk_usnap_end(uint64_t fence); int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out); -const void *vk_usnap_host(uint64_t addr, uint64_t n, uint64_t align); // the copy's bytes; NULL: not copied +const uint8_t *vk_usnap_map(void); // a host copy of the current arena, at the offsets vk_usnap_find gives +const uint8_t *vk_usnap_bytes(uint64_t addr, uint64_t n); // the copy of [addr, addr+n) as of the submit, NULL: none void vk_usnap_drop(uint64_t addr, uint64_t n); // the replay makes the GPU write [addr, addr+n) of guest memory uint64_t vk_usnap_gen(void); // changes whenever a binding may resolve differently void vk_usnap_shutdown(void); // vk_exec_draw_shutdown: frees the arenas +// vk_mvprev.c: last frame's copy of the scene's object (c7) and bone (c5) blocks, for per-object motion vectors. +// vk_mvprev_step: the distance of the game's other copy of a vertex-stage block the snapshot also copies, 0: none. +// vk_mvprev_find: last frame's block of that object (prev, its address; how, MVPREV_*), 0: none. +enum { MVPREV_NONE, MVPREV_SELF, MVPREV_SIDE, MVPREV_NEARER }; +uint64_t vk_mvprev_step(int stage, int index, uint64_t size); +int vk_mvprev_on(void); +void vk_mvprev_want(uint32_t index_mask); // also keep these binding indices' last-frame copies (bit n: index n) // DLAA on and TOTK_DLAA_OBJMV not 0, as of the last present +int vk_mvprev_find(int index, uint64_t addr, uint64_t size, uint64_t *prev, int *how); +void vk_mvprev_frame(void); // each present, beside vk_dlaa_frame void vk_mem_mirror_begin(void); int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); int vk_mem_mirror_pending(void); @@ -391,6 +401,7 @@ uint32_t vk_texture_slots_gen(void); // its storage address is created, dropped, released or taken back into use. Unlocked; read it before the lookup it // guards (a lookup may create an entry). 0 when the slot was never registered: nothing is kept under 0. uint64_t vk_texture_slot_token(int id); +void vk_sampler_dlss_bias(float bias); // texture LOD bias added for DLSS upscaling (material samplers) uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage void vk_texture_storage_bound_locked(const VkTexImage *img); // bound as a storage image in the recording submission diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 798d8331..5ba6f542 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -102,6 +102,8 @@ static void present_image(const tex_t *t, const int crop[4], uint64_t at_ns, uin static volatile int32_t n_skipped; vk_exec_frame_log(t, frame); vk_dlaa_frame(); + vk_mvsurvey_frame(); + vk_mvprev_frame(); vk_mem_census(frame); if (vk_dump_armed) vk_dump_present(frame); __atomic_store_n(&gfx_counters.validation_errors, (uint64_t)vk_validation_errors_total(), __ATOMIC_RELAXED); diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index c4fd62aa..536b88f7 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -236,7 +236,7 @@ static struct { VkImageView out_view, mv_view; NgxHandle *feat; uint64_t used; // evals count when it last ran - } sizes[3], *last; // last: the one the previous DLAA frame ran with + } sizes[5], *last; // last: the one the previous DLAA frame ran with VkImage last_color; // the target DLAA ran on last frame: another one means a new history NgxParam *params; int failed, reset; @@ -250,8 +250,14 @@ static struct { uint64_t holds; // camera (vk_dlaa_draw) and motion vectors (mv_pass) uint64_t cam_addr; // this frame's scene camera block (c4 of a scene draw's vertex stage), 0: none seen - double cam_prev[16]; // last frame's view-projection (row-major), valid while cam_prev_ok - int cam_prev_ok; + VkBuffer camb; // two CAM_SLOT slots: this frame's block and last frame's + VmaAllocation camb_mem; + VkBuffer mvm; // MVM_SLOTS pairs of blocks for dlaa_mv.comp: the identity and the reprojection (mv_pass) + VmaAllocation mvm_mem; + uint8_t *mvm_map; + uint32_t mvm_slot; + double cam_prev[16]; // last frame's view-projection (rows 3-6 of its block), for the reprojection + int cam_slot, cam_prev_ok; // the slot this frame writes / the other slot holds last frame's camera VkDescriptorSetLayout mv_dsl; VkPipelineLayout mv_layout; VkPipeline mv_pipe; @@ -304,7 +310,7 @@ static void free_size(struct Size *e) { // goes for a new one. static struct Size *size_for(VkCommandBuffer cb, uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) { struct Size *e = NULL, *lru = &D.sizes[0]; - for (int i = 0; i < 3; i++) { + for (int i = 0; i < 5; i++) { if (D.sizes[i].w == w && D.sizes[i].h == h && D.sizes[i].ow == ow && D.sizes[i].oh == oh) return &D.sizes[i]; if (!D.sizes[i].out && !e) e = &D.sizes[i]; if (D.sizes[i].used < lru->used) lru = &D.sizes[i]; @@ -316,7 +322,7 @@ static struct Size *size_for(VkCommandBuffer cb, uint32_t w, uint32_t h, uint32_ VkImageUsageFlags u = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; if (!image2d(ow, oh, VK_FORMAT_R16G16B16A16_SFLOAT, u, &e->out, &e->out_mem, &e->out_view) || - !image2d(w, h, VK_FORMAT_R16G16_SFLOAT, u, &e->mv, &e->mv_mem, &e->mv_view)) + !image2d(w, h, VK_FORMAT_R16G16_SFLOAT, u | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, &e->mv, &e->mv_mem, &e->mv_view)) return NULL; VkImageMemoryBarrier2 ib[2]; VkImage im[2] = {e->out, e->mv}; @@ -356,54 +362,87 @@ static NgxResourceVK res_image(VkImageView v, VkImage img, VkFormat f, uint32_t return r; } -// ---------------------------------------------------------------- motion vectors -// Camera motion only (runtime/shaders/dlaa_mv.comp): this frame's scene camera block is read from the replay's uniform -// snapshot (vk_usnap.c, as of the submit call), and the compute pass takes every pixel through last frame's -// view-projection times the inverse of this frame's. That matrix is made here in double precision: the shader once -// inverted the view-projection itself in 32-bit floats, and the rounding gave a still camera a few hundredths of a -// pixel of motion that changed with every jitter step, which DLSS followed, so the whole picture wobbled. Characters -// and anything else that moves on its own get the camera's motion only, which DLSS mostly copes with by rejecting -// their history. -#define CAM_BYTES 256u // vec4 rows 0-15: view, view-projection (3-6), projection, inverse view +// ---------------------------------------------------------------- timing (TOTK_DLAA_TIMES=1) +// GPU timestamps around the camera motion dispatch, the per-object motion pass and DLSS's evaluation, and the CPU +// time of NGX's call and of draining the recorder before it, summed and logged every TIMES_SUMMARY frames: +// dlaa times: gpu camera motion a ms, object motion b ms, DLSS c ms (frames timed); cpu: recorder drain d ms, NGX +// evaluate e ms, motion pass recording f ms +// Each frame takes its own slot of TIMES_SLOTS; a slot's results are read when it comes round again (by then the GPU is +// long done with it; not ready: that frame is left out). +enum { T_MV0, T_MV1, T_OBJ, T_DL0, T_DL1, T_N }; +#define TIMES_SLOTS 8u +#define TIMES_SUMMARY 600 +static struct { + int on; // -1 not read + VkQueryPool pool; + uint32_t slot; + int used[TIMES_SLOTS]; // the slot holds a frame's timestamps (all T_N written) + double gpu[3]; // ms summed: camera motion, object motion, DLSS + uint64_t frames, cpu_drain, cpu_ngx, cpu_mv; // ns summed +} T = {.on = -1}; -// m inverted in place (row-major 4x4, Gauss-Jordan with partial pivoting). 0: singular. -static int invert4(double m[16]) { - double a[4][8]; - for (int r = 0; r < 4; r++) - for (int c = 0; c < 8; c++) a[r][c] = c < 4 ? m[r * 4 + c] : (c - 4 == r); - for (int c = 0; c < 4; c++) { - int p = c; - for (int r = c + 1; r < 4; r++) - if (fabs(a[r][c]) > fabs(a[p][c])) p = r; - if (fabs(a[p][c]) < 1e-30) return 0; - for (int k = 0; k < 8; k++) { - double t = a[c][k]; - a[c][k] = a[p][k], a[p][k] = t; - } - double inv = 1.0 / a[c][c]; - for (int k = 0; k < 8; k++) a[c][k] *= inv; - for (int r = 0; r < 4; r++) { - if (r == c || a[r][c] == 0.0) continue; - double f = a[r][c]; - for (int k = 0; k < 8; k++) a[r][k] -= f * a[c][k]; +static int times_on(void) { + if (T.on < 0) { + T.on = rt_knob("DLAA_TIMES", "0"); + VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, + .queryType = VK_QUERY_TYPE_TIMESTAMP, .queryCount = TIMES_SLOTS * T_N}; + if (T.on && vkCreateQueryPool(vk_ctx()->device, &qi, NULL, &T.pool) != VK_SUCCESS) T.on = 0; + } + return T.on; +} + +static void times_mark(VkCommandBuffer cb, int which) { + if (times_on()) vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, T.pool, T.slot * T_N + (uint32_t)which); +} + +// At the start of a frame's timing (mv_pass): the slot's last results are read and it is reset. +static void times_begin(VkCommandBuffer cb) { + if (!times_on()) return; + T.slot = (T.slot + 1) % TIMES_SLOTS; + if (T.used[T.slot]) { + uint64_t q[T_N]; + if (vkGetQueryPoolResults(vk_ctx()->device, T.pool, T.slot * T_N, T_N, sizeof q, q, sizeof q[0], + VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) { + double ns = vk_ctx()->props.limits.timestampPeriod / 1e6; // ticks to ms + T.gpu[0] += (double)(q[T_MV1] - q[T_MV0]) * ns, T.gpu[1] += (double)(q[T_OBJ] - q[T_MV1]) * ns; + T.gpu[2] += (double)(q[T_DL1] - q[T_DL0]) * ns; + if (++T.frames % TIMES_SUMMARY == 0) { + rt_log("dlaa times: gpu camera motion %.3f ms, object motion %.3f ms, DLSS %.3f ms (%llu frames timed); " + "cpu: recorder drain %.3f ms, NGX evaluate %.3f ms, motion pass recording %.3f ms", + T.gpu[0] / TIMES_SUMMARY, T.gpu[1] / TIMES_SUMMARY, T.gpu[2] / TIMES_SUMMARY, + (unsigned long long)T.frames, T.cpu_drain / 1e6 / TIMES_SUMMARY, T.cpu_ngx / 1e6 / TIMES_SUMMARY, + T.cpu_mv / 1e6 / TIMES_SUMMARY); + T.gpu[0] = T.gpu[1] = T.gpu[2] = 0, T.cpu_drain = T.cpu_ngx = T.cpu_mv = 0; + } } } - for (int r = 0; r < 4; r++) - for (int c = 0; c < 4; c++) m[r * 4 + c] = a[r][c + 4]; - return 1; + T.used[T.slot] = 0; + vkCmdResetQueryPool(cb, T.pool, T.slot * T_N, T_N); } +// ---------------------------------------------------------------- motion vectors +// The camera's motion first (runtime/shaders/dlaa_mv.comp): this frame's scene camera block is copied out of the +// replay's uniform snapshot (vk_usnap.c, as of the submit call) into one slot of camb, last frame's stays in the other, +// and the compute pass reprojects every pixel's depth through the two (inv4 and mv_pass below). Then the per-object +// pass (vk_exec_draw.c, vk_mvobj_execute) writes the motion of whatever moves on its own over it. +#define CAM_BYTES 256u // vec4 rows 0-15: view, view-projection (3-6), projection, inverse view +#define CAM_BLOCK 0x940u // the whole block, kept for the per-object motion pass's vertex shaders (vk_exec_draw.c) +#define CAM_SLOT 0xa00u // camb slot stride +#define MVM_SLOTS 16u // frames the GPU may still be reading the reprojection blocks of + static int mv_init(void) { if (D.mv_ok >= 0) return D.mv_ok; D.mv_ok = 0; const VkCtx *c = vk_ctx(); - VkDescriptorSetLayoutBinding b[2] = { + VkDescriptorSetLayoutBinding b[4] = { {0, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, VK_SHADER_STAGE_COMPUTE_BIT, NULL}, - {1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, NULL}}; + {1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, NULL}, + {2, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, NULL}, + {3, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, NULL}}; VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, - .bindingCount = 2, .pBindings = b}; - VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 80}; + .bindingCount = 4, .pBindings = b}; + VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 16}; VkSamplerCreateInfo si = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, .magFilter = VK_FILTER_NEAREST, .minFilter = VK_FILTER_NEAREST, .mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST, .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, @@ -412,12 +451,24 @@ static int mv_init(void) { VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof dlaa_mv_spv, .pCode = dlaa_mv_spv}; VkShaderModule m = VK_NULL_HANDLE; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 2 * CAM_SLOT, + .usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; if (vkCreateDescriptorSetLayout(c->device, &dci, NULL, &D.mv_dsl) != VK_SUCCESS) goto fail; + VkBufferCreateInfo mbi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = MVM_SLOTS * 2 * CAM_BYTES, + .usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT}; + VmaAllocationCreateInfo mai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | + VMA_ALLOCATION_CREATE_MAPPED_BIT}; + VmaAllocationInfo minfo; + if (vmaCreateBuffer(c->vma, &mbi, &mai, &D.mvm, &D.mvm_mem, &minfo) != VK_SUCCESS) goto fail; + D.mvm_map = minfo.pMappedData; VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, .pSetLayouts = &D.mv_dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr}; if (vkCreatePipelineLayout(c->device, &li, NULL, &D.mv_layout) != VK_SUCCESS || vkCreateSampler(c->device, &si, NULL, &D.mv_samp) != VK_SUCCESS || - vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) + vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS || + vmaCreateBuffer(c->vma, &bi, &ai, &D.camb, &D.camb_mem, NULL) != VK_SUCCESS) goto fail; VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, @@ -432,48 +483,275 @@ fail: return 0; } +// The reprojection from this frame's clip space to last frame's, last * inverse(this), in double precision: the shader +// took each pixel back to world space through a float inverse of the view-projection, and with TotK's world +// coordinates in the thousands and its far plane the result was noise of a fraction of a pixel and more on distant +// surfaces, which DLSS followed into a shimmer like heat haze (roofs and trees, most of all in the upscaling modes). +// The shader is unchanged: its "this frame" block gets the identity and its "last frame" block this matrix, so it +// applies just the one matrix to each pixel. A still camera gives the identity, and zero motion exactly. +static int inv4(const double m[16], double o[16]) { + double a[4][8]; + for (int r = 0; r < 4; r++) + for (int k = 0; k < 8; k++) a[r][k] = k < 4 ? m[r * 4 + k] : (k - 4 == r); + for (int col = 0; col < 4; col++) { + int piv = col; + for (int r = col + 1; r < 4; r++) + if (fabs(a[r][col]) > fabs(a[piv][col])) piv = r; + if (fabs(a[piv][col]) < 1e-300) return 0; + for (int k = 0; k < 8; k++) { double t = a[col][k]; a[col][k] = a[piv][k]; a[piv][k] = t; } + double d = a[col][col]; + for (int k = 0; k < 8; k++) a[col][k] /= d; + for (int r = 0; r < 4; r++) + if (r != col) { + double f = a[r][col]; + for (int k = 0; k < 8; k++) a[r][k] -= f * a[col][k]; + } + } + for (int r = 0; r < 4; r++) + for (int k = 0; k < 4; k++) o[r * 4 + k] = a[r][k + 4]; + return 1; +} + // The pass, recorded through the recorder like the replay's own commands. 0: no camera this frame (the vectors stay // zero and DLSS starts a new history). -static int mv_pass(VkCommandBuffer cb, const struct Size *e, VkImageView zv, float jx, float jy) { +static int mv_pass(VkExec *ex, const struct Size *e, VkImageView zv, float jx, float jy) { + VkCommandBuffer cb = ex->cb; uint32_t w = e->w, h = e->h; - const float *cam = D.cam_addr ? vk_usnap_host(D.cam_addr, CAM_BYTES, 4) : NULL; - if (!mv_init() || !cam) { + VkDescriptorBufferInfo cam; + if (!mv_init() || !D.cam_addr || !vk_usnap_find(D.cam_addr, CAM_BLOCK, 4, &cam)) { D.mv_none++; D.cam_prev_ok = 0; return 0; } - double cur[16], prev[16], m[16]; - for (int i = 0; i < 16; i++) cur[i] = cam[12 + i]; // rows 3-6 - int had_prev = D.cam_prev_ok; - memcpy(prev, D.cam_prev, sizeof prev); - memcpy(D.cam_prev, cur, sizeof cur), D.cam_prev_ok = 1; - if (!had_prev || !invert4(cur)) return 0; // nothing to reproject into: zero motion, new history - for (int r = 0; r < 4; r++) - for (int c = 0; c < 4; c++) { - double v = 0; - for (int k = 0; k < 4; k++) v += prev[r * 4 + k] * cur[k * 4 + c]; - m[r * 4 + c] = v; + // this frame's view-projection as the snapshot holds it, for the reprojection; kept as last frame's for the next + const float *cr = (const float *)vk_usnap_bytes(D.cam_addr + 48, 64); + double cur_vp[16], m[16], ci[16]; + int have_m = 0; + if (cr) { + for (int i = 0; i < 16; i++) cur_vp[i] = cr[i]; + if (D.cam_prev_ok && inv4(cur_vp, ci)) { + for (int r = 0; r < 4; r++) + for (int k = 0; k < 4; k++) { + double v = 0; + for (int j = 0; j < 4; j++) v += D.cam_prev[r * 4 + j] * ci[j * 4 + k]; + m[r * 4 + k] = v; + } + have_m = 1; } + } + int s = D.cam_slot, had_prev = D.cam_prev_ok; + VkBufferCopy bc = {cam.offset, (VkDeviceSize)s * CAM_SLOT, CAM_BLOCK}; + vkCmdCopyBuffer(cb, cam.buffer, D.camb, 1, &bc); + full_barrier(cb); + D.cam_slot ^= 1, D.cam_prev_ok = cr != NULL; + if (cr) memcpy(D.cam_prev, cur_vp, sizeof cur_vp); + if (!had_prev || !have_m) return 0; // nothing to reproject into: zero motion, new history VkDescriptorImageInfo zi = {D.mv_samp, zv, VK_IMAGE_LAYOUT_GENERAL}; VkDescriptorImageInfo oi = {VK_NULL_HANDLE, e->mv_view, VK_IMAGE_LAYOUT_GENERAL}; - VkWriteDescriptorSet wr[2] = { + uint32_t ms = D.mvm_slot++ % MVM_SLOTS; + float *blk = (float *)(D.mvm_map + (size_t)ms * 2 * CAM_BYTES); // rows 3-6 of each block are read + memset(blk, 0, 2 * CAM_BYTES); + for (int r = 0; r < 4; r++) + for (int k = 0; k < 4; k++) + blk[12 + r * 4 + k] = r == k ? 1.0f : 0.0f, blk[64 + 12 + r * 4 + k] = (float)m[r * 4 + k]; + vmaFlushAllocation(vk_ctx()->vma, D.mvm_mem, (VkDeviceSize)ms * 2 * CAM_BYTES, 2 * CAM_BYTES); + VkDescriptorBufferInfo cur = {D.mvm, (VkDeviceSize)ms * 2 * CAM_BYTES, CAM_BYTES}; + VkDescriptorBufferInfo prev = {D.mvm, (VkDeviceSize)ms * 2 * CAM_BYTES + CAM_BYTES, CAM_BYTES}; + VkWriteDescriptorSet wr[4] = { {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 0, .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .pImageInfo = &zi}, {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 1, .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .pImageInfo = &oi}}; + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .pImageInfo = &oi}, + {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 2, .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, .pBufferInfo = &cur}, + {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 3, .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, .pBufferInfo = &prev}}; + times_begin(cb); + times_mark(cb, T_MV0); vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, D.mv_pipe); - rec_push(vk_pipeline_push_fn(), cb, VK_PIPELINE_BIND_POINT_COMPUTE, D.mv_layout, 0, 2, wr); - float pc[20]; - for (int r = 0; r < 4; r++) - for (int c = 0; c < 4; c++) pc[c * 4 + r] = (float)m[r * 4 + c]; // GLSL mat4 is column-major - pc[16] = (float)w, pc[17] = (float)h, pc[18] = jx, pc[19] = jy; + rec_push(vk_pipeline_push_fn(), cb, VK_PIPELINE_BIND_POINT_COMPUTE, D.mv_layout, 0, 4, wr); + float pc[4] = {(float)w, (float)h, jx, jy}; vkCmdPushConstants(cb, D.mv_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); vkCmdDispatch(cb, (w + 7) / 8, (h + 7) / 8, 1); full_barrier(cb); + times_mark(cb, T_MV1); + vk_mvcheck_camera(cb, e->mv, w, h, jx, jy); // TOTK_MV_CHECK (test) + // the characters' own motion over the camera's (vk_exec_draw.c, per-object motion) + uint64_t c0 = T.on > 0 ? kern_now_ns() : 0; + if (vk_mvobj_execute(ex, D.cam_addr, e->mv_view, zv, D.depth.format, w, h)) full_barrier(cb); + if (T.on > 0) T.cpu_mv += kern_now_ns() - c0; + times_mark(cb, T_OBJ); + if (T.on > 0) T.used[T.slot] = 1; // the DLSS marks follow in evaluate + vk_mvcheck_final(cb, ex->fence, e->mv, D.color.image, D.color.format, D.color.level, D.color.base_layer, w, h, jx, jy); D.mv_frames++; return 1; } +// Last frame's camera block for the per-object motion pass, recorded before this frame's mv_pass: that pass writes +// this frame's into slot cam_slot and leaves the other slot, last frame's, as it is. 0: none (no history). +int vk_dlaa_prev_camera(VkDescriptorBufferInfo *out, float jitter[2]) { + if (!dl_active() || !D.cam_prev_ok || !D.camb) return 0; + *out = (VkDescriptorBufferInfo){D.camb, (VkDeviceSize)(D.cam_slot ^ 1) * CAM_SLOT, CAM_BLOCK}; + jitter[0] = D.jit_ok ? D.jx : 0.0f, jitter[1] = D.jit_ok ? D.jy : 0.0f; + return 1; +} + +// ---------------------------------------------------------------- reactive mask +// Particles, smoke, sparks and other transparents have no motion vectors of their own: the motion image holds the +// surface's behind them, and DLSS carried their old colour along that surface (trails behind sparks, smoke smeared +// with the camera). DLSS takes a mask of where to trust this frame's colour over the history (NGX +// "DLSS.Input.Bias.Current.Color.Mask"). It is made from the picture itself, at no cost on the CPU: the light target is +// copied when the transparents start (the pass that arms DLAA, vk_dlaa_pass) and compared with what DLSS gets +// (runtime/dlaa_mask_spv.h, tools/make_dlaa_mask.py): the share of a pixel's colour that changed, from MASK_THR on, +// times MASK_GAIN. Off by default (TOTK_DLSS_REACTIVE=1 turns it on, TOTK_DLAA_MASKDEBUG=1 shows the mask instead of +// the picture): the pass that arms also draws the sky and blended fog layers over the solid scene, and the mask +// covered the sky, sunlit trees and roofs and a wide fog region besides the particles (seen with MASKDEBUG at 19:45 in +// the first save), where DLSS would give up its anti-aliasing. It needs the particle draws alone (drawn again into the +// mask, like the motion draws) before it can be on. In the upscaling modes the light target DLSS reads also holds the +// bloom by then; the copy would have to be compared at the end of the scene instead. +#include "dlaa_mask_spv.h" +#define MASK_THR 0.1f +#define MASK_GAIN 2.5f +#define SNAPS 3 + +static struct { + struct Snap { // per light target size and format: its copy at arming, and the mask + uint32_t w, h; + VkFormat f; + VkImage before, mask; + VmaAllocation before_mem, mask_mem; + VkImageView before_view, mask_view; + uint64_t used; + } s[SNAPS]; + struct Snap *cur; // this frame's copy, NULL: none + VkDescriptorSetLayout dsl; + VkPipelineLayout layout; + VkPipeline pipe; + int ok, knob; // ok: -1 not built, 0 failed, 1 ready; knob: -1 not read + uint64_t made, missed; // frames with a mask; frames DLSS ran without the copy +} R = {.ok = -1, .knob = -1}; + +static int mask_init(void) { + if (R.knob < 0) R.knob = rt_knob("DLSS_REACTIVE", "0"); + if (!R.knob) return 0; + if (R.ok >= 0) return R.ok; + R.ok = 0; + const VkCtx *c = vk_ctx(); + VkDescriptorSetLayoutBinding b[3] = { + {0, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, VK_SHADER_STAGE_COMPUTE_BIT, NULL}, + {1, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, VK_SHADER_STAGE_COMPUTE_BIT, NULL}, + {2, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, NULL}}; + VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, + .bindingCount = 3, .pBindings = b}; + VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 16}; + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof dlaa_mask_spv, + .pCode = dlaa_mask_spv}; + VkShaderModule m = VK_NULL_HANDLE; + if (vkCreateDescriptorSetLayout(c->device, &dci, NULL, &R.dsl) != VK_SUCCESS) goto fail; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, + .pSetLayouts = &R.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr}; + if (vkCreatePipelineLayout(c->device, &li, NULL, &R.layout) != VK_SUCCESS || + vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) + goto fail; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = R.layout}; + VkResult r = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &R.pipe); + vkDestroyShaderModule(c->device, m, NULL); + if (r != VK_SUCCESS) goto fail; + return R.ok = 1; +fail: + rt_log("dlaa: the reactive mask pass could not be set up: DLSS runs without it"); + return 0; +} + +static void free_snap(struct Snap *s) { + const VkCtx *c = vk_ctx(); + vkDeviceWaitIdle(c->device); // a fourth size: the evicted images may still be in flight + if (s->before_view) vkDestroyImageView(c->device, s->before_view, NULL); + if (s->mask_view) vkDestroyImageView(c->device, s->mask_view, NULL); + if (s->before) vmaDestroyImage(c->vma, s->before, s->before_mem); + if (s->mask) vmaDestroyImage(c->vma, s->mask, s->mask_mem); + memset(s, 0, sizeof *s); +} + +static struct Snap *snap_for(VkCommandBuffer cb, uint32_t w, uint32_t h, VkFormat f) { + static uint64_t tick; + struct Snap *e = NULL, *lru = &R.s[0]; + for (int i = 0; i < SNAPS; i++) { + if (R.s[i].w == w && R.s[i].h == h && R.s[i].f == f) return R.s[i].used = ++tick, &R.s[i]; + if (!R.s[i].before && !e) e = &R.s[i]; + if (R.s[i].used < lru->used) lru = &R.s[i]; + } + if (!e) free_snap(e = lru); + VkImageUsageFlags u = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + if (!image2d(w, h, f, u, &e->before, &e->before_mem, &e->before_view) || + !image2d(w, h, VK_FORMAT_R8_UNORM, u | VK_IMAGE_USAGE_STORAGE_BIT, &e->mask, &e->mask_mem, &e->mask_view)) { + free_snap(e); + return NULL; + } + VkImageMemoryBarrier2 ib[2]; + VkImage im[2] = {e->before, e->mask}; + for (int i = 0; i < 2; i++) + ib[i] = (VkImageMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = im[i], + .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 2, + .pImageMemoryBarriers = ib}; + vkCmdPipelineBarrier2(cb, &di); + e->w = w, e->h = h, e->f = f, e->used = ++tick; + return e; +} + +// At arming: the light target as it is before the transparents, outside any pass. +static void mask_snapshot(VkExec *ex) { + R.cur = NULL; + if (mask_init() <= 0) return; + const VkExecTarget *t = &D.color; + struct Snap *s = snap_for(ex->cb, t->width, t->height, t->format); + if (!s) return; + vk_exec_barrier(ex); + VkImageCopy c = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, t->level, t->base_layer, 1}, + .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .extent = {t->width, t->height, 1}}; + vkCmdCopyImage(ex->cb, t->image, VK_IMAGE_LAYOUT_GENERAL, s->before, VK_IMAGE_LAYOUT_GENERAL, 1, &c); + full_barrier(ex->cb); + ex->recorded += 2; + R.cur = s; +} + +// Before DLSS: the mask from the copy and the light target (view cv). NULL: none this frame. +static struct Snap *mask_pass(VkCommandBuffer cb, VkImageView cv, uint32_t w, uint32_t h) { + struct Snap *s = R.cur; + R.cur = NULL; + if (mask_init() <= 0) return NULL; + if (!s || s->w != w || s->h != h || s->f != D.color.format) return R.missed++, NULL; + VkDescriptorImageInfo bi = {D.mv_samp, s->before_view, VK_IMAGE_LAYOUT_GENERAL}; + VkDescriptorImageInfo ai = {D.mv_samp, cv, VK_IMAGE_LAYOUT_GENERAL}; + VkDescriptorImageInfo mi = {VK_NULL_HANDLE, s->mask_view, VK_IMAGE_LAYOUT_GENERAL}; + VkWriteDescriptorSet wr[3] = { + {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 0, .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .pImageInfo = &bi}, + {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 1, .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .pImageInfo = &ai}, + {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 2, .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .pImageInfo = &mi}}; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, R.pipe); + rec_push(vk_pipeline_push_fn(), cb, VK_PIPELINE_BIND_POINT_COMPUTE, R.layout, 0, 3, wr); + float pc[4] = {(float)w, (float)h, MASK_THR, MASK_GAIN}; + vkCmdPushConstants(cb, R.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); + vkCmdDispatch(cb, (w + 7) / 8, (h + 7) / 8, 1); + full_barrier(cb); + R.made++; + return s; +} + // ---------------------------------------------------------------- jitter // The scene's draws (every pass drawing with the scene depth, until the DLAA point) have their viewport moved by a // sub-pixel Halton(2,3) offset, which shifts what they rasterise as a jittered projection would; full-screen passes @@ -493,9 +771,39 @@ void vk_dlaa_viewport(VkExec *ex, VkViewport *v) { void vk_dlaa_draw(VkExec *ex) { if (!dl_active() || D.ended) return; + static int survey = -1; // TOTK_DLAA_SURVEY=1 (test): what is drawn into the armed light target + if (survey < 0) survey = rt_knob("DLAA_SURVEY", "0"); + static uint64_t dfn[16]; // scene draws writing depth, by depth test (reversed depth: GREATER/GEQUAL) + if (survey && ex->depth.image && ex->depth.image == D.jdepth && ex->pipe.ds.dt && ex->pipe.ds.dw) + dfn[ex->pipe.ds.df & 15]++; + if (survey && D.armed && ex->color[0].image == D.color.image) { + static uint64_t n[2][2][2], arms, frames, last_frame; // [blend][depth write][depth test] + const VkPipeState *s = &ex->pipe; + n[s->color.blend_en & 1][s->ds.dw][s->ds.dt]++; + uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (f != last_frame) last_frame = f, frames++; + static uint64_t ps; + if (ex->pass_serial != ps) ps = ex->pass_serial, arms++; + if (frames % 300 == 0) { + static uint64_t logged; + if (logged != frames) { + logged = frames; + rt_log("dlaa survey: scene draws writing depth by depth test (NVN func 1-8: never less equal lequal " + "greater notequal gequal always): %llu %llu %llu %llu %llu %llu %llu %llu", + (unsigned long long)dfn[1], (unsigned long long)dfn[2], (unsigned long long)dfn[3], + (unsigned long long)dfn[4], (unsigned long long)dfn[5], (unsigned long long)dfn[6], + (unsigned long long)dfn[7], (unsigned long long)dfn[8]); + rt_log("dlaa survey: %llu frames, %.1f passes a frame into the light target after arming; draws a frame " + "(blend/depth write/depth test): 000 %.1f 001 %.1f 010 %.1f 011 %.1f 100 %.1f 101 %.1f 110 %.1f 111 %.1f", + (unsigned long long)frames, (double)arms / frames, (double)n[0][0][0] / frames, (double)n[0][0][1] / frames, + (double)n[0][1][0] / frames, (double)n[0][1][1] / frames, (double)n[1][0][0] / frames, + (double)n[1][0][1] / frames, (double)n[1][1][0] / frames, (double)n[1][1][1] / frames); + } + } + } const p_bind_buf *b = &ex->ubo[0][1]; // constant bank c4: the replay keeps user banks from c3 on const VkExecTarget *z = &ex->depth; - if (b->size == 0x940 && b->addr && z->image && z->width >= 640 && z->format == VK_FORMAT_D32_SFLOAT && + if (b->size == 0x940 && b->addr && z->image && z->width >= DLAA_SCENE_MIN_W && z->format == VK_FORMAT_D32_SFLOAT && (!D.jdepth || z->image == D.jdepth)) D.cam_addr = b->addr; } @@ -503,10 +811,16 @@ void vk_dlaa_draw(VkExec *ex) { // The jitter moves on right after DLAA ran, not at the present: the replay can record the next frame's scene before // the present of this one is handled, and a step taken there landed in the middle of a frame (worse after an off/on // switch, which shifted that order). Jitter needs a frame DLSS ran on; a frame without the DLAA point stops it. -static void next_jitter(void) { +// The sequence is as long as NVIDIA's DLSS guide asks for the upscaling ratio, 8 x (output / scene)^2 phases (18 in +// Quality, 24 in Balanced, 32 in Performance, 72 in Ultra Performance), and 16 at least, the length DLAA has used from the start. +static void next_jitter(uint32_t w, uint32_t ow) { static int nj = -1; if (nj < 0) nj = !rt_knob("DLAA_JITTER", "1"); // TOTK_DLAA_JITTER=0: no jitter (test) - uint32_t i = D.jframe++ % 16 + 1; + double r = w ? (double)ow / w : 1.0; + uint32_t phases = (uint32_t)ceil(8.0 * r * r - 1e-6); + if (phases < 16) phases = 16; + if (phases > 72) phases = 72; + uint32_t i = D.jframe++ % phases + 1; D.jx = halton(i, 2) - 0.5f, D.jy = halton(i, 3) - 0.5f; D.jit_ok = !D.hold && !nj; } @@ -535,14 +849,21 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { float jx = D.jit_used ? D.ux : 0.0f, jy = D.jit_used ? D.uy : 0.0f; static int no_mv = -1; if (no_mv < 0) no_mv = !rt_knob("DLAA_MV", "1"); // TOTK_DLAA_MV=0: zero motion (test) - if (no_mv || !mv_pass(cb, e, zv, jx, jy)) { + if (no_mv || !mv_pass(ex, e, zv, jx, jy)) { VkClearColorValue zero = {{0, 0, 0, 0}}; VkImageSubresourceRange all = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; vkCmdClearColorImage(cb, e->mv, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &all); full_barrier(cb); if (!no_mv) D.reset = 1, D.why[1]++; // no camera: the history cannot be moved along } + VkImageView cv = vk_exec_target_view(t->img, t->format, VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT, t->level, + t->base_layer, 1, 0); + struct Snap *mk = D.mv_samp ? mask_pass(cb, cv, w, h) : NULL; + int timed = T.on > 0 && T.used[T.slot]; // mv_pass began this frame's slot + if (timed) times_mark(cb, T_DL0); + uint64_t c0 = timed ? kern_now_ns() : 0; rec_drain(); // NGX records into the command buffer itself: the recorder thread must have made every call so far + if (timed) T.cpu_drain += kern_now_ns() - c0; if (!e->feat) { p_setu(p, "CreationNodeMask", 1); p_setu(p, "VisibilityNodeMask", 1); @@ -554,15 +875,30 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { // IsHDR | MVLowRes | AutoExposure: linear scene light, motion vectors at the render size, no exposure texture p_seti(p, "DLSS.Feature.Create.Flags", (1 << 0) | (1 << 1) | (1 << 6)); p_seti(p, "DLSS.Enable.Output.Subrects", 0); + // The model per mode (NVSDK_NGX_DLSS_Hint_Render_Preset: 1 A ... 11 K, 12 L, 13 M), as NVIDIA recommends for + // DLSS 4.5 (310.x): K (transformer) for DLAA, Quality and Balanced, M for Performance, L for Ultra Performance. + // M and L cost more on cards without FP8 (RTX 20/30). TOTK_DLSS_PRESET= takes one for every mode, + // default the DLL's own choice. + const char *pv = cfg_get("TOTK_DLSS_PRESET"); + int pk = 11, pm = 13, pl = 12; + if (pv && *pv) { + int c = pv[0] | 0x20; + pk = pm = pl = c >= 'a' && c <= 'm' && !pv[1] ? c - 'a' + 1 : 0; // "default": 0, the DLL's choice + } + p_setu(p, "DLSS.Hint.Render.Preset.DLAA", (unsigned)pk); + p_setu(p, "DLSS.Hint.Render.Preset.UltraQuality", (unsigned)pk); + p_setu(p, "DLSS.Hint.Render.Preset.Quality", (unsigned)pk); + p_setu(p, "DLSS.Hint.Render.Preset.Balanced", (unsigned)pk); + p_setu(p, "DLSS.Hint.Render.Preset.Performance", (unsigned)pm); + p_setu(p, "DLSS.Hint.Render.Preset.UltraPerformance", (unsigned)pl); NgxResult r = N.create(cb, NGX_FEATURE_SUPERSAMPLING, p, &e->feat); if (!NGX_OK(r) || !e->feat) return dl_fail("DLSS feature creation", r); D.reset = 1; - rt_log("dlaa: DLSS feature created at %ux%u -> %ux%u, quality value %d (scene format %d, depth format %d)", w, - h, ow, oh, perf_quality(w, ow), (int)t->format, (int)z->format); + int q = perf_quality(w, ow), pr = q == 0 ? pm : q == 3 ? pl : pk; + rt_log("dlaa: DLSS feature created at %ux%u -> %ux%u, quality value %d, preset %c (scene format %d, depth format " + "%d)", w, h, ow, oh, q, pr ? 'A' + pr - 1 : '-', (int)t->format, (int)z->format); } if (D.last_color != t->image) D.reset = 1, D.why[2]++, D.last_color = t->image; - VkImageView cv = vk_exec_target_view(t->img, t->format, VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT, t->level, - t->base_layer, 1, 0); NgxResourceVK rc = res_image(cv, t->image, t->format, w, h, VK_IMAGE_ASPECT_COLOR_BIT, 0); NgxResourceVK ro = res_image(e->out_view, e->out, VK_FORMAT_R16G16B16A16_SFLOAT, ow, oh, VK_IMAGE_ASPECT_COLOR_BIT, 1); NgxResourceVK rz = res_image(zv, z->image, z->format, w, h, VK_IMAGE_ASPECT_DEPTH_BIT, 0); @@ -571,6 +907,9 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { p_setp(p, "Output", &ro); p_setp(p, "Depth", &rz); p_setp(p, "MotionVectors", &rm); + NgxResourceVK rk; + if (mk) rk = res_image(mk->mask_view, mk->mask, VK_FORMAT_R8_UNORM, w, h, VK_IMAGE_ASPECT_COLOR_BIT, 0); + p_setp(p, "DLSS.Input.Bias.Current.Color.Mask", mk ? &rk : NULL); // the parameters outlive the frame p_setf(p, "Jitter.Offset.X", jx); p_setf(p, "Jitter.Offset.Y", jy); p_setf(p, "Sharpness", 0.0f); @@ -582,30 +921,44 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { p_setf(p, "DLSS.Pre.Exposure", 1.0f); p_setf(p, "DLSS.Exposure.Scale", 1.0f); p_setf(p, "FrameTimeDeltaInMsec", 16.67f); + c0 = timed ? kern_now_ns() : 0; NgxResult r = N.evaluate(cb, e->feat, p, NULL); + if (timed) T.cpu_ngx += kern_now_ns() - c0; if (!NGX_OK(r)) return dl_fail("DLSS evaluation", r); + if (timed) times_mark(cb, T_DL1); // through the recorder, after NGX's own commands D.reset = 0; D.last = e, e->used = D.evals + 1; full_barrier(cb); + vk_mvcheck_output(cb, e->out, ow, oh); // TOTK_MV_CHECK (test) // the result into the scene target (DLAA: the bloom, the post passes and the HUD read it and draw over it) or into // the scaling pass's target (the HUD draws over it) VkImageBlit b = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .srcOffsets = {{0, 0, 0}, {(int32_t)ow, (int32_t)oh, 1}}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, dst->level, dst->base_layer, 1}, .dstOffsets = {{0, 0, 0}, {(int32_t)ow, (int32_t)oh, 1}}}; - vkCmdBlitImage(cb, e->out, VK_IMAGE_LAYOUT_GENERAL, dst->image, VK_IMAGE_LAYOUT_GENERAL, 1, &b, VK_FILTER_NEAREST); + // TOTK_DLAA_MVDEBUG=1 (test): the motion vectors in place of the picture, x in red and y in green, render pixels + // as scene light (the post passes still run over them) + static int mvdebug = -1; + if (mvdebug < 0) mvdebug = rt_knob("DLAA_MVDEBUG", "0"); + static int maskdebug = -1; + if (maskdebug < 0) maskdebug = rt_knob("DLAA_MASKDEBUG", "0"); + if (mvdebug || (maskdebug && mk)) b.srcOffsets[1] = (VkOffset3D){(int32_t)w, (int32_t)h, 1}; + vkCmdBlitImage(cb, mvdebug ? e->mv : maskdebug && mk ? mk->mask : e->out, VK_IMAGE_LAYOUT_GENERAL, dst->image, VK_IMAGE_LAYOUT_GENERAL, 1, &b, + VK_FILTER_NEAREST); full_barrier(cb); if (dst != t) vk_dlaa_swap_img = t->img, vk_dlaa_swap_view = e->out_view; // the scaling draw reads DLSS's picture ex->recorded += 4; vk_exec_draw_new_cb(); // NGX bound pipelines and descriptor sets of its own: the next draw binds everything again D.jdepth = z->image; - next_jitter(); + next_jitter(w, ow); if (++D.evals == 1 || D.evals % 3600 == 0) rt_log("dlaa: %llu frames anti-aliased (scene %ux%u, output %ux%u; %llu frames without the DLAA point, %llu with camera " - "motion, %llu without a camera, %llu jitter holds; resets: %llu size, %llu camera, %llu target)", + "motion, %llu without a camera, %llu jitter holds; resets: %llu size, %llu camera, %llu target; reactive mask %llu, " + "without its copy %llu)", (unsigned long long)D.evals, w, h, ow, oh, (unsigned long long)D.frames_missed, (unsigned long long)D.mv_frames, (unsigned long long)D.mv_none, (unsigned long long)D.holds, - (unsigned long long)D.why[0], (unsigned long long)D.why[1], (unsigned long long)D.why[2]); + (unsigned long long)D.why[0], (unsigned long long)D.why[1], (unsigned long long)D.why[2], + (unsigned long long)R.made, (unsigned long long)R.missed); } static int scene_color(VkFormat f) { @@ -615,10 +968,10 @@ static int scene_color(VkFormat f) { void vk_dlaa_pass(VkExec *ex) { const VkExecTarget *c = &ex->color[0], *z = &ex->depth; if (!dl_active() || D.done) return; - // The G-buffer: several colour targets with a D32F depth of at least 640 wide. Only the light target drawn after it + // The G-buffer: several colour targets with a D32F depth of at least DLAA_SCENE_MIN_W wide. Only the light target drawn after it // arms the frame; at night the game draws into the light target alone with the scene depth before the G-buffer too, // and DLAA then ran at a small pass ahead of the whole scene, which reached the screen jittered. - if (ex->pass_ncolor >= 3 && z->image && z->format == VK_FORMAT_D32_SFLOAT && z->width >= 640) { + if (ex->pass_ncolor >= 3 && z->image && z->format == VK_FORMAT_D32_SFLOAT && z->width >= DLAA_SCENE_MIN_W) { D.gbuf = 1, D.armed = 0; return; } @@ -627,10 +980,12 @@ void vk_dlaa_pass(VkExec *ex) { // D32F depth after the lighting (852x480 under a 1704x960 scene), and DLSS ran on that and wrote its picture into // the scene's own light target, which left every object black. if (D.gbuf && ex->pass_ncolor == 1 && c->image && scene_color(c->format) && z->image && z->format == VK_FORMAT_D32_SFLOAT && - c->width == z->width && c->height == z->height && c->width >= 640 && + c->width == z->width && c->height == z->height && c->width >= DLAA_SCENE_MIN_W && (!D.armed || (c->width >= D.color.width && c->height >= D.color.height)) && c->samples <= 1 && !c->is3d) { + int first = !D.armed; D.color = *c, D.depth = *z, D.armed = 1; + if (first) mask_snapshot(ex); // the lit scene, before the transparents (reactive mask) return; } if (!D.armed) return; @@ -652,8 +1007,24 @@ void vk_dlaa_pass(VkExec *ex) { } } +// TOTK_DLSS, read at each present (the menu switches it live): the scene size of the mode (resolution.c) and the +// texture bias that goes with it while DLSS runs. +static void dlss_mode(void) { + static char mode[32]; + static int bias_on = -1; + const char *m = cfg_value("TOTK_DLSS"); + if (strcmp(m, mode)) { + snprintf(mode, sizeof mode, "%s", m); + res_dlss_mode(m); + bias_on = -1; + } + int b = dl_active(); + if (b != bias_on) bias_on = b, vk_sampler_dlss_bias(b ? res_dlss_lod_bias() : 0.0f); +} + void vk_dlaa_frame(void) { vk_dlaa_swap_img = NULL, vk_dlaa_swap_view = VK_NULL_HANDLE; + dlss_mode(); int on = N.available && cfg_bool("TOTK_DLAA"); if (on != D.on) { rt_log("dlaa: %s", on ? "on" : "off"); @@ -753,4 +1124,5 @@ void vk_dlaa_frame(void) {} void vk_dlaa_draw(VkExec *ex) { (void)ex; } void vk_dlaa_viewport(VkExec *ex, VkViewport *v) { (void)ex, (void)v; } int vk_dlaa_skip_draw(VkExec *ex, const GfxShader *fs) { (void)ex, (void)fs; return 0; } +int vk_dlaa_prev_camera(VkDescriptorBufferInfo *out, float jitter[2]) { (void)out, (void)jitter; return 0; } #endif diff --git a/runtime/vk_dlaa.h b/runtime/vk_dlaa.h index 685a2524..82b98113 100644 --- a/runtime/vk_dlaa.h +++ b/runtime/vk_dlaa.h @@ -2,6 +2,10 @@ #pragma once #include "vk.h" +// The narrowest scene the DLSS pass and the motion passes recognise by its D32F depth: Ultra Performance at a 1080p +// output renders 640 wide, smaller windows less. +#define DLAA_SCENE_MIN_W 320u + int vk_dlaa_wanted(void); // the setting is on (the device may still lack DLSS) // vk_device.c: NGX's instance and device extensions join the lists (room for max names); vk_dlaa_init after the // device exists brings NGX up and checks that DLSS is available. @@ -20,6 +24,25 @@ void vk_dlaa_frame(void); // the viewport of the scene's draws by the frame's jitter (vk_dlaa_viewport, when the viewport is recorded). void vk_dlaa_draw(VkExec *ex); void vk_dlaa_viewport(VkExec *ex, VkViewport *v); +// vk_mvsurvey.c (TOTK_MV_SURVEY): notes the uniform blocks of each scene draw, beside vk_dlaa_draw; each present +// compares them with the previous frames' and logs a summary now and then. +void vk_mvsurvey_draw(VkExec *ex); +void vk_mvsurvey_frame(void); +void vk_mvprev_draw(VkExec *ex); // vk_mvprev.c: notes the scene draw's object and bone blocks +// Per-object motion (vk_exec_draw.c): the moving scene draws are listed again during the replay with +// their vertex shader's last-frame variant; vk_dlaa.c mv_pass records them over the camera motion into ex->cb (and +// leaves every binding of the replay to be set again; draws whose camera block is not cam are left out). +// vk_dlaa_prev_camera: last frame's camera block and this frame's jitter, 0 when there is no history. +int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth, VkFormat depth_fmt, uint32_t w, + uint32_t h); +int vk_dlaa_prev_camera(VkDescriptorBufferInfo *out, float jitter[2]); +// vk_mvcheck.c (TOTK_MV_CHECK, test): mv_pass hands it the motion image after the camera pass and, after the object +// pass, the motion image again and the scene light target it ran on; it measures how well the vectors fit the picture. +void vk_mvcheck_camera(VkCommandBuffer cb, VkImage mv, uint32_t w, uint32_t h, float jx, float jy); +void vk_mvcheck_final(VkCommandBuffer cb, uint64_t fence, VkImage mv, VkImage cimg, VkFormat cfmt, uint32_t level, + uint32_t layer, uint32_t w, uint32_t h, float jx, float jy); +const char *vk_mvobj_frame_masks(void); // vk_exec_draw.c: the last motion list, per position and changed banks +void vk_mvcheck_output(VkCommandBuffer cb, VkImage out, uint32_t ow, uint32_t oh); // after DLSS on a check frame // vk_exec_draw.c, once a draw's fragment shader is known: 1 skips the draw (the game's FXAA while TOTK_FXAA is 0, its // scaling draw when DLSS wrote the scaled picture). int vk_dlaa_skip_draw(VkExec *ex, const GfxShader *fs); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 4dd24d06..45bd084a 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -731,6 +731,8 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G // *present: the program has this stage. Returns NULL when it has not or the translation failed. pending (graphics // stages, vk_pipeline_async_on): a shader not translated yet is queued and NULL returned with *pending = 1. await: a // shader queued earlier is waited for (gfx_shader_await) instead. +static int sf_mvprev; // shader_for asks for the vertex shader's last-frame variant (per-object motion) + static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pending, int await) { const uint8_t *ctl, *code; uint32_t len; @@ -741,6 +743,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin GfxShaderAnswers a; uint64_t serial = ex->prog->serial; answers_for(ex, st, ctl, slot_mask_of(serial, st), &a); + a.mvprev = (uint8_t)(sf_mvprev && st == NVN_STAGE_VS); ShaderEnt *set = NULL; if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded @@ -758,7 +761,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin } int gs = st == NVN_STAGE_VS ? GFX_STAGE_VS : st == NVN_STAGE_FS ? GFX_STAGE_FS : GFX_STAGE_CS; const GfxShader *s = await ? gfx_shader_await(gs, ctl, code, len, &a) - : pending && vk_pipeline_async_on() ? gfx_shader_get_async(gs, ctl, code, len, &a, pending) + : pending && (vk_pipeline_async_on() || a.mvprev) ? gfx_shader_get_async(gs, ctl, code, len, &a, pending) : gfx_shader_get(gs, ctl, code, len, NULL, &a); if (pending && *pending) return NULL; if (s) slot_mask_learn(serial, st, s); @@ -895,7 +898,14 @@ static void limits_init(void) { __atomic_store_n(&LIM.ubo_max, l->maxUniformBufferRange, __ATOMIC_RELEASE); } +// Per-object motion (section at the end): while it records a draw again, these replace the uniform bindings of the +// vertex stage's slots 18-20 (last frame's camera, bones and place) and the fragment stage's slot 3 (its parameters). +static int mo_ovr_on; +static VkDescriptorBufferInfo mo_ovr[2][32]; // [vertex, fragment][slot]; buffer 0: not replaced + static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { + if (mo_ovr_on && (st == NVN_STAGE_VS || st == NVN_STAGE_FS) && slot >= 0 && slot < 32 && mo_ovr[st][slot].buffer) + return mo_ovr[st][slot]; if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); uint64_t max = LIM.ubo_max, align = LIM.ubo_align; VkDescriptorBufferInfo bi; @@ -1916,18 +1926,23 @@ static void bind_pipeline(VkExec *ex, VkPipelineBindPoint bp, VkPipeline p) { if (!k) B.st_valid = 0; } +static VkViewport draw_viewport(VkExec *ex) { + float x = 0, y = 0, w = (float)ex->pass_w, h = (float)ex->pass_h; + if (ex->has_vp) { + float k = ex->pass_scale > 1 ? (float)ex->pass_scale : 1.0f; + x = ex->vp[0][0] * k, y = ex->vp[0][1] * k, w = ex->vp[0][2] * k, h = ex->vp[0][3] * k; + } + if (w < 1) w = 1; + if (h < 1) h = 1; + VkViewport v = {x, y + h, w, -h, ex->depth_range[0], ex->depth_range[1]}; // the y flip: negative height + vk_dlaa_viewport(ex, &v); // the scene's draws move by the DLAA jitter + return v; +} + static void record_fixed(VkExec *ex) { VkCommandBuffer cb = ex->cb; if (ex->dirty & VK_EXEC_DIRTY_VIEWPORT) { - float x = 0, y = 0, w = (float)ex->pass_w, h = (float)ex->pass_h; - if (ex->has_vp) { - float k = ex->pass_scale > 1 ? (float)ex->pass_scale : 1.0f; - x = ex->vp[0][0] * k, y = ex->vp[0][1] * k, w = ex->vp[0][2] * k, h = ex->vp[0][3] * k; - } - if (w < 1) w = 1; - if (h < 1) h = 1; - VkViewport v = {x, y + h, w, -h, ex->depth_range[0], ex->depth_range[1]}; // the y flip: negative height - vk_dlaa_viewport(ex, &v); // the scene's draws move by the DLAA jitter + VkViewport v = draw_viewport(ex); vkCmdSetViewportWithCount(cb, 1, &v); ex->recorded++; } @@ -2189,6 +2204,8 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (ex->mark && vk_exec_frame.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; } vk_dlaa_draw(ex); + vk_mvsurvey_draw(ex); + vk_mvprev_draw(ex); record_fixed(ex); repeat_stats(ex, info); bind_resources(ex, info); @@ -2201,6 +2218,762 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { return 1; } +// ---- per-object motion +// The camera pass (vk_dlaa.c mv_pass) gives everything the camera's motion, so a character that moves with the camera, +// or on its own, gets the wrong one and DLSS throws its history away there. Each solid scene draw whose position reads +// an object's own uniform blocks (bones c5, place c7, c6; mvobj_draw_ says which are left out and why) is therefore +// drawn a second time (listed during the replay, MQ, and recorded by mv_pass) with its vertex shader's last-frame +// variant (translator patch 0015: the position again from last frame's blocks, bound at slot bank + 15, and last +// frame's camera, written to a spare output) and the fragment shader of vk_mvobj_fs.h, which writes last frame's window +// position minus this frame's. The draw keeps everything else of the original: vertex and index buffers, the other +// uniform blocks, the jittered viewport and the scene depth, tested with the game's own test and its equal case +// (LEQUAL, GEQUAL) without writes, so only the visible surface writes its vector. Last frame's blocks: +// vk_mvprev_find and mo_prev_block; the camera: vk_dlaa_prev_camera. A draw whose variant or pipeline is still +// building, or whose last-frame blocks are not in the snapshot, keeps the camera's motion. TOTK_DLAA_OBJMV=0 turns it +// off (vk_mvprev.c). +#include "vk_mvobj_fs.h" +static const char MO_FS_RES[] = "{\"key\":\"mvobj\",\"stage\":\"ps\",\"notes\":[],\"cbufs\":[{\"binding\":1,\"slot\":3}]," + "\"sbufs\":[],\"textures\":[],\"images\":[]}"; +static struct { + const GfxShader *fs[16]; // per input Location + int failed; + uint64_t cand, still, draws, nodata, building, ran, dropped; + uint64_t ns_check, ns_rec, n_rec, ns_emit, ns_still; + uint64_t ns_part[3], n_part[3], misses, cyc[3], ncyc, big; // checks: filter, data and stillness (memo misses), variant and pipeline // time spent: checks and lookups of every candidate; recording the draws again // cand: draws whose vertex shader reads c5 or c7 + uint64_t why[5]; // nodata by cause: camera, bones, place, bones copy, place copy + uint64_t no_out, storage, no_fs; // variant without the extra output, with storage resources; no fs + uint64_t other_cam; // listed draws left out: their camera block is not the scene camera + uint64_t lk[14][5]; // last-frame lookups per binding index: none, self, side, nearer, refused (another object) + uint64_t e_draws, e_pipe, e_dyn, e_push, e_pushw, e_sets, e_vb, e_setw, e_setreuse; // emitted commands (summary) +} MO; + +// The motion draws of the frame, kept as a list while the scene replays and recorded into the replay's own command +// buffer by vk_mvobj_execute, through the recorder thread (vk_rec.h) like every other draw: recording them straight +// into a secondary command buffer made the driver's work for each one on the replay thread. What a draw shares with +// the one before it (pipeline, dynamic state, viewport, buffers, descriptors) is recorded once; a push descriptor set +// is pushed again only where a binding changed (pushes update the set in place while its layout stays bound). +typedef struct { + VkPipeline pipe; + const VkPipeInfo *info; + uint32_t st, desc, vb; // indices into MQ.st, MQ.desc (the draw's packed descriptors) and MQ.vb + VkDescriptorSet set[2]; // allocated sets; VK_NULL_HANDLE for a pushed one + VkViewport v; + VkRect2D sc; + float bias[3]; + VkBuffer ib; + VkDeviceSize ib_off; + VkIndexType it; + uint32_t count, instances, base_instance; + int32_t base_vertex; + uint64_t cam; // the draw's camera block (c4), this frame's +} MoDraw; +typedef struct { VkBuffer b[17]; VkDeviceSize o[17], sz[17], str[17]; } MoVb; +static struct { + uint64_t frame, fence; // the frame and submission the list belongs to + VkFormat depth_fmt; + MoDraw *d; + VkPipeState *st; + VkDescSlot *desc; + MoVb *vb; + int n, cap, nst, cst, nvb, cvb; + uint32_t ndesc, cdesc; + NVNbufferRange vin[17]; // the vertex ranges MQ.vb[nvb - 1] was resolved from, under vk_mem_gen vgen + uint64_t vgen; + float sup_in[3]; // the support buffer last filled for the list and its inputs + VkDescriptorBufferInfo sup; + float par_in[4]; // the fragment shader's parameters last written and their inputs + VkDescriptorBufferInfo par; + VkDescriptorSetLayout aset_layout[2]; // the last allocated set per set number, with its descriptors + VkDescriptorSet aset[2]; + uint32_t aset_n[2]; + VkDescSlot aset_d[2][MAX_DESC]; +} MQ; + +static void *mo_grow(void *p, int *cap, int need, size_t elem) { + if (need <= *cap) return p; + int c = *cap ? *cap : 256; + while (c < need) c *= 2; + if (!(p = realloc(p, elem * (size_t)c))) rt_fatal("vk_exec: out of memory"); + *cap = c; + return p; +} + +// The position bank masks of the draws considered (gfx_shader_mvprev_banks), with counts, for the summary. +static struct { uint32_t mask; uint64_t n; int ubo, ssbo, tex; } MM[32]; // ubo, ssbo, tex: the first vertex shader's +static void mo_mask_note(uint32_t banks, const GfxShader *vs) { + for (int i = 0; i < 32; i++) + if (MM[i].mask == banks || !MM[i].n) { + if (!MM[i].n) { + const GfxShaderRes *r; + MM[i].ubo = gfx_shader_resources(vs, GFX_RES_UBO, &r), MM[i].ssbo = gfx_shader_resources(vs, GFX_RES_SSBO, &r); + MM[i].tex = gfx_shader_resources(vs, GFX_RES_TEX, &r); + } + MM[i].mask = banks, MM[i].n++; + return; + } +} + +static void mo_drop(void) { + if (MQ.n) MO.dropped++; + MQ.n = MQ.nst = MQ.nvb = 0, MQ.ndesc = 0, MQ.frame = 0; +} + +static uint64_t mo_gen = 1; // the list's snapshot (mo_prev_cached), bumped by mo_begin +static struct { uint32_t mask, chg, n; } MF[16]; // this list's draws per position banks and changed banks +static char fm_last[400]; // the last executed list's, formatted (vk_mvobj_frame_masks) + +// Starts this frame's list (the old one, never executed, is dropped). +static void mo_begin(VkExec *ex, uint64_t frame) { + mo_drop(); + mo_gen++; + memset(MF, 0, sizeof MF); + MQ.frame = frame, MQ.fence = ex->fence, MQ.depth_fmt = ex->depth.format; + memset(MQ.vin, 0, sizeof MQ.vin); + MQ.vgen = 0, MQ.sup.buffer = MQ.par.buffer = VK_NULL_HANDLE; + memset(MQ.aset, 0, sizeof MQ.aset); +} + +static const GfxShader *mo_fs(int loc) { + if (loc < 0 || loc >= 16) return NULL; + if (!MO.fs[loc]) { + uint32_t w[sizeof mvobj_fs_spv / 4]; + memcpy(w, mvobj_fs_spv, sizeof w); + w[MVOBJ_FS_LOCATION_WORD] = (uint32_t)loc; + MO.fs[loc] = gfx_shader_own(w, sizeof w / 4, MO_FS_RES); + if (!MO.fs[loc]) MO.failed = 1, rt_log("vk_exec: the per-object motion fragment shader is malformed: off"); + } + return MO.fs[loc]; +} + +// The write of binding b from its packed descriptors. +static VkWriteDescriptorSet mo_write(const VkPipeBinding *b, const VkDescSlot *d) { + VkWriteDescriptorSet w = {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = b->binding, + .descriptorCount = b->count ? b->count : 1, .descriptorType = b->type}; + if (b->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER || b->type == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE || + b->type == VK_DESCRIPTOR_TYPE_SAMPLER) + w.pImageInfo = &d->image; + else + w.pBufferInfo = &d->buffer; // VkDescSlot is 24 bytes, as VkDescriptorBufferInfo: arrays line up + return w; +} + +// The descriptors of a motion draw, packed into MQ.desc at q->desc without the binding caches of the command buffer +// being replayed (B, DS, RS), which then stay true for its next draw: uniform buffers (through ubo_info, so with +// mo_ovr), the support buffer and sampled textures. An allocated set is written here (or the last one reused when it +// would hold the same); a pushed set is pushed by mo_emit. 0 for a pipeline with storage buffers, images or texel +// buffers (not drawn again). +static int mo_desc(VkExec *ex, const VkPipeInfo *info, MoDraw *q) { + if (info->n_bindings > MAX_BINDINGS) return 0; + uint32_t total = 0; + for (uint32_t i = 0; i < info->n_bindings; i++) total += info->bindings[i].count ? info->bindings[i].count : 1; + if (total > MAX_DESC) return 0; + int cdesc = (int)MQ.cdesc; + MQ.desc = mo_grow(MQ.desc, &cdesc, (int)(MQ.ndesc + total), sizeof *MQ.desc); + MQ.cdesc = (uint32_t)cdesc; + VkDescSlot *d = MQ.desc + MQ.ndesc; + memset(d, 0, total * sizeof *d); // unused bytes zero: equal descriptors compare equal + for (uint32_t i = 0, nd = 0; i < info->n_bindings; i++) { + const VkPipeBinding *b = &info->bindings[i]; + const GfxShaderRes *r = b->res; + uint32_t cnt = b->count ? b->count : 1; + VkDescSlot *o = &d[nd]; + int st = nvn_stage_of(b->stage); + if (b->kind == VK_PIPE_SUPPORT) { + float in[3]; + support_in(ex, in); + if (!MQ.sup.buffer || memcmp(in, MQ.sup_in, sizeof in)) { + VkBuffer sb; + VkDeviceSize so; + support_fill(scratch(ex, SUPPORT_BYTES, &sb, &so), in[0], in[1], in[2]); + MQ.sup = (VkDescriptorBufferInfo){sb, so, SUPPORT_BYTES}; + memcpy(MQ.sup_in, in, sizeof in); + } + o->buffer = MQ.sup; + } else if (b->kind == GFX_RES_UBO) { + for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ubo_info(ex, st, r->slot + (int)k); + } else if (b->kind == GFX_RES_SSBO && !(r->flags & 1)) { // read only: instance data of foliage and such + for (uint32_t k = 0; k < cnt; k++) o[k].buffer = ssbo_info(ex, st, r, 0); + } else if (b->kind == GFX_RES_TEX && b->type != VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER) { + vk_texture_lock(); + for (uint32_t k = 0; k < cnt; k++) { + int slot = r->slot + (int)k; + tex_info(ex, r, slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0, 0, r->separate, &o[k].image); + } + vk_texture_unlock(); + } else { + return 0; + } + nd += cnt; + } + uint32_t n_sets = info->n_sets ? info->n_sets : 1; + q->set[0] = q->set[1] = VK_NULL_HANDLE; + for (uint32_t s = 0, i0 = 0, k0 = 0; s < n_sets && s < 2; s++) { + uint32_t i1 = i0, k1 = k0; + while (i1 < info->n_bindings && (n_sets == 1 || info->bindings[i1].set == s)) + k1 += info->bindings[i1].count ? info->bindings[i1].count : 1, i1++; + if (i1 > i0 && !(s ? info->push1 : info->push)) { + VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout; + if (MQ.aset[s] && MQ.aset_layout[s] == sl && MQ.aset_n[s] == k1 - k0 && + !memcmp(MQ.aset_d[s], d + k0, (k1 - k0) * sizeof *d)) { + q->set[s] = MQ.aset[s]; + MO.e_setreuse++; + } else { + static VkWriteDescriptorSet w[MAX_BINDINGS]; + VkDescriptorSet set = alloc_set(ex, sl); + for (uint32_t i = i0, k = k0; i < i1; k += info->bindings[i].count ? info->bindings[i].count : 1, i++) + w[i - i0] = mo_write(&info->bindings[i], d + k), w[i - i0].dstSet = set; + vkUpdateDescriptorSets(vk_ctx()->device, i1 - i0, w, 0, NULL); + q->set[s] = MQ.aset[s] = set, MQ.aset_layout[s] = sl, MQ.aset_n[s] = k1 - k0; + memcpy(MQ.aset_d[s], d + k0, (k1 - k0) * sizeof *d); + MO.e_setw++; + } + } + i0 = i1, k0 = k1; + } + q->desc = MQ.ndesc; + MQ.ndesc += total; + return 1; +} + +// The draw's vertex buffers as bind_vertex_buffers resolves them, into MQ.vb; the previous entry again while the +// ranges and vk_mem_gen are the same (most motion draws in a row are one model's parts). +static uint32_t mo_vertex_buffers(VkExec *ex) { + uint64_t gen = vk_mem_gen(); + if (MQ.nvb && MQ.vgen == gen && !memcmp(MQ.vin, ex->vbuf, sizeof MQ.vin)) { + const MoVb *l = &MQ.vb[MQ.nvb - 1]; + int same = 1; + if (!vk_pipeline_vertex_input_dynamic()) + for (int b = 0; b < 16 && same; b++) same = l->str[b] == ex->pipe.vstream[b].stride; + if (same) return (uint32_t)MQ.nvb - 1; + } + MQ.vb = mo_grow(MQ.vb, &MQ.cvb, MQ.nvb + 1, sizeof *MQ.vb); + MoVb *v = &MQ.vb[MQ.nvb]; + for (int b = 0; b < 17; b++) { + v->b[b] = Z.zbuf, v->o[b] = 0, v->sz[b] = ZERO_BYTES, v->str[b] = b < 16 ? ex->pipe.vstream[b].stride : 0; + if (b == VK_PIPE_ZERO_BINDING) continue; + const NVNbufferRange *r = &ex->vbuf[b]; + VkMemSpan sp; + if (!r->address || !vk_mem_resolve(r->address, 1, &sp)) continue; + VK_MEM_USE(sp); + v->b[b] = sp.buffer, v->o[b] = sp.offset, v->sz[b] = r->size && r->size < sp.avail ? r->size : sp.avail; + } + memcpy(MQ.vin, ex->vbuf, sizeof MQ.vin); + MQ.vgen = gen; + return (uint32_t)MQ.nvb++; +} + +// Records the list into cb (the replay's command buffer, inside the motion pass's rendering), skipping what the draw +// before already set. +static void mo_emit(VkCommandBuffer cb) { + int dyn_vi = vk_pipeline_vertex_input_dynamic(); + VkPipeline pipe = VK_NULL_HANDLE; + uint32_t st = UINT32_MAX, vb = UINT32_MAX; + VkPipelineLayout lay[2] = {0}; + const VkPipeInfo *pinfo[2] = {0}; + int pdesc[2] = {-1, -1}; // the pushed set's descriptors (MQ.desc index) under lay/pinfo, -1 none + VkDescriptorSet bound[2] = {0}; + const MoDraw *p = NULL; + for (int n = 0; n < MQ.n; p = &MQ.d[n++]) { + const MoDraw *q = &MQ.d[n]; + const VkPipeInfo *info = q->info; + if (q->pipe != pipe) vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, q->pipe), pipe = q->pipe, MO.e_pipe++; + if (q->st != st) vk_pipeline_set_dynamic(cb, &MQ.st[q->st]), st = q->st, MO.e_dyn++; + if (!p || memcmp(&q->v, &p->v, sizeof q->v)) vkCmdSetViewportWithCount(cb, 1, &q->v); + if (!p || memcmp(&q->sc, &p->sc, sizeof q->sc)) vkCmdSetScissorWithCount(cb, 1, &q->sc); + if (!p || memcmp(q->bias, p->bias, sizeof q->bias)) vkCmdSetDepthBias(cb, q->bias[0], q->bias[1], q->bias[2]); + uint32_t n_sets = info->n_sets ? info->n_sets : 1; + const VkDescSlot *d = MQ.desc + q->desc; + for (uint32_t s = 0, i0 = 0, k0 = 0; s < n_sets && s < 2; s++) { + uint32_t i1 = i0, k1 = k0; + while (i1 < info->n_bindings && (n_sets == 1 || info->bindings[i1].set == s)) + k1 += info->bindings[i1].count ? info->bindings[i1].count : 1, i1++; + if (i1 > i0) { + if (s ? info->push1 : info->push) { + // the bindings that differ from the last push of this layout and pipeline (all after another) + int same = lay[s] == info->layout && pinfo[s] == info && pdesc[s] >= 0; + const VkDescSlot *pd = same ? MQ.desc + pdesc[s] : NULL; + VkWriteDescriptorSet w[MAX_BINDINGS]; + uint32_t nw = 0; + for (uint32_t i = i0, k = k0; i < i1; i++) { + uint32_t cnt = info->bindings[i].count ? info->bindings[i].count : 1; + if (!same || memcmp(d + k, pd + k, cnt * sizeof *d)) w[nw++] = mo_write(&info->bindings[i], d + k); + k += cnt; + } + if (nw) rec_push(vk_pipeline_push_fn(), cb, VK_PIPELINE_BIND_POINT_GRAPHICS, info->layout, s, nw, w); + MO.e_push += nw != 0, MO.e_pushw += nw; + lay[s] = info->layout, pinfo[s] = info, pdesc[s] = (int)q->desc, bound[s] = VK_NULL_HANDLE; + } else if (q->set[s] != bound[s] || lay[s] != info->layout) { + vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, info->layout, s, 1, &q->set[s], 0, NULL); + bound[s] = q->set[s], lay[s] = info->layout, pinfo[s] = info, pdesc[s] = -1, MO.e_sets++; + } + } + i0 = i1, k0 = k1; + } + if (q->vb != vb) { + const MoVb *v = &MQ.vb[q->vb]; + vkCmdBindVertexBuffers2(cb, 0, 17, v->b, v->o, v->sz, dyn_vi ? NULL : v->str); + vb = q->vb, MO.e_vb++; + } + if (!p || q->ib != p->ib || q->ib_off != p->ib_off || q->it != p->it) vkCmdBindIndexBuffer(cb, q->ib, q->ib_off, q->it); + vkCmdDrawIndexed(cb, q->count, q->instances, 0, q->base_vertex, q->base_instance); + MO.e_draws++; + } +} + +static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0); +static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off) { + if (MO.failed || !vk_mvprev_on() || !B.st_valid) return; + static int timed = -1; + if (timed < 0) timed = rt_knob("MVOBJ_TIMES", "0"); // TOTK_MVOBJ_TIMES=1: the summary's times (the clock reads cost) + if (!timed) return mvobj_draw_(ex, d, ib, ib_off, 0); + uint64_t t0 = kern_now_ns(); + mvobj_draw_(ex, d, ib, ib_off, t0); + MO.ns_check += kern_now_ns() - t0; +} + +// TOTK_MVOBJ_BANKS=1 (test): for scene draws, the words of each other uniform bank (c3, c6, c8-c16) that changed +// since the same shader's draw last frame, logged at frame 3000: where the time that moves wind and water lives +static void mvobj_banks(VkExec *ex) { + static int on = -1; + if (on < 0) on = rt_knob("MVOBJ_BANKS", "0"); + if (!on || !B.st_valid) return; + uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (frame < 2999 || frame > 3000) return; + static struct { const GfxShader *vs; int bank; uint64_t frame; float w[64]; int n; } tab[4096]; + static int logged; + const GfxShaderRes *ur; + int nu = gfx_shader_resources(B.st.vs, GFX_RES_UBO, &ur); + for (int i = 0; i < nu; i++) { + int bank = ur[i].slot; + if (bank < 3 || bank == 4 || bank == 5 || bank == 7 || bank > 16) continue; + const p_bind_buf *u = &ex->ubo[NVN_STAGE_VS][bank - 3]; + uint32_t n = u->size < 256 ? (uint32_t)u->size / 4 : 64; + const float *now = n ? (const float *)vk_usnap_bytes(u->addr, n * 4) : NULL; + if (!now) continue; + uint32_t k = (uint32_t)(((uintptr_t)B.st.vs ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull >> 52); + if (frame == 2999) { + tab[k].vs = B.st.vs, tab[k].bank = bank, tab[k].frame = frame, tab[k].n = (int)n; + memcpy(tab[k].w, now, n * 4); + } else if (tab[k].vs == B.st.vs && tab[k].bank == bank && tab[k].frame == 2999 && logged < 80) { + char line[700]; + int o = 0, nd = 0; + for (int j = 0; j < tab[k].n && j < (int)n && o < 600; j++) + if (memcmp(&tab[k].w[j], &now[j], 4)) nd++, o += snprintf(line + o, sizeof line - o, " [%d] %g>%g", j, tab[k].w[j], now[j]); + if (nd) logged++, rt_log("mvbanks: vs %p c%d size %#llx: %d words%s", (const void *)B.st.vs, bank, + (unsigned long long)u->size, nd, line); + tab[k].frame = 0; + } + } +} + +// 1: old is cur a little changed or the same (every float within 10 %, at least 1.0 and at most 4.0); *changed: not +// the same. The cap: TotK's world positions are in the thousands, and 10 % of them let another object's block a few +// units away pass as this one moved; nothing moves 4 units in a frame. +static int mo_similar(const uint8_t *cur, const uint8_t *old, uint64_t n, int *changed) { + *changed = 0; + if (!memcmp(cur, old, n)) return 1; + // word by word as integers; float arithmetic only on two normal floats (the blocks hold integers too, and + // denormal arithmetic took this loop 1.9 million cycles a block) + const uint32_t *x = (const uint32_t *)cur, *y = (const uint32_t *)old; + for (uint64_t w = 0; w < n / 4; w++) { + if (x[w] == y[w]) continue; + uint32_t ex = x[w] >> 23 & 0xff, ey = y[w] >> 23 & 0xff; + if (!ex || ex == 0xff || !ey || ey == 0xff) continue; // not both normal floats: an integer or a tiny value + float fx, fy; + memcpy(&fx, &x[w], 4), memcpy(&fy, &y[w], 4); + float dd = fabsf(fx - fy), tol = 0.1f * fabsf(fy); + if (dd > (tol < 1.0f ? 1.0f : tol > 4.0f ? 4.0f : tol)) return 0; + } + *changed = 1; + return 1; +} + +// Last frame's copy of a block, accepted when it is this frame's block a little changed (mo_similar, +// which see): a block of another object the address rule found would make motion up. 1 and *out: found (the block itself +// when it is unchanged, *changed 0); 0: none, the draw keeps this frame's block there. +static int mo_prev_block(int index, const p_bind_buf *u, VkDescriptorBufferInfo *out, int *changed) { + uint64_t max = LIM.ubo_max, n = u->size < max ? u->size : max, pa; + int how; + *changed = 0; + // c9 and c11 keep this frame's block. With the copies the address rule finds for c11 and last frame's c9, the + // c4 c7 c9 c11 draws (walls, cliffs, arches) got several pixels of motion under a still camera (TOTK_MV_CHECK, + // those draws alone: 34 % of the picture moving; with c11 kept, 0.8 %). With c11 kept, their c9 still changes every + // frame while nothing moves (the check's per-draw line: about 120 such draws a frame changed in c9 alone), the + // object scale the picture shows is 0.6-0.8 of theirs, and with Link stopped against a cliff 40-77 % of the picture + // got 1-3 px of motion that flipped direction, three times the warp error of the camera's vectors. With c9 kept too, + // the turning run is as good (object warp error 0.46 of the camera's). Why their copies are wrong is not known. + // TOTK_MVPREV_SKIP= (test) replaces the default a00 (bit n: cn). + static long skip = -2; + if (skip == -2) { + const char *e = cfg_get("TOTK_MVPREV_SKIP"); + skip = e && *e ? strtol(e, NULL, 16) : 1L << 9 | 1L << 11; + } + if (skip >> (index + 3) & 1) return 0; + uint64_t c0 = __builtin_ia32_rdtsc(); + int found = vk_mvprev_find(index, u->addr, u->size, &pa, &how); + uint64_t c1 = __builtin_ia32_rdtsc(); + MO.cyc[0] += c1 - c0, MO.ncyc++, MO.big += n; + if (index >= 0 && index < 14) MO.lk[index][found ? how : MVPREV_NONE]++; + if (!found || !vk_usnap_find(pa, n, LIM.ubo_align, out)) return 0; + if (how == MVPREV_SELF) return 1; // bound at the same address last frame: not written again + const uint8_t *map = vk_usnap_map(), *cur = vk_usnap_bytes(u->addr, n); + uint64_t c2 = __builtin_ia32_rdtsc(); + MO.cyc[1] += c2 - c1; + if (!cur) return 0; + const uint8_t *old = map + out->offset; + static long trust = -2; // TOTK_MVPREV_TRUST= (test): the copy found is taken without the check + if (trust == -2) { + const char *e = cfg_get("TOTK_MVPREV_TRUST"); + trust = e && *e ? strtol(e, NULL, 16) : 0; + } + if (trust >> (index + 3) & 1) { + *changed = memcmp(cur, old, n) != 0; + return 1; + } + int r = mo_similar(cur, old, n, changed); + MO.cyc[2] += __builtin_ia32_rdtsc() - c2; + if (!r && index >= 0 && index < 14) MO.lk[index][4]++; + return r; +} + +// mo_prev_block once per block and list: many draws share their blocks (the scene's c14, a model's c7 and c9 across +// its parts), and the lookup with its comparison was most of the checks' time. +static int mo_prev_cached(int index, const p_bind_buf *u, VkDescriptorBufferInfo *out, int *changed) { + static struct { uint64_t gen, addr, size; int index, r, changed; VkDescriptorBufferInfo out; } bc[4096]; + uint32_t i = (uint32_t)((u->addr ^ (uint64_t)index << 56 ^ u->size << 40) * 0x9E3779B97F4A7C15ull >> 52); + if (bc[i].gen == mo_gen && bc[i].addr == u->addr && bc[i].size == u->size && bc[i].index == index) { + *out = bc[i].out, *changed = bc[i].changed; + return bc[i].r; + } + int r = mo_prev_block(index, u, out, changed); + bc[i].gen = mo_gen, bc[i].addr = u->addr, bc[i].size = u->size, bc[i].index = index; + bc[i].r = r, bc[i].changed = *changed, bc[i].out = *out; + return r; +} + +static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0) { + mvobj_banks(ex); + const VkExecTarget *z = &ex->depth; + if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || B.st.samples > 1 || B.st.raster_discard) + return; + // solid geometry only, which tests and writes the scene depth: effects, particles and screen quads write none, and + // drawn again they would put their motion on the surfaces behind them (DLSS takes those from the surface) + if (!B.st.ds.dt || !B.st.ds.dw) return; + // a vertex shader that reads a user bank besides the camera's (c4) may move on its own + const GfxShaderRes *ur; + int nu = gfx_shader_resources(B.st.vs, GFX_RES_UBO, &ur), reads = 0; + for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4; + if (!reads) return; + MO.cand++; + uint64_t tp = t0 ? kern_now_ns() : 0; + if (t0) MO.ns_part[0] += tp - t0, MO.n_part[0]++; + uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (MQ.frame != frame || MQ.fence != ex->fence || MQ.depth_fmt != z->format) mo_begin(ex, frame); + if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); + // the variant: the translation of the same code and answers, kept per shader once found; it names the banks the + // position depends on (0: none, the shader is skipped from then on) + static struct { const GfxShader *of, *vs; uint32_t banks; } vmemo[1024]; + uint32_t vi = (uint32_t)((uintptr_t)B.st.vs * 0x9E3779B97F4A7C15ull >> 54); + if (vmemo[vi].of != B.st.vs) { + int has, pending = 0; + sf_mvprev = 1; + const GfxShader *v = shader_for(ex, NVN_STAGE_VS, &has, &pending, 0); + sf_mvprev = 0; + if (!v || pending) { + MO.building++; + return; + } + GfxShaderIo io[32]; + uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(B.st.vs, io); + uint32_t banks = extra ? gfx_shader_mvprev_banks(v) & 0x1fff8u : 0; + vmemo[vi].of = B.st.vs, vmemo[vi].vs = v, vmemo[vi].banks = banks; + vk_mvprev_want((banks & ~(1u << 4)) >> 3); // binding index = bank - 3 + } + const GfxShader *vs = vmemo[vi].vs; + uint32_t banks = vmemo[vi].banks; + if (!banks || !(banks & ~(1u << 4))) { + MO.no_out++; + return; + } + // a position without the scene camera (c4) has a view of its own (c8 c9 c10 [c13]: its c8 is at a new address every + // frame, last frame's is never found): drawn again it would get this frame's view and no camera motion + // (TOTK_MV_CHECK while turning: 6.2 px end-point error against the camera vectors' 1.7) + static int noc4 = -1; + if (noc4 < 0) noc4 = rt_knob("MVOBJ_NOC4", "0"); // TOTK_MVOBJ_NOC4=1 (test): drawn again all the same + if (!(banks >> 4 & 1) && !noc4) { + MO.no_out++; + return; + } + // only positions that read an object's own blocks: bones (c5), place (c7) or c6. The rest (terrain and grass: c14 + // c15 c16, with or without c9) measured no better than the camera's motion (TOTK_MV_CHECK, turning, those draws + // alone: object warp error 0.112 against the camera's 0.113, the object part 0.4 of what the picture shows) and + // were half the draws checked and listed. TOTK_MVOBJ_ALL=1 (test) takes them again. + static int all = -1; + if (all < 0) all = rt_knob("MVOBJ_ALL", "0"); + if (!all && !(banks & (1u << 5 | 1u << 6 | 1u << 7))) { + MO.no_out++; + return; + } + // TOTK_MVOBJ_ONLY= (test): only draws whose position reads no bank outside the mask (bit n: cn) + static long only = -2; + if (only == -2) { + const char *e = cfg_get("TOTK_MVOBJ_ONLY"); + only = e && *e ? strtol(e, NULL, 16) : -1; + } + if (only >= 0 && banks & ~(uint32_t)only & ~(1u << 4)) return; + static long exact = -2; // TOTK_MVOBJ_EXACT= (test): only draws with exactly these position banks + if (exact == -2) { + const char *e = cfg_get("TOTK_MVOBJ_EXACT"); + exact = e && *e ? strtol(e, NULL, 16) : -1; + } + if (exact >= 0 && banks != (uint32_t)exact) return; + mo_mask_note(banks, B.st.vs); + // last frame's camera and blocks; held still when none of the position's blocks changed (most of the scenery) + VkDescriptorBufferInfo cam, prevb[17]; + float jit[2]; + if (!vk_dlaa_prev_camera(&cam, jit)) { + MO.nodata++, MO.why[0]++; + return; + } + static struct { uint64_t frame, key; int moving; VkDescriptorBufferInfo b[17]; uint32_t have; } memo[8192]; + uint64_t key = banks; + for (int b = 3; b <= 16; b++) + if (banks >> b & 1 && b != 4) key = (key ^ ex->ubo[NVN_STAGE_VS][b - 3].addr) * 0x100000001b3ull; + uint32_t mi = (uint32_t)(key * 0xff51afd7ed558ccdull >> 51); + uint32_t have = 0; + if (memo[mi].frame == frame && memo[mi].key == key) { + if (!memo[mi].moving) { + if (t0) MO.ns_still += kern_now_ns() - tp; + return (void)MO.still++; + } + memcpy(prevb, memo[mi].b, sizeof prevb), have = memo[mi].have; + } else { + MO.misses++; + int moving = 0; + for (int b = 3; b <= 16; b++) { + if (!(banks >> b & 1) || b == 4) continue; + const p_bind_buf *u = &ex->ubo[NVN_STAGE_VS][b - 3]; + int changed; + if (u->addr && u->size && mo_prev_cached(b - 3, u, &prevb[b], &changed)) + have |= 1u << b, moving |= changed << b; // the banks whose block changed + } + memo[mi].frame = frame, memo[mi].key = key, memo[mi].moving = moving; + memcpy(memo[mi].b, prevb, sizeof prevb), memo[mi].have = have; + if (!moving) { + if (t0) MO.ns_still += kern_now_ns() - tp; + return (void)MO.still++; + } + } + if (t0) MO.ns_part[1] += kern_now_ns() - tp, MO.n_part[1]++, tp = kern_now_ns(); + GfxShaderIo io[32]; + uint32_t extra = gfx_shader_outputs(vs, io) & ~gfx_shader_outputs(B.st.vs, io); + const GfxShaderRes *r; + if (gfx_shader_resources(vs, GFX_RES_IMG, &r)) { + MO.storage++; + return; + } + const GfxShader *fs = mo_fs(31 - __builtin_clz(extra)); + if (!fs) { + MO.no_fs++; + return; + } + VkPipeState ms = B.st; + ms.vs = vs, ms.fs = fs, ms.feedback = 0; + memset(ms.color_fmt, 0, sizeof ms.color_fmt); + ms.color_fmt[0] = NVN_FORMAT_RG16F; + // blending off (blend_en below); the factors stay the game's, valid NVN values + ms.color.blend_en = 0, ms.color.logic_op = 0; + ms.chmask.mask = 0x3; // r and g of target 0 + ms.ms.a2c = 0; + // the game's own test with its equal case (LESS -> LEQUAL, GREATER -> GEQUAL): the surface the draw left in the + // scene depth passes, anything nearer hides it (TOTK_MVOBJ_EQUAL=1: EQUAL, for comparison) + static int eq = -1; + if (eq < 0) eq = rt_knob("MVOBJ_EQUAL", "0"); + int df = B.st.ds.df; + df = eq ? NVN_DEPTH_FUNC_EQUAL : df == NVN_DEPTH_FUNC_LESS ? NVN_DEPTH_FUNC_LEQUAL + : df == NVN_DEPTH_FUNC_GREATER ? NVN_DEPTH_FUNC_GEQUAL : df == NVN_DEPTH_FUNC_LEQUAL || df == NVN_DEPTH_FUNC_GEQUAL ? df + : NVN_DEPTH_FUNC_EQUAL; + ms.ds.dt = 1, ms.ds.dw = 0, ms.ds.df = (uint64_t)df, ms.ds.st = 0; + // the pipeline per state, kept: the lookup hashes and locks the whole state + static struct { VkPipeState st; VkPipeline p; const VkPipeInfo *info; int used; } pmemo[64]; + static int plast = -1; // the entry of the last draw: a model's parts in a row share the state, no hash then + uint32_t pi; + if (plast >= 0 && pmemo[plast].used && !memcmp(&pmemo[plast].st, &ms, sizeof ms)) { + pi = (uint32_t)plast; + } else { + uint64_t ph = 0xcbf29ce484222325ull; + for (size_t i = 0; i < sizeof ms; i += 8) { + uint64_t w8; + memcpy(&w8, (const uint8_t *)&ms + i, sizeof ms - i < 8 ? sizeof ms - i : 8); + ph = (ph ^ w8) * 0x100000001b3ull; + } + pi = (uint32_t)(ph >> 58); + } + const VkPipeInfo *info = NULL; + VkPipeline pipe; + int pending = 0; + if (pmemo[pi].used && !memcmp(&pmemo[pi].st, &ms, sizeof ms)) { + info = pmemo[pi].info; + pipe = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); // its optimised build once swapped in + if (!pipe) return; + } else { + pipe = vk_pipeline_graphics_async(&ms, &info, &pending, 0); + if (pending || !pipe || !info) { + MO.building += pending != 0; + return; + } + pmemo[pi].st = ms, pmemo[pi].p = pipe, pmemo[pi].info = info, pmemo[pi].used = 1; + } + plast = (int)pi; + uint64_t t1 = kern_now_ns(); + if (t0) MO.ns_part[2] += t1 - tp, MO.n_part[2]++; + if (t0) MO.ns_check -= t1 - t0; // counted as recording below + // into the frame's list, past the primary's binding caches (mo_desc); mo_emit records it + MQ.d = mo_grow(MQ.d, &MQ.cap, MQ.n + 1, sizeof *MQ.d); + MoDraw *q = &MQ.d[MQ.n]; + float par_in[4] = {(float)z->width, (float)z->height, jit[0], jit[1]}; + if (!MQ.par.buffer || memcmp(par_in, MQ.par_in, sizeof par_in)) { + VkBuffer pbuf; + VkDeviceSize poff; + memcpy(scratch(ex, 16, &pbuf, &poff), par_in, sizeof par_in); + MQ.par = (VkDescriptorBufferInfo){pbuf, poff, 16}; + memcpy(MQ.par_in, par_in, sizeof par_in); + } + memset(mo_ovr, 0, sizeof mo_ovr); + for (int bk = 3; bk <= 16; bk++) // last frame's block, or this frame's where none was found + mo_ovr[NVN_STAGE_VS][bk + 15] = bk == 4 ? cam : have >> bk & 1 ? prevb[bk] : ubo_info(ex, NVN_STAGE_VS, bk); + mo_ovr[NVN_STAGE_FS][3] = MQ.par; + mo_ovr_on = 1; + int bound = mo_desc(ex, info, q); + mo_ovr_on = 0; + if (!bound) { + MO.storage++; + return; + } + q->pipe = pipe, q->info = info; + if (!MQ.nst || memcmp(&MQ.st[MQ.nst - 1], &ms, sizeof ms)) { + MQ.st = mo_grow(MQ.st, &MQ.cst, MQ.nst + 1, sizeof *MQ.st); + MQ.st[MQ.nst++] = ms; + } + q->st = (uint32_t)MQ.nst - 1; + q->v = draw_viewport(ex), q->sc = vk_exec_scissor(ex); + q->bias[0] = ex->poly_offset.units, q->bias[1] = ex->poly_offset.clamp, q->bias[2] = ex->poly_offset.factor; + q->vb = mo_vertex_buffers(ex); + q->ib = ib, q->ib_off = ib_off; + q->it = d->itype == NVN_INDEX_TYPE_UNSIGNED_SHORT ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32; + q->count = (uint32_t)d->count, q->instances = d->instances > 0 ? (uint32_t)d->instances : 1; + q->base_vertex = d->base_vertex, q->base_instance = (uint32_t)d->base_instance; + q->cam = ex->ubo[NVN_STAGE_VS][1].addr; + MQ.n++; + MO.draws++; + for (int i = 0; i < 16; i++) // the frame's draws per position banks and changed banks (vk_mvobj_frame_masks) + if (!MF[i].n || (MF[i].mask == banks && MF[i].chg == (uint32_t)memo[mi].moving)) { + MF[i].mask = banks, MF[i].chg = (uint32_t)memo[mi].moving, MF[i].n++; + break; + } + if (t0) MO.ns_rec += kern_now_ns() - t1, MO.n_rec++; +} + +int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth, VkFormat depth_fmt, uint32_t w, + uint32_t h) { + static uint64_t frames; + if (++frames % 60 == 0) + rt_log("vk_exec: per-object motion: %.2f ms a frame checking, %.2f ms listing (%.1f us a draw), %.2f ms recording", + MO.ns_check / 1e6 / (frames ? frames : 1), MO.ns_rec / 1e6 / (frames ? frames : 1), + MO.n_rec ? MO.ns_rec / 1e3 / MO.n_rec : 0.0, MO.ns_emit / 1e6 / (frames ? frames : 1)); + if (frames % 600 == 0) { + char l[1400]; + int o = 0; + for (int i = 0; i < 32 && MM[i].n && o < 1300; i++) + o += snprintf(l + o, sizeof l - o, " %#x:%llu (u%d s%d t%d)", MM[i].mask, (unsigned long long)MM[i].n, MM[i].ubo, + MM[i].ssbo, MM[i].tex); + if (o) rt_log("vk_exec: per-object motion position banks (bit n: cn) and draws:%s", l); + o = 0; + for (int i = 0; i < 14 && o < 1300; i++) + if (MO.lk[i][0] + MO.lk[i][1] + MO.lk[i][2] + MO.lk[i][3]) + o += snprintf(l + o, sizeof l - o, " c%d %llu/%llu/%llu/%llu/%llu", i + 3, (unsigned long long)MO.lk[i][0], + (unsigned long long)MO.lk[i][1], (unsigned long long)MO.lk[i][2], + (unsigned long long)MO.lk[i][3], (unsigned long long)MO.lk[i][4]); + if (o) rt_log("vk_exec: per-object motion last-frame lookups (none/self/side/nearer/refused):%s", l); + } + if (frames % 60 == 0 && MO.ran) + rt_log("vk_exec: per-object motion recorded a frame: %.0f draws, %.0f pipeline binds, %.0f dynamic states, %.0f " + "pushes of %.1f bindings, %.0f set binds, %.0f vertex buffer binds; sets written %.0f, reused %.0f", + (double)MO.e_draws / MO.ran, (double)MO.e_pipe / MO.ran, (double)MO.e_dyn / MO.ran, (double)MO.e_push / MO.ran, + MO.e_push ? (double)MO.e_pushw / MO.e_push : 0.0, (double)MO.e_sets / MO.ran, (double)MO.e_vb / MO.ran, + (double)MO.e_setw / MO.ran, (double)MO.e_setreuse / MO.ran); + if (frames % 60 == 0) + rt_log("vk_exec: per-object motion checks: filter %.2f ms (%.0f a frame), data %.2f ms (%.0f), variant and " + "pipeline %.2f ms (%.0f); %.0f objects looked up; still draws %.2f ms", MO.ns_part[0] / 1e6 / frames, + (double)MO.n_part[0] / frames, + MO.ns_part[1] / 1e6 / frames, (double)MO.n_part[1] / frames, MO.ns_part[2] / 1e6 / frames, + (double)MO.n_part[2] / frames, (double)MO.misses / frames, MO.ns_still / 1e6 / frames); + if (frames % 60 == 0 && MO.ncyc) + rt_log("vk_exec: per-object motion lookups: %llu cycles finding last frame's, %llu the copies, %llu comparing, " + "a lookup; %llu bytes a lookup", (unsigned long long)(MO.cyc[0] / MO.ncyc), + (unsigned long long)(MO.cyc[1] / MO.ncyc), (unsigned long long)(MO.cyc[2] / MO.ncyc), + (unsigned long long)(MO.big / MO.ncyc)); + if (frames % 60 == 0) + rt_log("vk_exec: per-object motion: %llu draws read place or bones, %llu of them still, %llu drawn again, %llu without last frame's data, %llu " + "waiting for a shader or pipeline; %llu frames ran, %llu dropped (no data: camera %llu, bones %llu, place %llu, " + "bones copy %llu, place copy %llu; uniform alignment %llu; variant without its output %llu, with storage %llu, " + "no fragment shader %llu; another camera %llu)", + (unsigned long long)MO.cand, (unsigned long long)MO.still, (unsigned long long)MO.draws, (unsigned long long)MO.nodata, + (unsigned long long)MO.building, + (unsigned long long)MO.ran, (unsigned long long)MO.dropped, (unsigned long long)MO.why[0], + (unsigned long long)MO.why[1], (unsigned long long)MO.why[2], (unsigned long long)MO.why[3], + (unsigned long long)MO.why[4], (unsigned long long)LIM.ubo_align, (unsigned long long)MO.no_out, + (unsigned long long)MO.storage, (unsigned long long)MO.no_fs, (unsigned long long)MO.other_cam); + if (!MQ.n) return 0; + static int exec = -1; + if (exec < 0) exec = rt_knob("MVOBJ_EXEC", "1"); // TOTK_MVOBJ_EXEC=0: listed, never recorded (test) + if (!exec) { + mo_drop(); + return 0; + } + uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (MQ.frame != frame || MQ.fence != ex->fence || MQ.depth_fmt != depth_fmt) { + mo_drop(); + return 0; + } + int fo = 0; + for (int i = 0; i < 16 && MF[i].n && fo < (int)sizeof fm_last - 40; i++) + fo += snprintf(fm_last + fo, sizeof fm_last - fo, " %#x/%#x:%u", MF[i].mask, MF[i].chg, MF[i].n); + if (!fo) fm_last[0] = 0; + // only draws with the scene camera block (vk_dlaa.c mv_pass's): last frame's camera they get is that block's, and + // a draw of another view would move by the difference between the two views + int kept = 0; + for (int i = 0; i < MQ.n; i++) + if (MQ.d[i].cam == cam) MQ.d[kept++] = MQ.d[i]; + else MO.other_cam++; + MQ.n = kept; + if (!MQ.n) { + mo_drop(); + return 0; + } + uint64_t t0 = kern_now_ns(); + VkCommandBuffer cb = ex->cb; + VkRenderingAttachmentInfo ca = {.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, .imageView = mv, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE}; + VkRenderingAttachmentInfo da = {.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, .imageView = depth, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE}; + VkRenderingInfo ri = {.sType = VK_STRUCTURE_TYPE_RENDERING_INFO, .renderArea = {{0, 0}, {w, h}}, .layerCount = 1, + .colorAttachmentCount = 1, .pColorAttachments = &ca, .pDepthAttachment = &da}; + vkCmdBeginRendering(cb, &ri); // through the recorder (vk_rec.h), in order with the draws and the end + mo_emit(cb); + vkCmdEndRendering(cb); + // the draws left their pipeline, descriptors, buffers and dynamic state bound: the next game draw sets everything + vk_exec_draw_new_cb(); + ex->dirty = VK_EXEC_DIRTY_ALL; + ex->recorded += MQ.n; + MQ.n = MQ.nst = MQ.nvb = 0, MQ.ndesc = 0, MQ.frame = 0; + MO.ran++; + MO.ns_emit += kern_now_ns() - t0; + return 1; +} + static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType type) { VkBuffer b; VkDeviceSize o; @@ -2292,6 +3065,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { vkCmdDrawIndexed(ex->cb, (uint32_t)d->count, d->instances > 0 ? (uint32_t)d->instances : 1, 0, d->base_vertex, (uint32_t)d->base_instance); ex->recorded++; + if (d->itype != NVN_INDEX_TYPE_UNSIGNED_BYTE) mvobj_draw(ex, d, sp.buffer, sp.offset); } // DrawElementsIndirect: the arguments are read by the GPU from the pool buffer (they are often written by an earlier @@ -2499,3 +3273,6 @@ void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) { default: rt_fatal("vk: op %s is not a draw op", nvn_op_name(op)); } } + +// The last executed list's draws as " /:" (bit n: cn), for TOTK_MV_CHECK's lines. +const char *vk_mvobj_frame_masks(void) { return fm_last; } diff --git a/runtime/vk_mvcheck.c b/runtime/vk_mvcheck.c new file mode 100644 index 00000000..16660b24 --- /dev/null +++ b/runtime/vk_mvcheck.c @@ -0,0 +1,474 @@ +// TOTK_MV_CHECK= (test): measures how well DLSS's motion vectors explain the picture, without screenshots or +// judging by eye. Every DLAA frames, vk_dlaa.c mv_pass copies to host memory the scene light target of that +// frame and of the frame before (before DLSS, so jittered), and the motion image twice: after the camera pass (camera +// only) and after the per-object pass (what DLSS gets). Once the GPU is done, the completion thread compares them: +// warp error: the frame before, sampled bilinearly at p + mv + d, against this frame at p, as a relative luminance +// difference |a - b| / (a + b) averaged over pixels; d = last frame's jitter minus this frame's (the vectors are +// between unjittered positions, the pictures are jittered: content at x is drawn at x + jitter). Lower is better, +// and the correct vectors give the lowest. Over every pixel: zero motion, the camera's and the final vectors, and +// the final ones with d subtracted or left out (the jitter convention). Over the object pixels (final and camera +// differ by more than a quarter pixel): the camera's, the final ones and variants of the object part (final - +// camera) scaled by k = -1, 0.5, 1.5, 2 or with one axis negated. The variant with the lowest error names the +// correction the object vectors need, if any ("best"). +// flow: on up to FLOW_MAX object pixels with some texture, the offset whose 5x5 luminance patch of the frame before +// matches best, searched over a window around both the camera and the final vector and refined to a sub-pixel by +// a parabola: the measured motion. The end-point error of the camera and final vectors (+ d) against it, median in +// render pixels, and how often the final vector is the nearer. +// output (DLSS's picture, which the vectors are for): on about OUT_SAMPLES pixels picked independently of the vectors, +// the picture's own motion is measured with a wide search (12 px around the camera vector); where it differs from +// the camera's by over 1.5 px the pixel moves on its own. At each sample, DLSS's output (at the same unjittered +// place) against this frame's input, as a relative luminance difference: on the moving samples it grows where +// DLSS drags old colour along wrong vectors (ghosting, smearing), on the static ones it is the plain difference +// anti-aliasing and upscaling make. Their ratio compares runs with the per-object pass on and off (it does not +// depend on the pass's own vectors) and modes. +// One "mv_check:" line per check. Disocclusion and changing light add error to every candidate alike, so the comparison +// between candidates counts, not the absolute values. Copies and analysis cost a few ms on check frames only. +#include +#include +#include +#include +#include + +#include "cfg.h" +#include "host.h" +#include "nvn_api.h" +#include "vk.h" +#include "vk_exec.h" +#include "vk_dlaa.h" + +#define FLOW_MAX 20000 +#define OBJ_MIN 0.25f // final and camera vectors differ by more than this (render pixels): an object pixel + +#define OUT_SAMPLES 4000 +enum { PREV, CUR, MV_CAM, MV_FIN, OUT, NBUF }; +static struct { + int period; // 0 off, -1 not read yet + VkBuffer buf[NBUF]; + VmaAllocation mem[NBUF]; + uint8_t *map[NBUF]; + uint64_t cap[NBUF]; + uint32_t w, h, ow, oh; // ow, oh: DLSS's output in OUT, 0 none this check + int want_out; // the check frame's output is to be copied (vk_mvcheck_output) + char masks[400]; // the check frame's motion draws (vk_mvobj_frame_masks) + VkFormat cfmt; + uint64_t n; // camera-stage calls (DLAA frames with motion) + uint64_t prev_frame; // gfx frame the PREV colour is of, 0 none + float prev_u[2], u[2], uprev[2]; + int check; // this frame is a check frame (camera stage copied MV_CAM) + int busy; // analysis pending: the buffers are not touched + uint64_t done; +} K = {.period = -1}; + +static int on(void) { + if (K.period < 0) { + const char *v = cfg_get("TOTK_MV_CHECK"); + K.period = v && *v ? atoi(v) : 0; + if (K.period < 0) K.period = 0; + if (K.period && K.period < 4) K.period = 4; + if (K.period) rt_log("mv_check: motion vectors measured every %d frames", K.period); + } + return K.period > 0; +} + +static int cbytes(VkFormat f) { + return f == VK_FORMAT_B10G11R11_UFLOAT_PACK32 ? 4 : f == VK_FORMAT_R16G16B16A16_SFLOAT ? 8 : 0; +} + +// Host buffers of at least the size asked for; a larger size gets new ones and leaves the old (test only). +static int ensure(int i, uint64_t bytes) { + if (K.cap[i] >= bytes) return 1; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bytes, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; + VmaAllocationInfo info; + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &K.buf[i], &K.mem[i], &info) != VK_SUCCESS || !info.pMappedData) { + rt_log("mv_check: host buffer of %llu bytes failed: off", (unsigned long long)bytes); + K.period = 0; + return 0; + } + K.map[i] = info.pMappedData, K.cap[i] = bytes; + return 1; +} + +static void barrier(VkCommandBuffer cb) { + VkMemoryBarrier2 m = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT | VK_ACCESS_2_MEMORY_READ_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_2_HOST_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT | VK_ACCESS_2_MEMORY_READ_BIT | + VK_ACCESS_2_HOST_READ_BIT}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &m}; + vkCmdPipelineBarrier2(cb, &di); +} + +static void copy(VkCommandBuffer cb, int i, VkImage img, uint32_t level, uint32_t layer, uint32_t w, uint32_t h) { + barrier(cb); + VkBufferImageCopy r = {.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, layer, 1}, .imageExtent = {w, h, 1}}; + vkCmdCopyImageToBuffer(cb, img, VK_IMAGE_LAYOUT_GENERAL, K.buf[i], 1, &r); + barrier(cb); +} + +// ---- decoding + +static float half(uint16_t v) { + uint32_t s = (uint32_t)(v >> 15) << 31, e = v >> 10 & 31, m = v & 1023, f; + if (!e) return (s ? -1.0f : 1.0f) * ldexpf((float)m, -24); + f = e == 31 ? s | 0x7f800000u | m << 13 : s | (e + 112) << 23 | m << 13; + float r; + memcpy(&r, &f, 4); + return r; +} + +static float ufloat(uint32_t v, int mbits) { // unsigned 5-bit exponent mini-float + uint32_t e = v >> mbits, m = v & ((1u << mbits) - 1); + if (e == 31) return 65504.0f; + return e ? ldexpf(1.0f + (float)m / (float)(1u << mbits), (int)e - 15) : ldexpf((float)m / (float)(1u << mbits), -14); +} + +static void luminance(const uint8_t *src, VkFormat f, uint32_t n, float *out) { + for (uint32_t i = 0; i < n; i++) { + float r, g, b; + if (f == VK_FORMAT_B10G11R11_UFLOAT_PACK32) { + uint32_t v; + memcpy(&v, src + 4 * (size_t)i, 4); + r = ufloat(v & 0x7ff, 6), g = ufloat(v >> 11 & 0x7ff, 6), b = ufloat(v >> 22, 5); + } else { + uint16_t c[4]; + memcpy(c, src + 8 * (size_t)i, 8); + r = half(c[0]), g = half(c[1]), b = half(c[2]); + } + float y = 0.2126f * r + 0.7152f * g + 0.0722f * b; + out[i] = isfinite(y) && y > 0 ? y : 0; + } +} + +// ---- analysis + +typedef struct { const float *cur, *prev; uint32_t w, h; } Pic; + +static int sample(const Pic *p, float x, float y, float *out) { // bilinear, pixel index coordinates + if (!(x >= 0 && y >= 0 && x <= (float)(p->w - 1) && y <= (float)(p->h - 1))) return 0; + uint32_t x0 = (uint32_t)x, y0 = (uint32_t)y, x1 = x0 + 1 < p->w ? x0 + 1 : x0, y1 = y0 + 1 < p->h ? y0 + 1 : y0; + float fx = x - (float)x0, fy = y - (float)y0; + const float *a = p->prev + (size_t)y0 * p->w, *b = p->prev + (size_t)y1 * p->w; + *out = (a[x0] * (1 - fx) + a[x1] * fx) * (1 - fy) + (b[x0] * (1 - fx) + b[x1] * fx) * fy; + return 1; +} + +typedef struct { double sum; uint64_t n; } Acc; +static void acc(Acc *a, const Pic *p, uint32_t x, uint32_t y, float mx, float my) { + float b; + if (!sample(p, (float)x + mx, (float)y + my, &b)) return; + float c = p->cur[(size_t)y * p->w + x]; + a->sum += fabsf(c - b) / (c + b + 1e-4f), a->n++; +} +static double mean(const Acc *a) { return a->n ? a->sum / (double)a->n : NAN; } + +static float sad5(const Pic *p, uint32_t x, uint32_t y, int dx, int dy) { + float s = 0; + for (int j = -2; j <= 2; j++) + for (int i = -2; i <= 2; i++) { + int px = (int)x + i + dx, py = (int)y + j + dy; + if (px < 0 || py < 0 || px >= (int)p->w || py >= (int)p->h) return INFINITY; + float c = p->cur[(size_t)(y + j) * p->w + x + i], b = p->prev[(size_t)py * p->w + px]; + s += fabsf(c - b) / (c + b + 1e-4f); + } + return s; +} + +static float parabola(float l, float c, float r) { // offset of the minimum of three samples at -1, 0, 1 + float den = l - 2 * c + r; + if (!(den > 1e-9f) || !isfinite(l) || !isfinite(r)) return 0; + float o = 0.5f * (l - r) / den; + return o < -0.5f ? -0.5f : o > 0.5f ? 0.5f : o; // a flat curve says nothing past the neighbours +} + +static int by_float(const void *a, const void *b) { + float x = *(const float *)a, y = *(const float *)b; + return x < y ? -1 : x > y; +} + +// The object pixels' measured motion (sub-pixel), searched over the box around both vectors with a margin of 3; +// 0 when the patch is flat or the match is not clearly better than the box's average. +static int flow(const Pic *p, uint32_t x, uint32_t y, float ax, float ay, float bx, float by, float out[2]) { + float mu = 0, var = 0; + for (int j = -2; j <= 2; j++) + for (int i = -2; i <= 2; i++) mu += p->cur[(size_t)(y + j) * p->w + x + i]; + mu /= 25; + for (int j = -2; j <= 2; j++) + for (int i = -2; i <= 2; i++) { + float d = p->cur[(size_t)(y + j) * p->w + x + i] - mu; + var += d * d; + } + if (mu <= 0 || sqrtf(var / 25) < 0.05f * mu) return 0; + int x0 = (int)floorf(fminf(ax, bx)) - 3, x1 = (int)ceilf(fmaxf(ax, bx)) + 3; + int y0 = (int)floorf(fminf(ay, by)) - 3, y1 = (int)ceilf(fmaxf(ay, by)) + 3; + if (x1 - x0 > 40 || y1 - y0 > 40) return 0; + float best = INFINITY, total = 0; + int bdx = 0, bdy = 0, n = 0; + for (int dy = y0; dy <= y1; dy++) + for (int dx = x0; dx <= x1; dx++) { + float s = sad5(p, x, y, dx, dy); + if (!isfinite(s)) continue; + total += s, n++; + if (s < best) best = s, bdx = dx, bdy = dy; + } + if (!n || !isfinite(best) || best > 0.5f * total / (float)n) return 0; + out[0] = (float)bdx + parabola(sad5(p, x, y, bdx - 1, bdy), best, sad5(p, x, y, bdx + 1, bdy)); + out[1] = (float)bdy + parabola(sad5(p, x, y, bdx, bdy - 1), best, sad5(p, x, y, bdx, bdy + 1)); + return 1; +} + +// Totals over every check (logged every TOTAL_EVERY checks): end-point error histograms in HBIN px bins of the object +// samples (camera, final) and of the static samples (the measurement's own noise floor), and the object warp errors. +#define HBIN 0.05f +#define HN 640 +#define TOTAL_EVERY 20 +#define KN 120 // histogram of per-sample scale ratios, -2 to 4 in steps of 0.05 +enum { H_CAM, H_FIN, H_STATIC, NH }; +static struct { uint64_t h[NH][HN], n[NH], checks, wrong, kh[KN], kn; double obj[2], objn, knum, kden, out[2], outn[2]; uint64_t cal[3]; } T; // cal: checks where k=0.5, k=1 fit the static pixels best; checks + +static void hist_add(int k, float e) { + int b = e >= 0 && e < HBIN * HN ? (int)(e / HBIN) : HN - 1; // NaN and huge errors in the last bin + T.h[k][b < HN ? b : HN - 1]++, T.n[k]++; +} +static float hist_q(int k, double q) { // quantile q of histogram k, bin centre + uint64_t want = (uint64_t)(q * (double)T.n[k]), s = 0; + for (int b = 0; b < HN; b++) + if ((s += T.h[k][b]) > want) return ((float)b + 0.5f) * HBIN; + return NAN; +} + +static float kmedian(void) { + uint64_t s = 0; + for (int b = 0; b < KN; b++) + if ((s += T.kh[b]) * 2 > T.kn) return -2 + ((float)b + 0.5f) * 0.05f; + return NAN; +} + +static uint32_t mix(uint32_t x, uint32_t y) { // a fixed pseudo-random pick of pixels, spread over the picture + uint32_t v = x * 0x9E3779B1u ^ y * 0x85EBCA77u; + v ^= v >> 15, v *= 0x2C1B3C6Du, v ^= v >> 12; + return v; +} + +static void analyse(void *arg) { + (void)arg; + VmaAllocator vma = vk_ctx()->vma; + for (int i = 0; i < NBUF; i++) vmaInvalidateAllocation(vma, K.mem[i], 0, VK_WHOLE_SIZE); + uint32_t w = K.w, h = K.h, n = w * h; + float *cur = malloc(sizeof(float) * n), *prev = malloc(sizeof(float) * n), *e_cam = NULL, *e_fin = NULL, + *e_st = NULL; + if (!cur || !prev) goto out; + luminance(K.map[CUR], K.cfmt, n, cur); + luminance(K.map[PREV], K.cfmt, n, prev); + Pic p = {cur, prev, w, h}; + const uint16_t *A = (const uint16_t *)K.map[MV_CAM], *B = (const uint16_t *)K.map[MV_FIN]; + float dx = K.uprev[0] - K.u[0], dy = K.uprev[1] - K.u[1]; + enum { ZERO, CAM, FIN, FIN_NEG_D, FIN_NO_D, NALL }; + enum { O_CAM, O_FIN, O_KM1, O_K05, O_K15, O_K2, O_NX, O_NY, NOBJ }; + static const char *const oname[NOBJ] = {"camera", "final", "k=-1", "k=0.5", "k=1.5", "k=2", "-x", "-y"}; + Acc all[NALL] = {0}, obj[NOBJ] = {0}, cal[3] = {0}; + // the object pixels first, for the sampling rates: about FLOW_MAX object and FLOW_MAX / 4 static samples + uint64_t nobj = 0, huge = 0; // huge: camera vectors over 256 px or not finite + double cam_len = 0; + for (uint32_t y = 2; y + 2 < h; y++) + for (uint32_t x = 2; x + 2 < w; x++) { + size_t i = (size_t)y * w + x; + float ax = half(A[2 * i]), ay = half(A[2 * i + 1]), bx = half(B[2 * i]), by = half(B[2 * i + 1]); + if (isfinite(ax) && isfinite(ay)) cam_len += hypotf(ax, ay); + huge += !isfinite(ax) || !isfinite(ay) || hypotf(ax, ay) > 256; + nobj += fabsf(bx - ax) > OBJ_MIN || fabsf(by - ay) > OBJ_MIN; + } + uint32_t rate_obj = nobj > FLOW_MAX ? (uint32_t)(0xffffffffull * FLOW_MAX / nobj) : 0xffffffffu; + uint64_t nst = (uint64_t)n - nobj; + uint32_t rate_st = nst > FLOW_MAX / 4 ? (uint32_t)(0xffffffffull * (FLOW_MAX / 4) / nst) : 0xffffffffu; + e_cam = malloc(sizeof(float) * FLOW_MAX * 2), e_fin = malloc(sizeof(float) * FLOW_MAX * 2), + e_st = malloc(sizeof(float) * FLOW_MAX); + if (!e_cam || !e_fin || !e_st) goto out; + int nflow = 0, nearer = 0, tried = 0, nst_f = 0; + double knum = 0, kden = 0; + // object samples the final vector misses by more than 1.5 px where the camera's fits better: where and how + struct { int n; uint32_t x0, y0, x1, y1; double ox, oy, mx, my; } bad = {0, UINT32_MAX, UINT32_MAX, 0, 0, 0, 0, 0, 0}; + for (uint32_t y = 2; y + 2 < h; y++) + for (uint32_t x = 2; x + 2 < w; x++) { + size_t i = (size_t)y * w + x; + float ax = half(A[2 * i]), ay = half(A[2 * i + 1]), bx = half(B[2 * i]), by = half(B[2 * i + 1]); + if (!isfinite(ax) || !isfinite(ay) || !isfinite(bx) || !isfinite(by)) continue; + acc(&all[ZERO], &p, x, y, dx, dy); + acc(&all[CAM], &p, x, y, ax + dx, ay + dy); + acc(&all[FIN], &p, x, y, bx + dx, by + dy); + acc(&all[FIN_NEG_D], &p, x, y, bx - dx, by - dy); + acc(&all[FIN_NO_D], &p, x, y, bx, by); + float ox = bx - ax, oy = by - ay, f[2]; + if (fabsf(ox) <= OBJ_MIN && fabsf(oy) <= OBJ_MIN) { + // calibration: static pixels with a small camera vector (known right), at k = 0.5, 1, 1.5 of it + float al = hypotf(ax, ay); + if (al > 0.25f && al < 2) { + acc(&cal[0], &p, x, y, 0.5f * ax + dx, 0.5f * ay + dy), acc(&cal[1], &p, x, y, ax + dx, ay + dy); + acc(&cal[2], &p, x, y, 1.5f * ax + dx, 1.5f * ay + dy); + } + if (mix(x, y) > rate_st || nst_f >= FLOW_MAX) continue; + if (flow(&p, x, y, ax + dx, ay + dy, ax + dx, ay + dy, f)) e_st[nst_f++] = hypotf(ax + dx - f[0], ay + dy - f[1]); + continue; + } + acc(&obj[O_CAM], &p, x, y, ax + dx, ay + dy); + acc(&obj[O_FIN], &p, x, y, bx + dx, by + dy); + acc(&obj[O_KM1], &p, x, y, ax - ox + dx, ay - oy + dy); + acc(&obj[O_K05], &p, x, y, ax + 0.5f * ox + dx, ay + 0.5f * oy + dy); + acc(&obj[O_K15], &p, x, y, ax + 1.5f * ox + dx, ay + 1.5f * oy + dy); + acc(&obj[O_K2], &p, x, y, ax + 2 * ox + dx, ay + 2 * oy + dy); + acc(&obj[O_NX], &p, x, y, ax - ox + dx, by + dy); + acc(&obj[O_NY], &p, x, y, bx + dx, ay - oy + dy); + if (mix(x, y) > rate_obj || nflow >= 2 * FLOW_MAX) continue; + tried++; + if (!flow(&p, x, y, ax + dx, ay + dy, bx + dx, by + dy, f)) continue; + float ec = hypotf(ax + dx - f[0], ay + dy - f[1]), ef = hypotf(bx + dx - f[0], by + dy - f[1]); + e_cam[nflow] = ec, e_fin[nflow] = ef, nflow++; + nearer += ef < ec; + if (fabsf(ox) > 0.5f || fabsf(oy) > 0.5f) { // the scale of the object part the measured motion shows + float mx = f[0] - ax - dx, my = f[1] - ay - dy, oo = ox * ox + oy * oy, kr = (mx * ox + my * oy) / oo; + knum += mx * ox + my * oy, kden += oo; + int kb = (int)floorf((kr + 2) / 0.05f); + T.kh[kb < 0 ? 0 : kb >= KN ? KN - 1 : kb]++, T.kn++; + } + if (ef > 1.5f && ef > ec) { + bad.n++, bad.ox += ox, bad.oy += oy, bad.mx += f[0] - ax - dx, bad.my += f[1] - ay - dy; + if (x < bad.x0) bad.x0 = x; + if (y < bad.y0) bad.y0 = y; + if (x > bad.x1) bad.x1 = x; + if (y > bad.y1) bad.y1 = y; + } + } + int best = 0; + for (int k = 1; k < NOBJ; k++) + if (obj[k].n && (!obj[best].n || mean(&obj[k]) < mean(&obj[best]))) best = k; + char line[512]; + int o = snprintf(line, sizeof line, "object pixels %.1f%% (%llu):", 100.0 * (double)nobj / (double)n, + (unsigned long long)nobj); + for (int k = 0; k < NOBJ && o < (int)sizeof line; k++) o += snprintf(line + o, sizeof line - o, " %s %.4f", oname[k], mean(&obj[k])); + if (o < (int)sizeof line) snprintf(line + o, sizeof line - o, "; best %s", nobj ? oname[best] : "-"); + // DLSS's output against this frame's input, on samples moving on their own and static ones (the picture's motion) + char outl[200] = ""; + if (K.ow && K.oh) { + float *ol = malloc(sizeof(float) * (size_t)K.ow * K.oh); + if (ol) { + luminance(K.map[OUT], VK_FORMAT_R16G16B16A16_SFLOAT, K.ow * K.oh, ol); + Pic op = {NULL, ol, K.ow, K.oh}; // sample() reads prev + float sx = (float)K.ow / (float)w, sy = (float)K.oh / (float)h; + uint32_t rate = (uint32_t)(0xffffffffull * OUT_SAMPLES / n); + Acc mov = {0}, sta = {0}; + for (uint32_t y = 2; y + 2 < h; y++) + for (uint32_t x = 2; x + 2 < w; x++) { + if ((mix(x, y) ^ 0x5bd1e995u) * 0x2C1B3C6Du > rate) continue; + size_t i = (size_t)y * w + x; + float ax = half(A[2 * i]) + dx, ay = half(A[2 * i + 1]) + dy, f[2], ov; + if (!isfinite(ax) || !isfinite(ay) || !flow(&p, x, y, ax - 12, ay - 12, ax + 12, ay + 12, f)) continue; + if (!sample(&op, ((float)x + 0.5f - K.u[0]) * sx - 0.5f, ((float)y + 0.5f - K.u[1]) * sy - 0.5f, &ov)) + continue; + float c = cur[i], e = fabsf(ov - c) / (ov + c + 1e-4f); + Acc *a = hypotf(f[0] - ax, f[1] - ay) > 1.5f ? &mov : &sta; + a->sum += e, a->n++; + } + snprintf(outl, sizeof outl, "; output against input: moving on their own %.4f (%llu), static %.4f (%llu), " + "ratio %.3f", mean(&mov), (unsigned long long)mov.n, mean(&sta), (unsigned long long)sta.n, + mean(&mov) / mean(&sta)); + if (mov.n) T.out[0] += mov.sum, T.outn[0] += (double)mov.n; + if (sta.n) T.out[1] += sta.sum, T.outn[1] += (double)sta.n; + free(ol); + } + } + float mc = NAN, mf = NAN, ms = NAN; + for (int k = 0; k < nflow; k++) hist_add(H_CAM, e_cam[k]), hist_add(H_FIN, e_fin[k]); + for (int k = 0; k < nst_f; k++) hist_add(H_STATIC, e_st[k]); + if (nflow) { + qsort(e_cam, (size_t)nflow, sizeof *e_cam, by_float), qsort(e_fin, (size_t)nflow, sizeof *e_fin, by_float); + mc = e_cam[nflow / 2], mf = e_fin[nflow / 2]; + } + if (nst_f) qsort(e_st, (size_t)nst_f, sizeof *e_st, by_float), ms = e_st[nst_f / 2]; + char badl[200] = ""; + if (bad.n) + snprintf(badl, sizeof badl, "; %d missed (over 1.5 px, camera nearer) in x %u-%u y %u-%u, object part (%.2f %.2f), " + "measured (%.2f %.2f)", bad.n, bad.x0, bad.x1, bad.y0, bad.y1, bad.ox / bad.n, bad.oy / bad.n, + bad.mx / bad.n, bad.my / bad.n); + rt_log("mv_check: #%llu %ux%u, jitter (%.2f %.2f) after (%.2f %.2f), camera motion %.2f px; warp error, every pixel: " + "zero %.4f camera %.4f final %.4f (jitter subtracted %.4f, left out %.4f); camera vectors over 256 px %llu; %s; " + "flow on %d of %d object " + "samples: median error camera %.2f px, final %.2f px, final nearer %.0f%%; static %.2f px (%d); scale %.2f; calibration, static pixels under 2 px of camera motion (%llu): k=0.5 %.4f k=1 %.4f " + "k=1.5 %.4f%s%s; motion draws (position banks/changed banks:draws)%s", + (unsigned long long)++K.done, w, h, K.u[0], K.u[1], K.uprev[0], K.uprev[1], cam_len / (double)n, + mean(&all[ZERO]), mean(&all[CAM]), mean(&all[FIN]), mean(&all[FIN_NEG_D]), mean(&all[FIN_NO_D]), (unsigned long long)huge, line, + nflow, tried, mc, mf, nflow ? 100.0 * nearer / nflow : 0.0, ms, nst_f, kden > 0 ? knum / kden : NAN, (unsigned long long)cal[1].n, mean(&cal[0]), mean(&cal[1]), mean(&cal[2]), badl, outl, K.masks); + if (cal[1].n > 1000) T.cal[0] += mean(&cal[0]) < mean(&cal[1]) && mean(&cal[0]) < mean(&cal[2]), T.cal[1] += mean(&cal[1]) <= mean(&cal[0]) && mean(&cal[1]) <= mean(&cal[2]), T.cal[2]++; + T.knum += knum, T.kden += kden; + T.checks++; + if (obj[O_FIN].n) { + T.obj[0] += obj[O_CAM].sum, T.obj[1] += obj[O_FIN].sum, T.objn += (double)obj[O_FIN].n; + T.wrong += nobj * 200 > n && best != O_FIN; // over 0.5 % of the picture and the vectors are not the best fit + } + if (T.checks % TOTAL_EVERY == 0) + rt_log("mv_check total: %llu checks, %llu with over 0.5%% object pixels fit better by a variant; object warp error " + "camera %.4f final %.4f; end-point error (median / 90%% / 99%%) object samples camera %.2f / %.2f / %.2f " + "px, final %.2f / %.2f / %.2f px (%llu); static samples %.2f / %.2f / %.2f px (%llu); object scale %.3f (median %.2f of %llu); calibration: k=0.5 best in %llu, k=1 in %llu of %llu checks; " + "output against input: moving on their own %.4f (%.0f), static %.4f (%.0f), ratio %.3f", + (unsigned long long)T.checks, (unsigned long long)T.wrong, T.objn ? T.obj[0] / T.objn : NAN, + T.objn ? T.obj[1] / T.objn : NAN, hist_q(H_CAM, 0.5), hist_q(H_CAM, 0.9), hist_q(H_CAM, 0.99), + hist_q(H_FIN, 0.5), hist_q(H_FIN, 0.9), hist_q(H_FIN, 0.99), (unsigned long long)T.n[H_FIN], + hist_q(H_STATIC, 0.5), hist_q(H_STATIC, 0.9), hist_q(H_STATIC, 0.99), (unsigned long long)T.n[H_STATIC], + T.kden > 0 ? T.knum / T.kden : NAN, kmedian(), (unsigned long long)T.kn, (unsigned long long)T.cal[0], (unsigned long long)T.cal[1], + (unsigned long long)T.cal[2], T.outn[0] ? T.out[0] / T.outn[0] : NAN, T.outn[0], + T.outn[1] ? T.out[1] / T.outn[1] : NAN, T.outn[1], + T.outn[0] && T.outn[1] ? (T.out[0] / T.outn[0]) / (T.out[1] / T.outn[1]) : NAN); +out: + free(cur), free(prev), free(e_cam), free(e_fin), free(e_st); + __atomic_store_n(&K.busy, 0, __ATOMIC_RELEASE); +} + +// ---- capture (vk_dlaa.c mv_pass) + +void vk_mvcheck_camera(VkCommandBuffer cb, VkImage mv, uint32_t w, uint32_t h, float jx, float jy) { + K.check = 0; + if (!on()) return; + K.n++; + uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (K.n % (uint64_t)K.period || __atomic_load_n(&K.busy, __ATOMIC_ACQUIRE) || K.prev_frame + 1 != frame || + K.w != w || K.h != h) + return; + if (!ensure(MV_CAM, (uint64_t)w * h * 4) || !ensure(MV_FIN, (uint64_t)w * h * 4)) return; + copy(cb, MV_CAM, mv, 0, 0, w, h); + K.check = 1, K.u[0] = jx, K.u[1] = jy, K.uprev[0] = K.prev_u[0], K.uprev[1] = K.prev_u[1]; +} + +void vk_mvcheck_final(VkCommandBuffer cb, uint64_t fence, VkImage mv, VkImage cimg, VkFormat cfmt, uint32_t level, + uint32_t layer, uint32_t w, uint32_t h, float jx, float jy) { + if (!on()) return; + uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + int b = cbytes(cfmt); + int want_prev = (K.n + 1) % (uint64_t)K.period == 0 && !__atomic_load_n(&K.busy, __ATOMIC_ACQUIRE); + if (!b) return (void)(K.check = 0, K.prev_frame = 0); + if (K.check) { + if (!ensure(CUR, (uint64_t)w * h * (uint64_t)b)) return; + copy(cb, MV_FIN, mv, 0, 0, w, h); + copy(cb, CUR, cimg, level, layer, w, h); + K.check = 0, K.prev_frame = 0, K.ow = K.oh = 0, K.want_out = 1; + snprintf(K.masks, sizeof K.masks, "%s", vk_mvobj_frame_masks()); + __atomic_store_n(&K.busy, 1, __ATOMIC_RELEASE); + vk_completion_add(fence, analyse, NULL); + return; + } + if (!want_prev) return; + if (!ensure(PREV, (uint64_t)w * h * (uint64_t)b)) return; + copy(cb, PREV, cimg, level, layer, w, h); + K.prev_frame = frame, K.prev_u[0] = jx, K.prev_u[1] = jy, K.w = w, K.h = h, K.cfmt = cfmt; +} + +// After DLSS ran on a check frame (vk_dlaa.c evaluate): its output (RGBA16F, ow x oh) for the analysis, which runs once +// the submission is done. +void vk_mvcheck_output(VkCommandBuffer cb, VkImage out, uint32_t ow, uint32_t oh) { + if (!K.want_out) return; + K.want_out = 0; + if (!ensure(OUT, (uint64_t)ow * oh * 8)) return; + copy(cb, OUT, out, 0, 0, ow, oh); + K.ow = ow, K.oh = oh; +} diff --git a/runtime/vk_mvobj_fs.h b/runtime/vk_mvobj_fs.h new file mode 100644 index 00000000..19b1a16b --- /dev/null +++ b/runtime/vk_mvobj_fs.h @@ -0,0 +1,39 @@ +// Generated by tools/make_mvobj_fs.py: the per-object motion fragment shader (vk_mvobj.c). Do not edit. +#define MVOBJ_FS_LOCATION_WORD 24 // the input Location literal, patched to the variant's +static const uint32_t mvobj_fs_spv[] = { + 0x07230203, 0x00010000, 0x00000000, 0x00000034, 0x00000000, 0x00020011, 0x00000001, 0x0003000e, + 0x00000000, 0x00000001, 0x0008000f, 0x00000004, 0x00000001, 0x6e69616d, 0x00000000, 0x0000000e, + 0x0000000f, 0x00000010, 0x00030010, 0x00000001, 0x00000007, 0x00040047, 0x0000000e, 0x0000001e, + 0x00000000, 0x00040047, 0x0000000f, 0x0000000b, 0x0000000f, 0x00040047, 0x00000010, 0x0000001e, + 0x00000000, 0x00030047, 0x0000000b, 0x00000002, 0x00050048, 0x0000000b, 0x00000000, 0x00000023, + 0x00000000, 0x00040047, 0x00000011, 0x00000022, 0x00000000, 0x00040047, 0x00000011, 0x00000021, + 0x00000001, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00030016, 0x00000004, + 0x00000020, 0x00040017, 0x00000005, 0x00000004, 0x00000002, 0x00040017, 0x00000006, 0x00000004, + 0x00000004, 0x00040015, 0x00000007, 0x00000020, 0x00000000, 0x00020014, 0x00000008, 0x00040020, + 0x00000009, 0x00000001, 0x00000006, 0x00040020, 0x0000000a, 0x00000003, 0x00000006, 0x0003001e, + 0x0000000b, 0x00000006, 0x00040020, 0x0000000c, 0x00000002, 0x0000000b, 0x00040020, 0x00000017, + 0x00000002, 0x00000006, 0x0004002b, 0x00000004, 0x00000012, 0x3f000000, 0x0004002b, 0x00000004, + 0x00000013, 0x00000000, 0x0004002b, 0x00000004, 0x00000014, 0x358637bd, 0x0004002b, 0x00000007, + 0x00000015, 0x00000000, 0x0007002c, 0x00000006, 0x00000016, 0x00000013, 0x00000013, 0x00000013, + 0x00000013, 0x0004003b, 0x00000009, 0x0000000e, 0x00000001, 0x0004003b, 0x00000009, 0x0000000f, + 0x00000001, 0x0004003b, 0x0000000a, 0x00000010, 0x00000003, 0x0004003b, 0x0000000c, 0x00000011, + 0x00000002, 0x00050036, 0x00000002, 0x00000001, 0x00000000, 0x00000003, 0x000200f8, 0x00000018, + 0x00050041, 0x00000017, 0x00000019, 0x00000011, 0x00000015, 0x0004003d, 0x00000006, 0x0000001a, + 0x00000019, 0x0004003d, 0x00000006, 0x0000001b, 0x0000000e, 0x0004003d, 0x00000006, 0x0000001c, + 0x0000000f, 0x0007004f, 0x00000005, 0x0000001d, 0x0000001c, 0x0000001c, 0x00000000, 0x00000001, + 0x0007004f, 0x00000005, 0x0000001e, 0x0000001a, 0x0000001a, 0x00000002, 0x00000003, 0x00050083, + 0x00000005, 0x0000001f, 0x0000001d, 0x0000001e, 0x00050051, 0x00000004, 0x00000020, 0x0000001b, + 0x00000000, 0x00050051, 0x00000004, 0x00000021, 0x0000001b, 0x00000001, 0x00050051, 0x00000004, + 0x00000022, 0x0000001b, 0x00000003, 0x00050088, 0x00000004, 0x00000023, 0x00000020, 0x00000022, + 0x00050088, 0x00000004, 0x00000024, 0x00000021, 0x00000022, 0x00050051, 0x00000004, 0x00000025, + 0x0000001a, 0x00000000, 0x00050051, 0x00000004, 0x00000026, 0x0000001a, 0x00000001, 0x00050085, + 0x00000004, 0x00000027, 0x00000023, 0x00000012, 0x00050081, 0x00000004, 0x00000028, 0x00000027, + 0x00000012, 0x00050085, 0x00000004, 0x00000029, 0x00000028, 0x00000025, 0x00050085, 0x00000004, + 0x0000002a, 0x00000024, 0x00000012, 0x00050083, 0x00000004, 0x0000002b, 0x00000012, 0x0000002a, + 0x00050085, 0x00000004, 0x0000002c, 0x0000002b, 0x00000026, 0x00050051, 0x00000004, 0x0000002d, + 0x0000001f, 0x00000000, 0x00050051, 0x00000004, 0x0000002e, 0x0000001f, 0x00000001, 0x00050083, + 0x00000004, 0x0000002f, 0x00000029, 0x0000002d, 0x00050083, 0x00000004, 0x00000030, 0x0000002c, + 0x0000002e, 0x00070050, 0x00000006, 0x00000031, 0x0000002f, 0x00000030, 0x00000013, 0x00000013, + 0x000500ba, 0x00000008, 0x00000032, 0x00000022, 0x00000014, 0x000600a9, 0x00000006, 0x00000033, + 0x00000032, 0x00000031, 0x00000016, 0x0003003e, 0x00000010, 0x00000033, 0x000100fd, 0x00010038 +}; diff --git a/runtime/vk_mvprev.c b/runtime/vk_mvprev.c new file mode 100644 index 00000000..c8b09ab5 --- /dev/null +++ b/runtime/vk_mvprev.c @@ -0,0 +1,217 @@ +// Last frame's object and bone blocks, for per-object motion vectors (TOTK_DLAA_OBJMV, on with DLAA). The banks kept +// start as c5, c7 and c14 and grow with what the vertex shader variants read (vk_mvprev_want); c7 and c5 are described +// here, the others follow the same two-copy rule. +// +// The scene's vertex shaders read an object's place from bank c7 (gsys_shape: one 3x4 matrix at the start of a +// 0x100 block) and a skinned object's bones from c5 (gsys_skeleton: a run of 3x4 matrices), of this frame only. The +// game writes both into two copies that take turns, the second right after the first: an object's block is at addr in +// one frame and at addr +- step in the next, step its size rounded up to 256 bytes (vk_mvsurvey.c measured it: every +// c7 pair 0x100 apart, c5 pairs their rounded size apart, the pair's contents the same object moved a little). The +// copy this frame does not write still holds last frame's when the frame is submitted, so the uniform snapshot +// (vk_usnap.c) also copies both neighbours of these blocks, and vk_mvprev_find picks last frame's: +// self the block was bound at the same address last frame too (an object the game did not write again): itself +// side one neighbour was bound in that bank last frame: that one +// nearer both were (the copies of adjacent objects interleave): the one with more of its first 16 words equal to +// this frame's +// none neither: a new object, no motion of its own +// The pick is checked against what was snapshotted of that block last frame (intact: the same first CHECK bytes), and +// a summary goes to the log every SUMMARY frames: +// mv_prev: c: blocks: self a %, side b %, nearer c %, none d %; intact e %; moved: still f %, under +// one unit g %, more h % +#include +#include +#include +#include + +#include "cfg.h" +#include "host.h" +#include "vk.h" +#include "vk_exec.h" +#include "vk_dlaa.h" + +#define CHECK 128u +#define TABLE (1u << 14) +#define SUMMARY 600 + +enum { BONES = 2, PLACE = 4, CLOCK = 11, BANKS = 14 }; // NVN binding index: banks c5, c7 and c14; c3..c16 + +// 16 bytes, so a table of TABLE entries stays in the cache: every scene draw probes it (the bytes of TOTK_MVPREV_CHECK +// live in a table of their own, by entry index, allocated only for the check) +typedef struct { + uint64_t addr; // 0: empty + int index; + uint32_t have; +} Seen; + +typedef struct { uint64_t blocks, self, side, nearer, none, checked, intact, still, small, big; } Stat; + +static struct { + int on; // 1 knob on and DLAA on this frame + int knob; // -1 not read + Seen *t[2]; // this frame's bound blocks, last frame's + uint8_t (*bytes[2])[CHECK]; // TOTK_MVPREV_CHECK: the first CHECK bytes of each entry of t[k], NULL without the check + int timed; // TOTK_MVOBJ_TIMES: the summary's times + uint32_t frames; + Stat st[BANKS]; // per binding index + uint64_t last[BANKS]; // the block the last scene draw bound per binding index + uint32_t mask; // binding indices kept (vk_mvprev_want): bones, place and clock from the start + uint64_t ns_draw, ns_frame; // time in vk_mvprev_draw and vk_mvprev_frame since the last summary +} P = {.knob = -1, .mask = 1u << BONES | 1u << PLACE | 1u << CLOCK}; + +static uint32_t slot_of(uint64_t addr, int index) { + uint64_t h = (addr ^ (uint64_t)index * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull; + return (uint32_t)(h >> 40) & (TABLE - 1); +} + +static Seen *find(Seen *t, uint64_t addr, int index, int add) { + for (uint32_t i = slot_of(addr, index), k = 0; k < TABLE; i = (i + 1) & (TABLE - 1), k++) { + if (t[i].addr == addr && t[i].index == index) return &t[i]; + if (!t[i].addr) { + if (!add) return NULL; + t[i].addr = addr, t[i].index = index; + return &t[i]; + } + } + return NULL; +} + +int vk_mvprev_on(void) { return P.on > 0; } + +void vk_mvprev_want(uint32_t index_mask) { + static int fixed = -1; + if (fixed < 0) fixed = !rt_knob("MVPREV_GROW", "1"); // TOTK_MVPREV_GROW=0: c5, c7 and c14 only (test) + if (!fixed) P.mask |= index_mask & ((1u << BANKS) - 1); +} + +uint64_t vk_mvprev_step(int stage, int index, uint64_t size) { + if (P.on <= 0 || stage != 0 || index < 0 || index >= BANKS || !(P.mask >> index & 1)) return 0; + return (size + 255) & ~(uint64_t)255; +} + +// Translation distance between the first matrices of two blocks (rows' fourth floats). +static float moved(const uint8_t *a, const uint8_t *b) { + const float *x = (const float *)a, *y = (const float *)b; + float dx = x[3] - y[3], dy = x[7] - y[7], dz = x[11] - y[11]; + return sqrtf(dx * dx + dy * dy + dz * dz); +} + +int vk_mvprev_find(int index, uint64_t addr, uint64_t size, uint64_t *prev, int *how) { + *how = MVPREV_NONE; + if (P.on <= 0 || !P.t[1]) return 0; + if (find(P.t[1], addr, index, 0)) { + *prev = addr, *how = MVPREV_SELF; + return 1; + } + uint64_t step = (size + 255) & ~(uint64_t)255, cand[2]; + int n = 0; + if (find(P.t[1], addr + step, index, 0)) cand[n++] = addr + step; + if (addr >= step && find(P.t[1], addr - step, index, 0)) cand[n++] = addr - step; + if (!n) return 0; + if (n == 2) { // the copies of adjacent objects interleave: the one with more words equal to this frame's + const uint32_t *c = (const uint32_t *)vk_usnap_bytes(addr, 64), *a = (const uint32_t *)vk_usnap_bytes(cand[0], 64), + *b = (const uint32_t *)vk_usnap_bytes(cand[1], 64); + if (!c || !a || !b) return 0; + int na = 0, nb = 0; + for (int w = 0; w < 16; w++) na += c[w] == a[w], nb += c[w] == b[w]; + if (nb > na) cand[0] = cand[1]; + *how = MVPREV_NEARER; + } else { + *how = MVPREV_SIDE; + } + *prev = cand[0]; + return 1; +} + +static void mvprev_draw_(VkExec *ex); +void vk_mvprev_draw(VkExec *ex) { + if (P.on <= 0) return; + if (!P.timed) return mvprev_draw_(ex); // the clock reads were 1 % of the render thread + uint64_t t0 = kern_now_ns(); + mvprev_draw_(ex); + P.ns_draw += kern_now_ns() - t0; +} + +static void mvprev_draw_(VkExec *ex) { + const VkExecTarget *z = &ex->depth; + if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return; + int check = P.bytes[0] != NULL; // TOTK_MVPREV_CHECK=1: the summary's checks (3.6 ms a frame) + for (int index = 0; index < BANKS; index++) { + if (!(P.mask >> index & 1)) continue; + const p_bind_buf *u = &ex->ubo[0][index]; + if (!u->addr || !u->size || u->addr == P.last[index]) continue; // the next draws keep the block: noted already + P.last[index] = u->addr; + Seen *k = find(P.t[0], u->addr, index, 1); + if (!k || k->have || !check) continue; // else only the address is noted + int w = index; + uint32_t want = u->size < CHECK ? (uint32_t)u->size : CHECK; + const uint8_t *cur = vk_usnap_bytes(u->addr, want); + if (!cur) continue; + memcpy(P.bytes[0][k - P.t[0]], cur, want), k->have = want; + Stat *s = &P.st[w]; + s->blocks++; + uint64_t prev; + int how; + if (!vk_mvprev_find(index, u->addr, u->size, &prev, &how)) { + s->none++; + continue; + } + if (how == MVPREV_SELF) s->self++; + else if (how == MVPREV_SIDE) s->side++; + else s->nearer++; + const Seen *last = find(P.t[1], prev, index, 0); + const uint8_t *now = vk_usnap_bytes(prev, want); + if (last && now && last->have >= want && want >= 48) { + s->checked++; + if (!memcmp(now, P.bytes[1][last - P.t[1]], want)) s->intact++; + float d = moved(cur, now); + if (d < 0.01f) s->still++; + else if (d < 1.0f) s->small++; + else s->big++; + } + } +} + +void vk_mvprev_frame(void) { + if (P.knob < 0) { + P.knob = rt_knob("DLAA_OBJMV", "1"); + if (P.knob) { + P.t[0] = calloc(TABLE, sizeof(Seen)), P.t[1] = calloc(TABLE, sizeof(Seen)); + if (!P.t[0] || !P.t[1]) P.knob = 0; + if (P.knob && rt_knob("MVPREV_CHECK", "0")) { + P.bytes[0] = calloc(TABLE, CHECK), P.bytes[1] = calloc(TABLE, CHECK); + if (!P.bytes[0] || !P.bytes[1]) free(P.bytes[0]), free(P.bytes[1]), P.bytes[0] = P.bytes[1] = NULL; + } + P.timed = rt_knob("MVOBJ_TIMES", "0"); + } + } + int on = P.knob && cfg_bool("TOTK_DLAA"); + if (on != (P.on > 0)) { + P.on = on ? 1 : -1; + if (P.t[0]) memset(P.t[0], 0, TABLE * sizeof(Seen)), memset(P.t[1], 0, TABLE * sizeof(Seen)); + return; + } + if (P.on <= 0) return; + uint64_t t0 = P.timed ? kern_now_ns() : 0; + Seen *t = P.t[1]; + P.t[1] = P.t[0], P.t[0] = t; + uint8_t (*b)[CHECK] = P.bytes[1]; + P.bytes[1] = P.bytes[0], P.bytes[0] = b; + memset(P.t[0], 0, TABLE * sizeof(Seen)); + memset(P.last, 0, sizeof P.last); + if (P.timed) P.ns_frame += kern_now_ns() - t0; + if (++P.frames % SUMMARY) return; + if (P.timed) rt_log("mv_prev: %.3f ms a frame in the draws, %.3f ms at the present", P.ns_draw / 1e6 / SUMMARY, + P.ns_frame / 1e6 / SUMMARY); + P.ns_draw = P.ns_frame = 0; + for (int w = 0; w < BANKS; w++) { + Stat *s = &P.st[w]; + if (!s->blocks) continue; + double b = (double)s->blocks, c = s->checked ? (double)s->checked : 1; + rt_log("mv_prev: c%d: %llu blocks: self %.1f %%, side %.1f %%, nearer %.1f %%, none %.1f %%; intact %.1f %% of " + "%llu; moved: still %.1f %%, under one unit %.1f %%, more %.1f %%", + w + 3, (unsigned long long)s->blocks, 100 * s->self / b, 100 * s->side / b, 100 * s->nearer / b, + 100 * s->none / b, 100 * s->intact / c, (unsigned long long)s->checked, 100 * s->still / c, + 100 * s->small / c, 100 * s->big / c); + } + memset(P.st, 0, sizeof P.st); +} diff --git a/runtime/vk_mvsurvey.c b/runtime/vk_mvsurvey.c new file mode 100644 index 00000000..0e782c21 --- /dev/null +++ b/runtime/vk_mvsurvey.c @@ -0,0 +1,258 @@ +// Motion-vector survey (TOTK_MV_SURVEY=1, off by default): measures whether per-object motion vectors can be made from +// the uniform blocks the scene's draws bind. Per-object vectors need each object's matrices of the previous frame; the +// game keeps none (its shaders read gsys_shape, gsys_skeleton and gsys_projection of this frame only), so they would +// have to be kept from the previous frame's uniform snapshot (vk_usnap.c). That works when a block's guest address +// stands for the same object from one frame to the next, which this survey measures. +// +// For every scene draw (vertex stage, scene depth bound: D32 at least DLAA_SCENE_MIN_W wide, as vk_dlaa_draw), each user bank +// (c3 + index) with a binding is noted: address, size and its first SNAP bytes as of the submit (the snapshot's copy). +// At each frame the bank's blocks are compared with the previous frame's and the one before: +// repeat1 / repeat2 blocks whose address was bound in that bank one / two frames ago +// same / near / far of the blocks repeated from last frame: bytes unchanged / every float within 5 % or 1.0 of +// last frame's (the same object, moved a little) / some float further (another object, likely) +// The game writes its per-object blocks into two copies that take turns, so a moving object's block is at one address +// in even frames and another in odd ones. To find the rule that pairs them, a bank's blocks bound two frames ago but +// not last frame (this frame's copy) are sorted by address, and so are last frame's blocks not bound this frame (the +// other copy); with equal counts they are paired in order: +// pairs, delta the paired blocks and the commonest address differences (this frame's minus last frame's) +// psame / pnear / pfar the pairs' contents, as same / near / far above (near: the same object, moved) +// unequal frames whose two lists had different lengths (no pairing that frame) +// What the bank holds: each block's leading 3x4 matrices (rows of a rotation times a scale: three rows of the same +// length, at right angles, finite), counted over the whole block as of the submit: +// mats 0 / 1 / 2-7 / 8+ blocks by that count (one: an object's place, gsys_shape; many: bones, gsys_skeleton) +// Every SURVEY_FRAMES frames the log gets two lines per bank with the window's totals: +// mv_survey: c: draws, blocks (), sizes ; repeat1 a %, repeat2 b %; +// same c %, near d %, far e % +// mv_survey: c: pairs ( frames unequal), delta ; psame a %, pnear b %, pfar c %; +// mats 0 x 1 x 2-7 x 8+ x, at most +#include +#include +#include +#include + +#include "host.h" +#include "vk.h" +#include "vk_exec.h" +#include "vk_dlaa.h" + +#define BANKS 18 +#define SNAP 256u +#define SURVEY_FRAMES 300 +#define TABLE (1u << 15) // blocks per frame, all banks; a town frame binds about 10,000 uniform ranges + +typedef struct { + uint64_t addr; // 0: empty slot + uint32_t size, n; + int bank; + uint8_t bytes[SNAP]; + uint32_t have; // bytes of `bytes` that were in the snapshot + uint32_t mats; // leading 3x4 matrices of the whole block, 0 when it was not in the snapshot +} Block; + +typedef struct { + uint64_t draws, blocks, rep1, rep2, same, near, far, unread; + uint32_t size[4]; + uint64_t size_n[4]; + uint64_t pairs, psame, pnear, pfar, unequal, mats[4]; + uint32_t mats_max; + int64_t delta[4]; + uint64_t delta_n[4]; +} Stat; + +static struct { + int on; + Block *t[3]; // this frame, last frame, the one before + const Block **x, **y; // pairing: this frame's copy, last frame's + uint32_t frames; + Stat st[BANKS]; +} S; + +static uint32_t slot_of(uint64_t addr, int bank) { + uint64_t h = (addr ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull; + return (uint32_t)(h >> 40) & (TABLE - 1); +} + +static Block *find(Block *t, uint64_t addr, int bank, int add) { + for (uint32_t i = slot_of(addr, bank), k = 0; k < TABLE; i = (i + 1) & (TABLE - 1), k++) { + if (t[i].addr == addr && t[i].bank == bank) return &t[i]; + if (!t[i].addr) { + if (!add) return NULL; + t[i].addr = addr, t[i].bank = bank; + return &t[i]; + } + } + return NULL; // full: the block goes uncounted +} + +static void note_size(Stat *s, uint32_t size) { + for (int i = 0; i < 4; i++) + if (s->size[i] == size || !s->size_n[i]) { + s->size[i] = size, s->size_n[i]++; + return; + } + s->size_n[3]++; // a fifth size: counted with the last +} + +static void note_delta(Stat *s, int64_t d) { + for (int i = 0; i < 4; i++) + if (s->delta[i] == d || !s->delta_n[i]) { + s->delta[i] = d, s->delta_n[i]++; + return; + } + s->delta_n[3]++; +} + +// Leading 3x4 matrices: three rows of about the same length (a uniform scale), at right angles, all finite. +static uint32_t lead_mats(const float *f, uint32_t nf) { + uint32_t k = 0; + for (; (k + 1) * 12 <= nf && k < 4096; k++) { + const float *m = f + k * 12; + float l[3]; + for (int r = 0; r < 3; r++) { + const float *v = m + r * 4; + if (!isfinite(v[0]) || !isfinite(v[1]) || !isfinite(v[2]) || !isfinite(v[3])) return k; + l[r] = sqrtf(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); + if (l[r] < 1e-3f || l[r] > 1e3f) return k; + } + if (fabsf(l[1] - l[0]) > 0.02f * l[0] || fabsf(l[2] - l[0]) > 0.02f * l[0]) return k; + for (int a = 0; a < 3; a++) + for (int b = a + 1; b < 3; b++) { + const float *u = m + a * 4, *v = m + b * 4; + if (fabsf(u[0] * v[0] + u[1] * v[1] + u[2] * v[2]) > 0.02f * l[a] * l[b]) return k; + } + } + return k; +} + +static int by_bank_addr(const void *a, const void *b) { + const Block *x = *(const Block *const *)a, *y = *(const Block *const *)b; + if (x->bank != y->bank) return x->bank < y->bank ? -1 : 1; + return x->addr < y->addr ? -1 : x->addr > y->addr; +} + +static void survey_init(void) { + S.on = rt_knob("MV_SURVEY", "0") ? 1 : -1; + if (S.on < 0) return; + for (int i = 0; i < 3; i++) S.t[i] = calloc(TABLE, sizeof(Block)); + S.x = malloc(TABLE * sizeof *S.x), S.y = malloc(TABLE * sizeof *S.y); + if (!S.t[0] || !S.t[1] || !S.t[2] || !S.x || !S.y) { + S.on = -1; + return; + } + rt_log("mv_survey: on (scene draws' uniform blocks, a summary every %d frames)", SURVEY_FRAMES); +} + +void vk_mvsurvey_draw(VkExec *ex) { + if (!S.on) survey_init(); + if (S.on < 0) return; + const VkExecTarget *z = &ex->depth; + if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return; + for (int b = 0; b < BANKS; b++) { + const p_bind_buf *u = &ex->ubo[0][b]; + if (!u->addr || !u->size) continue; + S.st[b].draws++; + Block *k = find(S.t[0], u->addr, b, 1); + if (!k) continue; + if (k->n++) continue; + k->size = (uint32_t)u->size; + uint32_t want = u->size < SNAP ? (uint32_t)u->size : SNAP; + VkDescriptorBufferInfo bi; + const uint8_t *src = vk_usnap_find(u->addr, want, 1, &bi) ? vk_usnap_map() + bi.offset : NULL; + k->have = src ? want : 0; + if (src) memcpy(k->bytes, src, want); + const uint8_t *all = vk_usnap_find(u->addr, u->size, 1, &bi) ? vk_usnap_map() + bi.offset : NULL; + k->mats = all ? lead_mats((const float *)all, (uint32_t)(u->size / 4)) : 0; + if (all) { + Stat *s = &S.st[b]; + s->mats[k->mats == 0 ? 0 : k->mats == 1 ? 1 : k->mats < 8 ? 2 : 3]++; + if (k->mats > s->mats_max) s->mats_max = k->mats; + } + } +} + +// Same object moved a little: every float within 5 % or 1.0 of last frame's. +static int compare(const Block *a, const Block *b) { + uint32_t n = a->have < b->have ? a->have : b->have; + if (!n) return -1; + if (!memcmp(a->bytes, b->bytes, n)) return 0; + const float *x = (const float *)a->bytes, *y = (const float *)b->bytes; + for (uint32_t i = 0; i < n / 4; i++) { + float d = fabsf(x[i] - y[i]); + if (isnan(d) || (d > 1.0f && d > 0.05f * fabsf(y[i]))) return 2; + } + return 1; +} + +void vk_mvsurvey_frame(void) { + if (S.on <= 0) return; + for (uint32_t i = 0; i < TABLE; i++) { + const Block *k = &S.t[0][i]; + if (!k->addr) continue; + Stat *s = &S.st[k->bank]; + s->blocks++; + note_size(s, k->size); + const Block *p = find(S.t[1], k->addr, k->bank, 0); + if (p) { + s->rep1++; + int c = compare(k, p); + if (c == 0) s->same++; + else if (c == 1) s->near++; + else if (c == 2) s->far++; + else s->unread++; + } + if (find(S.t[2], k->addr, k->bank, 0)) s->rep2++; + } + // pairing of the two copies, bank by bank + uint32_t nx = 0, ny = 0; + for (uint32_t i = 0; i < TABLE; i++) { + const Block *k = &S.t[0][i]; + if (k->addr && !find(S.t[1], k->addr, k->bank, 0) && find(S.t[2], k->addr, k->bank, 0)) S.x[nx++] = k; + k = &S.t[1][i]; + if (k->addr && !find(S.t[0], k->addr, k->bank, 0)) S.y[ny++] = k; + } + qsort(S.x, nx, sizeof *S.x, by_bank_addr); + qsort(S.y, ny, sizeof *S.y, by_bank_addr); + for (uint32_t i = 0, j = 0; i < nx;) { + int b = S.x[i]->bank; + uint32_t i1 = i, j0, j1; + while (i1 < nx && S.x[i1]->bank == b) i1++; + while (j < ny && S.y[j]->bank < b) j++; + for (j0 = j1 = j; j1 < ny && S.y[j1]->bank == b;) j1++; + Stat *s = &S.st[b]; + if (i1 - i != j1 - j0) s->unequal++; + else + for (uint32_t a = i, c = j0; a < i1; a++, c++) { + s->pairs++; + note_delta(s, (int64_t)(S.x[a]->addr - S.y[c]->addr)); + int r = compare(S.x[a], S.y[c]); + if (r == 0) s->psame++; + else if (r == 1) s->pnear++; + else if (r == 2) s->pfar++; + } + i = i1, j = j1; + } + Block *old = S.t[2]; + S.t[2] = S.t[1], S.t[1] = S.t[0], S.t[0] = old; + memset(S.t[0], 0, TABLE * sizeof(Block)); + if (++S.frames % SURVEY_FRAMES) return; + for (int b = 0; b < BANKS; b++) { + Stat *s = &S.st[b]; + if (!s->blocks) continue; + double bl = (double)s->blocks, r1 = s->rep1 ? (double)s->rep1 : 1; + rt_log("mv_survey: c%d: %llu draws, %llu blocks (%.0f a frame), sizes %#x x%llu %#x x%llu %#x x%llu; repeat1 %.1f %%, " + "repeat2 %.1f %%; same %.1f %%, near %.1f %%, far %.1f %%, unread %.1f %%", + b + 3, (unsigned long long)s->draws, (unsigned long long)s->blocks, bl / SURVEY_FRAMES, s->size[0], + (unsigned long long)s->size_n[0], s->size[1], (unsigned long long)s->size_n[1], s->size[2], + (unsigned long long)s->size_n[2], 100 * s->rep1 / bl, 100 * s->rep2 / bl, 100 * s->same / r1, + 100 * s->near / r1, 100 * s->far / r1, 100 * s->unread / r1); + double pr = s->pairs ? (double)s->pairs : 1; + rt_log("mv_survey: c%d: pairs %llu (%llu frames unequal), delta %#llx x%llu %#llx x%llu %#llx x%llu; psame %.1f %%, " + "pnear %.1f %%, pfar %.1f %%; mats 0 x%llu 1 x%llu 2-7 x%llu 8+ x%llu, at most %u", + b + 3, (unsigned long long)s->pairs, (unsigned long long)s->unequal, (unsigned long long)s->delta[0], + (unsigned long long)s->delta_n[0], (unsigned long long)s->delta[1], (unsigned long long)s->delta_n[1], + (unsigned long long)s->delta[2], (unsigned long long)s->delta_n[2], 100 * s->psame / pr, + 100 * s->pnear / pr, 100 * s->pfar / pr, (unsigned long long)s->mats[0], (unsigned long long)s->mats[1], + (unsigned long long)s->mats[2], (unsigned long long)s->mats[3], s->mats_max); + } + memset(S.st, 0, sizeof S.st); +} diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index b0953383..8c288eec 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -1913,6 +1913,8 @@ static void rec_fill(KeyRec *r, const VkPipeState *s) { // A pipeline was built for a draw: queue its record for the file. Caller holds P.lock. static void keys_note(const VkPipeState *s) { if (!A.on || !K.path[0]) return; + static const uint8_t none[32]; + if (s->fs && !memcmp(gfx_shader_input_id(s->fs), none, 32)) return; // the runtime's own shader (gfx_shader_own) KeyRec r; rec_fill(&r, s); if (!seen_add(rec_hash(&r))) return; diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 28bf7ab8..ffa2c72e 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -38,7 +38,7 @@ enum { R_PAD, R_BeginQuery, R_BeginRendering, R_BindDescriptorSets, R_BindIndexBuffer, R_BindPipeline, R_BindVertexBuffers2, R_BlitImage, R_ClearAttachments, R_ClearColorImage, R_ClearDepthStencilImage, R_CopyBuffer, R_CopyBufferToImage, R_CopyImage, R_CopyImageToBuffer, R_Dispatch, R_DispatchIndirect, R_Draw, R_DrawIndexed, R_DrawIndexedIndirect, - R_EndQuery, R_EndRendering, R_FillBuffer, R_PipelineBarrier2, R_PushConstants, R_ResetQueryPool, R_ResolveImage, + R_EndQuery, R_EndRendering, R_ExecuteCommands, R_FillBuffer, R_PipelineBarrier2, R_PushConstants, R_ResetQueryPool, R_ResolveImage, R_SetBlendConstants, R_SetCullMode, R_SetDepthBias, R_SetDepthBiasEnable, R_SetDepthCompareOp, R_SetDepthTestEnable, R_SetDepthWriteEnable, R_SetFrontFace, R_SetPrimitiveRestartEnable, R_SetPrimitiveTopology, R_SetRasterizerDiscardEnable, R_SetScissorWithCount, R_SetStencilCompareMask, R_SetStencilOp, R_SetStencilReference, @@ -491,6 +491,15 @@ void rc_vkCmdClearDepthStencilImage(VkCommandBuffer cb, VkImage i, VkImageLayout pub(); } +typedef struct { uint32_t n; const VkCommandBuffer *c; } A_ExecuteCommands; +void rc_vkCmdExecuteCommands(VkCommandBuffer cb, uint32_t n, const VkCommandBuffer *c) { + if (!OWNED(cb)) { vkCmdExecuteCommands(cb, n, c); return; } + A_ExecuteCommands *a = put(R_ExecuteCommands, sizeof *a + sizeof *c * n); + uint8_t *at = (uint8_t *)(a + 1); + a->n = n, a->c = tail_copy(&at, c, sizeof *c * n); + pub(); +} + typedef struct { VkPipelineLayout l; VkShaderStageFlags st; uint32_t o, n; const void *v; } A_PushConstants; void rc_vkCmdPushConstants(VkCommandBuffer cb, VkPipelineLayout l, VkShaderStageFlags st, uint32_t o, uint32_t n, const void *v) { @@ -677,6 +686,7 @@ static void exec_packet(uint32_t op, void *p) { break; } F(PushConstants, a->l, a->st, a->o, a->n, a->v) + F(ExecuteCommands, a->n, a->c) #undef F case R_EndRendering: vkCmdEndRendering(cb); break; case R_BlitImage: { A_Img *a = p; vkCmdBlitImage(cb, a->s, a->sl, a->d, a->dl, a->n, a->r, a->f); break; } diff --git a/runtime/vk_rec.h b/runtime/vk_rec.h index c4a1faf8..feddc707 100644 --- a/runtime/vk_rec.h +++ b/runtime/vk_rec.h @@ -80,6 +80,7 @@ void rec_pass_mark(VkCommandBuffer cb, uint32_t id, int end); X(EndRendering, (VkCommandBuffer cb), (cb)) \ X(FillBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkDeviceSize n, uint32_t v), (cb, b, o, n, v)) \ X(PipelineBarrier2, (VkCommandBuffer cb, const VkDependencyInfo *di), (cb, di)) \ + X(ExecuteCommands, (VkCommandBuffer cb, uint32_t n, const VkCommandBuffer *c), (cb, n, c)) \ X(PushConstants, (VkCommandBuffer cb, VkPipelineLayout l, VkShaderStageFlags st, uint32_t o, uint32_t n, \ const void *v), (cb, l, st, o, n, v)) \ X(ResetQueryPool, (VkCommandBuffer cb, VkQueryPool p, uint32_t f, uint32_t n), (cb, p, f, n)) \ @@ -119,6 +120,7 @@ REC_CMDS(REC_DECL) #define vkCmdBindPipeline rc_vkCmdBindPipeline #define vkCmdBindVertexBuffers2 rc_vkCmdBindVertexBuffers2 #define vkCmdBlitImage rc_vkCmdBlitImage +#define vkCmdExecuteCommands rc_vkCmdExecuteCommands #define vkCmdClearAttachments rc_vkCmdClearAttachments #define vkCmdClearColorImage rc_vkCmdClearColorImage #define vkCmdClearDepthStencilImage rc_vkCmdClearDepthStencilImage diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 46abe06f..0a63a3bb 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1679,15 +1679,8 @@ static VkSampler make_sampler(VkCtx *c, const samp_par *d) { si.maxLod = d->lod_max > 1000 ? VK_LOD_CLAMP_NONE : d->lod_max; if (mf <= NVN_MIN_FILTER_LINEAR) si.maxLod = si.minLod + 0.25f; // not mipmapped: stay on the base level if (si.maxLod < si.minLod) si.maxLod = si.minLod; - float lim = c->props.limits.maxSamplerLodBias, bias = d->lod_bias; - // DLSS upscaling (resolution.c): the scene's material textures, the filtered mipmapped ones that repeat, are - // sampled for the output's resolution; clamped ones (post effects, LUTs, the UI) keep the game's bias. Without DLSS - // running (dlaa = 0, or NGX without DLSS) nothing accumulates the extra detail, and the sharper mips only shimmer. - if (vk_dlaa_available() && !d->cmp_mode && mf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR && - (si.addressModeU == VK_SAMPLER_ADDRESS_MODE_REPEAT || si.addressModeU == VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT || - si.addressModeV == VK_SAMPLER_ADDRESS_MODE_REPEAT || si.addressModeV == VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT)) - bias += res_dlss_lod_bias(); - si.mipLodBias = fmaxf(-lim, fminf(lim, bias)); + float lim = c->props.limits.maxSamplerLodBias; + si.mipLodBias = fmaxf(-lim, fminf(lim, d->lod_bias)); // TOTK_ANISO raises the level for filtered, mipmapped colour samplers that repeat on U and V: the surface textures, // which tile, so ground and walls seen at a shallow angle stay sharp into the distance. Depth-compare (shadow) // samplers, nearest or unmipmapped ones (data maps, LUTs) and clamped ones keep the game's level: clamped samplers @@ -1755,11 +1748,32 @@ void vk_sampler_register(int id, const samp_par *s) { unlock(); } +// DLSS upscaling (vk_sampler_dlss_bias): the scene's material textures, the filtered mipmapped ones that repeat, are +// sampled for the output's resolution; clamped ones (post effects, LUTs, the UI) keep the game's bias. +static float dlss_bias; + +static int dlss_biased(const samp_par *k) { + int rep_s = k->ws == NVN_WRAP_MODE_REPEAT || k->ws == NVN_WRAP_MODE_MIRRORED_REPEAT; + int rep_t = k->wt == NVN_WRAP_MODE_REPEAT || k->wt == NVN_WRAP_MODE_MIRRORED_REPEAT; + return !k->cmp_mode && k->minf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR && (rep_s || rep_t); +} + +void vk_sampler_dlss_bias(float b) { + lock(); + if (b != dlss_bias) { + dlss_bias = b; + for (int i = 0; i < nsslots; i++) sslots[i].s = sslots[i].sp = VK_NULL_HANDLE; // looked up again (the key moves) + samp_gen++; + } + unlock(); +} + // Caller holds the lock. static VkSampler sampler_for(VkCtx *c, const samp_par *s) { samp_par k; memset(&k, 0, sizeof k); k = *s; + if (dlss_bias != 0 && dlss_biased(&k)) k.lod_bias += dlss_bias; // part of the key: samplers live until shutdown VkSampler v = (VkSampler)nvn_hm_get(&samplers, &k); if (!v) { v = make_sampler(c, &k); diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index ecac9ddd..13a353cd 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -34,6 +34,8 @@ typedef struct Arena { VkBuffer buf; VmaAllocation alloc; uint8_t *map; + uint8_t *cpu; // the same bytes in host memory, for the CPU's reads (vk_usnap_bytes): map is often device memory + // through the BAR, written fast but read back at about a microsecond a cache line uint64_t size; } Arena; @@ -96,6 +98,7 @@ static Arena *arena_get(uint64_t need) { return NULL; } a->map = info.pMappedData; + if (!(a->cpu = malloc(a->size))) rt_fatal("vk_usnap: out of memory"); rt_log("vk_usnap: new arena of %.1f MiB", a->size / 1048576.0); return a; } @@ -121,9 +124,35 @@ static void collect(const op_hdr *h, int depth) { if (h->op == OP_BIND_UBO) { const p_bind_buf *b = a; if (b->addr && b->size) { - if (U.ncol == U.ccol) U.col = grow(U.col, &U.ccol, sizeof *U.col); uint64_t n = b->size < MAX_RANGE ? b->size : MAX_RANGE; - U.col[U.ncol++] = (Rng){b->addr & ~(uint64_t)(ALIGN - 1), b->addr + n}; + // the vertex stage's object and bone blocks: also the game's other copy on either side, which still + // holds last frame's (vk_mvprev.c) + uint64_t step = vk_mvprev_step(b->stage, b->index, b->size); + // which neighbours lie in an imported pool, per block and vk_mem_gen: the same blocks are bound by + // many draws, and two resolves for each binding were a quarter of the snapshot's time + static struct { uint64_t addr, size, gen; int ok; } nc[1024]; + uint32_t ci = (uint32_t)((b->addr ^ n << 48) * 0x9E3779B97F4A7C15ull >> 54); + uint64_t gen = vk_mem_gen(); + int known = step && nc[ci].addr == b->addr && nc[ci].size == n && nc[ci].gen == gen, ok = 0; + for (int k = step ? -1 : 0; k <= (step ? 1 : 0); k++) { + uint64_t at = b->addr + (uint64_t)((int64_t)k * (int64_t)step); + if (k < 0 && b->addr < step) continue; + // a neighbour past its pool would make the whole merged region fail the pool check below, and + // the bindings in it read live memory again + VkMemSpan sp; + uint64_t lo = at & ~(uint64_t)(ALIGN - 1); + if (k && known) { + if (!(nc[ci].ok >> (k + 1) & 1)) continue; + } else if (k) { + if (!vk_mem_resolve(lo, at + n - lo, &sp) || sp.alloc || !sp.pool || sp.avail < at + n - lo || + (sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) + continue; + ok |= 1 << (k + 1); + } + if (U.ncol == U.ccol) U.col = grow(U.col, &U.ccol, sizeof *U.col); + U.col[U.ncol++] = (Rng){at & ~(uint64_t)(ALIGN - 1), at + n}; + } + if (step && !known) nc[ci].addr = b->addr, nc[ci].size = n, nc[ci].gen = gen, nc[ci].ok = ok; } } h = nvn_op_next(h); @@ -255,7 +284,8 @@ void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) { if (!U.nreg) return; if (!(U.arena = arena_get(total))) return; for (int i = 0; i < U.nreg; i++) - memcpy(U.arena->map + U.reg[i].off, (const void *)(uintptr_t)U.reg[i].lo, U.reg[i].hi - U.reg[i].lo); + memcpy(U.arena->cpu + U.reg[i].off, (const void *)(uintptr_t)U.reg[i].lo, U.reg[i].hi - U.reg[i].lo); + memcpy(U.arena->map, U.arena->cpu, total); // one sequential write into the device copy vmaFlushAllocation(vk_ctx()->vma, U.arena->alloc, 0, total); // a no-op on coherent memory U.on = 1; U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += total, U.copy_ns += kern_now_ns() - t0; @@ -290,12 +320,6 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI return 1; } -const void *vk_usnap_host(uint64_t addr, uint64_t n, uint64_t align) { - VkDescriptorBufferInfo b; - if (!vk_usnap_find(addr, n, align, &b)) return NULL; - return U.arena->map + b.offset; -} - void vk_usnap_drop(uint64_t addr, uint64_t n) { if (!U.mode || !n) return; if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr); @@ -307,10 +331,17 @@ void vk_usnap_drop(uint64_t addr, uint64_t n) { uint64_t vk_usnap_gen(void) { return U.gen; } +const uint8_t *vk_usnap_map(void) { return U.arena ? U.arena->cpu : NULL; } + +const uint8_t *vk_usnap_bytes(uint64_t addr, uint64_t n) { + VkDescriptorBufferInfo bi; + return vk_usnap_find(addr, n, 1, &bi) ? U.arena->cpu + bi.offset : NULL; +} + // The arenas back on the free list (vk_exec_draw_shutdown: the device is idle, every completion ran). void vk_usnap_shutdown(void) { plat_mutex_lock(&free_lock); - for (Arena *a = free_arenas, *next; a; a = next) next = a->next, vmaDestroyBuffer(vk_ctx()->vma, a->buf, a->alloc), free(a); + for (Arena *a = free_arenas, *next; a; a = next) next = a->next, vmaDestroyBuffer(vk_ctx()->vma, a->buf, a->alloc), free(a->cpu), free(a); free_arenas = NULL; plat_mutex_unlock(&free_lock); } diff --git a/tools/build_xlate.py b/tools/build_xlate.py index 4d8c47cb..210c1acf 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 14: - sys.exit(f"expected 14 patches, found {len(patches)}") + if len(patches) != 15: + sys.exit(f"expected 15 patches, found {len(patches)}") for patch in patches: # git apply works outside a repository when given a directory; --check first so a failure names the patch. for extra in (["--check"], []): diff --git a/tools/make_dlaa_mask.py b/tools/make_dlaa_mask.py new file mode 100644 index 00000000..8dfa00d0 --- /dev/null +++ b/tools/make_dlaa_mask.py @@ -0,0 +1,203 @@ +"""Writes runtime/dlaa_mask_spv.h: the SPIR-V of DLSS's reactive mask pass (vk_dlaa.c, mask_pass), assembled here like +tools/make_mvobj_fs.py (no GLSL compiler on the machine this was written on). + +The shader, as GLSL: + + #version 450 + layout(local_size_x = 8, local_size_y = 8) in; + layout(binding = 0) uniform sampler2D before; // the light target when the transparents start + layout(binding = 1) uniform sampler2D after; // the light target when DLSS runs + layout(binding = 2, r8) uniform writeonly image2D mask; + layout(push_constant) uniform P { vec4 p; }; // render size xy, threshold z, gain w + void main() { + vec2 g = vec2(gl_GlobalInvocationID.xy); + if (g.x < p.x && g.y < p.y) { + ivec2 c = ivec2(gl_GlobalInvocationID.xy); + vec3 b = texelFetch(before, c, 0).rgb, a = texelFetch(after, c, 0).rgb; + vec3 d = abs(a - b); + float dm = max(max(d.r, d.g), d.b); + float m = max(max(max(a.r, a.g), a.b), max(max(b.r, b.g), b.b)); + float r = dm / max(m, 1e-4); + imageStore(mask, c, vec4(clamp((r - p.z) * p.w, 0.0, 1.0))); + } + } + +The mask is the share of a pixel's colour the transparents changed (particles, smoke, sparks: what has no motion +vectors of its own), 0 where nothing was drawn over the solid scene. + + python tools/make_dlaa_mask.py +""" +import struct +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +OUT = ROOT / 'runtime' / 'dlaa_mask_spv.h' + +SECTIONS = ('cap', 'ext', 'mem', 'entry', 'exec', 'dec', 'types', 'func') + + +class Asm: + def __init__(self): + self.next = 1 + self.sections = {k: [] for k in SECTIONS} + + def id(self): + self.next += 1 + return self.next - 1 + + def op(self, sec, code, *words): + self.sections[sec].append([(len(words) + 1) << 16 | code, *words]) + + def words(self): + out = [0x07230203, 0x00010000, 0, self.next, 0] + for sec in SECTIONS: + for ins in self.sections[sec]: + out.extend(ins) + return out + + +def string(s): + b = s.encode() + b'\0' + b += b'\0' * (-len(b) % 4) + return list(struct.unpack('<%dI' % (len(b) // 4), b)) + + +def f32(x): + return struct.unpack('\n' + 'static const uint32_t dlaa_mask_spv[] = {\n ' + ',\n '.join(lines) + '\n};\n', newline='\n') +print(f'{OUT}: {len(w)} words') diff --git a/tools/make_mvobj_fs.py b/tools/make_mvobj_fs.py new file mode 100644 index 00000000..58f5e3c9 --- /dev/null +++ b/tools/make_mvobj_fs.py @@ -0,0 +1,163 @@ +"""Writes runtime/vk_mvobj_fs.h: the SPIR-V of the per-object motion fragment shader (vk_mvobj.c), assembled here +because it must be patched at run time (the Location of its input follows the vertex shader variant) and is small. + +The shader, as GLSL: + + #version 450 + layout(location = 0) in vec4 prev; // last frame's clip position (patch 0015); Location patched + layout(set = 0, binding = 1, std140) uniform P { vec4 p; }; // render size xy, this frame's jitter zw (pixels) + layout(location = 0) out vec4 o; + void main() { + vec2 win = gl_FragCoord.xy - p.zw; // this frame's window position, without the jitter + vec2 pn = prev.xy / prev.w; + vec2 pwin = vec2((pn.x * 0.5 + 0.5) * p.x, (0.5 - pn.y * 0.5) * p.y); + o = prev.w > 1e-6 ? vec4(pwin - win, 0.0, 0.0) : vec4(0.0); + } + +The vector is last frame's window position minus this frame's, in render pixels, as runtime/shaders/dlaa_mv.comp +writes it. Its descriptor set and binding are renumbered by vk_pipeline.c like a translated shader's (the .res.json +text in vk_mvobj.c names binding 1 as slot 3). + + python tools/make_mvobj_fs.py +""" +import struct +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +OUT = ROOT / 'runtime' / 'vk_mvobj_fs.h' + + +class Asm: + def __init__(self): + self.next = 1 + self.sections = {k: [] for k in ('cap', 'mem', 'entry', 'exec', 'dec', 'types', 'func')} + self.loc_word = None # index of the patched Location literal in the final words + + def id(self): + self.next += 1 + return self.next - 1 + + def op(self, sec, code, *words): + self.sections[sec].append([(len(words) + 1) << 16 | code, *words]) + return self.sections[sec][-1] + + def words(self): + out = [0x07230203, 0x00010000, 0, self.next, 0] + for sec in ('cap', 'mem', 'entry', 'exec', 'dec', 'types', 'func'): + for ins in self.sections[sec]: + if ins is self.loc_ins: + self.loc_word = len(out) + 3 + out.extend(ins) + return out + + +def string(s): + b = s.encode() + b'\0' + b += b'\0' * (-len(b) % 4) + return list(struct.unpack('<%dI' % (len(b) // 4), b)) + + +def f32(x): + return struct.unpack(' _job["sbuf_align"] is JsonNode n ? (int)n : 16; // "subgroup": n: lanes per subgroup when the host cannot pin 32 (AMD GCN/Vega run 64): ballots and votes span n lanes. public int QueryHostSubgroupSize() => _job["subgroup"] is JsonNode n ? (int)n : 32; + // "mvprev": true: the vertex shader also writes last frame's position, computed from slots bank + 15 in place of + // the user banks c3-c16 (patch 0015, runtime/vk_mvprev.c); res.json's "mvprev" names the output location (-1: + // none) and "mvprev_banks" the banks the position depends on (bit n: cn). + public bool QueryMotionPrevious() => _job["mvprev"] is JsonValue v && v.TryGetValue(out bool b) && b; + public int MotionPreviousLocation = -1, MotionPreviousBanks; + public void ReportMotionPrevious(int location, int banks) => (MotionPreviousLocation, MotionPreviousBanks) = (location, banks); } @@ -140,6 +146,7 @@ static class SwshXlate ["uses_frag_coord"] = p.Info.UsesFragCoord, ["frag_out_map"] = p.Info.FragmentOutputMap, ["tex_queried"] = new JsonArray(acc.TexQueried.Select(x => (JsonNode)x).ToArray()), ["img_queried"] = new JsonArray(acc.ImgQueried.Select(x => (JsonNode)x).ToArray()), }; + if (acc.QueryMotionPrevious()) (res["mvprev"], res["mvprev_banks"]) = (acc.MotionPreviousLocation, acc.MotionPreviousBanks); foreach (var b in p.Info.CBuffers) res["cbufs"].AsArray().Add((JsonNode)new JsonObject { ["binding"] = b.Binding, ["slot"] = b.Slot }); foreach (var b in p.Info.SBuffers) res["sbufs"].AsArray().Add((JsonNode)new JsonObject { ["binding"] = b.Binding, ["slot"] = b.Slot, ["cb_slot"] = b.SbCbSlot, ["cb_offset"] = b.SbCbOffset, ["flags"] = (int)b.Flags }); foreach (var (list, name) in new[] { (p.Info.Textures, "textures"), (p.Info.Images, "images") }) diff --git a/tools/xlate/patches/0015-motion-previous-position.patch b/tools/xlate/patches/0015-motion-previous-position.patch new file mode 100644 index 00000000..9e915daa --- /dev/null +++ b/tools/xlate/patches/0015-motion-previous-position.patch @@ -0,0 +1,376 @@ +--- a/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs ++++ b/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs +@@ -184,6 +184,14 @@ namespace Ryujinx.Graphics.Shader + return false; + } + ++ // TOTK: true when a vertex shader should also write last frame's position (Transforms/MotionPrevious.cs), ++ // the user output location it went to (-1: none) and the banks the position depends on (bit n: cn). ++ bool QueryMotionPrevious() => false; ++ ++ void ReportMotionPrevious(int location, int banks) ++ { ++ } ++ + /// + /// Queries host about the presence of the FrontFacing built-in variable bug. + /// +--- a/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs ++++ b/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs +@@ -65,7 +65,7 @@ namespace Ryujinx.Graphics.Shader.Translation + _stage = stage; + _stagePrefix = GetShaderStagePrefix(stage); + +- _cbSlotToBindingMap = new SetBindingPair[18]; ++ _cbSlotToBindingMap = new SetBindingPair[32]; // TOTK: slots 18-31 for Transforms/MotionPrevious.cs + _sbSlotToBindingMap = new SetBindingPair[16]; + _cbSlotToBindingMap.AsSpan().Fill(new(-1, -1)); + _sbSlotToBindingMap.AsSpan().Fill(new(-1, -1)); +--- /dev/null ++++ b/src/Ryujinx.Graphics.Shader/Translation/Transforms/MotionPrevious.cs +@@ -0,0 +1,330 @@ ++using Ryujinx.Graphics.Shader.IntermediateRepresentation; ++using System.Collections.Generic; ++using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper; ++ ++namespace Ryujinx.Graphics.Shader.Translation.Transforms ++{ ++ // TOTK: last frame's position for per-object motion vectors (runtime/vk_exec_draw.c, vk_mvprev.c). A vertex shader's ++ // position depends on uniform banks of this frame: the camera (c4), the object's place (c7), its bones (c5), the ++ // clock that moves wind and water (c14), and so on. Every operation the position depends on that reads one of the ++ // user banks c3..c16, directly or through other values, is computed a second time beside the original, reading slot ++ // bank + 15 (18..31) instead, where the runtime binds that bank's block of last frame; the position this gives is ++ // written to a free user output. Control flow stays the original's. Operations that cannot be repeated (stores, ++ // texture operations) keep their original result. The runtime gets the output location and the banks the position ++ // depends on (ReportMotionPrevious): an object with none of them changed since last frame held still. ++ static class MotionPrevious ++ { ++ public const int SlotOffset = 15; ++ ++ private static int Remap(int slot) => slot >= 3 && slot <= 16 ? slot + SlotOffset : -1; ++ ++ // The output location written (-1: none free, or no position store), and in banks the mask of the banks (bit ++ // n: cn) the position depends on. ++ public static int Run(BasicBlock[] blocks, ResourceManager resourceManager, AttributeUsage attributeUsage, out int banks) ++ { ++ banks = 0; ++ int location = -1; ++ ++ for (int l = Constants.MaxAttributes - 1; l >= 0; l--) ++ { ++ if ((attributeUsage.UsedOutputAttributes & (1 << l)) == 0) ++ { ++ location = l; ++ break; ++ } ++ } ++ ++ if (location < 0) ++ { ++ return -1; ++ } ++ ++ // 1. What the position depends on: back from the values the position stores write, through every ++ // operation and phi that defines them. ++ HashSet slice = new(); ++ Stack work = new(); ++ bool stores = false; ++ ++ foreach (BasicBlock block in blocks) ++ { ++ for (LinkedListNode node = block.Operations.First; node != null; node = node.Next) ++ { ++ if (IsPositionStore(node.Value)) ++ { ++ stores = true; ++ work.Push(((Operation)node.Value).GetSource(2)); ++ } ++ } ++ } ++ ++ if (!stores) ++ { ++ return -1; ++ } ++ ++ while (work.Count != 0) ++ { ++ Operand v = work.Pop(); ++ ++ if (v == null || v.Type != OperandType.LocalVariable || v.AsgOp is not INode def || !slice.Add(def)) ++ { ++ continue; ++ } ++ ++ for (int s = 0; s < def.SourcesCount; s++) ++ { ++ work.Push(def.GetSource(s)); ++ } ++ } ++ ++ // 2. The banks the slice reads, and the bindings of those it loads with a computed offset to their ++ // last-frame slot's (only these get one: a binding is declared in the SPIR-V whether used or not, and the ++ // host's layout comes from the used ones). Direct operands (c[slot][offset]) get theirs when the ++ // structured program is made. ++ Dictionary bindings = new(); ++ ++ foreach (INode n in slice) ++ { ++ if (n is Operation op && IsBankLoad(op, resourceManager, out int slot)) ++ { ++ banks |= 1 << slot; ++ ++ if (!bindings.ContainsKey(op.GetSource(0).Value)) ++ { ++ bindings[op.GetSource(0).Value] = resourceManager.GetConstantBufferBinding(Remap(slot)); ++ } ++ } ++ ++ for (int s = 0; s < n.SourcesCount; s++) ++ { ++ Operand src = n.GetSource(s); ++ ++ if (src != null && src.Type == OperandType.ConstantBuffer && Remap(src.GetCbufSlot()) >= 0) ++ { ++ banks |= 1 << src.GetCbufSlot(); ++ } ++ } ++ } ++ ++ if (banks == 0) ++ { ++ return -1; ++ } ++ ++ // 3. Which of the slice's values depend on those banks, to a fixed point (phis carry it around loops). ++ HashSet tainted = new(); ++ bool changed = true; ++ ++ while (changed) ++ { ++ changed = false; ++ ++ foreach (BasicBlock block in blocks) ++ { ++ for (LinkedListNode node = block.Operations.First; node != null; node = node.Next) ++ { ++ INode n = node.Value; ++ ++ if (!slice.Contains(n) || n.DestsCount == 0 || !Reads(n, tainted, bindings)) ++ { ++ continue; ++ } ++ ++ for (int d = 0; d < n.DestsCount; d++) ++ { ++ Operand dest = n.GetDest(d); ++ ++ if (dest != null && dest.Type == OperandType.LocalVariable && tainted.Add(dest)) ++ { ++ changed = true; ++ } ++ } ++ } ++ } ++ } ++ ++ // 4. A copy of each such operation beside it, with new destinations. ++ Dictionary prev = new(); ++ List<(INode Original, INode Copy)> copies = new(); ++ ++ foreach (BasicBlock block in blocks) ++ { ++ for (LinkedListNode node = block.Operations.First; node != null; node = node.Next) ++ { ++ INode n = node.Value; ++ ++ if (!slice.Contains(n) || n.DestsCount == 0 || !IsTainted(n, tainted)) ++ { ++ continue; ++ } ++ ++ INode copy = null; ++ ++ if (n is PhiNode phi) ++ { ++ Operand dest = Local(); ++ prev[phi.Dest] = dest; ++ copy = new PhiNode(dest); ++ } ++ else if (n is Operation op && op.GetType() == typeof(Operation) && op.Inst != Instruction.Store) ++ { ++ Operand[] dests = new Operand[op.DestsCount]; ++ ++ for (int d = 0; d < dests.Length; d++) ++ { ++ Operand dest = op.GetDest(d); ++ dests[d] = dest != null && dest.Type == OperandType.LocalVariable ? Local() : dest; ++ ++ if (dest != null && dest.Type == OperandType.LocalVariable) ++ { ++ prev[dest] = dests[d]; ++ } ++ } ++ ++ Operand[] sources = new Operand[op.SourcesCount]; ++ ++ for (int s = 0; s < sources.Length; s++) ++ { ++ sources[s] = op.GetSource(s); ++ } ++ ++ Operation c = op.StorageKind != StorageKind.None && dests.Length == 1 ++ ? new Operation(op.Inst, op.StorageKind, dests[0], sources) ++ : new Operation(op.Inst, op.Index, dests, sources); ++ c.ForcePrecise = op.ForcePrecise; ++ copy = c; ++ } ++ ++ if (copy != null) ++ { ++ node = block.Operations.AddAfter(node, copy); ++ copies.Add((n, copy)); ++ } ++ } ++ } ++ ++ // 5. The copies' sources: last frame's values, the remapped banks. ++ foreach ((INode original, INode copy) in copies) ++ { ++ if (copy is PhiNode phiCopy) ++ { ++ PhiNode phi = (PhiNode)original; ++ ++ for (int s = 0; s < phi.SourcesCount; s++) ++ { ++ phiCopy.AddSource(phi.GetBlock(s), Prev(phi.GetSource(s), prev)); ++ } ++ ++ continue; ++ } ++ ++ Operation op = (Operation)original, c = (Operation)copy; ++ ++ for (int s = 0; s < op.SourcesCount; s++) ++ { ++ Operand src = op.GetSource(s); ++ ++ if (s == 0 && op.Inst == Instruction.Load && op.StorageKind == StorageKind.ConstantBuffer && ++ src.Type == OperandType.Constant && bindings.TryGetValue(src.Value, out int binding)) ++ { ++ c.SetSource(s, Const(binding)); ++ } ++ else if (src.Type == OperandType.ConstantBuffer && Remap(src.GetCbufSlot()) >= 0) ++ { ++ c.SetSource(s, Cbuf(Remap(src.GetCbufSlot()), src.GetCbufOffset())); ++ } ++ else ++ { ++ c.SetSource(s, Prev(src, prev)); ++ } ++ } ++ } ++ ++ // 6. Each position store also writes last frame's position to the output. ++ foreach (BasicBlock block in blocks) ++ { ++ for (LinkedListNode node = block.Operations.First; node != null; node = node.Next) ++ { ++ if (!IsPositionStore(node.Value)) ++ { ++ continue; ++ } ++ ++ Operation op = (Operation)node.Value; ++ Operation store = new(Instruction.Store, StorageKind.Output, null, ++ Const((int)IoVariable.UserDefined), Const(location), op.GetSource(1), Prev(op.GetSource(2), prev)); ++ node = block.Operations.AddAfter(node, store); ++ } ++ } ++ ++ attributeUsage.SetOutputUserAttribute(location); ++ ++ return location; ++ } ++ ++ private static bool IsPositionStore(INode n) ++ { ++ return n is Operation op && op.Inst == Instruction.Store && op.StorageKind == StorageKind.Output && ++ op.SourcesCount == 3 && op.GetSource(0).Type == OperandType.Constant && ++ (IoVariable)op.GetSource(0).Value == IoVariable.Position; ++ } ++ ++ private static bool IsBankLoad(Operation op, ResourceManager resourceManager, out int slot) ++ { ++ slot = -1; ++ ++ return op.Inst == Instruction.Load && op.StorageKind == StorageKind.ConstantBuffer && ++ op.GetSource(0).Type == OperandType.Constant && ++ resourceManager.TryGetConstantBufferSlot(op.GetSource(0).Value, out slot) && Remap(slot) >= 0; ++ } ++ ++ private static Operand Prev(Operand src, Dictionary prev) ++ { ++ return src != null && prev.TryGetValue(src, out Operand p) ? p : src; ++ } ++ ++ private static bool IsTainted(INode n, HashSet tainted) ++ { ++ for (int d = 0; d < n.DestsCount; d++) ++ { ++ if (n.GetDest(d) is Operand dest && tainted.Contains(dest)) ++ { ++ return true; ++ } ++ } ++ ++ return false; ++ } ++ ++ private static bool Reads(INode n, HashSet tainted, Dictionary bindings) ++ { ++ if (n is Operation op && op.Inst == Instruction.Load && op.StorageKind == StorageKind.ConstantBuffer && ++ op.GetSource(0).Type == OperandType.Constant && bindings.ContainsKey(op.GetSource(0).Value)) ++ { ++ return true; ++ } ++ ++ for (int s = 0; s < n.SourcesCount; s++) ++ { ++ Operand src = n.GetSource(s); ++ ++ if (src == null) ++ { ++ continue; ++ } ++ ++ if (src.Type == OperandType.ConstantBuffer && Remap(src.GetCbufSlot()) >= 0) ++ { ++ return true; ++ } ++ ++ if (tainted.Contains(src)) ++ { ++ return true; ++ } ++ } ++ ++ return false; ++ } ++ } ++} +--- a/src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs ++++ b/src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs +@@ -301,6 +301,12 @@ namespace Ryujinx.Graphics.Shader.Translation + + Optimizer.RunPass(context); + TransformPasses.RunPass(context); ++ ++ if (i == 0 && Definitions.Stage == ShaderStage.Vertex && GpuAccessor.QueryMotionPrevious()) ++ { ++ int location = MotionPrevious.Run(cfg.Blocks, resourceManager, AttributeUsage, out int banks); ++ GpuAccessor.ReportMotionPrevious(location, banks); ++ } + } + + funcs[i] = new Function(cfg.Blocks, $"fun{i}", false, inArgumentsCount, outArgumentsCount); diff --git a/totk.ini.example b/totk.ini.example index 9067ebb1..07678367 100644 --- a/totk.ini.example +++ b/totk.ini.example @@ -199,11 +199,17 @@ # NVIDIA DLSS / DLAA for the 3D scene, on NVIDIA RTX GPUs with nvngx_dlss.dll (from NVIDIA's DLSS SDK) next to # totk.exe; elsewhere the setting does nothing. The HUD is drawn after it and is not touched. Read at start. # dlaa = 0 -# What DLSS works from: dlaa (the 3D scene at full size), quality, balanced or performance (the scene at 67, 58 or -# 50 % of the output, scaled up by DLSS; these replace render_scale and sharpen texture filtering to match). They apply -# only with dlaa = 1 and nvngx_dlss.dll next to totk.exe, on Windows; otherwise the scene follows render_scale. Read at -# start. +# What DLSS works from: dlaa (the 3D scene at full size), quality, balanced, performance or ultra_performance (the +# scene at 67, 58, 50 or 33 % of the output, scaled up by DLSS; these replace render_scale and sharpen texture filtering +# to match). They apply only with dlaa = 1 and nvngx_dlss.dll next to totk.exe, on Windows; otherwise the scene follows +# render_scale. Changes while the game runs take effect at once. The DLSS model follows NVIDIA's advice per mode +# (preset K for dlaa, quality and balanced, M for performance, L for ultra_performance); on slower RTX cards the older +# model E is cheaper: TOTK_DLSS_PRESET=E in the environment. # dlss = dlaa +# Per-object motion vectors for DLSS: characters, animals and moving objects get their own motion instead of the +# camera's, so DLSS keeps their history instead of smearing or rejecting it. Costs some CPU time each frame (about 1-2 ms +# on a laptop CPU); 0 leaves camera-only motion. Read at start. +# dlaa_objmv = 1 # The game's own FXAA after the post effects; with DLAA on it is not needed. Changes while the game runs take effect at # once. # fxaa = 1