mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-12 18:31:05 +00:00
Valiantsin DzerakhandClaude Opus 5.5
c33af6020b
AMD's RDNA2 driver places the depth compression metadata after the depth data. Above 1080p that tail of a depth target bound into the game's texture heap lay over the next texture, and every depth write scrambled it: a garbage band at the top of the screen, a pink glow, blocks in the grass and a blue checkered scene in the menu (RX 6600 to 6800, issue #185). Reproduced at 2560x1440 on an RDNA2 iGPU with driver 26.10.1; all 63 oversize images there were D16 or D32 depth targets. Colour images keep their place in the pool. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1741 lines
85 KiB
C
1741 lines
85 KiB
C
// Vulkan textures of the graphics layer: a VkImage cache keyed by storage
|
|
// GPU address plus descriptor, uploads through staging buffers with block-linear deswizzle and ASTC decode (vk_astc.c),
|
|
// re-uploads when page write generations (vk_memory.c) show the storage written, ReadTexels / WriteTexels, the texture
|
|
// and sampler descriptor slots, and VkSamplers.
|
|
//
|
|
// Behaviour ported from MacNativeGames games/totk/third_party/d2r_gfx/gfx/nvn @ b75be8cf (MIT, Copyright (c) 2026
|
|
// jhaizhou; origin d2r-mobile): which textures mirror their pool memory (nvn_mtl.m tex_mirrored, 3347-3362), the
|
|
// "GPU-owned" rule that stops re-uploads once the GPU rendered into a mirrored texture (nvn_mtl.m tex_mtl, 3366-3372),
|
|
// the live texture at a storage address that a slot of a finalized texture follows (hle_nvn.c:373-393), and
|
|
// ReadTexels / WriteTexels (nvn_mtl.m:10105-10195). Image creation, upload regions and samplers are ported from LAS
|
|
// Recomp runtime/nvn/vulkan/textures.c:235-330, 778-844 @ da35b83a (MIT, Copyright (c) 2026 Peiikai).
|
|
//
|
|
// Every image lives in VK_IMAGE_LAYOUT_GENERAL. Uploads are recorded into the upload batch of vk_memory.c; the replay
|
|
// submits it (vk_upload_flush) before a submission that samples the textures.
|
|
#include <math.h>
|
|
#include <stdarg.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <windows.h>
|
|
|
|
#include "cfg.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "nvn_api.h"
|
|
#include "nvn_hmap.h"
|
|
#include "nvn_tex.h"
|
|
#include "tex.h"
|
|
#include "vk.h"
|
|
#include "vk_exec.h"
|
|
|
|
typedef struct {
|
|
uint64_t addr;
|
|
nvn_texdesc d;
|
|
uint8_t has_layout, layout[8], pad[7];
|
|
} TexKey;
|
|
|
|
typedef struct Tex {
|
|
VkTexImage pub; // first: vk_texture_get hands out &pub
|
|
TexKey key;
|
|
tex_t t; // the newest texture object seen with this key
|
|
TexVkFormat f;
|
|
VmaAllocation alloc; // own memory; NULL when bound into pool memory
|
|
VmaAllocation bound_to; // the pool memory it is bound into
|
|
uint32_t wgen; // page generation of the last upload check
|
|
uint32_t checked; // submit_epoch of the last check from a draw binding
|
|
int uploaded, released;
|
|
uint32_t n_up; // uploads of this entry (vk_texture_debug)
|
|
int last_ch; // last vk_mem_changed result of make_fresh: 1 written, 0 clean, -1 not tracked
|
|
struct Tex *next_at; // other entries at the same storage address
|
|
uint8_t gpu_dirty; // mirrored, and the GPU wrote the image after the last write-back to the storage
|
|
uint32_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded
|
|
uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes)
|
|
uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one)
|
|
uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear,
|
|
// storage binding); ReadTexels waits for it instead of draining the replay queue
|
|
uint8_t listed; // in owned[]
|
|
uint32_t wt_n; // WriteTexels into the storage since the last upload
|
|
struct Tex *rel_prev, *rel_next; // while released: the list of released entries, oldest first
|
|
} Tex;
|
|
|
|
// A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the
|
|
// entry (ce), valid while the generation of its address (agen) is still cgen, the view and has_view handed out with it,
|
|
// and the submit_epoch of the last freshness check through the slot (cepoch). Hot fields first: a lookup that hits
|
|
// touches one or two cache lines.
|
|
typedef struct {
|
|
struct Tex *ce; // NULL: nothing cached (registered since, or never resolved)
|
|
uint32_t cgen; // agen of the slot's address when ce was resolved
|
|
uint32_t cepoch;
|
|
view_t cview;
|
|
uint8_t chas_view, used, has_view;
|
|
view_t v;
|
|
tex_t t;
|
|
} TexSlot;
|
|
typedef struct { samp_par p; VkSampler s, sp; uint8_t used; } SampSlot; // s: vk_sampler_for(&p) once looked up; sp:
|
|
// its point-sampled form (vk_sampler_point_by_id_locked)
|
|
|
|
static CRITICAL_SECTION tex_lock;
|
|
static INIT_ONCE tex_once = INIT_ONCE_STATIC_INIT;
|
|
static nvn_hmap by_key = {.klen = sizeof(TexKey)};
|
|
static nvn_hmap by_addr = {.klen = 8};
|
|
static TexSlot *tslots;
|
|
static int ntslots;
|
|
static SampSlot *sslots;
|
|
static int nsslots;
|
|
static nvn_hmap samplers = {.klen = sizeof(samp_par)};
|
|
static int nreleased;
|
|
static Tex *rel_head, *rel_tail; // released entries in release order (sweep_released drops from the head)
|
|
static uint64_t n_uploads, n_reuploads;
|
|
// Re-uploads since the last perf line (tex_lock): all, and those of textures a WriteTexels wrote since their last upload.
|
|
static uint64_t ru_n, ru_bytes, ru_ns, ru_wt_n, ru_wt_bytes, ru_wt_ns;
|
|
// TOTK_TEX_ALIAS (default on; =0: a new image over live storage starts empty): a new descriptor over the storage of a
|
|
// live image that the GPU wrote gets that image's content (alias_copy). Counts since the last log line.
|
|
static int tex_alias = -1;
|
|
static uint64_t n_alias_copied, n_alias_upload, n_alias_uninit;
|
|
static uint32_t submit_epoch = 1; // bumped per queue submit: draw bindings check a texture's storage once per submit
|
|
// Mirrored textures the GPU wrote into, sorted by address (vk_texture_cpu_stale, vk_texture_write_back). Changed under
|
|
// tex_lock and owned_lock; read under either.
|
|
static SRWLOCK owned_lock = SRWLOCK_INIT;
|
|
static Tex **owned;
|
|
static int nowned, cowned;
|
|
static uint64_t max_owned; // largest storage size in owned[]
|
|
static uint64_t gpu_writes; // GPU writes into mirrored textures so far: Tex.gpu_seq
|
|
// Bumped whenever the entry a slot resolves to can change: an entry created or destroyed, released or taken back into
|
|
// use (the slot lookup and the follow of a finalized slot read only by_key, by_addr and the released flags).
|
|
static uint32_t tex_gen = 1;
|
|
// Bumped with tex_gen and whenever a slot is registered: what vk_texture_by_id returns for any slot may have changed.
|
|
static uint32_t slots_gen = 1;
|
|
static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise
|
|
static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line
|
|
// tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so
|
|
// a slot's resolution stays valid while its address's generation does. TotK finalizes and initializes transient
|
|
// textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws,
|
|
// which emptied every cache keyed on the global generations.
|
|
#define AGEN_N 4096
|
|
static uint32_t agen[AGEN_N];
|
|
// Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered)
|
|
// and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are
|
|
// 20 bits), unlike tslots, which moves when it grows.
|
|
static uint64_t *sinfo;
|
|
static uint32_t reg_serial;
|
|
|
|
static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); }
|
|
|
|
static void tex_gen_bump(uint64_t addr) {
|
|
if (!++tex_gen) tex_gen = 1;
|
|
slots_gen++;
|
|
uint32_t *g = &agen[agen_idx(addr)];
|
|
__atomic_store_n(g, *g + 1, __ATOMIC_RELEASE);
|
|
}
|
|
|
|
static volatile long tex_ready;
|
|
static BOOL CALLBACK tex_init(PINIT_ONCE o, PVOID p, PVOID *ctx) {
|
|
(void)o, (void)p, (void)ctx;
|
|
InitializeCriticalSection(&tex_lock);
|
|
__atomic_store_n(&tex_ready, 1, __ATOMIC_RELEASE);
|
|
return TRUE;
|
|
}
|
|
static void lock(void) {
|
|
if (!__atomic_load_n(&tex_ready, __ATOMIC_ACQUIRE)) InitOnceExecuteOnce(&tex_once, tex_init, NULL, NULL);
|
|
EnterCriticalSection(&tex_lock);
|
|
}
|
|
static void unlock(void) { LeaveCriticalSection(&tex_lock); }
|
|
|
|
static uint32_t umax(uint32_t a, uint32_t b) { return a > b ? a : b; }
|
|
static uint32_t mip(uint32_t v, unsigned lv) { return umax(v >> lv, 1); }
|
|
|
|
static void make_key(const tex_t *t, TexKey *k) {
|
|
memset(k, 0, sizeof *k);
|
|
k->addr = (uintptr_t)t->mem;
|
|
k->d = t->d;
|
|
k->has_layout = t->has_layout;
|
|
memcpy(k->layout, t->layout, 8);
|
|
}
|
|
|
|
// Keys equal apart from the descriptor's swizzle selectors: the same storage, format and geometry.
|
|
static int same_but_swizzle(const TexKey *a, const TexKey *b) {
|
|
TexKey x = *a, y = *b;
|
|
x.d.swizzle_r = x.d.swizzle_g = x.d.swizzle_b = x.d.swizzle_a = 0;
|
|
y.d.swizzle_r = y.d.swizzle_g = y.d.swizzle_b = y.d.swizzle_a = 0;
|
|
return !memcmp(&x, &y, sizeof x);
|
|
}
|
|
|
|
// MNG nvn_mtl.m tex_mirrored: textures whose content the CPU provides in pool memory. Display, compressible, zcull,
|
|
// depth, multisample and GPU-only storage stays GPU-owned.
|
|
static int tex_mirrored(const tex_t *t) {
|
|
if (!t->mem || nvn_gpu_only((uintptr_t)t->mem)) return 0;
|
|
if (t->has_layout) return 1;
|
|
const pool_t *p = t->pool;
|
|
if (!p || !(p->flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) return 0;
|
|
if (t->d.flags & (NVN_TEXTURE_FLAGS_DISPLAY | NVN_TEXTURE_FLAGS_COMPRESSIBLE | NVN_TEXTURE_FLAGS_ADAPTIVE_ZCULL |
|
|
NVN_TEXTURE_FLAGS_DEPTH16_PREFER_FAST_CLEAR | NVN_TEXTURE_FLAGS_ZCULL_SUPPORT_STENCIL))
|
|
return 0;
|
|
if (t->d.samples > 1 || t->d.target == NVN_TEXTURE_TARGET_BUFFER) return 0;
|
|
if (t->d.format >= 0x31 && t->d.format <= 0x36) return 0; // stencil / depth formats
|
|
return 1;
|
|
}
|
|
|
|
static void log_limited(volatile long *n, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
|
|
static void log_limited(volatile long *n, const char *fmt, ...) {
|
|
if (InterlockedIncrement(n) > 16) return;
|
|
char buf[512];
|
|
va_list ap;
|
|
va_start(ap, fmt);
|
|
vsnprintf(buf, sizeof buf, fmt, ap);
|
|
va_end(ap);
|
|
rt_log("vk: %s", buf);
|
|
}
|
|
|
|
static void barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst,
|
|
VkAccessFlags2 da) {
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa,
|
|
.dstStageMask = dst, .dstAccessMask = da};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
|
vkCmdPipelineBarrier2(cb, &di);
|
|
}
|
|
#define ALL_RW VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT
|
|
|
|
// ---------------------------------------------------------------- images
|
|
static VkFormatFeatureFlags features_of(VkFormat f) {
|
|
VkFormatProperties p;
|
|
vkGetPhysicalDeviceFormatProperties(vk_ctx()->phys, f, &p);
|
|
return p.optimalTilingFeatures;
|
|
}
|
|
|
|
static VkImageAspectFlags aspect_of(const TexVkFormat *f) {
|
|
VkImageAspectFlags a = 0;
|
|
if (f->depth) a |= VK_IMAGE_ASPECT_DEPTH_BIT;
|
|
if (f->stencil) a |= VK_IMAGE_ASPECT_STENCIL_BIT;
|
|
return a ? a : VK_IMAGE_ASPECT_COLOR_BIT;
|
|
}
|
|
|
|
// Binds img into the pool memory behind its GPU address (physical pools, virtual pools through the page table) when
|
|
// the memory type, alignment and contiguous size allow; 0 otherwise.
|
|
// A depth / stencil image that needs more bytes than its NVN storage (guest_size) is not bound: AMD's RDNA2 driver
|
|
// (26.10.1) puts the depth compression metadata after the depth data, and above 1080p that tail lies over the next
|
|
// texture of the game's heap; each depth write then scrambled it (garbage band at the top, pink glow, blocks in the
|
|
// grass, a blue checkered scene in the menu). Colour images keep their place: the RX 9070 also sizes some of them
|
|
// above their storage, and nothing there showed that the GPU writes the excess.
|
|
static int format_compressed(VkFormat f) { // vkCmdClearColorImage takes no block-compressed format
|
|
return (f >= VK_FORMAT_BC1_RGB_UNORM_BLOCK && f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK) ||
|
|
(f >= VK_FORMAT_ASTC_4x4_SFLOAT_BLOCK && f <= VK_FORMAT_ASTC_12x12_SFLOAT_BLOCK);
|
|
}
|
|
|
|
static int bind_pool_memory(VkCtx *c, Tex *e, VkImage img, uint64_t guest_size) {
|
|
VkMemoryRequirements req;
|
|
vkGetImageMemoryRequirements(c->device, img, &req);
|
|
if ((e->f.depth || e->f.stencil) && req.size > guest_size) return 0;
|
|
VkMemSpan sp;
|
|
if (!vk_mem_resolve(e->pub.addr, req.size, &sp) || !sp.alloc || sp.alloc_offset % req.alignment) return 0;
|
|
VmaAllocationInfo info;
|
|
vmaGetAllocationInfo(c->vma, sp.alloc, &info);
|
|
if (!(req.memoryTypeBits >> info.memoryType & 1)) return 0;
|
|
if (vmaBindImageMemory2(c->vma, sp.alloc, sp.alloc_offset, img, NULL) != VK_SUCCESS) return 0;
|
|
e->bound_to = sp.alloc;
|
|
return 1;
|
|
}
|
|
|
|
// agl's BC4 encoder draws a BC texture's blocks through an RGBA16UI view, one texel per 8-byte block (refs/nxnative
|
|
// docs/agents/04-graphics.md, Breath of the Wild and Tears of the Kingdom sections). Vulkan does it with an uncompressed
|
|
// view of a BLOCK_TEXEL_VIEW_COMPATIBLE image; EXTENDED_USAGE lets the BC image carry the COLOR_ATTACHMENT usage that
|
|
// only such views support. Whether the device takes that combination for this format and shape:
|
|
static int block_views_ok(const VkImageCreateInfo *ci) {
|
|
VkPhysicalDeviceImageFormatInfo2 fi = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
|
|
.format = ci->format, .type = ci->imageType, .tiling = ci->tiling,
|
|
.usage = ci->usage, .flags = ci->flags};
|
|
VkImageFormatProperties2 p = {.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2};
|
|
if (vkGetPhysicalDeviceImageFormatProperties2(vk_ctx()->phys, &fi, &p) != VK_SUCCESS) return 0;
|
|
const VkImageFormatProperties *q = &p.imageFormatProperties;
|
|
return ci->extent.width <= q->maxExtent.width && ci->extent.height <= q->maxExtent.height &&
|
|
ci->extent.depth <= q->maxExtent.depth && ci->mipLevels <= q->maxMipLevels &&
|
|
ci->arrayLayers <= q->maxArrayLayers;
|
|
}
|
|
|
|
// UNDEFINED -> GENERAL; new depth / stencil images start cleared (LAS textures.c: the game copies depth
|
|
// without stencil, and the stencil of those images must not be whatever the memory held).
|
|
static void init_image(Tex *e, VkCommandBuffer cb) {
|
|
VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_NONE,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
|
|
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL,
|
|
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.image = e->pub.image, .subresourceRange = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib};
|
|
vkCmdPipelineBarrier2(cb, &di);
|
|
if (e->f.depth || e->f.stencil) {
|
|
VkClearDepthStencilValue zero = {0.0f, 0};
|
|
VkImageSubresourceRange rg = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
|
|
vkCmdClearDepthStencilImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &rg);
|
|
barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
|
|
} else if (vk_fix_zero() && !e->bound_to && !e->pub.mirrored && !format_compressed(e->f.vk)) {
|
|
// GPU-only storage that got memory of its own starts at zero, as the pool it stands for (vk_memory.c
|
|
// alloc_physical). Images aliasing pool memory are not cleared: that would erase what other textures rendered
|
|
// there; mirrored images are uploaded from their storage on first use.
|
|
VkClearColorValue zero = {0};
|
|
VkImageSubresourceRange rg = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
|
|
vkCmdClearColorImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &rg);
|
|
barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
|
|
}
|
|
}
|
|
|
|
// defer_init: the caller records the image's first transition (init_image or alias_copy).
|
|
static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) {
|
|
static volatile long n_unmapped, n_unsupported, n_failed, n_unbound;
|
|
VkCtx *c = vk_ctx();
|
|
const nvn_texdesc *d = &t->d;
|
|
if (!c || d->target == NVN_TEXTURE_TARGET_BUFFER) return NULL;
|
|
TexVkFormat f;
|
|
if (!tex_vk_format(d->format, &f)) {
|
|
log_limited(&n_unmapped, "texture format %#x has no Vulkan format: %s", d->format, f.reason);
|
|
return NULL;
|
|
}
|
|
if (!tex_vk_format_supported(&f)) {
|
|
log_limited(&n_unsupported, "texture format %#x: VkFormat %d lacks features %#x", d->format, (int)f.vk, (unsigned)f.need);
|
|
return NULL;
|
|
}
|
|
nvn_texlayout L;
|
|
nvn_tex_layout(d, &L);
|
|
uint32_t w = umax(d->width, 1), h = umax(d->height, 1), depth = 1, layers = 1;
|
|
VkImageType type = VK_IMAGE_TYPE_2D;
|
|
switch (d->target) {
|
|
case NVN_TEXTURE_TARGET_1D: type = VK_IMAGE_TYPE_1D; h = 1; break;
|
|
case NVN_TEXTURE_TARGET_1D_ARRAY: type = VK_IMAGE_TYPE_1D; h = 1; layers = L.layers; break;
|
|
case NVN_TEXTURE_TARGET_3D: type = VK_IMAGE_TYPE_3D; depth = umax(d->depth, 1); break;
|
|
case NVN_TEXTURE_TARGET_2D_ARRAY: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY:
|
|
case NVN_TEXTURE_TARGET_CUBEMAP: case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: layers = L.layers; break;
|
|
default: break;
|
|
}
|
|
uint32_t levels = umax(d->levels, 1), most = 1;
|
|
for (uint32_t s = umax(umax(w, h), depth); s > 1; s >>= 1) most++;
|
|
if (levels > most) levels = most;
|
|
uint32_t samples = d->samples > 1 ? (uint32_t)d->samples : 1;
|
|
if (samples > 1) levels = 1;
|
|
VkFormatFeatureFlags ff = features_of(f.vk);
|
|
VkImageUsageFlags usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
|
|
int ds = f.depth || f.stencil;
|
|
if (ds && (ff & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
|
|
if (!ds && (ff & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
|
|
if ((d->flags & NVN_TEXTURE_FLAGS_IMAGE) && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1)
|
|
usage |= VK_IMAGE_USAGE_STORAGE_BIT;
|
|
VkImageCreateFlags flags = ds ? 0 : VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; // format-reinterpreting views
|
|
// Cube compatible only for cube targets: the replay makes cube views of nothing else (vk_exec_draw.c get_view checks
|
|
// the image's own view type). The flag on every square array of 6+ layers changed how AMD's Windows driver lays such
|
|
// an image out, and terrain material arrays sampled through it came out in rainbow blocks that moved with the mip
|
|
// level (Savage Wind Recomp issue #59, BotW on an RX 580 and a 7800 XT).
|
|
if (type == VK_IMAGE_TYPE_2D && layers >= 6 && w == h && samples == 1 &&
|
|
(d->target == NVN_TEXTURE_TARGET_CUBEMAP || d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY))
|
|
flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
|
|
// NVN renders into a 3D texture slice by slice (a view's min layer is the depth plane, nvn_mtl.m), e.g. the
|
|
// grading LUT of the menu background blur. Vulkan needs a 2D view of the slice for that.
|
|
if (type == VK_IMAGE_TYPE_3D && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT;
|
|
VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = flags, .imageType = type, .format = f.vk,
|
|
.extent = {w, h, depth}, .mipLevels = levels, .arrayLayers = layers,
|
|
.samples = (VkSampleCountFlagBits)samples, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = usage,
|
|
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED};
|
|
int bw, bh, block_views = 0;
|
|
// TOTK_BLOCK_VIEWS_3D (default 0): a 3D BC image also gets block views. On RDNA2 cards (driver 26.10.1, issue #5)
|
|
// the 32x32x32 BC4 volume the game creates at the logo, made BLOCK_TEXEL_VIEW_COMPATIBLE and 2D_ARRAY_COMPATIBLE,
|
|
// took a GPU write just past the end of its memory: device lost at the first submit after it when the image ended
|
|
// its memory block. Draws into a 3D BC image go through a stand-in instead (vk_exec.c bc_target).
|
|
static int bv3d = -1;
|
|
if (bv3d < 0) {
|
|
bv3d = rt_knob("BLOCK_VIEWS_3D", "0");
|
|
rt_log("vk: 3D BC textures %s (TOTK_BLOCK_VIEWS_3D=%d)", bv3d ? "get block views (renderable)" : "have no block views",
|
|
bv3d);
|
|
}
|
|
if (!ds && samples == 1 && f.vk == f.ideal && nvn_fmt_bpb(d->format, &bw, &bh) && bw == 4 && bh == 4 &&
|
|
(type != VK_IMAGE_TYPE_3D || bv3d)) {
|
|
VkImageCreateInfo bc = ci; // BC formats: renderable through views of their blocks (block_views_ok)
|
|
bc.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
|
|
bc.flags |= VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
|
|
if (type == VK_IMAGE_TYPE_3D) bc.flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT;
|
|
if (block_views_ok(&bc)) ci = bc, usage = bc.usage, flags = bc.flags, block_views = 1;
|
|
}
|
|
// Feedback loops (vk.h VkCtx.feedback): every attachable image may be sampled by a draw of a pass that renders it,
|
|
// so it gets the usage now; adding it when a loop shows up would mean recreating the image (that lost the
|
|
// grass tile maps once).
|
|
int fb_usage = c->feedback == VK_FEEDBACK_LAYOUT && !block_views &&
|
|
(usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT));
|
|
if (fb_usage) ci.usage = usage |= VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
|
Tex *e = calloc(1, sizeof *e);
|
|
if (!e) rt_fatal("vk: out of memory for a texture");
|
|
e->key = *k;
|
|
e->t = *t;
|
|
e->f = f;
|
|
e->pub.addr = k->addr;
|
|
VkImage img = VK_NULL_HANDLE;
|
|
if (nvn_gpu_only(k->addr)) { // storage in a physical or virtual pool: alias the pool memory (section 3.2)
|
|
if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS && !bind_pool_memory(c, e, img, L.size)) {
|
|
vkDestroyImage(c->device, img, NULL);
|
|
img = VK_NULL_HANDLE;
|
|
log_limited(&n_unbound, "texture %#llx (%ux%u fmt %#x) cannot alias its pool memory; own memory instead",
|
|
(unsigned long long)k->addr, w, h, d->format);
|
|
}
|
|
}
|
|
if (!img) {
|
|
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
|
VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL);
|
|
if (r != VK_SUCCESS) {
|
|
log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r,
|
|
d->format, w, h, depth, d->target, levels, layers);
|
|
free(e);
|
|
return NULL;
|
|
}
|
|
}
|
|
VkTexImage *p = &e->pub;
|
|
p->image = img;
|
|
p->format = f.vk;
|
|
p->type = type;
|
|
p->usage = usage;
|
|
p->aspect = aspect_of(&f);
|
|
p->width = w, p->height = h, p->depth = depth, p->levels = levels, p->layers = layers, p->samples = samples;
|
|
p->view_type = type == VK_IMAGE_TYPE_1D ? (d->target == NVN_TEXTURE_TARGET_1D_ARRAY ? VK_IMAGE_VIEW_TYPE_1D_ARRAY : VK_IMAGE_VIEW_TYPE_1D)
|
|
: type == VK_IMAGE_TYPE_3D ? VK_IMAGE_VIEW_TYPE_3D
|
|
: d->target == NVN_TEXTURE_TARGET_CUBEMAP && (flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) ? VK_IMAGE_VIEW_TYPE_CUBE
|
|
: d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY && (flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) ? VK_IMAGE_VIEW_TYPE_CUBE_ARRAY
|
|
: layers > 1 || d->target == NVN_TEXTURE_TARGET_2D_ARRAY || d->target == NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY
|
|
? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D;
|
|
p->alpha_one = f.alpha_one;
|
|
memcpy(p->chan, f.chan, sizeof p->chan);
|
|
p->swz[0] = (uint8_t)d->swizzle_r, p->swz[1] = (uint8_t)d->swizzle_g;
|
|
p->swz[2] = (uint8_t)d->swizzle_b, p->swz[3] = (uint8_t)d->swizzle_a;
|
|
if (!(p->swz[0] | p->swz[1] | p->swz[2] | p->swz[3])) // descriptors built without nvnTextureBuilderSetDefaults
|
|
p->swz[0] = NVN_TEXTURE_SWIZZLE_R, p->swz[1] = NVN_TEXTURE_SWIZZLE_G, p->swz[2] = NVN_TEXTURE_SWIZZLE_B,
|
|
p->swz[3] = NVN_TEXTURE_SWIZZLE_A;
|
|
p->mirrored = (uint8_t)tex_mirrored(t);
|
|
p->bound = e->bound_to != NULL;
|
|
p->block_views = (uint8_t)block_views;
|
|
p->fb_usage = (uint8_t)fb_usage;
|
|
|
|
if (!defer_init) {
|
|
init_image(e, vk_upload_begin());
|
|
vk_upload_end();
|
|
}
|
|
return e;
|
|
}
|
|
|
|
static int owned_lower(uint64_t a) {
|
|
int lo = 0, hi = nowned;
|
|
while (lo < hi) {
|
|
int m = (lo + hi) / 2;
|
|
if (owned[m]->pub.addr < a) lo = m + 1;
|
|
else hi = m;
|
|
}
|
|
return lo;
|
|
}
|
|
|
|
static void own_add(Tex *e) {
|
|
if (e->listed) return;
|
|
AcquireSRWLockExclusive(&owned_lock);
|
|
if (nowned == cowned) {
|
|
cowned = cowned ? 2 * cowned : 256;
|
|
if (!(owned = realloc(owned, sizeof *owned * (size_t)cowned))) rt_fatal("vk: out of memory for the owned-texture list");
|
|
}
|
|
int i = owned_lower(e->pub.addr);
|
|
memmove(owned + i + 1, owned + i, sizeof *owned * (size_t)(nowned - i));
|
|
owned[i] = e;
|
|
__atomic_store_n(&nowned, nowned + 1, __ATOMIC_RELEASE);
|
|
if (e->t.L.size > max_owned) max_owned = e->t.L.size;
|
|
e->listed = 1;
|
|
ReleaseSRWLockExclusive(&owned_lock);
|
|
}
|
|
|
|
static void own_remove(Tex *e) {
|
|
if (!e->listed) return;
|
|
AcquireSRWLockExclusive(&owned_lock);
|
|
for (int i = owned_lower(e->pub.addr); i < nowned && owned[i]->pub.addr == e->pub.addr; i++)
|
|
if (owned[i] == e) {
|
|
memmove(owned + i, owned + i + 1, sizeof *owned * (size_t)(nowned - i - 1));
|
|
__atomic_store_n(&nowned, nowned - 1, __ATOMIC_RELEASE);
|
|
break;
|
|
}
|
|
e->listed = 0;
|
|
ReleaseSRWLockExclusive(&owned_lock);
|
|
}
|
|
|
|
static void rel_append(Tex *e) {
|
|
e->rel_next = NULL;
|
|
e->rel_prev = rel_tail;
|
|
if (rel_tail) rel_tail->rel_next = e;
|
|
else rel_head = e;
|
|
rel_tail = e;
|
|
}
|
|
|
|
static void rel_unlink(Tex *e) {
|
|
if (e->rel_prev) e->rel_prev->rel_next = e->rel_next;
|
|
else rel_head = e->rel_next;
|
|
if (e->rel_next) e->rel_next->rel_prev = e->rel_prev;
|
|
else rel_tail = e->rel_prev;
|
|
e->rel_prev = e->rel_next = NULL;
|
|
}
|
|
|
|
static void destroy_tex(Tex *e) {
|
|
own_remove(e);
|
|
tex_gen_bump(e->key.addr);
|
|
nvn_hm_del(&by_key, &e->key);
|
|
Tex *head = nvn_hm_get(&by_addr, &e->key.addr);
|
|
if (head == e) {
|
|
if (e->next_at) nvn_hm_put(&by_addr, &e->key.addr, e->next_at);
|
|
else nvn_hm_del(&by_addr, &e->key.addr);
|
|
} else {
|
|
for (Tex *p = head; p; p = p->next_at)
|
|
if (p->next_at == e) { p->next_at = e->next_at; break; }
|
|
}
|
|
if (e->released) nreleased--, rel_unlink(e);
|
|
vk_exec_image_gone(e->pub.image);
|
|
vk_defer_image(e->pub.image, e->alloc);
|
|
free(e);
|
|
}
|
|
|
|
// Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries the
|
|
// oldest one goes at each release (it comes back on demand, a mirrored one re-uploaded from its storage). TotK
|
|
// finalizes and initializes transient textures every frame, so a released entry is mostly taken back within frames.
|
|
// Dropping all of them at once when the count passed 4096 destroyed thousands of images in one frame: 333-422 ms
|
|
// frames every 40-50 s on a GTX 1050 Ti, then the textures still in use were created and uploaded again.
|
|
#define RELEASED_MAX 2048
|
|
static void sweep_released(void) {
|
|
while (nreleased > RELEASED_MAX && rel_head) destroy_tex(rel_head);
|
|
}
|
|
|
|
// TOTK_TEX_VRAM_PRESSURE (percent of the device-local budget, default 90; 0 = off): above it, each submit drops up to
|
|
// PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released images (each ASTC
|
|
// one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason above. Only entries
|
|
// that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their storage. A render target
|
|
// lives only in its image; dropping it brought the texture back empty (black flashes in a Depths boss fight, ~170
|
|
// "left uninitialised" aliases a minute).
|
|
#define PRESSURE_STEP 32
|
|
static int vram_pressure = -1;
|
|
static uint64_t n_pressure_drops;
|
|
|
|
static void sweep_pressure(void) { // tex_lock held
|
|
if (vram_pressure < 0) {
|
|
const char *s = cfg_get("TOTK_TEX_VRAM_PRESSURE");
|
|
vram_pressure = s && *s ? atoi(s) : 90;
|
|
if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = 90;
|
|
rt_log("vk: released textures under VRAM pressure: %s%d%%", vram_pressure ? "dropped above " : "off", vram_pressure);
|
|
}
|
|
if (!vram_pressure || !rel_head || submit_epoch & 7) return; // the budget query every 8 submits
|
|
uint64_t used, budget;
|
|
int exact;
|
|
if (!vk_mem_vram(&used, &budget, &exact) || used <= budget / 100 * (uint64_t)vram_pressure) return;
|
|
int dropped = 0;
|
|
for (Tex *e = rel_head, *next; e && dropped < PRESSURE_STEP; e = next) {
|
|
next = e->rel_next;
|
|
if (!e->pub.mirrored || e->pub.gpu_owned || e->gpu_dirty) continue;
|
|
destroy_tex(e);
|
|
dropped++, n_pressure_drops++;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- uploads
|
|
static uint32_t texel_alpha_one(int format, uint32_t *elem) {
|
|
switch (format) {
|
|
case 0x19: *elem = 1; return 0xFF;
|
|
case 0x1A: *elem = 1; return 0x7F;
|
|
case 0x1B: case 0x1C: *elem = 1; return 1;
|
|
case 0x1D: *elem = 2; return 0x3C00;
|
|
case 0x1E: *elem = 2; return 0xFFFF;
|
|
case 0x1F: *elem = 2; return 0x7FFF;
|
|
case 0x20: case 0x21: *elem = 2; return 1;
|
|
case 0x22: *elem = 4; return 0x3F800000;
|
|
default: *elem = 4; return 1; // 0x23, 0x24
|
|
}
|
|
}
|
|
|
|
// 3-component texels to 4 (n texels).
|
|
static void expand_rgb(int format, const uint8_t *src, uint8_t *dst, size_t n) {
|
|
uint32_t e, one = texel_alpha_one(format, &e);
|
|
for (size_t i = 0; i < n; i++, src += 3 * e, dst += 4 * e) {
|
|
memcpy(dst, src, 3 * e);
|
|
memcpy(dst + 3 * e, &one, e);
|
|
}
|
|
}
|
|
|
|
static void shrink_rgb(int format, const uint8_t *src, uint8_t *dst, size_t n) {
|
|
uint32_t e;
|
|
texel_alpha_one(format, &e);
|
|
for (size_t i = 0; i < n; i++, src += 4 * e, dst += 3 * e) memcpy(dst, src, 3 * e);
|
|
}
|
|
|
|
// TOTK_ASTC=check: the compute decoder's texels of n regions (gpu: the regions one after the other) against
|
|
// astc_decode_ldr of the same raw blocks (raw: the staging bytes the regions point into).
|
|
static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar, int n, const uint8_t *gpu) {
|
|
static volatile long n_bad;
|
|
static uint64_t n_tex, n_texels, n_diff_tex;
|
|
uint32_t bw = e->f.astc_bw, bh = e->f.astc_bh;
|
|
uint64_t diff = 0, first = UINT64_MAX;
|
|
uint32_t want = 0, got = 0;
|
|
int first_k = 0;
|
|
for (int k = 0; k < n; k++) {
|
|
const VkAstcRegion *r = &ar[k];
|
|
size_t sb = (size_t)((r->w + bw - 1) / bw) * ((r->h + bh - 1) / bh) * 16, st = (size_t)r->w * r->h * 4;
|
|
uint8_t *cpu = malloc(st);
|
|
for (uint32_t z = 0; z < r->slices; z++, gpu += st) {
|
|
astc_decode_ldr(raw + r->src + z * sb, r->w, r->h, bw, bh, e->f.astc_srgb, cpu);
|
|
if (!memcmp(cpu, gpu, st)) continue;
|
|
for (size_t i = 0; i < st / 4; i++)
|
|
if (memcmp(cpu + 4 * i, gpu + 4 * i, 4)) {
|
|
if (first == UINT64_MAX) {
|
|
first = i, first_k = k;
|
|
memcpy(&want, cpu + 4 * i, 4), memcpy(&got, gpu + 4 * i, 4);
|
|
}
|
|
diff++;
|
|
}
|
|
}
|
|
free(cpu);
|
|
n_texels += (uint64_t)r->w * r->h * r->slices;
|
|
}
|
|
n_tex++;
|
|
if (diff) {
|
|
n_diff_tex++;
|
|
log_limited(&n_bad, "ASTC check: texture %#llx fmt %#x %ux%u: %llu texels differ, first region %d texel %llu "
|
|
"cpu %08x gpu %08x", (unsigned long long)e->pub.addr, e->t.d.format, bw, bh,
|
|
(unsigned long long)diff, first_k, (unsigned long long)first, want, got);
|
|
}
|
|
if (n_tex == 1 || !(n_tex & 63) || diff)
|
|
rt_log("vk: ASTC check: %llu textures, %llu texels compared, %llu textures differ", (unsigned long long)n_tex,
|
|
(unsigned long long)n_texels, (unsigned long long)n_diff_tex);
|
|
}
|
|
|
|
// Copies the whole storage into the image (every level and layer the image has).
|
|
static void upload(Tex *e) {
|
|
static volatile long n_bad, n_undecoded;
|
|
const tex_t *t = &e->t;
|
|
const nvn_texdesc *d = &t->d;
|
|
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
|
|
if (!bpb || !t->mem) return;
|
|
uint64_t p0 = perf_begin(), u0 = kern_now_ns();
|
|
int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D;
|
|
uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers;
|
|
int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb;
|
|
int agpu = astc && vk_astc_gpu(), acheck = agpu && vk_astc_mode() == VK_ASTC_CHECK; // raw blocks to the GPU
|
|
uint32_t n = levels * layers, k = 0;
|
|
VkBufferImageCopy *rg = calloc(n, sizeof *rg);
|
|
VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL;
|
|
VkDeviceSize total = 0, decoded = 0;
|
|
for (uint32_t lv = 0; lv < levels; lv++)
|
|
for (uint32_t ly = 0; ly < layers; ly++) {
|
|
uint32_t w = mip(d->width, lv), h = one_d ? 1 : mip(d->height, lv), sl = is3d ? mip(d->depth, lv) : 1;
|
|
uint64_t texels = (uint64_t)w * h * sl, raw = (uint64_t)(w + bw - 1) / bw * bpb * ((h + bh - 1) / bh) * sl;
|
|
uint64_t out = agpu ? raw : astc ? texels * 4 : rgb ? texels * (uint64_t)bpb / 3 * 4 : raw;
|
|
rg[k++] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {e->pub.aspect, lv, ly, 1},
|
|
.imageExtent = {w, h, sl}};
|
|
total += (out + 15) & ~15ull;
|
|
decoded += texels * 4;
|
|
}
|
|
VkCommandBuffer cb = vk_upload_begin();
|
|
VkBuffer sb, rbuf = VK_NULL_HANDLE;
|
|
VmaAllocation ralloc = NULL;
|
|
uint8_t *stg = vk_staging(total, &sb, NULL), *rmap = acheck ? vk_staging(decoded, &rbuf, &ralloc) : NULL;
|
|
k = 0;
|
|
for (uint32_t lv = 0; lv < levels; lv++)
|
|
for (uint32_t ly = 0; ly < layers; ly++, k++) {
|
|
uint32_t w = rg[k].imageExtent.width, h = rg[k].imageExtent.height, sl = rg[k].imageExtent.depth;
|
|
uint32_t rb = (w + bw - 1) / bw * (uint32_t)bpb, rows = (h + bh - 1) / bh;
|
|
size_t raw = (size_t)rb * rows * sl;
|
|
uint8_t *dst = stg + rg[k].bufferOffset, *tmp = (astc && !agpu) || rgb ? malloc(raw) : NULL;
|
|
if (nvn_deswizzle(d, lv, ly, t->mem, tmp ? tmp : dst, rb)) {
|
|
log_limited(&n_bad, "texture %#llx fmt %#x target %d: level %u layer %u has no CPU layout; uploaded as zeros",
|
|
(unsigned long long)e->pub.addr, d->format, d->target, lv, ly);
|
|
memset(tmp ? tmp : dst, 0, raw);
|
|
}
|
|
if (agpu) {
|
|
ar[k] = (VkAstcRegion){.src = rg[k].bufferOffset, .w = w, .h = h, .slices = sl, .level = lv, .layer = ly};
|
|
} else if (astc) {
|
|
uint64_t a0 = perf_begin();
|
|
for (uint32_t z = 0; z < sl; z++)
|
|
astc_decode_ldr_mt(tmp + (size_t)z * rb * rows, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb,
|
|
dst + (size_t)z * w * h * 4);
|
|
perf_end(PERF_ASTC, a0);
|
|
rg[k].bufferRowLength = w, rg[k].bufferImageHeight = h;
|
|
} else if (rgb) {
|
|
expand_rgb(d->format, tmp, dst, (size_t)w * h * sl);
|
|
rg[k].bufferRowLength = w, rg[k].bufferImageHeight = h;
|
|
} else {
|
|
rg[k].bufferRowLength = rb / (uint32_t)bpb * (uint32_t)bw, rg[k].bufferImageHeight = rows * (uint32_t)bh;
|
|
}
|
|
free(tmp);
|
|
}
|
|
uint8_t *raw_copy = acheck ? malloc(total) : NULL;
|
|
if (raw_copy) memcpy(raw_copy, stg, total);
|
|
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
|
int checked = 0;
|
|
if (agpu) {
|
|
uint64_t a0 = perf_begin();
|
|
checked = vk_astc_record(cb, sb, total, ar, (int)n, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, e->pub.image,
|
|
e->pub.aspect, rbuf);
|
|
perf_end(PERF_ASTC, a0);
|
|
if (!checked)
|
|
log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was",
|
|
(unsigned long long)e->pub.addr, d->format, d->width, d->height);
|
|
} else {
|
|
vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, n, rg);
|
|
}
|
|
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
|
|
vk_upload_end();
|
|
if (acheck) { // waits for the GPU
|
|
vk_upload_flush(1);
|
|
if (checked) {
|
|
vmaInvalidateAllocation(vk_ctx()->vma, ralloc, 0, VK_WHOLE_SIZE);
|
|
astc_check(e, raw_copy, ar, (int)n, rmap);
|
|
}
|
|
vmaDestroyBuffer(vk_ctx()->vma, rbuf, ralloc);
|
|
}
|
|
free(raw_copy);
|
|
free(ar);
|
|
free(rg);
|
|
uint64_t dt = kern_now_ns() - u0;
|
|
if (e->uploaded) {
|
|
n_reuploads++, perf_count(PERF_CNT_TEX_REUP);
|
|
ru_n++, ru_bytes += t->L.size, ru_ns += dt;
|
|
if (e->wt_n) {
|
|
ru_wt_n++, ru_wt_bytes += t->L.size, ru_wt_ns += dt;
|
|
if (rt_trace & RT_TRACE_PERF)
|
|
rt_log("perf: frame %llu: re-upload after %u WriteTexels: texture %#llx fmt %#x %ux%u, %.1f KiB, %.3f ms",
|
|
(unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), e->wt_n,
|
|
(unsigned long long)e->pub.addr, d->format, d->width, d->height, t->L.size / 1024.0, dt / 1e6);
|
|
}
|
|
} else {
|
|
n_uploads++, perf_count(PERF_CNT_TEX_UP);
|
|
}
|
|
e->wt_n = 0;
|
|
e->uploaded = 1;
|
|
e->n_up++;
|
|
perf_end(PERF_TEX, p0);
|
|
}
|
|
|
|
// vk_mem_changed over the pages wholly inside e's storage (all of its pages when it fills none): a neighbour that shares
|
|
// the first or last page does not count as a write to e.
|
|
static int storage_rewritten(const Tex *e, uint32_t since, uint32_t *now) {
|
|
uint64_t lo = (e->pub.addr + 0xfff) & ~0xfffull, hi = (e->pub.addr + e->t.L.size) & ~0xfffull;
|
|
if (hi <= lo) lo = e->pub.addr, hi = e->pub.addr + e->t.L.size;
|
|
return vk_mem_changed(lo, hi - lo, since, now);
|
|
}
|
|
|
|
// A mirrored texture is uploaded on first use and again when its storage pages carry a newer write generation.
|
|
static void make_fresh(Tex *e) {
|
|
if (!e->pub.mirrored || e->pub.gpu_owned) return;
|
|
uint64_t size = e->t.L.size;
|
|
if (!e->uploaded) {
|
|
vk_mem_changed(e->pub.addr, size, 0, &e->wgen); // resets the write bits before the storage is read
|
|
upload(e);
|
|
return;
|
|
}
|
|
uint32_t now;
|
|
int ch = vk_mem_changed(e->pub.addr, size, e->wgen, &now);
|
|
e->wgen = now;
|
|
e->last_ch = ch;
|
|
if (ch) upload(e); // 1 written, -1 not tracked: both upload again
|
|
}
|
|
|
|
// make_fresh for draw bindings: the page check answers from this submission's write-watch snapshot (vk_memory.c
|
|
// vk_mem_changed_snap) instead of a system call per texture. A write the snapshot of the texture's region missed (made
|
|
// after it, during this replay) keeps its write-watch bit and is caught by the next submission's snapshot.
|
|
static void make_fresh_snap(Tex *e) {
|
|
if (!e->pub.mirrored || e->pub.gpu_owned) return;
|
|
uint32_t now;
|
|
n_checks++;
|
|
int ch = vk_mem_changed_snap(e->pub.addr, e->t.L.size, e->uploaded ? e->wgen : 0, submit_epoch, &now);
|
|
e->wgen = now;
|
|
if (!e->uploaded) { upload(e); return; }
|
|
e->last_ch = ch;
|
|
if (ch) upload(e);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- aliases (TOTK_TEX_ALIAS)
|
|
// Bytes per block and block size of the image's VkFormat (ASTC decodes to RGBA8, RGB expands to RGBA); 0 when the
|
|
// VkFormat substitutes another one of unknown size.
|
|
static int img_block(const Tex *e, int *bw, int *bh) {
|
|
if (e->f.astc_bc) { *bw = *bh = 4; return 16; }
|
|
if (e->f.astc_bw) { *bw = *bh = 1; return 4; }
|
|
int b = nvn_fmt_bpb(e->t.d.format, bw, bh);
|
|
if (e->f.expand_rgb) { *bw = *bh = 1; return b / 3 * 4; }
|
|
return e->f.vk == e->f.ideal ? b : 0;
|
|
}
|
|
|
|
// The new entry e can take src's content texel for texel: colour, one sample, the same shape and storage layout, and
|
|
// image formats of one block size.
|
|
static int alias_compatible(const Tex *e, const Tex *src) {
|
|
const VkTexImage *p = &e->pub, *q = &src->pub;
|
|
if (e->f.depth || e->f.stencil || src->f.depth || src->f.stencil || p->samples > 1 || q->samples > 1) return 0;
|
|
if (p->type != q->type || p->width != q->width || p->height != q->height || p->depth != q->depth) return 0;
|
|
if (e->t.L.size != src->t.L.size || nvn_tex_is_linear(&e->t.d) != nvn_tex_is_linear(&src->t.d)) return 0;
|
|
int ew, eh, sw, sh, eb = img_block(e, &ew, &eh), sb = img_block(src, &sw, &sh);
|
|
return eb && eb == sb && ew == sw && eh == sh;
|
|
}
|
|
|
|
// src holds content the storage bytes do not: the GPU wrote it and, for a mirrored texture, it was not written back
|
|
// and the CPU has not rewritten the storage since (the new entry's upload then reads the newer bytes).
|
|
static int alias_has_gpu_content(const Tex *src) {
|
|
if (!src->pub.gpu_owned) return 0;
|
|
if (!src->pub.mirrored) return 1;
|
|
uint32_t now;
|
|
return src->gpu_dirty && !storage_rewritten(src, src->gpu_gen, &now);
|
|
}
|
|
|
|
// Maxwell reads the same bytes through both descriptors: src's content goes into the new image e at the replay's
|
|
// current position, before any later command of the submission uses e. Through a buffer, since bound images share
|
|
// their memory and image copies may not overlap; e leaves UNDEFINED between the two halves, so its first transition
|
|
// cannot clobber src's bytes before they are read. 0 when no replay is recording (e is then not initialised yet).
|
|
static int alias_copy(Tex *e, const Tex *src) {
|
|
int reopen;
|
|
VkCommandBuffer cb = vk_exec_inline_begin(&reopen);
|
|
if (!cb) return 0;
|
|
int bw, bh, bpb = img_block(e, &bw, &bh);
|
|
uint32_t levels = e->pub.levels < src->pub.levels ? e->pub.levels : src->pub.levels;
|
|
uint32_t layers = e->pub.layers < src->pub.layers ? e->pub.layers : src->pub.layers, n = levels * layers, k = 0;
|
|
VkBufferImageCopy *rg = calloc(n, sizeof *rg);
|
|
if (!rg) rt_fatal("vk: out of memory for an alias copy");
|
|
VkDeviceSize total = 0;
|
|
for (uint32_t lv = 0; lv < levels; lv++)
|
|
for (uint32_t ly = 0; ly < layers; ly++, k++) {
|
|
uint32_t w = mip(e->pub.width, lv), h = mip(e->pub.height, lv), z = mip(e->pub.depth, lv);
|
|
rg[k] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, lv, ly, 1},
|
|
.imageExtent = {w, h, z}};
|
|
total += ((VkDeviceSize)(w + bw - 1) / bw * ((h + bh - 1) / bh) * z * bpb + 15) & ~15ull;
|
|
}
|
|
VkBuffer buf = vk_exec_inline_buffer(total);
|
|
if (!buf) {
|
|
free(rg);
|
|
vk_exec_inline_end(reopen);
|
|
return 0;
|
|
}
|
|
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
|
|
vkCmdCopyImageToBuffer(cb, src->pub.image, VK_IMAGE_LAYOUT_GENERAL, buf, n, rg);
|
|
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT};
|
|
VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
|
|
.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT,
|
|
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL,
|
|
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.image = e->pub.image,
|
|
.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
|
|
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb,
|
|
.imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib};
|
|
vkCmdPipelineBarrier2(cb, &di);
|
|
vkCmdCopyBufferToImage(cb, buf, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, n, rg);
|
|
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
|
|
vk_exec_inline_end(reopen);
|
|
free(rg);
|
|
return 1;
|
|
}
|
|
|
|
static Tex *entry_of(const tex_t *t) {
|
|
TexKey k;
|
|
make_key(t, &k);
|
|
Tex *e = nvn_hm_get(&by_key, &k);
|
|
if (!e) {
|
|
if (tex_alias < 0) {
|
|
tex_alias = cfg_bool("TOTK_TEX_ALIAS");
|
|
rt_log("vk: texture aliases: %s", tex_alias ? "a new image takes the live image's content" : "a new image starts empty");
|
|
}
|
|
if (!(e = create_tex(t, &k, tex_alias))) return NULL;
|
|
perf_count(PERF_CNT_TEX_NEW);
|
|
// released entries of another descriptor at this address are replaced by the new one; a live one holding
|
|
// GPU-written content is the alias source, preferring the one that differs only in swizzle (same_but_swizzle)
|
|
int alias = 0;
|
|
Tex *src = NULL;
|
|
for (Tex *p = nvn_hm_get(&by_addr, &k.addr), *nx; p; p = nx) {
|
|
nx = p->next_at;
|
|
if (p->released) { destroy_tex(p); continue; }
|
|
alias = 1;
|
|
if (tex_alias && (!src || (same_but_swizzle(&p->key, &k) && !same_but_swizzle(&src->key, &k))) &&
|
|
alias_compatible(e, p) && alias_has_gpu_content(p))
|
|
src = p;
|
|
}
|
|
if (alias) perf_count(PERF_CNT_TEX_ALIAS);
|
|
if (tex_alias) {
|
|
if (src && alias_copy(e, src)) {
|
|
n_alias_copied++;
|
|
if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes
|
|
vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen);
|
|
e->uploaded = 1;
|
|
e->pub.gpu_owned = 1;
|
|
}
|
|
} else {
|
|
init_image(e, vk_upload_begin());
|
|
vk_upload_end();
|
|
if (alias) {
|
|
if (e->pub.mirrored) n_alias_upload++;
|
|
else n_alias_uninit++;
|
|
}
|
|
}
|
|
}
|
|
e->next_at = nvn_hm_get(&by_addr, &k.addr);
|
|
nvn_hm_put(&by_addr, &k.addr, e);
|
|
nvn_hm_put(&by_key, &k, e);
|
|
tex_gen_bump(k.addr);
|
|
}
|
|
if (e->released) e->released = 0, nreleased--, rel_unlink(e), tex_gen_bump(k.addr);
|
|
e->t = *t;
|
|
return e;
|
|
}
|
|
|
|
// vk_dump: upload bookkeeping of an image vk_texture_get / vk_texture_by_id returned. pages_written: whether its
|
|
// storage pages were written since the last check (1, 0, -1 not tracked); the query leaves that check's state alone.
|
|
void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written) {
|
|
lock();
|
|
const Tex *e = (const Tex *)img; // pub is the first member
|
|
uint32_t now;
|
|
*uploads = e->n_up;
|
|
*last_change = e->last_ch;
|
|
*pages_written = e->pub.mirrored && e->uploaded ? vk_mem_changed(e->pub.addr, e->t.L.size, e->wgen, &now) : 0;
|
|
unlock();
|
|
}
|
|
|
|
const VkTexImage *vk_texture_get(const tex_t *t) {
|
|
if (!t || !t->mem) return NULL;
|
|
lock();
|
|
Tex *e = entry_of(t);
|
|
if (e) make_fresh(e);
|
|
unlock();
|
|
return e ? &e->pub : NULL;
|
|
}
|
|
|
|
// Small render targets the CPU reads without NVN's invalidate (vk_texture_write_back_small); small_pending: one may be
|
|
// dirty. wb_limit is the largest target (bytes) that goes back when the CPU waits for a fence: TOTK_WB_FENCE_LIMIT,
|
|
// read once. wb_trace: TOTK_TRACE=gfx, read together with it.
|
|
enum { WB_SMALL_DEFAULT = 64 << 10 };
|
|
static int small_pending;
|
|
static uint64_t wb_limit = WB_SMALL_DEFAULT;
|
|
static int wb_trace;
|
|
static int wb_cfg_read;
|
|
|
|
static void wb_cfg_init(void) {
|
|
if (wb_cfg_read) return;
|
|
int v = cfg_int("TOTK_WB_FENCE_LIMIT");
|
|
wb_limit = v < 0 ? 0 : (uint64_t)v;
|
|
wb_trace = (rt_trace & RT_TRACE_GFX) != 0;
|
|
wb_cfg_read = 1;
|
|
if (wb_limit != WB_SMALL_DEFAULT) rt_log("vk: fence write-back limit %llu bytes", (unsigned long long)wb_limit);
|
|
}
|
|
|
|
void vk_texture_gpu_written(const tex_t *t) {
|
|
if (!t || !t->mem) return;
|
|
vk_exec_frame_written(t);
|
|
lock();
|
|
Tex *e = entry_of(t);
|
|
if (e) {
|
|
make_fresh(e); // a partial render keeps what the storage held before
|
|
e->pub.gpu_owned = 1;
|
|
uint64_t f = vk_exec_recording_fence();
|
|
if (f > e->wr_fence) e->wr_fence = f;
|
|
if (e->pub.mirrored) {
|
|
e->gpu_dirty = 1;
|
|
storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render
|
|
e->gpu_seq = ++gpu_writes;
|
|
e->gpu_fence = vk_exec_recording_fence();
|
|
wb_cfg_init();
|
|
if (e->t.L.size <= wb_limit) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE);
|
|
own_add(e);
|
|
}
|
|
}
|
|
unlock();
|
|
}
|
|
|
|
void vk_texture_release(const tex_t *t) {
|
|
if (!t || !t->mem) return;
|
|
TexKey k;
|
|
make_key(t, &k);
|
|
lock();
|
|
Tex *e = nvn_hm_get(&by_key, &k);
|
|
if (e && !e->released) {
|
|
e->released = 1;
|
|
nreleased++;
|
|
rel_append(e);
|
|
tex_gen_bump(k.addr);
|
|
sweep_released();
|
|
}
|
|
unlock();
|
|
}
|
|
|
|
void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc) {
|
|
lock();
|
|
Tex **list = malloc(sizeof *list * (size_t)(by_key.n + 1));
|
|
int n = 0;
|
|
for (uint32_t i = 0; i < by_key.cap; i++) {
|
|
Tex *e = by_key.vals[i];
|
|
if (e && ((e->key.addr >= lo && e->key.addr - lo < size) || (alloc && e->bound_to == alloc))) list[n++] = e;
|
|
}
|
|
for (int i = 0; i < n; i++) destroy_tex(list[i]);
|
|
free(list);
|
|
unlock();
|
|
}
|
|
|
|
// ---------------------------------------------------------------- descriptor slots
|
|
void vk_texture_register(int id, const tex_t *t, const view_t *v) {
|
|
if (id < 0 || id >= (1 << 20)) rt_fatal("vk: texture slot %d out of range", id);
|
|
lock();
|
|
if (id >= ntslots) {
|
|
int n = ntslots ? ntslots : 1024;
|
|
while (n <= id) n *= 2;
|
|
TexSlot *s = realloc(tslots, sizeof *s * (size_t)n);
|
|
if (!s) rt_fatal("vk: out of memory for texture slots");
|
|
memset(s + ntslots, 0, sizeof *s * (size_t)(n - ntslots));
|
|
tslots = s, ntslots = n;
|
|
}
|
|
if (!sinfo) {
|
|
uint64_t *p = calloc(1 << 20, sizeof *p);
|
|
if (!p) rt_fatal("vk: out of memory for texture slot tokens");
|
|
__atomic_store_n(&sinfo, p, __ATOMIC_RELEASE);
|
|
}
|
|
TexSlot *s = &tslots[id];
|
|
s->t = *t;
|
|
s->used = 1;
|
|
s->ce = NULL;
|
|
slots_gen++;
|
|
if (!++reg_serial) reg_serial = 1;
|
|
__atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE);
|
|
s->has_view = v != NULL;
|
|
if (v) s->v = *v;
|
|
else memset(&s->v, 0, sizeof s->v);
|
|
unlock();
|
|
}
|
|
|
|
// The entry slot s resolves to, and the view and has_view a caller gets with it (the slot's, with the descriptor's
|
|
// swizzle when the entry is another descriptor at the same storage). NULL when the slot names no texture.
|
|
static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) {
|
|
*view = s->v;
|
|
*has_view = s->has_view;
|
|
TexKey k;
|
|
make_key(&s->t, &k);
|
|
Tex *e = nvn_hm_get(&by_key, &k);
|
|
if (!e || e->released) { // MNG hle_nvn.c: a finalized slot follows the live texture at its storage
|
|
// Prefer the live entry that differs from the slot descriptor only in its swizzle: transient render targets
|
|
// share storage, and the newest one at the address is usually another size. The map screen samples its
|
|
// 2250x1080 map target through an (R,G,B,1) descriptor and followed a 960x60 target at the same address.
|
|
Tex *own = e, *any = NULL;
|
|
e = NULL;
|
|
for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) {
|
|
if (p->released) continue;
|
|
if (!any) any = p;
|
|
if (same_but_swizzle(&p->key, &k)) { e = p; break; }
|
|
}
|
|
if (!e) e = any ? any : own;
|
|
}
|
|
if (!e) e = s->t.mem ? entry_of(&s->t) : NULL;
|
|
// A followed texture is another descriptor at the same storage: its image holds the content, but sampling takes the
|
|
// slot descriptor's swizzle. The UI renders text into RGBA8 and samples it through a (A,A,A,A) descriptor; with the
|
|
// render target's RGBA swizzle the text blur saw R = 1 everywhere and drew solid boxes behind tips and labels.
|
|
if (e && !(s->has_view && s->v.set_swizzle)) {
|
|
const nvn_texdesc *d = &s->t.d;
|
|
uint8_t sw[4] = {(uint8_t)d->swizzle_r, (uint8_t)d->swizzle_g, (uint8_t)d->swizzle_b, (uint8_t)d->swizzle_a};
|
|
if (!(sw[0] | sw[1] | sw[2] | sw[3])) // descriptors built without nvnTextureBuilderSetDefaults
|
|
sw[0] = NVN_TEXTURE_SWIZZLE_R, sw[1] = NVN_TEXTURE_SWIZZLE_G, sw[2] = NVN_TEXTURE_SWIZZLE_B,
|
|
sw[3] = NVN_TEXTURE_SWIZZLE_A;
|
|
if (memcmp(sw, e->pub.swz, 4)) {
|
|
if (!s->has_view) memset(view, 0, sizeof *view);
|
|
memcpy(view->swz, sw, 4);
|
|
view->set_swizzle = 1;
|
|
*has_view = 1;
|
|
}
|
|
}
|
|
return e;
|
|
}
|
|
|
|
void vk_texture_lock(void) { lock(); }
|
|
void vk_texture_unlock(void) { unlock(); }
|
|
uint32_t vk_texture_slots_gen(void) { return slots_gen; }
|
|
|
|
uint64_t vk_texture_slot_token(int id) {
|
|
const uint64_t *si = __atomic_load_n(&sinfo, __ATOMIC_ACQUIRE);
|
|
if (!si || id < 0 || id >= (1 << 20)) return 0;
|
|
uint64_t v = __atomic_load_n(&si[id], __ATOMIC_ACQUIRE);
|
|
return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0;
|
|
}
|
|
uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); }
|
|
|
|
const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) {
|
|
lock();
|
|
const VkTexImage *r = vk_texture_by_id_locked(id, view, has_view);
|
|
unlock();
|
|
return r;
|
|
}
|
|
|
|
const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
|
|
if (id < 0 || id >= ntslots || !tslots[id].used) return NULL;
|
|
TexSlot *s = &tslots[id];
|
|
uint32_t g = agen[sinfo[id] & 0xffff];
|
|
if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the
|
|
view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may
|
|
int hv; // succeed later
|
|
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it
|
|
s->cview = v, s->chas_view = (uint8_t)hv;
|
|
s->cgen = g;
|
|
s->cepoch = 0;
|
|
}
|
|
Tex *e = s->ce;
|
|
// Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a
|
|
// replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not
|
|
// mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do.
|
|
if (e && s->cepoch != submit_epoch) {
|
|
if (!e->uploaded || e->checked != submit_epoch) {
|
|
make_fresh_snap(e);
|
|
e->checked = submit_epoch;
|
|
}
|
|
s->cepoch = submit_epoch;
|
|
}
|
|
if (view) *view = has_view ? s->cview : s->v;
|
|
if (has_view) *has_view = view ? s->chas_view : s->has_view;
|
|
return e ? &e->pub : NULL;
|
|
}
|
|
|
|
// vk_dump: the swizzle selectors of the descriptor registered in slot id; 0 when the slot is empty.
|
|
int vk_texture_slot_swz(int id, uint8_t swz[4]) {
|
|
lock();
|
|
int ok = id >= 0 && id < ntslots && tslots[id].used;
|
|
if (ok) {
|
|
const nvn_texdesc *d = &tslots[id].t.d;
|
|
swz[0] = (uint8_t)d->swizzle_r, swz[1] = (uint8_t)d->swizzle_g, swz[2] = (uint8_t)d->swizzle_b;
|
|
swz[3] = (uint8_t)d->swizzle_a;
|
|
}
|
|
unlock();
|
|
return ok;
|
|
}
|
|
|
|
uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); }
|
|
|
|
void vk_texture_new_submit(void) {
|
|
lock();
|
|
if (!++submit_epoch) submit_epoch = 1;
|
|
sweep_pressure();
|
|
static uint64_t log_at;
|
|
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) / 60;
|
|
if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) {
|
|
log_at = f;
|
|
rt_log("perf: frame %llu: texture aliases: content copied %llu, filled from storage %llu, left uninitialised %llu",
|
|
(unsigned long long)f * 60, (unsigned long long)n_alias_copied, (unsigned long long)n_alias_upload,
|
|
(unsigned long long)n_alias_uninit);
|
|
n_alias_copied = n_alias_upload = n_alias_uninit = 0;
|
|
rt_log("perf: frame %llu: texture re-uploads %llu (%.1f MiB, %.2f ms), after WriteTexels %llu (%.1f MiB, %.2f ms)",
|
|
(unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6,
|
|
(unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6);
|
|
ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = 0;
|
|
uint64_t ck, calls, untr;
|
|
vk_mem_snap_stats(&ck, &calls, &untr);
|
|
rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu",
|
|
(unsigned long long)f * 60, (unsigned long long)ck, (unsigned long long)n_checks, (unsigned long long)calls,
|
|
(unsigned long long)untr);
|
|
n_checks = 0;
|
|
rt_log("perf: frame %llu: released textures kept %d, dropped under VRAM pressure %llu", (unsigned long long)f * 60,
|
|
nreleased, (unsigned long long)n_pressure_drops);
|
|
n_pressure_drops = 0;
|
|
}
|
|
unlock();
|
|
}
|
|
|
|
// ---------------------------------------------------------------- texel reads and writes
|
|
typedef struct { unsigned level, layer0; int nz, is3d, bw, bh, bpb; size_t wb, hb, row, img, x0, y0; } Region;
|
|
|
|
static int region_of(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img, Region *g) {
|
|
g->bpb = nvn_fmt_bpb(t->d.format, &g->bw, &g->bh);
|
|
if (!g->bpb || r->width <= 0 || r->height <= 0 || r->depth < 0 || r->xoffset < 0 || r->yoffset < 0 || r->zoffset < 0 ||
|
|
r->xoffset % g->bw || r->yoffset % g->bh)
|
|
return 0;
|
|
g->level = v && v->set_levels ? v->base_level : 0;
|
|
g->layer0 = v && v->set_layers ? v->min_layer : 0;
|
|
g->nz = r->depth ? r->depth : 1;
|
|
g->is3d = t->d.target == NVN_TEXTURE_TARGET_3D;
|
|
g->wb = (size_t)((r->width + g->bw - 1) / g->bw) * (size_t)g->bpb;
|
|
g->hb = (size_t)(r->height + g->bh - 1) / (size_t)g->bh;
|
|
g->row = row ? row : g->wb;
|
|
g->img = img ? img : g->row * g->hb;
|
|
g->x0 = (size_t)(r->xoffset / g->bw) * (size_t)g->bpb;
|
|
g->y0 = (size_t)(r->yoffset / g->bh);
|
|
return 1;
|
|
}
|
|
|
|
// Pool bytes <-> caller rows of the region alone (nvn_texels_box). MacNativeGames' nvn_mtl.m goes through a
|
|
// linear copy of the whole level; for a small region of a large texture that cost tens of milliseconds while the guest
|
|
// thread held its virtual core and the caller the texture lock (monsters2 replay, frames 765-790).
|
|
static int storage_texels(const tex_t *t, const Region *g, const NVNcopyRegion *r, uint8_t *mem, int write) {
|
|
for (int z = 0; z < g->nz; z++) {
|
|
unsigned layer = g->is3d ? 0 : g->layer0 + (unsigned)r->zoffset + (unsigned)z;
|
|
uint32_t slice = g->is3d ? (uint32_t)(r->zoffset + z) : 0;
|
|
if (nvn_texels_box(&t->d, g->level, layer, (uint32_t)g->x0, (uint32_t)g->y0, slice, (uint32_t)g->wb,
|
|
(uint32_t)g->hb, t->mem, mem + (size_t)z * g->img, g->row, !write))
|
|
return 0;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
static int read_aspect(const Tex *e, VkImageAspectFlags *a) {
|
|
if (!e->f.depth && !e->f.stencil) { *a = VK_IMAGE_ASPECT_COLOR_BIT; return 1; }
|
|
switch (e->t.d.format) { // packed depth-stencil storage has no single-aspect copy
|
|
case 0x31: *a = VK_IMAGE_ASPECT_STENCIL_BIT; return 1;
|
|
case 0x32: case 0x34: *a = VK_IMAGE_ASPECT_DEPTH_BIT; return 1;
|
|
default: return 0;
|
|
}
|
|
}
|
|
|
|
// Image <-> caller rows through a staging buffer; the GPU copy is waited for.
|
|
static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t *mem, int write) {
|
|
static volatile long n_astc, n_ds;
|
|
VkCtx *c = vk_ctx();
|
|
VkImageAspectFlags aspect;
|
|
if (e->f.astc_bw && !write) {
|
|
log_limited(&n_astc, "ReadTexels of ASTC texture %#llx: the image holds decoded RGBA8; zeros returned",
|
|
(unsigned long long)e->pub.addr);
|
|
return 0;
|
|
}
|
|
if (!read_aspect(e, &aspect)) {
|
|
log_limited(&n_ds, "texels of packed depth-stencil format %#x are not copied", e->t.d.format);
|
|
return 0;
|
|
}
|
|
if (g->level >= e->pub.levels) return 0;
|
|
uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height;
|
|
int rgb = e->f.expand_rgb, astc = e->f.astc_bw != 0, agpu = astc && write && vk_astc_gpu();
|
|
size_t texels = (size_t)w * h, ib = agpu ? 0 : astc ? 4 : rgb ? (size_t)g->bpb / 3 * 4 : 0;
|
|
size_t slice = ib ? texels * ib : g->wb * g->hb; // agpu: the raw blocks, decoded by vk_astc_record
|
|
VkBuffer sb;
|
|
VmaAllocation sa;
|
|
uint8_t *stg = vk_staging(slice * (size_t)g->nz, &sb, &sa);
|
|
VkBufferImageCopy *rg = calloc((size_t)g->nz, sizeof *rg);
|
|
for (int z = 0; z < g->nz; z++) {
|
|
uint8_t *s = stg + slice * (size_t)z;
|
|
const uint8_t *m = mem + (size_t)z * g->img;
|
|
if (write) {
|
|
if (agpu) {
|
|
for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb);
|
|
} else if (astc) {
|
|
uint8_t *tight = malloc(g->wb * g->hb);
|
|
for (size_t y = 0; y < g->hb; y++) memcpy(tight + y * g->wb, m + y * g->row, g->wb);
|
|
uint64_t a0 = perf_begin();
|
|
astc_decode_ldr_mt(tight, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, s);
|
|
perf_end(PERF_ASTC, a0);
|
|
free(tight);
|
|
} else if (rgb) {
|
|
for (size_t y = 0; y < g->hb; y++) expand_rgb(e->t.d.format, m + y * g->row, s + y * w * ib, w);
|
|
} else {
|
|
for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb);
|
|
}
|
|
}
|
|
rg[z] = (VkBufferImageCopy){.bufferOffset = slice * (size_t)z,
|
|
.imageSubresource = {aspect, g->level, g->is3d ? 0 : g->layer0 + (uint32_t)r->zoffset + (uint32_t)z, 1},
|
|
.imageOffset = {r->xoffset, r->yoffset, g->is3d ? r->zoffset + z : 0},
|
|
.imageExtent = {w, h, 1}};
|
|
}
|
|
VkCommandBuffer cb = vk_upload_begin();
|
|
if (write) {
|
|
vmaFlushAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
|
|
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
|
|
int done = 0;
|
|
// A BC3 image takes whole 4x4 blocks, or a part that ends on the mip's edge; a region of a 5x5 or 6x6 footprint
|
|
// may end inside a block (vk_astc_record checks the start corner).
|
|
uint32_t mw = e->t.d.width >> g->level, mh = e->t.d.height >> g->level;
|
|
int bc_cut = e->f.astc_bc && (((w & 3) && (uint32_t)r->xoffset + w != (mw ? mw : 1)) ||
|
|
((h & 3) && (uint32_t)r->yoffset + h != (mh ? mh : 1)));
|
|
if (agpu && !bc_cut) { // bc_cut: nothing recorded, logged below as not decoded
|
|
VkAstcRegion *ar = calloc((size_t)g->nz, sizeof *ar);
|
|
for (int z = 0; z < g->nz; z++)
|
|
ar[z] = (VkAstcRegion){.src = slice * (size_t)z, .w = w, .h = h, .slices = 1, .level = rg[z].imageSubresource.mipLevel,
|
|
.layer = rg[z].imageSubresource.baseArrayLayer, .offset = rg[z].imageOffset};
|
|
uint64_t a0 = perf_begin();
|
|
done = vk_astc_record(cb, sb, slice * (size_t)g->nz, ar, g->nz, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc,
|
|
e->pub.image, aspect, VK_NULL_HANDLE);
|
|
perf_end(PERF_ASTC, a0);
|
|
free(ar);
|
|
}
|
|
if (!agpu) vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)g->nz, rg);
|
|
else if (!done) log_limited(&n_astc, "WriteTexels of ASTC texture %#llx not decoded on the GPU: left as it was",
|
|
(unsigned long long)e->pub.addr);
|
|
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
|
|
} else {
|
|
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
|
|
vkCmdCopyImageToBuffer(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, sb, (uint32_t)g->nz, rg);
|
|
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
|
|
VK_ACCESS_2_HOST_READ_BIT);
|
|
}
|
|
vk_upload_end();
|
|
vk_upload_flush(1);
|
|
if (!write) {
|
|
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
|
|
for (int z = 0; z < g->nz; z++) {
|
|
const uint8_t *s = stg + slice * (size_t)z;
|
|
uint8_t *m = mem + (size_t)z * g->img;
|
|
for (size_t y = 0; y < g->hb; y++) {
|
|
if (rgb) shrink_rgb(e->t.d.format, s + y * w * ib, m + y * g->row, w);
|
|
else memcpy(m + y * g->row, s + y * g->wb, g->wb);
|
|
}
|
|
}
|
|
}
|
|
free(rg);
|
|
vmaDestroyBuffer(c->vma, sb, sa);
|
|
return 1;
|
|
}
|
|
|
|
static void zero_rows(const Region *g, uint8_t *dst) {
|
|
for (int z = 0; z < g->nz; z++)
|
|
for (size_t y = 0; y < g->hb; y++) memset(dst + (size_t)z * g->img + y * g->row, 0, g->wb);
|
|
}
|
|
|
|
int vk_texture_read_image(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) {
|
|
Region g;
|
|
if (!t || !region_of(t, v, r, row, img, &g)) return 0;
|
|
lock();
|
|
Tex *e = entry_of(t);
|
|
int ok = 0;
|
|
if (e) {
|
|
make_fresh(e);
|
|
ok = image_texels(e, &g, r, dst, 0);
|
|
}
|
|
unlock();
|
|
if (!ok) zero_rows(&g, dst);
|
|
return ok;
|
|
}
|
|
|
|
// ReadTexels: the pool bytes are the content of a mirrored texture the GPU has not rendered into; every other texture
|
|
// is read back from its image.
|
|
void vk_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) {
|
|
static volatile long n_bad;
|
|
Region g;
|
|
if (!region_of(t, v, r, row, img, &g)) {
|
|
log_limited(&n_bad, "ReadTexels: invalid region %d,%d,%d %dx%dx%d (format %#x)", r->xoffset, r->yoffset,
|
|
r->zoffset, r->width, r->height, r->depth, t->d.format);
|
|
return;
|
|
}
|
|
lock();
|
|
Tex *e = entry_of(t);
|
|
int ok = 0;
|
|
if (e && e->pub.mirrored && !e->pub.gpu_owned) ok = storage_texels(t, &g, r, dst, 0);
|
|
else if (e) {
|
|
make_fresh(e);
|
|
ok = image_texels(e, &g, r, dst, 0);
|
|
}
|
|
unlock();
|
|
if (!ok) zero_rows(&g, dst);
|
|
}
|
|
|
|
// WriteTexels: a mirrored texture gets the texels in its pool bytes (and in its image once the GPU owns it); every
|
|
// other texture in its image.
|
|
void vk_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row, uint64_t img) {
|
|
static volatile long n_bad;
|
|
Region g;
|
|
if (!region_of(t, v, r, row, img, &g) || (v && v->set_format && v->format != t->d.format)) {
|
|
log_limited(&n_bad, "WriteTexels: invalid region %d,%d,%d %dx%dx%d or view format (format %#x)", r->xoffset,
|
|
r->yoffset, r->zoffset, r->width, r->height, r->depth, t->d.format);
|
|
return;
|
|
}
|
|
lock();
|
|
Tex *e = entry_of(t);
|
|
if (e && e->pub.mirrored) {
|
|
if (!storage_texels(t, &g, r, (uint8_t *)src, 1))
|
|
log_limited(&n_bad, "WriteTexels: region outside level %u of texture %#llx", g.level, (unsigned long long)e->pub.addr);
|
|
vk_guest_written((uintptr_t)t->mem, t->L.size);
|
|
if (e->pub.gpu_owned) image_texels(e, &g, r, (uint8_t *)src, 1);
|
|
else if (e->uploaded) e->wt_n++;
|
|
} else if (e) {
|
|
image_texels(e, &g, r, (uint8_t *)src, 1);
|
|
}
|
|
unlock();
|
|
}
|
|
|
|
// ---------------------------------------------------------------- CPU reads of GPU-written storage
|
|
// A mirrored texture the GPU rendered into holds its texels in its image only; its storage keeps what the CPU last put
|
|
// there. The game may still read that storage with the CPU: TotK moves its generated grass maps between pools with
|
|
// memcpy once the GPU is done, and the copy then carried zeros. Such a read writes the image back first.
|
|
static void write_back(Tex *e) {
|
|
const nvn_texdesc *d = &e->t.d;
|
|
int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D;
|
|
uint32_t levels = linear ? 1 : e->pub.levels, layers = linear || is3d ? 1 : e->pub.layers;
|
|
for (uint32_t lv = 0; lv < levels; lv++)
|
|
for (uint32_t ly = 0; ly < layers; ly++) {
|
|
view_t v = {.set_levels = 1, .set_layers = 1, .base_level = (uint8_t)lv, .min_layer = (uint16_t)ly};
|
|
NVNcopyRegion r = {.width = (int)mip(d->width, lv),
|
|
.height = d->target == 0 || d->target == 3 ? 1 : (int)mip(d->height, lv),
|
|
.depth = is3d ? (int)mip(d->depth, lv) : 1};
|
|
Region g;
|
|
if (!region_of(&e->t, &v, &r, 0, 0, &g)) continue;
|
|
uint8_t *buf = malloc(g.img * (size_t)g.nz);
|
|
if (!buf) rt_fatal("vk: out of memory for a texture write-back");
|
|
if (image_texels(e, &g, &r, buf, 0)) storage_texels(&e->t, &g, &r, buf, 1);
|
|
free(buf);
|
|
}
|
|
e->gpu_dirty = 0;
|
|
}
|
|
|
|
// Mirrored textures overlapping [addr, addr+size) that the GPU wrote since their last write-back.
|
|
static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) {
|
|
int n = 0;
|
|
for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) {
|
|
Tex *e = owned[i];
|
|
if (__atomic_load_n(&e->gpu_dirty, __ATOMIC_RELAXED) && e->pub.addr + e->t.L.size > addr) {
|
|
if (!hit) return 1;
|
|
hit[n++] = e;
|
|
}
|
|
}
|
|
return n;
|
|
}
|
|
|
|
// A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c
|
|
// tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels.
|
|
void vk_texture_storage_bound_locked(const VkTexImage *img) {
|
|
Tex *e = (Tex *)img; // pub is the first member
|
|
uint64_t f = vk_exec_recording_fence();
|
|
if (f > e->wr_fence) e->wr_fence = f;
|
|
}
|
|
|
|
uint64_t vk_texture_last_write(const tex_t *t) {
|
|
lock();
|
|
Tex *e = entry_of(t);
|
|
uint64_t f = e ? e->wr_fence : 0;
|
|
unlock();
|
|
return f;
|
|
}
|
|
|
|
size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) {
|
|
Region g;
|
|
if (!region_of(t, v, r, row, img, &g)) return 0;
|
|
return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb;
|
|
}
|
|
|
|
int vk_texture_cpu_stale(uint64_t addr, uint64_t size) {
|
|
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
|
|
AcquireSRWLockShared(&owned_lock);
|
|
int dirty = owned_dirty(addr, size, NULL);
|
|
ReleaseSRWLockShared(&owned_lock);
|
|
return dirty;
|
|
}
|
|
|
|
// The newest submission recorded to write a GPU-written mirrored texture overlapping [addr, addr+size); 0: none.
|
|
uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size) {
|
|
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
|
|
lock();
|
|
uint64_t f = 0;
|
|
for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) {
|
|
const Tex *e = owned[i];
|
|
if (!e->gpu_dirty || e->pub.addr + e->t.L.size <= addr) continue;
|
|
if (e->gpu_fence > f) f = e->gpu_fence;
|
|
if (e->wr_fence > f) f = e->wr_fence;
|
|
}
|
|
unlock();
|
|
return f;
|
|
}
|
|
|
|
// The image is the newest content only while nothing wrote the storage since the GPU render: TotK finalizes grass maps,
|
|
// reuses their storage for new data and memcpys that data on, and writing the old image back first handed the copy
|
|
// a stale map or stale resource bytes (wrong textures in the distance). Every hit is checked before any is written
|
|
// back, since a write-back is itself a CPU write to storage another hit may share; the hits then go back oldest render
|
|
// first, so where two descriptors share storage the newer render wins.
|
|
void vk_texture_write_back(uint64_t addr, uint64_t size) {
|
|
lock();
|
|
Tex **hit = malloc(sizeof *hit * (size_t)(nowned + 1));
|
|
if (!hit) rt_fatal("vk: out of memory for a texture write-back");
|
|
int n = owned_dirty(addr, size, hit), k = 0; // owned[] cannot change while tex_lock is held
|
|
for (int i = 0; i < n; i++) {
|
|
uint32_t now;
|
|
if (storage_rewritten(hit[i], hit[i]->gpu_gen, &now) == 1) hit[i]->gpu_dirty = 0;
|
|
else hit[k++] = hit[i];
|
|
}
|
|
for (int i = 1; i < k; i++)
|
|
for (int j = i; j > 0 && hit[j - 1]->gpu_seq > hit[j]->gpu_seq; j--) {
|
|
Tex *x = hit[j];
|
|
hit[j] = hit[j - 1], hit[j - 1] = x;
|
|
}
|
|
for (int i = 0; i < k; i++) write_back(hit[i]);
|
|
for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB);
|
|
for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP);
|
|
free(hit);
|
|
unlock();
|
|
}
|
|
|
|
// TotK's grass system renders a 128x128 RGBA8 map of cut and trampled grass into a CPU-cached pool every frame and its
|
|
// CPU code reads the pool bytes without nvnMemoryPoolInvalidateMappedRange (cut reactions: blades, insects, the cut
|
|
// itself). On the Switch the bytes are there once the frame's fence has passed. So when the
|
|
// CPU waits for a fence, mirrored targets up to wb_limit bytes rendered by work up to that fence go back to their
|
|
// storage, unless the CPU rewrote it since. Larger targets still go back on an invalidate, memcpy or ReadTexels.
|
|
int vk_texture_small_dirty(void) { return __atomic_load_n(&small_pending, __ATOMIC_ACQUIRE); }
|
|
|
|
// TOTK_TRACE=gfx: a dirty target the fence write-back leaves alone because it is over the limit (what a CPU read
|
|
// after the wait would miss), or sends back although it is over the default limit (what a raised limit changed). One
|
|
// line per target address and outcome; the table is small and a full one stops logging.
|
|
static void wb_note(const Tex *e, int skipped) {
|
|
static uint64_t seen[2][64];
|
|
static int n_seen[2];
|
|
for (int i = 0; i < n_seen[skipped]; i++)
|
|
if (seen[skipped][i] == e->pub.addr) return;
|
|
if (n_seen[skipped] == 64) return;
|
|
seen[skipped][n_seen[skipped]++] = e->pub.addr;
|
|
const pool_t *p = e->t.pool;
|
|
unsigned fl = p ? p->flags : 0;
|
|
rt_log("vk: fence write-back %s %#llx %ux%u fmt %#x size %llu pool %s (limit %llu)",
|
|
skipped ? "skipped-for-size" : "over-default", (unsigned long long)e->pub.addr, e->t.d.width, e->t.d.height,
|
|
e->t.d.format, (unsigned long long)e->t.L.size,
|
|
!p ? "none" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_CACHED) ? "cpu-cached" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED) ? "cpu-uncached" : "other",
|
|
(unsigned long long)wb_limit);
|
|
}
|
|
|
|
void vk_texture_write_back_small(uint64_t fence) {
|
|
lock();
|
|
wb_cfg_init();
|
|
__atomic_store_n(&small_pending, 0, __ATOMIC_RELAXED);
|
|
AcquireSRWLockShared(&owned_lock);
|
|
int n = 0, later = 0;
|
|
for (int i = 0; i < nowned; i++) n += owned[i]->gpu_dirty && owned[i]->t.L.size <= wb_limit;
|
|
Tex **hit = n ? malloc(sizeof *hit * (size_t)n) : NULL;
|
|
if (n && !hit) rt_fatal("vk: out of memory for a texture write-back");
|
|
n = 0;
|
|
for (int i = 0; i < nowned; i++) {
|
|
Tex *e = owned[i];
|
|
if (!e->gpu_dirty) continue;
|
|
if (e->t.L.size > wb_limit) {
|
|
if (wb_trace && e->gpu_fence <= fence) {
|
|
wb_note(e, 1);
|
|
perf_count(PERF_CNT_WB_SKIP);
|
|
}
|
|
continue;
|
|
}
|
|
if (e->gpu_fence > fence) later = 1;
|
|
else {
|
|
if (wb_trace && e->t.L.size > WB_SMALL_DEFAULT) wb_note(e, 0);
|
|
hit[n++] = e;
|
|
}
|
|
}
|
|
ReleaseSRWLockShared(&owned_lock);
|
|
if (later) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE);
|
|
int k = 0;
|
|
for (int i = 0; i < n; i++) {
|
|
uint32_t now;
|
|
if (storage_rewritten(hit[i], hit[i]->gpu_gen, &now) == 1) hit[i]->gpu_dirty = 0;
|
|
else hit[k++] = hit[i];
|
|
}
|
|
for (int i = 1; i < k; i++)
|
|
for (int j = i; j > 0 && hit[j - 1]->gpu_seq > hit[j]->gpu_seq; j--) {
|
|
Tex *x = hit[j];
|
|
hit[j] = hit[j - 1], hit[j - 1] = x;
|
|
}
|
|
for (int i = 0; i < k; i++) write_back(hit[i]);
|
|
for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB);
|
|
for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP);
|
|
free(hit);
|
|
unlock();
|
|
}
|
|
|
|
// ---------------------------------------------------------------- samplers (LAS textures.c)
|
|
static VkCompareOp compare_op(uint32_t f) {
|
|
switch (f) {
|
|
case NVN_COMPARE_FUNC_NEVER: return VK_COMPARE_OP_NEVER;
|
|
case NVN_COMPARE_FUNC_LESS: return VK_COMPARE_OP_LESS;
|
|
case NVN_COMPARE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL;
|
|
case NVN_COMPARE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL;
|
|
case NVN_COMPARE_FUNC_GREATER: return VK_COMPARE_OP_GREATER;
|
|
case NVN_COMPARE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL;
|
|
case NVN_COMPARE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL;
|
|
default: return VK_COMPARE_OP_ALWAYS;
|
|
}
|
|
}
|
|
|
|
static struct { int init, mirror_clamp_edge, minmax, border_no_format; } sf;
|
|
|
|
static void sampler_features(VkCtx *c) {
|
|
if (sf.init) return;
|
|
VkPhysicalDeviceCustomBorderColorFeaturesEXT cbf = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT};
|
|
VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES,
|
|
.pNext = c->has_custom_border_color ? &cbf : NULL};
|
|
VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12};
|
|
vkGetPhysicalDeviceFeatures2(c->phys, &f2);
|
|
sf.mirror_clamp_edge = f12.samplerMirrorClampToEdge;
|
|
sf.minmax = f12.samplerFilterMinmax;
|
|
sf.border_no_format = c->has_custom_border_color && cbf.customBorderColors && cbf.customBorderColorWithoutFormat;
|
|
sf.init = 1;
|
|
}
|
|
|
|
static VkSampler make_sampler(VkCtx *c, const samp_par *d) {
|
|
sampler_features(c);
|
|
VkSamplerCreateInfo si = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO};
|
|
si.magFilter = d->magf == NVN_MAG_FILTER_LINEAR ? VK_FILTER_LINEAR : VK_FILTER_NEAREST;
|
|
uint32_t mf = d->minf;
|
|
si.minFilter = mf == 1 || mf == 3 || mf == 5 ? VK_FILTER_LINEAR : VK_FILTER_NEAREST;
|
|
si.mipmapMode = mf == 4 || mf == 5 ? VK_SAMPLER_MIPMAP_MODE_LINEAR : VK_SAMPLER_MIPMAP_MODE_NEAREST;
|
|
VkSamplerAddressMode modes[3];
|
|
const uint8_t wrap[3] = {d->ws, d->wt, d->wr};
|
|
int border = 0;
|
|
for (int i = 0; i < 3; i++) {
|
|
switch (wrap[i]) {
|
|
case NVN_WRAP_MODE_CLAMP: case NVN_WRAP_MODE_CLAMP_TO_EDGE: modes[i] = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; break;
|
|
case NVN_WRAP_MODE_MIRROR_CLAMP: case NVN_WRAP_MODE_MIRROR_CLAMP_TO_EDGE:
|
|
modes[i] = sf.mirror_clamp_edge ? VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE : VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
|
|
break;
|
|
case NVN_WRAP_MODE_MIRROR_CLAMP_TO_BORDER: case NVN_WRAP_MODE_CLAMP_TO_BORDER:
|
|
modes[i] = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
|
|
border = 1;
|
|
break;
|
|
case NVN_WRAP_MODE_MIRRORED_REPEAT: modes[i] = VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT; break;
|
|
default: modes[i] = VK_SAMPLER_ADDRESS_MODE_REPEAT;
|
|
}
|
|
}
|
|
si.addressModeU = modes[0], si.addressModeV = modes[1], si.addressModeW = modes[2];
|
|
si.minLod = d->lod_min;
|
|
si.maxLod = d->lod_max > 1000 ? VK_LOD_CLAMP_NONE : d->lod_max;
|
|
if (mf <= NVN_MIN_FILTER_LINEAR) si.maxLod = si.minLod + 0.25f; // not mipmapped: stay on the base level
|
|
if (si.maxLod < si.minLod) si.maxLod = si.minLod;
|
|
float lim = c->props.limits.maxSamplerLodBias;
|
|
si.mipLodBias = fmaxf(-lim, fminf(lim, d->lod_bias));
|
|
if (d->aniso > 1) {
|
|
si.anisotropyEnable = VK_TRUE;
|
|
si.maxAnisotropy = fminf(d->aniso, c->props.limits.maxSamplerAnisotropy);
|
|
}
|
|
if (d->cmp_mode) {
|
|
si.compareEnable = VK_TRUE;
|
|
si.compareOp = compare_op(d->cmp_func);
|
|
}
|
|
VkSamplerCustomBorderColorCreateInfoEXT cb = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT};
|
|
VkSamplerReductionModeCreateInfo rm = {.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO};
|
|
const void **tail = &si.pNext;
|
|
if (border) {
|
|
const float *b = d->border;
|
|
if (b[0] == 0 && b[1] == 0 && b[2] == 0 && b[3] == 0) si.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
|
|
else if (b[0] == 0 && b[1] == 0 && b[2] == 0 && b[3] == 1) si.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK;
|
|
else if (b[0] == 1 && b[1] == 1 && b[2] == 1 && b[3] == 1) si.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
|
|
else if (sf.border_no_format) {
|
|
memcpy(cb.customBorderColor.float32, b, 16);
|
|
si.borderColor = VK_BORDER_COLOR_FLOAT_CUSTOM_EXT;
|
|
*tail = &cb;
|
|
tail = &cb.pNext;
|
|
} else si.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
|
|
}
|
|
if (d->reduction && sf.minmax && !si.compareEnable) {
|
|
rm.reductionMode = d->reduction == NVN_SAMPLER_REDUCTION_MIN ? VK_SAMPLER_REDUCTION_MODE_MIN : VK_SAMPLER_REDUCTION_MODE_MAX;
|
|
*tail = &rm;
|
|
}
|
|
VkSampler s;
|
|
VkResult r = vkCreateSampler(c->device, &si, NULL, &s);
|
|
if (r != VK_SUCCESS) rt_fatal("vk: vkCreateSampler failed (%d)", (int)r);
|
|
return s;
|
|
}
|
|
|
|
void vk_sampler_register(int id, const samp_par *s) {
|
|
if (id < 0 || id >= (1 << 12)) rt_fatal("vk: sampler slot %d out of range", id);
|
|
lock();
|
|
if (id >= nsslots) {
|
|
int n = nsslots ? nsslots : 256;
|
|
while (n <= id) n *= 2;
|
|
SampSlot *p = realloc(sslots, sizeof *p * (size_t)n);
|
|
if (!p) rt_fatal("vk: out of memory for sampler slots");
|
|
memset(p + nsslots, 0, sizeof *p * (size_t)(n - nsslots));
|
|
sslots = p, nsslots = n;
|
|
}
|
|
sslots[id].p = *s;
|
|
sslots[id].s = sslots[id].sp = VK_NULL_HANDLE;
|
|
sslots[id].used = 1;
|
|
samp_gen++;
|
|
unlock();
|
|
}
|
|
|
|
// Caller holds the lock.
|
|
static VkSampler sampler_for(VkCtx *c, const samp_par *s) {
|
|
samp_par k;
|
|
memset(&k, 0, sizeof k);
|
|
k = *s;
|
|
VkSampler v = (VkSampler)nvn_hm_get(&samplers, &k);
|
|
if (!v) {
|
|
v = make_sampler(c, &k);
|
|
nvn_hm_put(&samplers, &k, (void *)v);
|
|
}
|
|
return v;
|
|
}
|
|
|
|
VkSampler vk_sampler_for(const samp_par *s) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || !s) return VK_NULL_HANDLE;
|
|
lock();
|
|
VkSampler v = sampler_for(c, s);
|
|
unlock();
|
|
return v;
|
|
}
|
|
|
|
// Samplers live until shutdown, so a slot keeps the one its parameters map to until it is registered again.
|
|
VkSampler vk_sampler_by_id_locked(int id) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return VK_NULL_HANDLE;
|
|
VkSampler v = VK_NULL_HANDLE;
|
|
if (id >= 0 && id < nsslots && sslots[id].used) {
|
|
SampSlot *s = &sslots[id];
|
|
if (!s->s) s->s = sampler_for(c, &s->p);
|
|
v = s->s;
|
|
}
|
|
return v;
|
|
}
|
|
|
|
VkSampler vk_sampler_point_by_id_locked(int id) {
|
|
VkCtx *c = vk_ctx();
|
|
if (!c || id < 0 || id >= nsslots || !sslots[id].used) return VK_NULL_HANDLE;
|
|
SampSlot *s = &sslots[id];
|
|
if (!s->sp) {
|
|
samp_par k = s->p;
|
|
k.magf = NVN_MAG_FILTER_NEAREST;
|
|
k.minf = k.minf == NVN_MIN_FILTER_LINEAR ? NVN_MIN_FILTER_NEAREST :
|
|
k.minf > NVN_MIN_FILTER_LINEAR ? NVN_MIN_FILTER_NEAREST_MIPMAP_NEAREST : k.minf;
|
|
k.aniso = 1;
|
|
s->sp = sampler_for(c, &k);
|
|
}
|
|
return s->sp;
|
|
}
|
|
|
|
VkSampler vk_sampler_by_id(int id) {
|
|
lock();
|
|
VkSampler v = vk_sampler_by_id_locked(id);
|
|
unlock();
|
|
return v;
|
|
}
|
|
|
|
int vk_sampler_params(int id, samp_par *out) {
|
|
lock();
|
|
int ok = id >= 0 && id < nsslots && sslots[id].used;
|
|
if (ok) *out = sslots[id].p;
|
|
unlock();
|
|
return ok;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- statistics and shutdown
|
|
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) {
|
|
lock();
|
|
if (images) *images = by_key.n;
|
|
if (uploads) *uploads = n_uploads;
|
|
if (reuploads) *reuploads = n_reuploads;
|
|
unlock();
|
|
}
|
|
|
|
void vk_texture_census(VkTexCensus *s) {
|
|
VkCtx *c = vk_ctx();
|
|
memset(s, 0, sizeof *s);
|
|
lock();
|
|
for (uint32_t i = 0; i < by_key.cap; i++) {
|
|
const Tex *e = by_key.vals[i];
|
|
if (!e) continue;
|
|
uint64_t bytes = 0;
|
|
if (e->alloc && c) {
|
|
VmaAllocationInfo ai;
|
|
vmaGetAllocationInfo(c->vma, e->alloc, &ai);
|
|
bytes = ai.size;
|
|
}
|
|
if (e->released) s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes;
|
|
else s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes;
|
|
}
|
|
unlock();
|
|
}
|
|
|
|
void vk_texture_shutdown(void) {
|
|
VkCtx *c = vk_ctx();
|
|
lock();
|
|
Tex **list = malloc(sizeof *list * (size_t)(by_key.n + 1));
|
|
int n = 0;
|
|
for (uint32_t i = 0; i < by_key.cap; i++)
|
|
if (by_key.vals[i]) list[n++] = by_key.vals[i];
|
|
for (int i = 0; i < n; i++) destroy_tex(list[i]);
|
|
free(list);
|
|
for (uint32_t i = 0; i < samplers.cap; i++)
|
|
if (samplers.vals[i] && c) vkDestroySampler(c->device, (VkSampler)samplers.vals[i], NULL);
|
|
free(by_key.keys), free(by_key.vals), free(by_addr.keys), free(by_addr.vals), free(samplers.keys), free(samplers.vals);
|
|
by_key = (nvn_hmap){.klen = sizeof(TexKey)};
|
|
by_addr = (nvn_hmap){.klen = 8};
|
|
samplers = (nvn_hmap){.klen = sizeof(samp_par)};
|
|
free(tslots), free(sslots);
|
|
tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0;
|
|
rel_head = rel_tail = NULL;
|
|
if (sinfo) memset(sinfo, 0, sizeof *sinfo << 20); // no slot is registered: tokens 0
|
|
unlock();
|
|
}
|