Files
DroidDeck/tools/venus/layer/bc.c
T
The412BannerandDersim Abbas 9796345987 Mali (Venus): David Cimo's Venus route sources and recipes
From DavidCimo/droiddeck-mali, branch mali-venus @ 4f838891 (prototype 13; his 12 commits on top
of DroidDeck 0.3.1, GPL-3.0), by David Cimo:
- tools/venus: Mesa Venus vtest patch (incl. the per-queue submit_mutex fix and the cached-memory
  frame readback), virglrenderer Android patch, the Mali compatibility layer (the six features
  DXVK needs: fill modes, multi-viewport, robustness2, clip/cull distance, BC1-BC7) and his tools.
- tools/linuxfs/preload/venus.c: keeps Zink's kopper under Venus.
- tools/gamescope/patches/0127 (his 0114): shm buffers uploaded on every commit.
- docs/development/mali-venus.md: his design notes.
- tools/venus/ci-build.sh and build-venus.yml: his build recipes as one CI job that packs the
  component (vtest server, ICD, layer) as venus.tzst for the apk.

Co-authored-by: Dersim Abbas <derabb0620@gmail.com>
2026-10-12 23:41:52 -04:00

870 lines
34 KiB
C

/*
* BC1-BC7 textures on a GPU without them: each BC image is created in an uncompressed format, and
* every upload into one is decoded on the CPU while the copy is recorded. The decoded texels go
* through a staging buffer of the layer's own, which lives until the command buffer is reset.
* Uploads must come from memory the application has mapped, as DXVK's staging memory always is.
*/
#include "mali_compat.h"
#include <stdlib.h>
#include <string.h>
#define BCDEC_IMPLEMENTATION
#define BCDEC_STATIC
#define BCDEC_BC4BC5_PRECISE
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#include "bcdec.h"
#pragma GCC diagnostic pop
#define HANDLE(h) ((uint64_t)(uintptr_t)(h))
#define STAGING_BLOCK_SIZE (16u << 20)
#define STAGING_KEPT (64u << 20)
enum bc_kind { BC1, BC2, BC3, BC4, BC5, BC6H, BC7 };
struct bc_format {
VkFormat format;
VkFormat substitute;
enum bc_kind kind;
uint32_t block_bytes;
uint32_t texel_bytes;
bool is_signed;
/* BC1's RGB formats have no alpha; its 1-bit transparency must read as opaque. */
bool opaque;
};
static const struct bc_format bc_formats[] = {
{VK_FORMAT_BC1_RGB_UNORM_BLOCK, VK_FORMAT_R8G8B8A8_UNORM, BC1, 8, 4, false, true},
{VK_FORMAT_BC1_RGB_SRGB_BLOCK, VK_FORMAT_R8G8B8A8_SRGB, BC1, 8, 4, false, true},
{VK_FORMAT_BC1_RGBA_UNORM_BLOCK, VK_FORMAT_R8G8B8A8_UNORM, BC1, 8, 4, false, false},
{VK_FORMAT_BC1_RGBA_SRGB_BLOCK, VK_FORMAT_R8G8B8A8_SRGB, BC1, 8, 4, false, false},
{VK_FORMAT_BC2_UNORM_BLOCK, VK_FORMAT_R8G8B8A8_UNORM, BC2, 16, 4, false, false},
{VK_FORMAT_BC2_SRGB_BLOCK, VK_FORMAT_R8G8B8A8_SRGB, BC2, 16, 4, false, false},
{VK_FORMAT_BC3_UNORM_BLOCK, VK_FORMAT_R8G8B8A8_UNORM, BC3, 16, 4, false, false},
{VK_FORMAT_BC3_SRGB_BLOCK, VK_FORMAT_R8G8B8A8_SRGB, BC3, 16, 4, false, false},
{VK_FORMAT_BC4_UNORM_BLOCK, VK_FORMAT_R8_UNORM, BC4, 8, 1, false, false},
{VK_FORMAT_BC4_SNORM_BLOCK, VK_FORMAT_R8_SNORM, BC4, 8, 1, true, false},
{VK_FORMAT_BC5_UNORM_BLOCK, VK_FORMAT_R8G8_UNORM, BC5, 16, 2, false, false},
{VK_FORMAT_BC5_SNORM_BLOCK, VK_FORMAT_R8G8_SNORM, BC5, 16, 2, true, false},
{VK_FORMAT_BC6H_UFLOAT_BLOCK, VK_FORMAT_R16G16B16A16_SFLOAT, BC6H, 16, 8, false, false},
{VK_FORMAT_BC6H_SFLOAT_BLOCK, VK_FORMAT_R16G16B16A16_SFLOAT, BC6H, 16, 8, true, false},
{VK_FORMAT_BC7_UNORM_BLOCK, VK_FORMAT_R8G8B8A8_UNORM, BC7, 16, 4, false, false},
{VK_FORMAT_BC7_SRGB_BLOCK, VK_FORMAT_R8G8B8A8_SRGB, BC7, 16, 4, false, false},
};
static const struct bc_format *find_bc(VkFormat format)
{
for (size_t i = 0; i < sizeof(bc_formats) / sizeof(bc_formats[0]); i++) {
if (bc_formats[i].format == format)
return &bc_formats[i];
}
return NULL;
}
VkFormat bc_substitute(VkFormat format)
{
const struct bc_format *bc = find_bc(format);
return bc ? bc->substitute : VK_FORMAT_UNDEFINED;
}
/* What a real BC format allows: sampling and copies. */
#define BC_FEATURES \
(VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT | \
VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_MINMAX_BIT | VK_FORMAT_FEATURE_TRANSFER_SRC_BIT | \
VK_FORMAT_FEATURE_TRANSFER_DST_BIT | VK_FORMAT_FEATURE_BLIT_SRC_BIT)
#define BC_USAGE (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT)
static const struct bc_format *physical_bc(VkPhysicalDevice physical_device, VkFormat format, struct instance **inst)
{
*inst = instance_from_handle(physical_device);
if (!(physical_device_gaps(*inst, physical_device) & GAP_BC))
return NULL;
return find_bc(format);
}
VKAPI_ATTR void VKAPI_CALL GetPhysicalDeviceFormatProperties(VkPhysicalDevice physical_device, VkFormat format,
VkFormatProperties *properties)
{
struct instance *inst;
const struct bc_format *bc = physical_bc(physical_device, format, &inst);
inst->GetPhysicalDeviceFormatProperties(physical_device, bc ? bc->substitute : format, properties);
if (bc) {
properties->optimalTilingFeatures &= BC_FEATURES;
properties->linearTilingFeatures = 0;
properties->bufferFeatures = 0;
}
}
VKAPI_ATTR void VKAPI_CALL GetPhysicalDeviceFormatProperties2(VkPhysicalDevice physical_device, VkFormat format,
VkFormatProperties2 *properties)
{
struct instance *inst;
const struct bc_format *bc = physical_bc(physical_device, format, &inst);
inst->GetPhysicalDeviceFormatProperties2(physical_device, bc ? bc->substitute : format, properties);
if (!bc)
return;
properties->formatProperties.optimalTilingFeatures &= BC_FEATURES;
properties->formatProperties.linearTilingFeatures = 0;
properties->formatProperties.bufferFeatures = 0;
for (VkBaseOutStructure *s = properties->pNext; s; s = s->pNext) {
if (s->sType == VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3) {
VkFormatProperties3 *p3 = (VkFormatProperties3 *)s;
p3->optimalTilingFeatures &= (VkFormatFeatureFlags2)BC_FEATURES;
p3->linearTilingFeatures = 0;
p3->bufferFeatures = 0;
} else if (s->sType == VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT) {
((VkDrmFormatModifierPropertiesListEXT *)s)->drmFormatModifierCount = 0;
} else if (s->sType == VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_2_EXT) {
((VkDrmFormatModifierPropertiesList2EXT *)s)->drmFormatModifierCount = 0;
}
}
}
VKAPI_ATTR VkResult VKAPI_CALL GetPhysicalDeviceImageFormatProperties(
VkPhysicalDevice physical_device, VkFormat format, VkImageType type, VkImageTiling tiling,
VkImageUsageFlags usage, VkImageCreateFlags flags, VkImageFormatProperties *properties)
{
struct instance *inst;
const struct bc_format *bc = physical_bc(physical_device, format, &inst);
if (!bc)
return inst->GetPhysicalDeviceImageFormatProperties(physical_device, format, type, tiling, usage, flags,
properties);
if (tiling != VK_IMAGE_TILING_OPTIMAL || (usage & ~BC_USAGE))
return VK_ERROR_FORMAT_NOT_SUPPORTED;
return inst->GetPhysicalDeviceImageFormatProperties(physical_device, bc->substitute, type, tiling, usage,
flags & ~VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT,
properties);
}
/* A view format list naming BC formats, as one naming their substitutes. */
struct format_list {
VkImageFormatListCreateInfo *list;
const VkFormat *original;
VkFormat formats[16];
};
static void substitute_format_list(const void *chain, VkFormat substitute, struct format_list *saved)
{
saved->list = NULL;
for (const VkBaseInStructure *s = chain; s; s = s->pNext) {
if (s->sType != VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO)
continue;
VkImageFormatListCreateInfo *list = (VkImageFormatListCreateInfo *)s;
uint32_t count = 0;
saved->formats[count++] = substitute;
for (uint32_t i = 0; i < list->viewFormatCount && count < 16; i++) {
VkFormat format = bc_substitute(list->pViewFormats[i]);
/* Uncompressed views of the compressed blocks have nothing to alias any more. */
if (format == VK_FORMAT_UNDEFINED)
continue;
bool seen = false;
for (uint32_t k = 0; k < count; k++)
seen |= saved->formats[k] == format;
if (!seen)
saved->formats[count++] = format;
}
saved->list = list;
saved->original = list->pViewFormats;
list->pViewFormats = saved->formats;
list->viewFormatCount = count;
return;
}
}
static void restore_format_list(struct format_list *saved, uint32_t original_count)
{
if (saved->list) {
saved->list->pViewFormats = saved->original;
saved->list->viewFormatCount = original_count;
}
}
static uint32_t format_list_count(const void *chain)
{
for (const VkBaseInStructure *s = chain; s; s = s->pNext) {
if (s->sType == VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO)
return ((const VkImageFormatListCreateInfo *)s)->viewFormatCount;
}
return 0;
}
VKAPI_ATTR VkResult VKAPI_CALL GetPhysicalDeviceImageFormatProperties2(
VkPhysicalDevice physical_device, const VkPhysicalDeviceImageFormatInfo2 *info,
VkImageFormatProperties2 *properties)
{
struct instance *inst;
const struct bc_format *bc = physical_bc(physical_device, info->format, &inst);
if (!bc)
return inst->GetPhysicalDeviceImageFormatProperties2(physical_device, info, properties);
if (info->tiling != VK_IMAGE_TILING_OPTIMAL || (info->usage & ~BC_USAGE))
return VK_ERROR_FORMAT_NOT_SUPPORTED;
VkPhysicalDeviceImageFormatInfo2 copy = *info;
copy.format = bc->substitute;
copy.flags &= ~VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT;
uint32_t list_count = format_list_count(info->pNext);
struct format_list saved;
substitute_format_list(info->pNext, bc->substitute, &saved);
VkResult result = inst->GetPhysicalDeviceImageFormatProperties2(physical_device, &copy, properties);
restore_format_list(&saved, list_count);
return result;
}
struct bc_image {
const struct bc_format *format;
uint32_t array_layers;
};
static void substitute_image_info(const struct bc_format *bc, VkImageCreateInfo *info)
{
info->format = bc->substitute;
info->flags &= ~VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT;
info->usage &= BC_USAGE;
}
VKAPI_ATTR VkResult VKAPI_CALL CreateImage(VkDevice device, const VkImageCreateInfo *info,
const VkAllocationCallbacks *allocator, VkImage *image)
{
struct device *dev = device_from_handle(device);
const struct bc_format *bc = find_bc(info->format);
if (!bc)
return dev->CreateImage(device, info, allocator, image);
VkImageCreateInfo copy = *info;
substitute_image_info(bc, &copy);
uint32_t list_count = format_list_count(info->pNext);
struct format_list saved;
substitute_format_list(info->pNext, bc->substitute, &saved);
VkResult result = dev->CreateImage(device, &copy, allocator, image);
restore_format_list(&saved, list_count);
if (result != VK_SUCCESS)
return result;
struct bc_image *tracked = malloc(sizeof(*tracked));
tracked->format = bc;
tracked->array_layers = info->arrayLayers;
pthread_mutex_lock(&dev->lock);
map_put(&dev->images, HANDLE(*image), tracked);
pthread_mutex_unlock(&dev->lock);
return VK_SUCCESS;
}
VKAPI_ATTR void VKAPI_CALL DestroyImage(VkDevice device, VkImage image, const VkAllocationCallbacks *allocator)
{
struct device *dev = device_from_handle(device);
pthread_mutex_lock(&dev->lock);
free(map_remove(&dev->images, HANDLE(image)));
pthread_mutex_unlock(&dev->lock);
dev->DestroyImage(device, image, allocator);
}
static const struct bc_image *tracked_image(struct device *dev, VkImage image)
{
pthread_mutex_lock(&dev->lock);
const struct bc_image *tracked = map_get(&dev->images, HANDLE(image));
pthread_mutex_unlock(&dev->lock);
return tracked;
}
VKAPI_ATTR VkResult VKAPI_CALL CreateImageView(VkDevice device, const VkImageViewCreateInfo *info,
const VkAllocationCallbacks *allocator, VkImageView *view)
{
struct device *dev = device_from_handle(device);
const struct bc_image *tracked = tracked_image(dev, info->image);
if (!tracked)
return dev->CreateImageView(device, info, allocator, view);
VkImageViewCreateInfo copy = *info;
VkFormat format = bc_substitute(info->format);
copy.format = format != VK_FORMAT_UNDEFINED ? format : tracked->format->substitute;
return dev->CreateImageView(device, &copy, allocator, view);
}
VKAPI_ATTR void VKAPI_CALL GetDeviceImageMemoryRequirements(VkDevice device,
const VkDeviceImageMemoryRequirements *info,
VkMemoryRequirements2 *requirements)
{
struct device *dev = device_from_handle(device);
const struct bc_format *bc = find_bc(info->pCreateInfo->format);
if (!bc) {
dev->GetDeviceImageMemoryRequirements(device, info, requirements);
return;
}
VkImageCreateInfo image = *info->pCreateInfo;
substitute_image_info(bc, &image);
VkDeviceImageMemoryRequirements copy = *info;
copy.pCreateInfo = &image;
dev->GetDeviceImageMemoryRequirements(device, &copy, requirements);
}
VKAPI_ATTR void VKAPI_CALL GetDeviceImageSparseMemoryRequirements(
VkDevice device, const VkDeviceImageMemoryRequirements *info, uint32_t *count,
VkSparseImageMemoryRequirements2 *requirements)
{
struct device *dev = device_from_handle(device);
const struct bc_format *bc = find_bc(info->pCreateInfo->format);
if (!bc) {
dev->GetDeviceImageSparseMemoryRequirements(device, info, count, requirements);
return;
}
VkImageCreateInfo image = *info->pCreateInfo;
substitute_image_info(bc, &image);
VkDeviceImageMemoryRequirements copy = *info;
copy.pCreateInfo = &image;
dev->GetDeviceImageSparseMemoryRequirements(device, &copy, count, requirements);
}
VKAPI_ATTR void VKAPI_CALL GetDeviceImageSubresourceLayout(VkDevice device,
const VkDeviceImageSubresourceInfo *info,
VkSubresourceLayout2 *layout)
{
struct device *dev = device_from_handle(device);
const struct bc_format *bc = find_bc(info->pCreateInfo->format);
if (!bc) {
dev->GetDeviceImageSubresourceLayout(device, info, layout);
return;
}
VkImageCreateInfo image = *info->pCreateInfo;
substitute_image_info(bc, &image);
VkDeviceImageSubresourceInfo copy = *info;
copy.pCreateInfo = &image;
dev->GetDeviceImageSubresourceLayout(device, &copy, layout);
}
struct bound_buffer {
VkDeviceMemory memory;
VkDeviceSize offset;
};
static void track_binding(struct device *dev, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize offset)
{
struct bound_buffer *bound = malloc(sizeof(*bound));
bound->memory = memory;
bound->offset = offset;
pthread_mutex_lock(&dev->lock);
free(map_remove(&dev->buffers, HANDLE(buffer)));
map_put(&dev->buffers, HANDLE(buffer), bound);
pthread_mutex_unlock(&dev->lock);
}
VKAPI_ATTR VkResult VKAPI_CALL BindBufferMemory(VkDevice device, VkBuffer buffer, VkDeviceMemory memory,
VkDeviceSize offset)
{
struct device *dev = device_from_handle(device);
VkResult result = dev->BindBufferMemory(device, buffer, memory, offset);
if (result == VK_SUCCESS)
track_binding(dev, buffer, memory, offset);
return result;
}
VKAPI_ATTR VkResult VKAPI_CALL BindBufferMemory2(VkDevice device, uint32_t count, const VkBindBufferMemoryInfo *infos)
{
struct device *dev = device_from_handle(device);
VkResult result = dev->BindBufferMemory2(device, count, infos);
if (result == VK_SUCCESS) {
for (uint32_t i = 0; i < count; i++)
track_binding(dev, infos[i].buffer, infos[i].memory, infos[i].memoryOffset);
}
return result;
}
VKAPI_ATTR void VKAPI_CALL DestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks *allocator)
{
struct device *dev = device_from_handle(device);
pthread_mutex_lock(&dev->lock);
free(map_remove(&dev->buffers, HANDLE(buffer)));
pthread_mutex_unlock(&dev->lock);
dev->DestroyBuffer(device, buffer, allocator);
}
/* The memory's start as the application sees it mapped. */
static void track_mapping(struct device *dev, VkDeviceMemory memory, VkDeviceSize offset, void *data)
{
pthread_mutex_lock(&dev->lock);
map_put(&dev->memories, HANDLE(memory), (uint8_t *)data - offset);
pthread_mutex_unlock(&dev->lock);
}
static void forget_mapping(struct device *dev, VkDeviceMemory memory)
{
pthread_mutex_lock(&dev->lock);
map_remove(&dev->memories, HANDLE(memory));
pthread_mutex_unlock(&dev->lock);
}
VKAPI_ATTR VkResult VKAPI_CALL MapMemory(VkDevice device, VkDeviceMemory memory, VkDeviceSize offset,
VkDeviceSize size, VkMemoryMapFlags flags, void **data)
{
struct device *dev = device_from_handle(device);
VkResult result = dev->MapMemory(device, memory, offset, size, flags, data);
if (result == VK_SUCCESS)
track_mapping(dev, memory, offset, *data);
return result;
}
VKAPI_ATTR VkResult VKAPI_CALL MapMemory2(VkDevice device, const VkMemoryMapInfo *info, void **data)
{
struct device *dev = device_from_handle(device);
VkResult result = dev->MapMemory2(device, info, data);
if (result == VK_SUCCESS)
track_mapping(dev, info->memory, info->offset, *data);
return result;
}
VKAPI_ATTR void VKAPI_CALL UnmapMemory(VkDevice device, VkDeviceMemory memory)
{
struct device *dev = device_from_handle(device);
forget_mapping(dev, memory);
dev->UnmapMemory(device, memory);
}
VKAPI_ATTR VkResult VKAPI_CALL UnmapMemory2(VkDevice device, const VkMemoryUnmapInfo *info)
{
struct device *dev = device_from_handle(device);
forget_mapping(dev, info->memory);
return dev->UnmapMemory2(device, info);
}
VKAPI_ATTR void VKAPI_CALL FreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks *allocator)
{
struct device *dev = device_from_handle(device);
forget_mapping(dev, memory);
dev->FreeMemory(device, memory, allocator);
}
struct staging_block {
VkBuffer buffer;
VkDeviceMemory memory;
uint8_t *data;
VkDeviceSize size;
VkDeviceSize used;
struct staging_block *next;
};
struct command_buffer {
VkCommandPool pool;
struct staging_block *staging;
};
static void destroy_block(struct device *dev, struct staging_block *block)
{
dev->DestroyBuffer(dev->handle, block->buffer, NULL);
dev->FreeMemory(dev->handle, block->memory, NULL);
free(block);
}
static struct staging_block *create_block(struct device *dev, VkDeviceSize size)
{
struct staging_block *block = calloc(1, sizeof(*block));
block->size = size;
VkBufferCreateInfo buffer_info = {
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.size = size,
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
};
if (dev->CreateBuffer(dev->handle, &buffer_info, NULL, &block->buffer) != VK_SUCCESS) {
free(block);
return NULL;
}
VkMemoryRequirements requirements;
dev->GetBufferMemoryRequirements(dev->handle, block->buffer, &requirements);
const VkMemoryPropertyFlags wanted = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
uint32_t type = UINT32_MAX;
for (uint32_t i = 0; i < dev->memory_properties.memoryTypeCount && type == UINT32_MAX; i++) {
if ((requirements.memoryTypeBits & (1u << i)) &&
(dev->memory_properties.memoryTypes[i].propertyFlags & wanted) == wanted)
type = i;
}
VkMemoryAllocateInfo allocate_info = {
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
.allocationSize = requirements.size,
.memoryTypeIndex = type,
};
void *data = NULL;
if (type == UINT32_MAX || dev->AllocateMemory(dev->handle, &allocate_info, NULL, &block->memory) != VK_SUCCESS) {
dev->DestroyBuffer(dev->handle, block->buffer, NULL);
free(block);
return NULL;
}
if (dev->BindBufferMemory(dev->handle, block->buffer, block->memory, 0) != VK_SUCCESS ||
dev->MapMemory(dev->handle, block->memory, 0, VK_WHOLE_SIZE, 0, &data) != VK_SUCCESS) {
destroy_block(dev, block);
return NULL;
}
block->data = data;
return block;
}
/* Called with the device lock held: the command buffer is done with its staging. */
static void recycle_staging(struct device *dev, struct command_buffer *state)
{
VkDeviceSize kept = 0;
for (struct staging_block *b = dev->free_staging; b; b = b->next)
kept += b->size;
while (state->staging) {
struct staging_block *block = state->staging;
state->staging = block->next;
if (kept + block->size > STAGING_KEPT) {
destroy_block(dev, block);
continue;
}
block->used = 0;
block->next = dev->free_staging;
dev->free_staging = block;
kept += block->size;
}
}
/* Room for `size` bytes in a staging buffer that lives as long as the command buffer's recording. */
static struct staging_block *staging_for(struct device *dev, VkCommandBuffer cmd, VkDeviceSize size,
VkDeviceSize *offset)
{
pthread_mutex_lock(&dev->lock);
struct command_buffer *state = map_get(&dev->command_buffers, HANDLE(cmd));
if (!state) {
state = calloc(1, sizeof(*state));
map_put(&dev->command_buffers, HANDLE(cmd), state);
}
struct staging_block *block = state->staging;
VkDeviceSize start = block ? (block->used + 63) & ~(VkDeviceSize)63 : 0;
if (!block || start + size > block->size) {
struct staging_block **link = &dev->free_staging;
while (*link && (*link)->size < size)
link = &(*link)->next;
block = *link;
if (block)
*link = block->next;
else
block = create_block(dev, size > STAGING_BLOCK_SIZE ? size : STAGING_BLOCK_SIZE);
if (block) {
block->next = state->staging;
state->staging = block;
}
start = 0;
}
if (block)
block->used = start + size;
pthread_mutex_unlock(&dev->lock);
*offset = start;
return block;
}
/* Where the application has the buffer's contents mapped, or NULL. */
static const uint8_t *mapped_buffer(struct device *dev, VkBuffer buffer)
{
pthread_mutex_lock(&dev->lock);
const struct bound_buffer *bound = map_get(&dev->buffers, HANDLE(buffer));
const uint8_t *base = bound ? map_get(&dev->memories, HANDLE(bound->memory)) : NULL;
const uint8_t *data = base ? base + bound->offset : NULL;
pthread_mutex_unlock(&dev->lock);
return data;
}
static void decode_block(const struct bc_format *bc, const uint8_t *block, uint8_t *texels)
{
switch (bc->kind) {
case BC1:
bcdec_bc1(block, texels, 16);
if (bc->opaque) {
for (int i = 0; i < 16; i++)
texels[i * 4 + 3] = 0xff;
}
break;
case BC2:
bcdec_bc2(block, texels, 16);
break;
case BC3:
bcdec_bc3(block, texels, 16);
break;
case BC4:
bcdec_bc4(block, texels, 4, bc->is_signed);
break;
case BC5:
bcdec_bc5(block, texels, 8, bc->is_signed);
break;
case BC6H: {
uint16_t rgb[16 * 3];
bcdec_bc6h_half(block, rgb, 12, bc->is_signed);
uint16_t *rgba = (uint16_t *)texels;
for (int i = 0; i < 16; i++) {
rgba[i * 4 + 0] = rgb[i * 3 + 0];
rgba[i * 4 + 1] = rgb[i * 3 + 1];
rgba[i * 4 + 2] = rgb[i * 3 + 2];
rgba[i * 4 + 3] = 0x3c00;
}
break;
}
case BC7:
bcdec_bc7(block, texels, 16);
break;
}
}
struct upload {
VkDeviceSize offset;
uint32_t row_length;
uint32_t image_height;
VkImageSubresourceLayers subresource;
VkOffset3D image_offset;
VkExtent3D extent;
};
static uint32_t warned;
static void warn_once(uint32_t bit, const char *message)
{
if (!(__atomic_fetch_or(&warned, bit, __ATOMIC_RELAXED) & bit))
layer_log("%s", message);
}
static void upload_region(struct device *dev, VkCommandBuffer cmd, const struct bc_image *image,
const uint8_t *source, VkImage dst, VkImageLayout layout, const struct upload *region)
{
const struct bc_format *bc = image->format;
uint32_t width = region->extent.width, height = region->extent.height;
uint32_t layers = region->subresource.layerCount;
if (layers == VK_REMAINING_ARRAY_LAYERS)
layers = image->array_layers - region->subresource.baseArrayLayer;
uint32_t slices = region->extent.depth * layers;
uint32_t row_texels = region->row_length ? region->row_length : width;
uint32_t height_texels = region->image_height ? region->image_height : height;
size_t source_row = (size_t)((row_texels + 3) / 4) * bc->block_bytes;
size_t source_slice = (size_t)((height_texels + 3) / 4) * source_row;
size_t row_bytes = (size_t)width * bc->texel_bytes;
VkDeviceSize size = (VkDeviceSize)row_bytes * height * slices;
if (!size)
return;
VkDeviceSize offset;
struct staging_block *block = staging_for(dev, cmd, size, &offset);
if (!block) {
warn_once(1u << 0, "out of staging memory for BC texture uploads; textures left empty");
return;
}
uint8_t *out = block->data + offset;
const uint8_t *in = source + region->offset;
uint8_t texels[16 * 8];
for (uint32_t s = 0; s < slices; s++) {
for (uint32_t by = 0; by < (height + 3) / 4; by++) {
for (uint32_t bx = 0; bx < (width + 3) / 4; bx++) {
decode_block(bc, in + s * source_slice + by * source_row + (size_t)bx * bc->block_bytes, texels);
for (uint32_t y = 0; y < 4 && by * 4 + y < height; y++) {
uint32_t columns = width - bx * 4 < 4 ? width - bx * 4 : 4;
memcpy(out + ((size_t)s * height + by * 4 + y) * row_bytes + (size_t)bx * 4 * bc->texel_bytes,
texels + (size_t)y * 4 * bc->texel_bytes, (size_t)columns * bc->texel_bytes);
}
}
}
}
VkBufferImageCopy copy = {
.bufferOffset = offset,
.imageSubresource = region->subresource,
.imageOffset = region->image_offset,
.imageExtent = region->extent,
};
copy.imageSubresource.layerCount = layers;
dev->CmdCopyBufferToImage(cmd, block->buffer, dst, layout, 1, &copy);
}
VKAPI_ATTR void VKAPI_CALL CmdCopyBufferToImage(VkCommandBuffer cmd, VkBuffer buffer, VkImage image,
VkImageLayout layout, uint32_t count,
const VkBufferImageCopy *regions)
{
struct device *dev = device_from_handle(cmd);
const struct bc_image *tracked = tracked_image(dev, image);
if (!tracked) {
dev->CmdCopyBufferToImage(cmd, buffer, image, layout, count, regions);
return;
}
const uint8_t *source = mapped_buffer(dev, buffer);
if (!source) {
warn_once(1u << 1, "BC texture upload from memory that is not mapped; texture left empty");
return;
}
for (uint32_t i = 0; i < count; i++) {
struct upload region = {regions[i].bufferOffset, regions[i].bufferRowLength, regions[i].bufferImageHeight,
regions[i].imageSubresource, regions[i].imageOffset, regions[i].imageExtent};
upload_region(dev, cmd, tracked, source, image, layout, &region);
}
}
VKAPI_ATTR void VKAPI_CALL CmdCopyBufferToImage2(VkCommandBuffer cmd, const VkCopyBufferToImageInfo2 *info)
{
struct device *dev = device_from_handle(cmd);
const struct bc_image *tracked = tracked_image(dev, info->dstImage);
if (!tracked) {
dev->CmdCopyBufferToImage2(cmd, info);
return;
}
const uint8_t *source = mapped_buffer(dev, info->srcBuffer);
if (!source) {
warn_once(1u << 1, "BC texture upload from memory that is not mapped; texture left empty");
return;
}
for (uint32_t i = 0; i < info->regionCount; i++) {
const VkBufferImageCopy2 *r = &info->pRegions[i];
struct upload region = {r->bufferOffset, r->bufferRowLength, r->bufferImageHeight,
r->imageSubresource, r->imageOffset, r->imageExtent};
upload_region(dev, cmd, tracked, source, info->dstImage, info->dstImageLayout, &region);
}
}
/* Reading compressed blocks back would hand out decoded texels the buffer has no room for. */
VKAPI_ATTR void VKAPI_CALL CmdCopyImageToBuffer(VkCommandBuffer cmd, VkImage image, VkImageLayout layout,
VkBuffer buffer, uint32_t count, const VkBufferImageCopy *regions)
{
struct device *dev = device_from_handle(cmd);
if (tracked_image(dev, image)) {
warn_once(1u << 2, "BC texture read back into a buffer; skipped");
return;
}
dev->CmdCopyImageToBuffer(cmd, image, layout, buffer, count, regions);
}
VKAPI_ATTR void VKAPI_CALL CmdCopyImageToBuffer2(VkCommandBuffer cmd, const VkCopyImageToBufferInfo2 *info)
{
struct device *dev = device_from_handle(cmd);
if (tracked_image(dev, info->srcImage)) {
warn_once(1u << 2, "BC texture read back into a buffer; skipped");
return;
}
dev->CmdCopyImageToBuffer2(cmd, info);
}
/* Between two BC images the texels line up; between BC and anything else they cannot. */
VKAPI_ATTR void VKAPI_CALL CmdCopyImage(VkCommandBuffer cmd, VkImage src, VkImageLayout src_layout, VkImage dst,
VkImageLayout dst_layout, uint32_t count, const VkImageCopy *regions)
{
struct device *dev = device_from_handle(cmd);
if (!tracked_image(dev, src) != !tracked_image(dev, dst)) {
warn_once(1u << 3, "copy between a BC texture and an uncompressed one; skipped");
return;
}
dev->CmdCopyImage(cmd, src, src_layout, dst, dst_layout, count, regions);
}
VKAPI_ATTR void VKAPI_CALL CmdCopyImage2(VkCommandBuffer cmd, const VkCopyImageInfo2 *info)
{
struct device *dev = device_from_handle(cmd);
if (!tracked_image(dev, info->srcImage) != !tracked_image(dev, info->dstImage)) {
warn_once(1u << 3, "copy between a BC texture and an uncompressed one; skipped");
return;
}
dev->CmdCopyImage2(cmd, info);
}
VKAPI_ATTR VkResult VKAPI_CALL AllocateCommandBuffers(VkDevice device, const VkCommandBufferAllocateInfo *info,
VkCommandBuffer *buffers)
{
struct device *dev = device_from_handle(device);
VkResult result = dev->AllocateCommandBuffers(device, info, buffers);
if (result != VK_SUCCESS)
return result;
pthread_mutex_lock(&dev->lock);
for (uint32_t i = 0; i < info->commandBufferCount; i++) {
struct command_buffer *state = calloc(1, sizeof(*state));
state->pool = info->commandPool;
map_put(&dev->command_buffers, HANDLE(buffers[i]), state);
}
pthread_mutex_unlock(&dev->lock);
return result;
}
VKAPI_ATTR void VKAPI_CALL FreeCommandBuffers(VkDevice device, VkCommandPool pool, uint32_t count,
const VkCommandBuffer *buffers)
{
struct device *dev = device_from_handle(device);
pthread_mutex_lock(&dev->lock);
for (uint32_t i = 0; i < count; i++) {
struct command_buffer *state = buffers[i] ? map_remove(&dev->command_buffers, HANDLE(buffers[i])) : NULL;
if (state) {
recycle_staging(dev, state);
free(state);
}
}
pthread_mutex_unlock(&dev->lock);
dev->FreeCommandBuffers(device, pool, count, buffers);
}
static void recycle_command_buffer(struct device *dev, VkCommandBuffer cmd)
{
pthread_mutex_lock(&dev->lock);
struct command_buffer *state = map_get(&dev->command_buffers, HANDLE(cmd));
if (state)
recycle_staging(dev, state);
pthread_mutex_unlock(&dev->lock);
}
VKAPI_ATTR VkResult VKAPI_CALL BeginCommandBuffer(VkCommandBuffer cmd, const VkCommandBufferBeginInfo *info)
{
struct device *dev = device_from_handle(cmd);
recycle_command_buffer(dev, cmd);
return dev->BeginCommandBuffer(cmd, info);
}
VKAPI_ATTR VkResult VKAPI_CALL ResetCommandBuffer(VkCommandBuffer cmd, VkCommandBufferResetFlags flags)
{
struct device *dev = device_from_handle(cmd);
recycle_command_buffer(dev, cmd);
return dev->ResetCommandBuffer(cmd, flags);
}
/* Every command buffer of the pool is done with its staging; `forget` drops them as well. */
static void recycle_pool(struct device *dev, VkCommandPool pool, bool forget)
{
pthread_mutex_lock(&dev->lock);
struct handle_map *map = &dev->command_buffers;
for (uint32_t i = 0; i < map->capacity;) {
struct command_buffer *state = map->values[i];
if (!map->keys[i] || state->pool != pool) {
i++;
continue;
}
recycle_staging(dev, state);
if (forget) {
/* Removal shifts a later entry into this slot, so look at it again. */
free(map_remove(map, map->keys[i]));
continue;
}
i++;
}
pthread_mutex_unlock(&dev->lock);
}
VKAPI_ATTR VkResult VKAPI_CALL ResetCommandPool(VkDevice device, VkCommandPool pool, VkCommandPoolResetFlags flags)
{
struct device *dev = device_from_handle(device);
recycle_pool(dev, pool, false);
return dev->ResetCommandPool(device, pool, flags);
}
VKAPI_ATTR void VKAPI_CALL DestroyCommandPool(VkDevice device, VkCommandPool pool,
const VkAllocationCallbacks *allocator)
{
struct device *dev = device_from_handle(device);
recycle_pool(dev, pool, true);
dev->DestroyCommandPool(device, pool, allocator);
}
void bc_destroy_device(struct device *dev)
{
struct handle_map *maps[] = {&dev->images, &dev->buffers, &dev->command_buffers};
for (size_t m = 0; m < sizeof(maps) / sizeof(maps[0]); m++) {
for (uint32_t i = 0; i < maps[m]->capacity; i++) {
if (!maps[m]->keys[i])
continue;
if (maps[m] == &dev->command_buffers)
recycle_staging(dev, maps[m]->values[i]);
free(maps[m]->values[i]);
}
map_free(maps[m]);
}
map_free(&dev->memories);
while (dev->free_staging) {
struct staging_block *block = dev->free_staging;
dev->free_staging = block->next;
destroy_block(dev, block);
}
}