perf(vkbridge): persistent server pipeline cache

Each device gets a VkPipelineCache loaded from <cache-dir>/<vendor>-<device>-
<cacheUUID>.vkpc (header checked against the GPU first). Pipelines created
without a cache use it; empty app caches are seeded from it and merged back on
destroy. Saved atomically every 10s while dirty (the app stops the server with
SIGKILL) and on device destroy/process cleanup. The app passes
--cache-dir <cacheDir>/vkbridge-pipelines; host tests use VKBRIDGE_CACHE_DIR.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ahmad Sayed Alsayed EbrahimandClaude Opus 5.5 committed 2026-10-10 14:28:13 +03:00
1 parent a4bd2eb271
commit 23c4eaf062
6 files changed
+305 -3

No files matched your search

+1
View File
@@ -105,6 +105,7 @@ void vkb_proc_cleanup(vkb_srv_proc *p)
if (t->is_device) {
VKB_INFO("process %d exited without destroying device %u; cleaning up", p->pid, t->id);
if (t->real.vkDeviceWaitIdle) t->real.vkDeviceWaitIdle(t->device);
vkb_pcache_device_destroy(t);
vkb_emu_device_destroy(t);
vkb_mem_free_all(t);
t->real.vkDestroyDevice(t->device, NULL);
@@ -294,6 +294,7 @@ void vkb_sv_vkCreateDevice(vkb_srv_call *c)
r = VK_ERROR_OUT_OF_HOST_MEMORY;
} else {
vkb_emu_device_init(t, emu);
vkb_pcache_device_init(t);
VKB_INFO("device %u created on %s (%u extensions, memory sharing %s)", id, k ? k->name : "?", n,
vkb_mem_strategy_name(strategy));
for (uint32_t i = 0; i < n; i++) VKB_DBG(" device extension %s", exts[i]);
@@ -316,6 +317,7 @@ void vkb_sv_vkDestroyDevice(vkb_srv_call *c)
return;
}
vkb_table_unregister(t);
vkb_pcache_device_destroy(t);
vkb_emu_device_destroy(t);
vkb_mem_free_all(t);
t->real.vkDestroyDevice(dev, NULL);
+5 -2
View File
@@ -2,7 +2,7 @@
*
* vkbridge server entry point.
*
* libvkbridge_server.so --socket <path> [--log <file>] [--vulkan <libvulkan>] [--selftest]
* libvkbridge_server.so --socket <path> [--log <file>] [--vulkan <libvulkan>] [--cache-dir <dir>] [--selftest]
*
* Packaged as a "library" so Android extracts it into the native library directory, from where
* the app executes it like proot. It loads the system Vulkan loader (on the device: the vendor's
@@ -148,7 +148,7 @@ static int listen_on(const char *path)
static void usage(void)
{
fprintf(stderr, "usage: vkbridge-server --socket PATH [--log FILE] [--vulkan LIB] [--selftest] [--verbose]\n");
fprintf(stderr, "usage: vkbridge-server --socket PATH [--log FILE] [--vulkan LIB] [--cache-dir DIR] [--selftest] [--verbose]\n");
}
int main(int argc, char **argv)
@@ -156,11 +156,13 @@ int main(int argc, char **argv)
const char *sock_path = getenv("VKBRIDGE_SOCKET");
const char *log_path = getenv("VKBRIDGE_SERVER_LOG");
const char *vk_lib = getenv("VKBRIDGE_SERVER_VULKAN");
const char *cache_dir = getenv("VKBRIDGE_CACHE_DIR");
int selftest_only = 0;
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "--socket") && i + 1 < argc) sock_path = argv[++i];
else if (!strcmp(argv[i], "--log") && i + 1 < argc) log_path = argv[++i];
else if (!strcmp(argv[i], "--vulkan") && i + 1 < argc) vk_lib = argv[++i];
else if (!strcmp(argv[i], "--cache-dir") && i + 1 < argc) cache_dir = argv[++i];
else if (!strcmp(argv[i], "--selftest")) selftest_only = 1;
else if (!strcmp(argv[i], "--verbose")) vkb_verbose = 1;
else {
@@ -196,6 +198,7 @@ int main(int argc, char **argv)
}
vkb_dispatch_load_instance(&vkb_global_dt, vkb_gipa, VK_NULL_HANDLE);
vkb_pcache_set_dir(cache_dir);
vkb_selftest_all();
if (selftest_only) return vkb_npds > 0 ? 0 : 1;
+287
View File
@@ -0,0 +1,287 @@
/* SPDX-License-Identifier: GPL-3.0-or-later
*
* Persistent pipeline cache. Mali compiles every pipeline from scratch in each new process unless
* it is handed a VkPipelineCache; games through DXVK pass none. With a cache directory set
* (--cache-dir), each device gets a cache loaded from <dir>/<vendor>-<device>-<cacheUUID>.vkpc:
* - pipelines created without a cache use it;
* - caches the app creates empty start from it, and are merged back into it when destroyed;
* - it is written back every few seconds while new pipelines come in (the app stops the server
* with SIGKILL, so there is no reliable last chance) and when the device goes away.
* The file's header is checked against the GPU before the driver sees it.
*/
#define _GNU_SOURCE
#include "vkb_server.h"
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#define PC_MAX_FILE (256u << 20)
#define PC_SAVE_PERIOD_S 10
typedef struct vkb_pcache {
vkb_srv_table *t;
VkPipelineCache cache;
char path[512];
int dirty;
size_t saved_size;
PFN_vkCreateGraphicsPipelines create_gfx;
PFN_vkCreateComputePipelines create_compute;
PFN_vkCreatePipelineCache create_cache;
PFN_vkDestroyPipelineCache destroy_cache;
struct vkb_pcache *next;
} vkb_pcache;
static const char *pc_dir;
static pthread_mutex_t pc_lock = PTHREAD_MUTEX_INITIALIZER;
static vkb_pcache *pc_list;
static int pc_thread_started;
void vkb_pcache_set_dir(const char *dir)
{
if (!dir || !*dir) return;
mkdir(dir, 0700);
pc_dir = dir;
VKB_INFO("pipeline cache directory %s", dir);
}
static vkb_pcache *cur(void)
{
vkb_srv_table *t = vkb_srv_current_table();
return t ? t->pcache : NULL;
}
/* The header every pipeline cache starts with (VkPipelineCacheHeaderVersionOne). */
static int header_ok(const vkb_srv_table *t, const uint8_t *data, size_t size)
{
VkPipelineCacheHeaderVersionOne h;
if (size < sizeof(h)) return 0;
memcpy(&h, data, sizeof(h));
VkPhysicalDeviceProperties props;
t->real.vkGetPhysicalDeviceProperties(t->physical, &props);
return h.headerSize >= sizeof(h) && h.headerVersion == VK_PIPELINE_CACHE_HEADER_VERSION_ONE &&
h.vendorID == props.vendorID && h.deviceID == props.deviceID &&
!memcmp(h.pipelineCacheUUID, props.pipelineCacheUUID, VK_UUID_SIZE);
}
static uint8_t *read_file(const char *path, size_t *size)
{
*size = 0;
int fd = open(path, O_RDONLY | O_CLOEXEC);
if (fd < 0) return NULL;
struct stat st;
uint8_t *buf = NULL;
if (fstat(fd, &st) == 0 && st.st_size > 0 && (uint64_t)st.st_size <= PC_MAX_FILE && (buf = malloc((size_t)st.st_size))) {
size_t got = 0;
while (got < (size_t)st.st_size) {
ssize_t r = read(fd, buf + got, (size_t)st.st_size - got);
if (r <= 0) break;
got += (size_t)r;
}
if (got == (size_t)st.st_size) {
*size = got;
} else {
free(buf);
buf = NULL;
}
}
close(fd);
return buf;
}
/* Writes the cache next to its file and renames it over, so a kill mid-write loses nothing. */
static void save(vkb_pcache *p)
{
vkb_srv_table *t = p->t;
size_t size = 0;
if (t->real.vkGetPipelineCacheData(t->device, p->cache, &size, NULL) != VK_SUCCESS || !size || size > PC_MAX_FILE) return;
if (size == p->saved_size) {
/* Nothing new was compiled (merges of what it already had leave the size alone). */
p->dirty = 0;
return;
}
uint8_t *buf = malloc(size);
if (!buf) return;
VkResult r = t->real.vkGetPipelineCacheData(t->device, p->cache, &size, buf);
if (r == VK_SUCCESS) {
char tmp[600];
snprintf(tmp, sizeof(tmp), "%s.%d.tmp", p->path, (int)getpid());
int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600);
int ok = fd >= 0;
for (size_t done = 0; ok && done < size;) {
ssize_t w = write(fd, buf + done, size - done);
if (w < 0 && errno == EINTR) continue;
if (w <= 0) ok = 0;
else done += (size_t)w;
}
if (fd >= 0 && fsync(fd) != 0) ok = 0;
if (fd >= 0) close(fd);
if (ok && rename(tmp, p->path) == 0) {
VKB_INFO("device %u: pipeline cache saved (%zu KB)", t->id, size >> 10);
p->saved_size = size;
p->dirty = 0;
} else {
VKB_WARN("device %u: pipeline cache not saved to %s: %s", t->id, p->path, strerror(errno));
unlink(tmp);
}
}
free(buf);
}
static void *saver(void *arg)
{
(void)arg;
for (;;) {
struct timespec ts = {PC_SAVE_PERIOD_S, 0};
nanosleep(&ts, NULL);
pthread_mutex_lock(&pc_lock);
for (vkb_pcache *p = pc_list; p; p = p->next)
if (__atomic_load_n(&p->dirty, __ATOMIC_RELAXED)) save(p);
pthread_mutex_unlock(&pc_lock);
}
return NULL;
}
/* ------------------------------------------------------------------ wrappers */
static VkResult VKAPI_CALL pc_CreateGraphicsPipelines(VkDevice device, VkPipelineCache cache, uint32_t n,
const VkGraphicsPipelineCreateInfo *ci, const VkAllocationCallbacks *a,
VkPipeline *out)
{
vkb_pcache *p = cur();
if (!cache) {
cache = p->cache;
__atomic_store_n(&p->dirty, 1, __ATOMIC_RELAXED);
}
return p->create_gfx(device, cache, n, ci, a, out);
}
static VkResult VKAPI_CALL pc_CreateComputePipelines(VkDevice device, VkPipelineCache cache, uint32_t n,
const VkComputePipelineCreateInfo *ci, const VkAllocationCallbacks *a,
VkPipeline *out)
{
vkb_pcache *p = cur();
if (!cache) {
cache = p->cache;
__atomic_store_n(&p->dirty, 1, __ATOMIC_RELAXED);
}
return p->create_compute(device, cache, n, ci, a, out);
}
static VkResult VKAPI_CALL pc_CreatePipelineCache(VkDevice device, const VkPipelineCacheCreateInfo *ci,
const VkAllocationCallbacks *a, VkPipelineCache *out)
{
vkb_pcache *p = cur();
if (ci->initialDataSize) return p->create_cache(device, ci, a, out);
size_t size = 0;
uint8_t *buf = NULL;
if (p->t->real.vkGetPipelineCacheData(device, p->cache, &size, NULL) == VK_SUCCESS && size && (buf = malloc(size)) &&
p->t->real.vkGetPipelineCacheData(device, p->cache, &size, buf) != VK_SUCCESS)
size = 0;
VkPipelineCacheCreateInfo seeded = *ci;
seeded.initialDataSize = buf ? size : 0;
seeded.pInitialData = buf;
VkResult r = p->create_cache(device, &seeded, a, out);
free(buf);
return r;
}
static void VKAPI_CALL pc_DestroyPipelineCache(VkDevice device, VkPipelineCache cache, const VkAllocationCallbacks *a)
{
vkb_pcache *p = cur();
if (cache && cache != p->cache && p->t->real.vkMergePipelineCaches(device, p->cache, 1, &cache) == VK_SUCCESS)
__atomic_store_n(&p->dirty, 1, __ATOMIC_RELAXED);
p->destroy_cache(device, cache, a);
}
/* ------------------------------------------------------------------ device hooks */
void vkb_pcache_device_init(vkb_srv_table *t)
{
if (!pc_dir || !t->real.vkCreatePipelineCache || !t->real.vkGetPipelineCacheData || !t->real.vkMergePipelineCaches) return;
vkb_pcache *p = calloc(1, sizeof(*p));
if (!p) return;
p->t = t;
VkPhysicalDeviceProperties props;
t->real.vkGetPhysicalDeviceProperties(t->physical, &props);
char uuid[2 * VK_UUID_SIZE + 1];
for (int i = 0; i < VK_UUID_SIZE; i++) snprintf(uuid + 2 * i, 3, "%02x", props.pipelineCacheUUID[i]);
snprintf(p->path, sizeof(p->path), "%s/%04x-%04x-%s.vkpc", pc_dir, props.vendorID, props.deviceID, uuid);
size_t size;
uint8_t *data = read_file(p->path, &size);
if (data && !header_ok(t, data, size)) {
VKB_WARN("device %u: %s is not this GPU's pipeline cache; starting empty", t->id, p->path);
free(data);
data = NULL;
size = 0;
}
VkPipelineCacheCreateInfo ci = {VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, NULL, 0, data ? size : 0, data};
VkResult r = t->real.vkCreatePipelineCache(t->device, &ci, NULL, &p->cache);
if (r != VK_SUCCESS && data) {
VKB_WARN("device %u: driver refused the saved pipeline cache (%d); starting empty", t->id, r);
ci.initialDataSize = 0;
ci.pInitialData = NULL;
r = t->real.vkCreatePipelineCache(t->device, &ci, NULL, &p->cache);
}
free(data);
if (r != VK_SUCCESS) {
VKB_WARN("device %u: no pipeline cache (%d)", t->id, r);
free(p);
return;
}
p->saved_size = size;
VKB_INFO("device %u: pipeline cache %s (%zu KB loaded)", t->id, p->path, size >> 10);
vkb_dispatch *dt = &t->dt;
p->create_gfx = dt->vkCreateGraphicsPipelines;
p->create_compute = dt->vkCreateComputePipelines;
p->create_cache = dt->vkCreatePipelineCache;
p->destroy_cache = dt->vkDestroyPipelineCache;
dt->vkCreateGraphicsPipelines = pc_CreateGraphicsPipelines;
dt->vkCreateComputePipelines = pc_CreateComputePipelines;
dt->vkCreatePipelineCache = pc_CreatePipelineCache;
dt->vkDestroyPipelineCache = pc_DestroyPipelineCache;
t->pcache = p;
pthread_mutex_lock(&pc_lock);
p->next = pc_list;
pc_list = p;
if (!pc_thread_started) {
pthread_t th;
pthread_attr_t at;
pthread_attr_init(&at);
pthread_attr_setdetachstate(&at, PTHREAD_CREATE_DETACHED);
pc_thread_started = pthread_create(&th, &at, saver, NULL) == 0;
pthread_attr_destroy(&at);
}
pthread_mutex_unlock(&pc_lock);
}
/* Before the device is destroyed: last save, then the cache goes. */
void vkb_pcache_device_destroy(vkb_srv_table *t)
{
vkb_pcache *p = t->pcache;
if (!p) return;
pthread_mutex_lock(&pc_lock);
for (vkb_pcache **pp = &pc_list; *pp; pp = &(*pp)->next) {
if (*pp == p) {
*pp = p->next;
break;
}
}
if (p->dirty) save(p);
pthread_mutex_unlock(&pc_lock);
t->dt.vkCreateGraphicsPipelines = p->create_gfx;
t->dt.vkCreateComputePipelines = p->create_compute;
t->dt.vkCreatePipelineCache = p->create_cache;
t->dt.vkDestroyPipelineCache = p->destroy_cache;
t->real.vkDestroyPipelineCache(t->device, p->cache, NULL);
t->pcache = NULL;
free(p);
}
+6
View File
@@ -85,6 +85,7 @@ typedef struct vkb_srv_table {
pthread_mutex_t lock;
vkb_srv_mem *mems[256]; /* hash by handle */
struct vkb_emu_device *emu;
struct vkb_pcache *pcache; /* server_pcache.c */
} vkb_srv_table;
/* server_main.c */
@@ -92,6 +93,11 @@ extern PFN_vkGetInstanceProcAddr vkb_gipa;
extern vkb_dispatch vkb_global_dt;
extern int vkb_verbose;
/* server_pcache.c */
void vkb_pcache_set_dir(const char *dir);
void vkb_pcache_device_init(vkb_srv_table *t);
void vkb_pcache_device_destroy(vkb_srv_table *t);
/* server_conn.c */
void vkb_srv_serve(int listen_fd);
vkb_srv_table *vkb_table_get(uint32_t id);
@@ -34,7 +34,10 @@ class VkBridgeComponent(private val socket: File, private val logFile: File) : S
socket.parentFile?.mkdirs()
socket.delete()
exited = false
val cmd = listOf(binary.absolutePath, "--socket", socket.absolutePath, "--log", logFile.absolutePath)
// Pipeline cache kept across sessions (Mali recompiles everything in a new process otherwise).
val cacheDir = File(app().cacheDir, "vkbridge-pipelines").apply { mkdirs() }
val cmd = listOf(binary.absolutePath, "--socket", socket.absolutePath, "--log", logFile.absolutePath,
"--cache-dir", cacheDir.absolutePath)
.joinToString(" ") { it.replace("\\", "\\\\").replace(" ", "\\ ") }
val env = arrayOf("HOME=" + app().filesDir)
pid = HostProcess.start(cmd, env, app().filesDir, { status ->