From 92f2696a0cd541d23d8f06ccdb50ee35fd779afc Mon Sep 17 00:00:00 2001 From: XxJSONDeruloxX Date: Thu, 15 Oct 2026 18:21:39 -0400 Subject: [PATCH] fix: use SteamVR compositor on Quest --- app/src/main/cpp/questxr/quest_xr.cpp | 1 + docs/agent-control.md | 2 +- docs/quest.md | 34 +- .../overlay/usr/local/bin/droiddeck-steamvr | 28 +- tools/tests/test_steamvr_helper.py | 33 +- tools/tests/test_vr_wsi.py | 23 ++ tools/vr/build.sh | 7 +- tools/vr/src/driver_droiddeck.cpp | 2 + tools/vr/src/vk_layer_droiddeck_vr.c | 47 ++- tools/vr/src/wsi_capture.cpp | 351 ++++++++++++++++++ tools/vr/src/wsi_capture.h | 17 + tools/vr/src/xr_frame.h | 4 +- tools/vr/src/xr_frame_channel.h | 121 +++++- tools/vr/tests/test_wsi_capture.cpp | 78 ++++ tools/vr/tests/test_xr_frames.cpp | 4 +- tools/vr/tests/test_xr_shared.cpp | 67 ++++ 16 files changed, 793 insertions(+), 26 deletions(-) create mode 100644 tools/tests/test_vr_wsi.py create mode 100644 tools/vr/src/wsi_capture.cpp create mode 100644 tools/vr/src/wsi_capture.h create mode 100644 tools/vr/tests/test_wsi_capture.cpp create mode 100644 tools/vr/tests/test_xr_shared.cpp diff --git a/app/src/main/cpp/questxr/quest_xr.cpp b/app/src/main/cpp/questxr/quest_xr.cpp index c31ec541f..26acbcb18 100644 --- a/app/src/main/cpp/questxr/quest_xr.cpp +++ b/app/src/main/cpp/questxr/quest_xr.cpp @@ -555,6 +555,7 @@ private: } if (glGetError() != GL_NO_ERROR) throw std::runtime_error("Frame texture upload failed"); sourceHeader_ = frame.header; + receiver_.Recycle(std::move(frame.pixels)); uploadedFrames_++; if (verifyPixels_ || uploadedFrames_ % 72 == 1) health(); } diff --git a/docs/agent-control.md b/docs/agent-control.md index a7bd42db4..26e08a057 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -129,7 +129,7 @@ tools/droiddeckctl wake FOV and dimensions. It works over wired or wireless authorized ADB. Newer builds also include `source.png`, the incoming stereo texture before the Android eye draw, and sRGB attachment/write state in the metadata for color debugging. -Source PNGs are tagged as linear and eye PNGs as sRGB. Transfers use Android's +Source PNGs use the incoming frame's color space and eye PNGs use sRGB. Transfers use Android's gzip utility to avoid sending full uncompressed images over wireless ADB. A unique request ID prevents stale images from passing as a fresh capture; no valid XR layer produces an explicit error. Captures exclude Meta system overlays and diff --git a/docs/quest.md b/docs/quest.md index 28ec53063..efa539906 100644 --- a/docs/quest.md +++ b/docs/quest.md @@ -70,7 +70,7 @@ Host allocation and preload-chain tests do not establish headset operation. In a running Steam session, open the session menu's SteamVR tab, install SteamVR once, then choose Enter SteamVR. Cardboard has its own tab and can remain off. On supported Quest devices this opens the standard app's OpenXR activity. The guest driver receives headset poses, Touch controller input -and display timing, and returns owned stereo images with their render poses. +and display timing, and returns owned stereo images tagged with the latest eye poses. SteamVR haptics return through the tracking bridge. No root or separate XR APK is required. Other devices retain their existing SteamVR start behavior. @@ -86,6 +86,23 @@ invalid seated poses, causing Thumper to remain behind its tracking-loss fade. On Quest, supplying both transforms made all three origins valid; seated height started within a millimetre of zero while standing and raw height stayed unchanged. +Quest uses SteamVR's normal Linux compositor through its display component. +The Vulkan layer captures only the HMD display swapchain, retains its present +waits, and chains a GPU copy before presentation. Three bounded capture buffers +allow rendering and transfer to overlap. Color conversion happens on the GPU; +the readback allocation travels as a DMA-BUF descriptor through `SCM_RIGHTS`. +Android brackets its CPU read with `DMA_BUF_IOCTL_SYNC`, copies to owned pixels, +then acknowledges that buffer. Missing acknowledgements quarantine the transfer +instead of allowing a producer to overwrite a possible reader. This still +includes a CPU read and GLES upload; it is not GPU image import into OpenXR. + +The helper enables SteamVR's Linux Vulkan asynchronous queue and disables motion +smoothing. It excludes the pinned Steam Frame package's optional CVP backend, +whose library constructor exits on Quest's unsupported SoC. The original library +is retained with a `.droiddeck-disabled` suffix. SteamVR binaries are not patched. +Cardboard retains its existing direct-mode path; `questframes` remains an explicit +diagnostic override for the earlier driver presenter. + The panel compositor detaches before OpenXR starts displaying guest frames. Session suspension accounts for the immersive activity while the panel is hidden. Closing the immersive activity stops SteamVR. The panel remains available for @@ -105,10 +122,19 @@ On Quest 3, a 45-second hardware run maintained focused tracked 6DoF, both controllers active, and advancing stereo images. A headset tester confirmed controller models and trigger animation. That run delivered about 8.5 new stereo images per second despite a 90 Hz XR session. -This branch retains the safe owned-frame transfer path; reducing transfer cost, -validating game input and testing actual VR games remain necessary. +That earlier measurement used the driver-direct presenter. With SteamVR's normal +compositor, a native Vulkan scene passed its first submission and continued +submitting stereo frames. Thumper and The Thrill of the Fight produced clean +immersive scenes. SUPERHOT's Windows OpenXR build still showed shared-image +corruption. A subsequent 60-second shared-buffer dashboard run delivered about +43 new stereo images per second. Reusing Android pixel buffers raised the next +60-second run to about 68 received and 65 uploaded images per second, with valid +focused tracking and no rejected frames. These dashboard results do not establish refresh-rate gameplay: game input, +sustained game performance, and exact association with SteamVR's final compositor +pose still require validation. -The frame channel carries linear RGBA8. Android samples it as linear and writes +The frame channel supports linear and sRGB RGBA8. SteamVR's final display image +uses sRGB display values; Android decodes those during texture sampling and writes to an sRGB eye swapchain, encoding the display values once. A Quest color-chart capture verified that 64, 128, 192 and 224 grey patches retain those values; the previous sRGB source upload darkened them to approximately 13, 55, 134 and diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steamvr b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steamvr index 53e438f99..a28918692 100644 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steamvr +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steamvr @@ -78,7 +78,33 @@ JSON export VK_INSTANCE_LAYERS="VK_LAYER_DROIDDECK_vr${VK_INSTANCE_LAYERS:+:$VK_INSTANCE_LAYERS}" export VK_LOADER_LAYERS_DISABLE="${VK_LOADER_LAYERS_DISABLE:-~implicit~}" if [ -S "${XDG_RUNTIME_DIR:-/run/droiddeck}/droiddeck-xr-frames" ] && [ -z "${DROIDDECK_VR_PRESENT:-}" ]; then - export DROIDDECK_VR_PRESENT=questframes + export DROIDDECK_VR_PRESENT=questwsi + fi + if [ "${DROIDDECK_VR_PRESENT:-}" = questwsi ]; then + python3 - "$HOME/.local/share/Steam/config/steamvr.vrsettings" <<'PY' || exit 1 +import json, os, sys +path = sys.argv[1] +try: + with open(path) as source: + settings = json.load(source) +except FileNotFoundError: + settings = {} +steamvr = settings.setdefault('steamvr', {}) +steamvr['enableLinuxVulkanAsync'] = True +steamvr['motionSmoothing'] = False +temporary = path + '.droiddeck.tmp' +with open(temporary, 'w') as destination: + json.dump(settings, destination, indent=3) + destination.write('\n') +os.replace(temporary, path) +PY + # The pinned Frame package's CVP constructor exits on Quest's unsupported SoC. + cvp=$steamvr/drivers/cv/bin/linuxarm64/libArcturusPerception.so + if [ -f "$cvp" ]; then + [ ! -e "$cvp.droiddeck-disabled" ] || { echo "CVP backup already exists: $cvp" >&2; exit 1; } + mv "$cvp" "$cvp.droiddeck-disabled" || exit 1 + echo "Disabled Frame-only CVP backend for Quest; SteamVR uses its normal compositor" + fi fi if [ "${DROIDDECK_VR_PRESENT:-}" = questframes ]; then export DROIDDECK_VR_PRODUCER_LEASE=${DROIDDECK_VR_PRODUCER_LEASE:-1} diff --git a/tools/tests/test_steamvr_helper.py b/tools/tests/test_steamvr_helper.py index b7f80a006..686d84c98 100644 --- a/tools/tests/test_steamvr_helper.py +++ b/tools/tests/test_steamvr_helper.py @@ -1,4 +1,5 @@ import os +import json import shutil import socket import subprocess @@ -83,11 +84,39 @@ class SteamVrHelperTest(unittest.TestCase): '"${DROIDDECK_VR_PRESENT:-unset}" "${DROIDDECK_VR_PRODUCER_LEASE:-unset}"') return self.start().read_text().strip() - def test_quest_receiver_automatically_enables_retained_producer_ownership(self): + def test_quest_receiver_uses_normal_compositor_and_excludes_unsupported_frame_backend(self): self.runtime.mkdir() + cvp = self.steamvr / 'drivers/cv/bin/linuxarm64/libArcturusPerception.so' + cvp.parent.mkdir(parents=True) + cvp.write_bytes(b'frame-only backend') with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as receiver: receiver.bind(str(self.runtime / 'droiddeck-xr-frames')) - self.assertIn('present=questframes lease=1', self.lease_log()) + self.assertIn('present=questwsi lease=unset', self.lease_log()) + settings = json.loads((self.home / '.local/share/Steam/config/steamvr.vrsettings').read_text()) + self.assertEqual(settings['steamvr'], {'enableLinuxVulkanAsync': True, 'motionSmoothing': False}) + self.assertFalse(cvp.exists()) + self.assertEqual(Path(str(cvp) + '.droiddeck-disabled').read_bytes(), b'frame-only backend') + + def test_quest_settings_preserve_other_preferences_and_start_is_idempotent(self): + self.environment['DROIDDECK_VR_PRESENT'] = 'questwsi' + path = self.home / '.local/share/Steam/config/steamvr.vrsettings' + path.parent.mkdir(parents=True) + path.write_text(json.dumps({'steamvr': {'somePreference': 42}, 'dashboard': {'somePreference': True}})) + self.start() + self.start() + settings = json.loads(path.read_text()) + self.assertEqual(settings['steamvr']['somePreference'], 42) + self.assertEqual(settings['dashboard'], {'somePreference': True}) + + def test_quest_refuses_to_overwrite_malformed_settings(self): + self.environment['DROIDDECK_VR_PRESENT'] = 'questwsi' + path = self.home / '.local/share/Steam/config/steamvr.vrsettings' + path.parent.mkdir(parents=True) + path.write_text('invalid json') + result = subprocess.run(['sh', str(self.helper), 'start'], env=self.environment, + capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(path.read_text(), 'invalid json') def test_explicit_quest_presenter_enables_the_lease(self): self.environment['DROIDDECK_VR_PRESENT'] = 'questframes' diff --git a/tools/tests/test_vr_wsi.py b/tools/tests/test_vr_wsi.py new file mode 100644 index 000000000..89cdf5220 --- /dev/null +++ b/tools/tests/test_vr_wsi.py @@ -0,0 +1,23 @@ +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +@unittest.skipUnless(shutil.which('g++'), 'needs a Linux C++ compiler') +class VrWsiTest(unittest.TestCase): + def test_compositor_handoff_and_frame_transport(self): + repository = Path(__file__).resolve().parents[2] + headers = next((p for p in (Path('/usr/include'), repository / 'build/vr/include') + if (p / 'vulkan/vulkan.h').is_file()), None) + if headers is None: + self.skipTest('needs Vulkan headers') + with tempfile.TemporaryDirectory() as directory: + for name in ('test_wsi_capture', 'test_xr_shared', 'test_xr_frames'): + with self.subTest(name=name): + executable = Path(directory) / name + subprocess.run(['g++', '-std=c++17', '-O2', '-pthread', '-Wall', '-Wextra', '-Werror', + '-Wno-missing-field-initializers', '-I' + str(headers), + str(repository / 'tools/vr/tests' / (name + '.cpp')), '-o', str(executable)], check=True) + subprocess.run([str(executable)], check=True, timeout=20) diff --git a/tools/vr/build.sh b/tools/vr/build.sh index 61e0fc2e5..1fbe42373 100755 --- a/tools/vr/build.sh +++ b/tools/vr/build.sh @@ -40,8 +40,11 @@ compile() { -Itools/vr/third_party/wayland -Iapp/src/main/cpp/waylandcomp/generated \ -o build/vr/out/droiddeck/bin/linuxarm64/driver_droiddeck.so tools/vr/src/driver_droiddeck.cpp tools/vr/src/presenter.cpp \ build/vr/xdg-shell-protocol.o -Lbuild/vr/stub -lwayland-client -ldl - aarch64-linux-gnu-gcc -std=c11 -shared -fPIC -O2 -Wall -Wextra -Wno-unused-parameter -fvisibility=hidden -pthread \ - -Wl,-z,defs -Wl,-z,nodelete -Ibuild/vr/include -o build/vr/out/layer/libVkLayer_droiddeck_vr.so tools/vr/src/vk_layer_droiddeck_vr.c -ldl + aarch64-linux-gnu-gcc -std=c11 -c -fPIC -O2 -Wall -Wextra -Wno-unused-parameter -fvisibility=hidden -pthread \ + -DDROIDDECK_WSI_CAPTURE -Ibuild/vr/include -o build/vr/vk_layer.o tools/vr/src/vk_layer_droiddeck_vr.c + aarch64-linux-gnu-g++ -std=c++17 -shared -fPIC -O2 -Wall -Wextra -Wno-missing-field-initializers -fvisibility=hidden -pthread \ + -static-libstdc++ -static-libgcc -Wl,-z,defs -Wl,-z,nodelete -Ibuild/vr/include \ + -o build/vr/out/layer/libVkLayer_droiddeck_vr.so build/vr/vk_layer.o tools/vr/src/wsi_capture.cpp -ldl aarch64-linux-gnu-strip --strip-unneeded build/vr/out/droiddeck/bin/linuxarm64/driver_droiddeck.so \ build/vr/out/layer/libVkLayer_droiddeck_vr.so') if [ "$native" = 1 ]; then diff --git a/tools/vr/src/driver_droiddeck.cpp b/tools/vr/src/driver_droiddeck.cpp index 7aef171ef..a43a94965 100644 --- a/tools/vr/src/driver_droiddeck.cpp +++ b/tools/vr/src/driver_droiddeck.cpp @@ -569,6 +569,8 @@ public: VRProperties()->SetBoolProperty(c, Prop_IsOnDesktop_Bool, false); VRProperties()->SetBoolProperty(c, Prop_DisplayDebugMode_Bool, false); directMode_ = VRSettings()->GetBool(kSettingsSection, "directMode"); + const char *presentation = std::getenv("DROIDDECK_VR_PRESENT"); + if (xrMode_ && presentation && !std::strcmp(presentation, "questwsi")) directMode_ = false; VRProperties()->SetBoolProperty(c, Prop_HasDriverDirectModeComponent_Bool, directMode_); if (directMode_) { VRProperties()->SetBoolProperty(c, Prop_DriverDirectModeSendsVsyncEvents_Bool, true); diff --git a/tools/vr/src/vk_layer_droiddeck_vr.c b/tools/vr/src/vk_layer_droiddeck_vr.c index ed0213cb8..71ca63fd9 100644 --- a/tools/vr/src/vk_layer_droiddeck_vr.c +++ b/tools/vr/src/vk_layer_droiddeck_vr.c @@ -13,6 +13,9 @@ #include #define DROIDDECK_PRODUCER_SERVER #include "producer_lease.h" +#ifdef DROIDDECK_WSI_CAPTURE +#include "wsi_capture.h" +#endif #define LAYER_NAME "VK_LAYER_DROIDDECK_vr" #define EXPORT __attribute__((visibility("default"))) @@ -258,6 +261,9 @@ static VkResult VKAPI_CALL CreateDevice(VkPhysicalDevice pd, const VkDeviceCreat } } pthread_mutex_unlock(&g_lock); +#ifdef DROIDDECK_WSI_CAPTURE + if (in) droiddeck_wsi_device(*out, pd, in->handle, in->gipa, next_gdpa, ci); +#endif return VK_SUCCESS; } @@ -267,6 +273,9 @@ static void producer_destroy_device(struct dev *device, const VkAllocationCallba static void VKAPI_CALL DestroyDevice(VkDevice device, const VkAllocationCallbacks *alloc) { struct dev *d = dev_for(device); if (!d) return; +#ifdef DROIDDECK_WSI_CAPTURE + droiddeck_wsi_destroy(device); +#endif if (d->producer) producer_destroy_device(d, alloc); forget_device_semaphores(device); PFN_vkDestroyDevice f = d->destroy; @@ -281,6 +290,7 @@ static void VKAPI_CALL DestroyDevice(VkDevice device, const VkAllocationCallback #define FAKE_DISPLAY ((VkDisplayKHR)(uintptr_t)0xdd0001) #define FAKE_MODE ((VkDisplayModeKHR)(uintptr_t)0xdd0002) +static VkSurfaceKHR g_hmd_surface; static uint32_t g_mode_w = 1920, g_mode_h = 1080, g_mode_mhz = 60000; static void mode_init(void) { @@ -383,6 +393,7 @@ static VkResult VKAPI_CALL CreateDisplayPlaneSurfaceKHR(VkInstance i, const VkDi if (!headless) { say("%s: no headless surface to stand in", "vkCreateDisplayPlaneSurfaceKHR"); return VK_ERROR_INITIALIZATION_FAILED; } VkHeadlessSurfaceCreateInfoEXT hci = {VK_STRUCTURE_TYPE_HEADLESS_SURFACE_CREATE_INFO_EXT, NULL, 0}; VkResult r = headless(i, &hci, a, s); + if (r == VK_SUCCESS) g_hmd_surface = *s; say("the display's surface is a headless one (%s)", r == VK_SUCCESS ? "ok" : "failed"); return r; } @@ -441,12 +452,24 @@ static VkResult VKAPI_CALL CreateSwapchainKHR(VkDevice d, const VkSwapchainCreat VkSwapchainKHR *out) { PFN_vkCreateSwapchainKHR next = NEXT(d, CreateSwapchainKHR); VkSwapchainCreateInfoKHR sized = *ci; +#ifdef DROIDDECK_WSI_CAPTURE + const char *capture_mode = getenv("DROIDDECK_VR_PRESENT"); + const int capture = capture_mode && !strcmp(capture_mode, "questwsi") + && !strcmp(program_invocation_short_name, "vrcompositor") && ci->surface == g_hmd_surface; + if (capture) + sized.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; +#endif mode_init(); if (sized.imageExtent.width == UINT32_MAX || sized.imageExtent.height == UINT32_MAX) sized.imageExtent = (VkExtent2D){g_mode_w, g_mode_h}; ci = &sized; VkResult r = next(d, ci, a, out); - if (r == VK_SUCCESS) return r; + if (r == VK_SUCCESS) { +#ifdef DROIDDECK_WSI_CAPTURE + if (capture) droiddeck_wsi_swapchain(d, *out, ci); +#endif + return r; + } char line[256]; int n = snprintf(line, sizeof(line), "format %d space %d mode %d images %u %ux%u usage 0x%x chain:", ci->imageFormat, ci->imageColorSpace, ci->presentMode, ci->minImageCount, ci->imageExtent.width, ci->imageExtent.height, @@ -462,16 +485,30 @@ static VkResult VKAPI_CALL CreateSwapchainKHR(VkDevice d, const VkSwapchainCreat if (r != VK_SUCCESS) { mine.imageFormat = VK_FORMAT_B8G8R8A8_UNORM; mine.imageUsage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; +#ifdef DROIDDECK_WSI_CAPTURE + if (capture) mine.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; +#endif mine.compositeAlpha = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR; mine.preTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; r = next(d, &mine, a, out); } say("headless swapchain: %s", r == VK_SUCCESS ? "ok" : "still refused"); +#ifdef DROIDDECK_WSI_CAPTURE + if (r == VK_SUCCESS && capture) droiddeck_wsi_swapchain(d, *out, &mine); +#endif return r; } +static void VKAPI_CALL DestroySwapchainKHR(VkDevice device, VkSwapchainKHR swapchain, const VkAllocationCallbacks *allocator) { +#ifdef DROIDDECK_WSI_CAPTURE + droiddeck_wsi_forget(device, swapchain); +#endif + NEXT(device, DestroySwapchainKHR)(device, swapchain, allocator); +} + static PFN_vkVoidFunction display_device_hook(const char *name) { HOOK_AS("vkCreateSwapchainKHR", CreateSwapchainKHR) + HOOK_AS("vkDestroySwapchainKHR", DestroySwapchainKHR) HOOK_AS("vkDisplayPowerControlEXT", DisplayPowerControlEXT) HOOK_AS("vkRegisterDeviceEventEXT", RegisterDeviceEventEXT) HOOK_AS("vkRegisterDisplayEventEXT", RegisterDisplayEventEXT) @@ -873,7 +910,11 @@ static VkResult VKAPI_CALL QueuePresentKHR(VkQueue queue, const VkPresentInfoKHR struct dev *device = dev_for(queue); PFN_vkQueuePresentKHR next = (PFN_vkQueuePresentKHR)device->gdpa(device->handle, "vkQueuePresentKHR"); if (device->producer) { pthread_mutex_lock(&g_submission_lock); producer_record(device, queue); } +#ifdef DROIDDECK_WSI_CAPTURE + VkResult result = droiddeck_wsi_present(device->handle, queue, info, next); +#else VkResult result = next(queue, info); +#endif if (device->producer) pthread_mutex_unlock(&g_submission_lock); return result; } @@ -928,9 +969,9 @@ static VkResult VKAPI_CALL CreateImage(VkDevice d, const VkImageCreateInfo *ci, } if (!ext) return r; char line[256]; - int n = snprintf(line, sizeof(line), "%s image %ux%u format %d flags 0x%x usage 0x%x tiling %d handles 0x%x list", + int n = snprintf(line, sizeof(line), "%s image %ux%u format %d flags 0x%x usage 0x%x tiling %d layers %u mips %u samples %u handles 0x%x list", program_invocation_short_name, ci->extent.width, ci->extent.height, ci->format, ci->flags, ci->usage, - ci->tiling, ext->handleTypes); + ci->tiling, ci->arrayLayers, ci->mipLevels, ci->samples, ext->handleTypes); for (uint32_t i = 0; list && i < list->viewFormatCount && n < (int)sizeof(line) - 8; i++) n += snprintf(line + n, sizeof(line) - n, " %d", list->pViewFormats[i]); say("%s", line); diff --git a/tools/vr/src/wsi_capture.cpp b/tools/vr/src/wsi_capture.cpp new file mode 100644 index 000000000..ab874ce84 --- /dev/null +++ b/tools/vr/src/wsi_capture.cpp @@ -0,0 +1,351 @@ +#define VK_NO_PROTOTYPES +#include "wsi_capture.h" +#include "xr_tracking_file.h" +#include "xr_frame_channel.h" +#include +#include +#include +#include +#include +#include + +namespace { +#define FUNCTIONS(X) \ + X(GetSwapchainImagesKHR) X(CreateCommandPool) X(DestroyCommandPool) X(AllocateCommandBuffers) \ + X(ResetCommandBuffer) X(BeginCommandBuffer) X(EndCommandBuffer) X(CmdPipelineBarrier) \ + X(CmdCopyImageToBuffer) X(CreateBuffer) X(DestroyBuffer) X(GetBufferMemoryRequirements) \ + X(AllocateMemory) X(FreeMemory) X(BindBufferMemory) X(MapMemory) X(UnmapMemory) \ + X(InvalidateMappedMemoryRanges) X(CreateFence) X(DestroyFence) X(ResetFences) X(WaitForFences) X(QueueSubmit) \ + X(CreateSemaphore) X(DestroySemaphore) X(GetMemoryFdKHR) X(CreateImage) X(DestroyImage) \ + X(GetImageMemoryRequirements) X(BindImageMemory) X(CmdBlitImage) +struct Device { + VkDevice handle{}; + VkPhysicalDeviceMemoryProperties memory{}; + std::unordered_map queues; +#define MEMBER(name) PFN_vk##name name{}; + FUNCTIONS(MEMBER) +#undef MEMBER +}; +struct CaptureSlot { + Device *device{}; + VkCommandPool pool{}; + VkCommandBuffer command{}; + VkBuffer buffer{}; + VkDeviceMemory memory{}; + VkImage rgbaImage{}; + VkDeviceMemory rgbaMemory{}; + int sharedFd = -1; + bool rgbaInitialized{}; + void *mapping{}; + VkFence fence{}; + bool busy{}, queued{}; + DroidDeckXrFrameHeader header{}; + ~CaptureSlot() { + if (!device) return; + if (busy && fence) device->WaitForFences(device->handle, 1, &fence, VK_TRUE, UINT64_MAX); + if (sharedFd >= 0) close(sharedFd); + if (fence) device->DestroyFence(device->handle, fence, nullptr); + if (pool) device->DestroyCommandPool(device->handle, pool, nullptr); + if (buffer) device->DestroyBuffer(device->handle, buffer, nullptr); + if (rgbaImage) device->DestroyImage(device->handle, rgbaImage, nullptr); + if (rgbaMemory) device->FreeMemory(device->handle, rgbaMemory, nullptr); + if (mapping) device->UnmapMemory(device->handle, memory); + if (memory) device->FreeMemory(device->handle, memory, nullptr); + } + bool Initialize(Device &d, VkExtent2D extent, VkFormat format, uint32_t family) { + device = &d; + VkCommandPoolCreateInfo pc{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pc.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pc.queueFamilyIndex = family; + if (d.CreateCommandPool(d.handle, &pc, nullptr, &pool)) return false; + VkCommandBufferAllocateInfo ca{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + ca.commandPool = pool; ca.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; ca.commandBufferCount = 1; + if (d.AllocateCommandBuffers(d.handle, &ca, &command)) return false; + VkBufferCreateInfo bc{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; + external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; bc.pNext = &external; + bc.size = uint64_t(extent.width) * extent.height * 4; bc.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + if (d.CreateBuffer(d.handle, &bc, nullptr, &buffer)) return false; + VkMemoryRequirements requirements{}; d.GetBufferMemoryRequirements(d.handle, buffer, &requirements); + uint32_t type = d.memory.memoryTypeCount; + for (uint32_t i = 0; i < d.memory.memoryTypeCount; i++) { + auto flags = d.memory.memoryTypes[i].propertyFlags; + if (!(requirements.memoryTypeBits & (1u << i)) || !(flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) continue; + type = i; + if (flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) break; + } + if (type == d.memory.memoryTypeCount) return false; + VkMemoryAllocateInfo alloc{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + VkExportMemoryAllocateInfo exported{VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO}; + exported.handleTypes = external.handleTypes; alloc.pNext = &exported; + alloc.allocationSize = requirements.size; alloc.memoryTypeIndex = type; + std::fprintf(stderr, "[droiddeck-vr-layer] WSI readback memory type %u flags %x\n", type, d.memory.memoryTypes[type].propertyFlags); + if (d.AllocateMemory(d.handle, &alloc, nullptr, &memory) || d.BindBufferMemory(d.handle, buffer, memory, 0)) return false; + if (d.MapMemory(d.handle, memory, 0, VK_WHOLE_SIZE, 0, &mapping)) return false; + VkMemoryGetFdInfoKHR getFd{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; + getFd.memory = memory; getFd.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; + if (d.GetMemoryFdKHR(d.handle, &getFd, &sharedFd)) return false; + VkImageCreateInfo image{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + image.imageType = VK_IMAGE_TYPE_2D; image.extent = {extent.width, extent.height, 1}; + image.mipLevels = image.arrayLayers = 1; image.samples = VK_SAMPLE_COUNT_1_BIT; + image.format = (format == VK_FORMAT_R8G8B8A8_SRGB || format == VK_FORMAT_B8G8R8A8_SRGB) + ? VK_FORMAT_R8G8B8A8_SRGB : VK_FORMAT_R8G8B8A8_UNORM; + image.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + if (d.CreateImage(d.handle, &image, nullptr, &rgbaImage)) return false; + d.GetImageMemoryRequirements(d.handle, rgbaImage, &requirements); + type = 0; + while (type < d.memory.memoryTypeCount && !(requirements.memoryTypeBits & (1u << type))) type++; + if (type == d.memory.memoryTypeCount) return false; + alloc.pNext = nullptr; alloc.allocationSize = requirements.size; alloc.memoryTypeIndex = type; + if (d.AllocateMemory(d.handle, &alloc, nullptr, &rgbaMemory) + || d.BindImageMemory(d.handle, rgbaImage, rgbaMemory, 0)) return false; + VkFenceCreateInfo fc{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + if (d.CreateFence(d.handle, &fc, nullptr, &fence)) return false; + return true; + } +}; +struct Swapchain { + explicit Swapchain(const char *trackingPath = "/run/droiddeck/" DROIDDECK_XR_FILE) : tracking(trackingPath) {} + Device *device{}; + VkExtent2D extent{}; + VkFormat format{}; + std::vector images; + std::vector ready; + std::array slots; + uint64_t frame{}; + XrTrackingFile tracking; + XrFrameSender sender; + bool initialized{}, attempted{}; + uint32_t family{}; + std::mutex sendMutex; + std::condition_variable sendReady; + bool stopping{}, failed{}; + std::thread sendThread; + ~Swapchain() { + { std::lock_guard lock(sendMutex); stopping = true; sendReady.notify_all(); } + if (sendThread.joinable()) sendThread.join(); + for (auto semaphore : ready) if (semaphore) device->DestroySemaphore(device->handle, semaphore, nullptr); + } + void Run() noexcept { + try { Loop(); } + catch (...) { + std::lock_guard lock(sendMutex); failed = true; + std::fprintf(stderr, "[droiddeck-vr-layer] WSI worker failed; restart SteamVR\n"); + } + } + void Loop() { + for (;;) { + CaptureSlot *slot; + { + std::unique_lock lock(sendMutex); + sendReady.wait(lock, [this] { + return stopping || std::any_of(slots.begin(), slots.end(), [](const auto &item) { return item.queued; }); + }); + slot = nullptr; + for (auto &item : slots) if (item.queued && (!slot || item.header.frame_id < slot->header.frame_id)) slot = &item; + if (!slot) return; + slot->queued = false; + } + auto &d = *device; + const auto header = slot->header; + auto &fence = slot->fence; + auto &memory = slot->memory; + auto sharedFd = slot->sharedFd; + int64_t begin = XrTrackingFile::Now(); + VkResult result = d.WaitForFences(d.handle, 1, &fence, VK_TRUE, UINT64_MAX); + int64_t waited = XrTrackingFile::Now(); + VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; range.memory = memory; range.size = VK_WHOLE_SIZE; + if (result == VK_SUCCESS) result = d.InvalidateMappedMemoryRanges(d.handle, 1, &range); + int64_t invalidated = XrTrackingFile::Now(); + if (result == VK_SUCCESS && XrTrackingFile::Now() - header.captured_ns < 200000000LL) { + sender.SendShared("/run/droiddeck/" DROIDDECK_XR_FRAME_SOCKET, header, sharedFd); + } + if (header.frame_id % 120 == 0) std::fprintf(stderr, "[droiddeck-vr-layer] WSI ms wait=%.2f invalidate=%.2f shared=%.2f\n", + (waited-begin)/1e6, (invalidated-waited)/1e6, (XrTrackingFile::Now()-invalidated)/1e6); + std::lock_guard lock(sendMutex); + slot->busy = false; + if (result != VK_SUCCESS || sender.SharedLeaseLost()) { + std::fprintf(stderr, "[droiddeck-vr-layer] WSI buffer quarantined: result=%d leaseLost=%d; restart SteamVR\n", result, sender.SharedLeaseLost()); + failed = true; return; + } + } + } + bool Initialize(VkSwapchainKHR chain, VkQueue queue) { + if (initialized) return true; + if (attempted) return false; + attempted = true; + auto &d = *device; + auto queueFamily = d.queues.find(queue); + if (queueFamily == d.queues.end()) return false; + family = queueFamily->second; + uint32_t count = 0; + if (d.GetSwapchainImagesKHR(d.handle, chain, &count, nullptr) || !count) return false; + images.resize(count); + if (d.GetSwapchainImagesKHR(d.handle, chain, &count, images.data())) return false; + ready.resize(count); + VkSemaphoreCreateInfo semaphoreInfo{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; + for (auto &semaphore : ready) if (d.CreateSemaphore(d.handle, &semaphoreInfo, nullptr, &semaphore)) return false; + for (auto &slot : slots) if (!slot.Initialize(d, extent, format, family)) return false; + sendThread = std::thread([this] { Run(); }); + initialized = true; + std::fprintf(stderr, "[droiddeck-vr-layer] capturing compositor swapchain %ux%u format %d\n", extent.width, extent.height, format); + return true; + } + VkResult Fail(VkResult result, CaptureSlot &slot) { + std::lock_guard lock(sendMutex); + failed = true; slot.busy = false; + return result; + } + bool Header(DroidDeckXrFrameHeader &header) { + DroidDeckXrTracking packet{}; + if (!tracking.Read(&packet) || !XrTrackingFile::Fresh(packet)) return false; + header = {}; + header.magic = DROIDDECK_XR_FRAME_MAGIC; header.version = DROIDDECK_XR_FRAME_VERSION; + header.width = extent.width; header.height = extent.height; + header.bytes = extent.width * extent.height * 4; + header.color_space = DROIDDECK_XR_COLOR_SRGB_RGBA8; + header.frame_id = ++frame; header.captured_ns = XrTrackingFile::Now(); + for (int eye = 0; eye < 2; eye++) { + const auto &pose = packet.eyes[eye].pose; + const float x = pose.orientation[0], y = pose.orientation[1], z = pose.orientation[2], w = pose.orientation[3]; + float matrix[3][4] = {{1-2*(y*y+z*z),2*(x*y-w*z),2*(x*z+w*y),pose.position[0]}, + {2*(x*y+w*z),1-2*(x*x+z*z),2*(y*z-w*x),pose.position[1]}, + {2*(x*z-w*y),2*(y*z+w*x),1-2*(x*x+y*y),pose.position[2]}}; + std::memcpy(header.eye_pose[eye], matrix, sizeof(matrix)); + const auto *fov = packet.eyes[eye].fov; + float left = std::tan(fov[0]), right = std::tan(fov[1]), up = std::tan(fov[2]), down = std::tan(fov[3]); + auto &p = header.projection[eye]; + p[0][0] = 2/(right-left); p[0][2] = (right+left)/(right-left); + p[1][1] = 2/(up-down); p[1][2] = (up+down)/(up-down); + p[2][2] = -1; p[2][3] = -0.2f; p[3][2] = -1; + } + return xr_frame_valid(header); + } + VkResult Capture(VkQueue queue, uint32_t index, const VkPresentInfoKHR &present, VkSemaphore *completion) { + auto &d = *device; + if (index >= images.size()) return VK_ERROR_OUT_OF_DATE_KHR; + CaptureSlot *selected = nullptr; + { + std::lock_guard lock(sendMutex); + if (failed) return VK_ERROR_DEVICE_LOST; + for (auto &slot : slots) if (!slot.busy) { selected = &slot; slot.busy = true; break; } + } + if (!selected) return VK_SUCCESS; + auto &slot = *selected; + DroidDeckXrFrameHeader header{}; + if (!Header(header)) { std::lock_guard lock(sendMutex); slot.busy = false; return VK_SUCCESS; } + auto &command = slot.command; + auto &buffer = slot.buffer; + auto &fence = slot.fence; + auto &rgbaImage = slot.rgbaImage; + auto &rgbaInitialized = slot.rgbaInitialized; + VkResult result = d.ResetCommandBuffer(command, 0); + if (result) return Fail(result, slot); + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + if ((result = d.BeginCommandBuffer(command, &begin))) return Fail(result, slot); + VkImageMemoryBarrier image{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + image.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; image.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + image.oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; image.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + image.srcQueueFamilyIndex = image.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image.image = images[index]; image.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + d.CmdPipelineBarrier(command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &image); + VkBufferImageCopy copy{}; copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {extent.width, extent.height, 1}; + VkImageMemoryBarrier rgba = image; + rgba.image = rgbaImage; rgba.srcAccessMask = rgbaInitialized ? VK_ACCESS_TRANSFER_READ_BIT : 0; + rgba.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + rgba.oldLayout = rgbaInitialized ? VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL : VK_IMAGE_LAYOUT_UNDEFINED; + rgba.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + d.CmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &rgba); + VkImageBlit blit{}; blit.srcSubresource = blit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.srcOffsets[1] = blit.dstOffsets[1] = {static_cast(extent.width), static_cast(extent.height), 1}; + d.CmdBlitImage(command, images[index], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, rgbaImage, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_NEAREST); + rgba.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; rgba.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + rgba.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; rgba.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + d.CmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &rgba); + d.CmdCopyImageToBuffer(command, rgbaImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, buffer, 1, ©); + image.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; image.dstAccessMask = 0; + image.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; image.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + VkBufferMemoryBarrier host{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + host.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; host.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + host.srcQueueFamilyIndex = host.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; host.buffer = buffer; host.size = VK_WHOLE_SIZE; + d.CmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + 0, 0, nullptr, 1, &host, 1, &image); + if ((result = d.EndCommandBuffer(command)) || (result = d.ResetFences(d.handle, 1, &fence))) return Fail(result, slot); + std::vector stages(present.waitSemaphoreCount, VK_PIPELINE_STAGE_TRANSFER_BIT); + VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + submit.waitSemaphoreCount = present.waitSemaphoreCount; submit.pWaitSemaphores = present.pWaitSemaphores; + submit.pWaitDstStageMask = stages.data(); submit.commandBufferCount = 1; submit.pCommandBuffers = &command; + submit.signalSemaphoreCount = 1; submit.pSignalSemaphores = &ready[index]; + if ((result = d.QueueSubmit(queue, 1, &submit, fence))) return Fail(result, slot); + rgbaInitialized = true; + { std::lock_guard lock(sendMutex); slot.header = header; slot.queued = true; sendReady.notify_all(); } + *completion = ready[index]; + return VK_SUCCESS; + } +}; +std::mutex mutex; +std::unordered_map> devices; +std::unordered_map> swapchains; +} + +extern "C" void droiddeck_wsi_device(VkDevice device, VkPhysicalDevice physical, VkInstance instance, + PFN_vkGetInstanceProcAddr gipa, PFN_vkGetDeviceProcAddr gdpa, const VkDeviceCreateInfo *create) { + const char *mode = getenv("DROIDDECK_VR_PRESENT"); + if (!mode || strcmp(mode, "questwsi") || strcmp(program_invocation_short_name, "vrcompositor")) return; + auto d = std::make_unique(); d->handle = device; +#define LOAD(name) d->name = reinterpret_cast(gdpa(device, "vk" #name)); if (!d->name) return; + FUNCTIONS(LOAD) +#undef LOAD + auto properties = reinterpret_cast(gipa(instance, "vkGetPhysicalDeviceMemoryProperties")); + properties(physical, &d->memory); + auto getQueue = reinterpret_cast(gdpa(device, "vkGetDeviceQueue")); + for (uint32_t index = 0; index < create->queueCreateInfoCount; index++) { + const auto &info = create->pQueueCreateInfos[index]; + if (info.flags) continue; + for (uint32_t number = 0; number < info.queueCount; number++) { + VkQueue queue{}; getQueue(device, info.queueFamilyIndex, number, &queue); + if (queue) d->queues[queue] = info.queueFamilyIndex; + } + } + std::lock_guard lock(mutex); devices[device] = std::move(d); +} +extern "C" void droiddeck_wsi_destroy(VkDevice device) { + std::lock_guard lock(mutex); + for (auto it = swapchains.begin(); it != swapchains.end();) + if (it->second->device->handle == device) it = swapchains.erase(it); else ++it; + devices.erase(device); +} +extern "C" void droiddeck_wsi_swapchain(VkDevice device, VkSwapchainKHR chain, const VkSwapchainCreateInfoKHR *create) { + std::lock_guard lock(mutex); + auto it = devices.find(device); if (it == devices.end()) return; + if (create->imageFormat != VK_FORMAT_R8G8B8A8_UNORM && create->imageFormat != VK_FORMAT_R8G8B8A8_SRGB && + create->imageFormat != VK_FORMAT_B8G8R8A8_UNORM && create->imageFormat != VK_FORMAT_B8G8R8A8_SRGB) return; + auto chainState = std::make_unique(); chainState->device = it->second.get(); + chainState->extent = create->imageExtent; chainState->format = create->imageFormat; + swapchains[chain] = std::move(chainState); +} +extern "C" void droiddeck_wsi_forget(VkDevice device, VkSwapchainKHR chain) { + (void)device; std::lock_guard lock(mutex); swapchains.erase(chain); +} +extern "C" VkResult droiddeck_wsi_present(VkDevice device, VkQueue queue, const VkPresentInfoKHR *present, PFN_vkQueuePresentKHR next) try { + (void)device; std::lock_guard lock(mutex); + if (present->swapchainCount != 1) return next(queue, present); + auto it = swapchains.find(present->pSwapchains[0]); + if (it == swapchains.end()) return next(queue, present); + if (!it->second->Initialize(present->pSwapchains[0], queue)) return VK_ERROR_INITIALIZATION_FAILED; + auto family = it->second->device->queues.find(queue); + if (family == it->second->device->queues.end() || family->second != it->second->family) + return VK_ERROR_INITIALIZATION_FAILED; + VkSemaphore completion{}; + VkResult result = it->second->Capture(queue, present->pImageIndices[0], *present, &completion); + if (result) return result; + if (!completion) return next(queue, present); + // Reacquiring this swapchain image also proves its previous present wait semaphore is reusable. + VkPresentInfoKHR copy = *present; copy.waitSemaphoreCount = 1; copy.pWaitSemaphores = &completion; + return next(queue, ©); +} catch (...) { + std::fprintf(stderr, "[droiddeck-vr-layer] WSI capture failed; restart SteamVR\n"); + return VK_ERROR_OUT_OF_HOST_MEMORY; +} diff --git a/tools/vr/src/wsi_capture.h b/tools/vr/src/wsi_capture.h new file mode 100644 index 000000000..d56aab4ea --- /dev/null +++ b/tools/vr/src/wsi_capture.h @@ -0,0 +1,17 @@ +#pragma once +#include + +#ifdef __cplusplus +extern "C" { +#endif +void droiddeck_wsi_device(VkDevice device, VkPhysicalDevice physical, VkInstance instance, + PFN_vkGetInstanceProcAddr gipa, PFN_vkGetDeviceProcAddr gdpa, + const VkDeviceCreateInfo *create); +void droiddeck_wsi_destroy(VkDevice device); +void droiddeck_wsi_swapchain(VkDevice device, VkSwapchainKHR swapchain, const VkSwapchainCreateInfoKHR *create); +void droiddeck_wsi_forget(VkDevice device, VkSwapchainKHR swapchain); +VkResult droiddeck_wsi_present(VkDevice device, VkQueue queue, const VkPresentInfoKHR *present, + PFN_vkQueuePresentKHR next); +#ifdef __cplusplus +} +#endif diff --git a/tools/vr/src/xr_frame.h b/tools/vr/src/xr_frame.h index f3d67ea20..05e1c8e16 100644 --- a/tools/vr/src/xr_frame.h +++ b/tools/vr/src/xr_frame.h @@ -4,6 +4,7 @@ #define DROIDDECK_XR_FRAME_MAGIC 0x46584444u #define DROIDDECK_XR_FRAME_VERSION 1u +#define DROIDDECK_XR_FRAME_SHARED_VERSION 2u #define DROIDDECK_XR_FRAME_SOCKET "droiddeck-xr-frames" #define DROIDDECK_XR_FRAME_MAX_BYTES (64u * 1024u * 1024u) #define DROIDDECK_XR_COLOR_LINEAR_RGBA8 1u @@ -22,7 +23,8 @@ static_assert(sizeof(DroidDeckXrFrameHeader) == 264, "XR frame protocol layout") inline bool xr_frame_valid(const DroidDeckXrFrameHeader &header) { uint64_t bytes = (uint64_t)header.width * header.height * 4; - if (header.magic != DROIDDECK_XR_FRAME_MAGIC || header.version != DROIDDECK_XR_FRAME_VERSION + if (header.magic != DROIDDECK_XR_FRAME_MAGIC || (header.version != DROIDDECK_XR_FRAME_VERSION + && header.version != DROIDDECK_XR_FRAME_SHARED_VERSION) || !header.frame_id || header.width < 2 || (header.width & 1u) || !header.height || header.width > 8192 || header.height > 4096 || bytes > DROIDDECK_XR_FRAME_MAX_BYTES || header.bytes != bytes || (header.color_space != DROIDDECK_XR_COLOR_LINEAR_RGBA8 diff --git a/tools/vr/src/xr_frame_channel.h b/tools/vr/src/xr_frame_channel.h index ceffe5a9b..530a3e1a3 100644 --- a/tools/vr/src/xr_frame_channel.h +++ b/tools/vr/src/xr_frame_channel.h @@ -12,6 +12,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -46,6 +49,55 @@ inline bool xr_frame_peer(int descriptor) { && size == sizeof(peer) && peer.uid == getuid(); } +struct XrFrameDescriptor { + int fd = -1; + ~XrFrameDescriptor() { if (fd >= 0) close(fd); } + XrFrameDescriptor() = default; + XrFrameDescriptor(const XrFrameDescriptor &) = delete; + XrFrameDescriptor &operator=(const XrFrameDescriptor &) = delete; +}; + +inline bool xr_frame_header(int socket, DroidDeckXrFrameHeader &header, XrFrameDescriptor &shared, int64_t deadline) { + for (;;) { + int64_t remaining = deadline - xr_frame_now(); + if (remaining <= 0) return false; + pollfd ready{socket, POLLIN, 0}; + int result = poll(&ready, 1, (int)((remaining + 999999) / 1000000)); + if (result < 0 && errno == EINTR) continue; + if (result <= 0 || !(ready.revents & POLLIN)) return false; + alignas(cmsghdr) unsigned char control[CMSG_SPACE(8 * sizeof(int))]{}; + iovec data{&header, sizeof(header)}; + msghdr message{}; message.msg_iov = &data; message.msg_iovlen = 1; + message.msg_control = control; message.msg_controllen = sizeof(control); + ssize_t bytes = recvmsg(socket, &message, MSG_DONTWAIT | MSG_CMSG_CLOEXEC); + if (bytes < 0 && (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)) continue; + if (bytes <= 0) return false; + unsigned count = 0; + for (auto *item = CMSG_FIRSTHDR(&message); item; item = CMSG_NXTHDR(&message, item)) { + if (item->cmsg_level != SOL_SOCKET || item->cmsg_type != SCM_RIGHTS) continue; + size_t descriptors = (item->cmsg_len - CMSG_LEN(0)) / sizeof(int); + auto *fds = reinterpret_cast(CMSG_DATA(item)); + for (size_t index = 0; index < descriptors; index++) { + if (++count == 1) shared.fd = fds[index]; else close(fds[index]); + } + } + if ((message.msg_flags & (MSG_CTRUNC | MSG_TRUNC)) || count > 1) return false; + return xr_frame_io(socket, reinterpret_cast(&header) + bytes, sizeof(header) - bytes, false, deadline); + } +} + +inline bool xr_frame_shared_pixels(int descriptor, unsigned char *pixels, size_t bytes) { + struct stat status{}; + if (fstat(descriptor, &status) != 0 || status.st_size < static_cast(bytes)) return false; + dma_buf_sync sync{DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ}; + if (ioctl(descriptor, DMA_BUF_IOCTL_SYNC, &sync) != 0) return false; + void *mapping = mmap(nullptr, bytes, PROT_READ, MAP_SHARED, descriptor, 0); + bool success = mapping != MAP_FAILED; + if (success) { std::memcpy(pixels, mapping, bytes); munmap(mapping, bytes); } + sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ; + return ioctl(descriptor, DMA_BUF_IOCTL_SYNC, &sync) == 0 && success; +} + struct DroidDeckXrPixels { DroidDeckXrFrameHeader header{}; std::vector pixels; @@ -88,6 +140,8 @@ public: filename_.clear(); if (lock_ >= 0) close(lock_); lock_ = -1; + std::lock_guard lock(mutex_); + pending_ = {}; recycled_.clear(); } bool Take(DroidDeckXrPixels *out) { std::lock_guard lock(mutex_); @@ -96,6 +150,10 @@ public: pending_ = {}; return true; } + void Recycle(std::vector pixels) { + std::lock_guard lock(mutex_); + if (recycled_.size() < 3) recycled_.push_back(std::move(pixels)); + } uint64_t Rejected() const { return rejected_; } uint64_t Received() const { return received_; } private: @@ -107,6 +165,7 @@ private: std::thread thread_; std::mutex mutex_; DroidDeckXrPixels pending_; + std::vector> recycled_; void Loop() { while (!stopping_) { pollfd ready{listener_, POLLIN, 0}; @@ -117,20 +176,33 @@ private: uint64_t lastFrame = 0; while (!stopping_) { DroidDeckXrPixels frame; + XrFrameDescriptor shared; int64_t deadline = xr_frame_now() + 200000000LL; - if (!xr_frame_io(client, &frame.header, sizeof(frame.header), false, deadline)) break; + if (!xr_frame_header(client, frame.header, shared, deadline)) break; int64_t now = xr_frame_now(); if (!xr_frame_valid(frame.header) || frame.header.captured_ns > now - || now - frame.header.captured_ns > 500000000LL || frame.header.frame_id <= lastFrame) { rejected_++; break; } - try { frame.pixels.resize(frame.header.bytes); } + || now - frame.header.captured_ns > 500000000LL || frame.header.frame_id <= lastFrame + || (frame.header.version == DROIDDECK_XR_FRAME_SHARED_VERSION) != (shared.fd >= 0)) { rejected_++; break; } + try { + { + std::lock_guard lock(mutex_); + if (!recycled_.empty()) { + frame.pixels = std::move(recycled_.back()); recycled_.pop_back(); + } + } + frame.pixels.resize(frame.header.bytes); + } catch (...) { rejected_++; break; } deadline = xr_frame_now() + 200000000LL; - if (!xr_frame_io(client, frame.pixels.data(), frame.pixels.size(), false, deadline)) { rejected_++; break; } + bool copied = shared.fd >= 0 ? xr_frame_shared_pixels(shared.fd, frame.pixels.data(), frame.pixels.size()) + : xr_frame_io(client, frame.pixels.data(), frame.pixels.size(), false, deadline); + if (!copied) { rejected_++; break; } if (xr_frame_now() - frame.header.captured_ns > 500000000LL) { rejected_++; break; } uint64_t acknowledgement = frame.header.frame_id; lastFrame = acknowledgement; { std::lock_guard lock(mutex_); + if (!pending_.pixels.empty() && recycled_.size() < 3) recycled_.push_back(std::move(pending_.pixels)); pending_ = std::move(frame); } received_++; @@ -148,7 +220,22 @@ public: XrFrameSender &operator=(const XrFrameSender &) = delete; ~XrFrameSender() { Disconnect(); } bool Send(const std::string &filename, const DroidDeckXrFrameHeader &header, const unsigned char *pixels) { - if (!pixels || !xr_frame_valid(header) || filename.empty() || filename.size() >= sizeof(sockaddr_un::sun_path)) return false; + return SendFrame(filename, header, pixels, -1); + } + bool SendShared(const std::string &filename, DroidDeckXrFrameHeader header, int shared) { + if (shared < 0 || sharedLeaseLost_) return false; + header.version = DROIDDECK_XR_FRAME_SHARED_VERSION; + return SendFrame(filename, header, nullptr, shared); + } + bool SharedLeaseLost() const { return sharedLeaseLost_; } +private: + int descriptor_ = -1; + bool sharedLeaseLost_ = false; + std::string filename_; + bool SendFrame(const std::string &filename, const DroidDeckXrFrameHeader &header, const unsigned char *pixels, int shared) { + if ((!pixels && shared < 0) || !xr_frame_valid(header) || filename.empty() + || filename.size() >= sizeof(sockaddr_un::sun_path) + || (header.version == DROIDDECK_XR_FRAME_SHARED_VERSION) != (shared >= 0)) return false; if (filename_ != filename) Disconnect(); int64_t deadline = xr_frame_now() + 200000000LL; if (descriptor_ < 0) { @@ -169,14 +256,28 @@ public: filename_ = filename; } uint64_t acknowledgement = 0; - if (!xr_frame_io(descriptor_, const_cast(&header), sizeof(header), true, deadline) - || !xr_frame_io(descriptor_, const_cast(pixels), header.bytes, true, deadline) + bool sent = false; + if (shared >= 0) { + iovec data{const_cast(&header), sizeof(header)}; + alignas(cmsghdr) unsigned char control[CMSG_SPACE(sizeof(int))]{}; + msghdr message{}; message.msg_iov = &data; message.msg_iovlen = 1; + message.msg_control = control; message.msg_controllen = sizeof(control); + auto *item = CMSG_FIRSTHDR(&message); item->cmsg_level = SOL_SOCKET; item->cmsg_type = SCM_RIGHTS; + item->cmsg_len = CMSG_LEN(sizeof(int)); std::memcpy(CMSG_DATA(item), &shared, sizeof(shared)); + ssize_t bytes; + do { bytes = sendmsg(descriptor_, &message, MSG_NOSIGNAL | MSG_DONTWAIT); } while (bytes < 0 && errno == EINTR); + if (bytes > 0) { + sharedLeaseLost_ = true; + sent = xr_frame_io(descriptor_, reinterpret_cast(data.iov_base) + bytes, + sizeof(header) - bytes, true, deadline); + } + } else sent = xr_frame_io(descriptor_, const_cast(&header), sizeof(header), true, deadline) + && xr_frame_io(descriptor_, const_cast(pixels), header.bytes, true, deadline); + if (!sent || !xr_frame_io(descriptor_, &acknowledgement, sizeof(acknowledgement), false, deadline) || acknowledgement != header.frame_id) { Disconnect(); return false; } + sharedLeaseLost_ = false; return true; } -private: - int descriptor_ = -1; - std::string filename_; void Disconnect() { if (descriptor_ >= 0) close(descriptor_); descriptor_ = -1; filename_.clear(); } }; diff --git a/tools/vr/tests/test_wsi_capture.cpp b/tools/vr/tests/test_wsi_capture.cpp new file mode 100644 index 000000000..7b26adfee --- /dev/null +++ b/tools/vr/tests/test_wsi_capture.cpp @@ -0,0 +1,78 @@ +#include "../src/wsi_capture.cpp" +#include + +static VkResult resetResult = VK_SUCCESS; +static unsigned submitted, presented, copied, blitted; +static VkSemaphore originalWaits[2]{(VkSemaphore)10, (VkSemaphore)11}; +static VkSemaphore expectedPresent; +static VkResult VKAPI_CALL Present(VkQueue, const VkPresentInfoKHR *info) { + presented++; + if (expectedPresent) assert(info->waitSemaphoreCount == 1 && info->pWaitSemaphores[0] == expectedPresent); + else assert(info->waitSemaphoreCount == 2 && info->pWaitSemaphores[0] == originalWaits[0]); + return VK_SUCCESS; +} + +int main() { + char path[] = "/tmp/droiddeck-wsi-XXXXXX"; int fd = mkstemp(path); assert(fd >= 0); + DroidDeckXrTracking tracking{}; + tracking.magic = DROIDDECK_XR_MAGIC; tracking.version = DROIDDECK_XR_VERSION; tracking.active = 1; + tracking.sampled_ns = xr_frame_now(); tracking.period_ns = 11111111; + for (auto &eye : tracking.eyes) { + eye.width = eye.height = 4; eye.pose.orientation[3] = 1; eye.pose.flags = 15; + eye.fov[0] = eye.fov[3] = -0.7f; eye.fov[1] = eye.fov[2] = 0.7f; + } + assert(write(fd, &tracking, sizeof(tracking)) == sizeof(tracking)); close(fd); + auto device = std::make_unique(); + device->ResetCommandBuffer = [](VkCommandBuffer, VkCommandBufferResetFlags) { return resetResult; }; + device->BeginCommandBuffer = [](VkCommandBuffer, const VkCommandBufferBeginInfo *) { return VK_SUCCESS; }; + device->EndCommandBuffer = [](VkCommandBuffer) { return VK_SUCCESS; }; + device->ResetFences = [](VkDevice, uint32_t, const VkFence *) { return VK_SUCCESS; }; + device->CmdPipelineBarrier = [](VkCommandBuffer, VkPipelineStageFlags, VkPipelineStageFlags, + VkDependencyFlags, uint32_t, const VkMemoryBarrier *, uint32_t buffers, const VkBufferMemoryBarrier *host, + uint32_t images, const VkImageMemoryBarrier *image) { + assert(images == 1 && image->srcQueueFamilyIndex == VK_QUEUE_FAMILY_IGNORED); + if (buffers) assert(host->dstAccessMask == VK_ACCESS_HOST_READ_BIT); + }; + device->CmdBlitImage = [](VkCommandBuffer, VkImage, VkImageLayout source, VkImage, VkImageLayout destination, + uint32_t count, const VkImageBlit *blit, VkFilter filter) { + assert(source == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL && destination == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + assert(count == 1 && filter == VK_FILTER_NEAREST && blit->srcOffsets[1].x == 8); blitted++; + }; + device->CmdCopyImageToBuffer = [](VkCommandBuffer, VkImage, VkImageLayout layout, VkBuffer, + uint32_t count, const VkBufferImageCopy *copy) { + assert(layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL && count == 1 && copy->imageExtent.width == 8); copied++; + }; + device->QueueSubmit = [](VkQueue, uint32_t count, const VkSubmitInfo *info, VkFence) { + assert(count == 1 && info->waitSemaphoreCount == 2 && info->signalSemaphoreCount == 1); + for (int i = 0; i < 2; i++) assert(info->pWaitSemaphores[i] == originalWaits[i] + && info->pWaitDstStageMask[i] == VK_PIPELINE_STAGE_TRANSFER_BIT); + submitted++; return VK_SUCCESS; + }; + device->DestroySemaphore = [](VkDevice, VkSemaphore, const VkAllocationCallbacks *) {}; + VkDevice handle = (VkDevice)1; VkQueue queue = (VkQueue)2; + device->handle = handle; device->queues[queue] = 7; + auto chain = std::make_unique(path); + chain->device = device.get(); chain->family = 7; chain->initialized = true; chain->extent = {8, 4}; + chain->images = {(VkImage)20, (VkImage)21, (VkImage)22}; + chain->ready = {(VkSemaphore)30, (VkSemaphore)31, (VkSemaphore)32}; + devices[handle] = std::move(device); + VkSwapchainKHR swapchain = (VkSwapchainKHR)3; swapchains[swapchain] = std::move(chain); + VkPresentInfoKHR info{VK_STRUCTURE_TYPE_PRESENT_INFO_KHR}; + uint32_t index = 0; info.pImageIndices = &index; info.swapchainCount = 1; info.pSwapchains = &swapchain; + info.waitSemaphoreCount = 2; info.pWaitSemaphores = originalWaits; + for (index = 0; index < 3; index++) { + expectedPresent = (VkSemaphore)(uintptr_t)(30 + index); + assert(droiddeck_wsi_present(handle, queue, &info, Present) == VK_SUCCESS); + } + index = 0; expectedPresent = VK_NULL_HANDLE; + assert(droiddeck_wsi_present(handle, queue, &info, Present) == VK_SUCCESS); + assert(submitted == 3 && presented == 4 && copied == 3 && blitted == 3); + for (const auto &slot : swapchains[swapchain]->slots) assert(slot.busy && slot.queued && xr_frame_valid(slot.header)); + swapchains[swapchain]->slots[0].busy = false; + resetResult = VK_ERROR_DEVICE_LOST; + assert(droiddeck_wsi_present(handle, queue, &info, Present) == VK_ERROR_DEVICE_LOST); + assert(droiddeck_wsi_present(handle, queue, &info, Present) == VK_ERROR_DEVICE_LOST); + assert(submitted == 3 && presented == 4); + droiddeck_wsi_destroy(handle); unlink(path); + std::puts("WSI synchronization passed: original waits retained, per-image completion, bounded in-flight buffers and fatal error propagation"); +} diff --git a/tools/vr/tests/test_xr_frames.cpp b/tools/vr/tests/test_xr_frames.cpp index cc514eae7..ecc03088d 100644 --- a/tools/vr/tests/test_xr_frames.cpp +++ b/tools/vr/tests/test_xr_frames.cpp @@ -43,7 +43,7 @@ int main(int argc, char **argv) { auto valid = Header(); assert(xr_frame_valid(valid)); valid.color_space = DROIDDECK_XR_COLOR_SRGB_RGBA8; assert(xr_frame_valid(valid)); auto unknownColor = valid; unknownColor.color_space = 3; assert(!xr_frame_valid(unknownColor)); - auto invalid = valid; invalid.version++; assert(!xr_frame_valid(invalid)); + auto invalid = valid; invalid.version = 3; assert(!xr_frame_valid(invalid)); invalid = valid; invalid.width = 3; assert(!xr_frame_valid(invalid)); invalid = valid; invalid.bytes--; assert(!xr_frame_valid(invalid)); invalid = valid; invalid.width = 8192; invalid.height = 4096; assert(!xr_frame_valid(invalid)); @@ -111,7 +111,7 @@ int main(int argc, char **argv) { assert(acknowledged == delayedHeader.frame_id && receiver.Take(&taken)); close(delayed); receiver.Stop(); assert(receiver.Start(socketPath)); - int descriptor = Connect(socketPath); auto header = Header(); header.version++; + int descriptor = Connect(socketPath); auto header = Header(); header.version = 3; assert(xr_frame_io(descriptor, &header, sizeof(header), true, xr_frame_now() + 200000000LL)); close(descriptor); WaitRejected(receiver, 1); assert(!receiver.Take(&taken)); descriptor = Connect(socketPath); header = Header(); header.captured_ns += 1000000000LL; diff --git a/tools/vr/tests/test_xr_shared.cpp b/tools/vr/tests/test_xr_shared.cpp new file mode 100644 index 000000000..fae4efde6 --- /dev/null +++ b/tools/vr/tests/test_xr_shared.cpp @@ -0,0 +1,67 @@ +#include "../src/xr_frame_channel.h" +#include +#include +#include + +static DroidDeckXrFrameHeader Header() { + DroidDeckXrFrameHeader header{}; + header.magic = DROIDDECK_XR_FRAME_MAGIC; header.version = DROIDDECK_XR_FRAME_VERSION; + header.width = 8; header.height = 4; header.bytes = 128; + header.color_space = DROIDDECK_XR_COLOR_SRGB_RGBA8; header.frame_id = 1; header.captured_ns = xr_frame_now(); + for (int eye = 0; eye < 2; eye++) { + for (int axis = 0; axis < 3; axis++) header.eye_pose[eye][axis][axis] = 1; + header.projection[eye][0][0] = header.projection[eye][1][1] = 1; + header.projection[eye][3][2] = -1; + } + return header; +} + +int main() { + char directory[] = "/tmp/droiddeck-shared-XXXXXX"; assert(mkdtemp(directory)); + std::string path = std::string(directory) + "/frames"; + int listener = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); assert(listener >= 0); + sockaddr_un address{}; address.sun_family = AF_UNIX; std::strcpy(address.sun_path, path.c_str()); + assert(bind(listener, reinterpret_cast(&address), sizeof(address)) == 0); + assert(listen(listener, 1) == 0); + int buffer = syscall(SYS_memfd_create, "frame-test", 1); assert(buffer >= 0); + assert(ftruncate(buffer, 128) == 0); + unsigned char pixels[128]; std::memset(pixels, 27, sizeof(pixels)); + assert(pwrite(buffer, pixels, sizeof(pixels), 0) == sizeof(pixels)); + auto header = Header(); + XrFrameSender sender; + std::thread peer([&] { + int client = accept(listener, nullptr, nullptr); assert(client >= 0); + DroidDeckXrFrameHeader received{}; XrFrameDescriptor shared; + assert(xr_frame_header(client, received, shared, xr_frame_now() + 1000000000LL)); + assert(received.version == DROIDDECK_XR_FRAME_SHARED_VERSION && xr_frame_valid(received)); + assert(shared.fd >= 0 && (fcntl(shared.fd, F_GETFD) & FD_CLOEXEC)); + unsigned char copy[128]; assert(pread(shared.fd, copy, sizeof(copy), 0) == sizeof(copy)); + for (auto value : copy) assert(value == 27); + assert(xr_frame_io(client, &received.frame_id, sizeof(received.frame_id), true, xr_frame_now() + 1000000000LL)); + close(client); + }); + assert(sender.SendShared(path, header, buffer)); peer.join(); + assert(!sender.SharedLeaseLost()); + XrFrameSender lost; + std::thread disconnected([&] { + int client = accept(listener, nullptr, nullptr); assert(client >= 0); + DroidDeckXrFrameHeader received{}; XrFrameDescriptor shared; + assert(xr_frame_header(client, received, shared, xr_frame_now() + 1000000000LL)); + assert(shared.fd >= 0); + close(client); + }); + assert(!lost.SendShared(path, header, buffer)); disconnected.join(); + assert(lost.SharedLeaseLost()); + assert(!lost.SendShared(path, header, buffer)); + close(listener); unlink(path.c_str()); + XrFrameSender absent; + assert(!absent.SendShared(path, header, buffer)); assert(!absent.SharedLeaseLost()); + XrFrameReceiver receiver; assert(receiver.Start(path)); + XrFrameSender invalid; + // An ordinary file is not a DMA-BUF and must never be acknowledged as one. + assert(!invalid.SendShared(path, Header(), buffer)); + assert(invalid.SharedLeaseLost() && receiver.Rejected() == 1); + DroidDeckXrPixels frame; assert(!receiver.Take(&frame)); + receiver.Stop(); close(buffer); unlink((path + ".lock").c_str()); rmdir(directory); + std::puts("XR shared handoff passed: descriptor ownership, copied payload, acknowledgement, disconnect quarantine, absent peer and invalid buffer rejection"); +}