From 679e3d89349d43bf6f511e0395cbdde94324c210 Mon Sep 17 00:00:00 2001 From: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:48:00 -0400 Subject: [PATCH] feat: add immersive Quest SteamVR to the standard app (#910) --- .github/workflows/build.yml | 22 +- app/build.gradle | 2 + app/src/main/AndroidManifest.xml | 14 + app/src/main/cpp/CMakeLists.txt | 1 + app/src/main/cpp/questxr/CMakeLists.txt | 13 + app/src/main/cpp/questxr/gpu_frames.h | 201 +++++ app/src/main/cpp/questxr/quest_xr.cpp | 777 ++++++++++++++++++ app/src/main/cpp/waylandcomp/src/vk_present.c | 33 +- app/src/main/cpp/waylandcomp/src/vk_present.h | 1 + .../cpp/waylandcomp/src/waylandcomp_jni.c | 6 + .../com/droiddeck/launcher/QuestXrActivity.kt | 118 +++ .../com/droiddeck/launcher/SessionActivity.kt | 9 +- .../launcher/session/SessionFiles.kt | 3 + .../launcher/session/SessionService.kt | 38 +- .../launcher/session/SessionVisibility.kt | 4 + .../launcher/ui/SessionDrawerPage.kt | 2 + .../droiddeck/launcher/ui/SessionOverlay.kt | 41 +- .../launcher/wayland/CompositorHost.kt | 9 + .../launcher/wayland/WaylandCompositor.java | 1 + app/src/main/res/values/strings.xml | 4 + .../launcher/session/SessionVisibilityTest.kt | 28 + .../launcher/ui/SessionDrawerPageTest.kt | 4 +- docs/agent-control.md | 15 +- docs/quest.md | 148 +++- tools/droiddeckctl | 104 ++- .../overlay/usr/local/bin/droiddeck-agent | 56 +- .../overlay/usr/local/bin/droiddeck-steamvr | 51 +- .../usr/local/bin/droiddeck-steamvr-game | 21 + .../usr/local/bin/droiddeck-steamvr-settings | 80 ++ tools/release/check_android_libraries.py | 2 +- tools/release/check_session_assets.py | 11 +- tools/test_quest_xr.py | 134 +++ tools/tests/test_droiddeck_agent.py | 30 + tools/tests/test_droiddeckctl_xr.py | 119 +++ tools/tests/test_quest_xr_validation.py | 85 ++ tools/tests/test_session_assets.py | 6 + tools/tests/test_steamvr_game.py | 65 ++ tools/tests/test_steamvr_helper.py | 204 +++++ tools/tests/test_vr_chaperone.py | 25 + tools/tests/test_vr_producer_lease.py | 45 + tools/tests/test_vr_semaphore_layer.py | 26 + tools/tests/test_vr_wsi.py | 23 + tools/vr/build.sh | 9 +- tools/vr/src/driver_droiddeck.cpp | 285 ++++++- tools/vr/src/presenter.cpp | 282 ++++++- tools/vr/src/presenter.h | 6 +- tools/vr/src/producer_lease.h | 207 +++++ tools/vr/src/vk_layer_droiddeck_vr.c | 520 +++++++----- tools/vr/src/wsi_capture.cpp | 404 +++++++++ tools/vr/src/wsi_capture.h | 17 + tools/vr/src/xr_chaperone.h | 40 + tools/vr/src/xr_display_mode.cpp | 11 + tools/vr/src/xr_display_mode.h | 9 + tools/vr/src/xr_frame.h | 53 ++ tools/vr/src/xr_frame_channel.h | 289 +++++++ tools/vr/src/xr_frame_math.h | 57 ++ tools/vr/src/xr_gpu_channel.h | 118 +++ tools/vr/src/xr_touch_input.h | 33 + tools/vr/src/xr_tracking.h | 59 ++ tools/vr/src/xr_tracking_file.h | 68 ++ tools/vr/src/xr_vsync_timing.h | 36 + tools/vr/tests/test_presenter_shutdown.cpp | 23 + tools/vr/tests/test_producer_gpu.c | 143 ++++ tools/vr/tests/test_producer_lease.c | 115 +++ tools/vr/tests/test_semaphore_layer.c | 240 ++++++ tools/vr/tests/test_wsi_capture.cpp | 95 +++ tools/vr/tests/test_xr_chaperone.cpp | 30 + tools/vr/tests/test_xr_frames.cpp | 152 ++++ tools/vr/tests/test_xr_gpu.cpp | 80 ++ tools/vr/tests/test_xr_shared.cpp | 67 ++ tools/vr/tests/test_xr_touch.cpp | 51 ++ tools/vr/tests/test_xr_tracking.cpp | 73 ++ tools/vr/tests/test_xr_vsync.cpp | 52 ++ 73 files changed, 5916 insertions(+), 289 deletions(-) create mode 100644 app/src/main/cpp/questxr/CMakeLists.txt create mode 100644 app/src/main/cpp/questxr/gpu_frames.h create mode 100644 app/src/main/cpp/questxr/quest_xr.cpp create mode 100644 app/src/main/java/com/droiddeck/launcher/QuestXrActivity.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/session/SessionVisibility.kt create mode 100644 app/src/test/java/com/droiddeck/launcher/session/SessionVisibilityTest.kt create mode 100755 tools/linuxfs/overlay/usr/local/bin/droiddeck-steamvr-game create mode 100644 tools/linuxfs/overlay/usr/local/bin/droiddeck-steamvr-settings create mode 100644 tools/test_quest_xr.py create mode 100644 tools/tests/test_droiddeckctl_xr.py create mode 100644 tools/tests/test_quest_xr_validation.py create mode 100644 tools/tests/test_steamvr_game.py create mode 100644 tools/tests/test_steamvr_helper.py create mode 100644 tools/tests/test_vr_chaperone.py create mode 100644 tools/tests/test_vr_producer_lease.py create mode 100644 tools/tests/test_vr_semaphore_layer.py create mode 100644 tools/tests/test_vr_wsi.py create mode 100644 tools/vr/src/producer_lease.h create mode 100644 tools/vr/src/wsi_capture.cpp create mode 100644 tools/vr/src/wsi_capture.h create mode 100644 tools/vr/src/xr_chaperone.h create mode 100644 tools/vr/src/xr_display_mode.cpp create mode 100644 tools/vr/src/xr_display_mode.h create mode 100644 tools/vr/src/xr_frame.h create mode 100644 tools/vr/src/xr_frame_channel.h create mode 100644 tools/vr/src/xr_frame_math.h create mode 100644 tools/vr/src/xr_gpu_channel.h create mode 100644 tools/vr/src/xr_touch_input.h create mode 100644 tools/vr/src/xr_tracking.h create mode 100644 tools/vr/src/xr_tracking_file.h create mode 100644 tools/vr/src/xr_vsync_timing.h create mode 100644 tools/vr/tests/test_presenter_shutdown.cpp create mode 100644 tools/vr/tests/test_producer_gpu.c create mode 100644 tools/vr/tests/test_producer_lease.c create mode 100644 tools/vr/tests/test_semaphore_layer.c create mode 100644 tools/vr/tests/test_wsi_capture.cpp create mode 100644 tools/vr/tests/test_xr_chaperone.cpp create mode 100644 tools/vr/tests/test_xr_frames.cpp create mode 100644 tools/vr/tests/test_xr_gpu.cpp create mode 100644 tools/vr/tests/test_xr_shared.cpp create mode 100644 tools/vr/tests/test_xr_touch.cpp create mode 100644 tools/vr/tests/test_xr_tracking.cpp create mode 100644 tools/vr/tests/test_xr_vsync.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7bd2d1d080..c76edb7626 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -193,7 +193,7 @@ jobs: run: | set -euo pipefail sudo apt-get update -qq - sudo apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu >/dev/null + sudo apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libvulkan1 mesa-vulkan-drivers >/dev/null D=app/src/main/assets/linuxfs mkdir -p "$D" aarch64-linux-gnu-g++ -shared -fPIC -O2 -Wall -Wno-attributes -Wno-nonnull-compare \ @@ -396,6 +396,24 @@ jobs: set -euo pipefail bash tools/linuxfs/check-preloads.sh app/src/main/assets/linuxfs + - name: Check Quest tracking IPC + run: | + g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_tracking.cpp -o /tmp/test-xr-tracking + /tmp/test-xr-tracking + g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_xr_touch.cpp -o /tmp/test-xr-touch + /tmp/test-xr-touch + g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_vsync.cpp -o /tmp/test-xr-vsync + /tmp/test-xr-vsync + g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_frames.cpp -o /tmp/test-xr-frames + /tmp/test-xr-frames + cc -Itools/vr/third_party/wayland -c tools/vr/src/wayland_client_stub.c -o /tmp/test-wayland-stub.o + cc -Itools/vr/third_party/wayland -c app/src/main/cpp/waylandcomp/generated/xdg-shell-protocol.c -o /tmp/test-xdg-shell.o + g++ -std=c++17 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-missing-field-initializers -pthread -Ibuild/vr/include \ + -Itools/vr/third_party/wayland -Iapp/src/main/cpp/waylandcomp/generated \ + tools/vr/tests/test_presenter_shutdown.cpp tools/vr/src/presenter.cpp /tmp/test-wayland-stub.o /tmp/test-xdg-shell.o \ + -ldl -o /tmp/test-presenter-shutdown + timeout 10 /tmp/test-presenter-shutdown + - name: Rust toolchain for the store engines (aarch64-linux-android) id: rust run: | @@ -429,7 +447,7 @@ jobs: path: | app/.cxx app/build/intermediates/cxx - key: cxx-${{ env.NDK_VERSION }}-${{ hashFiles('app/src/main/cpp/**') }} + key: cxx-${{ env.NDK_VERSION }}-${{ hashFiles('app/src/main/cpp/**', 'tools/vr/src/xr_*.h') }} restore-keys: | cxx-${{ env.NDK_VERSION }}- diff --git a/app/build.gradle b/app/build.gradle index 2938645666..d3ed3382b5 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -150,6 +150,7 @@ android { } buildFeatures { + prefab true compose true buildConfig true } @@ -225,6 +226,7 @@ if (!skipRust) android.sourceSets.main.jniLibs.srcDir(rustJniLibsDir) tasks.named('preBuild').configure { dependsOn(buildRustStores) } dependencies { + implementation 'org.khronos.openxr:openxr_loader_for_android:1.1.49' implementation 'com.github.luben:zstd-jni:1.5.7-4@aar' implementation 'org.apache.commons:commons-compress:1.27.1' implementation 'org.tukaani:xz:1.10' diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 2320652d70..e1e6e82f62 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -22,6 +22,7 @@ + + + + + + + + + + +#include +#include +#include +#include "xr_gpu_channel.h" + +class QuestGpuFrames { +public: + bool Start(const std::string &filename, EGLDisplay display) { + const char *extensions = reinterpret_cast(glGetString(GL_EXTENSIONS)); + const char *eglExtensions = eglQueryString(display, EGL_EXTENSIONS); + if (!extensions || !eglExtensions || !strstr(extensions, "GL_EXT_external_buffer") + || !strstr(eglExtensions, "EGL_ANDROID_get_native_client_buffer")) return false; + storage_ = reinterpret_cast(eglGetProcAddress("glBufferStorageExternalEXT")); + client_ = reinterpret_cast(eglGetProcAddress("eglGetNativeClientBufferANDROID")); + if (strstr(extensions, "GL_OES_texture_view")) view_ = reinterpret_cast(eglGetProcAddress("glTextureViewOES")); + else if (strstr(extensions, "GL_EXT_texture_view")) view_ = reinterpret_cast(eglGetProcAddress("glTextureViewEXT")); + if (!storage_ || !client_ || !view_ || filename.size() >= sizeof(sockaddr_un::sun_path)) return false; + const char *source = "#version 310 es\nlayout(local_size_x=8,local_size_y=8)in;" + "layout(std430,binding=0)readonly buffer Source{uint pixels[];};" + "layout(rgba8,binding=0)writeonly uniform highp image2D target;uniform bool bgra;" + "void main(){ivec2 p=ivec2(gl_GlobalInvocationID.xy),size=imageSize(target);" + "if(any(greaterThanEqual(p,size)))return;vec4 value=unpackUnorm4x8(pixels[p.y*size.x+p.x]);" + "imageStore(target,p,bgra?value.bgra:value);}"; + GLuint shader = glCreateShader(GL_COMPUTE_SHADER); glShaderSource(shader, 1, &source, nullptr); glCompileShader(shader); + program_ = glCreateProgram(); glAttachShader(program_, shader); glLinkProgram(program_); glDeleteShader(shader); + GLint linked{}; glGetProgramiv(program_, GL_LINK_STATUS, &linked); + if (!linked) { Stop(); return false; } + lock_ = open((filename + ".lock").c_str(), O_CREAT | O_RDWR | O_CLOEXEC | O_NOFOLLOW, 0600); + struct stat status{}; + if (lock_ < 0 || fstat(lock_, &status) || !S_ISREG(status.st_mode) || status.st_uid != getuid() + || flock(lock_, LOCK_EX | LOCK_NB)) { Stop(); return false; } + if (lstat(filename.c_str(), &status) == 0) { + if (!S_ISSOCK(status.st_mode) || status.st_uid != getuid() || unlink(filename.c_str())) { Stop(); return false; } + } else if (errno != ENOENT) { Stop(); return false; } + listener_ = socket(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC | SOCK_NONBLOCK, 0); + sockaddr_un address{}; address.sun_family = AF_UNIX; strcpy(address.sun_path, filename.c_str()); + if (listener_ < 0 || bind(listener_, reinterpret_cast(&address), sizeof(address))) { Stop(); return false; } + filename_ = filename; + if (chmod(filename.c_str(), 0600) || listen(listener_, 2)) { Stop(); return false; } + return true; + } + + bool Take(DroidDeckXrFrameHeader &header, GLuint &texture) { + if (listener_ < 0) return false; + if (peer_ < 0) { + peer_ = accept4(listener_, nullptr, nullptr, SOCK_CLOEXEC | SOCK_NONBLOCK); + if (peer_ < 0) return false; + if (!xr_frame_peer(peer_)) { Disconnect(); return false; } + configured_ = false; deadline_ = xr_frame_now() + 2000000000LL; + } + pollfd ready{peer_, POLLIN, 0}; + if (poll(&ready, 1, 0) <= 0) { + if (!configured_ && xr_frame_now() > deadline_) Disconnect(); + return false; + } + if (!(ready.revents & POLLIN)) { Disconnect(); return false; } + if (!configured_) { + XrGpuRequest request; + if (!xr_gpu_receive(peer_, &request, sizeof(request)) || request.magic != XR_GPU_MAGIC + || request.version != XR_GPU_VERSION || !xr_gpu_dimensions(request.width, request.height) + || !Allocate(request.width, request.height)) { Disconnect(); return false; } + XrGpuPoolInfo info; info.width = width_; info.height = height_; + for (unsigned index = 0; index < XR_GPU_SLOTS; index++) info.sizes[index] = sizes_[index]; + configured_ = xr_gpu_send(peer_, &info, sizeof(info), fds_.data(), fds_.size()); + for (int &fd : fds_) { if (fd >= 0) close(fd); fd = -1; } + if (!configured_) Disconnect(); + return false; + } + XrGpuFrame frame; + if (!xr_gpu_receive(peer_, &frame, sizeof(frame))) { Disconnect(); return false; } + auto state = xr_gpu_frame_state(frame, width_, height_, lastFrame_, xr_frame_now()); + if (state == XrGpuFrameState::Invalid) { + rejected_++; Disconnect(); return false; + } + lastFrame_ = frame.header.frame_id; + if (state == XrGpuFrameState::Stale) { + // No GPU read was submitted, so the producer can safely reclaim a dropped frame. + dropped_++; + if (!xr_gpu_send(peer_, &frame.header.frame_id, sizeof(frame.header.frame_id))) Disconnect(); + return false; + } + if (!Output()) { Disconnect(); return false; } + glMemoryBarrier(GL_ALL_BARRIER_BITS); + glUseProgram(program_); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, buffers_[frame.slot]); + glUniform1i(glGetUniformLocation(program_, "bgra"), !!(frame.flags & XR_GPU_BGRA)); + glBindImageTexture(0, raw_, 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGBA8); + glDispatchCompute((width_ + 7) / 8, (height_ + 7) / 8, 1); + glMemoryBarrier(GL_TEXTURE_FETCH_BARRIER_BIT | GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); + // The producer may overwrite this slot only after the native GPU has finished reading it. + glFinish(); + if (glGetError() != GL_NO_ERROR) { rejected_++; Disconnect(); return false; } + if (!xr_gpu_send(peer_, &frame.header.frame_id, sizeof(frame.header.frame_id))) Disconnect(); + received_++; + header = frame.header; + texture = header.color_space == DROIDDECK_XR_COLOR_SRGB_RGBA8 ? srgb_ : raw_; + return true; + } + + uint64_t Received() const { return received_; } + uint64_t Rejected() const { return rejected_; } + uint64_t Dropped() const { return dropped_; } + void Stop() { + Disconnect(); DropPool(); + glDeleteTextures(1, &srgb_); glDeleteTextures(1, &raw_); srgb_ = raw_ = 0; + if (program_) glDeleteProgram(program_); program_ = 0; + if (listener_ >= 0) close(listener_); listener_ = -1; + if (!filename_.empty()) unlink(filename_.c_str()); filename_.clear(); + if (lock_ >= 0) close(lock_); lock_ = -1; + } +private: + int listener_ = -1, peer_ = -1, lock_ = -1; + std::string filename_; + bool configured_ = false; + int64_t deadline_{}; + uint32_t width_{}, height_{}, outputWidth_{}, outputHeight_{}; + uint64_t lastFrame_{}, received_{}, rejected_{}, dropped_{}; + std::array hardware_{}; + std::array buffers_{}; + std::array fds_{{-1, -1, -1}}; + std::array sizes_{}; + GLuint raw_{}, srgb_{}, program_{}; + PFNGLBUFFERSTORAGEEXTERNALEXTPROC storage_{}; + PFNEGLGETNATIVECLIENTBUFFERANDROIDPROC client_{}; + PFNGLTEXTUREVIEWOESPROC view_{}; + + void Disconnect() { if (peer_ >= 0) close(peer_); peer_ = -1; configured_ = false; lastFrame_ = 0; } + void DropPool() { + if (buffers_[0]) glFinish(); + glDeleteBuffers(buffers_.size(), buffers_.data()); buffers_.fill(0); + for (auto *&buffer : hardware_) { if (buffer) AHardwareBuffer_release(buffer); buffer = nullptr; } + for (int &fd : fds_) { if (fd >= 0) close(fd); fd = -1; } + } + static int AllocationFd(AHardwareBuffer *buffer, uint64_t bytes, uint64_t &allocation) { + int pair[2]; + if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, pair)) return -1; + if (AHardwareBuffer_sendHandleToUnixSocket(buffer, pair[0])) { close(pair[0]); close(pair[1]); return -1; } + close(pair[0]); + int selected = -1; unsigned candidates = 0; + for (;;) { + unsigned char data[1024]; iovec bytesReceived{data, sizeof(data)}; + alignas(cmsghdr) unsigned char control[CMSG_SPACE(32 * sizeof(int))]{}; + msghdr message{}; message.msg_iov = &bytesReceived; message.msg_iovlen = 1; + message.msg_control = control; message.msg_controllen = sizeof(control); + ssize_t result = recvmsg(pair[1], &message, MSG_CMSG_CLOEXEC); + if (result < 0 && errno == EINTR) continue; + if (result <= 0) break; + for (auto *item = CMSG_FIRSTHDR(&message); item; item = CMSG_NXTHDR(&message, item)) { + if (item->cmsg_level != SOL_SOCKET || item->cmsg_type != SCM_RIGHTS) continue; + auto *fds = reinterpret_cast(CMSG_DATA(item)); + for (unsigned index = 0; index < (item->cmsg_len - CMSG_LEN(0)) / sizeof(int); index++) { + struct stat status{}; + if (!fstat(fds[index], &status) && status.st_size >= static_cast(bytes) + && status.st_size <= static_cast(bytes + 65536)) { + candidates++; if (selected >= 0) close(selected); + selected = fds[index]; allocation = status.st_size; + } else close(fds[index]); + } + } + if (message.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) { candidates = 0; break; } + } + close(pair[1]); + if (candidates != 1) { if (selected >= 0) close(selected); return -1; } + return selected; + } + bool Allocate(uint32_t width, uint32_t height) { + DropPool(); width_ = width; height_ = height; + const uint64_t bytes = uint64_t(width) * height * 4; + GLint64 limit{}; glGetInteger64v(GL_MAX_SHADER_STORAGE_BLOCK_SIZE, &limit); + if (bytes > static_cast(limit)) return false; + AHardwareBuffer_Desc description{}; description.width = bytes; description.height = description.layers = 1; + description.format = AHARDWAREBUFFER_FORMAT_BLOB; + description.usage = AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER; + glGenBuffers(buffers_.size(), buffers_.data()); + for (unsigned index = 0; index < XR_GPU_SLOTS; index++) { + if (AHardwareBuffer_allocate(&description, &hardware_[index])) { DropPool(); return false; } + fds_[index] = AllocationFd(hardware_[index], bytes, sizes_[index]); + if (fds_[index] < 0) { DropPool(); return false; } + glBindBuffer(GL_SHADER_STORAGE_BUFFER, buffers_[index]); + storage_(GL_SHADER_STORAGE_BUFFER, 0, bytes, client_(hardware_[index]), 0); + if (glGetError() != GL_NO_ERROR) { DropPool(); return false; } + } + glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0); + return true; + } + bool Output() { + if (raw_ && outputWidth_ == width_ && outputHeight_ == height_) return true; + glDeleteTextures(1, &srgb_); glDeleteTextures(1, &raw_); srgb_ = raw_ = 0; + glGenTextures(1, &raw_); glBindTexture(GL_TEXTURE_2D, raw_); + glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, width_, height_); + glGenTextures(1, &srgb_); view_(srgb_, GL_TEXTURE_2D, raw_, GL_SRGB8_ALPHA8, 0, 1, 0, 1); + for (GLuint texture : {raw_, srgb_}) { + glBindTexture(GL_TEXTURE_2D, texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } + outputWidth_ = width_; outputHeight_ = height_; + return glGetError() == GL_NO_ERROR; + } +}; diff --git a/app/src/main/cpp/questxr/quest_xr.cpp b/app/src/main/cpp/questxr/quest_xr.cpp new file mode 100644 index 0000000000..09423c7d1a --- /dev/null +++ b/app/src/main/cpp/questxr/quest_xr.cpp @@ -0,0 +1,777 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "xr_tracking.h" +#include "xr_frame_channel.h" +#include "xr_frame_math.h" +#include "gpu_frames.h" + +static void check(XrResult result, const char *operation) { + if (XR_FAILED(result)) throw std::runtime_error(std::string(operation) + ": " + std::to_string(result)); +} + +struct Eye { + XrSwapchain swapchain = XR_NULL_HANDLE; + int width = 0, height = 0; + std::vector images; +}; + +class QuestXR { +public: + explicit QuestXR(android_app *app) : app_(app) {} + ~QuestXR() { cleanup(); } + + void run() { + initialize(); + while (!app_->destroyRequested && !exit_) { + android_poll_source *source = nullptr; + int events = 0; + int pollTimeout = running_ ? 0 : 100; + for (int eventCount = 0; eventCount < 32; eventCount++) { + if (ALooper_pollOnce(pollTimeout, nullptr, &events, reinterpret_cast(&source)) < 0) break; + if (source) source->process(app_, source); + if (app_->destroyRequested) break; + pollTimeout = 0; + } + receiveFrame(); + pollEvents(); + if (running_ && !app_->destroyRequested) frame(); + } + if (exit_ && !app_->destroyRequested) ANativeActivity_finish(app_->activity); + } + + void health(const char *error = "") { + int steamvrState = -1; + if (statusCallback_) { + steamvrState = env_->CallIntMethod(app_->activity->clazz, statusCallback_); + if (env_->ExceptionCheck()) { env_->ExceptionClear(); steamvrState = -1; } + } + std::string path = std::string(app_->activity->externalDataPath) + "/xr-health.json"; + std::string temp = path + ".tmp"; + FILE *file = std::fopen(temp.c_str(), "w"); + if (!file) return; + std::fprintf(file, + "{\"frames\":%llu,\"rendered\":%llu,\"sessionState\":%d,\"headFlags\":%llu," + "\"headPosition\":[%.6f,%.6f,%.6f],\"controllersActive\":[%s,%s]," + "\"periodNs\":%lld,\"surfaceSwapchain\":%s,\"source\":\"%s\",\"guestFrames\":%lld,\"steamvrState\":%d," + "\"receivedFrames\":%llu,\"rejectedFrames\":%llu,\"uploadedFrames\":%llu,\"sourceFrameId\":%llu," + "\"gpuReceivedFrames\":%llu,\"gpuDroppedFrames\":%llu,\"gpuVerifiedFrameId\":%llu,\"gpuChecksum\":%u,\"error\":\"%s\"}\n", + (unsigned long long)frames_, (unsigned long long)rendered_, state_, + (unsigned long long)head_.locationFlags, + head_.pose.position.x, head_.pose.position.y, head_.pose.position.z, + handActive_[0] ? "true" : "false", handActive_[1] ? "true" : "false", + (long long)period_, "false", "frames", + 0LL, steamvrState, (unsigned long long)(receiver_.Received() + gpuFrames_.Received()), + (unsigned long long)(receiver_.Rejected() + gpuFrames_.Rejected()), + (unsigned long long)uploadedFrames_, (unsigned long long)sourceHeader_.frame_id, + (unsigned long long)gpuFrames_.Received(), (unsigned long long)gpuFrames_.Dropped(), + (unsigned long long)verifiedFrame_, gpuChecksum_, error); + std::fclose(file); + std::rename(temp.c_str(), path.c_str()); + } + +private: + android_app *app_; + JNIEnv *env_ = nullptr; + XrInstance instance_ = XR_NULL_HANDLE; + XrSession session_ = XR_NULL_HANDLE; + XrSystemId system_ = XR_NULL_SYSTEM_ID; + XrSpace world_ = XR_NULL_HANDLE, view_ = XR_NULL_HANDLE; + XrActionSet actions_ = XR_NULL_HANDLE; + XrAction grip_ = XR_NULL_HANDLE, trigger_ = XR_NULL_HANDLE, vibration_ = XR_NULL_HANDLE; + XrAction squeeze_ = XR_NULL_HANDLE, stick_ = XR_NULL_HANDLE, stickClick_ = XR_NULL_HANDLE; + XrAction primary_ = XR_NULL_HANDLE, secondary_ = XR_NULL_HANDLE, menu_ = XR_NULL_HANDLE; + XrAction triggerTouch_ = XR_NULL_HANDLE, stickTouch_ = XR_NULL_HANDLE, primaryTouch_ = XR_NULL_HANDLE; + XrAction secondaryTouch_ = XR_NULL_HANDLE, thumbrestTouch_ = XR_NULL_HANDLE; + DroidDeckXrHand inputs_[2]{}; + PFN_xrConvertTimeToTimespecTimeKHR convertTime_ = nullptr; + bool timespecSupported_ = false; + PFN_xrPerfSettingsSetPerformanceLevelEXT setPerformance_ = nullptr; + bool performanceSupported_ = false; + XrPath hands_[2]{}; + XrSpace handSpaces_[2]{}; + XrSpaceLocation head_{XR_TYPE_SPACE_LOCATION}; + XrSpaceLocation handLocations_[2]{{XR_TYPE_SPACE_LOCATION}, {XR_TYPE_SPACE_LOCATION}}; + bool handActive_[2]{}; + bool running_ = false, exit_ = false, quitRequested_ = false; + bool guestAttached_ = false, verifyPixels_ = false; + XrFrameReceiver receiver_; + QuestGpuFrames gpuFrames_; + DroidDeckXrFrameHeader sourceHeader_{}; + uint64_t uploadedFrames_ = 0, verifiedFrame_ = 0; + uint32_t gpuChecksum_ = 0; + GLuint sourceTexture_ = 0, sourceProgram_ = 0; + GLuint cpuTexture_ = 0; + jmethodID frameCallback_ = nullptr; + jmethodID activeCallback_ = nullptr; + jmethodID statusCallback_ = nullptr; + DroidDeckXrTracking *tracking_ = nullptr; + DroidDeckXrHaptics *haptics_ = nullptr; + uint64_t lastPulse_[2]{}; + XrSessionState state_ = XR_SESSION_STATE_UNKNOWN; + uint64_t frames_ = 0, rendered_ = 0; + XrDuration period_ = 0; + EGLDisplay display_ = EGL_NO_DISPLAY; + EGLContext context_ = EGL_NO_CONTEXT; + EGLSurface surface_ = EGL_NO_SURFACE; + std::array eyes_; + GLuint framebuffer_ = 0, vertexArray_ = 0; + GLint sourceEncoding_ = 0, eyeEncoding_ = 0; + + std::string captureRequest() { + if (frames_ % 9) return {}; + std::string file = std::string(app_->activity->externalDataPath) + "/xr-capture.request"; + FILE *request = std::fopen(file.c_str(), "r"); + if (!request) return {}; + char token[34]{}; + size_t size = std::fread(token, 1, sizeof(token) - 1, request); + std::fclose(request); + unlink(file.c_str()); + if (size != 32 || std::strspn(token, "0123456789abcdef") != 32) return {}; + unlink((std::string(app_->activity->externalDataPath) + "/xr-capture.json").c_str()); + return token; + } + + bool captureTexture(GLuint texture, int width, int height, const char *name) { + glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture, 0); + GLint encoding = 0; + glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_FRAMEBUFFER_ATTACHMENT_COLOR_ENCODING, &encoding); + if (!std::strcmp(name, "source")) sourceEncoding_ = encoding; + else eyeEncoding_ = encoding; + std::vector pixels((size_t)width * height * 4); + glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, pixels.data()); + bool ok = glGetError() == GL_NO_ERROR; + glBindFramebuffer(GL_FRAMEBUFFER, 0); + if (!ok) return false; + std::string file = std::string(app_->activity->externalDataPath) + + "/xr-capture-" + name + ".ppm"; + std::string temporary = file + ".tmp"; + FILE *output = std::fopen(temporary.c_str(), "wb"); + if (!output) return false; + ok = std::fprintf(output, "P6\n%d %d\n255\n", width, height) > 0; + std::vector row((size_t)width * 3); + for (int y = height - 1; y >= 0 && ok; y--) { + for (int x = 0; x < width; x++) + std::memcpy(row.data() + x * 3, pixels.data() + ((size_t)y * width + x) * 4, 3); + ok = std::fwrite(row.data(), 1, row.size(), output) == row.size(); + } + if (std::fclose(output) != 0) ok = false; + if (!ok) { unlink(temporary.c_str()); return false; } + return std::rename(temporary.c_str(), file.c_str()) == 0; + } + + void captureResult(const std::string &token, bool ok, bool valid, + const XrCompositionLayerProjectionView (&views)[2], XrTime displayTime) { + std::string file = std::string(app_->activity->externalDataPath) + "/xr-capture.json"; + std::string temporary = file + ".tmp"; + FILE *output = std::fopen(temporary.c_str(), "w"); + if (!output) return; + std::fprintf(output, "{\"id\":\"%s\",\"ok\":%s,\"validLayer\":%s,\"frame\":%llu," + "\"sourceFrameId\":%llu,\"displayTime\":%lld,\"sessionState\":%d,\"eyes\":[", + token.c_str(), ok ? "true" : "false", valid ? "true" : "false", + (unsigned long long)frames_, (unsigned long long)sourceHeader_.frame_id, (long long)displayTime, state_); + for (int eye = 0; eye < 2; eye++) { + const auto &pose = views[eye].pose; + const auto &fov = views[eye].fov; + std::fprintf(output, "%s{\"width\":%d,\"height\":%d,\"position\":[%g,%g,%g]," + "\"orientation\":[%g,%g,%g,%g],\"fov\":[%g,%g,%g,%g]}", + eye ? "," : "", eyes_[eye].width, eyes_[eye].height, + pose.position.x, pose.position.y, pose.position.z, + pose.orientation.x, pose.orientation.y, pose.orientation.z, pose.orientation.w, + fov.angleLeft, fov.angleRight, fov.angleUp, fov.angleDown); + } + const char *extensions = reinterpret_cast(glGetString(GL_EXTENSIONS)); + const bool srgbControl = extensions && std::strstr(extensions, "GL_EXT_sRGB_write_control"); + std::fprintf(output, "],\"sourceImage\":true,\"sourceColorSpace\":\"%s\",\"sourceEncoding\":%d,\"eyeEncoding\":%d," + "\"srgbWriteControl\":%s,\"srgbWriteEnabled\":%s}\n", + sourceHeader_.color_space == DROIDDECK_XR_COLOR_SRGB_RGBA8 ? "srgb" : "linear", sourceEncoding_, eyeEncoding_, + srgbControl ? "true" : "false", srgbControl && glIsEnabled(0x8DB9) ? "true" : "false"); + if (std::fclose(output) == 0) std::rename(temporary.c_str(), file.c_str()); + } + + XrPath path(const char *text) { + XrPath result{}; + check(xrStringToPath(instance_, text, &result), "xrStringToPath"); + return result; + } + + void initialize() { + app_->userData = this; + app_->onInputEvent = [](android_app *app, AInputEvent *event) -> int32_t { + if (AInputEvent_getType(event) != AINPUT_EVENT_TYPE_KEY || AKeyEvent_getKeyCode(event) != AKEYCODE_BACK + || AKeyEvent_getAction(event) != AKEY_EVENT_ACTION_UP) return 0; + auto *runtime = static_cast(app->userData); + if (runtime->running_) { + XrResult result = xrRequestExitSession(runtime->session_); + if (XR_SUCCEEDED(result)) runtime->quitRequested_ = true; + else __android_log_print(ANDROID_LOG_ERROR, "DroidDeckXR", "exit request failed: %d", result); + } else ANativeActivity_finish(app->activity); + return 1; + }; + app_->activity->vm->AttachCurrentThread(&env_, nullptr); + jclass activityClass = env_->GetObjectClass(app_->activity->clazz); + jmethodID getIntent = env_->GetMethodID(activityClass, "getIntent", "()Landroid/content/Intent;"); + jobject intent = env_->CallObjectMethod(app_->activity->clazz, getIntent); + jclass intentClass = env_->GetObjectClass(intent); + jmethodID getBoolean = env_->GetMethodID(intentClass, "getBooleanExtra", "(Ljava/lang/String;Z)Z"); + jstring extra = env_->NewStringUTF("frame_verify"); + verifyPixels_ = env_->CallBooleanMethod(intent, getBoolean, extra, JNI_FALSE); + env_->DeleteLocalRef(extra); + activeCallback_ = env_->GetMethodID(activityClass, "onXrActive", "(Z)V"); + frameCallback_ = env_->GetMethodID(activityClass, "onXrFrame", "(J)V"); + statusCallback_ = env_->GetMethodID(activityClass, "onXrStatus", "()I"); + if (env_->ExceptionCheck()) { env_->ExceptionClear(); throw std::runtime_error("XR activity bridge methods unavailable"); } + env_->DeleteLocalRef(intentClass); env_->DeleteLocalRef(intent); env_->DeleteLocalRef(activityClass); + PFN_xrInitializeLoaderKHR initializeLoader = nullptr; + check(xrGetInstanceProcAddr(XR_NULL_HANDLE, "xrInitializeLoaderKHR", + reinterpret_cast(&initializeLoader)), "loader init entrypoint"); + XrLoaderInitInfoAndroidKHR loader{XR_TYPE_LOADER_INIT_INFO_ANDROID_KHR}; + loader.applicationVM = app_->activity->vm; + loader.applicationContext = app_->activity->clazz; + check(initializeLoader(reinterpret_cast(&loader)), "xrInitializeLoaderKHR"); + uint32_t count = 0; + check(xrEnumerateInstanceExtensionProperties(nullptr, 0, &count, nullptr), "extensions count"); + std::vector extensions(count, {XR_TYPE_EXTENSION_PROPERTIES}); + check(xrEnumerateInstanceExtensionProperties(nullptr, count, &count, extensions.data()), "extensions"); + for (const auto &extension : extensions) { + if (!std::strcmp(extension.extensionName, XR_KHR_CONVERT_TIMESPEC_TIME_EXTENSION_NAME)) + timespecSupported_ = true; + if (!std::strcmp(extension.extensionName, XR_EXT_PERFORMANCE_SETTINGS_EXTENSION_NAME)) + performanceSupported_ = true; + } + std::vector enabled{XR_KHR_ANDROID_CREATE_INSTANCE_EXTENSION_NAME, XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME}; + if (timespecSupported_) enabled.push_back(XR_KHR_CONVERT_TIMESPEC_TIME_EXTENSION_NAME); + if (performanceSupported_) enabled.push_back(XR_EXT_PERFORMANCE_SETTINGS_EXTENSION_NAME); + XrInstanceCreateInfoAndroidKHR androidInfo{XR_TYPE_INSTANCE_CREATE_INFO_ANDROID_KHR}; + androidInfo.applicationVM = app_->activity->vm; + androidInfo.applicationActivity = app_->activity->clazz; + XrInstanceCreateInfo create{XR_TYPE_INSTANCE_CREATE_INFO}; + create.next = &androidInfo; + std::strcpy(create.applicationInfo.applicationName, "DroidDeck XR"); + create.applicationInfo.applicationVersion = 1; + create.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 0, 0); + create.enabledExtensionCount = enabled.size(); + create.enabledExtensionNames = enabled.data(); + check(xrCreateInstance(&create, &instance_), "xrCreateInstance"); + if (performanceSupported_) { + XrResult result = xrGetInstanceProcAddr(instance_, "xrPerfSettingsSetPerformanceLevelEXT", + reinterpret_cast(&setPerformance_)); + if (XR_FAILED(result)) setPerformance_ = nullptr; + } + if (timespecSupported_) check(xrGetInstanceProcAddr(instance_, "xrConvertTimeToTimespecTimeKHR", + reinterpret_cast(&convertTime_)), "time conversion entrypoint"); + XrSystemGetInfo systemInfo{XR_TYPE_SYSTEM_GET_INFO}; + systemInfo.formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY; + check(xrGetSystem(instance_, &systemInfo, &system_), "xrGetSystem"); + PFN_xrGetOpenGLESGraphicsRequirementsKHR requirementsFunction = nullptr; + check(xrGetInstanceProcAddr(instance_, "xrGetOpenGLESGraphicsRequirementsKHR", + reinterpret_cast(&requirementsFunction)), "GLES requirements entrypoint"); + XrGraphicsRequirementsOpenGLESKHR requirements{XR_TYPE_GRAPHICS_REQUIREMENTS_OPENGL_ES_KHR}; + check(requirementsFunction(instance_, system_, &requirements), "GLES requirements"); + if (requirements.minApiVersionSupported > XR_MAKE_VERSION(3, 0, 0) + || requirements.maxApiVersionSupported < XR_MAKE_VERSION(3, 0, 0)) + throw std::runtime_error("Runtime does not support GLES 3.0"); + display_ = eglGetDisplay(EGL_DEFAULT_DISPLAY); + if (!eglInitialize(display_, nullptr, nullptr)) throw std::runtime_error("eglInitialize failed"); + const EGLint attributes[] = {EGL_RENDERABLE_TYPE, EGL_OPENGL_ES3_BIT, EGL_SURFACE_TYPE, EGL_PBUFFER_BIT | EGL_WINDOW_BIT, + EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_ALPHA_SIZE, 8, EGL_DEPTH_SIZE, 24, EGL_NONE}; + EGLConfig config{}; + EGLint found = 0; + if (!eglChooseConfig(display_, attributes, &config, 1, &found) || found != 1) + throw std::runtime_error("EGL config unavailable"); + const EGLint contextAttributes[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE}; + context_ = eglCreateContext(display_, config, EGL_NO_CONTEXT, contextAttributes); + const EGLint surfaceAttributes[] = {EGL_WIDTH, 16, EGL_HEIGHT, 16, EGL_NONE}; + surface_ = eglCreatePbufferSurface(display_, config, surfaceAttributes); + if (!eglMakeCurrent(display_, surface_, surface_, context_)) throw std::runtime_error("eglMakeCurrent failed"); + XrGraphicsBindingOpenGLESAndroidKHR binding{XR_TYPE_GRAPHICS_BINDING_OPENGL_ES_ANDROID_KHR}; + binding.display = display_; binding.config = config; binding.context = context_; + XrSessionCreateInfo sessionInfo{XR_TYPE_SESSION_CREATE_INFO}; + sessionInfo.next = &binding; sessionInfo.systemId = system_; + check(xrCreateSession(instance_, &sessionInfo, &session_), "xrCreateSession"); + XrReferenceSpaceCreateInfo reference{XR_TYPE_REFERENCE_SPACE_CREATE_INFO}; + reference.poseInReferenceSpace.orientation.w = 1; + reference.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_STAGE; + XrResult referenceResult = xrCreateReferenceSpace(session_, &reference, &world_); + if (referenceResult == XR_ERROR_REFERENCE_SPACE_UNSUPPORTED) { + reference.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_LOCAL; + reference.poseInReferenceSpace.position.y = -1.6f; + check(xrCreateReferenceSpace(session_, &reference, &world_), "local standing space"); + } else check(referenceResult, "stage space"); + reference.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW; + reference.poseInReferenceSpace.position.y = 0; + check(xrCreateReferenceSpace(session_, &reference, &view_), "view space"); + initializeActions(); + initializeImages(); + initializeGraphics(); + initializeTracking(); + { + std::string filename = std::string(app_->activity->internalDataPath) + "/.wayland-rt/" DROIDDECK_XR_FRAME_SOCKET; + if (!receiver_.Start(filename)) throw std::runtime_error("XR frame receiver unavailable"); + bool gpu = !verifyPixels_ && gpuFrames_.Start(std::string(app_->activity->internalDataPath) + "/.wayland-rt/" DROIDDECK_XR_GPU_SOCKET, display_); + __android_log_print(ANDROID_LOG_INFO, "DroidDeckXR", "GPU frame channel available=%d", gpu); + } + health(); + } + + void initializeActions() { + hands_[0] = path("/user/hand/left"); hands_[1] = path("/user/hand/right"); + XrActionSetCreateInfo setInfo{XR_TYPE_ACTION_SET_CREATE_INFO}; + std::strcpy(setInfo.actionSetName, "quest"); std::strcpy(setInfo.localizedActionSetName, "Quest controllers"); + check(xrCreateActionSet(instance_, &setInfo, &actions_), "action set"); + auto action = [&](const char *name, XrActionType type, XrAction *handle) { + XrActionCreateInfo info{XR_TYPE_ACTION_CREATE_INFO}; + std::strcpy(info.actionName, name); std::strcpy(info.localizedActionName, name); + info.actionType = type; info.countSubactionPaths = 2; info.subactionPaths = hands_; + check(xrCreateAction(actions_, &info, handle), "create action"); + }; + action("grip_pose", XR_ACTION_TYPE_POSE_INPUT, &grip_); + action("trigger", XR_ACTION_TYPE_FLOAT_INPUT, &trigger_); + action("squeeze", XR_ACTION_TYPE_FLOAT_INPUT, &squeeze_); + action("stick", XR_ACTION_TYPE_VECTOR2F_INPUT, &stick_); + action("stick_click", XR_ACTION_TYPE_BOOLEAN_INPUT, &stickClick_); + action("primary", XR_ACTION_TYPE_BOOLEAN_INPUT, &primary_); + action("secondary", XR_ACTION_TYPE_BOOLEAN_INPUT, &secondary_); + action("menu", XR_ACTION_TYPE_BOOLEAN_INPUT, &menu_); + action("trigger_touch", XR_ACTION_TYPE_BOOLEAN_INPUT, &triggerTouch_); + action("stick_touch", XR_ACTION_TYPE_BOOLEAN_INPUT, &stickTouch_); + action("primary_touch", XR_ACTION_TYPE_BOOLEAN_INPUT, &primaryTouch_); + action("secondary_touch", XR_ACTION_TYPE_BOOLEAN_INPUT, &secondaryTouch_); + action("thumbrest_touch", XR_ACTION_TYPE_BOOLEAN_INPUT, &thumbrestTouch_); + action("vibration", XR_ACTION_TYPE_VIBRATION_OUTPUT, &vibration_); + std::vector bindings; + for (const char *hand : {"left", "right"}) { + std::string root = std::string("/user/hand/") + hand; + bindings.push_back({grip_, path((root + "/input/grip/pose").c_str())}); + bindings.push_back({trigger_, path((root + "/input/trigger/value").c_str())}); + bindings.push_back({triggerTouch_, path((root + "/input/trigger/touch").c_str())}); + bindings.push_back({squeeze_, path((root + "/input/squeeze/value").c_str())}); + bindings.push_back({stick_, path((root + "/input/thumbstick").c_str())}); + bindings.push_back({stickClick_, path((root + "/input/thumbstick/click").c_str())}); + bindings.push_back({stickTouch_, path((root + "/input/thumbstick/touch").c_str())}); + bindings.push_back({thumbrestTouch_, path((root + "/input/thumbrest/touch").c_str())}); + bindings.push_back({primary_, path((root + (std::strcmp(hand, "left") == 0 ? "/input/x/click" : "/input/a/click")).c_str())}); + bindings.push_back({secondary_, path((root + (std::strcmp(hand, "left") == 0 ? "/input/y/click" : "/input/b/click")).c_str())}); + bindings.push_back({primaryTouch_, path((root + (std::strcmp(hand, "left") == 0 ? "/input/x/touch" : "/input/a/touch")).c_str())}); + bindings.push_back({secondaryTouch_, path((root + (std::strcmp(hand, "left") == 0 ? "/input/y/touch" : "/input/b/touch")).c_str())}); + bindings.push_back({vibration_, path((root + "/output/haptic").c_str())}); + } + bindings.push_back({menu_, path("/user/hand/left/input/menu/click")}); + XrInteractionProfileSuggestedBinding suggestion{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING}; + suggestion.interactionProfile = path("/interaction_profiles/oculus/touch_controller"); + suggestion.countSuggestedBindings = bindings.size(); suggestion.suggestedBindings = bindings.data(); + check(xrSuggestInteractionProfileBindings(instance_, &suggestion), "Touch bindings"); + XrSessionActionSetsAttachInfo attach{XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO}; + attach.countActionSets = 1; attach.actionSets = &actions_; + check(xrAttachSessionActionSets(session_, &attach), "attach actions"); + for (int hand = 0; hand < 2; hand++) { + XrActionSpaceCreateInfo space{XR_TYPE_ACTION_SPACE_CREATE_INFO}; + space.action = grip_; space.subactionPath = hands_[hand]; space.poseInActionSpace.orientation.w = 1; + check(xrCreateActionSpace(session_, &space, &handSpaces_[hand]), "controller space"); + } + } + + void initializeImages() { + uint32_t count = 0; + check(xrEnumerateViewConfigurationViews(instance_, system_, XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, + 0, &count, nullptr), "view count"); + if (count != 2) throw std::runtime_error("Expected two stereo views"); + std::vector views(count, {XR_TYPE_VIEW_CONFIGURATION_VIEW}); + check(xrEnumerateViewConfigurationViews(instance_, system_, XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, + count, &count, views.data()), "view configuration"); + check(xrEnumerateSwapchainFormats(session_, 0, &count, nullptr), "format count"); + std::vector formats(count); + check(xrEnumerateSwapchainFormats(session_, count, &count, formats.data()), "formats"); + int64_t format = 0; + for (int64_t candidate : formats) if (candidate == GL_SRGB8_ALPHA8) format = candidate; + if (!format) throw std::runtime_error("sRGB swapchain unavailable"); + for (int eye = 0; eye < 2; eye++) { + Eye &target = eyes_[eye]; + target.width = views[eye].recommendedImageRectWidth; + target.height = views[eye].recommendedImageRectHeight; + XrSwapchainCreateInfo create{XR_TYPE_SWAPCHAIN_CREATE_INFO}; + create.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT; + create.format = format; create.sampleCount = 1; + create.width = target.width; create.height = target.height; + create.faceCount = 1; create.arraySize = 1; create.mipCount = 1; + check(xrCreateSwapchain(session_, &create, &target.swapchain), "swapchain"); + check(xrEnumerateSwapchainImages(target.swapchain, 0, &count, nullptr), "image count"); + target.images.resize(count, {XR_TYPE_SWAPCHAIN_IMAGE_OPENGL_ES_KHR}); + check(xrEnumerateSwapchainImages(target.swapchain, count, &count, + reinterpret_cast(target.images.data())), "images"); + } + } + + void initializeTracking() { + std::string directory = std::string(app_->activity->internalDataPath) + "/.wayland-rt"; + if (mkdir(directory.c_str(), 0700) != 0 && errno != EEXIST) + throw std::runtime_error("XR tracking directory unavailable"); + auto mapFile = [&](const char *name, size_t size) { + std::string filename = directory + "/" + name; + int descriptor = open(filename.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0600); + if (descriptor < 0) throw std::runtime_error("XR IPC file unavailable"); + if (ftruncate(descriptor, size) != 0) { + close(descriptor); throw std::runtime_error("XR IPC file size failed"); + } + void *mapping = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, descriptor, 0); + close(descriptor); + if (mapping == MAP_FAILED) throw std::runtime_error("XR IPC map failed"); + return mapping; + }; + tracking_ = static_cast(mapFile(DROIDDECK_XR_FILE, sizeof(DroidDeckXrTracking))); + haptics_ = static_cast(mapFile(DROIDDECK_XR_HAPTIC_FILE, sizeof(DroidDeckXrHaptics))); + haptics_->magic = DROIDDECK_XR_HAPTIC_MAGIC; haptics_->version = DROIDDECK_XR_VERSION; + for (int hand = 0; hand < 2; hand++) lastPulse_[hand] = haptics_->hands[hand].id; + } + + static DroidDeckXrPose posePacket(const XrPosef &pose, XrSpaceLocationFlags flags) { + return {{pose.orientation.x, pose.orientation.y, pose.orientation.z, pose.orientation.w}, + {pose.position.x, pose.position.y, pose.position.z}, (uint32_t)flags}; + } + + void publishTracking(const XrView views[2], XrTime predicted) { + if (!tracking_) return; + timespec now{}; clock_gettime(CLOCK_MONOTONIC, &now); + DroidDeckXrTracking packet{}; + packet.magic = DROIDDECK_XR_MAGIC; packet.version = DROIDDECK_XR_VERSION; + packet.active = state_ == XR_SESSION_STATE_VISIBLE || state_ == XR_SESSION_STATE_FOCUSED; + packet.sampled_ns = (int64_t)now.tv_sec * 1000000000LL + now.tv_nsec; + if (convertTime_) { + timespec displayTime{}; + check(convertTime_(instance_, predicted, &displayTime), "display time conversion"); + packet.predicted_ns = (int64_t)displayTime.tv_sec * 1000000000LL + displayTime.tv_nsec; + } + packet.period_ns = period_; + packet.head = posePacket(head_.pose, head_.locationFlags); + for (int eye = 0; eye < 2; eye++) { + packet.eyes[eye].pose = posePacket(views[eye].pose, head_.locationFlags); + packet.eyes[eye].fov[0] = views[eye].fov.angleLeft; + packet.eyes[eye].fov[1] = views[eye].fov.angleRight; + packet.eyes[eye].fov[2] = views[eye].fov.angleUp; + packet.eyes[eye].fov[3] = views[eye].fov.angleDown; + packet.eyes[eye].width = eyes_[eye].width; packet.eyes[eye].height = eyes_[eye].height; + packet.hands[eye] = inputs_[eye]; + packet.hands[eye].pose = posePacket(handLocations_[eye].pose, handLocations_[eye].locationFlags); + packet.hands[eye].active = handActive_[eye]; + } + uint32_t sequence = __atomic_load_n(&tracking_->seq, __ATOMIC_RELAXED); + __atomic_store_n(&tracking_->seq, sequence | 1u, __ATOMIC_SEQ_CST); + __atomic_thread_fence(__ATOMIC_SEQ_CST); + std::memcpy(tracking_, &packet, offsetof(DroidDeckXrTracking, seq)); + std::memcpy(reinterpret_cast(tracking_) + offsetof(DroidDeckXrTracking, active), + reinterpret_cast(&packet) + offsetof(DroidDeckXrTracking, active), + sizeof(packet) - offsetof(DroidDeckXrTracking, active)); + __atomic_store_n(&tracking_->seq, (sequence | 1u) + 1u, __ATOMIC_RELEASE); + } + + void attachGuest(bool attach) { + if (attach == guestAttached_) return; + env_->CallVoidMethod(app_->activity->clazz, activeCallback_, static_cast(attach)); + if (env_->ExceptionCheck()) { + env_->ExceptionDescribe(); env_->ExceptionClear(); throw std::runtime_error("Guest XR handshake failed"); + } + guestAttached_ = attach; + } + + void pollHaptics() { + if (!haptics_) return; + DroidDeckXrHaptics packet{}; + uint32_t sequence = __atomic_load_n(&haptics_->seq, __ATOMIC_ACQUIRE); + if (sequence & 1u) return; + std::memcpy(&packet, haptics_, sizeof(packet)); + __atomic_thread_fence(__ATOMIC_ACQUIRE); + if (__atomic_load_n(&haptics_->seq, __ATOMIC_RELAXED) != sequence + || packet.magic != DROIDDECK_XR_HAPTIC_MAGIC || packet.version != DROIDDECK_XR_VERSION) return; + for (int hand = 0; hand < 2; hand++) { + const auto &pulse = packet.hands[hand]; + if (pulse.id == lastPulse_[hand]) continue; + lastPulse_[hand] = pulse.id; + if (state_ != XR_SESSION_STATE_FOCUSED || !handActive_[hand] + || pulse.duration_ns < 1000000 || pulse.duration_ns > 1000000000 + || !std::isfinite(pulse.amplitude) || pulse.amplitude < 0 || pulse.amplitude > 1 + || !std::isfinite(pulse.frequency) || pulse.frequency < 0) continue; + XrHapticActionInfo info{XR_TYPE_HAPTIC_ACTION_INFO}; info.action = vibration_; info.subactionPath = hands_[hand]; + XrHapticVibration vibration{XR_TYPE_HAPTIC_VIBRATION}; + vibration.duration = pulse.duration_ns; vibration.amplitude = pulse.amplitude; vibration.frequency = pulse.frequency; + check(xrApplyHapticFeedback(session_, &info, reinterpret_cast(&vibration)), "guest haptics"); + } + } + + GLuint shader(GLenum type, const char *text) { + GLuint handle = glCreateShader(type); + glShaderSource(handle, 1, &text, nullptr); glCompileShader(handle); + GLint success = 0; glGetShaderiv(handle, GL_COMPILE_STATUS, &success); + if (!success) { glDeleteShader(handle); throw std::runtime_error("Shader compile failed"); } + return handle; + } + + void initializeGraphics() { + glGenFramebuffers(1, &framebuffer_); + glGenVertexArrays(1, &vertexArray_); + GLuint vertex = shader(GL_VERTEX_SHADER, + "#version 300 es\nout vec2 uv; uniform float eye; void main(){vec2 coordinate=vec2(float((gl_VertexID<<1)&2),float(gl_VertexID&2));uv=vec2(coordinate.x*0.5+eye*0.5,1.0-coordinate.y);gl_Position=vec4(coordinate*2.0-1.0,0.0,1.0);}"); + GLuint fragment = shader(GL_FRAGMENT_SHADER, + "#version 300 es\nprecision highp float; in vec2 uv; uniform sampler2D source; uniform float eye; out vec4 outputColor; void main(){float edge=0.5/float(textureSize(source,0).x);vec2 coordinate=vec2(clamp(uv.x,eye*0.5+edge,(eye+1.0)*0.5-edge),uv.y);outputColor=texture(source,coordinate);}"); + sourceProgram_ = glCreateProgram(); glAttachShader(sourceProgram_, vertex); glAttachShader(sourceProgram_, fragment); + glLinkProgram(sourceProgram_); glDeleteShader(vertex); glDeleteShader(fragment); + GLint linked = 0; glGetProgramiv(sourceProgram_, GL_LINK_STATUS, &linked); + if (!linked) throw std::runtime_error("Frame shader link failed"); + glGenTextures(1, &sourceTexture_); glBindTexture(GL_TEXTURE_2D, sourceTexture_); + cpuTexture_ = sourceTexture_; + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } + + void receiveFrame() { + if (gpuFrames_.Take(sourceHeader_, sourceTexture_)) { + uploadedFrames_++; + if (uploadedFrames_ % 72 == 1) health(); + return; + } + DroidDeckXrPixels frame; + if (!receiver_.Take(&frame)) return; + GLint maximum = 0; glGetIntegerv(GL_MAX_TEXTURE_SIZE, &maximum); + if (frame.header.width > static_cast(maximum) || frame.header.height > static_cast(maximum)) + throw std::runtime_error("XR source exceeds GPU texture limit"); + sourceTexture_ = cpuTexture_; + glBindTexture(GL_TEXTURE_2D, sourceTexture_); + GLenum encoding = frame.header.color_space == DROIDDECK_XR_COLOR_SRGB_RGBA8 ? GL_SRGB8_ALPHA8 : GL_RGBA8; + glTexImage2D(GL_TEXTURE_2D, 0, encoding, frame.header.width, frame.header.height, 0, + GL_RGBA, GL_UNSIGNED_BYTE, frame.pixels.data()); + if (verifyPixels_) { + glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); + glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, 0); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, sourceTexture_, 0); + if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + throw std::runtime_error("Frame readback framebuffer incomplete"); + std::vector readback(frame.header.bytes); + glReadPixels(0, 0, frame.header.width, frame.header.height, GL_RGBA, GL_UNSIGNED_BYTE, readback.data()); + if (readback != frame.pixels) throw std::runtime_error("Frame GPU readback does not match owned pixels"); + gpuChecksum_ = 2166136261u; + for (unsigned char value : readback) gpuChecksum_ = (gpuChecksum_ ^ value) * 16777619u; + verifiedFrame_ = frame.header.frame_id; + glBindFramebuffer(GL_FRAMEBUFFER, 0); + } + if (glGetError() != GL_NO_ERROR) throw std::runtime_error("Frame texture upload failed"); + sourceHeader_ = frame.header; + receiver_.Recycle(std::move(frame.pixels)); + uploadedFrames_++; + if (verifyPixels_ || uploadedFrames_ % 72 == 1) health(); + } + + void render(Eye &eye, uint32_t image) { + glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, eye.images[image].image, 0); + if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + throw std::runtime_error("Incomplete eye framebuffer"); + { + glViewport(0, 0, eye.width, eye.height); glDisable(GL_DEPTH_TEST); + glUseProgram(sourceProgram_); glBindVertexArray(vertexArray_); + glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, sourceTexture_); + glUniform1i(glGetUniformLocation(sourceProgram_, "source"), 0); + glUniform1f(glGetUniformLocation(sourceProgram_, "eye"), &eye == &eyes_[1] ? 1.f : 0.f); + glDrawArrays(GL_TRIANGLES, 0, 3); + } + glBindFramebuffer(GL_FRAMEBUFFER, 0); + if (glGetError() != GL_NO_ERROR) throw std::runtime_error("OpenGL render error"); + glFlush(); + } + + void pollEvents() { + XrEventDataBuffer event{XR_TYPE_EVENT_DATA_BUFFER}; + XrResult result; + while ((result = xrPollEvent(instance_, &event)) == XR_SUCCESS) { + if (event.type == XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED) { + state_ = reinterpret_cast(&event)->state; + __android_log_print(ANDROID_LOG_INFO, "DroidDeckXR", "session state=%d", state_); + attachGuest(state_ == XR_SESSION_STATE_VISIBLE || state_ == XR_SESSION_STATE_FOCUSED); + if (state_ == XR_SESSION_STATE_READY) { + XrSessionBeginInfo begin{XR_TYPE_SESSION_BEGIN_INFO}; + begin.primaryViewConfigurationType = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + check(xrBeginSession(session_, &begin), "xrBeginSession"); running_ = true; + if (setPerformance_) { + for (auto domain : {XR_PERF_SETTINGS_DOMAIN_CPU_EXT, XR_PERF_SETTINGS_DOMAIN_GPU_EXT}) { + XrResult request = setPerformance_(session_, domain, XR_PERF_SETTINGS_LEVEL_SUSTAINED_HIGH_EXT); + __android_log_print(ANDROID_LOG_INFO, "DroidDeckXR", "sustained performance domain=%d result=%d", domain, request); + } + } + } else if (state_ == XR_SESSION_STATE_STOPPING) { + check(xrEndSession(session_), "xrEndSession"); running_ = false; + if (quitRequested_) exit_ = true; + } else if (state_ == XR_SESSION_STATE_EXITING || state_ == XR_SESSION_STATE_LOSS_PENDING) exit_ = true; + health(); + } else if (event.type == XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING) exit_ = true; + else if (event.type == XR_TYPE_EVENT_DATA_PERF_SETTINGS_EXT) { + const auto &performance = *reinterpret_cast(&event); + __android_log_print(ANDROID_LOG_INFO, "DroidDeckXR", "performance domain=%d subdomain=%d level=%d->%d", + performance.domain, performance.subDomain, performance.fromLevel, performance.toLevel); + } + event = {XR_TYPE_EVENT_DATA_BUFFER}; + } + check(result, "xrPollEvent"); + } + + void frame() { + XrFrameWaitInfo wait{XR_TYPE_FRAME_WAIT_INFO}; + XrFrameState frameState{XR_TYPE_FRAME_STATE}; + check(xrWaitFrame(session_, &wait, &frameState), "xrWaitFrame"); + period_ = frameState.predictedDisplayPeriod; + XrFrameBeginInfo begin{XR_TYPE_FRAME_BEGIN_INFO}; check(xrBeginFrame(session_, &begin), "xrBeginFrame"); + check(xrLocateSpace(view_, world_, frameState.predictedDisplayTime, &head_), "head pose"); + if (state_ == XR_SESSION_STATE_FOCUSED) { + XrActiveActionSet active{actions_, XR_NULL_PATH}; + XrActionsSyncInfo sync{XR_TYPE_ACTIONS_SYNC_INFO}; sync.countActiveActionSets = 1; sync.activeActionSets = &active; + check(xrSyncActions(session_, &sync), "xrSyncActions"); + for (int hand = 0; hand < 2; hand++) { + XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO}; get.action = grip_; get.subactionPath = hands_[hand]; + XrActionStatePose pose{XR_TYPE_ACTION_STATE_POSE}; check(xrGetActionStatePose(session_, &get, &pose), "grip state"); + handActive_[hand] = pose.isActive; + check(xrLocateSpace(handSpaces_[hand], world_, frameState.predictedDisplayTime, &handLocations_[hand]), "hand pose"); + get.action = trigger_; + XrActionStateFloat trigger{XR_TYPE_ACTION_STATE_FLOAT}; check(xrGetActionStateFloat(session_, &get, &trigger), "trigger state"); + inputs_[hand].trigger = trigger.isActive ? trigger.currentState : 0; + get.action = squeeze_; + XrActionStateFloat squeeze{XR_TYPE_ACTION_STATE_FLOAT}; check(xrGetActionStateFloat(session_, &get, &squeeze), "squeeze state"); + inputs_[hand].squeeze = squeeze.isActive ? squeeze.currentState : 0; + get.action = stick_; + XrActionStateVector2f stick{XR_TYPE_ACTION_STATE_VECTOR2F}; check(xrGetActionStateVector2f(session_, &get, &stick), "stick state"); + inputs_[hand].stick[0] = stick.isActive ? stick.currentState.x : 0; + inputs_[hand].stick[1] = stick.isActive ? stick.currentState.y : 0; + inputs_[hand].buttons = 0; + const std::pair buttons[] = { + {stickClick_, DROIDDECK_XR_STICK_CLICK}, {primary_, DROIDDECK_XR_PRIMARY_CLICK}, + {secondary_, DROIDDECK_XR_SECONDARY_CLICK}, {menu_, DROIDDECK_XR_MENU_CLICK}, + {triggerTouch_, DROIDDECK_XR_TRIGGER_TOUCH}, {stickTouch_, DROIDDECK_XR_STICK_TOUCH}, + {primaryTouch_, DROIDDECK_XR_PRIMARY_TOUCH}, {secondaryTouch_, DROIDDECK_XR_SECONDARY_TOUCH}, + {thumbrestTouch_, DROIDDECK_XR_THUMBREST_TOUCH}}; + for (const auto &button : buttons) { + get.action = button.first; + XrActionStateBoolean value{XR_TYPE_ACTION_STATE_BOOLEAN}; check(xrGetActionStateBoolean(session_, &get, &value), "button state"); + if (value.isActive && value.currentState) inputs_[hand].buttons |= button.second; + } + } + } else handActive_[0] = handActive_[1] = false; + pollHaptics(); + XrViewLocateInfo locate{XR_TYPE_VIEW_LOCATE_INFO}; locate.viewConfigurationType = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + locate.displayTime = frameState.predictedDisplayTime; locate.space = world_; + XrViewState viewState{XR_TYPE_VIEW_STATE}; uint32_t count = 0; + XrView views[2]{{XR_TYPE_VIEW}, {XR_TYPE_VIEW}}; + check(xrLocateViews(session_, &locate, &viewState, 2, &count, views), "xrLocateViews"); + publishTracking(views, frameState.predictedDisplayTime); + if (guestAttached_) { + timespec now{}; clock_gettime(CLOCK_MONOTONIC, &now); + env_->CallVoidMethod(app_->activity->clazz, frameCallback_, (jlong)now.tv_sec * 1000000000LL + now.tv_nsec); + if (env_->ExceptionCheck()) { env_->ExceptionClear(); throw std::runtime_error("Guest frame callback failed"); } + } + XrCompositionLayerProjectionView projection[2]{{XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW}, {XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW}}; + bool valid = frameState.shouldRender && count == 2 && + (viewState.viewStateFlags & (XR_VIEW_STATE_ORIENTATION_VALID_BIT | XR_VIEW_STATE_POSITION_VALID_BIT)) + == (XR_VIEW_STATE_ORIENTATION_VALID_BIT | XR_VIEW_STATE_POSITION_VALID_BIT); + { + valid = valid && sourceHeader_.frame_id && xr_frame_now() - sourceHeader_.captured_ns < 500000000LL; + if (valid) for (int eye = 0; eye < 2; eye++) { + auto source = xr_frame_view(sourceHeader_, eye); + views[eye].pose.orientation = {source.orientation[0], source.orientation[1], source.orientation[2], source.orientation[3]}; + views[eye].pose.position = {source.position[0], source.position[1], source.position[2]}; + views[eye].fov = {source.fov[0], source.fov[1], source.fov[2], source.fov[3]}; + } + } + const std::string capture = captureRequest(); + bool captured = valid; + if (!capture.empty() && valid) + captured = captureTexture(sourceTexture_, sourceHeader_.width, sourceHeader_.height, "source"); + if (valid) for (int eye = 0; eye < 2; eye++) { + uint32_t image = 0; + XrSwapchainImageAcquireInfo acquire{XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO}; + check(xrAcquireSwapchainImage(eyes_[eye].swapchain, &acquire, &image), "acquire image"); + XrSwapchainImageWaitInfo imageWait{XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO}; imageWait.timeout = XR_INFINITE_DURATION; + check(xrWaitSwapchainImage(eyes_[eye].swapchain, &imageWait), "wait image"); + render(eyes_[eye], image); + if (!capture.empty()) captured = captureTexture(eyes_[eye].images[image].image, eyes_[eye].width, eyes_[eye].height, eye ? "right" : "left") && captured; + XrSwapchainImageReleaseInfo release{XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO}; + check(xrReleaseSwapchainImage(eyes_[eye].swapchain, &release), "release image"); + projection[eye].pose = views[eye].pose; projection[eye].fov = views[eye].fov; + projection[eye].subImage.swapchain = eyes_[eye].swapchain; + projection[eye].subImage.imageRect.extent = {eyes_[eye].width, eyes_[eye].height}; + } + XrCompositionLayerProjection layer{XR_TYPE_COMPOSITION_LAYER_PROJECTION}; + layer.space = world_; layer.viewCount = 2; layer.views = projection; + const XrCompositionLayerBaseHeader *layers[] = {reinterpret_cast(&layer)}; + XrFrameEndInfo end{XR_TYPE_FRAME_END_INFO}; end.displayTime = frameState.predictedDisplayTime; + end.environmentBlendMode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; end.layerCount = valid ? 1 : 0; end.layers = layers; + check(xrEndFrame(session_, &end), "xrEndFrame"); + if (!capture.empty()) captureResult(capture, captured, valid, projection, frameState.predictedDisplayTime); + frames_++; if (valid) rendered_++; + if (frames_ % 72 == 0) { + health(); + __android_log_print(ANDROID_LOG_INFO, "DroidDeckXR", "frames=%llu rendered=%llu head=%llu hands=%d,%d period=%lld", + (unsigned long long)frames_, (unsigned long long)rendered_, (unsigned long long)head_.locationFlags, + handActive_[0], handActive_[1], (long long)period_); + } + } + + void cleanup() { + receiver_.Stop(); + gpuFrames_.Stop(); + try { attachGuest(false); } catch (const std::exception &error) { + __android_log_print(ANDROID_LOG_ERROR, "DroidDeckXR", "cleanup: %s", error.what()); + } + if (tracking_) { + __atomic_store_n(&tracking_->active, 0u, __ATOMIC_RELEASE); + munmap(tracking_, sizeof(DroidDeckXrTracking)); + } + if (haptics_) munmap(haptics_, sizeof(DroidDeckXrHaptics)); + if (display_ != EGL_NO_DISPLAY && context_ != EGL_NO_CONTEXT) { + glDeleteFramebuffers(1, &framebuffer_); glDeleteVertexArrays(1, &vertexArray_); + glDeleteTextures(1, &cpuTexture_); glDeleteProgram(sourceProgram_); + } + for (auto &eye : eyes_) if (eye.swapchain) xrDestroySwapchain(eye.swapchain); + for (auto space : handSpaces_) if (space) xrDestroySpace(space); + if (view_) xrDestroySpace(view_); if (world_) xrDestroySpace(world_); + if (session_) xrDestroySession(session_); + if (actions_) xrDestroyActionSet(actions_); + if (instance_) xrDestroyInstance(instance_); + if (display_ != EGL_NO_DISPLAY) { + eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT); + if (surface_ != EGL_NO_SURFACE) eglDestroySurface(display_, surface_); + if (context_ != EGL_NO_CONTEXT) eglDestroyContext(display_, context_); + eglTerminate(display_); + } + if (env_) app_->activity->vm->DetachCurrentThread(); + } +}; + +void android_main(android_app *app) { + QuestXR runtime(app); + try { runtime.run(); } + catch (const std::exception &error) { + __android_log_print(ANDROID_LOG_ERROR, "DroidDeckXR", "%s", error.what()); + runtime.health(error.what()); + ANativeActivity_finish(app->activity); + } +} diff --git a/app/src/main/cpp/waylandcomp/src/vk_present.c b/app/src/main/cpp/waylandcomp/src/vk_present.c index 9405f00ed7..96d0b415d3 100644 --- a/app/src/main/cpp/waylandcomp/src/vk_present.c +++ b/app/src/main/cpp/waylandcomp/src/vk_present.c @@ -138,6 +138,9 @@ static pthread_mutex_t g_req_lock = PTHREAD_MUTEX_INITIALIZER; static ANativeWindow *g_req_window; static int g_req_pending; static int g_req_detached; +static uint64_t g_req_generation, g_req_applied; +static pthread_cond_t g_req_condition = PTHREAD_COND_INITIALIZER; + static volatile int g_mode = VKP_MODE_STRETCH, g_align = VKP_ALIGN_CENTER; @@ -218,26 +221,53 @@ void vk_present_set_window(ANativeWindow *window) { if (g_req_pending && g_req_window && g_req_window != window) superseded = g_req_window; g_req_window = window; g_req_pending = 1; + g_req_generation++; if (!window) g_req_detached = 1; pthread_mutex_unlock(&g_req_lock); if (superseded) ANativeWindow_release(superseded); } +static void window_request_applied(uint64_t generation) { + pthread_mutex_lock(&g_req_lock); + g_req_applied = generation; + pthread_cond_broadcast(&g_req_condition); + pthread_mutex_unlock(&g_req_lock); +} + +int vkp_wait_window_request(int timeout_ms) { + if (timeout_ms <= 0 || timeout_ms > 10000) return 0; + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += timeout_ms / 1000; + deadline.tv_nsec += (timeout_ms % 1000) * 1000000L; + if (deadline.tv_nsec >= 1000000000L) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000L; } + pthread_mutex_lock(&g_req_lock); + const uint64_t generation = g_req_generation; + int error = 0; + while (g_req_applied < generation && error == 0) + error = pthread_cond_timedwait(&g_req_condition, &g_req_lock, &deadline); + int applied = g_req_applied >= generation; + pthread_mutex_unlock(&g_req_lock); + return applied; +} + int vkp_apply_window_request(void) { ANativeWindow *w; pthread_mutex_lock(&g_req_lock); if (!g_req_pending) { pthread_mutex_unlock(&g_req_lock); return 0; } w = g_req_window; const int detached = g_req_detached; + const uint64_t generation = g_req_generation; g_req_window = NULL; g_req_pending = 0; g_req_detached = 0; pthread_mutex_unlock(&g_req_lock); if (w == g_window) { if (w) ANativeWindow_release(w); - if (!w || !detached) return 0; + if (!w || !detached) { window_request_applied(generation); return 0; } sc_layer_window_gone(); destroy_swapchain(); + window_request_applied(generation); droiddeck_log("gpu", "screen surface rebound: rebuilding the swapchain at its new size"); return 1; } @@ -248,6 +278,7 @@ int vkp_apply_window_request(void) { g_window = w; g_swap_retry_at_ns = 0; g_swap_fail_logged = 0; + window_request_applied(generation); droiddeck_log("gpu", w ? "screen surface attached" : "screen surface gone: presenting paused"); return 1; } diff --git a/app/src/main/cpp/waylandcomp/src/vk_present.h b/app/src/main/cpp/waylandcomp/src/vk_present.h index a965807e7b..48d32ab4dc 100644 --- a/app/src/main/cpp/waylandcomp/src/vk_present.h +++ b/app/src/main/cpp/waylandcomp/src/vk_present.h @@ -51,6 +51,7 @@ const char *vkp_gpu_name(void); int vkp_ready(void); int vkp_has_window(void); +int vkp_wait_window_request(int timeout_ms); int vkp_device_lost(void); int vkp_render(int scene_w, int scene_h, const struct vkp_draw *draws, int n); diff --git a/app/src/main/cpp/waylandcomp/src/waylandcomp_jni.c b/app/src/main/cpp/waylandcomp/src/waylandcomp_jni.c index 207c52cd5d..accbd08a11 100644 --- a/app/src/main/cpp/waylandcomp/src/waylandcomp_jni.c +++ b/app/src/main/cpp/waylandcomp/src/waylandcomp_jni.c @@ -592,6 +592,12 @@ static char *dup_bytes(JNIEnv *env, jbyteArray arr, int *len) { return buf; } +JNIEXPORT jboolean JNICALL +Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeWaitSurface( + JNIEnv *env, jclass clazz, jint timeout_ms) { + return vkp_wait_window_request(timeout_ms) ? JNI_TRUE : JNI_FALSE; +} + JNIEXPORT void JNICALL Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeSetClipboardText(JNIEnv *env, jclass clazz, jbyteArray utf8) { int len; diff --git a/app/src/main/java/com/droiddeck/launcher/QuestXrActivity.kt b/app/src/main/java/com/droiddeck/launcher/QuestXrActivity.kt new file mode 100644 index 0000000000..3709b5fa50 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/QuestXrActivity.kt @@ -0,0 +1,118 @@ +package com.droiddeck.launcher + +import android.app.NativeActivity +import android.os.Bundle +import android.os.SystemClock +import android.util.Log +import android.widget.Toast +import com.droiddeck.launcher.gpu.SteamVr +import com.droiddeck.launcher.session.SessionService +import com.droiddeck.launcher.session.SessionState +import com.droiddeck.launcher.wayland.CompositorHost +import com.droiddeck.launcher.wayland.WaylandCompositor + +class QuestXrActivity : NativeActivity() { + private var steamVrStarted = false + @Volatile private var closing = false + @Volatile private var steamVrStatus = SteamVr.Status.UNAVAILABLE + private var sessionId: String? = null + override fun onCreate(savedInstanceState: Bundle?) { + sessionId = SessionState.sessionId + SessionService.setXrActivityVisible(this, true) + super.onCreate(savedInstanceState) + } + + fun onXrActive(active: Boolean) { + if (active) { + check(CompositorHost.isStarted) { "DroidDeck compositor is not running" } + SessionService.resume(this) + WaylandCompositor.nativeSetCardboard(false, 1, 0.064f, 3f, 4f, 0f, 0f, 90f, 0f, 0f) + WaylandCompositor.nativeSetFrameGenArmed(false, 0, 0) + WaylandCompositor.nativeSetZeroCopy(false) + CompositorHost.attachXr() + WaylandCompositor.nativeVsync(System.nanoTime()) + check(WaylandCompositor.nativeWaitSurface(3000)) { "XR frame GPU handshake timed out" } + } + SessionService.setXrActivityVisible(this, active) + if (active) startSteamVr() + } + + private fun startSteamVr() { + if (!steamVrStarted) { + steamVrStarted = true + Thread({ + try { + val started = synchronized(steamVrLock) { + if (closing || !sessionAlive()) false else { + steamVrOwner = this + SteamVr.run(this, "start") + } + } + if (!started) { + if (!closing) leaveXr("SteamVR start failed", sessionAlive()) + return@Thread + } + val startupDeadline = SystemClock.elapsedRealtime() + 30_000 + var lastRunning: Long? = null + while (!closing) { + if (!sessionAlive()) { + leaveXr("Steam session ended", false) + break + } + steamVrStatus = SteamVr.status(this) + val now = SystemClock.elapsedRealtime() + if (steamVrStatus == SteamVr.Status.RUNNING) lastRunning = now + val deadline = lastRunning?.plus(5_000) ?: startupDeadline + if (steamVrStatus != SteamVr.Status.RUNNING && now >= deadline) { + leaveXr("SteamVR stopped or did not finish starting", true) + break + } + Thread.sleep(1000) + } + } finally { + synchronized(steamVrLock) { + // A retiring activity must not stop a replacement's SteamVR session. + if (steamVrOwner === this) { + if (sessionAlive()) SteamVr.run(this, "stop") + steamVrOwner = null + } + } + } + }, "quest-steamvr-start").start() + } + } + + private fun sessionAlive() = SessionState.running && !SessionState.stopRequested && + sessionId != null && SessionState.sessionId == sessionId + + private fun leaveXr(reason: String, failed: Boolean) { + Log.w("DroidDeckXR", reason) + closing = true + runOnUiThread { + if (failed) Toast.makeText(this, R.string.cardboard_steamvr_failed, Toast.LENGTH_LONG).show() + finish() + } + } + + fun onXrFrame(frameTimeNanos: Long) { + WaylandCompositor.nativeVsync(frameTimeNanos) + } + + fun onXrStatus(): Int = steamVrStatus.ordinal + + override fun onDestroy() { + closing = true + SessionService.setXrActivityVisible(this, false) + super.onDestroy() + } + + override fun onStop() { + SessionService.setXrActivityVisible(this, false) + super.onStop() + } + + companion object { + private val steamVrLock = Any() + private var steamVrOwner: QuestXrActivity? = null + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt b/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt index 4b2bb6b2a1..72dae53b38 100644 --- a/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt +++ b/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt @@ -521,8 +521,8 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { releaseDrawerDirection() } } - LaunchedEffect(cardboard.enabled, steamVrSwitchedPicture) { - while (cardboard.enabled || steamVrSwitchedPicture) { + LaunchedEffect(cardboard.enabled, steamVrSwitchedPicture, drawerOpen, drawerPage) { + while (cardboard.enabled || steamVrSwitchedPicture || (drawerOpen && drawerPage == SessionDrawerPage.STEAMVR)) { val now = kotlinx.coroutines.withContext(kotlinx.coroutines.Dispatchers.IO) { SteamVr.status(this@SessionActivity) } @@ -559,6 +559,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { onCardboard = { c -> SessionPrefs.setCardboard(this@SessionActivity, c); applyCardboard(c) }, onCardboardRecenter = { headTracker?.recenter() }, steamVr = steamVrStatus, + immersiveSteamVr = com.droiddeck.launcher.session.QuestExperimental.isSupported(), onSteamVr = { action -> steamVrAction(action) }, textureAnisotropy = if (drawerGameProfileId == null) textureAnisotropy else drawerTextureAnisotropy, textureLodBias = if (drawerGameProfileId == null) textureLodBias else drawerTextureLodBias, @@ -2384,6 +2385,10 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { } private fun steamVrAction(action: String) { + if (action == "start" && com.droiddeck.launcher.session.QuestExperimental.isSupported()) { + startActivity(android.content.Intent(this, QuestXrActivity::class.java)) + return + } if (action == "start") { switchPictureForSteamVr() steamVrSeenRunning = false diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt index b61ac122c7..3a61f1b472 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt @@ -71,6 +71,8 @@ object SessionFiles { "usr/local/bin/droiddeck-steam-games" to "usr/local/bin/droiddeck-steam-games", "usr/local/bin/droiddeck-pad-defaults" to "usr/local/bin/droiddeck-pad-defaults", "usr/local/bin/droiddeck-steamvr" to "usr/local/bin/droiddeck-steamvr", + "usr/local/bin/droiddeck-steamvr-settings" to "usr/local/bin/droiddeck-steamvr-settings", + "usr/local/bin/droiddeck-steamvr-game" to "usr/local/bin/droiddeck-steamvr-game", "usr/local/bin/droiddeck-bwrap" to "usr/local/bin/droiddeck-bwrap", "usr/local/bin/droiddeck-flatpak" to "usr/local/bin/droiddeck-flatpak", "usr/local/bin/droiddeck-flatpak-setup" to "usr/local/bin/droiddeck-flatpak-setup", @@ -104,6 +106,7 @@ object SessionFiles { listOf("NOTICE", "GPL-2", "GPL-3", "LGPL-2.1", "7zip-License", "7zip-unRarLicense").map { "usr/local/share/licenses/droiddeck-msitools/$it" } val vr = listOf( "droiddeck/bin/linuxarm64/driver_droiddeck.so", + "droiddeck/bin/linuxarm64/xr-display-mode", "droiddeck/driver.vrdrivermanifest", "droiddeck/resources/settings/default.vrsettings", "layer/libVkLayer_droiddeck_vr.so", diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt index 7a3ac4fd4c..e1554e1555 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt @@ -71,6 +71,8 @@ class SessionService : Service() { private var suspendPolicy = SessionPrefs.SUSPEND_MANUAL private var steamDownloadsInBackground = false private var activityVisible = true + private var xrActivityVisible = false + private val hasVisibleSurface get() = sessionHasVisibleSurface(activityVisible, screenOn, xrActivityVisible) private var screenOn = true private var manualPauseRequested = false private var steamSleepToken: String? = null @@ -170,6 +172,17 @@ class SessionService : Service() { updateSuspendPolicy() return START_NOT_STICKY } + ACTION_XR_ACTIVITY_VISIBLE -> { + xrActivityVisible = true + resumeSession(panelVisible = false) + return START_NOT_STICKY + } + ACTION_XR_ACTIVITY_HIDDEN -> { + xrActivityVisible = false + suspendAttemptFailed = false + updateSuspendPolicy() + return START_NOT_STICKY + } ACTION_SUSPEND_POLICY_CHANGED -> { if (!SessionState.running) return START_NOT_STICKY suspendPolicy = SessionPrefs.suspendPolicy(this, SessionState.mode) @@ -205,6 +218,7 @@ class SessionService : Service() { suspendPolicy = SessionPrefs.suspendPolicy(this, SessionState.mode) steamDownloadsInBackground = SessionPrefs.steamDownloadsInBackground(this) activityVisible = true + xrActivityVisible = false screenOn = (getSystemService(Context.POWER_SERVICE) as? PowerManager)?.isInteractive ?: true manualPauseRequested = false suspendDownloadNow = false @@ -919,8 +933,8 @@ class SessionService : Service() { Log.i(TAG, "queued Steam game $gameId for the running client") } - private fun resumeSession() { - activityVisible = true + private fun resumeSession(panelVisible: Boolean = true) { + if (panelVisible) activityVisible = true screenOn = (getSystemService(Context.POWER_SERVICE) as? PowerManager)?.isInteractive ?: screenOn manualPauseRequested = false suspendDownloadNow = false @@ -1111,11 +1125,11 @@ class SessionService : Service() { private fun updateSuspendPolicy() { if (!SessionState.running) return - if (suspendPolicy == SessionPrefs.SUSPEND_MANUAL && (!activityVisible || !screenOn)) { + if (suspendPolicy == SessionPrefs.SUSPEND_MANUAL && !hasVisibleSurface) { manualPauseRequested = true } val controller = suspendController ?: return - val hidden = !activityVisible || !screenOn + val hidden = !hasVisibleSurface val downloadsBlockingSuspend = steamDownloadsInBackground && SessionState.mode == MODE_STEAM && suspendPolicy != SessionPrefs.SUSPEND_NEVER && hidden && downloadActive && !suspendDownloadNow if (downloadsBlockingSuspend) acquireLocks() @@ -1137,6 +1151,7 @@ class SessionService : Service() { SessionEvents.record(if (shouldSuspend) "session.suspend_requested" else "session.resume_requested", mapOf( "policy" to suspendPolicy, "activityVisible" to activityVisible, "screenOn" to screenOn, "steamSleep" to (steamSleepToken != null), + "xrActivityVisible" to xrActivityVisible, "nativeSleep" to (nativeSleepToken != null), )) if (shouldSuspend) { @@ -1194,7 +1209,7 @@ class SessionService : Service() { } private fun resumeSteamSleepOnReturn() { - if (suspendPolicy in setOf(SessionPrefs.SUSPEND_AUTO, SessionPrefs.SUSPEND_NATIVE) && activityVisible && screenOn) { + if (suspendPolicy in setOf(SessionPrefs.SUSPEND_AUTO, SessionPrefs.SUSPEND_NATIVE) && hasVisibleSurface) { if (steamSleepToken == null && (nativeSleepToken == null || nativeSleepFallback)) requestSteamWake() completeNativeSleep() completeSteamSleep() @@ -1305,6 +1320,7 @@ class SessionService : Service() { if (!SessionState.running) return SessionState.running = false } + xrActivityVisible = false updateDownloadMonitoring() val stoppedGen = sessionGen val stoppedSession = SessionState.sessionId @@ -1357,8 +1373,8 @@ class SessionService : Service() { } override fun onTaskRemoved(rootIntent: Intent?) { - if (pipTask) { - Log.i(TAG, "PiP task dismissed - retaining the session under its background policy") + if (pipTask || xrActivityVisible) { + Log.i(TAG, "panel task dismissed - retaining the session under its active surface policy") activityVisible = false updateSuspendPolicy() return @@ -1559,6 +1575,8 @@ class SessionService : Service() { private const val ACTION_SUSPEND_POLICY_CHANGED = "com.droiddeck.launcher.SUSPEND_POLICY_CHANGED" private const val ACTION_ACTIVITY_VISIBLE = "com.droiddeck.launcher.ACTIVITY_VISIBLE" private const val ACTION_ACTIVITY_HIDDEN = "com.droiddeck.launcher.ACTIVITY_HIDDEN" + private const val ACTION_XR_ACTIVITY_VISIBLE = "com.droiddeck.launcher.XR_ACTIVITY_VISIBLE" + private const val ACTION_XR_ACTIVITY_HIDDEN = "com.droiddeck.launcher.XR_ACTIVITY_HIDDEN" private const val ACTION_TRACK_AUXILIARY = "com.droiddeck.launcher.TRACK_AUXILIARY" private const val ACTION_STOP_AUXILIARY = "com.droiddeck.launcher.STOP_AUXILIARY" private const val ACTION_AUXILIARY_EXITED = "com.droiddeck.launcher.AUXILIARY_EXITED" @@ -1618,6 +1636,12 @@ class SessionService : Service() { context.startService(Intent(context, SessionService::class.java).setAction(action)) } + fun setXrActivityVisible(context: Context, visible: Boolean) { + if (!SessionState.running) return + val action = if (visible) ACTION_XR_ACTIVITY_VISIBLE else ACTION_XR_ACTIVITY_HIDDEN + context.startService(Intent(context, SessionService::class.java).setAction(action)) + } + fun resume(context: Context) { if (!SessionState.running) return context.startService(Intent(context, SessionService::class.java).setAction(ACTION_RESUME)) diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionVisibility.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionVisibility.kt new file mode 100644 index 0000000000..33b456ecca --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionVisibility.kt @@ -0,0 +1,4 @@ +package com.droiddeck.launcher.session + +internal fun sessionHasVisibleSurface(panelVisible: Boolean, screenOn: Boolean, xrVisible: Boolean): Boolean = + xrVisible || (panelVisible && screenOn) diff --git a/app/src/main/java/com/droiddeck/launcher/ui/SessionDrawerPage.kt b/app/src/main/java/com/droiddeck/launcher/ui/SessionDrawerPage.kt index 898900f450..576587f7bf 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/SessionDrawerPage.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/SessionDrawerPage.kt @@ -5,6 +5,8 @@ internal enum class SessionDrawerPage(val firstControl: String) { SECOND_SCREEN("second-screen:none"), DISPLAY("hud"), EFFECTS("look"), + STEAMVR("steamvr"), + CARDBOARD("cardboard"), GAMES("anisotropy"), SESSION("suspend"), } diff --git a/app/src/main/java/com/droiddeck/launcher/ui/SessionOverlay.kt b/app/src/main/java/com/droiddeck/launcher/ui/SessionOverlay.kt index 775804a62f..c784d7a365 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/SessionOverlay.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/SessionOverlay.kt @@ -42,6 +42,8 @@ import androidx.compose.material.icons.outlined.PictureInPictureAlt import androidx.compose.material.icons.automirrored.outlined.ExitToApp import androidx.compose.material.icons.outlined.Settings import androidx.compose.material.icons.outlined.SportsEsports +import androidx.compose.material.icons.outlined.ViewInAr +import androidx.compose.material.icons.outlined.Vrpano import androidx.compose.material3.Icon import androidx.compose.material3.RadioButton import androidx.compose.material3.RadioButtonDefaults @@ -151,6 +153,8 @@ private val SessionDrawerPage.title: Int get() = when (this) { SessionDrawerPage.CONTROLLER -> R.string.drawer_page_controller SessionDrawerPage.SECOND_SCREEN -> R.string.drawer_second_screen SessionDrawerPage.DISPLAY -> R.string.drawer_page_display + SessionDrawerPage.STEAMVR -> R.string.steamvr_title + SessionDrawerPage.CARDBOARD -> R.string.cardboard_title SessionDrawerPage.EFFECTS -> R.string.drawer_page_effects SessionDrawerPage.GAMES -> R.string.game_settings_title SessionDrawerPage.SESSION -> R.string.mode_session @@ -160,6 +164,8 @@ private val SessionDrawerPage.icon get() = when (this) { SessionDrawerPage.CONTROLLER -> Icons.Outlined.SportsEsports SessionDrawerPage.SECOND_SCREEN -> Icons.Outlined.ScreenShare SessionDrawerPage.DISPLAY -> Icons.Outlined.DesktopWindows + SessionDrawerPage.STEAMVR -> Icons.Outlined.ViewInAr + SessionDrawerPage.CARDBOARD -> Icons.Outlined.Vrpano SessionDrawerPage.EFFECTS -> Icons.Outlined.AutoFixHigh SessionDrawerPage.GAMES -> Icons.Outlined.Layers SessionDrawerPage.SESSION -> Icons.Outlined.Settings @@ -209,6 +215,7 @@ class DrawerActions( val cardboard: Cardboard = Cardboard.OFF, val cardboardSensor: Boolean = false, val steamVr: SteamVr.Status? = null, + val immersiveSteamVr: Boolean = false, val textureAnisotropy: Int? = 0, val textureLodBias: String? = TextureFiltering.LOD_BIAS_OFF, val secondScreenMode: SecondScreenMode, @@ -452,12 +459,13 @@ internal fun SessionDrawer(open: Boolean, requestedPage: SessionDrawerPage, cont stringResource(R.string.display_sharpness), null, a.upscaleSharpness, 0..100, step = 5, format = { "$it%" }, modifier = focus.track(page, "upscale-sharpness"), onChange = a.onUpscaleSharpness) } - CardboardGroup(host, a) { key -> focus.track(page, key) } } SessionDrawerPage.EFFECTS -> { ScreenEffectsGroup(host, a, advancedEffectsExpanded, onAdvancedToggle = { advancedEffectsExpanded = !advancedEffectsExpanded }) { key -> focus.track(page, key) } } + SessionDrawerPage.STEAMVR -> SteamVrGroup(a) { key -> focus.track(page, key) } + SessionDrawerPage.CARDBOARD -> CardboardGroup(host, a) { key -> focus.track(page, key) } SessionDrawerPage.CONTROLLER -> { SettingsGroup(stringResource(R.string.drawer_page_controller)) { a.padType?.let { type -> @@ -915,17 +923,34 @@ private fun CardboardGroup(host: MenuHost, a: DrawerActions, track: (String) -> } if (c.headTracking && a.cardboardSensor) ActionRow(stringResource(R.string.cardboard_recenter), null, stringResource(R.string.cardboard_recenter_button), a.onCardboardRecenter) + } +} + +@Composable +private fun SteamVrGroup(a: DrawerActions, track: (String) -> Modifier) { + SettingsGroup(stringResource(R.string.steamvr_title)) { val vrLabel = stringResource(R.string.cardboard_steamvr) + val enterLabel = stringResource(if (a.immersiveSteamVr) R.string.steamvr_enter else R.string.cardboard_steamvr_start) + val enterHint = stringResource(if (a.immersiveSteamVr) R.string.steamvr_quest_ready_hint else R.string.cardboard_steamvr_ready_hint) + @Composable fun button(label: String, hint: String?, text: String, action: String) { + SettingsRow(label, hint) { + SecondaryButton(text, modifier = track("steamvr"), onClick = { a.onSteamVr(action) }) + } + } when (a.steamVr) { - SteamVr.Status.NOT_INSTALLED -> ActionRow(vrLabel, stringResource(R.string.cardboard_steamvr_install_hint), - stringResource(R.string.cardboard_steamvr_install), { a.onSteamVr("install") }) + SteamVr.Status.NOT_INSTALLED -> button(vrLabel, stringResource(R.string.cardboard_steamvr_install_hint), + stringResource(R.string.cardboard_steamvr_install), "install") SteamVr.Status.INSTALLING -> SettingsRow(vrLabel, stringResource(R.string.cardboard_steamvr_installing)) {} - SteamVr.Status.READY -> ActionRow(vrLabel, stringResource(R.string.cardboard_steamvr_ready_hint), - stringResource(R.string.cardboard_steamvr_start), { a.onSteamVr("start") }) - SteamVr.Status.RUNNING -> ActionRow(vrLabel, stringResource(R.string.cardboard_steamvr_running_hint), - stringResource(R.string.cardboard_steamvr_stop), { a.onSteamVr("stop") }) + SteamVr.Status.READY -> button(vrLabel, enterHint, enterLabel, "start") + SteamVr.Status.RUNNING -> { + if (a.immersiveSteamVr) button(vrLabel, enterHint, enterLabel, "start") + SettingsRow(vrLabel, stringResource(R.string.cardboard_steamvr_running_hint)) { + SecondaryButton(stringResource(R.string.cardboard_steamvr_stop), + modifier = track(if (a.immersiveSteamVr) "steamvr-stop" else "steamvr"), onClick = { a.onSteamVr("stop") }) + } + } SteamVr.Status.UNAVAILABLE -> SettingsRow(vrLabel, stringResource(R.string.cardboard_steamvr_unavailable)) {} - null -> {} + null -> SettingsRow(vrLabel, stringResource(R.string.steamvr_checking)) {} } } } diff --git a/app/src/main/java/com/droiddeck/launcher/wayland/CompositorHost.kt b/app/src/main/java/com/droiddeck/launcher/wayland/CompositorHost.kt index 9df4d4dfd0..efb1fbdd73 100644 --- a/app/src/main/java/com/droiddeck/launcher/wayland/CompositorHost.kt +++ b/app/src/main/java/com/droiddeck/launcher/wayland/CompositorHost.kt @@ -7,6 +7,7 @@ import com.droiddeck.launcher.session.SessionState object CompositorHost { @Volatile private var started = false + @Volatile private var vsyncRunning = false private val vsync = object : Choreographer.FrameCallback { @@ -71,6 +72,14 @@ object CompositorHost { private var attached: Surface? = null + @Synchronized + fun attachXr() { + check(started) { "DroidDeck compositor is not running" } + pauseVsync() + attached = null + WaylandCompositor.nativeSetSurface(null) + } + @Synchronized fun detach(surface: Surface?) { if (!started) return diff --git a/app/src/main/java/com/droiddeck/launcher/wayland/WaylandCompositor.java b/app/src/main/java/com/droiddeck/launcher/wayland/WaylandCompositor.java index 85f9464c1e..248c2b227d 100644 --- a/app/src/main/java/com/droiddeck/launcher/wayland/WaylandCompositor.java +++ b/app/src/main/java/com/droiddeck/launcher/wayland/WaylandCompositor.java @@ -148,6 +148,7 @@ public final class WaylandCompositor { String nativeLibDir); public static native void nativeSetSurface(Surface surface); + public static native boolean nativeWaitSurface(int timeoutMs); public static native void nativeSendPointer(int action, int x, int y); diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 06968444bf..a50a313226 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -595,6 +595,10 @@ This device has no rotation sensor. Recenter view Recenter + SteamVR + Enter SteamVR + Enter immersive VR with headset tracking and Touch controllers. + Checking SteamVR… SteamVR (experimental) Install Downloads Valve\'s ARM64 SteamVR (571 MB). diff --git a/app/src/test/java/com/droiddeck/launcher/session/SessionVisibilityTest.kt b/app/src/test/java/com/droiddeck/launcher/session/SessionVisibilityTest.kt new file mode 100644 index 0000000000..b35ea287e3 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/session/SessionVisibilityTest.kt @@ -0,0 +1,28 @@ +package com.droiddeck.launcher.session + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class SessionVisibilityTest { + @Test fun xrVisibilitySurvivesPanelHandoffAndPanelScreenState() { + for (panelVisible in listOf(false, true)) { + for (screenOn in listOf(false, true)) { + assertTrue(sessionHasVisibleSurface(panelVisible, screenOn, true)) + } + } + } + + @Test fun normalPanelStillRequiresAnInteractiveScreen() { + assertTrue(sessionHasVisibleSurface(true, true, false)) + assertFalse(sessionHasVisibleSurface(false, true, false)) + assertFalse(sessionHasVisibleSurface(true, false, false)) + assertFalse(sessionHasVisibleSurface(false, false, false)) + } + + @Test fun losingXrVisibilityReturnsToNormalSuspendRules() { + assertTrue(sessionHasVisibleSurface(false, false, true)) + assertFalse(sessionHasVisibleSurface(false, false, false)) + assertTrue(sessionHasVisibleSurface(true, true, false)) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/ui/SessionDrawerPageTest.kt b/app/src/test/java/com/droiddeck/launcher/ui/SessionDrawerPageTest.kt index 6c844ead17..bed0fceb28 100644 --- a/app/src/test/java/com/droiddeck/launcher/ui/SessionDrawerPageTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/ui/SessionDrawerPageTest.kt @@ -20,7 +20,9 @@ class SessionDrawerPageTest { assertEquals(sessionDrawerPages(false), connected.filter { it != SessionDrawerPage.SECOND_SCREEN }) for (pages in listOf(connected, sessionDrawerPages(false))) { assertEquals(SessionDrawerPage.EFFECTS, pages.step(SessionDrawerPage.DISPLAY, 1)) - assertEquals(SessionDrawerPage.GAMES, pages.step(SessionDrawerPage.EFFECTS, 1)) + assertEquals(SessionDrawerPage.STEAMVR, pages.step(SessionDrawerPage.EFFECTS, 1)) + assertEquals(SessionDrawerPage.CARDBOARD, pages.step(SessionDrawerPage.STEAMVR, 1)) + assertEquals(SessionDrawerPage.GAMES, pages.step(SessionDrawerPage.CARDBOARD, 1)) assertEquals(SessionDrawerPage.SESSION, pages.step(SessionDrawerPage.GAMES, 1)) } } diff --git a/docs/agent-control.md b/docs/agent-control.md index e6484c0bbc..26e08a057b 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -117,12 +117,25 @@ tools/droiddeckctl prefs set session some_key true --type bool tools/droiddeckctl logs latest ./session-artifacts tools/droiddeckctl screenshot ./screen.png --display 4 +tools/droiddeckctl xr-frame ./xr-capture tools/droiddeckctl record ./clip.mp4 --seconds 8 --display 0 tools/droiddeckctl pacing --seconds 30 tools/droiddeckctl displays tools/droiddeckctl wake ``` +`xr-frame` captures both app eye textures while SteamVR is open, saving `left.png`, +`right.png` and `capture.json` with the matching source frame, projection poses, +FOV and dimensions. It works over wired or wireless authorized ADB. +Newer builds also include `source.png`, the incoming stereo texture before the +Android eye draw, and sRGB attachment/write state in the metadata for color debugging. +Source PNGs use the incoming frame's color space and eye PNGs use sRGB. Transfers use Android's +gzip utility to avoid sending full uncompressed images over wireless ADB. +A unique request ID prevents stale images from passing as a fresh capture; no valid XR +layer produces an explicit error. Captures exclude Meta system overlays and +the headset compositor's final reprojection. GPU readback runs only on request +and can briefly affect frame timing. Remote files are overwritten each time. + A sleeping device suspends the session and hides the app's windows from `ui dump`; `wake` first. `pacing` reads SurfaceFlinger's timestats for the game's layer (the session's SurfaceView when the @@ -176,7 +189,7 @@ Automator show them: | `play-steam` | Play on the Steam and Games pages | | `setting-` | every toggle and choice row, by its key (`setting-logs`, `setting-agent-commands`, ...) | | `tab-` | a page's tab strip (Setup's Overview, Controller, Session, Launcher...), by position | -| `drawer-tab-controller`, `-second_screen`, `-display`, `-effects`, `-games`, `-session` | the session drawer's pages | +| `drawer-tab-controller`, `-second_screen`, `-display`, `-effects`, `-steamvr`, `-cardboard`, `-games`, `-session` | the session drawer's pages | | `drawer-stop`, `drawer-stop-confirm`, `drawer-stop-cancel` | stopping a session from the drawer | ## Scenarios diff --git a/docs/quest.md b/docs/quest.md index 597fb5ac5d..067b708921 100644 --- a/docs/quest.md +++ b/docs/quest.md @@ -1,7 +1,7 @@ # Experimental Quest 2/3/3S support -Quest 3/3S runs DroidDeck as a 2D panel. This does not add immersive VR or -OpenXR support. Quest 2 has a separate experimental Adreno 6xx driver path. +Quest runs DroidDeck as a 2D panel, with an experimental immersive SteamVR path +in the standard app. Quest 2 has a separate experimental Adreno 6xx driver path. Quest 2 selects `turnip26.3.0-quest2`, a pinned Banners-Turnip A6xx/A7xx build with the same GPUOBJ adapter. It does not select the Quest 3 A7xx binary. @@ -64,3 +64,147 @@ On a headset, check the Steam login panel, sign-in, game installation and then an owned game's rendering/input. Capture the device/OS, session logs, `files/quest-startup.txt` and any `files/quest-last-crash.txt` failure. Host allocation and preload-chain tests do not establish headset operation. + +## Experimental immersive SteamVR + +In a running Steam session, open the session menu's SteamVR tab, install SteamVR +once, then choose Enter SteamVR. Cardboard has its own tab and can remain off. On supported Quest devices this opens the standard app's +OpenXR activity. The guest driver receives headset poses, Touch controller input +and display timing, and returns owned stereo images tagged with the latest eye poses. +SteamVR haptics return through the tracking bridge. No root or separate XR APK +is required. Other devices retain their existing SteamVR start behavior. + +The left Touch menu button is SteamVR's system button and toggles its dashboard. +The right Meta button remains reserved by the headset OS. + +The driver supplies SteamVR's version-5 chaperone configuration through +`Prop_DriverProvidedChaperoneJson_String`, following Valve's driver API. Standing +tracking keeps the OpenXR stage coordinates; seated tracking starts at the head's +position and yaw. No synthetic play area or collision walls are advertised. +Without a seated transform, SteamVR reported valid raw and standing poses but +invalid seated poses, causing Thumper to remain behind its tracking-loss fade. +On Quest, supplying both transforms made all three origins valid; seated height +started within a millimetre of zero while standing and raw height stayed unchanged. + +Quest uses SteamVR's normal Linux compositor through its display component. +The Vulkan layer captures only the HMD display swapchain, retains its present +waits, and chains a GPU copy before presentation. Three bounded capture buffers +allow rendering and transfer to overlap. Where native GLES supports +`GL_EXT_external_buffer` and texture views, Android allocates three +`AHardwareBuffer` BLOBs and passes their DMA-BUF allocations to Linux through +`SCM_RIGHTS`. Vulkan imports them as buffers and transfers foreign queue-family +ownership before and after copying the display image. A native compute shader +reads each shared buffer into a private RGBA8 texture; an sRGB texture view +preserves hardware transfer decoding. Native GPU completion precedes the +acknowledgement, so Linux cannot overwrite an active reader. This removes the CPU +pixel copy and upload, but still includes GPU copies and explicit completion waits. + +If capability negotiation or buffer import fails before frames are submitted, +the existing CPU transport remains available. It brackets the DMA-BUF read with +`DMA_BUF_IOCTL_SYNC`, copies to owned pixels, then acknowledges and uploads them. +Both channels authenticate the peer UID and bound buffer sizes. Missing +acknowledgements after submission quarantine the transfer and fail presentation +instead of falling back while a reader might still own a buffer. + +The helper temporarily enables SteamVR's Linux Vulkan asynchronous queue and disables motion +smoothing, backing up those preferences and restoring them on stop or the next start +after an interrupted session. It excludes the pinned Steam Frame package's optional CVP backend, +whose library constructor exits on Quest's unsupported SoC. The original library +is retained with a `.droiddeck-disabled` suffix. SteamVR binaries are not patched. +Cardboard retains its existing direct-mode path; `questframes` remains an explicit +diagnostic override for the earlier driver presenter. + +The panel compositor detaches before OpenXR starts displaying guest frames. +Session suspension accounts for the immersive activity while the panel is hidden. +Closing the immersive activity stops SteamVR. The panel remains available for +returning to the normal Steam session. +The immersive activity belongs to the Steam session that opened it. Ending that +session closes XR; a failed startup or stopped SteamVR also returns to the panel +with an error instead of leaving a black scene. Enter SteamVR can then start a +fresh connection. + +SteamVR scene-application and dashboard events also guide Gamescope's game focus. +While the dashboard is hidden, the guest agent resolves the live scene process's +Steam app ID and promotes its focusable window. Opening the dashboard or ending +the scene restores the previous order if no other client has replaced it. This +keeps keyboard/gamepad input with the VR scene without game-specific IDs. + +With Steam already running, `python3 tools/test_quest_xr.py --serial SERIAL` +launches the immersive activity and verifies advancing owned images, tracked +6DoF, focus and SteamVR status. Add `--require-controllers` while holding both +controllers. Results include telemetry and logcat in `build/questxr-validation`. +Host tests and APK installation alone do not establish headset operation. + +On Quest 3, a 45-second hardware run maintained focused tracked 6DoF, both +controllers active, and advancing stereo images. A headset tester confirmed +controller models and trigger animation. That run delivered about 8.5 new stereo +images per second despite a 90 Hz XR session. +That earlier measurement used the driver-direct presenter. With SteamVR's normal +compositor, a native Vulkan scene passed its first submission and continued +submitting stereo frames. Thumper and The Thrill of the Fight produced clean +immersive scenes. SUPERHOT's Windows OpenXR build initially showed shared-image +corruption: DXVK supplied a restricted mutable-image format list while SteamVR +created its corresponding opaque shared images without one. Turnip can select +different memory layouts for those descriptions. On the Quest bridge, the layer +normalizes the known opaque-FD stereo-array creation chains to SteamVR's unrestricted list, +without changing application-owned single-layer OpenVR images, DMA-BUF/modifier +images or other targets. A headset capture then +showed clean SUPERHOT scene geometry; interactive gameplay remains unverified. +A subsequent 60-second shared-buffer dashboard run delivered about +43 new stereo images per second. Reusing Android pixel buffers raised the next +60-second run to about 68 received and 65 uploaded images per second, with valid +focused tracking and no rejected frames. These dashboard results do not establish refresh-rate gameplay: game input, +sustained game performance, and exact association with SteamVR's final compositor +pose still require validation. + +The native GPU-buffer channel passed 256 full-size buffer reuse cycles, including +head/tail pixel checks and sRGB texture-view decoding. A subsequent SUPERHOT +scene run delivered about 43 new stereo images/s and 58 XR draws/s over one +minute, with focused tracking and no rejected frames. The earlier CPU-path +SUPERHOT run uploaded about 28 images/s; these are separate runs, not a controlled +benchmark. Capturing the source and eye images can delay frame consumption; +the GPU receiver acknowledges stale frames without reading them and keeps the +last private image until a fresh frame arrives. Missing completion still fails +closed. `frame_verify` explicitly selects the CPU path for byte comparison. + +The frame channel supports linear and sRGB RGBA8. SteamVR's final display image +uses sRGB display values; Android decodes those during texture sampling and writes +to an sRGB eye swapchain, encoding the display values once. A Quest color-chart +capture verified that 64, 128, 192 and 224 grey patches retain those values; +the previous sRGB source upload darkened them to approximately 13, 55, 134 and +190. The transport remains 8-bit SDR; this correction does not add HDR or higher +precision gradients. + +When available, `XR_EXT_performance_settings` requests sustained-high CPU and GPU +performance for the running OpenXR session. These are runtime-managed hints; +device power and thermal policies remain authoritative. Unsupported or rejected +requests do not prevent startup. This does not change render resolution. + +The manifest declares the supported Quest models using Meta's standard +`com.oculus.supportedDevices` metadata. Without it, Quest 3 used compatibility +mode and its refresh-rate API returned an empty capability list. With the +declaration, the test runtime reported 72/80/90/120 Hz; a fresh normal launch +used 72 Hz in both OpenXR telemetry and SteamVR's HMD property. DroidDeck leaves +the refresh choice to the runtime. No forced refresh override is included. + +The OpenVR driver maps OpenXR's asymmetric eye FOV to OpenVR's lower/upper Y +projection bounds. A live Quest 3 check confirmed that SteamVR and native +OpenXR now report the same vertical projection center; the previous mapping +mirrored that offset. This applies to every VR scene, without game-specific +rendering changes. + +### Experimental flat games in SteamVR + +Start SteamVR first, then add `droiddeck-steamvr-game %command%` to a flat +game's Steam launch options. Remove that prefix to restore normal panel launch. +This runs a nested, unmodified Gamescope OpenVR backend for that game. Gamescope +owns the GPU overlay, placement and input routing; no CPU screen capture or +PipeWire service is used. The existing DroidDeck Vulkan synchronization layer +remains enabled. On Quest, a connected XR receiver selects Gamescope's Vulkan +texture sharing instead of DMA-BUF modifiers. Explicit environment overrides +are preserved; other devices retain the upstream modifier default. + +An upstream GPU cube overlay has been observed on Quest 3. Flat-game stability, +controller interaction and performance still require hardware validation. This +uses Valve's public Gamescope backend; the exact Steam Frame OS launch setup +has not been verified. diff --git a/tools/droiddeckctl b/tools/droiddeckctl index d88327a59d..65be5580cf 100755 --- a/tools/droiddeckctl +++ b/tools/droiddeckctl @@ -5,13 +5,18 @@ from __future__ import annotations import argparse import base64 +import gzip import json import os import re import shutil import subprocess +import struct import sys +import tempfile import time +import uuid +import zlib from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -382,8 +387,101 @@ def input_request(args: argparse.Namespace) -> dict: raise ControlError(f"Unknown input type {kind}", "INVALID_INPUT") +def xr_ppm_to_png(data: bytes, linear: bool = False) -> bytes: + header = re.match(rb"P6\n([0-9]+) ([0-9]+)\n255\n", data) + if not header: + raise ControlError("Invalid XR image header", "XR_CAPTURE_FAILED") + width, height = map(int, header.groups()) + pixels = data[header.end():] + if not (0 < width <= 16384 and 0 < height <= 16384) or len(pixels) != width * height * 3: + raise ControlError("Invalid XR image dimensions or incomplete pixels", "XR_CAPTURE_FAILED") + def chunk(kind: bytes, value: bytes) -> bytes: + return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value)) + stride = width * 3 + rows = b''.join(b'\0' + pixels[start:start + stride] for start in range(0, len(pixels), stride)) + transfer = chunk(b'gAMA', struct.pack('>I', 100000)) if linear else chunk(b'sRGB', b'\0') + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + transfer + + chunk(b'IDAT', zlib.compress(rows)) + chunk(b'IEND', b'')) + + +def xr_capture(adb: str, serial: str, destination: Path, timeout: float) -> dict: + if not 0 < timeout <= 120: + raise ControlError("XR capture timeout must be between 0 and 120 seconds", "INVALID_TIMEOUT") + remote = f'/sdcard/Android/data/{PACKAGE}/files/xr-capture' + token = uuid.uuid4().hex + with tempfile.TemporaryDirectory() as temporary: + request = Path(temporary) / 'request' + request.write_text(token, encoding='ascii') + pushed = run_adb(adb, serial, 'push', str(request), remote + '.request.tmp') + if pushed.returncode: + raise ControlError(pushed.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED') + moved = run_adb(adb, serial, 'shell', 'mv', remote + '.request.tmp', remote + '.request') + if moved.returncode: + raise ControlError(moved.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED') + deadline = time.monotonic() + timeout + metadata = None + while time.monotonic() < deadline: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + result = run_adb(adb, serial, 'shell', 'cat', remote + '.json', timeout=remaining) + except ControlError as error: + if not isinstance(error.__cause__, subprocess.TimeoutExpired): + raise + break + try: + answer = json.loads(result.stdout) + if answer.get('id') == token: + metadata = answer + break + except (ValueError, AttributeError): + pass + time.sleep(0.2) + if metadata is None: + try: + run_adb(adb, serial, 'shell', 'rm', '-f', remote + '.request', timeout=1) + except ControlError: + pass + raise ControlError('No fresh XR capture. Enter SteamVR using a build with XR capture support.', 'TIMEOUT') + if not metadata.get('ok'): + detail = 'No valid XR projection layer is being submitted' if not metadata.get('validLayer') else 'GPU readback or image write failed' + raise ControlError(detail, 'XR_CAPTURE_FAILED') + images = {} + for eye in (('left', 'right', 'source') if metadata.get('sourceImage') else ('left', 'right')): + pulled = run_adb(adb, serial, 'exec-out', 'gzip', '-c', remote + '-' + eye + '.ppm', timeout=120) + if pulled.returncode: + raise ControlError(pulled.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED') + try: + pixels = gzip.decompress(pulled.stdout) + except (OSError, EOFError, zlib.error) as error: + raise ControlError('Incomplete compressed XR image', 'XR_CAPTURE_FAILED') from error + images[eye] = xr_ppm_to_png(pixels, linear=eye == 'source' and metadata.get('sourceColorSpace') == 'linear') + # Detect overlapping captures before publishing images with another request's metadata. + check = run_adb(adb, serial, 'shell', 'cat', remote + '.json') + try: + unchanged = json.loads(check.stdout).get('id') == token + except (ValueError, AttributeError): + unchanged = False + if not unchanged: + raise ControlError('XR capture was replaced by another request; retry', 'XR_CAPTURE_FAILED') + try: + destination.mkdir(parents=True, exist_ok=True) + for eye, image in images.items(): + (destination / (eye + '.png')).write_bytes(image) + (destination / 'capture.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8') + except OSError as error: + raise ControlError(str(error), 'FILE_IO') from error + return {'ok': True, 'command': 'xr-frame', 'deviceSerial': serial, 'capture': metadata, + 'left': str((destination / 'left.png').resolve()), 'right': str((destination / 'right.png').resolve()), + **({'source': str((destination / 'source.png').resolve())} if 'source' in images else {}), + 'scope': 'App eye textures before headset compositor reprojection; excludes Meta system overlays'} + + def execute(args: argparse.Namespace, adb: str, serial: str) -> dict: command = args.command + if command == 'xr-frame': + return xr_capture(adb, serial, Path(args.path), args.timeout) if command == "state": return add_serial(get_state(adb, serial), serial) @@ -717,6 +815,10 @@ def build_parser() -> argparse.ArgumentParser: shot.add_argument("path") shot.add_argument("--display", type=int, help="logical display id (see displays); default the default display") + xr = commands.add_parser('xr-frame', help='capture both submitted XR eyes and pose metadata') + xr.add_argument('path', help='output directory for left.png, right.png and capture.json') + xr.add_argument('--timeout', type=float, default=15) + rec = commands.add_parser("record", help="record the screen to an mp4") rec.add_argument("path") rec.add_argument("--seconds", type=float, default=10) @@ -738,7 +840,7 @@ EXIT_SESSION = { } EXIT_TIMEOUT = {"TIMEOUT", "ARTIFACT_TIMEOUT"} EXIT_FILE = {"NO_SESSION_LOGS", "LOG_PULL_FAILED", "SCREENSHOT_FAILED", "FILE_IO", "RECORD_FAILED", "PUSH_FAILED", - "PACING_FAILED"} + "PACING_FAILED", "XR_CAPTURE_FAILED"} EXIT_DENIED = {"AGENT_COMMANDS_DISABLED", "PROTECTED_PREFS", "PROTECTED_CONTROL"} diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-agent b/tools/linuxfs/overlay/usr/local/bin/droiddeck-agent index 504321cc26..631eb9ad5e 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-agent +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-agent @@ -177,6 +177,11 @@ class X: return list(struct.unpack("<%dI" % n, data)) return data + def cardinals(self, window, name, values): + self.s.sendall(struct.pack("