diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7bd2d1d080..19006c522e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -396,6 +396,11 @@ jobs: set -euo pipefail bash tools/linuxfs/check-preloads.sh app/src/main/assets/linuxfs + - name: Check Quest tracking IPC + run: | + g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_tracking.cpp -o /tmp/test-xr-tracking + /tmp/test-xr-tracking + - name: Rust toolchain for the store engines (aarch64-linux-android) id: rust run: | @@ -429,7 +434,7 @@ jobs: path: | app/.cxx app/build/intermediates/cxx - key: cxx-${{ env.NDK_VERSION }}-${{ hashFiles('app/src/main/cpp/**') }} + key: cxx-${{ env.NDK_VERSION }}-${{ hashFiles('app/src/main/cpp/**', 'questxr/src/main/cpp/**', 'tools/vr/src/xr_tracking.h') }} restore-keys: | cxx-${{ env.NDK_VERSION }}- diff --git a/docs/development/quest-openxr.md b/docs/development/quest-openxr.md index cb35da1ba6..406e8231e3 100644 --- a/docs/development/quest-openxr.md +++ b/docs/development/quest-openxr.md @@ -46,8 +46,6 @@ After declaring optional controller/hand capability and restarting VR Shell, the app launched. Automated tests should report launch blockers rather than treating a successful `am start` as successful rendering. -## Remaining stages - ## Milestone 2: Android projection surface and guest integration The Android Surface producer test passed repeatedly, including 1,079 submitted @@ -75,6 +73,30 @@ association and return haptics still need implementation/validation. The companion's release build now targets Android 14 and passes release lint. +## Milestone 3: input, timing and return haptics + +The driver now supplies eye-to-head transforms and runtime-derived IPD, consumes +Touch trigger/squeeze/stick/button state, reports predicted pose timestamps and +paces vsync against converted OpenXR display timing. Vive-profile compatibility +maps primary buttons to trackpad click and secondary/menu buttons to application +menu. A versioned 64-byte feedback channel carries bounded driver haptic events +back to OpenXR. Repeated surface tests still submit 1,079 stereo frames at 72 Hz. +The native and Kotlin targets compile, release lint passes, and the ARM64 Linux +driver links with no undefined symbols. IPC tests cover valid packets, odd +seqlocks, version mismatch, stale/future timestamps, inactive state, excessive +dimensions and truncated files; they run in CI as well. + +The first full integration CI APK compiled successfully but failed the ELF +closure check because its system-library list omitted `libGLESv3.so`. The list +now recognizes that Android system dependency. A subsequent full CI run is +required before deployment of the integrated main-app build. + +On-device prerequisites: the installed app's Debugging tools toggle explicitly +rejects agent UI activation (`PROTECTED_CONTROL`); it must be switched on-device. +Steam is signed out and both Touch controllers remain inactive. These block +guest command diagnostics and actual game/controller/haptic validation, but do +not block native rendering or independent engineering. No guard was bypassed. + ## Remaining stages - Android/Linux eye-image bridge and frame ownership/synchronization. diff --git a/questxr/src/main/cpp/quest_xr.cpp b/questxr/src/main/cpp/quest_xr.cpp index de233178c1..8a90f237ce 100644 --- a/questxr/src/main/cpp/quest_xr.cpp +++ b/questxr/src/main/cpp/quest_xr.cpp @@ -148,6 +148,8 @@ private: jmethodID surfaceCallback_ = nullptr, frameCallback_ = nullptr; jobject androidJavaSurface_ = nullptr; DroidDeckXrTracking *tracking_ = nullptr; + DroidDeckXrHaptics *haptics_ = nullptr; + uint64_t lastPulse_[2]{}; XrSwapchain androidSwapchain_ = XR_NULL_HANDLE; ANativeWindow *androidWindow_ = nullptr; EGLSurface androidSurface_ = EGL_NO_SURFACE; @@ -380,16 +382,22 @@ private: std::string directory = std::string(app_->activity->internalDataPath) + "/.wayland-rt"; if (mkdir(directory.c_str(), 0700) != 0 && errno != EEXIST) throw std::runtime_error("XR tracking directory unavailable"); - std::string filename = directory + "/" DROIDDECK_XR_FILE; - int descriptor = open(filename.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0600); - if (descriptor < 0) throw std::runtime_error("XR tracking file unavailable"); - if (ftruncate(descriptor, sizeof(DroidDeckXrTracking)) != 0) { - close(descriptor); throw std::runtime_error("XR tracking file size failed"); - } - void *mapping = mmap(nullptr, sizeof(DroidDeckXrTracking), PROT_READ | PROT_WRITE, MAP_SHARED, descriptor, 0); - close(descriptor); - if (mapping == MAP_FAILED) throw std::runtime_error("XR tracking map failed"); - tracking_ = static_cast(mapping); + auto mapFile = [&](const char *name, size_t size) { + std::string filename = directory + "/" + name; + int descriptor = open(filename.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0600); + if (descriptor < 0) throw std::runtime_error("XR IPC file unavailable"); + if (ftruncate(descriptor, size) != 0) { + close(descriptor); throw std::runtime_error("XR IPC file size failed"); + } + void *mapping = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, descriptor, 0); + close(descriptor); + if (mapping == MAP_FAILED) throw std::runtime_error("XR IPC map failed"); + return mapping; + }; + tracking_ = static_cast(mapFile(DROIDDECK_XR_FILE, sizeof(DroidDeckXrTracking))); + haptics_ = static_cast(mapFile(DROIDDECK_XR_HAPTIC_FILE, sizeof(DroidDeckXrHaptics))); + haptics_->magic = DROIDDECK_XR_HAPTIC_MAGIC; haptics_->version = DROIDDECK_XR_VERSION; + for (int hand = 0; hand < 2; hand++) lastPulse_[hand] = haptics_->hands[hand].id; } static DroidDeckXrPose posePacket(const XrPosef &pose, XrSpaceLocationFlags flags) { @@ -442,6 +450,30 @@ private: guestAttached_ = attach; } + void pollHaptics() { + if (!haptics_) return; + DroidDeckXrHaptics packet{}; + uint32_t sequence = __atomic_load_n(&haptics_->seq, __ATOMIC_ACQUIRE); + if (sequence & 1u) return; + std::memcpy(&packet, haptics_, sizeof(packet)); + __atomic_thread_fence(__ATOMIC_ACQUIRE); + if (__atomic_load_n(&haptics_->seq, __ATOMIC_RELAXED) != sequence + || packet.magic != DROIDDECK_XR_HAPTIC_MAGIC || packet.version != DROIDDECK_XR_VERSION) return; + for (int hand = 0; hand < 2; hand++) { + const auto &pulse = packet.hands[hand]; + if (pulse.id == lastPulse_[hand]) continue; + lastPulse_[hand] = pulse.id; + if (state_ != XR_SESSION_STATE_FOCUSED || !handActive_[hand] + || pulse.duration_ns < 1000000 || pulse.duration_ns > 1000000000 + || !std::isfinite(pulse.amplitude) || pulse.amplitude < 0 || pulse.amplitude > 1 + || !std::isfinite(pulse.frequency) || pulse.frequency < 0) continue; + XrHapticActionInfo info{XR_TYPE_HAPTIC_ACTION_INFO}; info.action = vibration_; info.subactionPath = hands_[hand]; + XrHapticVibration vibration{XR_TYPE_HAPTIC_VIBRATION}; + vibration.duration = pulse.duration_ns; vibration.amplitude = pulse.amplitude; vibration.frequency = pulse.frequency; + check(xrApplyHapticFeedback(session_, &info, reinterpret_cast(&vibration)), "guest haptics"); + } + } + GLuint shader(GLenum type, const char *text) { GLuint handle = glCreateShader(type); glShaderSource(handle, 1, &text, nullptr); glCompileShader(handle); @@ -577,6 +609,7 @@ private: pressed_[hand] = pressed; } } else handActive_[0] = handActive_[1] = false; + pollHaptics(); XrViewLocateInfo locate{XR_TYPE_VIEW_LOCATE_INFO}; locate.viewConfigurationType = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; locate.displayTime = frameState.predictedDisplayTime; locate.space = world_; XrViewState viewState{XR_TYPE_VIEW_STATE}; uint32_t count = 0; @@ -641,6 +674,7 @@ private: __atomic_store_n(&tracking_->active, 0u, __ATOMIC_RELEASE); munmap(tracking_, sizeof(DroidDeckXrTracking)); } + if (haptics_) munmap(haptics_, sizeof(DroidDeckXrHaptics)); if (display_ != EGL_NO_DISPLAY && context_ != EGL_NO_CONTEXT) { glDeleteFramebuffers(1, &framebuffer_); glDeleteProgram(program_); glDeleteBuffers(1, &buffer_); glDeleteVertexArrays(1, &vertexArray_); diff --git a/tools/release/check_android_libraries.py b/tools/release/check_android_libraries.py index 80b543e803..5d6c775561 100644 --- a/tools/release/check_android_libraries.py +++ b/tools/release/check_android_libraries.py @@ -8,7 +8,7 @@ from zipfile import ZipFile SYSTEM = set('libc.so libm.so libdl.so liblog.so libandroid.so libz.so libvulkan.so libstdc++.so ' - 'libGLESv2.so libEGL.so libnativewindow.so libjnigraphics.so libaaudio.so ' + 'libGLESv2.so libGLESv3.so libEGL.so libnativewindow.so libjnigraphics.so libaaudio.so ' 'libOpenSLES.so libmediandk.so libcamera2ndk.so libsync.so libneuralnetworks.so'.split()) EXPORTS = ['Java_com_droiddeck_launcher_stores_StoresNative_nativeVersion', 'Java_com_droiddeck_launcher_stores_gog_GogNative_nativeStart', diff --git a/tools/vr/src/driver_droiddeck.cpp b/tools/vr/src/driver_droiddeck.cpp index 09a9066ab8..2239d83681 100644 --- a/tools/vr/src/driver_droiddeck.cpp +++ b/tools/vr/src/driver_droiddeck.cpp @@ -88,6 +88,27 @@ HmdQuaternion_t QuatFromMatrix(const float m[9]) { return q; } +HmdMatrix34_t XrPoseMatrix(const DroidDeckXrPose &pose) { + const float qx = pose.orientation[0], qy = pose.orientation[1], qz = pose.orientation[2], qw = pose.orientation[3]; + HmdMatrix34_t matrix{}; + matrix.m[0][0] = 1 - 2 * (qy * qy + qz * qz); matrix.m[0][1] = 2 * (qx * qy - qw * qz); matrix.m[0][2] = 2 * (qx * qz + qw * qy); + matrix.m[1][0] = 2 * (qx * qy + qw * qz); matrix.m[1][1] = 1 - 2 * (qx * qx + qz * qz); matrix.m[1][2] = 2 * (qy * qz - qw * qx); + matrix.m[2][0] = 2 * (qx * qz - qw * qy); matrix.m[2][1] = 2 * (qy * qz + qw * qx); matrix.m[2][2] = 1 - 2 * (qx * qx + qy * qy); + for (int axis = 0; axis < 3; axis++) matrix.m[axis][3] = pose.position[axis]; + return matrix; +} + +HmdMatrix34_t XrEyeToHead(const DroidDeckXrPose &head, const DroidDeckXrPose &eye) { + HmdMatrix34_t headMatrix = XrPoseMatrix(head), eyeMatrix = XrPoseMatrix(eye), result{}; + for (int row = 0; row < 3; row++) { + for (int column = 0; column < 3; column++) + for (int inner = 0; inner < 3; inner++) result.m[row][column] += headMatrix.m[inner][row] * eyeMatrix.m[inner][column]; + for (int inner = 0; inner < 3; inner++) + result.m[row][3] += headMatrix.m[inner][row] * (eyeMatrix.m[inner][3] - headMatrix.m[inner][3]); + } + return result; +} + class VirtualDisplay : public IVRVirtualDisplay { public: void Present(const PresentInfo_t *info, uint32_t size) override { @@ -121,20 +142,29 @@ public: static void Start() { if (running_.exchange(true)) return; thread_ = std::thread([] { + XrTrackingFile xrFile; auto next = std::chrono::steady_clock::now() + kInterval; while (running_) { + auto interval = kInterval; + DroidDeckXrTracking xr; + if (xrFile.Read(&xr) && XrTrackingFile::Fresh(xr)) { + interval = std::chrono::nanoseconds(xr.period_ns); + auto tick = std::chrono::steady_clock::time_point(std::chrono::nanoseconds(xr.predicted_ns)); + auto now = std::chrono::steady_clock::now(); + if (xr.predicted_ns > 0 && tick > now && tick < now + interval * 3) next = tick; + } auto announce = next - kLead; std::this_thread::sleep_until(announce); auto now = std::chrono::steady_clock::now(); - double offset = std::chrono::duration(announce - now).count(); + double offset = std::chrono::duration(next - now).count(); VRServerDriverHost()->VsyncEvent(offset); { std::lock_guard lock(mutex_); last_ = next; } - next += kInterval; + next += interval; now = std::chrono::steady_clock::now(); - while (next <= now) next += kInterval; + while (next <= now) next += interval; } }); } @@ -372,6 +402,7 @@ public: void DebugRequest(const char *, char *resp, uint32_t size) override { if (size) resp[0] = 0; } DriverPose_t GetPose() override { return pose_; } + VRInputComponentHandle_t HapticHandle() const { return haptic_; } void UpdateXr(const DroidDeckXrHand &hand) { if (id_ == k_unTrackedDeviceIndexInvalid) return; @@ -389,6 +420,11 @@ public: VRDriverInput()->UpdateBooleanComponent(grip_, hand.active && hand.squeeze > 0.6f, 0); VRDriverInput()->UpdateScalarComponent(padX_, hand.active ? hand.stick[0] : 0, 0); VRDriverInput()->UpdateScalarComponent(padY_, hand.active ? hand.stick[1] : 0, 0); + VRDriverInput()->UpdateBooleanComponent(padClick_, hand.active && (hand.buttons & 3u), 0); + VRDriverInput()->UpdateBooleanComponent(padTouch_, hand.active && + (std::abs(hand.stick[0]) + std::abs(hand.stick[1]) > 0.01f || (hand.buttons & 1u)), 0); + VRDriverInput()->UpdateBooleanComponent(menu_, hand.active && (hand.buttons & 12u), 0); + VRDriverInput()->UpdateBooleanComponent(system_, false, 0); } void Update(const HmdQuaternion_t &head, const PadState *pad) { @@ -462,6 +498,15 @@ public: VRProperties()->SetStringProperty(c, Prop_ManufacturerName_String, "DroidDeck"); VRProperties()->SetStringProperty(c, Prop_TrackingSystemName_String, "droiddeck"); VRProperties()->SetFloatProperty(c, Prop_UserIpdMeters_Float, 0.064f); + if (xrMode_) { + VRServerDriverHost()->SetDisplayEyeToHead(id_, XrEyeToHead(xr.head, xr.eyes[0].pose), XrEyeToHead(xr.head, xr.eyes[1].pose)); + float distanceSquared = 0; + for (int axis = 0; axis < 3; axis++) { + float difference = xr.eyes[0].pose.position[axis] - xr.eyes[1].pose.position[axis]; + distanceSquared += difference * difference; + } + VRProperties()->SetFloatProperty(c, Prop_UserIpdMeters_Float, std::sqrt(distanceSquared)); + } VRProperties()->SetFloatProperty(c, Prop_UserHeadToEyeDepthMeters_Float, 0.0f); VRProperties()->SetFloatProperty(c, Prop_DisplayFrequency_Float, hz); VRProperties()->SetFloatProperty(c, Prop_SecondsFromVsyncToPhotons_Float, 0.011f); @@ -522,6 +567,7 @@ public: if (p.poseIsValid) { p.qRotation = {xr.head.orientation[3], xr.head.orientation[0], xr.head.orientation[1], xr.head.orientation[2]}; for (int axis = 0; axis < 3; axis++) p.vecPosition[axis] = xr.head.position[axis]; + if (xr.predicted_ns > 0) p.poseTimeOffset = (xr.predicted_ns - XrTrackingFile::Now()) / 1000000000.0; } return p; } @@ -656,17 +702,49 @@ public: } return VRInitError_None; } - void Cleanup() override { VR_CLEANUP_SERVER_DRIVER_CONTEXT(); } + void Cleanup() override { + if (haptics_) { munmap(haptics_, sizeof(*haptics_)); haptics_ = nullptr; } + VR_CLEANUP_SERVER_DRIVER_CONTEXT(); + } const char *const *GetInterfaceVersions() override { return k_InterfaceVersions; } void RunFrame() override { VREvent_t e; - while (VRServerDriverHost()->PollNextEvent(&e, sizeof(e))) {} + while (VRServerDriverHost()->PollNextEvent(&e, sizeof(e))) { + if (e.eventType == VREvent_Input_HapticVibration) { + const auto &pulse = e.data.hapticVibration; + int hand = pulse.componentHandle == left_.HapticHandle() ? 0 : pulse.componentHandle == right_.HapticHandle() ? 1 : -1; + if (hand >= 0) SendHaptic(hand, pulse); + } + } } bool ShouldBlockStandbyMode() override { return false; } void EnterStandby() override {} void LeaveStandby() override {} private: + DroidDeckXrHaptics *haptics_ = nullptr; + void SendHaptic(int hand, const VREvent_HapticVibration_t &pulse) { + if (!std::isfinite(pulse.fAmplitude) || !std::isfinite(pulse.fFrequency) || !std::isfinite(pulse.fDurationSeconds)) return; + if (!haptics_) { + int descriptor = open("/run/droiddeck/" DROIDDECK_XR_HAPTIC_FILE, O_RDWR | O_CLOEXEC); + if (descriptor < 0) return; + struct stat status{}; + if (fstat(descriptor, &status) != 0 || status.st_size != sizeof(*haptics_)) { close(descriptor); return; } + void *mapping = mmap(nullptr, sizeof(*haptics_), PROT_READ | PROT_WRITE, MAP_SHARED, descriptor, 0); + close(descriptor); + if (mapping == MAP_FAILED) return; + haptics_ = static_cast(mapping); + } + uint32_t sequence = __atomic_load_n(&haptics_->seq, __ATOMIC_RELAXED); + __atomic_store_n(&haptics_->seq, sequence | 1u, __ATOMIC_SEQ_CST); + __atomic_thread_fence(__ATOMIC_SEQ_CST); + auto &target = haptics_->hands[hand]; + target.id++; + target.duration_ns = (int64_t)(std::clamp(pulse.fDurationSeconds, 0.001f, 1.f) * 1000000000.0); + target.amplitude = std::clamp(pulse.fAmplitude, 0.f, 1.f); + target.frequency = std::max(pulse.fFrequency, 0.f); + __atomic_store_n(&haptics_->seq, (sequence | 1u) + 1u, __ATOMIC_RELEASE); + } CardboardHmd hmd_; CardboardController left_{false}, right_{true}; }; diff --git a/tools/vr/src/xr_tracking.h b/tools/vr/src/xr_tracking.h index 6258d32ae6..8289e66c9c 100644 --- a/tools/vr/src/xr_tracking.h +++ b/tools/vr/src/xr_tracking.h @@ -32,3 +32,19 @@ struct DroidDeckXrTracking { }; static_assert(sizeof(DroidDeckXrTracking) == 296, "XR tracking IPC layout"); + +#define DROIDDECK_XR_HAPTIC_MAGIC 0x48584444u +#define DROIDDECK_XR_HAPTIC_FILE "droiddeck-xr-haptics" + +struct DroidDeckXrPulse { + uint64_t id; + int64_t duration_ns; + float amplitude, frequency; +}; + +struct DroidDeckXrHaptics { + uint32_t magic, version, seq, reserved; + DroidDeckXrPulse hands[2]; +}; + +static_assert(sizeof(DroidDeckXrHaptics) == 64, "XR haptics IPC layout"); diff --git a/tools/vr/src/xr_tracking_file.h b/tools/vr/src/xr_tracking_file.h index 5295ee207c..c8e442e91c 100644 --- a/tools/vr/src/xr_tracking_file.h +++ b/tools/vr/src/xr_tracking_file.h @@ -2,6 +2,7 @@ #include "xr_tracking.h" #include #include +#include #include #include #include @@ -10,11 +11,12 @@ class XrTrackingFile { public: + explicit XrTrackingFile(const char *path = "/run/droiddeck/" DROIDDECK_XR_FILE) : path_(path) {} ~XrTrackingFile() { if (mapping_) munmap(mapping_, sizeof(*mapping_)); } bool Read(DroidDeckXrTracking *packet) { std::lock_guard lock(mutex_); if (!mapping_) { - int descriptor = open("/run/droiddeck/" DROIDDECK_XR_FILE, O_RDONLY | O_CLOEXEC); + int descriptor = open(path_.c_str(), O_RDONLY | O_CLOEXEC); if (descriptor < 0) return false; struct stat status{}; if (fstat(descriptor, &status) != 0 || status.st_size != sizeof(*mapping_)) { @@ -48,4 +50,5 @@ public: private: DroidDeckXrTracking *mapping_ = nullptr; std::mutex mutex_; + std::string path_; }; diff --git a/tools/vr/tests/test_xr_tracking.cpp b/tools/vr/tests/test_xr_tracking.cpp new file mode 100644 index 0000000000..be3354584f --- /dev/null +++ b/tools/vr/tests/test_xr_tracking.cpp @@ -0,0 +1,47 @@ +#include "../src/xr_tracking_file.h" +#include +#include + +int main() { + char filename[] = "/tmp/droiddeck-xr-test-XXXXXX"; + int descriptor = mkstemp(filename); + assert(descriptor >= 0); + assert(ftruncate(descriptor, sizeof(DroidDeckXrTracking)) == 0); + auto *writer = static_cast(mmap(nullptr, sizeof(DroidDeckXrTracking), + PROT_READ | PROT_WRITE, MAP_SHARED, descriptor, 0)); + assert(writer != MAP_FAILED); + { + XrTrackingFile reader(filename); + DroidDeckXrTracking packet{}; + assert(!reader.Read(&packet)); + writer->magic = DROIDDECK_XR_MAGIC; writer->version = DROIDDECK_XR_VERSION; + writer->active = 1; writer->sampled_ns = XrTrackingFile::Now(); writer->period_ns = 13888889; + writer->eyes[0].width = 1680; writer->eyes[0].height = 1760; + writer->head.orientation[3] = 1; writer->head.position[1] = 1.6f; writer->head.flags = 15; + assert(reader.Read(&packet)); assert(XrTrackingFile::Fresh(packet)); + assert(packet.head.position[1] == 1.6f); + __atomic_store_n(&writer->seq, 1u, __ATOMIC_RELEASE); + assert(!reader.Read(&packet)); + __atomic_store_n(&writer->seq, 2u, __ATOMIC_RELEASE); + writer->version++; + assert(!reader.Read(&packet)); + writer->version = DROIDDECK_XR_VERSION; + writer->sampled_ns = XrTrackingFile::Now() - 1000000000LL; + assert(reader.Read(&packet)); assert(!XrTrackingFile::Fresh(packet)); + writer->sampled_ns = XrTrackingFile::Now() + 1000000000LL; + assert(reader.Read(&packet)); assert(!XrTrackingFile::Fresh(packet)); + writer->sampled_ns = XrTrackingFile::Now(); writer->active = 0; + assert(reader.Read(&packet)); assert(!XrTrackingFile::Fresh(packet)); + writer->active = 1; writer->eyes[0].width = 9000; + assert(reader.Read(&packet)); assert(!XrTrackingFile::Fresh(packet)); + } + munmap(writer, sizeof(DroidDeckXrTracking)); + assert(ftruncate(descriptor, 16) == 0); + { + XrTrackingFile reader(filename); + DroidDeckXrTracking packet{}; + assert(!reader.Read(&packet)); + } + close(descriptor); unlink(filename); + std::puts("XR IPC validation passed: valid, seqlock, version, stale, future, inactive, dimensions, truncated"); +}