feat: capture submitted Quest XR eye frames

app/src/main/cpp/questxr/quest_xr.cpp

@@ -131,6 +131,69 @@ private:
     std::array<Eye, 2> eyes_;
     GLuint framebuffer_ = 0, vertexArray_ = 0;
 
+    std::string captureRequest() {
+        if (frames_ % 9) return {};
+        std::string file = std::string(app_->activity->externalDataPath) + "/xr-capture.request";
+        FILE *request = std::fopen(file.c_str(), "r");
+        if (!request) return {};
+        char token[34]{};
+        size_t size = std::fread(token, 1, sizeof(token) - 1, request);
+        std::fclose(request);
+        unlink(file.c_str());
+        if (size != 32 || std::strspn(token, "0123456789abcdef") != 32) return {};
+        unlink((std::string(app_->activity->externalDataPath) + "/xr-capture.json").c_str());
+        return token;
+    }
+
+    bool captureEye(const Eye &eye, uint32_t image, int index) {
+        glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, eye.images[image].image, 0);
+        std::vector<unsigned char> pixels((size_t)eye.width * eye.height * 4);
+        glReadPixels(0, 0, eye.width, eye.height, GL_RGBA, GL_UNSIGNED_BYTE, pixels.data());
+        bool ok = glGetError() == GL_NO_ERROR;
+        glBindFramebuffer(GL_FRAMEBUFFER, 0);
+        if (!ok) return false;
+        std::string file = std::string(app_->activity->externalDataPath) +
+            (index ? "/xr-capture-right.ppm" : "/xr-capture-left.ppm");
+        std::string temporary = file + ".tmp";
+        FILE *output = std::fopen(temporary.c_str(), "wb");
+        if (!output) return false;
+        ok = std::fprintf(output, "P6\n%d %d\n255\n", eye.width, eye.height) > 0;
+        std::vector<unsigned char> row((size_t)eye.width * 3);
+        for (int y = eye.height - 1; y >= 0 && ok; y--) {
+            for (int x = 0; x < eye.width; x++)
+                std::memcpy(row.data() + x * 3, pixels.data() + ((size_t)y * eye.width + x) * 4, 3);
+            ok = std::fwrite(row.data(), 1, row.size(), output) == row.size();
+        }
+        if (std::fclose(output) != 0) ok = false;
+        if (!ok) { unlink(temporary.c_str()); return false; }
+        return std::rename(temporary.c_str(), file.c_str()) == 0;
+    }
+
+    void captureResult(const std::string &token, bool ok, bool valid,
+                       const XrCompositionLayerProjectionView (&views)[2], XrTime displayTime) {
+        std::string file = std::string(app_->activity->externalDataPath) + "/xr-capture.json";
+        std::string temporary = file + ".tmp";
+        FILE *output = std::fopen(temporary.c_str(), "w");
+        if (!output) return;
+        std::fprintf(output, "{\"id\":\"%s\",\"ok\":%s,\"validLayer\":%s,\"frame\":%llu,"
+            "\"sourceFrameId\":%llu,\"displayTime\":%lld,\"sessionState\":%d,\"eyes\":[",
+            token.c_str(), ok ? "true" : "false", valid ? "true" : "false",
+            (unsigned long long)frames_, (unsigned long long)sourceHeader_.frame_id, (long long)displayTime, state_);
+        for (int eye = 0; eye < 2; eye++) {
+            const auto &pose = views[eye].pose;
+            const auto &fov = views[eye].fov;
+            std::fprintf(output, "%s{\"width\":%d,\"height\":%d,\"position\":[%g,%g,%g],"
+                "\"orientation\":[%g,%g,%g,%g],\"fov\":[%g,%g,%g,%g]}",
+                eye ? "," : "", eyes_[eye].width, eyes_[eye].height,
+                pose.position.x, pose.position.y, pose.position.z,
+                pose.orientation.x, pose.orientation.y, pose.orientation.z, pose.orientation.w,
+                fov.angleLeft, fov.angleRight, fov.angleUp, fov.angleDown);
+        }
+        std::fputs("]}\n", output);
+        if (std::fclose(output) == 0) std::rename(temporary.c_str(), file.c_str());
+    }
+
     XrPath path(const char *text) {
         XrPath result{};
         check(xrStringToPath(instance_, text, &result), "xrStringToPath");
@@ -590,6 +653,8 @@ private:
                 views[eye].fov = {source.fov[0], source.fov[1], source.fov[2], source.fov[3]};
             }
         }
+        const std::string capture = captureRequest();
+        bool captured = valid;
         if (valid) for (int eye = 0; eye < 2; eye++) {
             uint32_t image = 0;
             XrSwapchainImageAcquireInfo acquire{XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO};
@@ -597,6 +662,7 @@ private:
             XrSwapchainImageWaitInfo imageWait{XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO}; imageWait.timeout = XR_INFINITE_DURATION;
             check(xrWaitSwapchainImage(eyes_[eye].swapchain, &imageWait), "wait image");
             render(eyes_[eye], image);
+            if (!capture.empty()) captured = captureEye(eyes_[eye], image, eye) && captured;
             XrSwapchainImageReleaseInfo release{XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO};
             check(xrReleaseSwapchainImage(eyes_[eye].swapchain, &release), "release image");
             projection[eye].pose = views[eye].pose; projection[eye].fov = views[eye].fov;
@@ -609,6 +675,7 @@ private:
         XrFrameEndInfo end{XR_TYPE_FRAME_END_INFO}; end.displayTime = frameState.predictedDisplayTime;
         end.environmentBlendMode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; end.layerCount = valid ? 1 : 0; end.layers = layers;
         check(xrEndFrame(session_, &end), "xrEndFrame");
+        if (!capture.empty()) captureResult(capture, captured, valid, projection, frameState.predictedDisplayTime);
         frames_++; if (valid) rendered_++;
         if (frames_ % 72 == 0) {
             health();

docs/agent-control.md

@@ -117,12 +117,21 @@ tools/droiddeckctl prefs set session some_key true --type bool
 
 tools/droiddeckctl logs latest ./session-artifacts
 tools/droiddeckctl screenshot ./screen.png --display 4
+tools/droiddeckctl xr-frame ./xr-capture
 tools/droiddeckctl record ./clip.mp4 --seconds 8 --display 0
 tools/droiddeckctl pacing --seconds 30
 tools/droiddeckctl displays
 tools/droiddeckctl wake
 ```
 
+`xr-frame` captures both app eye textures while SteamVR is open, saving `left.png`,
+`right.png` and `capture.json` with the matching source frame, projection poses,
+FOV and dimensions. It works over wired or wireless authorized ADB. A unique
+request ID prevents stale images from passing as a fresh capture; no valid XR
+layer produces an explicit error. Captures exclude Meta system overlays and
+the headset compositor's final reprojection. GPU readback runs only on request
+and can briefly affect frame timing. Remote files are overwritten each time.
+
 A sleeping device suspends the session and hides the app's windows from `ui dump`; `wake` first.
 
 `pacing` reads SurfaceFlinger's timestats for the game's layer (the session's SurfaceView when the

tools/droiddeckctl

@@ -10,8 +10,12 @@ import os
 import re
 import shutil
 import subprocess
+import struct
 import sys
+import tempfile
 import time
+import uuid
+import zlib
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -382,8 +386,82 @@ def input_request(args: argparse.Namespace) -> dict:
     raise ControlError(f"Unknown input type {kind}", "INVALID_INPUT")
 
 
+def xr_ppm_to_png(data: bytes) -> bytes:
+    header = re.match(rb"P6\n([0-9]+) ([0-9]+)\n255\n", data)
+    if not header:
+        raise ControlError("Invalid XR image header", "XR_CAPTURE_FAILED")
+    width, height = map(int, header.groups())
+    pixels = data[header.end():]
+    if not (0 < width <= 16384 and 0 < height <= 16384) or len(pixels) != width * height * 3:
+        raise ControlError("Invalid XR image dimensions or incomplete pixels", "XR_CAPTURE_FAILED")
+    def chunk(kind: bytes, value: bytes) -> bytes:
+        return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value))
+    stride = width * 3
+    rows = b''.join(b'\0' + pixels[start:start + stride] for start in range(0, len(pixels), stride))
+    return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) +
+            chunk(b'IDAT', zlib.compress(rows)) + chunk(b'IEND', b''))
+
+
+def xr_capture(adb: str, serial: str, destination: Path, timeout: float) -> dict:
+    if not 0 < timeout <= 120:
+        raise ControlError("XR capture timeout must be between 0 and 120 seconds", "INVALID_TIMEOUT")
+    remote = f'/sdcard/Android/data/{PACKAGE}/files/xr-capture'
+    token = uuid.uuid4().hex
+    with tempfile.TemporaryDirectory() as temporary:
+        request = Path(temporary) / 'request'
+        request.write_text(token, encoding='ascii')
+        pushed = run_adb(adb, serial, 'push', str(request), remote + '.request.tmp')
+        if pushed.returncode:
+            raise ControlError(pushed.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED')
+        moved = run_adb(adb, serial, 'shell', 'mv', remote + '.request.tmp', remote + '.request')
+        if moved.returncode:
+            raise ControlError(moved.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED')
+        deadline = time.monotonic() + timeout
+        metadata = None
+        while time.monotonic() < deadline:
+            result = run_adb(adb, serial, 'shell', 'cat', remote + '.json')
+            try:
+                answer = json.loads(result.stdout)
+                if answer.get('id') == token:
+                    metadata = answer
+                    break
+            except (ValueError, AttributeError):
+                pass
+            time.sleep(0.2)
+        if metadata is None:
+            run_adb(adb, serial, 'shell', 'rm', '-f', remote + '.request')
+            raise ControlError('No fresh XR capture. Enter SteamVR using a build with XR capture support.', 'TIMEOUT')
+        if not metadata.get('ok'):
+            detail = 'No valid XR projection layer is being submitted' if not metadata.get('validLayer') else 'GPU readback or image write failed'
+            raise ControlError(detail, 'XR_CAPTURE_FAILED')
+        images = {}
+        for eye in ('left', 'right'):
+            local = Path(temporary) / (eye + '.ppm')
+            pulled = run_adb(adb, serial, 'pull', remote + '-' + eye + '.ppm', str(local), timeout=120)
+            if pulled.returncode:
+                raise ControlError(pulled.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED')
+            images[eye] = xr_ppm_to_png(local.read_bytes())
+        # Detect overlapping captures before publishing images with another request's metadata.
+        check = run_adb(adb, serial, 'shell', 'cat', remote + '.json')
+        try:
+            unchanged = json.loads(check.stdout).get('id') == token
+        except (ValueError, AttributeError):
+            unchanged = False
+        if not unchanged:
+            raise ControlError('XR capture was replaced by another request; retry', 'XR_CAPTURE_FAILED')
+    destination.mkdir(parents=True, exist_ok=True)
+    for eye, image in images.items():
+        (destination / (eye + '.png')).write_bytes(image)
+    (destination / 'capture.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
+    return {'ok': True, 'command': 'xr-frame', 'deviceSerial': serial, 'capture': metadata,
+            'left': str((destination / 'left.png').resolve()), 'right': str((destination / 'right.png').resolve()),
+            'scope': 'App eye textures before headset compositor reprojection; excludes Meta system overlays'}
+
+
 def execute(args: argparse.Namespace, adb: str, serial: str) -> dict:
     command = args.command
+    if command == 'xr-frame':
+        return xr_capture(adb, serial, Path(args.path), args.timeout)
 
     if command == "state":
         return add_serial(get_state(adb, serial), serial)
@@ -717,6 +795,10 @@ def build_parser() -> argparse.ArgumentParser:
     shot.add_argument("path")
     shot.add_argument("--display", type=int, help="logical display id (see displays); default the default display")
 
+    xr = commands.add_parser('xr-frame', help='capture both submitted XR eyes and pose metadata')
+    xr.add_argument('path', help='output directory for left.png, right.png and capture.json')
+    xr.add_argument('--timeout', type=float, default=15)
+
     rec = commands.add_parser("record", help="record the screen to an mp4")
     rec.add_argument("path")
     rec.add_argument("--seconds", type=float, default=10)
@@ -738,7 +820,7 @@ EXIT_SESSION = {
 }
 EXIT_TIMEOUT = {"TIMEOUT", "ARTIFACT_TIMEOUT"}
 EXIT_FILE = {"NO_SESSION_LOGS", "LOG_PULL_FAILED", "SCREENSHOT_FAILED", "FILE_IO", "RECORD_FAILED", "PUSH_FAILED",
-             "PACING_FAILED"}
+             "PACING_FAILED", "XR_CAPTURE_FAILED"}
 EXIT_DENIED = {"AGENT_COMMANDS_DISABLED", "PROTECTED_PREFS", "PROTECTED_CONTROL"}
 
 

tools/tests/test_droiddeckctl_xr.py

@@ -0,0 +1,77 @@
+import importlib.machinery
+import importlib.util
+import json
+from pathlib import Path
+import struct
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+import zlib
+
+TOOLS = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(TOOLS))
+loader = importlib.machinery.SourceFileLoader('ctl_xr', str(TOOLS / 'droiddeckctl'))
+spec = importlib.util.spec_from_loader(loader.name, loader)
+ctl = importlib.util.module_from_spec(spec)
+loader.exec_module(ctl)
+
+
+class XrCaptureTest(unittest.TestCase):
+    def test_png_keeps_rgb_rows_and_dimensions(self):
+        pixels = bytes([255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255])
+        png = ctl.xr_ppm_to_png(b'P6\n2 2\n255\n' + pixels)
+        self.assertEqual(png[:8], b'\x89PNG\r\n\x1a\n')
+        self.assertEqual(struct.unpack('>II', png[16:24]), (2, 2))
+        position = png.index(b'IDAT')
+        size = struct.unpack('>I', png[position - 4:position])[0]
+        self.assertEqual(zlib.decompress(png[position + 4:position + 4 + size]),
+                         b'\0' + pixels[:6] + b'\0' + pixels[6:])
+
+    def test_incomplete_or_invalid_images_are_rejected(self):
+        for data in (b'P6\n2 2\n255\nabc', b'P6\n0 1\n255\n', b'P6\n999999 1\n255\n', b'junk'):
+            with self.subTest(data=data), self.assertRaises(ctl.ControlError):
+                ctl.xr_ppm_to_png(data)
+
+    def capture(self, root, failed=False, replaced=False, no_response=False):
+        calls = []
+        token = None
+        reads = 0
+        def adb(_adb, _serial, *args, **kwargs):
+            nonlocal token, reads
+            calls.append(args)
+            stdout = b''
+            if args[0] == 'push':
+                token = Path(args[1]).read_text()
+            if args[:2] == ('shell', 'cat'):
+                reads += 1
+                stdout = json.dumps({'id': ('old' if reads == 1 or no_response or (replaced and reads > 2) else token),
+                                     'ok': not failed, 'validLayer': not failed}).encode()
+            if args[0] == 'pull':
+                Path(args[2]).write_bytes(b'P6\n1 1\n255\n\x12\x34\x56')
+            return subprocess.CompletedProcess(args, 0, stdout, b'')
+        with patch.object(ctl, 'run_adb', side_effect=adb), patch.object(ctl.time, 'sleep'):
+            result = ctl.xr_capture('adb', 'quest', root, 0.01 if no_response else 2)
+        return result, calls
+
+    def test_stale_response_is_ignored_and_matching_pair_is_published(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory) / 'capture'
+            result, calls = self.capture(root)
+            self.assertTrue(result['ok'])
+            self.assertEqual((root / 'left.png').read_bytes(), (root / 'right.png').read_bytes())
+            self.assertEqual(len([c for c in calls if c[:2] == ('shell', 'cat')]), 3)
+            self.assertTrue((root / 'capture.json').is_file())
+
+    def test_invalid_layer_replaced_capture_and_timeout_publish_no_images(self):
+        for flags in ({'failed': True}, {'replaced': True}, {'no_response': True}):
+            with self.subTest(flags=flags), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory) / 'capture'
+                with self.assertRaises(ctl.ControlError):
+                    self.capture(root, **flags)
+                self.assertFalse(root.exists())
+
+
+if __name__ == '__main__':
+    unittest.main()