From 52d351908f8aef1b1c522d064fd1477ef77d26c9 Mon Sep 17 00:00:00 2001 From: XxJSONDeruloxX Date: Thu, 15 Oct 2026 16:08:10 -0400 Subject: [PATCH] feat: capture submitted Quest XR eye frames --- app/src/main/cpp/questxr/quest_xr.cpp | 67 +++++++++++++++++++++ docs/agent-control.md | 9 +++ tools/droiddeckctl | 84 ++++++++++++++++++++++++++- tools/tests/test_droiddeckctl_xr.py | 77 ++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 tools/tests/test_droiddeckctl_xr.py diff --git a/app/src/main/cpp/questxr/quest_xr.cpp b/app/src/main/cpp/questxr/quest_xr.cpp index 33f07a58c..c9256ee38 100644 --- a/app/src/main/cpp/questxr/quest_xr.cpp +++ b/app/src/main/cpp/questxr/quest_xr.cpp @@ -131,6 +131,69 @@ private: std::array eyes_; GLuint framebuffer_ = 0, vertexArray_ = 0; + std::string captureRequest() { + if (frames_ % 9) return {}; + std::string file = std::string(app_->activity->externalDataPath) + "/xr-capture.request"; + FILE *request = std::fopen(file.c_str(), "r"); + if (!request) return {}; + char token[34]{}; + size_t size = std::fread(token, 1, sizeof(token) - 1, request); + std::fclose(request); + unlink(file.c_str()); + if (size != 32 || std::strspn(token, "0123456789abcdef") != 32) return {}; + unlink((std::string(app_->activity->externalDataPath) + "/xr-capture.json").c_str()); + return token; + } + + bool captureEye(const Eye &eye, uint32_t image, int index) { + glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, eye.images[image].image, 0); + std::vector pixels((size_t)eye.width * eye.height * 4); + glReadPixels(0, 0, eye.width, eye.height, GL_RGBA, GL_UNSIGNED_BYTE, pixels.data()); + bool ok = glGetError() == GL_NO_ERROR; + glBindFramebuffer(GL_FRAMEBUFFER, 0); + if (!ok) return false; + std::string file = std::string(app_->activity->externalDataPath) + + (index ? "/xr-capture-right.ppm" : "/xr-capture-left.ppm"); + std::string temporary = file + ".tmp"; + FILE *output = std::fopen(temporary.c_str(), "wb"); + if (!output) return false; + ok = std::fprintf(output, "P6\n%d %d\n255\n", eye.width, eye.height) > 0; + std::vector row((size_t)eye.width * 3); + for (int y = eye.height - 1; y >= 0 && ok; y--) { + for (int x = 0; x < eye.width; x++) + std::memcpy(row.data() + x * 3, pixels.data() + ((size_t)y * eye.width + x) * 4, 3); + ok = std::fwrite(row.data(), 1, row.size(), output) == row.size(); + } + if (std::fclose(output) != 0) ok = false; + if (!ok) { unlink(temporary.c_str()); return false; } + return std::rename(temporary.c_str(), file.c_str()) == 0; + } + + void captureResult(const std::string &token, bool ok, bool valid, + const XrCompositionLayerProjectionView (&views)[2], XrTime displayTime) { + std::string file = std::string(app_->activity->externalDataPath) + "/xr-capture.json"; + std::string temporary = file + ".tmp"; + FILE *output = std::fopen(temporary.c_str(), "w"); + if (!output) return; + std::fprintf(output, "{\"id\":\"%s\",\"ok\":%s,\"validLayer\":%s,\"frame\":%llu," + "\"sourceFrameId\":%llu,\"displayTime\":%lld,\"sessionState\":%d,\"eyes\":[", + token.c_str(), ok ? "true" : "false", valid ? "true" : "false", + (unsigned long long)frames_, (unsigned long long)sourceHeader_.frame_id, (long long)displayTime, state_); + for (int eye = 0; eye < 2; eye++) { + const auto &pose = views[eye].pose; + const auto &fov = views[eye].fov; + std::fprintf(output, "%s{\"width\":%d,\"height\":%d,\"position\":[%g,%g,%g]," + "\"orientation\":[%g,%g,%g,%g],\"fov\":[%g,%g,%g,%g]}", + eye ? "," : "", eyes_[eye].width, eyes_[eye].height, + pose.position.x, pose.position.y, pose.position.z, + pose.orientation.x, pose.orientation.y, pose.orientation.z, pose.orientation.w, + fov.angleLeft, fov.angleRight, fov.angleUp, fov.angleDown); + } + std::fputs("]}\n", output); + if (std::fclose(output) == 0) std::rename(temporary.c_str(), file.c_str()); + } + XrPath path(const char *text) { XrPath result{}; check(xrStringToPath(instance_, text, &result), "xrStringToPath"); @@ -590,6 +653,8 @@ private: views[eye].fov = {source.fov[0], source.fov[1], source.fov[2], source.fov[3]}; } } + const std::string capture = captureRequest(); + bool captured = valid; if (valid) for (int eye = 0; eye < 2; eye++) { uint32_t image = 0; XrSwapchainImageAcquireInfo acquire{XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO}; @@ -597,6 +662,7 @@ private: XrSwapchainImageWaitInfo imageWait{XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO}; imageWait.timeout = XR_INFINITE_DURATION; check(xrWaitSwapchainImage(eyes_[eye].swapchain, &imageWait), "wait image"); render(eyes_[eye], image); + if (!capture.empty()) captured = captureEye(eyes_[eye], image, eye) && captured; XrSwapchainImageReleaseInfo release{XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO}; check(xrReleaseSwapchainImage(eyes_[eye].swapchain, &release), "release image"); projection[eye].pose = views[eye].pose; projection[eye].fov = views[eye].fov; @@ -609,6 +675,7 @@ private: XrFrameEndInfo end{XR_TYPE_FRAME_END_INFO}; end.displayTime = frameState.predictedDisplayTime; end.environmentBlendMode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; end.layerCount = valid ? 1 : 0; end.layers = layers; check(xrEndFrame(session_, &end), "xrEndFrame"); + if (!capture.empty()) captureResult(capture, captured, valid, projection, frameState.predictedDisplayTime); frames_++; if (valid) rendered_++; if (frames_ % 72 == 0) { health(); diff --git a/docs/agent-control.md b/docs/agent-control.md index 3b72bc867..797fc493f 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -117,12 +117,21 @@ tools/droiddeckctl prefs set session some_key true --type bool tools/droiddeckctl logs latest ./session-artifacts tools/droiddeckctl screenshot ./screen.png --display 4 +tools/droiddeckctl xr-frame ./xr-capture tools/droiddeckctl record ./clip.mp4 --seconds 8 --display 0 tools/droiddeckctl pacing --seconds 30 tools/droiddeckctl displays tools/droiddeckctl wake ``` +`xr-frame` captures both app eye textures while SteamVR is open, saving `left.png`, +`right.png` and `capture.json` with the matching source frame, projection poses, +FOV and dimensions. It works over wired or wireless authorized ADB. A unique +request ID prevents stale images from passing as a fresh capture; no valid XR +layer produces an explicit error. Captures exclude Meta system overlays and +the headset compositor's final reprojection. GPU readback runs only on request +and can briefly affect frame timing. Remote files are overwritten each time. + A sleeping device suspends the session and hides the app's windows from `ui dump`; `wake` first. `pacing` reads SurfaceFlinger's timestats for the game's layer (the session's SurfaceView when the diff --git a/tools/droiddeckctl b/tools/droiddeckctl index d88327a59..d70bbf642 100755 --- a/tools/droiddeckctl +++ b/tools/droiddeckctl @@ -10,8 +10,12 @@ import os import re import shutil import subprocess +import struct import sys +import tempfile import time +import uuid +import zlib from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -382,8 +386,82 @@ def input_request(args: argparse.Namespace) -> dict: raise ControlError(f"Unknown input type {kind}", "INVALID_INPUT") +def xr_ppm_to_png(data: bytes) -> bytes: + header = re.match(rb"P6\n([0-9]+) ([0-9]+)\n255\n", data) + if not header: + raise ControlError("Invalid XR image header", "XR_CAPTURE_FAILED") + width, height = map(int, header.groups()) + pixels = data[header.end():] + if not (0 < width <= 16384 and 0 < height <= 16384) or len(pixels) != width * height * 3: + raise ControlError("Invalid XR image dimensions or incomplete pixels", "XR_CAPTURE_FAILED") + def chunk(kind: bytes, value: bytes) -> bytes: + return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value)) + stride = width * 3 + rows = b''.join(b'\0' + pixels[start:start + stride] for start in range(0, len(pixels), stride)) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(rows)) + chunk(b'IEND', b'')) + + +def xr_capture(adb: str, serial: str, destination: Path, timeout: float) -> dict: + if not 0 < timeout <= 120: + raise ControlError("XR capture timeout must be between 0 and 120 seconds", "INVALID_TIMEOUT") + remote = f'/sdcard/Android/data/{PACKAGE}/files/xr-capture' + token = uuid.uuid4().hex + with tempfile.TemporaryDirectory() as temporary: + request = Path(temporary) / 'request' + request.write_text(token, encoding='ascii') + pushed = run_adb(adb, serial, 'push', str(request), remote + '.request.tmp') + if pushed.returncode: + raise ControlError(pushed.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED') + moved = run_adb(adb, serial, 'shell', 'mv', remote + '.request.tmp', remote + '.request') + if moved.returncode: + raise ControlError(moved.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED') + deadline = time.monotonic() + timeout + metadata = None + while time.monotonic() < deadline: + result = run_adb(adb, serial, 'shell', 'cat', remote + '.json') + try: + answer = json.loads(result.stdout) + if answer.get('id') == token: + metadata = answer + break + except (ValueError, AttributeError): + pass + time.sleep(0.2) + if metadata is None: + run_adb(adb, serial, 'shell', 'rm', '-f', remote + '.request') + raise ControlError('No fresh XR capture. Enter SteamVR using a build with XR capture support.', 'TIMEOUT') + if not metadata.get('ok'): + detail = 'No valid XR projection layer is being submitted' if not metadata.get('validLayer') else 'GPU readback or image write failed' + raise ControlError(detail, 'XR_CAPTURE_FAILED') + images = {} + for eye in ('left', 'right'): + local = Path(temporary) / (eye + '.ppm') + pulled = run_adb(adb, serial, 'pull', remote + '-' + eye + '.ppm', str(local), timeout=120) + if pulled.returncode: + raise ControlError(pulled.stderr.decode(errors='replace'), 'XR_CAPTURE_FAILED') + images[eye] = xr_ppm_to_png(local.read_bytes()) + # Detect overlapping captures before publishing images with another request's metadata. + check = run_adb(adb, serial, 'shell', 'cat', remote + '.json') + try: + unchanged = json.loads(check.stdout).get('id') == token + except (ValueError, AttributeError): + unchanged = False + if not unchanged: + raise ControlError('XR capture was replaced by another request; retry', 'XR_CAPTURE_FAILED') + destination.mkdir(parents=True, exist_ok=True) + for eye, image in images.items(): + (destination / (eye + '.png')).write_bytes(image) + (destination / 'capture.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8') + return {'ok': True, 'command': 'xr-frame', 'deviceSerial': serial, 'capture': metadata, + 'left': str((destination / 'left.png').resolve()), 'right': str((destination / 'right.png').resolve()), + 'scope': 'App eye textures before headset compositor reprojection; excludes Meta system overlays'} + + def execute(args: argparse.Namespace, adb: str, serial: str) -> dict: command = args.command + if command == 'xr-frame': + return xr_capture(adb, serial, Path(args.path), args.timeout) if command == "state": return add_serial(get_state(adb, serial), serial) @@ -717,6 +795,10 @@ def build_parser() -> argparse.ArgumentParser: shot.add_argument("path") shot.add_argument("--display", type=int, help="logical display id (see displays); default the default display") + xr = commands.add_parser('xr-frame', help='capture both submitted XR eyes and pose metadata') + xr.add_argument('path', help='output directory for left.png, right.png and capture.json') + xr.add_argument('--timeout', type=float, default=15) + rec = commands.add_parser("record", help="record the screen to an mp4") rec.add_argument("path") rec.add_argument("--seconds", type=float, default=10) @@ -738,7 +820,7 @@ EXIT_SESSION = { } EXIT_TIMEOUT = {"TIMEOUT", "ARTIFACT_TIMEOUT"} EXIT_FILE = {"NO_SESSION_LOGS", "LOG_PULL_FAILED", "SCREENSHOT_FAILED", "FILE_IO", "RECORD_FAILED", "PUSH_FAILED", - "PACING_FAILED"} + "PACING_FAILED", "XR_CAPTURE_FAILED"} EXIT_DENIED = {"AGENT_COMMANDS_DISABLED", "PROTECTED_PREFS", "PROTECTED_CONTROL"} diff --git a/tools/tests/test_droiddeckctl_xr.py b/tools/tests/test_droiddeckctl_xr.py new file mode 100644 index 000000000..f5bfc2258 --- /dev/null +++ b/tools/tests/test_droiddeckctl_xr.py @@ -0,0 +1,77 @@ +import importlib.machinery +import importlib.util +import json +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch +import zlib + +TOOLS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(TOOLS)) +loader = importlib.machinery.SourceFileLoader('ctl_xr', str(TOOLS / 'droiddeckctl')) +spec = importlib.util.spec_from_loader(loader.name, loader) +ctl = importlib.util.module_from_spec(spec) +loader.exec_module(ctl) + + +class XrCaptureTest(unittest.TestCase): + def test_png_keeps_rgb_rows_and_dimensions(self): + pixels = bytes([255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255]) + png = ctl.xr_ppm_to_png(b'P6\n2 2\n255\n' + pixels) + self.assertEqual(png[:8], b'\x89PNG\r\n\x1a\n') + self.assertEqual(struct.unpack('>II', png[16:24]), (2, 2)) + position = png.index(b'IDAT') + size = struct.unpack('>I', png[position - 4:position])[0] + self.assertEqual(zlib.decompress(png[position + 4:position + 4 + size]), + b'\0' + pixels[:6] + b'\0' + pixels[6:]) + + def test_incomplete_or_invalid_images_are_rejected(self): + for data in (b'P6\n2 2\n255\nabc', b'P6\n0 1\n255\n', b'P6\n999999 1\n255\n', b'junk'): + with self.subTest(data=data), self.assertRaises(ctl.ControlError): + ctl.xr_ppm_to_png(data) + + def capture(self, root, failed=False, replaced=False, no_response=False): + calls = [] + token = None + reads = 0 + def adb(_adb, _serial, *args, **kwargs): + nonlocal token, reads + calls.append(args) + stdout = b'' + if args[0] == 'push': + token = Path(args[1]).read_text() + if args[:2] == ('shell', 'cat'): + reads += 1 + stdout = json.dumps({'id': ('old' if reads == 1 or no_response or (replaced and reads > 2) else token), + 'ok': not failed, 'validLayer': not failed}).encode() + if args[0] == 'pull': + Path(args[2]).write_bytes(b'P6\n1 1\n255\n\x12\x34\x56') + return subprocess.CompletedProcess(args, 0, stdout, b'') + with patch.object(ctl, 'run_adb', side_effect=adb), patch.object(ctl.time, 'sleep'): + result = ctl.xr_capture('adb', 'quest', root, 0.01 if no_response else 2) + return result, calls + + def test_stale_response_is_ignored_and_matching_pair_is_published(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / 'capture' + result, calls = self.capture(root) + self.assertTrue(result['ok']) + self.assertEqual((root / 'left.png').read_bytes(), (root / 'right.png').read_bytes()) + self.assertEqual(len([c for c in calls if c[:2] == ('shell', 'cat')]), 3) + self.assertTrue((root / 'capture.json').is_file()) + + def test_invalid_layer_replaced_capture_and_timeout_publish_no_images(self): + for flags in ({'failed': True}, {'replaced': True}, {'no_response': True}): + with self.subTest(flags=flags), tempfile.TemporaryDirectory() as directory: + root = Path(directory) / 'capture' + with self.assertRaises(ctl.ControlError): + self.capture(root, **flags) + self.assertFalse(root.exists()) + + +if __name__ == '__main__': + unittest.main()