diff --git a/app/src/main/cpp/questxr/quest_xr.cpp b/app/src/main/cpp/questxr/quest_xr.cpp index c9256ee386..4e9f52b028 100644 --- a/app/src/main/cpp/questxr/quest_xr.cpp +++ b/app/src/main/cpp/questxr/quest_xr.cpp @@ -130,6 +130,7 @@ private: EGLSurface surface_ = EGL_NO_SURFACE; std::array eyes_; GLuint framebuffer_ = 0, vertexArray_ = 0; + GLint sourceEncoding_ = 0, eyeEncoding_ = 0; std::string captureRequest() { if (frames_ % 9) return {}; @@ -145,24 +146,29 @@ private: return token; } - bool captureEye(const Eye &eye, uint32_t image, int index) { + bool captureTexture(GLuint texture, int width, int height, const char *name) { glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, eye.images[image].image, 0); - std::vector pixels((size_t)eye.width * eye.height * 4); - glReadPixels(0, 0, eye.width, eye.height, GL_RGBA, GL_UNSIGNED_BYTE, pixels.data()); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture, 0); + GLint encoding = 0; + glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_FRAMEBUFFER_ATTACHMENT_COLOR_ENCODING, &encoding); + if (!std::strcmp(name, "source")) sourceEncoding_ = encoding; + else eyeEncoding_ = encoding; + std::vector pixels((size_t)width * height * 4); + glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, pixels.data()); bool ok = glGetError() == GL_NO_ERROR; glBindFramebuffer(GL_FRAMEBUFFER, 0); if (!ok) return false; std::string file = std::string(app_->activity->externalDataPath) + - (index ? "/xr-capture-right.ppm" : "/xr-capture-left.ppm"); + "/xr-capture-" + name + ".ppm"; std::string temporary = file + ".tmp"; FILE *output = std::fopen(temporary.c_str(), "wb"); if (!output) return false; - ok = std::fprintf(output, "P6\n%d %d\n255\n", eye.width, eye.height) > 0; - std::vector row((size_t)eye.width * 3); - for (int y = eye.height - 1; y >= 0 && ok; y--) { - for (int x = 0; x < eye.width; x++) - std::memcpy(row.data() + x * 3, pixels.data() + ((size_t)y * eye.width + x) * 4, 3); + ok = std::fprintf(output, "P6\n%d %d\n255\n", width, height) > 0; + std::vector row((size_t)width * 3); + for (int y = height - 1; y >= 0 && ok; y--) { + for (int x = 0; x < width; x++) + std::memcpy(row.data() + x * 3, pixels.data() + ((size_t)y * width + x) * 4, 3); ok = std::fwrite(row.data(), 1, row.size(), output) == row.size(); } if (std::fclose(output) != 0) ok = false; @@ -190,7 +196,11 @@ private: pose.orientation.x, pose.orientation.y, pose.orientation.z, pose.orientation.w, fov.angleLeft, fov.angleRight, fov.angleUp, fov.angleDown); } - std::fputs("]}\n", output); + const char *extensions = reinterpret_cast(glGetString(GL_EXTENSIONS)); + const bool srgbControl = extensions && std::strstr(extensions, "GL_EXT_sRGB_write_control"); + std::fprintf(output, "],\"sourceImage\":true,\"sourceColorSpace\":\"linear\",\"sourceEncoding\":%d,\"eyeEncoding\":%d," + "\"srgbWriteControl\":%s,\"srgbWriteEnabled\":%s}\n", sourceEncoding_, eyeEncoding_, + srgbControl ? "true" : "false", srgbControl && glIsEnabled(0x8DB9) ? "true" : "false"); if (std::fclose(output) == 0) std::rename(temporary.c_str(), file.c_str()); } @@ -525,7 +535,8 @@ private: if (frame.header.width > static_cast(maximum) || frame.header.height > static_cast(maximum)) throw std::runtime_error("XR source exceeds GPU texture limit"); glBindTexture(GL_TEXTURE_2D, sourceTexture_); - glTexImage2D(GL_TEXTURE_2D, 0, GL_SRGB8_ALPHA8, frame.header.width, frame.header.height, 0, + // Direct-mode output is linear; the sRGB eye framebuffer performs the display encoding. + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, frame.header.width, frame.header.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, frame.pixels.data()); if (verifyPixels_) { glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); @@ -655,6 +666,8 @@ private: } const std::string capture = captureRequest(); bool captured = valid; + if (!capture.empty() && valid) + captured = captureTexture(sourceTexture_, sourceHeader_.width, sourceHeader_.height, "source"); if (valid) for (int eye = 0; eye < 2; eye++) { uint32_t image = 0; XrSwapchainImageAcquireInfo acquire{XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO}; @@ -662,7 +675,7 @@ private: XrSwapchainImageWaitInfo imageWait{XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO}; imageWait.timeout = XR_INFINITE_DURATION; check(xrWaitSwapchainImage(eyes_[eye].swapchain, &imageWait), "wait image"); render(eyes_[eye], image); - if (!capture.empty()) captured = captureEye(eyes_[eye], image, eye) && captured; + if (!capture.empty()) captured = captureTexture(eyes_[eye].images[image].image, eyes_[eye].width, eyes_[eye].height, eye ? "right" : "left") && captured; XrSwapchainImageReleaseInfo release{XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO}; check(xrReleaseSwapchainImage(eyes_[eye].swapchain, &release), "release image"); projection[eye].pose = views[eye].pose; projection[eye].fov = views[eye].fov; diff --git a/docs/agent-control.md b/docs/agent-control.md index 797fc493fc..73d66e0a41 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -126,8 +126,10 @@ tools/droiddeckctl wake `xr-frame` captures both app eye textures while SteamVR is open, saving `left.png`, `right.png` and `capture.json` with the matching source frame, projection poses, -FOV and dimensions. It works over wired or wireless authorized ADB. A unique -request ID prevents stale images from passing as a fresh capture; no valid XR +FOV and dimensions. It works over wired or wireless authorized ADB. +Newer builds also include `source.png`, the incoming stereo texture before the +Android eye draw, and sRGB attachment/write state in the metadata for color debugging. +A unique request ID prevents stale images from passing as a fresh capture; no valid XR layer produces an explicit error. Captures exclude Meta system overlays and the headset compositor's final reprojection. GPU readback runs only on request and can briefly affect frame timing. Remote files are overwritten each time. diff --git a/tools/droiddeckctl b/tools/droiddeckctl index d70bbf642c..345fd00f36 100755 --- a/tools/droiddeckctl +++ b/tools/droiddeckctl @@ -435,7 +435,7 @@ def xr_capture(adb: str, serial: str, destination: Path, timeout: float) -> dict detail = 'No valid XR projection layer is being submitted' if not metadata.get('validLayer') else 'GPU readback or image write failed' raise ControlError(detail, 'XR_CAPTURE_FAILED') images = {} - for eye in ('left', 'right'): + for eye in (('left', 'right', 'source') if metadata.get('sourceImage') else ('left', 'right')): local = Path(temporary) / (eye + '.ppm') pulled = run_adb(adb, serial, 'pull', remote + '-' + eye + '.ppm', str(local), timeout=120) if pulled.returncode: @@ -455,6 +455,7 @@ def xr_capture(adb: str, serial: str, destination: Path, timeout: float) -> dict (destination / 'capture.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8') return {'ok': True, 'command': 'xr-frame', 'deviceSerial': serial, 'capture': metadata, 'left': str((destination / 'left.png').resolve()), 'right': str((destination / 'right.png').resolve()), + **({'source': str((destination / 'source.png').resolve())} if 'source' in images else {}), 'scope': 'App eye textures before headset compositor reprojection; excludes Meta system overlays'} diff --git a/tools/tests/test_droiddeckctl_xr.py b/tools/tests/test_droiddeckctl_xr.py index f5bfc2258a..7d3efbc691 100644 --- a/tools/tests/test_droiddeckctl_xr.py +++ b/tools/tests/test_droiddeckctl_xr.py @@ -34,7 +34,7 @@ class XrCaptureTest(unittest.TestCase): with self.subTest(data=data), self.assertRaises(ctl.ControlError): ctl.xr_ppm_to_png(data) - def capture(self, root, failed=False, replaced=False, no_response=False): + def capture(self, root, failed=False, replaced=False, no_response=False, source=False): calls = [] token = None reads = 0 @@ -47,7 +47,7 @@ class XrCaptureTest(unittest.TestCase): if args[:2] == ('shell', 'cat'): reads += 1 stdout = json.dumps({'id': ('old' if reads == 1 or no_response or (replaced and reads > 2) else token), - 'ok': not failed, 'validLayer': not failed}).encode() + 'ok': not failed, 'validLayer': not failed, 'sourceImage': source}).encode() if args[0] == 'pull': Path(args[2]).write_bytes(b'P6\n1 1\n255\n\x12\x34\x56') return subprocess.CompletedProcess(args, 0, stdout, b'') @@ -72,6 +72,13 @@ class XrCaptureTest(unittest.TestCase): self.capture(root, **flags) self.assertFalse(root.exists()) + def test_source_capture_is_pulled_when_supported_by_the_headset(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / 'capture' + result, calls = self.capture(root, source=True) + self.assertEqual(result['source'], str((root / 'source.png').resolve())) + self.assertEqual(len([c for c in calls if c[0] == 'pull']), 3) + if __name__ == '__main__': unittest.main() diff --git a/tools/vr/src/driver_droiddeck.cpp b/tools/vr/src/driver_droiddeck.cpp index 77ed8eb7c9..50bdb42d84 100644 --- a/tools/vr/src/driver_droiddeck.cpp +++ b/tools/vr/src/driver_droiddeck.cpp @@ -260,7 +260,8 @@ public: if (xrWidth_) { frame_.render.magic = DROIDDECK_XR_FRAME_MAGIC; frame_.render.version = DROIDDECK_XR_FRAME_VERSION; frame_.render.width = xrWidth_; frame_.render.height = xrHeight_; - frame_.render.bytes = xrWidth_ * xrHeight_ * 4; frame_.render.color_space = 1; + frame_.render.bytes = xrWidth_ * xrHeight_ * 4; + frame_.render.color_space = DROIDDECK_XR_COLOR_LINEAR_RGBA8; } for (int e = 0; e < 2; e++) { frame_.texture[e] = eyes[e].hTexture; diff --git a/tools/vr/src/xr_frame.h b/tools/vr/src/xr_frame.h index 335948786c..3d841528f3 100644 --- a/tools/vr/src/xr_frame.h +++ b/tools/vr/src/xr_frame.h @@ -6,6 +6,7 @@ #define DROIDDECK_XR_FRAME_VERSION 1u #define DROIDDECK_XR_FRAME_SOCKET "droiddeck-xr-frames" #define DROIDDECK_XR_FRAME_MAX_BYTES (64u * 1024u * 1024u) +#define DROIDDECK_XR_COLOR_LINEAR_RGBA8 1u struct DroidDeckXrFrameHeader { uint32_t magic, version, width, height; @@ -23,7 +24,7 @@ inline bool xr_frame_valid(const DroidDeckXrFrameHeader &header) { if (header.magic != DROIDDECK_XR_FRAME_MAGIC || header.version != DROIDDECK_XR_FRAME_VERSION || !header.frame_id || header.width < 2 || (header.width & 1u) || !header.height || header.width > 8192 || header.height > 4096 || bytes > DROIDDECK_XR_FRAME_MAX_BYTES - || header.bytes != bytes || header.color_space != 1 || header.captured_ns <= 0) return false; + || header.bytes != bytes || header.color_space != DROIDDECK_XR_COLOR_LINEAR_RGBA8 || header.captured_ns <= 0) return false; for (int eye = 0; eye < 2; eye++) { for (const auto &row : header.eye_pose[eye]) for (float value : row) if (!std::isfinite(value)) return false; for (const auto &row : header.projection[eye]) for (float value : row) if (!std::isfinite(value)) return false; diff --git a/tools/vr/tests/test_xr_frames.cpp b/tools/vr/tests/test_xr_frames.cpp index 76b8cc53ce..9a2acdd744 100644 --- a/tools/vr/tests/test_xr_frames.cpp +++ b/tools/vr/tests/test_xr_frames.cpp @@ -7,7 +7,7 @@ static DroidDeckXrFrameHeader Header(uint64_t id = 1) { DroidDeckXrFrameHeader header{}; header.magic = DROIDDECK_XR_FRAME_MAGIC; header.version = DROIDDECK_XR_FRAME_VERSION; - header.width = 8; header.height = 4; header.bytes = 128; header.color_space = 1; + header.width = 8; header.height = 4; header.bytes = 128; header.color_space = DROIDDECK_XR_COLOR_LINEAR_RGBA8; header.frame_id = id; header.captured_ns = xr_frame_now(); for (int eye = 0; eye < 2; eye++) { for (int axis = 0; axis < 3; axis++) header.eye_pose[eye][axis][axis] = 1;