From dbf114a0f23b2b341712f995c126056b769ca2ef Mon Sep 17 00:00:00 2001 From: Moroni Granja Date: Sun, 1 Nov 2026 01:26:14 -0300 Subject: [PATCH] fix: draw the guest cursor into the glasses picture (#1925) --- app/src/main/cpp/waylandcomp/src/cardboard.c | 7 + app/src/main/cpp/waylandcomp/src/cardboard.h | 1 + app/src/main/cpp/waylandcomp/src/compositor.c | 190 +++++++++++++++++- .../cpp/waylandcomp/src/waylandcomp_jni.c | 8 + .../com/droiddeck/launcher/SessionActivity.kt | 51 ++++- .../launcher/wayland/WaylandCompositor.java | 2 + docs/smart-glasses-support.md | 4 + tools/tests/cursor_composite_harness.c | 64 ++++++ tools/tests/test_cursor_composite.py | 54 +++++ 9 files changed, 369 insertions(+), 12 deletions(-) create mode 100644 tools/tests/cursor_composite_harness.c create mode 100644 tools/tests/test_cursor_composite.py diff --git a/app/src/main/cpp/waylandcomp/src/cardboard.c b/app/src/main/cpp/waylandcomp/src/cardboard.c index fc09d2f030..03c4343166 100644 --- a/app/src/main/cpp/waylandcomp/src/cardboard.c +++ b/app/src/main/cpp/waylandcomp/src/cardboard.c @@ -186,6 +186,13 @@ int vkp_cardboard_active(void) { return on; } +int vkp_cardboard_source(void) { + pthread_mutex_lock(&g_lock); + const int source = g_set.enabled ? g_set.source : CARDBOARD_SOURCE_SCREEN; + pthread_mutex_unlock(&g_lock); + return source; +} + int vkp_cardboard_pose_changed(void) { if (!g_changed) return 0; g_changed = 0; diff --git a/app/src/main/cpp/waylandcomp/src/cardboard.h b/app/src/main/cpp/waylandcomp/src/cardboard.h index 88d5ea3ff7..2950faec2f 100644 --- a/app/src/main/cpp/waylandcomp/src/cardboard.h +++ b/app/src/main/cpp/waylandcomp/src/cardboard.h @@ -27,6 +27,7 @@ void vkp_cardboard_set_render_height(int height); void vkp_cardboard_bind_device(VkDevice dev, const VkPhysicalDeviceMemoryProperties *memprops); int vkp_cardboard_active(void); +int vkp_cardboard_source(void); int vkp_cardboard_pose_changed(void); void vkp_cardboard_begin_frame(void); VkImage vkp_cardboard_run(VkCommandBuffer cmd, VkImage src, int src_w, int src_h, int out_w, int out_h); diff --git a/app/src/main/cpp/waylandcomp/src/compositor.c b/app/src/main/cpp/waylandcomp/src/compositor.c index 6a9fdacd26..68039b4443 100644 --- a/app/src/main/cpp/waylandcomp/src/compositor.c +++ b/app/src/main/cpp/waylandcomp/src/compositor.c @@ -446,6 +446,17 @@ static int g_cursor_w, g_cursor_h; static int g_cursor_hidden = 1; static int g_cursor_serial; +/* The pointer image as a compositor-owned texture, for outputs that cannot show + * the phone's Compose overlay (the glasses surface owns the picture there). */ +#define CURSOR_MAX_DRAWS 2 /* one per eye when the picture is side-by-side */ +static struct vkp_image *g_cursor_img; +static int g_cursor_img_w, g_cursor_img_h, g_cursor_img_serial = -1; +static atomic_int g_cursor_composite; +/* Last request from the app, used only to skip repeated calls. The applied flag below is + * written by the caller and read by the render path; message 22 only wakes the event loop, so + * a queued message can never put back an older value. */ +static atomic_int g_cursor_composite_want; + static void cursor_publish_shown(void) { pthread_mutex_lock(&g_cursor_lock); if (g_cursor_w > 0) { g_cursor_hidden = 0; g_cursor_serial++; } @@ -1736,6 +1747,8 @@ static void bind_output(struct wl_client *c, void *data, uint32_t ver, uint32_t struct draw_list { struct vkp_draw *d; int n, cap; }; +static int add_cursor(struct draw_list *dl, int scene_w); + static struct vkp_image *surface_image(struct surface *s) { if (!s->has_content) return NULL; if (s->shm_img) return s->shm_img; @@ -1909,7 +1922,9 @@ static int hdr_plan(const struct draw_list *dl, int w, int h, int fx_on, int fra } if (un && g_zero_copy) { hs->color = droiddeck_color_of(un->resource); hs->who = un; return 3; } hs->hf.tonemap = !droiddeck_color_output(); - if (dl->n <= 0 || !(hs->is_hdr = calloc((size_t)dl->n, 1))) return 0; + /* The pointer draws are appended after this plan: reserve their slots (zeroed, + * so they count as SDR content above the game) instead of reading past the end. */ + if (dl->n <= 0 || !(hs->is_hdr = calloc((size_t)dl->n + CURSOR_MAX_DRAWS, 1))) return 0; for (int i = 0; i < dl->n; i++) { struct surface *s = surface_for_image(dl->d[i].img); if (!is_hdr_surface(s)) continue; @@ -2185,6 +2200,36 @@ static void render_scene(void) { hdr_note_precedence(hdr_route == 3 ? hs.who : NULL, fx_on, framegen); } if (hdr_route == 3) { fx_on = 0; framegen = 0; } + /* A game the compositor cannot import (an AHardwareBuffer-only surface, or HDR frames it + * has no texture for) is shown on its own display layer and is not part of the scene, so + * composing would leave the game out of the picture: the pointer stays out instead, the + * same trade-off as HDR route 3. Both are decided before the pointer is appended, because + * the appended draws change the draw list they are matched against. */ + struct surface *ahb_only = ahb_layer_only_candidate(w, h); + struct surface *layer_only = ahb_only ? ahb_only : hdr_unimported_fullscreen(&dl, w, h); + const int cursor_on = hdr_route == 3 || layer_only ? 0 : add_cursor(&dl, w); + static int cursor_layer_said; + if (layer_only && atomic_load(&g_cursor_composite) && !cursor_layer_said) { + cursor_layer_said = 1; + droiddeck_log("cursor", "the pointer stays out of the picture: the game's frames cannot be imported, " + "so the game is shown on its own display layer"); + } else if (!layer_only) { + cursor_layer_said = 0; + } + /* With cardboard the scene is composed anyway (the pass needs the whole picture), so the + * pointer reaches the output without switching routes: the HDR composition path would drop + * the cardboard pass, which is 8-bit only. */ + static int cursor_hdr_said; + if (cursor_on && hdr_route == 0 && hs.n_hdr > 0 && !cardboard) { + hdr_route = 2; + if (!cursor_hdr_said) { + cursor_hdr_said = 1; + droiddeck_log("cursor", "the pointer needs a composed scene: the HDR picture goes through the screen " + "swapchain instead of the game's own display layer"); + } + } else { + cursor_hdr_said = 0; + } if (hdr_route == 1) { static int fell_back; vkp_apply_window_request(); @@ -2210,21 +2255,21 @@ static void render_scene(void) { else if (rendered && how == 2) droiddeck_color_frame_shown(hs.color, DROIDDECK_HDR_TONEMAPPED, 0); else if (rendered) hdr_copy = "the HDR composition pass is unavailable on this driver"; } - if (hdr_route == 0 && vkp_quest_layer_presentation() && !framegen && !cardboard && - !ahb_layer_only_candidate(w, h)) { + if (hdr_route == 0 && vkp_quest_layer_presentation() && !framegen && !cardboard && !cursor_on && + !ahb_only) { /* The complete SDR scene stays on one display layer. Do not hide/re-show GAME * on each desktop frame just because it is not a single fullscreen window. */ rendered = vkp_render(w, h, dl.d, dl.n) == 0; } else if (hdr_route == 0 || hdr_route == 3) { const int pass_on = fx_on || framegen; - int li = (g_zero_copy && !framegen && !cardboard) ? layer_candidate(&dl, w, h) : -1; + int li = (g_zero_copy && !framegen && !cardboard && !cursor_on) ? layer_candidate(&dl, w, h) : -1; int over = li >= 0 ? dl.n - 1 - li : 0; const int fx_blocks = (li >= 0 && fx_on && over > 0); if (fx_blocks) { li = -1; over = 0; } const int ov_blocks = (li >= 0 && over > 0 && !fx_blocks && !sc_layer_overlay_affordable()); if (ov_blocks) { li = -1; over = 0; } const int blocked = framegen ? 1 : (fx_blocks ? 2 : (ov_blocks ? 3 : 0)); - if (g_zero_copy && blocked != g_zero_copy_paused) { + if (g_zero_copy && !cursor_on && blocked != g_zero_copy_paused) { g_zero_copy_paused = blocked; if (blocked == 1) droiddeck_log("framegen", "zero-copy paused: frame generation needs the compositor pass"); @@ -2237,9 +2282,9 @@ static void render_scene(void) { droiddeck_log("effects", "zero-copy resumed: the game is back on its own display layer"); } struct surface *ls = li >= 0 ? surface_for_image(dl.d[li].img) : NULL; - if (li < 0) ls = ahb_layer_only_candidate(w, h); - if (li < 0 && !ls) ls = hdr_unimported_fullscreen(&dl, w, h); + if (li < 0) ls = layer_only; if (cardboard) ls = NULL; + if (cursor_on) ls = NULL; if (ls && pass_on && li < 0 && !g_zero_copy_fx_skip_said) { g_zero_copy_fx_skip_said = 1; droiddeck_log(framegen ? "framegen" : "effects", @@ -2409,6 +2454,7 @@ static void pointer_set_cursor(struct wl_client *c, struct wl_resource *r, uint3 if (g_cursor_surface) g_cursor_surface->role = ROLE_NONE; g_cursor_surface = NULL; cursor_publish_hidden(); + if (atomic_load(&g_cursor_composite)) schedule_render(); return; } struct surface *s = wl_resource_get_user_data(surface); @@ -2416,8 +2462,10 @@ static void pointer_set_cursor(struct wl_client *c, struct wl_resource *r, uint3 if (s->role != ROLE_NONE && s->role != ROLE_CURSOR) return; s->role = ROLE_CURSOR; g_cursor_surface = s; + pthread_mutex_lock(&g_cursor_lock); g_cursor_hx = hx; g_cursor_hy = hy; + pthread_mutex_unlock(&g_cursor_lock); struct wl_resource *buffer = surface_take_idle(s); if (buffer) { cursor_publish_buffer(s, buffer); @@ -2428,6 +2476,7 @@ static void pointer_set_cursor(struct wl_client *c, struct wl_resource *r, uint3 cursor_publish_pixels((const uint8_t *)s->idle_px, s->idle_w, s->idle_h, (size_t)s->idle_w * 4, hx, hy); g_cursor_shown = s; } + if (atomic_load(&g_cursor_composite)) schedule_render(); } static void pointer_release(struct wl_client *c, struct wl_resource *r) { wl_resource_destroy(r); } static const struct wl_pointer_interface pointer_impl = { @@ -2609,6 +2658,105 @@ static int g_ptr_known; static double g_raw_x, g_raw_y; static int g_raw_valid; +static int pointer_locked_for_cursor(void) { + return g_active_constraint && g_active_constraint->is_lock; +} + +static int cursor_scene_positions(int scene_w, int source, int ptr_x, int hx, int out[2]) { + out[0] = ptr_x - hx; out[1] = -1; + if (source != CARDBOARD_SOURCE_SBS || scene_w <= 1) return 1; + /* The pointer decides which eye it is in; the hotspot only shifts the image drawn there, + * so a hotspot that crosses the seam cannot put the second copy in the wrong eye. */ + const int half = scene_w / 2; + out[1] = (ptr_x < half ? ptr_x + half : ptr_x - half) - hx; + return 2; +} + +/* Crops one pointer copy to the eye it belongs to (eye < 0 crops nothing). Returns 0 when no + * part of the copy falls inside that eye, so the caller skips it. */ +static int cursor_clip_to_eye(int scene_w, int eye, int x0, int w, int *dx, int *sx, int *dw) { + int left = x0, right = x0 + w; + if (eye >= 0 && scene_w > 1) { + const int half = scene_w / 2; + const int band = eye ? half : 0; + if (left < band) left = band; + if (right > band + half) right = band + half; + } + if (right <= left) return 0; + *dx = left; + *sx = left - x0; + *dw = right - left; + return 1; +} + +/* Appends the guest pointer as the topmost draw of the scene. */ +static int add_cursor(struct draw_list *dl, int scene_w) { + if (!atomic_load(&g_cursor_composite) || !g_ptr_known || pointer_locked_for_cursor()) return 0; + + int hidden, w, h, hx, hy, serial; + uint32_t *copy = NULL; + pthread_mutex_lock(&g_cursor_lock); + hidden = g_cursor_hidden; + w = g_cursor_w; h = g_cursor_h; + hx = g_cursor_hx; hy = g_cursor_hy; + serial = g_cursor_serial; + if (!hidden && w > 0 && h > 0 && serial != g_cursor_img_serial) { + copy = malloc((size_t)w * (size_t)h * 4); + if (copy) memcpy(copy, g_cursor_px, (size_t)w * (size_t)h * 4); + } + pthread_mutex_unlock(&g_cursor_lock); + if (hidden || w <= 0 || h <= 0) return 0; + + if (copy) { + if (g_cursor_img && (g_cursor_img_w != w || g_cursor_img_h != h)) { + vkp_wait_idle(); + vkp_image_destroy(g_cursor_img); + g_cursor_img = NULL; + } + if (!g_cursor_img) g_cursor_img = vkp_image_create_shm(w, h); + if (g_cursor_img) { + g_cursor_img_w = w; g_cursor_img_h = h; + vkp_image_upload_shm(g_cursor_img, copy, w * 4); + g_cursor_img_serial = serial; + } + free(copy); + } + if (!g_cursor_img || g_cursor_img_serial != serial) return 0; + + int xs[CURSOR_MAX_DRAWS]; + const int source = vkp_cardboard_source(); + const int ptr_x = (int)(g_ptr_x + 0.5); + const int count = cursor_scene_positions(scene_w, source, ptr_x, hx, xs); + static int said_w = -1, said_source = -1, said_count = -1; + if (scene_w != said_w || source != said_source || count != said_count) { + said_w = scene_w; said_source = source; said_count = count; + droiddeck_log("cursor", "pointer drawn %d time(s) in a %d wide scene (%s picture)", count, scene_w, + source == CARDBOARD_SOURCE_SBS ? "side-by-side" : "single"); + } + if (dl->n + count > dl->cap) { + int cap = dl->cap ? dl->cap : 32; + while (cap < dl->n + count) cap *= 2; + struct vkp_draw *d = realloc(dl->d, (size_t)cap * sizeof(*d)); + if (!d) return 0; + dl->d = d; dl->cap = cap; + } + /* The cardboard pass samples each half of the scene as one eye, so a copy that crosses the + * seam would show up as a fragment at the opposite edge of the other eye: crop each copy to + * the eye it belongs to and drop it when nothing of it lands there. */ + const int sbs = source == CARDBOARD_SOURCE_SBS && scene_w > 1; + const int pointer_eye = ptr_x < scene_w / 2 ? 0 : 1; + int appended = 0; + for (int i = 0; i < count; i++) { + int dx, sx, cw; + const int eye = sbs ? (i ? 1 - pointer_eye : pointer_eye) : -1; + if (!cursor_clip_to_eye(scene_w, eye, xs[i], w, &dx, &sx, &cw)) continue; + dl->d[dl->n++] = (struct vkp_draw){g_cursor_img, (float)sx, 0, (float)cw, (float)h, + dx, (int)(g_ptr_y + 0.5) - hy, cw, h, 1}; + appended++; + } + return appended ? 1 : 0; +} + extern void droiddeck_on_pointer_lock(int locked, int x, int y); static void sync_lock_notify(void) { @@ -2617,6 +2765,9 @@ static void sync_lock_notify(void) { if (locked == last) return; last = locked; droiddeck_on_pointer_lock(locked, (int)g_ptr_x, (int)g_ptr_y); + /* The drawn pointer follows the lock state, so a game that stops presenting after it grabs + * the pointer would keep the last composited cursor on screen without a redraw here. */ + if (atomic_load(&g_cursor_composite)) schedule_render(); } static void constraint_describe(const struct constraint *k, char *out, size_t size) { @@ -2696,7 +2847,12 @@ static void constraint_activate(struct constraint *k) { constraint_end(g_active_constraint, 1, "replaced by a newer request"); k->active = 1; g_active_constraint = k; + const double was_x = g_ptr_x, was_y = g_ptr_y; if (!k->is_lock) confine_clamp(k, &g_ptr_x, &g_ptr_y); + /* Confining can move the pointer while the lock state stays the same, so sync_lock_notify() + * below stays quiet; the drawn pointer follows the position, so repaint here. */ + if ((g_ptr_x != was_x || g_ptr_y != was_y) && atomic_load(&g_cursor_composite)) + schedule_render(); constraint_origin(k, &ox, &oy); pointer_focus(k->surface->resource, wl_fixed_from_double(g_ptr_x - ox), wl_fixed_from_double(g_ptr_y - oy)); int focused = 0; @@ -2755,6 +2911,7 @@ static void constraints_pointer_gone(struct wl_resource *pointer) { static void constraints_surface_commit(struct surface *s) { struct constraint *k; + const double was_x = g_ptr_x, was_y = g_ptr_y; wl_list_for_each(k, &g_constraints, link) { if (k->surface != s) continue; if (k->pending_hint_set) { @@ -2778,6 +2935,10 @@ static void constraints_surface_commit(struct surface *s) { if (k->active && !k->is_lock) confine_clamp(k, &g_ptr_x, &g_ptr_y); } } + /* A position hint or a newly committed confine region can move the pointer while the lock + * state stays the same, so nothing else would repaint the composited cursor. */ + if ((g_ptr_x != was_x || g_ptr_y != was_y) && atomic_load(&g_cursor_composite)) + schedule_render(); } static void constraint_resource_destroy(struct wl_resource *r) { @@ -2963,6 +3124,7 @@ static void pointer_input(double x, double y, int relative, uint32_t button, int if (k) confine_clamp(k, &x, &y); g_ptr_x = x; g_ptr_y = y; g_ptr_known = 1; + if (atomic_load(&g_cursor_composite)) schedule_render(); if (k) { target = k->surface; } else if (g_desktop) { @@ -3442,6 +3604,7 @@ static int on_input_readable(int fd, uint32_t mask, void *data) { case 12: axis_event(m.p1, m.p2, 1); break; case 20: stream_take(); break; case 21: if (!m.p1) stream_forget_all(); schedule_render(); break; + case 22: schedule_render(); break; default: deliver_pointer(&m); break; } } @@ -3516,6 +3679,19 @@ void droiddeck_wayland_send_scene_input(int type, int a, int b) { (void)n; } +/* Any thread: draw the pointer into the compositor's own output (the glasses + * surface owns the picture, so the phone's Compose overlay never reaches it). */ +void droiddeck_wayland_set_cursor_composite(int on) { + const int want = on ? 1 : 0; + if (atomic_exchange(&g_cursor_composite_want, want) == want) return; + atomic_store(&g_cursor_composite, want); + droiddeck_log("cursor", "compositing the pointer into the output: %s", on ? "on" : "off"); + if (g_input_pipe[1] < 0) return; + struct input_msg m = { 22, on ? 1 : 0, 0, 0, 0 }; + ssize_t n = write(g_input_pipe[1], &m, sizeof(m)); + (void)n; +} + const char *droiddeck_client_name(struct wl_client *client) { return client_name(client); } struct wl_display *droiddeck_get_display(void) { return g_display; } diff --git a/app/src/main/cpp/waylandcomp/src/waylandcomp_jni.c b/app/src/main/cpp/waylandcomp/src/waylandcomp_jni.c index e3d0f5c5f4..adc2372dec 100644 --- a/app/src/main/cpp/waylandcomp/src/waylandcomp_jni.c +++ b/app/src/main/cpp/waylandcomp/src/waylandcomp_jni.c @@ -30,6 +30,7 @@ extern void droiddeck_wayland_send_key(int evdev, int state); extern int droiddeck_focus_takes_touch(void); extern int droiddeck_cursor_input_pos(int *x, int *y); extern void droiddeck_wayland_send_scene_input(int type, int a, int b); +extern void droiddeck_wayland_set_cursor_composite(int on); extern void droiddeck_wayland_vsync(int64_t frame_time_ns); extern void droiddeck_stream_set(int on); extern void droiddeck_stream_frame(AHardwareBuffer *ahb); @@ -292,6 +293,13 @@ Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeCursorSerial(JNIEnv return droiddeck_cursor_serial(); } +JNIEXPORT void JNICALL +Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeSetCursorComposite( + JNIEnv *env, jclass clazz, jboolean on) { + (void)env; (void)clazz; + droiddeck_wayland_set_cursor_composite(on ? 1 : 0); +} + JNIEXPORT void JNICALL Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeSendKey( JNIEnv *env, jclass clazz, jint evdev, jint state) { diff --git a/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt b/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt index e88966f905..93af09d07e 100644 --- a/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt +++ b/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt @@ -733,9 +733,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { }) scrollTouch = ScrollTouchGestures(PointerGestures.slop(this), android.view.ViewConfiguration.get(this).scaledMinimumFlingVelocity.toFloat(), scrollListener) - val noCursor = android.view.PointerIcon.getSystemIcon(this, android.view.PointerIcon.TYPE_NULL) - root.pointerIcon = noCursor - surfaceView.pointerIcon = noCursor + applyPlatformPointer(appUiOwnsPointer()) loading = LoadingState(this, steam = loadingMode() == SessionService.MODE_STEAM) if (!SessionState.running) { @@ -834,10 +832,12 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { isFocusableInTouchMode = true defaultFocusHighlightEnabled = false setContent { + val uiPointer = appUiOwnsPointer() androidx.compose.runtime.SideEffect { com.droiddeck.launcher.glasses.GlassesMenuMirror.setSessionMenuVisible(this@SessionActivity, (drawerOpen || achievementsOpen || pcKeyboardOpen) && !pipUi && steamVrStatus != SteamVr.Status.RUNNING) } + androidx.compose.runtime.SideEffect { applyPlatformPointer(uiPointer) } androidx.compose.runtime.DisposableEffect(Unit) { onDispose { com.droiddeck.launcher.glasses.GlassesMenuMirror.setSessionMenuVisible(this@SessionActivity, false) } } @@ -879,7 +879,8 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { } } else { com.droiddeck.launcher.ui.FocusGlideHost(androidx.compose.ui.Modifier.fillMaxSize()) { - CursorOverlay({ cursorPos }, cursorVisible, resources.displayMetrics.density, + CursorOverlay({ cursorPos }, cursorVisible && !uiPointer, + resources.displayMetrics.density, cursorImage, cursorHotX, cursorHotY, cursorImageScale) if (memoryHud.text.isNotEmpty()) HudText(memoryHud.text, hudColor, hudOpacity) val phonePanel = glassesOutputActive && !loading.visible && !drawerOpen && !pcKeyboardOpen @@ -2203,6 +2204,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { surfaceW = 0 surfaceH = 0 glassesOutputActive = false + if (CompositorHost.isStarted) WaylandCompositor.nativeSetCursorComposite(false) CompositorHost.detach(holder.surface) } @@ -2636,7 +2638,18 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { if (joyConVr?.onMotionEvent(event) == true) return true if (PadBridge.isControllerMotionEvent(event) && padMouseMode() && padMouse.onMotion(event)) return true if (padBridge?.onMotionEvent(event) == true) return true - if (event.isFromSource(android.view.InputDevice.SOURCE_MOUSE) && !drawerOpen && onMouse(event)) return true + if (event.isFromSource(android.view.InputDevice.SOURCE_MOUSE) && !drawerOpen) { + /* The app's own controls get the pointer first. With the picture on the glasses the + * phone shows its panel, and hovering or clicking a control there must reach it + * instead of moving and clicking the guest pointer on the glasses. Whatever the UI + * leaves alone — the rest of the screen — still drives the guest pointer. */ + if (appUiOwnsPointer()) { + if (super.dispatchGenericMotionEvent(event)) return true + if (onMouse(event)) return true + return false + } + if (onMouse(event)) return true + } return super.dispatchGenericMotionEvent(event) } @@ -3630,6 +3643,33 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { if (want) view.requestPointerCapture() else view.releasePointerCapture() } + /** Null until the first application, so the initial state is applied rather than skipped. */ + private var platformPointerShown: Boolean? = null + + /** + * The app's own UI is what the mouse is over, so it needs a pointer of its own: the drawer, + * keyboard, achievements and PiP controls, and — once the glasses own the picture — the whole + * phone, which then shows [GlassesPhonePanel] instead of the game. + */ + private fun appUiOwnsPointer(): Boolean = + drawerOpen || pcKeyboardOpen || achievementsOpen || pipUi || glassesOutputActive + + /** + * Over the game neither this app nor Android draws a pointer: the compositor's cursor is the + * only one (Compose's on the phone, inside the picture on the glasses). Over the app's own UI + * the game gets no pointer events at all, so without this nothing visible follows the mouse + * while buttons still highlight and click. + */ + private fun applyPlatformPointer(shown: Boolean) { + if (!::sessionRoot.isInitialized || !::surfaceView.isInitialized) return + if (shown == platformPointerShown) return + platformPointerShown = shown + val icon = if (shown) null + else android.view.PointerIcon.getSystemIcon(this, android.view.PointerIcon.TYPE_NULL) + sessionRoot.pointerIcon = icon + surfaceView.pointerIcon = icon + } + private fun onCapturedPointer(event: MotionEvent): Boolean { if (!pointerLocked || !event.isFromSource(InputDevice.SOURCE_MOUSE_RELATIVE)) return false notePress(event) @@ -4316,6 +4356,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { private fun onOutputAttached(holder: SurfaceHolder) { if (holder !== outputHolder()) return glassesOutputActive = glassesScreen?.holder === holder + if (CompositorHost.isStarted) WaylandCompositor.nativeSetCursorComposite(glassesOutputActive) // Keep the phone's layout for touch/controller coordinates, but remove // its SurfaceView so it cannot retain the last game frame under the hint. surfaceView.visibility = if (glassesOutputActive) View.INVISIBLE else View.VISIBLE diff --git a/app/src/main/java/com/droiddeck/launcher/wayland/WaylandCompositor.java b/app/src/main/java/com/droiddeck/launcher/wayland/WaylandCompositor.java index a31847a333..073ed60ffe 100644 --- a/app/src/main/java/com/droiddeck/launcher/wayland/WaylandCompositor.java +++ b/app/src/main/java/com/droiddeck/launcher/wayland/WaylandCompositor.java @@ -163,6 +163,8 @@ public final class WaylandCompositor { public static native int nativeCursorSerial(); + public static native void nativeSetCursorComposite(boolean on); + public static final int CURSOR_BUF_INTS = 6 + 256 * 256; public static native void nativeSendKey(int evdev, int state); diff --git a/docs/smart-glasses-support.md b/docs/smart-glasses-support.md index 19f59c2009..0446bdb34e 100644 --- a/docs/smart-glasses-support.md +++ b/docs/smart-glasses-support.md @@ -50,6 +50,10 @@ DroidDeck saves separate **2D resolution**, **3D resolution**, and **VR resoluti In **Automatic** picture mode, ordinary flat content is duplicated in full into both eyes on an SBS display. **Virtual screen** adds a configurable screen plane and tracking. **Side-by-side game** is for content that already contains separate eye images. SteamVR uses its direct stereo view and field-of-view control. +With **game on glasses** the compositor draws the guest pointer into its own scene as the last blended draw and mirrors it into both eyes for a side-by-side picture, while the phone stops drawing its Compose cursor. Over the app's own UI — the drawer, the PC keyboard, the achievements panel, PiP controls and the phone panel shown while the glasses have the picture — Android's pointer returns instead, so those controls are still clickable with a mouse. + +The pointer is left out of the picture when it cannot be drawn where the game is: while a game holds the pointer lock (it draws its own), and for a game the compositor cannot import — an AHardwareBuffer-only surface, or HDR frames it has no texture for. Those games stay on their own display layer, because composing the scene would drop the game itself. An importable HDR game that would otherwise stay on its own layer is composed through the screen swapchain, so there the pointer does fit. + ## Setup and app-wide behavior 1. Connect the glasses through a compatible video/data connection. Accept USB access when requested. For XREAL 1S, Android must also expose the glasses' USB-Ethernet network; the app leaves Steam's Internet connection on its normal network. diff --git a/tools/tests/cursor_composite_harness.c b/tools/tests/cursor_composite_harness.c new file mode 100644 index 0000000000..f4dede88c3 --- /dev/null +++ b/tools/tests/cursor_composite_harness.c @@ -0,0 +1,64 @@ +#include +#include + +#include "cursor_constants.inc" +#include "cursor_scene_positions.inc" +#include "cursor_clip_to_eye.inc" + +int main(void) { + int out[2]; + int dx, sx, dw; + + assert(cursor_scene_positions(1920, CARDBOARD_SOURCE_SCREEN, 100, 0, out) == 1); + assert(out[0] == 100); + + assert(cursor_scene_positions(1920, CARDBOARD_SOURCE_FLAT, 100, 0, out) == 1); + assert(out[0] == 100); + + assert(cursor_scene_positions(1, CARDBOARD_SOURCE_SBS, 0, 0, out) == 1); + + assert(cursor_scene_positions(3840, CARDBOARD_SOURCE_SBS, 100, 0, out) == 2); + assert(out[0] == 100 && out[1] == 2020); + + assert(cursor_scene_positions(3840, CARDBOARD_SOURCE_SBS, 2000, 0, out) == 2); + assert(out[0] == 2000 && out[1] == 80); + + assert(cursor_scene_positions(3840, CARDBOARD_SOURCE_SBS, 1919, 0, out) == 2); + assert(out[0] == 1919 && out[1] == 3839); + + assert(cursor_scene_positions(3840, CARDBOARD_SOURCE_SBS, 1920, 0, out) == 2); + assert(out[0] == 1920 && out[1] == 0); + + /* A hotspot that pushes the image across the seam must not move the mirror to the + * other eye: the pointer decides the eye, the hotspot only shifts the image. */ + assert(cursor_scene_positions(1920, CARDBOARD_SOURCE_SBS, 960, 32, out) == 2); + assert(out[0] == 928 && out[1] == -32); + + assert(cursor_scene_positions(1920, CARDBOARD_SOURCE_SBS, 959, 32, out) == 2); + assert(out[0] == 927 && out[1] == 1887); + + /* Each copy is cropped to its own eye, so nothing bleeds into the other eye's half. */ + assert(cursor_clip_to_eye(1920, -1, 100, 32, &dx, &sx, &dw) == 1); + assert(dx == 100 && sx == 0 && dw == 32); + + assert(cursor_clip_to_eye(1920, 0, 100, 32, &dx, &sx, &dw) == 1); + assert(dx == 100 && sx == 0 && dw == 32); + + assert(cursor_clip_to_eye(1920, 0, 950, 32, &dx, &sx, &dw) == 1); + assert(dx == 950 && sx == 0 && dw == 10); + + assert(cursor_clip_to_eye(1920, 1, 950, 32, &dx, &sx, &dw) == 1); + assert(dx == 960 && sx == 10 && dw == 22); + + assert(cursor_clip_to_eye(1920, 1, 960, 32, &dx, &sx, &dw) == 1); + assert(dx == 960 && sx == 0 && dw == 32); + + assert(cursor_clip_to_eye(1920, 0, 928, 32, &dx, &sx, &dw) == 1); + assert(dx == 928 && sx == 0 && dw == 32); + + assert(cursor_clip_to_eye(1920, 1, 928, 32, &dx, &sx, &dw) == 0); + assert(cursor_clip_to_eye(1920, 0, -32, 32, &dx, &sx, &dw) == 0); + + puts("PASS: cursor stereo placement"); + return 0; +} diff --git a/tools/tests/test_cursor_composite.py b/tools/tests/test_cursor_composite.py new file mode 100644 index 0000000000..5edf706c8b --- /dev/null +++ b/tools/tests/test_cursor_composite.py @@ -0,0 +1,54 @@ +"""Exercise the compositor's real cursor placement rule for glasses output.""" +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +REPO = Path(__file__).resolve().parents[2] + + +class CursorCompositeTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + compiler = shutil.which(os.environ.get('CC', 'cc')) + if compiler is None: + raise unittest.SkipTest('A host C compiler is required') + header = (REPO / 'app/src/main/cpp/waylandcomp/src/cardboard.h').read_text() + sources = dict(re.findall(r'CARDBOARD_SOURCE_(SCREEN|SBS|FLAT)\s*=\s*(\d+)', header)) + for name in ('SCREEN', 'SBS', 'FLAT'): + if name not in sources: + raise AssertionError(f'cardboard.h no longer defines CARDBOARD_SOURCE_{name}') + if sources['SBS'] != '1': + raise AssertionError(f'CARDBOARD_SOURCE_SBS is {sources["SBS"]}, expected 1') + + source = (REPO / 'app/src/main/cpp/waylandcomp/src/compositor.c').read_text() + start = source.index('static int cursor_scene_positions(int scene_w, int source, int ptr_x, int hx, int out[2]) {') + end = source.index('\n}\n', start) + 3 + + cls.temp = tempfile.TemporaryDirectory(prefix='droiddeck-cursor-composite-') + cls.addClassCleanup(cls.temp.cleanup) + folder = Path(cls.temp.name) + (folder / 'cursor_scene_positions.inc').write_text(source[start:end]) + start = source.index('static int cursor_clip_to_eye(int scene_w, int eye, int x0, int w, int *dx, int *sx, int *dw) {') + end = source.index('\n}\n', start) + 3 + (folder / 'cursor_clip_to_eye.inc').write_text(source[start:end]) + (folder / 'cursor_constants.inc').write_text( + f'#define CARDBOARD_SOURCE_SCREEN {sources["SCREEN"]}\n' + f'#define CARDBOARD_SOURCE_SBS {sources["SBS"]}\n' + f'#define CARDBOARD_SOURCE_FLAT {sources["FLAT"]}\n') + cls.binary = folder / 'cursor-composite-test' + subprocess.run([compiler, '-std=c11', '-Wall', '-Wextra', '-Werror', '-I', str(folder), + str(REPO / 'tools/tests/cursor_composite_harness.c'), + '-o', str(cls.binary)], check=True, capture_output=True, text=True) + + def test_cursor_is_drawn_once_for_one_eye_and_mirrored_for_sbs(self): + result = subprocess.run([str(self.binary)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('PASS: cursor stereo placement', result.stdout) + + +if __name__ == '__main__': + unittest.main()