From f5ca6d00531db08e2d2517cb2f58999612da4a30 Mon Sep 17 00:00:00 2001 From: The412Banner Date: Sun, 20 Sep 2026 03:43:53 -0400 Subject: [PATCH] A pointer for the desktop: an arrow, touch gestures and a real mouse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pointer moved but nothing drew it — the nested desktop asks its parent compositor for a cursor and ours ignores the request — and touch knew only move and tap. Now the app draws the arrow where it last sent the pointer, touch works like a remote desktop (tap clicks, hold then lift right-clicks, hold then drag drags, two fingers scroll), and a mouse arrives as itself: hover moves, every button presses, the wheel scrolls, with Android's own pointer hidden so there is one arrow. --- .../com/steamdeck/launcher/SessionActivity.kt | 90 ++++++++++--- .../launcher/input/PointerGestures.kt | 121 ++++++++++++++++++ .../steamdeck/launcher/ui/CursorOverlay.kt | 35 +++++ 3 files changed, 231 insertions(+), 15 deletions(-) create mode 100644 app/src/main/java/com/steamdeck/launcher/input/PointerGestures.kt create mode 100644 app/src/main/java/com/steamdeck/launcher/ui/CursorOverlay.kt diff --git a/app/src/main/java/com/steamdeck/launcher/SessionActivity.kt b/app/src/main/java/com/steamdeck/launcher/SessionActivity.kt index 8ed35ef99..049455a8b 100644 --- a/app/src/main/java/com/steamdeck/launcher/SessionActivity.kt +++ b/app/src/main/java/com/steamdeck/launcher/SessionActivity.kt @@ -29,6 +29,7 @@ import com.steamdeck.launcher.input.EvdevKeys import com.steamdeck.launcher.input.KeyboardHost import com.steamdeck.launcher.input.OnScreenControls import com.steamdeck.launcher.input.PadBridge +import com.steamdeck.launcher.input.PointerGestures import com.steamdeck.launcher.runtime.LinuxRuntime import com.steamdeck.launcher.session.LoadingState import com.steamdeck.launcher.session.PerfHud @@ -36,6 +37,7 @@ import com.steamdeck.launcher.session.ProtonExtras import com.steamdeck.launcher.session.SessionPrefs import com.steamdeck.launcher.session.SessionService import com.steamdeck.launcher.session.SessionState +import com.steamdeck.launcher.ui.CursorOverlay import com.steamdeck.launcher.ui.DrawerActions import com.steamdeck.launcher.ui.FrameGenDialog import com.steamdeck.launcher.ui.HudText @@ -67,6 +69,11 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { private var onScreenControls: OnScreenControls? = null private var keyboard: KeyboardHost? = null private var watching = true + private lateinit var gestures: PointerGestures + private var cursorPos by mutableStateOf(androidx.compose.ui.geometry.Offset(-100f, -100f)) + private var cursorVisible by mutableStateOf(false) + private val cursorHide = Runnable { cursorVisible = false } + private val uiHandler = Handler(Looper.getMainLooper()) // Compose reads these; the activity writes them. private var drawerOpen by mutableStateOf(false) @@ -109,6 +116,12 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { padBridge = bridge onScreenControls = OnScreenControls(this, bridge).also { root.addView(it) } keyboard = KeyboardHost(this).also { root.addView(it) } + gestures = PointerGestures(PointerGestures.slop(this), pointerListener) + // One arrow, ours: Android draws a system pointer for a mouse over any window, and the + // session already draws the pointer it is sent. + val noCursor = android.view.PointerIcon.getSystemIcon(this, android.view.PointerIcon.TYPE_NULL) + root.pointerIcon = noCursor + surfaceView.pointerIcon = noCursor loading = LoadingState(this) hud = PerfHud(this) @@ -127,6 +140,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { root.addView(ComposeView(this).apply { setContent { SteamDeckTheme { + CursorOverlay(cursorPos, cursorVisible, resources.displayMetrics.density) if (hud.text.isNotEmpty()) HudText(hud.text) if (loading.visible) LoadingOverlay(loading.step, loading.percent, loading.elapsed, loading.hint, loading.ended) if (drawerOpen) SessionDrawer(DrawerActions( @@ -402,33 +416,79 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { override fun dispatchGenericMotionEvent(event: MotionEvent): Boolean { if (padBridge?.onMotionEvent(event) == true) return true + if (event.isFromSource(android.view.InputDevice.SOURCE_MOUSE) && !drawerOpen && onMouse(event)) return true return super.dispatchGenericMotionEvent(event) } /** - * Touch as a mouse: where you touch is where the pointer goes, and a tap is a left click. - * The compositor's pointer space is a fixed 1920x1080, and the picture is letterboxed inside - * the view when the panel is a different shape from the output, so a touch is mapped through - * the fitted rectangle. + * The pointer. Touch goes through [PointerGestures] (tap, hold, drag, two-finger scroll); a + * mouse arrives with real buttons and a wheel and is forwarded as it is. Every position is + * mapped through the letterboxed rectangle into the compositor's fixed 1920x1080 pointer + * space, and the arrow is drawn where the app last sent the pointer. */ - override fun onTouchEvent(event: MotionEvent): Boolean { - val width = surfaceView.width.takeIf { it > 0 } ?: return false - val height = surfaceView.height.takeIf { it > 0 } ?: return false - val action = when (event.actionMasked) { - MotionEvent.ACTION_DOWN -> 0 - MotionEvent.ACTION_MOVE -> 1 - MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> 2 - else -> return false + private val pointerListener = object : PointerGestures.Listener { + override fun onMove(x: Float, y: Float) = movePointer(x, y) + override fun onButton(button: Int, pressed: Boolean, x: Float, y: Float) { + movePointer(x, y) + WaylandCompositor.nativeSendSceneInput(3, button, if (pressed) 1 else 0) } + override fun onWheel(steps: Int) { WaylandCompositor.nativeSendSceneInput(4, steps, 0) } + override fun onLongPress() { + @Suppress("DEPRECATION") + (getSystemService(VIBRATOR_SERVICE) as? android.os.Vibrator)?.vibrate(15) + } + } + + private fun movePointer(x: Float, y: Float) { + val width = surfaceView.width.takeIf { it > 0 } ?: return + val height = surfaceView.height.takeIf { it > 0 } ?: return val out = SessionState.outputSize val scale = minOf(width / out.first.toFloat(), height / out.second.toFloat()) val drawnW = out.first * scale val drawnH = out.second * scale val left = (width - drawnW) / 2f val top = (height - drawnH) / 2f - val x = ((event.x - left) / drawnW * 1920f).toInt().coerceIn(0, 1919) - val y = ((event.y - top) / drawnH * 1080f).toInt().coerceIn(0, 1079) - WaylandCompositor.nativeSendPointer(action, x, y) + val px = ((x - left) / drawnW * 1920f).toInt().coerceIn(0, 1919) + val py = ((y - top) / drawnH * 1080f).toInt().coerceIn(0, 1079) + WaylandCompositor.nativeSendPointer(1, px, py) + showCursor(x.coerceIn(left, left + drawnW), y.coerceIn(top, top + drawnH)) + } + + /** The arrow stays on a desktop; in a Steam session it shows for a moment after each move. */ + private fun showCursor(x: Float, y: Float) { + cursorPos = androidx.compose.ui.geometry.Offset(x, y) + cursorVisible = true + uiHandler.removeCallbacks(cursorHide) + if (SessionState.mode != SessionService.MODE_DESKTOP) uiHandler.postDelayed(cursorHide, 2500) + } + + override fun onTouchEvent(event: MotionEvent): Boolean { + if (event.isFromSource(android.view.InputDevice.SOURCE_MOUSE)) return onMouse(event) + return gestures.onTouch(event) + } + + /** A mouse: hover moves, buttons press, the wheel scrolls. Android sends buttons as touch + * actions and hover as generic motion, so both paths land here. */ + private fun onMouse(event: MotionEvent): Boolean { + when (event.actionMasked) { + MotionEvent.ACTION_HOVER_MOVE, MotionEvent.ACTION_MOVE, MotionEvent.ACTION_HOVER_ENTER -> + movePointer(event.x, event.y) + MotionEvent.ACTION_BUTTON_PRESS, MotionEvent.ACTION_BUTTON_RELEASE -> { + movePointer(event.x, event.y) + val button = when (event.actionButton) { + MotionEvent.BUTTON_SECONDARY -> PointerGestures.BTN_RIGHT + MotionEvent.BUTTON_TERTIARY -> PointerGestures.BTN_MIDDLE + else -> PointerGestures.BTN_LEFT + } + WaylandCompositor.nativeSendSceneInput(3, button, if (event.actionMasked == MotionEvent.ACTION_BUTTON_PRESS) 1 else 0) + } + MotionEvent.ACTION_SCROLL -> { + val v = event.getAxisValue(MotionEvent.AXIS_VSCROLL) + if (v != 0f) WaylandCompositor.nativeSendSceneInput(4, -Math.round(v), 0) + } + MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP -> {} // the BUTTON_* events carry these + else -> return false + } return true } diff --git a/app/src/main/java/com/steamdeck/launcher/input/PointerGestures.kt b/app/src/main/java/com/steamdeck/launcher/input/PointerGestures.kt new file mode 100644 index 000000000..df60fc0c0 --- /dev/null +++ b/app/src/main/java/com/steamdeck/launcher/input/PointerGestures.kt @@ -0,0 +1,121 @@ +package com.steamdeck.launcher.input + +import android.os.Handler +import android.os.Looper +import android.view.MotionEvent +import android.view.ViewConfiguration +import kotlin.math.abs +import kotlin.math.roundToInt + +/** + * Touch as a mouse, the way a remote desktop does it: where you touch is where the pointer goes. + * + * - tap ............................ left click + * - drag ........................... moves the pointer, no button (hover) + * - hold, then lift ................ right click + * - hold, then drag ................ left-button drag (a window by its title bar, a selection) + * - two fingers up / down .......... scroll wheel + * + * The listener gets pointer positions in the view's pixels; the activity maps them onto the + * compositor. Mouse events are not handled here — a mouse has real buttons. + */ +class PointerGestures( + private val slopPx: Float, + private val listener: Listener, +) { + interface Listener { + fun onMove(x: Float, y: Float) + /** [button] is an evdev code: BTN_LEFT, BTN_RIGHT. */ + fun onButton(button: Int, pressed: Boolean, x: Float, y: Float) + fun onWheel(steps: Int) + fun onLongPress() + } + + private val handler = Handler(Looper.getMainLooper()) + private var downX = 0f + private var downY = 0f + private var downAt = 0L + private var moved = false + private var held = false // the long press fired + private var dragging = false // left button down after a hold + private var scrolling = false + private var scrollAnchorY = 0f + private val longPress = Runnable { + held = true + listener.onLongPress() + } + + fun onTouch(event: MotionEvent): Boolean { + when (event.actionMasked) { + MotionEvent.ACTION_DOWN -> { + downX = event.x; downY = event.y; downAt = event.eventTime + moved = false; held = false; dragging = false; scrolling = false + listener.onMove(event.x, event.y) + handler.postDelayed(longPress, HOLD_MS) + return true + } + MotionEvent.ACTION_POINTER_DOWN -> { + // A second finger: scrolling, and whatever the first one was doing is off. + handler.removeCallbacks(longPress) + if (dragging) { listener.onButton(BTN_LEFT, false, event.x, event.y); dragging = false } + scrolling = true + scrollAnchorY = event.getY(0) + return true + } + MotionEvent.ACTION_MOVE -> { + if (scrolling) { + val dy = event.getY(0) - scrollAnchorY + val steps = (dy / SCROLL_STEP_PX).toInt() + if (steps != 0) { + // Finger down = content up = wheel "up", which the compositor takes as negative. + listener.onWheel(-steps) + scrollAnchorY += steps * SCROLL_STEP_PX + } + return true + } + if (!moved && (abs(event.x - downX) > slopPx || abs(event.y - downY) > slopPx)) { + moved = true + handler.removeCallbacks(longPress) + if (held) { + // Held, then moved: a drag with the left button, from where the hold was. + dragging = true + listener.onButton(BTN_LEFT, true, downX, downY) + } + } + if (moved) listener.onMove(event.x, event.y) + return true + } + MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> { + handler.removeCallbacks(longPress) + val cancelled = event.actionMasked == MotionEvent.ACTION_CANCEL + when { + scrolling -> {} + dragging -> listener.onButton(BTN_LEFT, false, event.x, event.y) + held && !moved && !cancelled -> { + listener.onButton(BTN_RIGHT, true, downX, downY) + listener.onButton(BTN_RIGHT, false, downX, downY) + } + !moved && !cancelled && event.eventTime - downAt < TAP_MS -> { + listener.onButton(BTN_LEFT, true, downX, downY) + listener.onButton(BTN_LEFT, false, downX, downY) + } + } + scrolling = false; dragging = false; held = false + return true + } + MotionEvent.ACTION_POINTER_UP -> return true + } + return false + } + + companion object { + const val BTN_LEFT = 0x110 + const val BTN_RIGHT = 0x111 + const val BTN_MIDDLE = 0x112 + private const val HOLD_MS = 450L + private const val TAP_MS = 350L + private const val SCROLL_STEP_PX = 40f + + fun slop(context: android.content.Context) = ViewConfiguration.get(context).scaledTouchSlop.toFloat() + } +} diff --git a/app/src/main/java/com/steamdeck/launcher/ui/CursorOverlay.kt b/app/src/main/java/com/steamdeck/launcher/ui/CursorOverlay.kt new file mode 100644 index 000000000..173a5aa79 --- /dev/null +++ b/app/src/main/java/com/steamdeck/launcher/ui/CursorOverlay.kt @@ -0,0 +1,35 @@ +package com.steamdeck.launcher.ui + +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.Path +import androidx.compose.ui.graphics.drawscope.Stroke + +/** + * The pointer, drawn by the app. The nested desktop asks its parent compositor to show a cursor + * and ours does not, and the Steam session has none either; this is the arrow for both, at the + * position the app last sent. Takes no touches. + */ +@Composable +fun CursorOverlay(position: Offset, visible: Boolean, scale: Float) { + if (!visible) return + Canvas(modifier = Modifier.fillMaxSize()) { + val s = 22f * scale + val path = Path().apply { + moveTo(position.x, position.y) + lineTo(position.x, position.y + s * 1.45f) + lineTo(position.x + s * 0.36f, position.y + s * 1.12f) + lineTo(position.x + s * 0.62f, position.y + s * 1.62f) + lineTo(position.x + s * 0.84f, position.y + s * 1.52f) + lineTo(position.x + s * 0.58f, position.y + s * 1.02f) + lineTo(position.x + s * 1.02f, position.y + s * 1.02f) + close() + } + drawPath(path, Color.White) + drawPath(path, Color(0xFF1B0730), style = Stroke(width = 1.5f * scale)) + } +}