A pointer for the desktop: an arrow, touch gestures and a real mouse

The pointer moved but nothing drew it — the nested desktop asks its parent
compositor for a cursor and ours ignores the request — and touch knew only
move and tap. Now the app draws the arrow where it last sent the pointer, touch
works like a remote desktop (tap clicks, hold then lift right-clicks, hold then
drag drags, two fingers scroll), and a mouse arrives as itself: hover moves,
every button presses, the wheel scrolls, with Android's own pointer hidden so
there is one arrow.

app/src/main/java/com/steamdeck/launcher/SessionActivity.kt

@@ -29,6 +29,7 @@ import com.steamdeck.launcher.input.EvdevKeys
 import com.steamdeck.launcher.input.KeyboardHost
 import com.steamdeck.launcher.input.OnScreenControls
 import com.steamdeck.launcher.input.PadBridge
+import com.steamdeck.launcher.input.PointerGestures
 import com.steamdeck.launcher.runtime.LinuxRuntime
 import com.steamdeck.launcher.session.LoadingState
 import com.steamdeck.launcher.session.PerfHud
@@ -36,6 +37,7 @@ import com.steamdeck.launcher.session.ProtonExtras
 import com.steamdeck.launcher.session.SessionPrefs
 import com.steamdeck.launcher.session.SessionService
 import com.steamdeck.launcher.session.SessionState
+import com.steamdeck.launcher.ui.CursorOverlay
 import com.steamdeck.launcher.ui.DrawerActions
 import com.steamdeck.launcher.ui.FrameGenDialog
 import com.steamdeck.launcher.ui.HudText
@@ -67,6 +69,11 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
     private var onScreenControls: OnScreenControls? = null
     private var keyboard: KeyboardHost? = null
     private var watching = true
+    private lateinit var gestures: PointerGestures
+    private var cursorPos by mutableStateOf(androidx.compose.ui.geometry.Offset(-100f, -100f))
+    private var cursorVisible by mutableStateOf(false)
+    private val cursorHide = Runnable { cursorVisible = false }
+    private val uiHandler = Handler(Looper.getMainLooper())
 
     // Compose reads these; the activity writes them.
     private var drawerOpen by mutableStateOf(false)
@@ -109,6 +116,12 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
         padBridge = bridge
         onScreenControls = OnScreenControls(this, bridge).also { root.addView(it) }
         keyboard = KeyboardHost(this).also { root.addView(it) }
+        gestures = PointerGestures(PointerGestures.slop(this), pointerListener)
+        // One arrow, ours: Android draws a system pointer for a mouse over any window, and the
+        // session already draws the pointer it is sent.
+        val noCursor = android.view.PointerIcon.getSystemIcon(this, android.view.PointerIcon.TYPE_NULL)
+        root.pointerIcon = noCursor
+        surfaceView.pointerIcon = noCursor
 
         loading = LoadingState(this)
         hud = PerfHud(this)
@@ -127,6 +140,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
         root.addView(ComposeView(this).apply {
             setContent {
                 SteamDeckTheme {
+                    CursorOverlay(cursorPos, cursorVisible, resources.displayMetrics.density)
                     if (hud.text.isNotEmpty()) HudText(hud.text)
                     if (loading.visible) LoadingOverlay(loading.step, loading.percent, loading.elapsed, loading.hint, loading.ended)
                     if (drawerOpen) SessionDrawer(DrawerActions(
@@ -402,33 +416,79 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
 
     override fun dispatchGenericMotionEvent(event: MotionEvent): Boolean {
         if (padBridge?.onMotionEvent(event) == true) return true
+        if (event.isFromSource(android.view.InputDevice.SOURCE_MOUSE) && !drawerOpen && onMouse(event)) return true
         return super.dispatchGenericMotionEvent(event)
     }
 
     /**
-     * Touch as a mouse: where you touch is where the pointer goes, and a tap is a left click.
-     * The compositor's pointer space is a fixed 1920x1080, and the picture is letterboxed inside
-     * the view when the panel is a different shape from the output, so a touch is mapped through
-     * the fitted rectangle.
+     * The pointer. Touch goes through [PointerGestures] (tap, hold, drag, two-finger scroll); a
+     * mouse arrives with real buttons and a wheel and is forwarded as it is. Every position is
+     * mapped through the letterboxed rectangle into the compositor's fixed 1920x1080 pointer
+     * space, and the arrow is drawn where the app last sent the pointer.
      */
-    override fun onTouchEvent(event: MotionEvent): Boolean {
-        val width = surfaceView.width.takeIf { it > 0 } ?: return false
-        val height = surfaceView.height.takeIf { it > 0 } ?: return false
-        val action = when (event.actionMasked) {
-            MotionEvent.ACTION_DOWN -> 0
-            MotionEvent.ACTION_MOVE -> 1
-            MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> 2
-            else -> return false
+    private val pointerListener = object : PointerGestures.Listener {
+        override fun onMove(x: Float, y: Float) = movePointer(x, y)
+        override fun onButton(button: Int, pressed: Boolean, x: Float, y: Float) {
+            movePointer(x, y)
+            WaylandCompositor.nativeSendSceneInput(3, button, if (pressed) 1 else 0)
         }
+        override fun onWheel(steps: Int) { WaylandCompositor.nativeSendSceneInput(4, steps, 0) }
+        override fun onLongPress() {
+            @Suppress("DEPRECATION")
+            (getSystemService(VIBRATOR_SERVICE) as? android.os.Vibrator)?.vibrate(15)
+        }
+    }
+
+    private fun movePointer(x: Float, y: Float) {
+        val width = surfaceView.width.takeIf { it > 0 } ?: return
+        val height = surfaceView.height.takeIf { it > 0 } ?: return
         val out = SessionState.outputSize
         val scale = minOf(width / out.first.toFloat(), height / out.second.toFloat())
         val drawnW = out.first * scale
         val drawnH = out.second * scale
         val left = (width - drawnW) / 2f
         val top = (height - drawnH) / 2f
-        val x = ((event.x - left) / drawnW * 1920f).toInt().coerceIn(0, 1919)
-        val y = ((event.y - top) / drawnH * 1080f).toInt().coerceIn(0, 1079)
-        WaylandCompositor.nativeSendPointer(action, x, y)
+        val px = ((x - left) / drawnW * 1920f).toInt().coerceIn(0, 1919)
+        val py = ((y - top) / drawnH * 1080f).toInt().coerceIn(0, 1079)
+        WaylandCompositor.nativeSendPointer(1, px, py)
+        showCursor(x.coerceIn(left, left + drawnW), y.coerceIn(top, top + drawnH))
+    }
+
+    /** The arrow stays on a desktop; in a Steam session it shows for a moment after each move. */
+    private fun showCursor(x: Float, y: Float) {
+        cursorPos = androidx.compose.ui.geometry.Offset(x, y)
+        cursorVisible = true
+        uiHandler.removeCallbacks(cursorHide)
+        if (SessionState.mode != SessionService.MODE_DESKTOP) uiHandler.postDelayed(cursorHide, 2500)
+    }
+
+    override fun onTouchEvent(event: MotionEvent): Boolean {
+        if (event.isFromSource(android.view.InputDevice.SOURCE_MOUSE)) return onMouse(event)
+        return gestures.onTouch(event)
+    }
+
+    /** A mouse: hover moves, buttons press, the wheel scrolls. Android sends buttons as touch
+     *  actions and hover as generic motion, so both paths land here. */
+    private fun onMouse(event: MotionEvent): Boolean {
+        when (event.actionMasked) {
+            MotionEvent.ACTION_HOVER_MOVE, MotionEvent.ACTION_MOVE, MotionEvent.ACTION_HOVER_ENTER ->
+                movePointer(event.x, event.y)
+            MotionEvent.ACTION_BUTTON_PRESS, MotionEvent.ACTION_BUTTON_RELEASE -> {
+                movePointer(event.x, event.y)
+                val button = when (event.actionButton) {
+                    MotionEvent.BUTTON_SECONDARY -> PointerGestures.BTN_RIGHT
+                    MotionEvent.BUTTON_TERTIARY -> PointerGestures.BTN_MIDDLE
+                    else -> PointerGestures.BTN_LEFT
+                }
+                WaylandCompositor.nativeSendSceneInput(3, button, if (event.actionMasked == MotionEvent.ACTION_BUTTON_PRESS) 1 else 0)
+            }
+            MotionEvent.ACTION_SCROLL -> {
+                val v = event.getAxisValue(MotionEvent.AXIS_VSCROLL)
+                if (v != 0f) WaylandCompositor.nativeSendSceneInput(4, -Math.round(v), 0)
+            }
+            MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP -> {} // the BUTTON_* events carry these
+            else -> return false
+        }
         return true
     }
 

app/src/main/java/com/steamdeck/launcher/input/PointerGestures.kt

@@ -0,0 +1,121 @@
+package com.steamdeck.launcher.input
+
+import android.os.Handler
+import android.os.Looper
+import android.view.MotionEvent
+import android.view.ViewConfiguration
+import kotlin.math.abs
+import kotlin.math.roundToInt
+
+/**
+ * Touch as a mouse, the way a remote desktop does it: where you touch is where the pointer goes.
+ *
+ *  - tap ............................ left click
+ *  - drag ........................... moves the pointer, no button (hover)
+ *  - hold, then lift ................ right click
+ *  - hold, then drag ................ left-button drag (a window by its title bar, a selection)
+ *  - two fingers up / down .......... scroll wheel
+ *
+ * The listener gets pointer positions in the view's pixels; the activity maps them onto the
+ * compositor. Mouse events are not handled here — a mouse has real buttons.
+ */
+class PointerGestures(
+    private val slopPx: Float,
+    private val listener: Listener,
+) {
+    interface Listener {
+        fun onMove(x: Float, y: Float)
+        /** [button] is an evdev code: BTN_LEFT, BTN_RIGHT. */
+        fun onButton(button: Int, pressed: Boolean, x: Float, y: Float)
+        fun onWheel(steps: Int)
+        fun onLongPress()
+    }
+
+    private val handler = Handler(Looper.getMainLooper())
+    private var downX = 0f
+    private var downY = 0f
+    private var downAt = 0L
+    private var moved = false
+    private var held = false          // the long press fired
+    private var dragging = false      // left button down after a hold
+    private var scrolling = false
+    private var scrollAnchorY = 0f
+    private val longPress = Runnable {
+        held = true
+        listener.onLongPress()
+    }
+
+    fun onTouch(event: MotionEvent): Boolean {
+        when (event.actionMasked) {
+            MotionEvent.ACTION_DOWN -> {
+                downX = event.x; downY = event.y; downAt = event.eventTime
+                moved = false; held = false; dragging = false; scrolling = false
+                listener.onMove(event.x, event.y)
+                handler.postDelayed(longPress, HOLD_MS)
+                return true
+            }
+            MotionEvent.ACTION_POINTER_DOWN -> {
+                // A second finger: scrolling, and whatever the first one was doing is off.
+                handler.removeCallbacks(longPress)
+                if (dragging) { listener.onButton(BTN_LEFT, false, event.x, event.y); dragging = false }
+                scrolling = true
+                scrollAnchorY = event.getY(0)
+                return true
+            }
+            MotionEvent.ACTION_MOVE -> {
+                if (scrolling) {
+                    val dy = event.getY(0) - scrollAnchorY
+                    val steps = (dy / SCROLL_STEP_PX).toInt()
+                    if (steps != 0) {
+                        // Finger down = content up = wheel "up", which the compositor takes as negative.
+                        listener.onWheel(-steps)
+                        scrollAnchorY += steps * SCROLL_STEP_PX
+                    }
+                    return true
+                }
+                if (!moved && (abs(event.x - downX) > slopPx || abs(event.y - downY) > slopPx)) {
+                    moved = true
+                    handler.removeCallbacks(longPress)
+                    if (held) {
+                        // Held, then moved: a drag with the left button, from where the hold was.
+                        dragging = true
+                        listener.onButton(BTN_LEFT, true, downX, downY)
+                    }
+                }
+                if (moved) listener.onMove(event.x, event.y)
+                return true
+            }
+            MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
+                handler.removeCallbacks(longPress)
+                val cancelled = event.actionMasked == MotionEvent.ACTION_CANCEL
+                when {
+                    scrolling -> {}
+                    dragging -> listener.onButton(BTN_LEFT, false, event.x, event.y)
+                    held && !moved && !cancelled -> {
+                        listener.onButton(BTN_RIGHT, true, downX, downY)
+                        listener.onButton(BTN_RIGHT, false, downX, downY)
+                    }
+                    !moved && !cancelled && event.eventTime - downAt < TAP_MS -> {
+                        listener.onButton(BTN_LEFT, true, downX, downY)
+                        listener.onButton(BTN_LEFT, false, downX, downY)
+                    }
+                }
+                scrolling = false; dragging = false; held = false
+                return true
+            }
+            MotionEvent.ACTION_POINTER_UP -> return true
+        }
+        return false
+    }
+
+    companion object {
+        const val BTN_LEFT = 0x110
+        const val BTN_RIGHT = 0x111
+        const val BTN_MIDDLE = 0x112
+        private const val HOLD_MS = 450L
+        private const val TAP_MS = 350L
+        private const val SCROLL_STEP_PX = 40f
+
+        fun slop(context: android.content.Context) = ViewConfiguration.get(context).scaledTouchSlop.toFloat()
+    }
+}

app/src/main/java/com/steamdeck/launcher/ui/CursorOverlay.kt

@@ -0,0 +1,35 @@
+package com.steamdeck.launcher.ui
+
+import androidx.compose.foundation.Canvas
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.runtime.Composable
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.geometry.Offset
+import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.Path
+import androidx.compose.ui.graphics.drawscope.Stroke
+
+/**
+ * The pointer, drawn by the app. The nested desktop asks its parent compositor to show a cursor
+ * and ours does not, and the Steam session has none either; this is the arrow for both, at the
+ * position the app last sent. Takes no touches.
+ */
+@Composable
+fun CursorOverlay(position: Offset, visible: Boolean, scale: Float) {
+    if (!visible) return
+    Canvas(modifier = Modifier.fillMaxSize()) {
+        val s = 22f * scale
+        val path = Path().apply {
+            moveTo(position.x, position.y)
+            lineTo(position.x, position.y + s * 1.45f)
+            lineTo(position.x + s * 0.36f, position.y + s * 1.12f)
+            lineTo(position.x + s * 0.62f, position.y + s * 1.62f)
+            lineTo(position.x + s * 0.84f, position.y + s * 1.52f)
+            lineTo(position.x + s * 0.58f, position.y + s * 1.02f)
+            lineTo(position.x + s * 1.02f, position.y + s * 1.02f)
+            close()
+        }
+        drawPath(path, Color.White)
+        drawPath(path, Color(0xFF1B0730), style = Stroke(width = 1.5f * scale))
+    }
+}