feat(input): a pen moves the pointer while hovering, clicks with its tip and right-clicks with its side button (#2625)

* feat(compositor): pen samples drive the pointer like a mouse

Input record 13 carries the pen's state (in proximity, tip, right and
middle button) and its position in input space as 24.8 fixed point.
Hover moves the pointer, the tip holds the left button, and a record
without the in-proximity bit releases every button. Under a pointer
lock only the buttons pass. A button pressed or released mid-stroke
keeps the implicit grab on the window the stroke started in, unless
that window closed.

* feat(input): a pen moves the pointer while hovering, clicks with its tip and right-clicks with its side button

In Mouse mode (the default) a pen's hover moves the pointer, the tip
holds the left button and the side button right-clicks; Touch keeps it a
finger. The side button can send a middle click or nothing instead, and
a Bluetooth pen key counts as the side button while the pen is near.
With glasses or a program's pointer lock the pen drives the touchpad.
A hover exit followed within 50 ms by more hover or a touch keeps the
pen near, so a pen at the edge of its hover range does not flicker out,
and a hover that sends nothing for 2 s leaves proximity. On the desktop
a pen that comes near releases KWin's grab of a hardware mouse, so the
pen moves KWin's pointer. Whatever cancels touches also ends the pen,
and stylus devices no longer count as hardware mice.

* feat(input): pen mode and side button settings in the Controls tab and the session drawer

A Pen group follows the touch settings once the phone has a stylus
device or a pen event was seen, which is remembered as penSeen and kept
by a settings reset. It holds Pen (Mouse, Touch) and Side button (Right
click, Middle click, Off); Side button is hidden in Touch. Before
Android 14 a touchscreen reports the stylus source without a pen, so
there it counts only with its first pen event. The strings are
translated into all 32 languages.

* docs: describe pen input and its limits

---------

Co-authored-by: alien2003 <11610652+alien2003@users.noreply.github.com>
This commit is contained in:
alien2003andalien2003 authored and GitHub committed 2026-11-15 09:38:01 -05:00
1 parent de70e37f30
commit decab6ea90
63 files changed
+2352 -38

No files matched your search

@@ -97,8 +97,11 @@ import com.droiddeck.launcher.input.PadState
import com.droiddeck.launcher.input.SteamControllerBle
import com.droiddeck.launcher.input.DirectTouchHold
import com.droiddeck.launcher.input.GestureTouchStreamGate
import com.droiddeck.launcher.input.PenRouter
import com.droiddeck.launcher.input.PenStream
import com.droiddeck.launcher.input.PointerGestures
import com.droiddeck.launcher.input.PointerSpace
import com.droiddeck.launcher.input.pointerSubset
import com.droiddeck.launcher.input.SecondScreenDisplay
import com.droiddeck.launcher.input.SecondScreenDisplays
import com.droiddeck.launcher.input.SecondScreenMode
@@ -266,6 +269,11 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
private var steamTouchOptions: com.droiddeck.launcher.input.SteamTouchConfig.Options? = null
private var routedTouchMode = SessionPrefs.TOUCH_AUTO
private var touchRoute: com.droiddeck.launcher.input.TouchRouter.Route? = null
private var penSettings by mutableStateOf(SessionPrefs.Pen())
private var penSeen by mutableStateOf(false)
private var penRoute: PenRouter.Route? = null
private var penDown = false
private val penStream = PenStream(WaylandCompositor::nativeSendPen)
private var keyboard: KeyboardHost? = null
private var desktopKeyboardButton: com.droiddeck.launcher.input.DesktopKeyboardButton? = null
private var lastUserPressMs = 0L
@@ -667,6 +675,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
private val deviceListener = object : InputManager.InputDeviceListener {
override fun onInputDeviceAdded(deviceId: Int) {
logInputDevice("connected", deviceId)
if (!penSeen && InputDevice.getDevice(deviceId)?.let { PenStream.isPenDevice(it.sources) } == true) notePenSeen()
joyConVr?.refresh()
padBridge?.syncConnectedControllers()
refreshControls()
@@ -677,6 +686,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
}
override fun onInputDeviceRemoved(deviceId: Int) {
physicalShortcuts.onDeviceRemoved(deviceId)
penStream.deviceRemoved(deviceId)
Log.i(TAG, "input device $deviceId disconnected")
dropMouseCursor()
backQamInput.onDeviceRemoved(deviceId)
@@ -771,6 +781,9 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
uiHandler.postDelayed(::updatePointerCapture, 250)
}
}
override fun onResolvePointerIcon(event: MotionEvent, pointerIndex: Int): android.view.PointerIcon? =
if (platformPointerShown == false && PenStream.isPen(event, pointerIndex)) pointerIcon
else super.onResolvePointerIcon(event, pointerIndex)
}
sessionRoot = root
surfaceView = SurfaceView(this)
@@ -852,7 +865,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
refreshGestureConfig()
root.isMotionEventSplittingEnabled = true
surfaceView.setOnTouchListener { _, event ->
if (!event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)) {
if (!event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN) && PenStream.penIndex(event) < 0) {
false
} else if (!gestureTouchStreamGate.accepts(GestureTouchStreamGate.Source.SURFACE, event)) {
false
@@ -1374,6 +1387,8 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
micEnabled = SessionState.micEnabled,
onMicEnabled = { SessionService.setMicEnabled(this@SessionActivity, it) },
touchAuto = getString(if (usingTouchpad()) R.string.session_touch_auto_touchpad else R.string.session_touch_auto_direct),
pen = penSettings.takeIf { penSeen },
onPen = { key, value -> SessionPrefs.setPenSetting(this@SessionActivity, key, value); readPrefs() },
fexPreset = fexPreset,
secondScreenMode = secondScreenMode,
secondScreenNone = !bottomOutputActive && !dockDrawer(),
@@ -2365,6 +2380,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
textureLodBias = SessionPrefs.textureLodBias(this)
val nextTouchMode = SessionPrefs.touchMode(this)
if (touchMode != nextTouchMode) {
cancelPen()
if (::touchpad.isInitialized) touchpad.cancel()
if (::directTouchHold.isInitialized) directTouchHold.cancel()
if (::twoFingerGestures.isInitialized) twoFingerGestures.cancel()
@@ -2378,6 +2394,11 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
if (nextTouchMode == SessionPrefs.TOUCH_OFF || !usingTouchpad()) cursorVisible = false
}
touchMode = nextTouchMode
val nextPen = SessionPrefs.pen(this)
if (penSettings != nextPen) cancelPen()
penSettings = nextPen
penStream.sideButton = nextPen.button
if (!penSeen && (SessionPrefs.penSeen(this) || PenStream.penConnected())) notePenSeen()
val nextRoutedTouchMode = effectiveTouchMode()
if (routedTouchMode != nextRoutedTouchMode) gestureTouchStreamGate.reset()
routedTouchMode = nextRoutedTouchMode
@@ -2753,9 +2774,20 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
if (::directTouchHold.isInitialized) directTouchHold.cancel()
if (::twoFingerGestures.isInitialized) twoFingerGestures.cancel()
if (::scrollTouch.isInitialized) scrollTouch.cancel()
if (sessionTouchDown || penDown) dismissTouch = true
cancelPen()
scrollTouchGesture = false
gesturePointerReady = false
if (sessionTouchDown) dismissTouch = true
}
private fun cancelPen(releaseKey: Boolean = false) {
penStream.cancel(releaseKey)
penDown = false
}
private fun notePenSeen() {
penSeen = true
SessionPrefs.setPenSeen(this)
}
private fun panelBounds(): android.graphics.Rect = if (Build.VERSION.SDK_INT >= 30) {
@@ -3219,6 +3251,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
return super.dispatchKeyEvent(event)
}
}
if (penStream.holdsKey(event) || (penSettings.mode != SessionPrefs.PEN_TOUCH && PenStream.isPenKey(event))) return penStream.onKey(event)
if (appTextFocus) return super.dispatchKeyEvent(event)
if (!pipUi) {
if (physicalInputBindings?.onKey(event, resumed && !drawerOpen && !pcKeyboardOpen && !achievementsOpen &&
@@ -3404,6 +3437,19 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
if (joyConVr?.onMotionEvent(event) == true) return true
if (PadBridge.isControllerMotionEvent(event) && padMouseMode() && padMouse.onMotion(event)) return true
if (padBridge?.onMotionEvent(event) == true) return true
val pen = PenStream.penIndex(event)
if (pen >= 0) when (routePen(event, pen)) {
PenRouter.Route.TOUCH -> {}
PenRouter.Route.PEN -> {
if (!appUiOwnsPointer()) return penSample(event, pen) || super.dispatchGenericMotionEvent(event)
if (super.dispatchGenericMotionEvent(event)) {
penStream.cancel()
return true
}
return !touchEditorOpen && penSample(event, pen)
}
else -> return super.dispatchGenericMotionEvent(event)
}
if (event.isFromSource(android.view.InputDevice.SOURCE_MOUSE) && !drawerOpen) {
if (appUiOwnsPointer()) {
if (super.dispatchGenericMotionEvent(event)) return true
@@ -3438,7 +3484,8 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
override fun dispatchTouchEvent(event: MotionEvent): Boolean {
if (pipUi) return true
notePress(event)
if (event.actionMasked == MotionEvent.ACTION_DOWN && event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)) {
if (event.actionMasked == MotionEvent.ACTION_DOWN &&
(event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN) || PenStream.penIndex(event) >= 0)) {
drawerControllerActive = false
gestureTouchStreamGate.reset()
}
@@ -4440,6 +4487,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
}
if (next == routedTouchMode) return
if (::touchscreenGestures.isInitialized) touchscreenGestures.cancel()
cancelPen()
gesturePointerReady = false
if (::touchpad.isInitialized) touchpad.cancel()
if (::directTouchHold.isInitialized) directTouchHold.cancel()
@@ -4608,7 +4656,9 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
private var sessionTouchDown = false
override fun onTouchEvent(event: MotionEvent): Boolean {
if (event.isFromSource(android.view.InputDevice.SOURCE_MOUSE)) return onMouse(event)
val pen = PenStream.penIndex(event)
val penRouted = if (pen >= 0) routePen(event, pen).takeIf { it != PenRouter.Route.TOUCH } else null
if (penRouted == null && event.isFromSource(android.view.InputDevice.SOURCE_MOUSE)) return onMouse(event)
when (event.actionMasked) {
MotionEvent.ACTION_DOWN -> {
sessionTouchDown = true
@@ -4627,19 +4677,14 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
if (event.actionMasked == MotionEvent.ACTION_UP || event.actionMasked == MotionEvent.ACTION_CANCEL) dismissTouch = false
return true
}
val uiBlocking = drawerOpen || achievementsOpen || gestureSettingsOpen || inputSettingsOpen || pipUi || SessionState.suspended
if (penRouted != null) return onPenTouch(event, pen, penRouted)
val uiBlocking = uiBlocksTouch()
if (event.actionMasked == MotionEvent.ACTION_DOWN && !uiBlocking && routedTouchMode != SessionPrefs.TOUCH_GESTURES) {
scrollTouchGesture = scrollTouchMode()
scrollRestX = 0f
scrollRestY = 0f
}
val snapshot = com.droiddeck.launcher.input.TouchRouter.Snapshot(
uiBlocking = uiBlocking,
mode = routedTouchMode,
scrollLayer = scrollTouchGesture,
steamSession = SessionState.mode == SessionService.MODE_STEAM,
steamDesktopUi = SessionState.steamUi == "desktop",
)
val snapshot = fingerTouchSnapshot(uiBlocking)
val route = com.droiddeck.launcher.input.TouchRouter.decide(snapshot)
if (route != touchRoute) {
Log.i(TAG, "touch route $touchRoute -> $route (mode=$routedTouchMode steamUi=${SessionState.steamUi} scrollLayer=$scrollTouchGesture)")
@@ -4667,17 +4712,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
if (threeFingerTap.active || wasActive) return true
}
if (route == com.droiddeck.launcher.input.TouchRouter.Route.TOUCHPAD) {
val rect = (if (pointerLocked) drawnRect() else pointerSurfaceRect()) ?: return false
if (touchpad.bounds != rect) {
val fresh = touchpad.bounds.width() <= 1f
touchpad.bounds = rect
if (fresh) touchpad.place(rect.centerX(), rect.centerY())
}
if (event.actionMasked == MotionEvent.ACTION_DOWN) {
touchpad.desktop = SessionState.mode == SessionService.MODE_DESKTOP
placeTouchpadAtGuest(rect)
}
touchpad.onTouch(event)
touchpadTouch(event)
} else {
dispatchDirectTouch(event)
}
@@ -4685,6 +4720,92 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
}
}
private fun uiBlocksTouch(): Boolean =
drawerOpen || achievementsOpen || gestureSettingsOpen || inputSettingsOpen || pipUi || SessionState.suspended
private fun fingerTouchSnapshot(uiBlocking: Boolean) = com.droiddeck.launcher.input.TouchRouter.Snapshot(
uiBlocking = uiBlocking,
mode = routedTouchMode,
scrollLayer = scrollTouchGesture,
steamSession = SessionState.mode == SessionService.MODE_STEAM,
steamDesktopUi = SessionState.steamUi == "desktop",
)
private fun touchpadTouch(event: MotionEvent): Boolean {
val rect = (if (pointerLocked) drawnRect() else pointerSurfaceRect()) ?: return false
if (touchpad.bounds != rect) {
val fresh = touchpad.bounds.width() <= 1f
touchpad.bounds = rect
if (fresh) touchpad.place(rect.centerX(), rect.centerY())
}
if (event.actionMasked == MotionEvent.ACTION_DOWN) {
touchpad.desktop = SessionState.mode == SessionService.MODE_DESKTOP
placeTouchpadAtGuest(rect)
}
return touchpad.onTouch(event)
}
private fun routePen(event: MotionEvent, pen: Int): PenRouter.Route {
if (!penSeen) notePenSeen()
val action = event.actionMasked
if (action == MotionEvent.ACTION_HOVER_ENTER || action == MotionEvent.ACTION_HOVER_MOVE || action == MotionEvent.ACTION_HOVER_EXIT) penDown = false
val own = PenStream.isPenAction(event, pen)
val starts = own && (action == MotionEvent.ACTION_DOWN || action == MotionEvent.ACTION_POINTER_DOWN)
val route = penRoute.takeIf { penDown && !starts } ?: decidePen()
if (own) penDown = starts
return route
}
private fun decidePen(): PenRouter.Route {
val mode = penSettings.mode
val route = PenRouter.decide(uiBlocksTouch(), mode, glassesOutputActive, programLocksPointer())
if (route != penRoute) {
if (penRoute == PenRouter.Route.PEN) penStream.cancel()
Log.i(TAG, "pen route $penRoute -> $route (mode=$mode)")
penRoute = route
}
return route
}
private fun onPenTouch(event: MotionEvent, pen: Int, route: PenRouter.Route): Boolean {
if (route == PenRouter.Route.RELATIVE && event.pointerCount > 1 &&
com.droiddeck.launcher.input.TouchRouter.decide(fingerTouchSnapshot(uiBlocksTouch())) ==
com.droiddeck.launcher.input.TouchRouter.Route.TOUCHPAD) {
touchpadTouch(event)
return true
}
if (event.pointerCount > 1) {
var fingers = 0
for (index in 0 until event.pointerCount) if (!PenStream.isPen(event, index)) fingers = fingers or (1 shl event.getPointerId(index))
event.pointerSubset(fingers)?.let { split -> try { onTouchEvent(split) } finally { split.recycle() } }
}
when (route) {
PenRouter.Route.PEN -> penSample(event, pen)
PenRouter.Route.RELATIVE -> {
val split = if (event.pointerCount == 1) MotionEvent.obtain(event) else event.pointerSubset(1 shl event.getPointerId(pen))
split?.let {
try {
if (it.actionMasked == MotionEvent.ACTION_UP && it.flags and MotionEvent.FLAG_CANCELED != 0) it.action = MotionEvent.ACTION_CANCEL
touchpadTouch(it)
} finally { it.recycle() }
}
}
else -> {}
}
return true
}
private fun penSample(event: MotionEvent, pen: Int): Boolean {
val area = pointerSurfaceRect() ?: return false
val view = outputView()
if (!penStream.onMotion(event, pen, area, view.width, view.height)) return false
if (penStream.inProximity) {
if (pointerLocked) releaseDesktopMouseGrab()
showCursor(event.getX(pen).coerceIn(area.left, area.right), event.getY(pen).coerceIn(area.top, area.bottom), fromMouse = true)
}
return true
}
private val directTouchSink = object : DirectTouchHold.Touch {
override fun down(pointerId: Int, x: Float, y: Float) = sendDirectTouch(0, pointerId, x, y)
override fun move(pointerId: Int, x: Float, y: Float) = sendDirectTouch(1, pointerId, x, y)
@@ -4770,7 +4891,8 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
}
private fun isHardwareMouse(d: InputDevice?): Boolean =
d != null && !d.isVirtual && (d.supportsSource(InputDevice.SOURCE_MOUSE) || d.supportsSource(InputDevice.SOURCE_MOUSE_RELATIVE))
d != null && !d.isVirtual && !d.supportsSource(InputDevice.SOURCE_STYLUS) &&
(d.supportsSource(InputDevice.SOURCE_MOUSE) || d.supportsSource(InputDevice.SOURCE_MOUSE_RELATIVE))
/** Android also sends the Back and Forward buttons as key events, so they are handled as keys. */
private fun guestMouseButton(actionButton: Int): Int? = when (actionButton) {
@@ -4809,7 +4931,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
pointerLocked && SessionState.mode != SessionService.MODE_DESKTOP
private fun desktopMouseGrabAllowed(): Boolean =
SessionState.mode == SessionService.MODE_DESKTOP && relativeMouseAttached() && !bottomOutputActive
SessionState.mode == SessionService.MODE_DESKTOP && !penStream.inProximity && relativeMouseAttached() && !bottomOutputActive
private fun tapDesktopGrabKey() {
val now = android.os.SystemClock.uptimeMillis()
@@ -5104,6 +5226,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
private fun releaseHeldGuestInput() {
mouseBackInput.cancel()
cancelPen(releaseKey = true)
heldButtons.release(1 or 2 or 8 or 16)
guestKeysDown.toList().forEach { sendGuestKey(it, false) }
heldKeys.release(8 or 16 or 32)
@@ -5195,6 +5318,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
scrollTouch.cancel()
directTouchHold.cancel()
twoFingerGestures.cancel()
cancelPen()
guestKeysDown.toList().forEach { sendGuestKey(it, false) }
heldKeys.clear()
heldButtons.clear()
@@ -5391,6 +5515,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
if (::touchpad.isInitialized) touchpad.cancel()
if (::directTouchHold.isInitialized) directTouchHold.cancel()
if (::twoFingerGestures.isInitialized) twoFingerGestures.cancel()
cancelPen(releaseKey = true)
uiHandler.removeCallbacks(clearRecognizedGesture)
uiHandler.removeCallbacks(cursorResync)
val screenOff = screenIsOff()
@@ -5697,7 +5822,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
}
if (shown != null) {
closeSecondScreen(reset = false)
val screen = com.droiddeck.launcher.ui.BottomGameScreen(this, shown, this, ::onBottomGameTouch)
val screen = com.droiddeck.launcher.ui.BottomGameScreen(this, shown, this, ::onBottomGameTouch, ::onBottomGameHover)
bottomGame = screen
surfaceW = 0
surfaceH = 0
@@ -5749,9 +5874,23 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
if (surfaceView.holder.surface?.isValid == true) surfaceCreated(surfaceView.holder)
}
private fun onBottomGameHover(event: MotionEvent): Boolean {
val pen = PenStream.penIndex(event)
if (pipUi || pen < 0) return false
return when (routePen(event, pen)) {
PenRouter.Route.TOUCH -> false
PenRouter.Route.PEN -> {
notePress(event)
penSample(event, pen)
true
}
else -> true
}
}
private fun onBottomGameTouch(event: MotionEvent): Boolean {
if (pipUi) return true
if (!event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)) return false
if (!event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN) && PenStream.penIndex(event) < 0) return false
notePress(event)
if (event.actionMasked == MotionEvent.ACTION_DOWN) {
drawerControllerActive = false
@@ -6052,6 +6191,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback, com.droidde
padBridge?.stop()
DeckControls.clearRouter(inputRouter)
inputRouter = null
cancelPen()
heldKeys.clear()
heldButtons.clear()
steamController?.stop()