fix: keep mapped desktop pointer visible and update it each frame

This commit is contained in:
delete committed 2026-10-24 01:43:34 +03:00
1 parent 45a496b802
commit 3dd6aeef46
7 files changed
+258 -9

No files matched your search

@@ -165,6 +165,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
private var physicalInputBindings: com.droiddeck.launcher.input.PhysicalInputBindings? = null
private var inputSettingsOpen by mutableStateOf(false)
@Volatile private var inputUsesLocalActions = false
private var inputUsesPointer = false
private var inputProfile = InputProfile.defaults(InputProfile.GAME)
private val inputMotion = InputMotion { inputRouter }
private val heldKeys = HeldInput { code, down -> WaylandCompositor.nativeSendKey(code, if (down) 1 else 0) }
@@ -172,7 +173,13 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
private val inputOutput = object : InputOutput {
override fun key(code: Int, down: Boolean) { heldKeys.set(4, code, down) }
override fun button(code: Int, down: Boolean) { heldButtons.set(4, code, down) }
override fun pointer(dx: Float, dy: Float) { WaylandCompositor.sendPointerDelta(dx, dy) }
override fun pointer(dx: Float, dy: Float) {
if (Looper.myLooper() == Looper.getMainLooper()) moveInputPointer(dx, dy)
else uiHandler.post {
if (resumed && !SessionState.suspended && !pipUi && !drawerOpen && !inputSettingsOpen)
moveInputPointer(dx, dy)
}
}
override fun scroll(steps: Int) { WaylandCompositor.nativeSendSceneInput(4, steps, 0) }
override fun requestMotion() { inputMotion.request() }
override fun command(command: InputCommand) {
@@ -568,11 +575,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
}
SessionState.deckPadListener = deckPadListener
openDeckRing("activity created")
bridge.setOnPlayerInput {
lastPadInputMs = android.os.SystemClock.uptimeMillis()
uiHandler.removeCallbacks(cursorHide)
cursorVisible = false
}
bridge.setOnPlayerInput(::onPadInput)
onScreenControls = OnScreenControls(this, bridge, onKeyboard = ::togglePcKeyboard).also { root.addView(it) }
keyboard = KeyboardHost(this).also { root.addView(it) }
touchpad = TouchpadGestures(PointerGestures.slop(this), pointerListener)
@@ -2429,13 +2432,36 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
showCursor(cx, cy)
}
private fun moveInputPointer(dx: Float, dy: Float) {
if (pointerLocked) {
WaylandCompositor.sendPointerDelta(dx, dy)
cursorVisible = false
return
}
val rect = drawnRect() ?: return
val position = cursorPos
val x = if (position.x >= 0f) position.x.coerceIn(rect.left, rect.right) else rect.centerX()
val y = if (position.y >= 0f) position.y.coerceIn(rect.top, rect.bottom) else rect.centerY()
val out = SessionState.outputSize
movePointer(x + dx * rect.width() / out.first, y + dy * rect.height() / out.second)
// Make frame-callback updates visible to drawing before this frame's traversal.
androidx.compose.runtime.snapshots.Snapshot.sendApplyNotifications()
}
private fun onPadInput() {
if (inputUsesPointer && !pointerLocked) return
lastPadInputMs = android.os.SystemClock.uptimeMillis()
uiHandler.removeCallbacks(cursorHide)
cursorVisible = false
}
private fun showCursor(x: Float, y: Float) {
cursorPos = androidx.compose.ui.geometry.Offset(x, y)
syncClientCursor()
uiHandler.removeCallbacks(cursorResync)
uiHandler.postDelayed(cursorResync, 60)
if (clientHidesCursor) { cursorVisible = false; return }
if (android.os.SystemClock.uptimeMillis() - lastPadInputMs < CURSOR_PAD_HOLD_MS) return
if (!inputUsesPointer && android.os.SystemClock.uptimeMillis() - lastPadInputMs < CURSOR_PAD_HOLD_MS) return
cursorVisible = true
uiHandler.removeCallbacks(cursorHide)
if (SessionState.mode == SessionService.MODE_STEAM && SessionState.steamUi != "desktop") uiHandler.postDelayed(cursorHide, 2500)
@@ -2535,6 +2561,11 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
onQam = ::tapQam,
onClose = { selectSecondScreenMode(SecondScreenMode.NONE) },
pad = padBridge,
pointerPosition = {
drawnRect()?.takeIf { cursorPos.x >= 0f && cursorPos.y >= 0f }?.let {
androidx.compose.ui.geometry.Offset((cursorPos.x - it.left) / it.width(), (cursorPos.y - it.top) / it.height())
}
},
)
secondScreenPresentation = presentation
}
@@ -2557,6 +2588,9 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
private fun refreshInputProfile() {
inputProfile = InputProfiles.active(this, loadingMode() == SessionService.MODE_DESKTOP)
inputUsesLocalActions = InputControl.padControls.any { inputProfile.action(InputRole.PHYSICAL, it) !is InputAction.Native }
inputUsesPointer = inputProfile.bindings.values.any { bindings ->
bindings.values.any { it is InputAction.Pointer || it is InputAction.MouseButton || it is InputAction.Scroll }
} || inputProfile.keys.any { it.action is InputAction.MouseButton }
physicalInputBindings?.configure(inputProfile.keys)
padBridge?.setInputProfile(inputProfile)
DeckControls.refresh()
@@ -2724,6 +2758,8 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
touchpad.bounds = rect
if (fresh) touchpad.place(rect.centerX(), rect.centerY())
}
if (event.actionMasked == MotionEvent.ACTION_DOWN && !pointerLocked && cursorPos.x >= 0f && cursorPos.y >= 0f)
touchpad.place(cursorPos.x, cursorPos.y)
return touchpad.onTouch(event)
}
val rect = drawnRect() ?: return false
@@ -79,7 +79,7 @@ class InputRouter(private val output: InputOutput) {
val cy = if (abs(channel.y) > 0.12f) channel.y else 0f
when (val action = channel.action) {
is InputAction.Pointer -> { x += cx * action.speed; y += cy * action.speed }
is InputAction.Scroll -> scroll -= cy * action.speed
is InputAction.Scroll -> scroll += cy * action.speed
else -> Unit
}
}
@@ -360,6 +360,8 @@ class DeckControlsPanel(
drawSurface(canvas, down)
val cx = width / 2f
labelPaint.textSize = min(height * 0.3f, 30 * density)
val labelWidth = labelPaint.measureText(label)
if (labelWidth > width * 0.84f) labelPaint.textSize *= width * 0.84f / labelWidth
labelPaint.color = if (down) ACCENT else TEXT
val labelY = height / 2f + labelPaint.textSize * (if (caption == null) 0.35f else 0.1f)
canvas.drawText(label, cx, labelY, labelPaint)
@@ -49,6 +49,7 @@ class SecondScreenPresentation(
private val onQam: () -> Unit,
private val onClose: () -> Unit,
private val pad: PadBridge? = null,
private val pointerPosition: () -> androidx.compose.ui.geometry.Offset? = { null },
) : Presentation(sessionActivity, display) {
private val root = LinearLayout(context).apply {
orientation = LinearLayout.VERTICAL
@@ -356,7 +357,12 @@ class SecondScreenPresentation(
canvas.drawColor(Color.BLACK)
}
override fun onTouchEvent(event: MotionEvent): Boolean = gestures.onTouch(event)
override fun onTouchEvent(event: MotionEvent): Boolean {
if (event.actionMasked == MotionEvent.ACTION_DOWN) pointerPosition()?.let {
gestures.place(it.x * width, it.y * height)
}
return gestures.onTouch(event)
}
}
private inner class TerminalClient : TerminalSessionClient, TerminalViewClient {
@@ -117,6 +117,35 @@ class InputRouterTest {
assertEquals(1, output.events.size)
}
@Test fun stickScrollFollowsUpAndDownForEveryInputRoleAndStopsOnRelease() {
for (role in InputRole.entries) {
router.setProfile(bind(InputControl.RS, InputAction.Scroll(10f), role))
val channel = router.channel(role, InputControl.RS)
output.events.clear()
channel.update(0f, -1f)
repeat(2) { assertTrue(router.tick(.05f)) }
assertEquals(listOf("motion", "scroll -1"), output.events)
channel.update(0f, 1f)
repeat(2) { assertTrue(router.tick(.05f)) }
assertEquals("scroll 1", output.events.last())
channel.release()
val count = output.events.size
assertFalse(router.tick(.05f))
assertEquals(count, output.events.size)
channel.close()
}
}
@Test fun trackpadScrollKeepsItsContactDirectionAndDoesNotScheduleStickMotion() {
router.setProfile(bind(InputControl.LEFT_PAD, InputAction.Scroll(10f), InputRole.SECONDARY))
val channel = router.channel(InputRole.SECONDARY, InputControl.LEFT_PAD)
channel.update(0f, 0f, true)
channel.update(0f, .5f, true)
channel.update(0f, 0f, true)
assertEquals(listOf("scroll 5", "scroll -5"), output.events)
assertFalse(router.tick(.05f))
}
@Test fun disablingRoutingReleasesKeysAndBlocksHeldInputOnResume() {
router.setProfile(bind(InputControl.A, InputAction.Keys(listOf(1))))
val source = router.pad(InputRole.PHYSICAL)
@@ -0,0 +1,149 @@
package com.droiddeck.launcher.input
import android.os.Looper
import android.view.SurfaceView
import androidx.compose.runtime.MutableState
import androidx.compose.runtime.snapshots.Snapshot
import androidx.compose.ui.geometry.Offset
import com.droiddeck.launcher.SessionActivity
import com.droiddeck.launcher.session.SessionService
import com.droiddeck.launcher.session.SessionState
import com.droiddeck.launcher.wayland.WaylandCompositor
import org.junit.After
import org.junit.Assert.*
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.Robolectric
import org.robolectric.RobolectricTestRunner
import org.robolectric.Shadows.shadowOf
import org.robolectric.annotation.Config
import org.robolectric.annotation.Implementation
import org.robolectric.annotation.Implements
import org.robolectric.util.ReflectionHelpers
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [28], manifest = Config.NONE, shadows = [SessionInputPointerTest.CompositorShadow::class, SessionInputPointerTest.WriterShadow::class])
class SessionInputPointerTest {
private lateinit var activity: SessionActivity
private lateinit var output: InputOutput
private var previousSize = SessionState.outputSize
private var previousMode = SessionState.mode
@Before fun setUp() {
activity = Robolectric.buildActivity(SessionActivity::class.java).get()
output = ReflectionHelpers.getField(activity, "inputOutput")
ReflectionHelpers.setField(activity, "surfaceView", SurfaceView(activity).apply { layout(0, 0, 1000, 600) })
SessionState.outputSize = 1600 to 900
SessionState.mode = SessionService.MODE_DESKTOP
InputProfiles.resetAll(activity)
InputProfiles.select(activity, InputProfile.DESKTOP)
refresh()
CompositorShadow.pointers.clear()
CompositorShadow.events.clear()
}
@After fun tearDown() {
shadowOf(Looper.getMainLooper()).idle()
SessionState.outputSize = previousSize
SessionState.mode = previousMode
}
@Test fun desktopMouseButtonsKeepTheVisibleCursorAndDeliverImmediately() {
state<Offset>("cursorPos").value = Offset(500f, 300f)
state<Boolean>("cursorVisible").value = true
output.button(272, true)
invoke("onPadInput")
output.button(272, false)
assertTrue(state<Boolean>("cursorVisible").value)
assertEquals(listOf(Triple(3, 272, 1), Triple(3, 272, 0)), CompositorShadow.events)
}
@Test fun nativeGameInputStillHidesCursor() {
InputProfiles.select(activity, InputProfile.GAME)
refresh()
state<Boolean>("cursorVisible").value = true
invoke("onPadInput")
assertFalse(state<Boolean>("cursorVisible").value)
}
@Test fun lowerScreenPointerBindingAlsoKeepsCursorVisible() {
InputProfiles.save(activity, InputProfile(InputProfile.GAME).withBinding(InputRole.SECONDARY, InputControl.RIGHT_PAD, InputAction.Pointer()))
InputProfiles.select(activity, InputProfile.GAME)
refresh()
state<Boolean>("cursorVisible").value = true
invoke("onPadInput")
assertTrue(state<Boolean>("cursorVisible").value)
}
@Test fun stickContinuesFromTrackpadPositionAndUpdatesVisibleAndGuestCoordinates() {
ReflectionHelpers.callInstanceMethod<Unit>(activity, "moveSecondScreenPointer",
ReflectionHelpers.ClassParameter.from(Float::class.javaPrimitiveType, 250f),
ReflectionHelpers.ClassParameter.from(Float::class.javaPrimitiveType, 150f),
ReflectionHelpers.ClassParameter.from(Float::class.javaPrimitiveType, 1000f),
ReflectionHelpers.ClassParameter.from(Float::class.javaPrimitiveType, 600f))
output.pointer(160f, 90f)
invoke("onPadInput")
assertEquals(Offset(350f, 215.625f), state<Offset>("cursorPos").value)
assertTrue(state<Boolean>("cursorVisible").value)
assertEquals(Triple(1, InputSpace.x(350f, 1000), InputSpace.y(215.625f, 600)), CompositorShadow.pointers.last())
}
@Test fun firstStickMovementStartsAtCenterAndClampsToPictureBounds() {
output.pointer(160f, 90f)
assertEquals(Offset(600f, 356.25f), state<Offset>("cursorPos").value)
output.pointer(10000f, -10000f)
assertEquals(Offset(1000f, 18.75f), state<Offset>("cursorPos").value)
}
@Test fun stickFrameNotifiesCursorDrawingBeforeReturningToTheLooper() {
val position = state<Offset>("cursorPos")
Snapshot.sendApplyNotifications()
var changed = false
val observer = Snapshot.registerApplyObserver { objects, _ ->
if (position in objects) changed = true
}
try {
output.pointer(20f, 10f)
assertTrue(changed)
} finally { observer.dispose() }
}
@Test fun pointerLockKeepsRelativeMotionAndHiddenCursor() {
ReflectionHelpers.setField(activity, "pointerLocked", true)
state<Boolean>("cursorVisible").value = true
output.pointer(2.5f, -1.25f)
assertEquals(listOf(Triple(6, 640, -320)), CompositorShadow.events)
assertTrue(CompositorShadow.pointers.isEmpty())
assertFalse(state<Boolean>("cursorVisible").value)
}
@Test fun clientRequestedHiddenCursorStaysHidden() {
ReflectionHelpers.setField(activity, "clientHidesCursor", true)
output.pointer(1f, 1f)
assertFalse(state<Boolean>("cursorVisible").value)
}
private fun refresh() = invoke("refreshInputProfile")
private fun invoke(name: String) = ReflectionHelpers.callInstanceMethod<Unit>(activity, name)
private fun <T> state(name: String): MutableState<T> = ReflectionHelpers.getField(activity, "$name\$delegate")
@Implements(value = WaylandCompositor::class, isInAndroidSdk = false)
class CompositorShadow {
companion object {
val pointers = mutableListOf<Triple<Int, Int, Int>>()
val events = mutableListOf<Triple<Int, Int, Int>>()
@JvmStatic @Implementation fun __staticInitializer__() {}
@JvmStatic @Implementation fun nativeSendPointer(action: Int, x: Int, y: Int) { pointers += Triple(action, x, y) }
@JvmStatic @Implementation fun nativeSendSceneInput(type: Int, code: Int, value: Int) { events += Triple(type, code, value) }
}
}
@Implements(value = FakeInputWriter::class, isInAndroidSdk = false)
class WriterShadow {
companion object {
@JvmStatic @Implementation fun __staticInitializer__() {}
@JvmStatic @Implementation fun writeDeckControls(index: Int, mask: Int, pads: ShortArray, pressure: ShortArray) {}
}
}
}
@@ -5,11 +5,13 @@ import android.hardware.input.InputManager
import android.view.Display
import android.view.InputDevice
import android.view.KeyEvent
import android.view.MotionEvent
import android.view.View
import android.view.ViewGroup
import android.widget.Button
import androidx.compose.ui.test.*
import androidx.compose.runtime.MutableState
import androidx.compose.ui.geometry.Offset
import com.droiddeck.launcher.R
import com.droiddeck.launcher.SessionActivity
import com.droiddeck.launcher.gpu.FrameGen
@@ -75,6 +77,31 @@ class SessionSecondScreenTest {
@Test fun steamTrackpadKeepsWorkingSteamShortcuts() = checkShortcuts(steam = true)
@Test fun trackpadTapUsesPositionLeftByAnotherPointerSource() {
val activity = Robolectric.buildActivity(SessionActivity::class.java).get()
val display = activity.getSystemService(DisplayManager::class.java).getDisplay(Display.DEFAULT_DISPLAY)
val positions = mutableListOf<Pair<Float, Float>>()
var position = Offset(0.2f, 0.7f)
val panel = SecondScreenPresentation(activity, display,
{ x, y, w, h -> positions += x / w to y / h }, { _, _ -> }, {}, {}, {}, {},
pointerPosition = { position })
try {
panel.create()
panel.showMode(SecondScreenMode.KEYBOARD_TRACKPAD)
val trackpad = descendants(panel.window!!.decorView).first { it.javaClass.simpleName == "TrackpadView" }
trackpad.layout(0, 0, 1000, 600)
for (next in listOf(Offset(0.2f, 0.7f), Offset(0.8f, 0.1f))) {
position = next
for ((action, time) in listOf(MotionEvent.ACTION_DOWN to 1000L, MotionEvent.ACTION_UP to 1010L)) {
val event = MotionEvent.obtain(1000, time, action, 300f, 300f, 0)
try { trackpad.onTouchEvent(event) } finally { event.recycle() }
}
assertEquals(next.x, positions.last().first, 0.001f)
assertEquals(next.y, positions.last().second, 0.001f)
}
} finally { panel.closeControls() }
}
@Test
@Config(sdk = [34], shadows = [PadBridgeRoutingTest.RegisteredKeyEvent::class])
fun controllerBumpersReachTheSecondScreenInBothSessionModes() {