diff --git a/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt b/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt index 489a918479..3fff669d3c 100644 --- a/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt +++ b/app/src/main/java/com/droiddeck/launcher/SessionActivity.kt @@ -165,6 +165,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { private var physicalInputBindings: com.droiddeck.launcher.input.PhysicalInputBindings? = null private var inputSettingsOpen by mutableStateOf(false) @Volatile private var inputUsesLocalActions = false + private var inputUsesPointer = false private var inputProfile = InputProfile.defaults(InputProfile.GAME) private val inputMotion = InputMotion { inputRouter } private val heldKeys = HeldInput { code, down -> WaylandCompositor.nativeSendKey(code, if (down) 1 else 0) } @@ -172,7 +173,13 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { private val inputOutput = object : InputOutput { override fun key(code: Int, down: Boolean) { heldKeys.set(4, code, down) } override fun button(code: Int, down: Boolean) { heldButtons.set(4, code, down) } - override fun pointer(dx: Float, dy: Float) { WaylandCompositor.sendPointerDelta(dx, dy) } + override fun pointer(dx: Float, dy: Float) { + if (Looper.myLooper() == Looper.getMainLooper()) moveInputPointer(dx, dy) + else uiHandler.post { + if (resumed && !SessionState.suspended && !pipUi && !drawerOpen && !inputSettingsOpen) + moveInputPointer(dx, dy) + } + } override fun scroll(steps: Int) { WaylandCompositor.nativeSendSceneInput(4, steps, 0) } override fun requestMotion() { inputMotion.request() } override fun command(command: InputCommand) { @@ -568,11 +575,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { } SessionState.deckPadListener = deckPadListener openDeckRing("activity created") - bridge.setOnPlayerInput { - lastPadInputMs = android.os.SystemClock.uptimeMillis() - uiHandler.removeCallbacks(cursorHide) - cursorVisible = false - } + bridge.setOnPlayerInput(::onPadInput) onScreenControls = OnScreenControls(this, bridge, onKeyboard = ::togglePcKeyboard).also { root.addView(it) } keyboard = KeyboardHost(this).also { root.addView(it) } touchpad = TouchpadGestures(PointerGestures.slop(this), pointerListener) @@ -2429,13 +2432,36 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { showCursor(cx, cy) } + private fun moveInputPointer(dx: Float, dy: Float) { + if (pointerLocked) { + WaylandCompositor.sendPointerDelta(dx, dy) + cursorVisible = false + return + } + val rect = drawnRect() ?: return + val position = cursorPos + val x = if (position.x >= 0f) position.x.coerceIn(rect.left, rect.right) else rect.centerX() + val y = if (position.y >= 0f) position.y.coerceIn(rect.top, rect.bottom) else rect.centerY() + val out = SessionState.outputSize + movePointer(x + dx * rect.width() / out.first, y + dy * rect.height() / out.second) + // Make frame-callback updates visible to drawing before this frame's traversal. + androidx.compose.runtime.snapshots.Snapshot.sendApplyNotifications() + } + + private fun onPadInput() { + if (inputUsesPointer && !pointerLocked) return + lastPadInputMs = android.os.SystemClock.uptimeMillis() + uiHandler.removeCallbacks(cursorHide) + cursorVisible = false + } + private fun showCursor(x: Float, y: Float) { cursorPos = androidx.compose.ui.geometry.Offset(x, y) syncClientCursor() uiHandler.removeCallbacks(cursorResync) uiHandler.postDelayed(cursorResync, 60) if (clientHidesCursor) { cursorVisible = false; return } - if (android.os.SystemClock.uptimeMillis() - lastPadInputMs < CURSOR_PAD_HOLD_MS) return + if (!inputUsesPointer && android.os.SystemClock.uptimeMillis() - lastPadInputMs < CURSOR_PAD_HOLD_MS) return cursorVisible = true uiHandler.removeCallbacks(cursorHide) if (SessionState.mode == SessionService.MODE_STEAM && SessionState.steamUi != "desktop") uiHandler.postDelayed(cursorHide, 2500) @@ -2535,6 +2561,11 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { onQam = ::tapQam, onClose = { selectSecondScreenMode(SecondScreenMode.NONE) }, pad = padBridge, + pointerPosition = { + drawnRect()?.takeIf { cursorPos.x >= 0f && cursorPos.y >= 0f }?.let { + androidx.compose.ui.geometry.Offset((cursorPos.x - it.left) / it.width(), (cursorPos.y - it.top) / it.height()) + } + }, ) secondScreenPresentation = presentation } @@ -2557,6 +2588,9 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { private fun refreshInputProfile() { inputProfile = InputProfiles.active(this, loadingMode() == SessionService.MODE_DESKTOP) inputUsesLocalActions = InputControl.padControls.any { inputProfile.action(InputRole.PHYSICAL, it) !is InputAction.Native } + inputUsesPointer = inputProfile.bindings.values.any { bindings -> + bindings.values.any { it is InputAction.Pointer || it is InputAction.MouseButton || it is InputAction.Scroll } + } || inputProfile.keys.any { it.action is InputAction.MouseButton } physicalInputBindings?.configure(inputProfile.keys) padBridge?.setInputProfile(inputProfile) DeckControls.refresh() @@ -2724,6 +2758,8 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback { touchpad.bounds = rect if (fresh) touchpad.place(rect.centerX(), rect.centerY()) } + if (event.actionMasked == MotionEvent.ACTION_DOWN && !pointerLocked && cursorPos.x >= 0f && cursorPos.y >= 0f) + touchpad.place(cursorPos.x, cursorPos.y) return touchpad.onTouch(event) } val rect = drawnRect() ?: return false diff --git a/app/src/main/java/com/droiddeck/launcher/input/InputRouter.kt b/app/src/main/java/com/droiddeck/launcher/input/InputRouter.kt index 921ca05d82..52af7ec41b 100644 --- a/app/src/main/java/com/droiddeck/launcher/input/InputRouter.kt +++ b/app/src/main/java/com/droiddeck/launcher/input/InputRouter.kt @@ -79,7 +79,7 @@ class InputRouter(private val output: InputOutput) { val cy = if (abs(channel.y) > 0.12f) channel.y else 0f when (val action = channel.action) { is InputAction.Pointer -> { x += cx * action.speed; y += cy * action.speed } - is InputAction.Scroll -> scroll -= cy * action.speed + is InputAction.Scroll -> scroll += cy * action.speed else -> Unit } } diff --git a/app/src/main/java/com/droiddeck/launcher/ui/DeckControlsPanel.kt b/app/src/main/java/com/droiddeck/launcher/ui/DeckControlsPanel.kt index 2cc80047af..c17ce567ff 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/DeckControlsPanel.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/DeckControlsPanel.kt @@ -360,6 +360,8 @@ class DeckControlsPanel( drawSurface(canvas, down) val cx = width / 2f labelPaint.textSize = min(height * 0.3f, 30 * density) + val labelWidth = labelPaint.measureText(label) + if (labelWidth > width * 0.84f) labelPaint.textSize *= width * 0.84f / labelWidth labelPaint.color = if (down) ACCENT else TEXT val labelY = height / 2f + labelPaint.textSize * (if (caption == null) 0.35f else 0.1f) canvas.drawText(label, cx, labelY, labelPaint) diff --git a/app/src/main/java/com/droiddeck/launcher/ui/SecondScreenPresentation.kt b/app/src/main/java/com/droiddeck/launcher/ui/SecondScreenPresentation.kt index 2ed7b1173e..44f3ac58f4 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/SecondScreenPresentation.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/SecondScreenPresentation.kt @@ -49,6 +49,7 @@ class SecondScreenPresentation( private val onQam: () -> Unit, private val onClose: () -> Unit, private val pad: PadBridge? = null, + private val pointerPosition: () -> androidx.compose.ui.geometry.Offset? = { null }, ) : Presentation(sessionActivity, display) { private val root = LinearLayout(context).apply { orientation = LinearLayout.VERTICAL @@ -356,7 +357,12 @@ class SecondScreenPresentation( canvas.drawColor(Color.BLACK) } - override fun onTouchEvent(event: MotionEvent): Boolean = gestures.onTouch(event) + override fun onTouchEvent(event: MotionEvent): Boolean { + if (event.actionMasked == MotionEvent.ACTION_DOWN) pointerPosition()?.let { + gestures.place(it.x * width, it.y * height) + } + return gestures.onTouch(event) + } } private inner class TerminalClient : TerminalSessionClient, TerminalViewClient { diff --git a/app/src/test/java/com/droiddeck/launcher/input/InputRouterTest.kt b/app/src/test/java/com/droiddeck/launcher/input/InputRouterTest.kt index 709861d5ea..2ad592fc4a 100644 --- a/app/src/test/java/com/droiddeck/launcher/input/InputRouterTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/input/InputRouterTest.kt @@ -117,6 +117,35 @@ class InputRouterTest { assertEquals(1, output.events.size) } + @Test fun stickScrollFollowsUpAndDownForEveryInputRoleAndStopsOnRelease() { + for (role in InputRole.entries) { + router.setProfile(bind(InputControl.RS, InputAction.Scroll(10f), role)) + val channel = router.channel(role, InputControl.RS) + output.events.clear() + channel.update(0f, -1f) + repeat(2) { assertTrue(router.tick(.05f)) } + assertEquals(listOf("motion", "scroll -1"), output.events) + channel.update(0f, 1f) + repeat(2) { assertTrue(router.tick(.05f)) } + assertEquals("scroll 1", output.events.last()) + channel.release() + val count = output.events.size + assertFalse(router.tick(.05f)) + assertEquals(count, output.events.size) + channel.close() + } + } + + @Test fun trackpadScrollKeepsItsContactDirectionAndDoesNotScheduleStickMotion() { + router.setProfile(bind(InputControl.LEFT_PAD, InputAction.Scroll(10f), InputRole.SECONDARY)) + val channel = router.channel(InputRole.SECONDARY, InputControl.LEFT_PAD) + channel.update(0f, 0f, true) + channel.update(0f, .5f, true) + channel.update(0f, 0f, true) + assertEquals(listOf("scroll 5", "scroll -5"), output.events) + assertFalse(router.tick(.05f)) + } + @Test fun disablingRoutingReleasesKeysAndBlocksHeldInputOnResume() { router.setProfile(bind(InputControl.A, InputAction.Keys(listOf(1)))) val source = router.pad(InputRole.PHYSICAL) diff --git a/app/src/test/java/com/droiddeck/launcher/input/SessionInputPointerTest.kt b/app/src/test/java/com/droiddeck/launcher/input/SessionInputPointerTest.kt new file mode 100644 index 0000000000..4f14310f0b --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/input/SessionInputPointerTest.kt @@ -0,0 +1,149 @@ +package com.droiddeck.launcher.input + +import android.os.Looper +import android.view.SurfaceView +import androidx.compose.runtime.MutableState +import androidx.compose.runtime.snapshots.Snapshot +import androidx.compose.ui.geometry.Offset +import com.droiddeck.launcher.SessionActivity +import com.droiddeck.launcher.session.SessionService +import com.droiddeck.launcher.session.SessionState +import com.droiddeck.launcher.wayland.WaylandCompositor +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.annotation.Implementation +import org.robolectric.annotation.Implements +import org.robolectric.util.ReflectionHelpers + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [28], manifest = Config.NONE, shadows = [SessionInputPointerTest.CompositorShadow::class, SessionInputPointerTest.WriterShadow::class]) +class SessionInputPointerTest { + private lateinit var activity: SessionActivity + private lateinit var output: InputOutput + private var previousSize = SessionState.outputSize + private var previousMode = SessionState.mode + + @Before fun setUp() { + activity = Robolectric.buildActivity(SessionActivity::class.java).get() + output = ReflectionHelpers.getField(activity, "inputOutput") + ReflectionHelpers.setField(activity, "surfaceView", SurfaceView(activity).apply { layout(0, 0, 1000, 600) }) + SessionState.outputSize = 1600 to 900 + SessionState.mode = SessionService.MODE_DESKTOP + InputProfiles.resetAll(activity) + InputProfiles.select(activity, InputProfile.DESKTOP) + refresh() + CompositorShadow.pointers.clear() + CompositorShadow.events.clear() + } + + @After fun tearDown() { + shadowOf(Looper.getMainLooper()).idle() + SessionState.outputSize = previousSize + SessionState.mode = previousMode + } + + @Test fun desktopMouseButtonsKeepTheVisibleCursorAndDeliverImmediately() { + state("cursorPos").value = Offset(500f, 300f) + state("cursorVisible").value = true + output.button(272, true) + invoke("onPadInput") + output.button(272, false) + assertTrue(state("cursorVisible").value) + assertEquals(listOf(Triple(3, 272, 1), Triple(3, 272, 0)), CompositorShadow.events) + } + + @Test fun nativeGameInputStillHidesCursor() { + InputProfiles.select(activity, InputProfile.GAME) + refresh() + state("cursorVisible").value = true + invoke("onPadInput") + assertFalse(state("cursorVisible").value) + } + + @Test fun lowerScreenPointerBindingAlsoKeepsCursorVisible() { + InputProfiles.save(activity, InputProfile(InputProfile.GAME).withBinding(InputRole.SECONDARY, InputControl.RIGHT_PAD, InputAction.Pointer())) + InputProfiles.select(activity, InputProfile.GAME) + refresh() + state("cursorVisible").value = true + invoke("onPadInput") + assertTrue(state("cursorVisible").value) + } + + @Test fun stickContinuesFromTrackpadPositionAndUpdatesVisibleAndGuestCoordinates() { + ReflectionHelpers.callInstanceMethod(activity, "moveSecondScreenPointer", + ReflectionHelpers.ClassParameter.from(Float::class.javaPrimitiveType, 250f), + ReflectionHelpers.ClassParameter.from(Float::class.javaPrimitiveType, 150f), + ReflectionHelpers.ClassParameter.from(Float::class.javaPrimitiveType, 1000f), + ReflectionHelpers.ClassParameter.from(Float::class.javaPrimitiveType, 600f)) + output.pointer(160f, 90f) + invoke("onPadInput") + assertEquals(Offset(350f, 215.625f), state("cursorPos").value) + assertTrue(state("cursorVisible").value) + assertEquals(Triple(1, InputSpace.x(350f, 1000), InputSpace.y(215.625f, 600)), CompositorShadow.pointers.last()) + } + + @Test fun firstStickMovementStartsAtCenterAndClampsToPictureBounds() { + output.pointer(160f, 90f) + assertEquals(Offset(600f, 356.25f), state("cursorPos").value) + output.pointer(10000f, -10000f) + assertEquals(Offset(1000f, 18.75f), state("cursorPos").value) + } + + @Test fun stickFrameNotifiesCursorDrawingBeforeReturningToTheLooper() { + val position = state("cursorPos") + Snapshot.sendApplyNotifications() + var changed = false + val observer = Snapshot.registerApplyObserver { objects, _ -> + if (position in objects) changed = true + } + try { + output.pointer(20f, 10f) + assertTrue(changed) + } finally { observer.dispose() } + } + + @Test fun pointerLockKeepsRelativeMotionAndHiddenCursor() { + ReflectionHelpers.setField(activity, "pointerLocked", true) + state("cursorVisible").value = true + output.pointer(2.5f, -1.25f) + assertEquals(listOf(Triple(6, 640, -320)), CompositorShadow.events) + assertTrue(CompositorShadow.pointers.isEmpty()) + assertFalse(state("cursorVisible").value) + } + + @Test fun clientRequestedHiddenCursorStaysHidden() { + ReflectionHelpers.setField(activity, "clientHidesCursor", true) + output.pointer(1f, 1f) + assertFalse(state("cursorVisible").value) + } + + private fun refresh() = invoke("refreshInputProfile") + private fun invoke(name: String) = ReflectionHelpers.callInstanceMethod(activity, name) + private fun state(name: String): MutableState = ReflectionHelpers.getField(activity, "$name\$delegate") + + @Implements(value = WaylandCompositor::class, isInAndroidSdk = false) + class CompositorShadow { + companion object { + val pointers = mutableListOf>() + val events = mutableListOf>() + @JvmStatic @Implementation fun __staticInitializer__() {} + @JvmStatic @Implementation fun nativeSendPointer(action: Int, x: Int, y: Int) { pointers += Triple(action, x, y) } + @JvmStatic @Implementation fun nativeSendSceneInput(type: Int, code: Int, value: Int) { events += Triple(type, code, value) } + } + } + + @Implements(value = FakeInputWriter::class, isInAndroidSdk = false) + class WriterShadow { + companion object { + @JvmStatic @Implementation fun __staticInitializer__() {} + @JvmStatic @Implementation fun writeDeckControls(index: Int, mask: Int, pads: ShortArray, pressure: ShortArray) {} + } + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/ui/SessionSecondScreenTest.kt b/app/src/test/java/com/droiddeck/launcher/ui/SessionSecondScreenTest.kt index 04d3f5632b..dfccf965db 100644 --- a/app/src/test/java/com/droiddeck/launcher/ui/SessionSecondScreenTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/ui/SessionSecondScreenTest.kt @@ -5,11 +5,13 @@ import android.hardware.input.InputManager import android.view.Display import android.view.InputDevice import android.view.KeyEvent +import android.view.MotionEvent import android.view.View import android.view.ViewGroup import android.widget.Button import androidx.compose.ui.test.* import androidx.compose.runtime.MutableState +import androidx.compose.ui.geometry.Offset import com.droiddeck.launcher.R import com.droiddeck.launcher.SessionActivity import com.droiddeck.launcher.gpu.FrameGen @@ -75,6 +77,31 @@ class SessionSecondScreenTest { @Test fun steamTrackpadKeepsWorkingSteamShortcuts() = checkShortcuts(steam = true) + @Test fun trackpadTapUsesPositionLeftByAnotherPointerSource() { + val activity = Robolectric.buildActivity(SessionActivity::class.java).get() + val display = activity.getSystemService(DisplayManager::class.java).getDisplay(Display.DEFAULT_DISPLAY) + val positions = mutableListOf>() + var position = Offset(0.2f, 0.7f) + val panel = SecondScreenPresentation(activity, display, + { x, y, w, h -> positions += x / w to y / h }, { _, _ -> }, {}, {}, {}, {}, + pointerPosition = { position }) + try { + panel.create() + panel.showMode(SecondScreenMode.KEYBOARD_TRACKPAD) + val trackpad = descendants(panel.window!!.decorView).first { it.javaClass.simpleName == "TrackpadView" } + trackpad.layout(0, 0, 1000, 600) + for (next in listOf(Offset(0.2f, 0.7f), Offset(0.8f, 0.1f))) { + position = next + for ((action, time) in listOf(MotionEvent.ACTION_DOWN to 1000L, MotionEvent.ACTION_UP to 1010L)) { + val event = MotionEvent.obtain(1000, time, action, 300f, 300f, 0) + try { trackpad.onTouchEvent(event) } finally { event.recycle() } + } + assertEquals(next.x, positions.last().first, 0.001f) + assertEquals(next.y, positions.last().second, 0.001f) + } + } finally { panel.closeControls() } + } + @Test @Config(sdk = [34], shadows = [PadBridgeRoutingTest.RegisteredKeyEvent::class]) fun controllerBumpersReachTheSecondScreenInBothSessionModes() {