feat: on-screen keys for any key or key combo (#1305)

This commit is contained in:
kingokksa authored and GitHub committed 2026-10-25 17:27:07 -04:00
1 parent 6e67dbb5be
commit 9f4794769c
48 files changed
+3169 -47

No files matched your search

@@ -75,8 +75,10 @@ import com.droiddeck.launcher.input.OnScreenControls
import com.droiddeck.launcher.input.PadBridge
import com.droiddeck.launcher.input.PadMouse
import com.droiddeck.launcher.input.PadState
import com.droiddeck.launcher.input.ScreenKeys
import com.droiddeck.launcher.input.SteamControllerBle
import com.droiddeck.launcher.input.DirectTouchHold
import com.droiddeck.launcher.input.GestureTouchStreamGate
import com.droiddeck.launcher.input.PointerGestures
import com.droiddeck.launcher.input.SecondScreenDisplay
import com.droiddeck.launcher.input.SecondScreenDisplays
@@ -199,10 +201,20 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
private var gestureConfig = com.droiddeck.launcher.input.TouchGestureConfig()
private var gestureGame: String? = null
private var gestureSettingsOpen by mutableStateOf(false)
private var screenKeyPickerOpen by mutableStateOf(false)
private var screenKeyBind: List<Int>? = null
private var screenKeyDir: Int? = null
private var screenKeyTextFocus = false
private var screenKeyEdit by mutableStateOf<com.droiddeck.launcher.input.ScreenKey?>(null)
private var screenLayoutsOpen by mutableStateOf(false)
private var screenKeysShown by mutableStateOf(true)
private var screenLayoutNames by mutableStateOf(listOf<String>())
private var screenKeys: ScreenKeys? = null
private var recognizedGesture by mutableStateOf<TouchscreenGestures.Gesture?>(null)
private var gesturePointerX = 0f
private var gesturePointerY = 0f
private var gesturePointerReady = false
private val gestureTouchStreamGate = GestureTouchStreamGate()
private val clearRecognizedGesture = Runnable { recognizedGesture = null }
private val gestureKeysDown = mutableSetOf<Int>()
private lateinit var scrollTouch: ScrollTouchGestures
@@ -536,6 +548,21 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
cursorVisible = false
}
onScreenControls = OnScreenControls(this, bridge, onKeyboard = ::togglePcKeyboard).also { root.addView(it) }
screenKeysShown = com.droiddeck.launcher.input.ControllerPrefs.screenKeysShown(this)
screenKeys = ScreenKeys(this) { code, down -> if (CompositorHost.isStarted) sendGuestKey(code, down) }.also { keys ->
keys.onTouchMiss = miss@{ event ->
if (routedTouchMode == SessionPrefs.TOUCH_GESTURES &&
!gestureTouchStreamGate.accepts(GestureTouchStreamGate.Source.SCREEN_KEYS, event)) return@miss
event.offsetLocation(keys.left - surfaceView.left.toFloat(), keys.top - surfaceView.top.toFloat())
onTouchEvent(event)
}
keys.onAdd = { screenKeyPickerOpen = true }
keys.onSelect = { codes -> screenKeyEdit = screenKeys?.stateOf(codes) }
keys.onEditingChanged = { if (!it) screenKeyEdit = null }
keys.steamStyle = { steamTouchControls?.style }
keys.visibility = if (screenKeysShown) View.VISIBLE else View.GONE
root.addView(keys)
}
keyboard = KeyboardHost(this).also { root.addView(it) }
touchpad = TouchpadGestures(PointerGestures.slop(this), pointerListener)
directTouchHold = DirectTouchHold(PointerGestures.slop(this), pointerListener) {
@@ -550,7 +577,12 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
// game surface and on-screen buttons, in either finger-down order.
root.isMotionEventSplittingEnabled = true
surfaceView.setOnTouchListener { _, event ->
routedTouchMode == SessionPrefs.TOUCH_GESTURES && onTouchEvent(event)
if (routedTouchMode != SessionPrefs.TOUCH_GESTURES) {
gestureTouchStreamGate.reset()
false
} else if (!gestureTouchStreamGate.accepts(GestureTouchStreamGate.Source.SURFACE, event)) {
false
} else onTouchEvent(event)
}
padMouse = PadMouse(padMouseListener)
bridge.setTouchRoute(object : PadBridge.TouchRoute {
@@ -741,6 +773,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
LaunchedEffect(drawerOpen, pcKeyboardOpen, achievementsOpen, gestureSettingsOpen, pipUi, SessionState.suspended) {
if (drawerOpen || pcKeyboardOpen || achievementsOpen || gestureSettingsOpen || pipUi || SessionState.suspended) {
cancelTouchGestures()
screenKeys?.stopEditing()
}
}
com.droiddeck.launcher.ui.GestureFeedback(recognizedGesture)
@@ -749,6 +782,84 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
onDismiss = { gestureSettingsOpen = false },
onSaved = { gestureSettingsOpen = false; refreshGestureConfig() },
)
if (screenKeyPickerOpen) {
val dir = screenKeyDir
com.droiddeck.launcher.ui.ScreenKeyPicker(
initial = if (dir != null) {
listOfNotNull((screenKeyEdit?.codes?.getOrNull(dir) ?: 0).takeIf { it != 0 })
} else {
screenKeyBind ?: emptyList()
},
onDismiss = { screenKeyPickerOpen = false; screenKeyBind = null; screenKeyDir = null },
onAdd = { codes ->
val bind = screenKeyBind
if (dir != null && bind != null) {
screenKeys?.setDirection(bind, dir, codes.lastOrNull() ?: 0)?.let { screenKeyEdit = it }
} else if (bind != null) {
screenKeys?.setCodes(bind, codes)?.let { screenKeyEdit = it }
} else {
screenKeys?.add(codes)
screenKeyEdit = screenKeys?.stateOf(codes)
}
screenKeyPickerOpen = false
screenKeyBind = null
screenKeyDir = null
},
)
}
screenKeyEdit?.let { key ->
com.droiddeck.launcher.ui.ScreenKeyEditor(
codes = key.codes, keyY = key.y,
size = key.size, opacity = key.opacity, shape = key.shape, toggle = key.toggle,
labels = key.labels,
text = key.text, hasImage = key.image.isNotEmpty(),
onResize = { screenKeys?.resize(key.codes, it); screenKeyEdit = screenKeys?.stateOf(key.codes) },
onOpacity = { screenKeys?.setOpacity(key.codes, it); screenKeyEdit = screenKeys?.stateOf(key.codes) },
onShape = { screenKeys?.setShape(key.codes, it); screenKeyEdit = screenKeys?.stateOf(key.codes) },
onToggle = { on -> screenKeys?.setToggle(key.codes, on); screenKeyEdit = screenKeys?.stateOf(key.codes) },
onLabels = { on -> screenKeys?.setLabels(key.codes, on); screenKeyEdit = screenKeys?.stateOf(key.codes) },
onDirection = { index -> screenKeyBind = key.codes; screenKeyDir = index; screenKeyPickerOpen = true },
onText = { screenKeys?.setText(key.codes, it); screenKeyEdit = screenKeys?.stateOf(key.codes) },
onImage = { screenKeys?.setImage(key.codes, it); screenKeyEdit = screenKeys?.stateOf(key.codes) },
onClearImage = { screenKeys?.setImage(key.codes, ""); screenKeyEdit = screenKeys?.stateOf(key.codes) },
onBind = { screenKeyBind = key.codes; screenKeyDir = null; screenKeyPickerOpen = true },
onTextFocus = { screenKeyTextFocus = true; keyboard?.hide() },
onTextBlur = { screenKeyTextFocus = false },
onRemove = { screenKeys?.remove(key.codes); screenKeyEdit = null },
onDismiss = { screenKeyEdit = null; screenKeyTextFocus = false; screenKeys?.deselect() },
)
}
if (screenLayoutsOpen) com.droiddeck.launcher.ui.ScreenLayoutPanel(
layouts = screenLayoutNames,
onSave = { name ->
screenKeys?.let { keys ->
com.droiddeck.launcher.input.ControllerPrefs.saveScreenLayout(this@SessionActivity, name, keys.export())
}
screenLayoutNames = layoutNames()
},
onLoad = { name ->
com.droiddeck.launcher.input.ControllerPrefs.screenLayouts(this@SessionActivity)[name]?.let { screenKeys?.load(it) }
screenLayoutsOpen = false
},
onDelete = { name ->
com.droiddeck.launcher.input.ControllerPrefs.deleteScreenLayout(this@SessionActivity, name)
screenLayoutNames = layoutNames()
},
onImport = { text ->
val width = (screenKeys?.width ?: 0).takeIf { it > 0 } ?: resources.displayMetrics.widthPixels
val keys = com.droiddeck.launcher.input.importScreenLayout(text, width, resources.displayMetrics.density)
if (keys != null) {
screenKeys?.load(com.droiddeck.launcher.input.ControllerPrefs.encodeScreenKeys(keys))
screenLayoutsOpen = false
}
},
exportText = {
val width = (screenKeys?.width ?: 0).takeIf { it > 0 } ?: resources.displayMetrics.widthPixels
val keys = com.droiddeck.launcher.input.ControllerPrefs.decodeScreenKeys(screenKeys?.export().orEmpty())
com.droiddeck.launcher.input.exportScreenLayout(keys, width, resources.displayMetrics.density)
},
onDismiss = { screenLayoutsOpen = false },
)
androidx.compose.runtime.LaunchedEffect(drawerOpen) {
if (drawerOpen) {
syncLossless()
@@ -939,6 +1050,19 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
},
onTouch = { v -> SessionPrefs.setTouchMode(this@SessionActivity, v); readPrefs() },
onGestureSettings = { drawerOpen = false; gestureSettingsOpen = true },
onScreenKeys = { drawerOpen = false; screenKeys?.startEditing() },
screenKeysShown = screenKeysShown,
onScreenKeysShown = { shown ->
screenKeysShown = shown
com.droiddeck.launcher.input.ControllerPrefs.setScreenKeysShown(this@SessionActivity, shown)
screenKeys?.stopEditing()
screenKeys?.visibility = if (shown) View.VISIBLE else View.GONE
},
onScreenLayouts = {
drawerOpen = false
screenLayoutNames = layoutNames()
screenLayoutsOpen = true
},
onFexPreset = { v -> SessionPrefs.setFexPreset(this@SessionActivity, v); readPrefs() },
onSecondScreenMode = ::selectSecondScreenMode,
onSecondScreenDisplay = ::selectSecondScreenDisplay,
@@ -1495,7 +1619,9 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
if (nextTouchMode == SessionPrefs.TOUCH_OFF || !usingTouchpad()) cursorVisible = false
}
touchMode = nextTouchMode
routedTouchMode = effectiveTouchMode()
val nextRoutedTouchMode = effectiveTouchMode()
if (routedTouchMode != nextRoutedTouchMode) gestureTouchStreamGate.reset()
routedTouchMode = nextRoutedTouchMode
frameGen = FrameGen.mode(this)
fexPreset = SessionPrefs.fexPreset(this)
suspendPolicy = SessionPrefs.suspendPolicy(this, SessionState.mode)
@@ -2053,6 +2179,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
return super.dispatchKeyEvent(event)
}
if (pipUi) return true
if (screenKeyTextFocus) return super.dispatchKeyEvent(event)
val pad = com.droiddeck.launcher.input.JoyCon.normalize(event)
if (pad !== event) return dispatchKeyEvent(pad)
val fromController = PadBridge.isControllerKeyEvent(event)
@@ -2254,8 +2381,12 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
super.dispatchKeyEvent(event)
}
private fun layoutNames(): List<String> =
com.droiddeck.launcher.input.ControllerPrefs.screenLayouts(this).keys.sorted()
private fun refreshGestureConfig() {
gestureConfig = com.droiddeck.launcher.input.GesturePrefs.read(this, gestureGame)
gestureTouchStreamGate.reset()
if (::touchscreenGestures.isInitialized) touchscreenGestures.configure(gestureConfig)
gesturePointerReady = false
recognizedGesture = null
@@ -2544,6 +2675,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
private fun refreshControls() {
applyUnfoldedControls()
updateOnScreenControls()
screenKeys?.refreshAppearance()
}
private fun applyUnfoldedControls() {
@@ -2629,6 +2761,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
if (::scrollTouch.isInitialized) scrollTouch.cancel()
if (CompositorHost.isStarted) WaylandCompositor.nativeSendTouch(3, -1, 0, 0)
if (next == SessionPrefs.TOUCH_OFF) cursorVisible = false
gestureTouchStreamGate.reset()
routedTouchMode = next
}
@@ -2819,6 +2952,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
device.plugged = false
uiHandler.postDelayed({ updateOnScreenControls() }, 700)
}
view.onStyle = { screenKeys?.refreshAppearance() }
root.addView(view, root.indexOfChild(onScreenControls) + 1)
steamTouchControls = view
steamTouchAvailable = true
@@ -2849,6 +2983,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
controls.visibility = View.GONE
steamTouchControls?.let { it.releaseAll(); it.visibility = View.GONE }
com.droiddeck.launcher.input.SteamTouchDevice.current?.plugged = false
screenKeys?.followSteam = false
applyTouchRouting()
return
}
@@ -2877,6 +3012,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
onScreenButtonsVisible = show
val drawn = show && !keyboardCoversPad()
if (steam != null) {
screenKeys?.followSteam = drawn
if (controls.visibility == View.VISIBLE) { controls.releaseAll(); controls.visibility = View.GONE }
if (!drawn) steam.releaseAll()
steam.visibility = if (drawn) View.VISIBLE else View.GONE
@@ -2885,6 +3021,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
Log.i(TAG, "on-screen controls: Steam's touch controller " + if (drawn) "shown" else "hidden")
return
}
screenKeys?.followSteam = false
if (drawn == (controls.visibility == View.VISIBLE)) return
if (!drawn) controls.releaseAll()
controls.visibility = if (drawn) View.VISIBLE else View.GONE
@@ -2970,6 +3107,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
pendingBackAction = null
releaseDrawerDirection()
onScreenControls?.releaseAll()
screenKeys?.releaseAll()
releasePads()
touchpad.cancel()
scrollTouch.cancel()