feat: agent bridge wayland and gamescope coverage (#406)

* feat: run a guest agent in every session for commands, focus and the Steam client's DevTools

* feat: give the agent bridge game, input, UI and experiment commands behind a Setup toggle

* feat: droiddeckctl commands for the new bridge, a scenario runner and contract tests

* feat: run cleanup steps after every scenario run so a failure cannot leak into the next

* feat: restart a session that died between scenario runs with start --reuse

* fix: give droiddeckctl stop its own, longer wait for the session's logs

* fix: harden debugging tools access and output
This commit is contained in:
Kurt Himebauch authored and GitHub committed 2026-10-08 03:50:58 +00:00
1 parent 7b1f0f25b2
commit 415b93f943
42 files changed
+3787 -168

No files matched your search

@@ -35,6 +35,7 @@ import com.droiddeck.launcher.runtime.LinuxRuntimeInstaller
import com.droiddeck.launcher.runtime.DeckyManager
import com.droiddeck.launcher.session.SessionService
import com.droiddeck.launcher.ui.PackageRow
import com.droiddeck.launcher.ui.exposeTestTags
import com.droiddeck.launcher.session.OfflineMode
import com.droiddeck.launcher.session.ProtonExtras
import com.droiddeck.launcher.session.ComponentsManager
@@ -355,6 +356,7 @@ class MainActivity : ComponentActivity() {
private var emulatorList by mutableStateOf<List<Library.Emulator>>(emptyList())
private var runningLabel by mutableStateOf<String?>(null)
private var logsEnabled by mutableStateOf(true)
private var agentCommands by mutableStateOf(false)
private var showRoms by mutableStateOf(false)
private var homeAppSelected by mutableStateOf(false)
private var homeScreenEnabled by mutableStateOf(false)
@@ -526,7 +528,7 @@ class MainActivity : ComponentActivity() {
updates.start()
setContent {
DroidDeckTheme(theme, appScale = appScale) {
com.droiddeck.launcher.ui.FocusGlideHost(androidx.compose.ui.Modifier.fillMaxSize()) {
com.droiddeck.launcher.ui.FocusGlideHost(androidx.compose.ui.Modifier.fillMaxSize().exposeTestTags()) {
val sm = settingsMode
val page: (@Composable () -> Unit)? = when {
sm != null -> { { ModeSettingsHost(sm) } }
@@ -546,7 +548,7 @@ class MainActivity : ComponentActivity() {
busy = busy, stage = stage, percent = percent,
desktopInstalled = desktopInstalled,
offlineAccount = offlineAccount, offline = offline,
frameGenLabel = frameGenLabel, romsDir = romsDir, logsEnabled = logsEnabled,
frameGenLabel = frameGenLabel, romsDir = romsDir, logsEnabled = logsEnabled, agentCommands = agentCommands,
steamGames = steamGames, emulators = emulatorList, running = runningLabel,
frameGen = FrameGen.mode(this),
lossless = lossless,
@@ -658,6 +660,10 @@ class MainActivity : ComponentActivity() {
SessionPrefs.setLogsEnabled(this, !SessionPrefs.logsEnabled(this))
logsEnabled = SessionPrefs.logsEnabled(this)
},
onAgentCommands = {
com.droiddeck.launcher.agent.AgentAccess.setCommandsEnabled(this, !com.droiddeck.launcher.agent.AgentAccess.commandsEnabled(this))
agentCommands = com.droiddeck.launcher.agent.AgentAccess.commandsEnabled(this)
},
onShareLogs = {
SessionLogShare.prepare(this, { SessionLogShare.latest(this) }) { zip ->
if (zip == null) android.widget.Toast.makeText(this, R.string.main_no_logs_yet, android.widget.Toast.LENGTH_LONG).show()
@@ -1492,6 +1498,7 @@ class MainActivity : ComponentActivity() {
frameGenLabel = FrameGen.label(this)
romsDir = SessionPrefs.romsDir(this).takeIf { it.isNotEmpty() }
logsEnabled = SessionPrefs.logsEnabled(this)
agentCommands = com.droiddeck.launcher.agent.AgentAccess.commandsEnabled(this)
runningLabel = if (SessionState.running) when (SessionState.mode) {
SessionService.MODE_DESKTOP -> getString(R.string.rail_desktop)
SessionService.MODE_RUN -> Library.nameForProgram(SessionState.program)
@@ -362,6 +362,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
val bridge = PadBridge(File(LinuxRuntime.sessionRoot(this), "dev/input"))
padBridge = bridge
SessionState.padBridge = bridge
padMotion = com.droiddeck.launcher.input.PadMotion(this) {
@Suppress("DEPRECATION")
(if (Build.VERSION.SDK_INT >= 30) display else windowManager.defaultDisplay)?.rotation ?: android.view.Surface.ROTATION_0
@@ -2052,6 +2053,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
closeSecondScreen(reset = false)
if (::hud.isInitialized) hud.stop()
padBridge?.stop()
if (SessionState.padBridge === padBridge) SessionState.padBridge = null
padMotion?.stop()
if (SessionState.deckPadListener === deckPadListener) SessionState.deckPadListener = null
if (SessionState.endListener === endListener) SessionState.endListener = null
@@ -0,0 +1,48 @@
package com.droiddeck.launcher.agent
import android.content.Context
/**
* Who may run code through the agent bridge. Reading state, starting and stopping sessions,
* launching and quitting games and sending input only need the shell (the provider's DUMP
* permission). Running commands in the guest, evaluating JavaScript in the Steam client, session
* environment, binary overrides and preference writes reach inside the app's sandbox - the guest
* holds the Steam login - which the shell of a stock device otherwise cannot. They require
* Setup > Session > Debugging tools to be on, in every build. The bridge itself cannot turn it on.
*/
object AgentAccess {
const val MAX_REQUEST_CHARS = 256 * 1024
private const val PREFS = "agent"
private const val KEY_COMMANDS = "commands"
/** The preferences file the bridge must never write: set-pref refuses it. */
const val PROTECTED_PREFS = PREFS
fun commandsAllowed(context: Context): Boolean =
prefs(context).getBoolean(KEY_COMMANDS, false)
/** The Setup toggle's own state. */
fun commandsEnabled(context: Context): Boolean = prefs(context).getBoolean(KEY_COMMANDS, false)
fun setCommandsEnabled(context: Context, enabled: Boolean) {
prefs(context).edit().putBoolean(KEY_COMMANDS, enabled).apply()
if (!enabled) runCatching { AgentEnv.revoke(context) }
.onFailure { android.util.Log.w("AgentAccess", "Could not finish clearing debugging experiments") }
}
fun requireCommands(context: Context) {
if (!commandsAllowed(context)) throw AgentException(
"AGENT_COMMANDS_DISABLED",
"Debugging tools are off: turn on Setup > Session > Debugging tools on the device",
)
}
fun requireStart(context: Context, request: org.json.JSONObject) {
if (request.optString("mode") == "run") requireCommands(context)
}
private fun prefs(context: Context) = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE)
}
/** A refusal with a stable code, answered as {"ok": false, "error": {code, message}}. */
class AgentException(val code: String, message: String) : RuntimeException(message)
@@ -10,13 +10,14 @@ import android.net.Uri
import android.os.Binder
import android.os.Bundle
import android.os.Process
import android.util.Base64
import com.droiddeck.launcher.BuildConfig
import com.droiddeck.launcher.frontend.GameLaunchLink
import com.droiddeck.launcher.runtime.LinuxRuntime
import com.droiddeck.launcher.runtime.LinuxRuntimeInstaller
import com.droiddeck.launcher.session.SessionArtifacts
import com.droiddeck.launcher.session.SessionEvents
import com.droiddeck.launcher.session.SessionPhase
import com.droiddeck.launcher.session.SessionPaths
import com.droiddeck.launcher.session.SessionService
import com.droiddeck.launcher.session.SessionState
import org.json.JSONArray
@@ -25,7 +26,14 @@ import java.io.File
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
/** A shell-only control surface, installed only by debug builds. */
/**
* The shell's control surface (docs/agent-control.md), in every build: the provider and its start
* Activity need android.permission.DUMP, held by the shell and privileged or explicitly granted callers. Commands that
* reach into the app's sandbox are further gated by [AgentAccess].
*
* A method's request is JSON, base64-encoded in the "request" extra; every answer is JSON in the
* "json" key of the returned Bundle, {"ok": true, ...} or {"ok": false, "error": {code, message}}.
*/
class AgentBridgeProvider : ContentProvider() {
override fun onCreate(): Boolean {
val appContext = context?.applicationContext ?: return false
@@ -48,17 +56,53 @@ class AgentBridgeProvider : ContentProvider() {
}
val response = try {
when (method) {
"state" -> state(context).put("ok", true)
"start" -> error("USE_DROIDDECKCTL", "Start sessions with tools/droiddeckctl so Android launches a visible Activity")
"stop" -> stop(context)
"resume" -> resume(context)
else -> error("UNKNOWN_COMMAND", "Unknown agent command '$method'")
val request = decode(extras?.getString(EXTRA_REQUEST))
// Binder identity is the shell's; everything below acts as the app.
val token = Binder.clearCallingIdentity()
try {
dispatch(context, method, request)
} finally {
Binder.restoreCallingIdentity(token)
}
} catch (e: AgentException) {
error(e.code, e.message ?: e.code)
} catch (e: Exception) {
error("COMMAND_FAILED", e.message ?: e.javaClass.simpleName)
}
return Bundle().apply { putString(RESULT_JSON, response.toString()) }
val json = response.toString().let {
if (it.length <= 384 * 1024) it else error("RESPONSE_TOO_LARGE", "Reduce the command output or select fewer fields").toString()
}
return Bundle().apply { putString(RESULT_JSON, json) }
}
private fun dispatch(context: Context, method: String, request: JSONObject): JSONObject = when (method) {
"state" -> state(context).put("ok", true)
"start" -> error("USE_DROIDDECKCTL", "Start sessions with tools/droiddeckctl so Android launches a visible Activity")
"stop" -> stop(context)
"resume" -> resume(context)
"launch" -> launch(context, request)
"focus" -> guest(context, JSONObject().put("kind", "focus"), 15_000)
"quit" -> guest(context, JSONObject().put("kind", "quit").put("appId", request.opt("appId"))
.put("timeout", request.optDouble("timeout", 15.0)), timeoutMs(request, 15.0, 30_000))
"input" -> AgentInput.handle(request)
"ui" -> AgentUi.handle(request)
"guest" -> {
AgentAccess.requireCommands(context)
guest(context, JSONObject(request.toString()).put("kind", "exec"), timeoutMs(request, 30.0, 10_000))
}
"cdp" -> {
AgentAccess.requireCommands(context)
guest(context, JSONObject(request.toString()).put("kind", "cdp"), timeoutMs(request, 15.0, 5_000))
}
"env" -> env(context, request)
"override" -> {
AgentAccess.requireCommands(context)
AgentEnv.override(context, request.optString("name")).put("ok", true).put("command", "override")
}
"prefs" -> prefs(context, request)
"access" -> JSONObject().put("ok", true).put("commands", AgentAccess.commandsAllowed(context))
.put("debugBuild", BuildConfig.DEBUG).put("inbox", AgentEnv.inbox(context).absolutePath)
else -> error("UNKNOWN_COMMAND", "Unknown agent command '$method'")
}
private fun state(context: Context): JSONObject {
@@ -91,13 +135,31 @@ class AgentBridgeProvider : ContentProvider() {
.put("status", SessionState.failureStatus ?: JSONObject.NULL)
session.put("failure", failure)
// gamescope's focus as last published by the guest agent, without the window lists
// (the focus command returns those, fresh).
val focus = AgentGuest.focus(context)?.let { snap ->
JSONObject()
.put("focusedApp", snap.opt("focusedApp") ?: JSONObject.NULL)
.put("focusedWindow", snap.opt("focusedWindow") ?: JSONObject.NULL)
.put("focusableApps", snap.optJSONArray("focusableApps") ?: JSONArray())
.put("baselayerAppIds", snap.optJSONArray("baselayerAppIds") ?: JSONArray())
.put("seq", snap.optInt("seq"))
.put("t", snap.optLong("t"))
}
session.put("focus", focus ?: JSONObject.NULL)
val hello = AgentGuest.hello(context)
return JSONObject()
.put("schema", 1)
.put("schema", SCHEMA)
.put("build", BuildConfig.BUILD_LABEL)
.put("appVersion", context.packageManager.getPackageInfo(context.packageName, 0).versionName)
.put("runtime", JSONObject()
.put("installed", LinuxRuntime.isInstalled(context))
.put("version", runtimeVersion ?: JSONObject.NULL))
.put("agent", JSONObject()
.put("guest", hello != null)
.put("guestFocus", hello?.optBoolean("focus") ?: false)
.put("commands", AgentAccess.commandsAllowed(context)))
.put("session", session)
}
@@ -131,6 +193,67 @@ class AgentBridgeProvider : ContentProvider() {
return JSONObject().put("ok", true).put("command", "resume")
}
private fun launch(context: Context, request: JSONObject): JSONObject {
val appId = request.optString("appId")
if (!GameLaunchLink.validId(appId)) throw AgentException("INVALID_APP_ID", "appId must be a decimal Steam app id")
if (!SessionState.running || SessionState.mode != SessionService.MODE_STEAM) {
throw AgentException("NO_STEAM_SESSION", "Games launch inside a running Steam session")
}
if (!SessionService.launchGame(context, appId)) throw AgentException("LAUNCH_REJECTED", "The session did not accept the launch")
SessionEvents.record("agent.launch_requested", mapOf("appId" to appId))
return JSONObject().put("ok", true).put("command", "launch").put("appId", appId)
}
private fun env(context: Context, request: JSONObject): JSONObject {
AgentAccess.requireCommands(context)
val result = when (val op = request.optString("op", "list")) {
"list" -> AgentEnv.list(context)
"set" -> {
val lines = request.optJSONArray("lines") ?: JSONArray()
AgentEnv.set(context, List(lines.length()) { lines.optString(it) }, request.optBoolean("persistent"))
}
"clear" -> AgentEnv.clear(context, request.optString("scope", "next"))
else -> throw AgentException("INVALID_REQUEST", "env op '$op' is not list, set or clear")
}
return result.put("ok", true).put("command", "env")
}
private fun prefs(context: Context, request: JSONObject): JSONObject {
AgentAccess.requireCommands(context)
val file = request.optString("file")
val result = when (val op = request.optString("op", "get")) {
"get" -> AgentPrefs.get(context, file)
"set" -> AgentPrefs.set(context, file, request.optString("key"), request.optString("type", "string"),
if (request.isNull("value")) null else request.optString("value"))
else -> throw AgentException("INVALID_REQUEST", "prefs op '$op' is not get or set")
}
return result.put("ok", true).put("command", "prefs")
}
/** A guest agent request; its own answer already carries ok or error. */
private fun guest(context: Context, request: JSONObject, timeoutMs: Long): JSONObject =
AgentGuest.call(context, request, timeoutMs)
/** The guest's own timeout plus slack for the round trip. */
private fun timeoutMs(request: JSONObject, defaultSeconds: Double, slackMs: Long): Long {
val seconds = if (request.has("timeout")) request.opt("timeout")?.toString()?.toDoubleOrNull()
?: throw AgentException("INVALID_REQUEST", "timeout must be a number") else defaultSeconds
if (!seconds.isFinite() || seconds <= 0 || seconds > 600) {
throw AgentException("INVALID_REQUEST", "timeout must be a positive number no greater than 600 seconds")
}
return (seconds * 1000).toLong() + slackMs
}
private fun decode(encoded: String?): JSONObject {
if (encoded.isNullOrBlank()) return JSONObject()
if (encoded.length > AgentAccess.MAX_REQUEST_CHARS) throw AgentException("INVALID_REQUEST", "The encoded request exceeds 256 KiB")
return try {
JSONObject(String(Base64.decode(encoded, Base64.DEFAULT), Charsets.UTF_8))
} catch (e: Exception) {
throw AgentException("INVALID_REQUEST", "The request extra is not base64 JSON")
}
}
private fun error(code: String, message: String) = JSONObject()
.put("ok", false)
.put("error", JSONObject().put("code", code).put("message", message))
@@ -143,6 +266,9 @@ class AgentBridgeProvider : ContentProvider() {
companion object {
const val RESULT_JSON = "json"
const val EXTRA_REQUEST = "request"
/** 2 added agent and session.focus; everything in 1 is unchanged. */
const val SCHEMA = 2
private val recoveryComplete = CountDownLatch(1)
fun awaitRecovery() {
@@ -0,0 +1,222 @@
package com.droiddeck.launcher.agent
import android.content.Context
import android.util.Log
import com.droiddeck.launcher.runtime.LinuxRuntime
import org.json.JSONArray
import org.json.JSONObject
import java.io.File
import java.nio.file.Files
import java.nio.file.LinkOption.NOFOLLOW_LINKS
import java.nio.file.StandardOpenOption.READ
/**
* Experiment switches an agent sets without touching Download/droiddeck-env or rebuilding the apk:
*
* - environment lines (KEY=VALUE) for the session, applied after the user's own droiddeck-env, so
* they win. "next" lines are taken by the next session that starts and dropped when it ends;
* "persistent" ones stay until cleared.
* - binary overrides: a file pushed to the app's external files (agent-inbox/<name>) is installed
* as /opt/droiddeck-agent/bin/<name> in the rootfs and put first on the next session's PATH -
* a gamescope build under test, say. Removed when that session ends.
*
* Kept in the app's own files (agent-env.json), never in a place the shell can write.
*/
object AgentEnv {
private const val TAG = "AgentEnv"
private const val FILE = "agent-env.json"
private const val GUEST_BIN = "/opt/droiddeck-agent/bin"
private const val BASE_PATH = "/usr/local/bin:/usr/bin:/bin"
private val NAME = Regex("[A-Za-z0-9._-]{1,64}")
private val KEY = Regex("[A-Za-z_][A-Za-z0-9_]*")
private val lock = Any()
fun inbox(context: Context): File = File(context.getExternalFilesDir(null), "agent-inbox")
fun list(context: Context): JSONObject = synchronized(lock) {
AgentAccess.requireCommands(context)
load(context)
}
/** Add [lines] for the next session ([persistent] = every session until cleared). */
fun set(context: Context, lines: List<String>, persistent: Boolean): JSONObject = synchronized(lock) {
AgentAccess.requireCommands(context)
if (lines.any { line -> KEY.matchEntire(line.substringBefore('=', "")) == null || !line.contains('=') || line.any { it == '\n' || it == '\r' || it == '\u0000' } }) {
throw AgentException("INVALID_ENV", "Environment entries must be single KEY=VALUE lines without NUL bytes")
}
val state = load(context)
val key = if (persistent) "persistent" else "pending"
val merged = merge(strings(state.optJSONArray(key)), lines)
state.put(key, JSONArray(merged))
save(context, state)
state
}
/** Drop lines: [scope] is "next", "persistent" or "all". */
fun clear(context: Context, scope: String): JSONObject = synchronized(lock) {
AgentAccess.requireCommands(context)
val state = load(context)
when (scope) {
"next" -> state.put("pending", JSONArray())
"persistent" -> state.put("persistent", JSONArray())
"all" -> { state.put("pending", JSONArray()); state.put("persistent", JSONArray()); state.put("pendingOverrides", JSONArray()) }
else -> throw AgentException("INVALID_SCOPE", "Scope must be next, persistent or all")
}
save(context, state)
state
}
/** Install agent-inbox/[name] as an override for the next session. */
fun override(context: Context, name: String): JSONObject = synchronized(lock) {
AgentAccess.requireCommands(context)
if (NAME.matchEntire(name) == null || name in setOf(".", "..")) throw AgentException("INVALID_NAME", "Override names are letters, digits, '.', '_' and '-'")
val source = File(inbox(context), name)
if (Files.isSymbolicLink(source.toPath())) throw AgentException("INVALID_NAME", "The override must be a regular file")
if (!source.isFile) throw AgentException("NO_SUCH_FILE", "Push the binary to ${source.path} first")
val target = File(bin(context), name)
target.parentFile?.mkdirs()
val staged = File.createTempFile("override-", ".tmp", target.parentFile)
try {
Files.newInputStream(source.toPath(), READ, NOFOLLOW_LINKS).use { input -> staged.outputStream().use { input.copyTo(it) } }
} catch (e: Exception) {
staged.delete()
throw AgentException("OVERRIDE_FAILED", "Could not stage the override")
}
if (!staged.setExecutable(true, false) || !staged.renameTo(target)) {
staged.delete()
throw AgentException("OVERRIDE_FAILED", "Could not install $name in the rootfs")
}
source.delete()
val state = load(context)
state.put("pendingOverrides", JSONArray(merge(strings(state.optJSONArray("pendingOverrides")), listOf(name))))
save(context, state)
state.put("installed", "$GUEST_BIN/$name")
}
/**
* At session start: the lines this session gets, after the user's droiddeck-env. Pending lines
* and overrides become this session's own, to be dropped by [endSession].
*/
fun beginSession(context: Context): List<String> = synchronized(lock) {
if (!AgentAccess.commandsAllowed(context)) {
revoke(context)
return@synchronized emptyList()
}
val state = load(context)
val active = strings(state.optJSONArray("pending"))
val overrides = strings(state.optJSONArray("pendingOverrides"))
state.put("active", JSONArray(active)).put("pending", JSONArray())
state.put("activeOverrides", JSONArray(overrides)).put("pendingOverrides", JSONArray())
save(context, state)
// Anything else there was left by a session that never reached endSession (the app died).
bin(context).listFiles()
?.filter { it.name !in overrides }?.forEach { it.delete() }
val lines = strings(state.optJSONArray("persistent")) + active
val withPath = if (overrides.isEmpty()) lines else lines + "PATH=$GUEST_BIN:${pathOf(lines)}"
if (withPath.isNotEmpty()) Log.i(TAG, "agent environment keys for this session: ${withPath.map { it.substringBefore('=') }}")
withPath
}
fun revoke(context: Context) = synchronized(lock) {
val state = load(context)
for (key in listOf("pending", "active", "persistent", "pendingOverrides", "activeOverrides")) state.put(key, JSONArray())
save(context, state)
bin(context).listFiles()?.forEach { it.delete() }
}
/** At session end: this session's lines and override binaries go. */
fun endSession(context: Context) = synchronized(lock) {
val state = load(context)
val overrides = strings(state.optJSONArray("activeOverrides"))
val pending = strings(state.optJSONArray("pendingOverrides"))
val bin = bin(context)
overrides.filter { it !in pending }.forEach { File(bin, it).delete() }
state.put("active", JSONArray()).put("activeOverrides", JSONArray())
save(context, state)
}
/** A PATH= line among [lines] (the user's or an agent's) is extended rather than replaced. */
private fun pathOf(lines: List<String>) =
lines.lastOrNull { it.startsWith("PATH=") }?.substringAfter('=')?.takeIf { it.isNotBlank() } ?: BASE_PATH
private fun bin(context: Context): File {
val root = LinuxRuntime.rootDir(context).canonicalFile.toPath()
val directory = root.resolve(GUEST_BIN.removePrefix("/"))
var path = directory
while (path != root) {
if (Files.isSymbolicLink(path)) throw AgentException("OVERRIDE_FAILED", "The override directory must not contain symbolic links")
path = path.parent
}
return directory.toFile()
}
private fun merge(existing: List<String>, added: List<String>): List<String> {
val keys = added.map { it.substringBefore('=') }.toSet()
return existing.filter { it.substringBefore('=') !in keys } + added
}
private fun strings(array: JSONArray?): List<String> =
if (array == null) emptyList() else List(array.length()) { array.optString(it) }.filter { it.isNotEmpty() }
private fun load(context: Context): JSONObject {
val file = File(context.filesDir, FILE)
val state = try { if (file.isFile) JSONObject(file.readText()) else JSONObject() } catch (e: Exception) { JSONObject() }
for (key in listOf("pending", "active", "persistent", "pendingOverrides", "activeOverrides")) {
if (!state.has(key)) state.put(key, JSONArray())
}
return state
}
private fun save(context: Context, state: JSONObject) {
val file = File(context.filesDir, FILE)
val staged = File(context.filesDir, "$FILE.tmp")
staged.writeText(state.toString())
if (!staged.renameTo(file)) {
staged.delete()
throw AgentException("ENV_WRITE_FAILED", "Could not save the debugging environment")
}
}
}
/** Typed writes to the app's SharedPreferences, for driving a settings matrix without the UI. */
object AgentPrefs {
fun set(context: Context, file: String, key: String, type: String, value: String?): JSONObject {
AgentAccess.requireCommands(context)
validateFile(file)
if (file == AgentAccess.PROTECTED_PREFS) throw AgentException("PROTECTED_PREFS", "The agent preferences are set on the device only")
if (file.isBlank() || key.isBlank()) throw AgentException("INVALID_PREF", "file and key are required")
val prefs = context.getSharedPreferences(file, Context.MODE_PRIVATE)
val previous = prefs.all[key]
val editor = prefs.edit()
try {
when (type) {
"remove" -> editor.remove(key)
"bool" -> editor.putBoolean(key, value?.toBooleanStrictOrNull() ?: throw NumberFormatException("not true/false"))
"int" -> editor.putInt(key, value!!.toInt())
"long" -> editor.putLong(key, value!!.toLong())
"float" -> editor.putFloat(key, value!!.toFloat())
"string" -> editor.putString(key, value ?: "")
else -> throw AgentException("INVALID_PREF", "type must be bool, int, long, float, string or remove")
}
} catch (e: RuntimeException) {
if (e is AgentException) throw e
throw AgentException("INVALID_PREF", "The value is not a $type")
}
if (!editor.commit()) throw AgentException("PREF_WRITE_FAILED", "Could not write $file/$key")
return JSONObject().put("file", file).put("key", key).put("type", type)
.put("previous", previous ?: JSONObject.NULL).put("value", prefs.all[key] ?: JSONObject.NULL)
}
fun get(context: Context, file: String): JSONObject {
AgentAccess.requireCommands(context)
validateFile(file)
val all = context.getSharedPreferences(file, Context.MODE_PRIVATE).all
return JSONObject().put("file", file).put("values", JSONObject(all.mapValues { it.value ?: JSONObject.NULL }))
}
private fun validateFile(file: String) {
if (!Regex("[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}").matches(file)) {
throw AgentException("INVALID_PREF", "Preference files must be simple names")
}
}
}
@@ -0,0 +1,128 @@
package com.droiddeck.launcher.agent
import android.content.Context
import android.os.FileObserver
import android.util.Log
import com.droiddeck.launcher.runtime.LinuxRuntime
import com.droiddeck.launcher.session.SessionEvents
import com.droiddeck.launcher.session.SessionState
import org.json.JSONObject
import java.io.File
import java.util.UUID
/**
* The app's side of the guest agent (tools/linuxfs/overlay/usr/local/bin/droiddeck-agent): requests
* go in as files under the session root's agent directory and answers come back the same way, so
* whatever runs does so inside the live session with its own environment.
*/
object AgentGuest {
private const val TAG = "AgentGuest"
private const val POLL_MS = 50L
private val lock = Any()
private var observer: FileObserver? = null
private var lastFocusKey: String? = null
fun dir(context: Context) = File(LinuxRuntime.sessionRoot(context), "agent")
/**
* Before each session: nothing of the last one's daemon, answers or focus may be read as
* current. Then watch for the new daemon's hello and focus snapshots, which go into the
* session's events.jsonl.
*/
fun reset(context: Context) {
synchronized(lock) {
observer?.stopWatching()
observer = null
lastFocusKey = null
val dir = dir(context)
dir.deleteRecursively()
dir.mkdirs()
@Suppress("DEPRECATION")
observer = object : FileObserver(dir.path, MOVED_TO or CLOSE_WRITE) {
override fun onEvent(event: Int, path: String?) {
when (path) {
"hello.json" -> readJson(File(dir, path))?.let {
SessionEvents.record("agent.ready", mapOf("pid" to it.optInt("pid"), "focus" to it.optBoolean("focus")))
}
"focus.json" -> readJson(File(dir, path))?.let(::recordFocus)
}
}
}.also { it.startWatching() }
}
}
/** At the end of a session: no more events for a folder that is being closed. */
fun stop() {
synchronized(lock) {
observer?.stopWatching()
observer = null
}
}
private fun recordFocus(snapshot: JSONObject) {
val fields = linkedMapOf<String, Any?>(
"focusedApp" to snapshot.opt("focusedApp"),
"focusableApps" to snapshot.optJSONArray("focusableApps")?.toString(),
"baselayerAppIds" to snapshot.optJSONArray("baselayerAppIds")?.toString(),
)
val key = fields.values.joinToString("|")
synchronized(lock) {
if (key == lastFocusKey) return
lastFocusKey = key
}
SessionEvents.record("focus.changed", fields + ("seq" to snapshot.optInt("seq")))
}
/** The daemon's hello, when it is running in the current session. */
fun hello(context: Context): JSONObject? {
if (!SessionState.running) return null
val hello = readJson(File(dir(context), "hello.json")) ?: return null
val pid = hello.optInt("pid", -1)
return hello.takeIf { pid > 1 && File("/proc/$pid").exists() }
}
/** The newest focus snapshot the daemon published, when a session is running. */
fun focus(context: Context): JSONObject? =
if (SessionState.running) readJson(File(dir(context), "focus.json")) else null
/** Send one request and wait for its answer. Throws [AgentException] with a stable code. */
fun call(context: Context, request: JSONObject, timeoutMs: Long): JSONObject {
if (request.optString("kind") in setOf("exec", "cdp")) AgentAccess.requireCommands(context)
if (!SessionState.running) throw AgentException("NO_ACTIVE_SESSION", "There is no running session")
hello(context) ?: throw AgentException("AGENT_UNAVAILABLE", "The guest agent is not running in this session (yet)")
val dir = dir(context)
val requests = File(dir, "req").apply { mkdirs() }
val answers = File(dir, "resp").apply { mkdirs() }
val id = UUID.randomUUID().toString()
val staged = File(requests, "$id.json.tmp")
val published = File(requests, "$id.json")
val answer = File(answers, "$id.json")
staged.writeText(request.toString())
if (!staged.renameTo(published)) {
staged.delete()
throw AgentException("AGENT_UNAVAILABLE", "Could not hand the request to the guest agent")
}
val deadline = System.currentTimeMillis() + timeoutMs
while (System.currentTimeMillis() < deadline) {
if (answer.isFile) {
if (answer.length() > 1024 * 1024) {
answer.delete()
throw AgentException("RESPONSE_TOO_LARGE", "The guest response exceeds 1 MiB")
}
val result = readJson(answer)
answer.delete()
return result ?: throw AgentException("INVALID_AGENT_RESPONSE", "The guest agent's answer was not JSON")
}
Thread.sleep(POLL_MS)
}
published.delete()
Log.w(TAG, "no answer to ${request.optString("kind")} in ${timeoutMs}ms")
throw AgentException("TIMEOUT", "The guest agent did not answer within ${timeoutMs / 1000}s")
}
private fun readJson(file: File): JSONObject? = try {
if (file.isFile && file.length() <= 1024 * 1024) JSONObject(file.readText()) else null
} catch (e: Exception) {
null
}
}
@@ -0,0 +1,221 @@
package com.droiddeck.launcher.agent
import com.droiddeck.launcher.input.PadState
import com.droiddeck.launcher.session.SessionState
import com.droiddeck.launcher.wayland.WaylandCompositor
import org.json.JSONObject
/**
* Input into the running session the way a player gives it: keys and touches through the
* compositor (what the session screen sends), and the virtual pad through the same PadBridge the
* on-screen controls drive, so games that ignore touch still see a controller.
*/
object AgentInput {
/** Out of the way of real fingers, which Android numbers from 0. */
private const val TOUCH_ID = 9
private const val TOUCH_SCALE_X = 1919f
private const val TOUCH_SCALE_Y = 1079f
fun handle(request: JSONObject): JSONObject {
if (!SessionState.running || !SessionState.firstFrameSeen) {
throw AgentException("SESSION_NOT_READY", "Input needs a running session that has drawn its first frame")
}
return when (val type = request.optString("type")) {
"key" -> key(request)
"text" -> text(request)
"tap" -> tap(request)
"swipe" -> swipe(request)
"pad" -> pad(request)
"stick" -> stick(request)
"trigger" -> trigger(request)
else -> throw AgentException("INVALID_INPUT", "Unknown input type '$type'")
}.put("ok", true).put("command", "input").put("type", request.optString("type"))
}
private fun key(request: JSONObject): JSONObject {
val name = request.optString("key")
val code = KEYS[name.uppercase()] ?: name.toIntOrNull()
?: throw AgentException("INVALID_INPUT", "Unknown key '$name' (a name such as ENTER or an evdev code)")
val modifiers = request.optJSONArray("modifiers")?.let { array ->
List(array.length()) { i ->
KEYS[array.optString(i).uppercase()] ?: throw AgentException("INVALID_INPUT", "Unknown modifier '${array.optString(i)}'")
}
}.orEmpty()
when (val action = request.optString("action", "tap")) {
"down" -> WaylandCompositor.nativeSendKey(code, 1)
"up" -> WaylandCompositor.nativeSendKey(code, 0)
"tap" -> {
modifiers.forEach { WaylandCompositor.nativeSendKey(it, 1) }
WaylandCompositor.nativeSendKey(code, 1)
hold(request, 40)
WaylandCompositor.nativeSendKey(code, 0)
modifiers.asReversed().forEach { WaylandCompositor.nativeSendKey(it, 0) }
}
else -> throw AgentException("INVALID_INPUT", "Key action '$action' is not tap, down or up")
}
return JSONObject().put("code", code)
}
/** Typed as key presses (US layout), which reaches X programs that have no text-input support. */
private fun text(request: JSONObject): JSONObject {
val text = request.optString("text")
if (request.optBoolean("commit")) {
WaylandCompositor.textInputCommit(text)
return JSONObject().put("length", text.length).put("via", "text-input")
}
val missing = text.filter { TYPED[it] == null }
if (missing.isNotEmpty()) throw AgentException("INVALID_INPUT", "Cannot type '${missing.toSet().joinToString("")}' as keys; use commit")
val delay = request.optLong("delayMs", 15)
for (ch in text) {
val (code, shift) = TYPED.getValue(ch)
if (shift) WaylandCompositor.nativeSendKey(KEYS.getValue("LEFTSHIFT"), 1)
WaylandCompositor.nativeSendKey(code, 1)
WaylandCompositor.nativeSendKey(code, 0)
if (shift) WaylandCompositor.nativeSendKey(KEYS.getValue("LEFTSHIFT"), 0)
Thread.sleep(delay)
}
return JSONObject().put("length", text.length).put("via", "keys")
}
private fun tap(request: JSONObject): JSONObject {
val (x, y) = point(request, "x", "y")
WaylandCompositor.nativeSendTouch(0, TOUCH_ID, x, y)
hold(request, 60)
WaylandCompositor.nativeSendTouch(2, TOUCH_ID, x, y)
return JSONObject().put("touch", org.json.JSONArray().put(x).put(y))
}
private fun swipe(request: JSONObject): JSONObject {
val (x1, y1) = point(request, "x", "y")
val (x2, y2) = point(request, "x2", "y2")
val duration = request.optLong("durationMs", 300).coerceIn(16, 10_000)
val steps = (duration / 16).toInt().coerceAtLeast(2)
WaylandCompositor.nativeSendTouch(0, TOUCH_ID, x1, y1)
for (i in 1..steps) {
Thread.sleep(duration / steps)
WaylandCompositor.nativeSendTouch(1, TOUCH_ID, x1 + (x2 - x1) * i / steps, y1 + (y2 - y1) * i / steps)
}
WaylandCompositor.nativeSendTouch(2, TOUCH_ID, x2, y2)
return JSONObject().put("from", org.json.JSONArray().put(x1).put(y1)).put("to", org.json.JSONArray().put(x2).put(y2))
}
/**
* A point in the compositor's touch space (0..1919 x 0..1079 whatever the session's size), from
* the session's output pixels (space "output", the default: what the guest draws at) or 0..1
* fractions (space "fraction").
*/
private fun point(request: JSONObject, xKey: String, yKey: String): Pair<Int, Int> {
if (!request.has(xKey) || !request.has(yKey)) throw AgentException("INVALID_INPUT", "$xKey and $yKey are required")
val x = request.getDouble(xKey)
val y = request.getDouble(yKey)
val (fx, fy) = when (val space = request.optString("space", "output")) {
"fraction" -> x to y
"output" -> {
val (w, h) = SessionState.outputSize
x / (w - 1).coerceAtLeast(1) to y / (h - 1).coerceAtLeast(1)
}
else -> throw AgentException("INVALID_INPUT", "space '$space' is not output or fraction")
}
return (fx.coerceIn(0.0, 1.0) * TOUCH_SCALE_X).toInt() to (fy.coerceIn(0.0, 1.0) * TOUCH_SCALE_Y).toInt()
}
private fun pad(request: JSONObject): JSONObject {
val bridge = SessionState.padBridge ?: throw AgentException("NO_PAD", "The session has no virtual pad")
val name = request.optString("button").lowercase()
if (name == "qam") {
bridge.triggerQam()
return JSONObject().put("button", name)
}
val button = BUTTONS[name]
if (button == null && name !in DPAD) throw AgentException("INVALID_INPUT", "Unknown pad button '$name'")
val apply = { s: PadState, down: Boolean ->
when (name) {
"up" -> s.up = down
"down" -> s.down = down
"left" -> s.left = down
"right" -> s.right = down
else -> s.press(button!!, down)
}
}
when (val action = request.optString("action", "tap")) {
"down" -> bridge.applyTouch { apply(it, true) }
"up" -> bridge.applyTouch { apply(it, false) }
"tap" -> {
bridge.applyTouch { apply(it, true) }
hold(request, 120)
bridge.applyTouch { apply(it, false) }
}
else -> throw AgentException("INVALID_INPUT", "Pad action '$action' is not tap, down or up")
}
return JSONObject().put("button", name)
}
private fun stick(request: JSONObject): JSONObject {
val bridge = SessionState.padBridge ?: throw AgentException("NO_PAD", "The session has no virtual pad")
val right = request.optString("stick", "left") == "right"
val x = request.optDouble("x", 0.0).toFloat().coerceIn(-1f, 1f)
val y = request.optDouble("y", 0.0).toFloat().coerceIn(-1f, 1f)
bridge.applyTouch { if (right) { it.rightX = x; it.rightY = y } else { it.leftX = x; it.leftY = y } }
hold(request, 300)
bridge.applyTouch { if (right) { it.rightX = 0f; it.rightY = 0f } else { it.leftX = 0f; it.leftY = 0f } }
return JSONObject().put("stick", if (right) "right" else "left")
}
private fun trigger(request: JSONObject): JSONObject {
val bridge = SessionState.padBridge ?: throw AgentException("NO_PAD", "The session has no virtual pad")
val right = request.optString("trigger", "right") == "right"
val value = request.optDouble("value", 1.0).toFloat().coerceIn(0f, 1f)
bridge.applyTouch { if (right) it.rightTrigger = value else it.leftTrigger = value }
hold(request, 200)
bridge.applyTouch { if (right) it.rightTrigger = 0f else it.leftTrigger = 0f }
return JSONObject().put("trigger", if (right) "right" else "left")
}
private fun hold(request: JSONObject, defaultMs: Long) =
Thread.sleep(request.optLong("holdMs", defaultMs).coerceIn(0, 10_000))
private val DPAD = setOf("up", "down", "left", "right")
private val BUTTONS = mapOf(
"a" to PadState.A, "b" to PadState.B, "x" to PadState.X, "y" to PadState.Y,
"lb" to PadState.LB, "rb" to PadState.RB, "select" to PadState.SELECT, "back" to PadState.SELECT,
"start" to PadState.START, "menu" to PadState.START, "l3" to PadState.L3, "r3" to PadState.R3,
"guide" to PadState.GUIDE, "steam" to PadState.GUIDE,
)
/** Linux evdev codes (input-event-codes.h) by name. */
private val KEYS: Map<String, Int> = buildMap {
"1234567890".forEachIndexed { i, c -> put(c.toString(), 2 + i) }
"QWERTYUIOP".forEachIndexed { i, c -> put(c.toString(), 16 + i) }
"ASDFGHJKL".forEachIndexed { i, c -> put(c.toString(), 30 + i) }
"ZXCVBNM".forEachIndexed { i, c -> put(c.toString(), 44 + i) }
(1..10).forEach { put("F$it", 58 + it) }
put("F11", 87); put("F12", 88)
putAll(mapOf(
"ESC" to 1, "MINUS" to 12, "EQUAL" to 13, "BACKSPACE" to 14, "TAB" to 15, "LEFTBRACE" to 26,
"RIGHTBRACE" to 27, "ENTER" to 28, "LEFTCTRL" to 29, "SEMICOLON" to 39, "APOSTROPHE" to 40,
"GRAVE" to 41, "LEFTSHIFT" to 42, "BACKSLASH" to 43, "COMMA" to 51, "DOT" to 52, "SLASH" to 53,
"RIGHTSHIFT" to 54, "LEFTALT" to 56, "SPACE" to 57, "CAPSLOCK" to 58, "RIGHTCTRL" to 97,
"RIGHTALT" to 100, "HOME" to 102, "UP" to 103, "PAGEUP" to 104, "LEFT" to 105, "RIGHT" to 106,
"END" to 107, "DOWN" to 108, "PAGEDOWN" to 109, "INSERT" to 110, "DELETE" to 111, "LEFTMETA" to 125,
"CTRL" to 29, "SHIFT" to 42, "ALT" to 56, "META" to 125, "SUPER" to 125, "RETURN" to 28, "ESCAPE" to 1,
))
}
/** US-layout characters as (evdev code, shift). */
private val TYPED: Map<Char, Pair<Int, Boolean>> = buildMap {
for (c in 'a'..'z') put(c, KEYS.getValue(c.uppercase()) to false)
for (c in 'A'..'Z') put(c, KEYS.getValue(c.toString()) to true)
for (c in '0'..'9') put(c, KEYS.getValue(c.toString()) to false)
"!@#$%^&*()".forEachIndexed { i, c -> put(c, KEYS.getValue(((i + 1) % 10).toString()) to true) }
putAll(mapOf(
' ' to (57 to false), '\n' to (28 to false), '\t' to (15 to false),
'-' to (12 to false), '_' to (12 to true), '=' to (13 to false), '+' to (13 to true),
'[' to (26 to false), '{' to (26 to true), ']' to (27 to false), '}' to (27 to true),
';' to (39 to false), ':' to (39 to true), '\'' to (40 to false), '"' to (40 to true),
'`' to (41 to false), '~' to (41 to true), '\\' to (43 to false), '|' to (43 to true),
',' to (51 to false), '<' to (51 to true), '.' to (52 to false), '>' to (52 to true),
'/' to (53 to false), '?' to (53 to true),
))
}
}
@@ -38,6 +38,10 @@ class AgentStartActivity : Activity() {
Log.e(TAG, "missing agent start request")
return null
}
if (encoded.length > AgentAccess.MAX_REQUEST_CHARS) {
Log.w(TAG, "agent start request is too large")
return null
}
return try {
JSONObject(String(Base64.decode(encoded, Base64.DEFAULT), Charsets.UTF_8))
} catch (e: Exception) {
@@ -47,6 +51,13 @@ class AgentStartActivity : Activity() {
}
private fun startSession(request: JSONObject) {
try {
AgentAccess.requireStart(this, request)
} catch (e: AgentException) {
Log.w(TAG, e.code)
finish()
return
}
val requestedMode = request.optString("mode")
val sessionMode = when (requestedMode) {
"steam" -> SessionService.MODE_STEAM
@@ -0,0 +1,157 @@
package com.droiddeck.launcher.agent
import android.os.Build
import android.os.Handler
import android.os.Looper
import android.view.View
import android.view.ViewGroup
import android.view.inspector.WindowInspector
import androidx.compose.ui.platform.ViewRootForTest
import androidx.compose.ui.semantics.SemanticsActions
import androidx.compose.ui.semantics.SemanticsConfiguration
import androidx.compose.ui.semantics.SemanticsNode
import androidx.compose.ui.semantics.SemanticsProperties
import androidx.compose.ui.semantics.getOrNull
import androidx.compose.ui.text.AnnotatedString
import org.json.JSONArray
import org.json.JSONObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicReference
/**
* The app's own UI as an agent sees it: every Compose node in every window of the process (popups
* and menus included), read from Compose's semantics rather than through UI Automator, which on a
* dual-screen device dumps whichever window has focus - often not this app's. Nodes carry their
* test tag (ui/TestTags.kt), text, content description, role, state and on-screen bounds; click
* and set-text act on a node by tag or text, so a test needs no coordinates.
*/
object AgentUi {
private const val TIMEOUT_MS = 5_000L
/** Setup's Debugging tools toggle: the bridge never acts on its own gate (AgentAccess). */
private const val GATE_TAG = "setting-agent-commands"
fun handle(request: JSONObject): JSONObject = when (val op = request.optString("op", "dump")) {
"dump" -> onMain { dump(request.optBoolean("all")) }
"click" -> onMain { act(request) { node -> perform(node, SemanticsActions.OnClick) } }
"set-text" -> onMain {
val text = request.optString("text")
act(request) { node ->
node.config.getOrNull(SemanticsActions.SetText)?.action?.invoke(AnnotatedString(text))
?: throw AgentException("NOT_EDITABLE", "That node takes no text")
}
}
else -> throw AgentException("INVALID_REQUEST", "ui op '$op' is not dump, click or set-text")
}.put("ok", true).put("command", "ui")
private fun roots(): List<Pair<View, ViewRootForTest>> {
if (Build.VERSION.SDK_INT < 29) throw AgentException("UNSUPPORTED", "The UI dump needs Android 10 or later")
val found = mutableListOf<Pair<View, ViewRootForTest>>()
fun walk(view: View) {
if (view is ViewRootForTest) found += view to view
if (view is ViewGroup) for (i in 0 until view.childCount) walk(view.getChildAt(i))
}
WindowInspector.getGlobalWindowViews().filter { it.isAttachedToWindow && it.isShown }.forEach(::walk)
return found
}
private fun dump(all: Boolean): JSONObject {
val windows = JSONArray()
for ((view, root) in roots()) {
val nodes = JSONArray()
val location = IntArray(2).also { view.getLocationOnScreen(it) }
fun visit(node: SemanticsNode) {
val json = describe(node, location)
if (all || json.has("tag") || json.has("text") || json.has("description") || json.optBoolean("clickable")) nodes.put(json)
node.children.forEach(::visit)
}
visit(root.semanticsOwner.rootSemanticsNode)
windows.put(JSONObject()
.put("display", view.display?.displayId ?: JSONObject.NULL)
.put("window", view.rootView.javaClass.simpleName)
.put("nodes", nodes))
}
return JSONObject().put("windows", windows)
}
private fun describe(node: SemanticsNode, origin: IntArray): JSONObject {
val config = node.config
val json = JSONObject().put("id", node.id)
config.getOrNull(SemanticsProperties.TestTag)?.let { json.put("tag", it) }
val text = textFields(config)
text.keys().forEach { json.put(it, text.get(it)) }
config.getOrNull(SemanticsProperties.Role)?.let { json.put("role", it.toString()) }
config.getOrNull(SemanticsProperties.ToggleableState)?.let { json.put("toggle", it.name) }
config.getOrNull(SemanticsProperties.Selected)?.let { json.put("selected", it) }
config.getOrNull(SemanticsProperties.Focused)?.let { json.put("focused", it) }
if (config.contains(SemanticsProperties.Disabled)) json.put("disabled", true)
if (config.contains(SemanticsActions.OnClick)) json.put("clickable", true)
if (config.contains(SemanticsActions.SetText)) json.put("editable", true)
val b = node.boundsInWindow
json.put("bounds", JSONArray()
.put(origin[0] + b.left.toInt()).put(origin[1] + b.top.toInt())
.put(origin[0] + b.right.toInt()).put(origin[1] + b.bottom.toInt()))
return json
}
internal fun textFields(config: SemanticsConfiguration): JSONObject {
val password = config.contains(SemanticsProperties.Password)
val json = JSONObject()
for ((key, value) in listOf(
"text" to config.getOrNull(SemanticsProperties.Text)?.joinToString(" ") { it.text },
"editableText" to config.getOrNull(SemanticsProperties.EditableText)?.text,
"description" to config.getOrNull(SemanticsProperties.ContentDescription)?.joinToString(" "),
"state" to config.getOrNull(SemanticsProperties.StateDescription),
)) value?.let { json.put(key, if (password) "<redacted>" else it) }
return json
}
/**
* The node a request names by "tag" (exact) or "match" (text or description containing it,
* ignoring case); "index" picks among several, in tree order.
*/
private fun act(request: JSONObject, action: (SemanticsNode) -> Unit): JSONObject {
val tag = request.optString("tag").takeIf { it.isNotEmpty() }
val text = if (tag == null) request.optString("match").takeIf { it.isNotEmpty() } else null
if (tag == null && text == null) throw AgentException("INVALID_REQUEST", "Name the node by tag or match")
val matches = mutableListOf<Pair<SemanticsNode, IntArray>>()
for ((view, root) in roots()) {
val origin = IntArray(2).also { view.getLocationOnScreen(it) }
fun visit(node: SemanticsNode) {
val config = node.config
val hit = if (tag != null) config.getOrNull(SemanticsProperties.TestTag) == tag
else (config.getOrNull(SemanticsProperties.Text)?.joinToString(" ") { it.text }
?: config.getOrNull(SemanticsProperties.ContentDescription)?.joinToString(" ") ?: "")
.contains(text!!, ignoreCase = true)
if (hit) matches += node to origin
node.children.forEach(::visit)
}
visit(root.semanticsOwner.rootSemanticsNode)
}
val index = request.optInt("index", 0)
val (node, origin) = matches.getOrNull(index)
?: throw AgentException("NO_SUCH_NODE", "No node with ${if (tag != null) "tag '$tag'" else "text '$text'"} (${matches.size} found)")
if (generateSequence(node) { it.parent }.any { it.config.getOrNull(SemanticsProperties.TestTag) == GATE_TAG }) {
throw AgentException("PROTECTED_CONTROL", "Debugging tools are switched on the device only")
}
action(node)
return JSONObject().put("matches", matches.size).put("node", describe(node, origin))
}
private fun perform(node: SemanticsNode, key: androidx.compose.ui.semantics.SemanticsPropertyKey<androidx.compose.ui.semantics.AccessibilityAction<() -> Boolean>>) {
val action = node.config.getOrNull(key)?.action ?: throw AgentException("NOT_CLICKABLE", "That node has no click action")
action()
}
private fun <T> onMain(block: () -> T): T {
if (Looper.myLooper() == Looper.getMainLooper()) return block()
val result = AtomicReference<Result<T>>()
val done = CountDownLatch(1)
Handler(Looper.getMainLooper()).post {
result.set(runCatching(block))
done.countDown()
}
if (!done.await(TIMEOUT_MS, TimeUnit.MILLISECONDS)) throw AgentException("TIMEOUT", "The UI thread did not answer")
return result.get().getOrThrow()
}
}
@@ -40,7 +40,7 @@ object HostProcess {
if (onLine == null) builder.redirectOutput(File("/dev/null"))
val process = builder.start()
val pid = pidOf(process)
Log.i(TAG, "started pid $pid: $command")
Log.i(TAG, "started pid $pid: ${argv.first()} (${argv.size - 1} arguments)")
if (onLine != null) {
Thread({
try {
@@ -59,7 +59,7 @@ object HostProcess {
}
pid
} catch (e: Exception) {
Log.e(TAG, "could not start: $command", e)
Log.e(TAG, "could not start: ${argv.first()}", e)
-1
}
}
@@ -41,6 +41,7 @@ object SessionFiles {
"libblfastpath.so" to "usr/local/lib/libblfastpath.so",
"libssbs.so" to "usr/local/lib/libssbs.so",
"usr/local/bin/droiddeck-session" to "usr/local/bin/droiddeck-session",
"usr/local/bin/droiddeck-agent" to "usr/local/bin/droiddeck-agent",
"usr/local/bin/steam-compatibility" to "usr/local/bin/steam-compatibility",
"usr/local/bin/droiddeck-clipboard" to "usr/local/bin/droiddeck-clipboard",
"usr/local/bin/droiddeck-steam-install" to "usr/local/bin/droiddeck-steam-install",
@@ -293,7 +293,7 @@ class SessionService : Service() {
val lines = FileUtils.readString(file)?.lines().orEmpty()
.map { it.trim() }
.filter { it.isNotEmpty() && !it.startsWith("#") && it.contains('=') && !it.startsWith("=") }
if (lines.isNotEmpty()) Log.i(TAG, "extra environment from ${file.path}: $lines")
if (lines.isNotEmpty()) Log.i(TAG, "extra environment keys from ${file.path}: ${lines.map { it.substringBefore('=') }}")
return lines
}
@@ -324,6 +324,7 @@ class SessionService : Service() {
val runtimeDir = File(filesDir, ".wayland-rt").apply { mkdirs() }
killStragglers()
SessionFiles.stage(this, root)
com.droiddeck.launcher.agent.AgentGuest.reset(this)
val sessionDir = openSessionFolder()
val sessionLog = File(sessionDir, "session.log")
@@ -423,7 +424,7 @@ class SessionService : Service() {
}
guest.add(program)
guest.addAll(SessionState.programArgs)
Log.i(TAG, "run: $program ${SessionState.programArgs.joinToString(" ")} under gamescope")
Log.i(TAG, "run: $program (${SessionState.programArgs.size} arguments) under gamescope")
}
// Android has no /dev/shm; the cache stands in for it and, unlike the real thing, keeps
@@ -659,6 +660,8 @@ class SessionService : Service() {
// (ZINK_DESCRIPTORS=lazy, MESA_*), gamescope's, the client's - whatever the experiment
// needs, without a build per attempt.
extraEnv().forEach { guest.add(it) }
// The agent bridge's experiment lines (AgentEnv), after the user's so an agent's win.
com.droiddeck.launcher.agent.AgentEnv.beginSession(this).forEach { guest.add(it) }
// Core masks, Bannerlator's two (cfca3912). The client's is sent whenever the override is
// on, even naming every core: it exists to undo the pin Steam applies to its own interface
// renderer, and the scheduler's default is exactly what that pin takes away. A game's is
@@ -1404,6 +1407,9 @@ class SessionService : Service() {
SessionState.pipActive = false
pipTask = false
SessionState.guestPid = -1
com.droiddeck.launcher.agent.AgentGuest.stop()
runCatching { com.droiddeck.launcher.agent.AgentEnv.endSession(this) }
.onFailure { Log.w(TAG, "clearing the agent's session environment", it) }
releaseLocks()
if (status == 0) {
SessionEvents.transition(SessionPhase.IDLE, "session.stopped", mapOf("status" to status))
@@ -80,6 +80,9 @@ object SessionState {
/** Told (on the thread that changed it) whenever [deckPad] changes. */
@Volatile var deckPadListener: (() -> Unit)? = null
/** The session screen's virtual pad, for the agent bridge's pad input (AgentInput). */
@Volatile var padBridge: com.droiddeck.launcher.input.PadBridge? = null
/** What the second screen shows, and on which display: kept for the session, so a screen
* turned off (sleep, a closed lid) or an activity recreated brings it back as it was. */
@Volatile var secondScreenMode = com.droiddeck.launcher.input.SecondScreenMode.NONE
@@ -1,5 +1,6 @@
package com.droiddeck.launcher.ui
import androidx.compose.ui.platform.testTag
import com.droiddeck.launcher.R
import androidx.compose.ui.res.stringResource
import androidx.compose.foundation.LocalIndication
@@ -71,7 +72,7 @@ internal fun GamesPage(s: FrontEndState, a: FrontEndActions, selected: String, o
Rise(0) { PageHeader(stringResource(R.string.content_games)) }
Rise(1) { Note(stringResource(if (s.shortcutPicker && s.shortcutLibraryScanning) R.string.game_shortcut_scanning else R.string.games_empty)) }
if (!s.shortcutPicker) Rise(2) {
Actions { PrimaryButton(stringResource(R.string.games_play_steam), enabled = !s.busy, main = true, icon = Icons.Filled.PlayArrow, modifier = Modifier.padding(top = 12.dp), onClick = a.onPlay) }
Actions { PrimaryButton(stringResource(R.string.games_play_steam), enabled = !s.busy, main = true, icon = Icons.Filled.PlayArrow, modifier = Modifier.padding(top = 12.dp).testTag("play-steam"), onClick = a.onPlay) }
}
GameFileFolderActions(s, a)
}
@@ -1,5 +1,6 @@
package com.droiddeck.launcher.ui
import androidx.compose.ui.platform.testTag
import com.droiddeck.launcher.R
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.focus.focusRequester
@@ -162,6 +163,7 @@ private fun RailItem(
contentAlignment = Alignment.Center,
modifier = Modifier
.then(if (frontFocus != null) Modifier.focusRequester(frontFocus.railFor(key)) else Modifier)
.testTag("rail-$key")
.graphicsLayer { scaleX = scale; scaleY = scale }
.size(width = if (iconOnly) 52.dp else 80.dp, height = height)
.clip(Shape14)
@@ -91,6 +91,8 @@ class FrontEndState(
val frameGenLabel: String,
val romsDir: String?,
val logsEnabled: Boolean,
/** Setup's Debugging tools toggle (AgentAccess): off unless the user turns it on. */
val agentCommands: Boolean = false,
val steamGames: List<Library.SteamGame>,
val emulators: List<Library.Emulator>,
val running: String?,
@@ -166,6 +168,7 @@ class FrontEndActions(
val onFiles: () -> Unit,
val onBrowseFiles: (File) -> Unit = {},
val onLogs: () -> Unit,
val onAgentCommands: () -> Unit = {},
val onShareLogs: () -> Unit = {},
val onClearLogs: () -> Unit = {},
val onOffline: () -> Unit,
@@ -307,6 +307,7 @@ internal fun SetupPanel(
}
}
ToggleRow(host, "logs", stringResource(R.string.setup_logs), stringResource(R.string.setup_logs_hint), s.logsEnabled) { a.onLogs() }
ToggleRow(host, "agent-commands", stringResource(R.string.setup_agent_commands), null, s.agentCommands) { a.onAgentCommands() }
ActionRow(stringResource(R.string.setup_latest_logs), stringResource(R.string.drawer_logs_hint), stringResource(R.string.drawer_share_logs), a.onShareLogs,
progress = com.droiddeck.launcher.session.SessionLogShare.progress)
ActionRow(stringResource(R.string.setup_saved_logs), stringResource(R.string.setup_saved_logs_hint, com.droiddeck.launcher.session.SessionPaths.KEEP_SESSIONS), stringResource(R.string.setup_clear_logs), a.onClearLogs)
@@ -1,5 +1,6 @@
package com.droiddeck.launcher.ui
import androidx.compose.ui.platform.testTag
import androidx.compose.ui.draw.alpha
import androidx.compose.animation.core.Animatable
import androidx.compose.foundation.Canvas
@@ -310,7 +311,7 @@ internal fun SessionDrawer(open: Boolean, requestedPage: SessionDrawerPage, cont
modifier = Modifier.fillMaxSize()
.clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) { host.open = null; a.onClose() },
)
androidx.compose.foundation.layout.BoxWithConstraints(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.CenterEnd) {
androidx.compose.foundation.layout.BoxWithConstraints(modifier = Modifier.fillMaxSize().exposeTestTags(), contentAlignment = Alignment.CenterEnd) {
// A 4:3 or near-square screen (under 440dp tall) gets tighter padding and a shorter tab row,
// so the settings keep most of the height; a narrow one keeps some of the game in view.
val short = maxHeight < 440.dp
@@ -621,7 +622,7 @@ private fun StopSessionButton(modifier: Modifier = Modifier, onClick: () -> Unit
val stopDescription = stringResource(R.string.stop_session)
Row(
verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(6.dp),
modifier = modifier.heightIn(min = 44.dp).semantics { contentDescription = stopDescription }
modifier = modifier.heightIn(min = 44.dp).semantics { contentDescription = stopDescription }.testTag("drawer-stop")
.clip(shape).background(fill).glideBorder(hot, shape, colors.error.copy(alpha = 0.9f), colors.error.copy(alpha = 0.55f))
.hoverable(src).clickable(interactionSource = src, indication = LocalIndication.current, onClick = onClick)
.controllerConfirm(onClick = onClick)
@@ -732,6 +733,7 @@ private fun DrawerPageTabs(page: SessionDrawerPage, pages: List<SessionDrawerPag
)
.glideBorder(focused, RoundedCornerShape(16.dp), colors.onBackground)
.semantics { contentDescription = pageDescription }
.testTag("drawer-tab-${item.name.lowercase()}")
.hoverable(source)
.clickable(interactionSource = source, indication = LocalIndication.current, onClick = select)
.controllerConfirm(onClick = select),
@@ -1172,14 +1174,14 @@ private fun StopConfirm(
ConfirmChoice(
stringResource(R.string.common_cancel), danger = false, enabled = open,
modifier = Modifier.graphicsLayer { alpha = items[1].value; translationY = (1f - items[1].value) * 6.dp.toPx() }
.focusRequester(cancelFocus).onFocusChanged { cancelFocused = it.isFocused },
.focusRequester(cancelFocus).onFocusChanged { cancelFocused = it.isFocused }.testTag("drawer-stop-cancel"),
onClick = onCancel,
)
ConfirmChoice(
stringResource(R.string.drawer_stop), danger = true, enabled = open,
modifier = Modifier.graphicsLayer { alpha = items[2].value; translationY = (1f - items[2].value) * 6.dp.toPx() }
.onFocusChanged { stopFocused = it.isFocused }
.onGloballyPositioned { stopCoords = it },
.onGloballyPositioned { stopCoords = it }.testTag("drawer-stop-confirm"),
) {
val at = IntArray(2).also { view.rootView.getLocationOnScreen(it) }
onStop(stopCoords?.takeIf { it.isAttached }?.boundsInWindow()?.translate(at[0].toFloat(), at[1].toFloat()))
@@ -1,5 +1,6 @@
package com.droiddeck.launcher.ui
import androidx.compose.ui.platform.testTag
import com.droiddeck.launcher.R
import androidx.compose.ui.res.stringResource
import android.view.KeyEvent
@@ -186,6 +187,7 @@ internal fun TabStrip(
color = if (on) pal.onSignal else if (hot) colors.onBackground else colors.onSurfaceVariant,
modifier = Modifier
.bringIntoViewRequester(reveal)
.testTag("tab-$i")
.then(focusRequesters?.getOrNull(i)?.let { Modifier.focusRequester(it) } ?: Modifier)
.clip(tabShape)
.background(if (on) pal.signal else if (hot) pal.signal.copy(alpha = 0.16f) else Color.Transparent)
@@ -456,7 +458,7 @@ fun <T> ChoiceRow(
val open = host.open == key
SettingsRow(label, hint, highlighted = open, hintLines = hintLines) {
Box {
ValueChip(options.firstOrNull { it.first == selected }?.second ?: "-", open, enabled, modifier = chipModifier) { host.open = if (open) null else key }
ValueChip(options.firstOrNull { it.first == selected }?.second ?: "-", open, enabled, modifier = chipModifier.testTag("setting-$key")) { host.open = if (open) null else key }
AnchoredMenu(open, onDismiss = { if (host.open == key) host.open = null }, title = label, note = note) { firstItemFocus ->
options.forEachIndexed { index, (value, text) ->
MenuItem(text, checked = value == selected, focusRequester = if (index == 0) firstItemFocus else null) {
@@ -472,7 +474,7 @@ fun <T> ChoiceRow(
@Composable
fun ToggleRow(host: MenuHost, key: String, label: String, hint: String?, checked: Boolean, enabled: Boolean = true, chipModifier: Modifier = Modifier, onChange: (Boolean) -> Unit) =
SettingsRow(label, hint) {
ToggleSwitch(checked, enabled, label, chipModifier) { host.open = null; onChange(it) }
ToggleSwitch(checked, enabled, label, chipModifier.testTag("setting-$key")) { host.open = null; onChange(it) }
}
/** An on/off switch: one tap or one A press flips it, where a menu of On and Off took three. */
@@ -1,5 +1,6 @@
package com.droiddeck.launcher.ui
import androidx.compose.ui.platform.testTag
import com.droiddeck.launcher.R
import androidx.compose.ui.res.stringResource
import androidx.compose.animation.core.FastOutSlowInEasing
@@ -91,7 +92,7 @@ internal fun SteamHome(s: FrontEndState, a: FrontEndActions, modifier: Modifier)
Spacer(Modifier.height(18.dp))
Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(10.dp)) {
// Enabled without a runtime: the session's loading screen installs it first.
PrimaryButton(stringResource(R.string.games_play_steam), enabled = !s.busy, main = true, large = true, icon = Icons.Filled.PlayArrow, onClick = a.onPlay)
PrimaryButton(stringResource(R.string.games_play_steam), enabled = !s.busy, main = true, large = true, icon = Icons.Filled.PlayArrow, modifier = Modifier.testTag("play-steam"), onClick = a.onPlay)
Cog(size = 54.dp, onClick = a.onSteamSettings)
}
}
@@ -0,0 +1,14 @@
package com.droiddeck.launcher.ui
import androidx.compose.ui.ExperimentalComposeUiApi
import androidx.compose.ui.Modifier
import androidx.compose.ui.semantics.semantics
import androidx.compose.ui.semantics.testTagsAsResourceId
/**
* Compose test tags as Android resource ids below this node, so UI Automator (`uiautomator dump`,
* droiddeckctl ui-dump) sees stable ids such as `rail-steam` or `setting-logs` rather than only
* labels that change with the language. Tags are named in docs/agent-control.md.
*/
@OptIn(ExperimentalComposeUiApi::class)
fun Modifier.exposeTestTags(): Modifier = semantics { testTagsAsResourceId = true }