From 61031e3c4b76f98261cf97ef526e6df68d512d90 Mon Sep 17 00:00:00 2001 From: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:44:39 -0500 Subject: [PATCH] feat(lab): build and run Lab builds of DroidDeck on the device itself (#2375) * feat(lab): load an on-device Lab build over the installed app * feat(lab): build Lab builds on the device and report the APK commit to agents * fix(lab): dd-agent connects loopback adb before starting Claude Code * feat(lab): tell the app's own agent to continue after a Lab build restart * fix(lab): size the build heap from the device's memory * feat(lab): keep earlier Lab builds and list them in Updates * feat(lab): notify when DroidDeck comes back on a Lab build or turns one off * feat(lab): estimate the time left and compile the installed commit ahead of the first build * fix(lab): report a failed command as a failed build and update progress every five seconds * feat(lab): a kept build for another version can be rebuilt * feat(lab): record which installed commit the build caches are warm for * fix(lab): the app's agent hands its builds to the app, so they outlive the agent's turn * perf(lab): BuildConfig describes the installed base, so a Lab commit does not change it * perf(lab): compile with the Kotlin daemon, since in-process Kotlin compiles always redo the whole app * fix(lab): a build whose last task was up to date no longer counts as failed * feat(lab): dd-lab-build --explain logs why tasks ran and Kotlin's build report * fix(lab): keep a waiting build when another is staged or Lab is turned off, and never restart during a session * fix(lab): keep the installed build's label and exact commit, and refuse AppLanguage changes * fix(lab): setup verifies its downloads and waits for adb before preparing * fix(agent): quote apostrophes in opened URLs and keep the sign-in pid --- .../main/java/com/droiddeck/launcher/App.kt | 6 +- .../launcher/agent/AgentBridgeProvider.kt | 2 + .../com/droiddeck/launcher/lab/LabNotice.kt | 60 ++++ .../com/droiddeck/launcher/lab/LabOverlay.kt | 276 ++++++++++++++++++ .../droiddeck/launcher/ui/BuildWatermark.kt | 3 +- .../com/droiddeck/launcher/ui/LabBuilds.kt | 162 ++++++++++ .../com/droiddeck/launcher/ui/UpdatesPage.kt | 7 +- app/src/main/res/values/strings.xml | 23 ++ .../droiddeck/launcher/lab/LabOverlayTest.kt | 154 ++++++++++ docs/lab-builds.md | 63 ++++ tools/agent/android-open | 9 + tools/agent/dd-agent | 5 + tools/agent/dd-gh-login | 24 ++ tools/agent/dd-lab-build | 226 ++++++++++++++ tools/agent/setup | 76 +++++ 15 files changed, 1091 insertions(+), 5 deletions(-) create mode 100644 app/src/main/java/com/droiddeck/launcher/lab/LabNotice.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/lab/LabOverlay.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/ui/LabBuilds.kt create mode 100644 app/src/test/java/com/droiddeck/launcher/lab/LabOverlayTest.kt create mode 100644 docs/lab-builds.md create mode 100755 tools/agent/android-open create mode 100755 tools/agent/dd-agent create mode 100755 tools/agent/dd-gh-login create mode 100755 tools/agent/dd-lab-build create mode 100755 tools/agent/setup diff --git a/app/src/main/java/com/droiddeck/launcher/App.kt b/app/src/main/java/com/droiddeck/launcher/App.kt index be81fc6960..b4c1621a47 100644 --- a/app/src/main/java/com/droiddeck/launcher/App.kt +++ b/app/src/main/java/com/droiddeck/launcher/App.kt @@ -7,8 +7,11 @@ import com.droiddeck.launcher.session.EnhancedSessionLogShare import com.droiddeck.launcher.session.GpuClockPin class App : Application() { - override fun attachBaseContext(base: android.content.Context) = + override fun attachBaseContext(base: android.content.Context) { super.attachBaseContext(com.droiddeck.launcher.core.AppLanguage.wrap(base)) + val process = if (android.os.Build.VERSION.SDK_INT >= 28) Application.getProcessName() else packageName + com.droiddeck.launcher.lab.LabOverlay.load(this, process == packageName) + } override fun onConfigurationChanged(newConfig: android.content.res.Configuration) { super.onConfigurationChanged(newConfig) @@ -39,6 +42,7 @@ class App : Application() { if (process == "$packageName:vr_recovery" || process == "$packageName:rgb_diagnostic" || process == "$packageName:shortcuts_sync" || process == "$packageName:bottom_desktop") return CrashHandler.install(this) + com.droiddeck.launcher.lab.LabNotice.started(this) com.droiddeck.launcher.core.SteamIntegration.syncShortcutsListing(this) com.droiddeck.launcher.session.QuestExperimental.initialize(this) com.droiddeck.launcher.session.SessionPrefs.applyNtsyncDefault(this) diff --git a/app/src/main/java/com/droiddeck/launcher/agent/AgentBridgeProvider.kt b/app/src/main/java/com/droiddeck/launcher/agent/AgentBridgeProvider.kt index 6b055221e5..93922c8050 100644 --- a/app/src/main/java/com/droiddeck/launcher/agent/AgentBridgeProvider.kt +++ b/app/src/main/java/com/droiddeck/launcher/agent/AgentBridgeProvider.kt @@ -288,6 +288,8 @@ class AgentBridgeProvider : ContentProvider() { return JSONObject() .put("schema", SCHEMA) .put("build", BuildConfig.BUILD_LABEL) + .put("apkCommit", com.droiddeck.launcher.lab.LabOverlay.apkCommit) + .put("lab", com.droiddeck.launcher.lab.LabOverlay.active?.label ?: JSONObject.NULL) .put("appVersion", context.packageManager.getPackageInfo(context.packageName, 0).versionName) .put("runtime", JSONObject() .put("installed", LinuxRuntime.isInstalled(context)) diff --git a/app/src/main/java/com/droiddeck/launcher/lab/LabNotice.kt b/app/src/main/java/com/droiddeck/launcher/lab/LabNotice.kt new file mode 100644 index 0000000000..85e1bab990 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/lab/LabNotice.kt @@ -0,0 +1,60 @@ +package com.droiddeck.launcher.lab + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import com.droiddeck.launcher.FrontEndActivity +import com.droiddeck.launcher.R +import com.droiddeck.launcher.stores.download.StoreDownloadService + +/** + * Says what happened to a Lab build when the app starts, so a restart onto a new build does not + * look like a crash, and a build Android or the safety checks turned off is not lost silently. + */ +object LabNotice { + private const val CHANNEL_ID = "lab" + private const val NOTIFICATION_ID = 4120 + + /** Where a start that just moved a Lab build into use opens; the agent that built it sets it. */ + @Volatile var openOnInstall: String = "updates" + + fun started(context: Context) { + val app = context.applicationContext + val active = LabOverlay.active + val skipped = LabOverlay.skipped + val current = LabOverlay.current(app) + when { + LabOverlay.promoted && active != null -> + post(app, app.getString(R.string.lab_installed_title), app.getString(R.string.lab_installed_text, active.label), openOnInstall) + skipped == LabOverlay.SKIPPED_APP_IMAGE && current != null -> + post(app, app.getString(R.string.lab_paused_title), app.getString(R.string.lab_paused_text, current.label), "updates") + skipped == LabOverlay.Reason.OTHER_INSTALL.name || skipped == LabOverlay.Reason.DID_NOT_START.name -> { + val reason = if (skipped == LabOverlay.Reason.OTHER_INSTALL.name) R.string.lab_off_other_install else R.string.lab_off_did_not_start + post(app, app.getString(R.string.lab_off_title), app.getString(reason), "updates") + } + } + } + + private fun post(app: Context, title: String, text: String, nav: String) { + val manager = app.getSystemService(NotificationManager::class.java) ?: return + manager.createNotificationChannel(NotificationChannel(CHANNEL_ID, app.getString(R.string.upd_lab), NotificationManager.IMPORTANCE_DEFAULT)) + val open = PendingIntent.getActivity( + app, NOTIFICATION_ID, + Intent(app, FrontEndActivity::class.java).putExtra(StoreDownloadService.EXTRA_NAV, nav) + .addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_REORDER_TO_FRONT), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ) + val notification = Notification.Builder(app, CHANNEL_ID) + .setSmallIcon(R.drawable.ic_stat_session) + .setContentTitle(title) + .setContentText(text) + .setStyle(Notification.BigTextStyle().bigText(text)) + .setContentIntent(open) + .setAutoCancel(true) + .build() + runCatching { manager.notify(NOTIFICATION_ID, notification) } + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/lab/LabOverlay.kt b/app/src/main/java/com/droiddeck/launcher/lab/LabOverlay.kt new file mode 100644 index 0000000000..7fb7f7a26d --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/lab/LabOverlay.kt @@ -0,0 +1,276 @@ +package com.droiddeck.launcher.lab + +import android.app.Activity +import android.app.Application +import android.content.Context +import android.content.Intent +import android.content.res.Resources +import android.content.res.loader.ResourcesLoader +import android.content.res.loader.ResourcesProvider +import android.os.Build +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.ParcelFileDescriptor +import android.util.Log +import androidx.annotation.RequiresApi +import androidx.annotation.VisibleForTesting +import com.droiddeck.launcher.BuildConfig +import org.json.JSONObject +import java.io.File + +/** + * A Lab build: app code and resources an on-device agent built from this install's own commit, + * loaded over the installed APK at startup so it can be tried without reinstalling or re-signing. + * + * `files/lab/next` holds a build waiting for the next start and `files/lab/active` the one in use, + * each with `overlay.apk` (the build's dex files and linked resources) and `overlay.json`. A build + * that is replaced, turned off or refused moves to `files/lab/builds`, so it can be used again. + */ +object LabOverlay { + data class Info(val label: String, val baseCommit: String, val baseApk: String, val built: Long = 0) + + enum class Reason { REPLACED, TURNED_OFF, OTHER_INSTALL, DID_NOT_START, UNREADABLE } + + /** A build kept in `files/lab/builds`; [usable] when it was made for this install. */ + data class Saved(val dir: File, val info: Info?, val reason: Reason?, val usable: Boolean) + + private const val TAG = "LabOverlay" + private const val MAX_FAILED_STARTS = 2 + private const val STABLE_AFTER_MS = 10_000L + private const val KEEP_BUILDS = 12 + const val SKIPPED_APP_IMAGE = "app image present" + + /** The installed APK's commit; this class always comes from the APK, never from a Lab build. */ + val apkCommit: String = BuildConfig.SOURCE_COMMIT + + @Volatile var active: Info? = null + private set + @Volatile var skipped: String? = null + private set + /** This start moved a newly staged build into use. */ + @Volatile var promoted = false + private set + + private var loader: Any? = null + + @VisibleForTesting + internal var injectDex: (ClassLoader, File) -> Unit = ::prependDex + @VisibleForTesting + internal var openResources: (File) -> Any? = { if (Build.VERSION.SDK_INT >= 30) createLoader(it) else null } + @VisibleForTesting + internal var appImage: (Context) -> Boolean = ::hasAppImage + + fun root(context: Context) = File(context.filesDir, "lab") + + /** Called from [Application.attachBaseContext], before any class the build may change is loaded. */ + fun load(app: Application, mainProcess: Boolean) { + active = null + skipped = null + promoted = false + try { + val root = root(app) + if (mainProcess) promoted = promote(root) + val dir = File(root, "active") + val apk = File(dir, "overlay.apk") + val meta = File(dir, "overlay.json") + if (!apk.isFile || !meta.isFile) return + val info = parse(meta.readText()) ?: return refuse(root, mainProcess, Reason.UNREADABLE) + if (!madeFor(app, info)) return refuse(root, mainProcess, Reason.OTHER_INSTALL) + if (Build.VERSION.SDK_INT < 30) return skip("needs Android 11") + // Classes in an app image are loaded before attachBaseContext and would ignore the overlay; + // dd-lab-build recompiles the app without one (cmd package compile -m verify). + if (appImage(app)) return skip(SKIPPED_APP_IMAGE) + if (mainProcess) { + val starts = failedStarts(root) + 1 + if (starts > MAX_FAILED_STARTS) return refuse(root, true, Reason.DID_NOT_START) + File(root, "starts").writeText(starts.toString()) + } + apk.setReadOnly() + loader = openResources(apk) + injectDex(app.classLoader, apk) + active = info + addLoader(app.resources) + app.registerActivityLifecycleCallbacks(Callbacks(root, mainProcess)) + } catch (e: Throwable) { + Log.w(TAG, "Lab build not loaded", e) + skipped = e.message ?: e.javaClass.simpleName + } + } + + /** The APK's path changes with every install and update of the app, so it pins a build to one install. */ + fun madeFor(context: Context, info: Info) = info.baseCommit == apkCommit && info.baseApk == context.applicationInfo.sourceDir + + /** The build in `active`, whether or not this start loaded it. */ + fun current(context: Context): Info? = read(File(root(context), "active")) + + /** A build staged for the next start. */ + fun pending(context: Context): Info? = read(File(root(context), "next")) + + /** Kept builds, newest first. */ + fun saved(context: Context): List = File(root(context), "builds").listFiles().orEmpty() + .filter { File(it, "overlay.apk").isFile } + .sortedByDescending { it.name.toLongOrNull() ?: 0L } + .map { dir -> + val info = read(dir) + val reason = File(dir, "reason").takeIf { it.isFile }?.readText()?.trim()?.let { r -> Reason.entries.firstOrNull { it.name == r } } + Saved(dir, info, reason, info != null && madeFor(context, info)) + } + + /** Leaves the Lab build and restarts the app on the installed APK. */ + fun turnOff(context: Context) { + val root = root(context) + archive(root, File(root, "active"), Reason.TURNED_OFF) + archive(root, File(root, "next"), Reason.TURNED_OFF) + File(root, "next").deleteRecursively() + File(root, "starts").delete() + restart(context) + } + + /** Stages [build] for the next start and keeps a build that was waiting; the caller restarts the app. */ + fun stage(context: Context, build: Saved) { + val root = root(context) + val next = File(root, "next") + val waiting = File(root, "next.replaced").apply { deleteRecursively() } + next.renameTo(waiting) + next.deleteRecursively() + File(build.dir, "reason").delete() + if (!build.dir.renameTo(next)) { + waiting.renameTo(next) + error("could not stage ${build.dir.name}") + } + archive(root, waiting, Reason.REPLACED) + waiting.deleteRecursively() + } + + fun delete(build: Saved) { + build.dir.deleteRecursively() + } + + /** + * Android's background optimization can give the APK an app image again, which keeps a Lab + * build from loading. Recompiling without one needs a shell, which the caller provides. + */ + fun needsRecompile(context: Context) = appImage(context) + + fun recompileCommand(context: Context) = "cmd package compile -m verify -f ${context.packageName}" + + fun restart(context: Context) { + val launch = context.packageManager.getLaunchIntentForPackage(context.packageName) + ?.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TASK) + if (launch != null) context.startActivity(launch) + Runtime.getRuntime().exit(0) + } + + @VisibleForTesting + internal fun parse(json: String): Info? = runCatching { + val o = JSONObject(json) + Info(o.getString("label"), o.getString("baseCommit"), o.getString("baseApk"), o.optLong("built")) + }.getOrNull() + + private fun read(dir: File): Info? = File(dir, "overlay.json").takeIf { it.isFile && File(dir, "overlay.apk").isFile }?.let { parse(it.readText()) } + + /** Moves `next` into use and keeps the build it replaces. Returns whether there was one to move. */ + @VisibleForTesting + internal fun promote(root: File): Boolean { + val next = File(root, "next") + if (!File(next, "overlay.apk").isFile || !File(next, "overlay.json").isFile) return false + val active = File(root, "active") + archive(root, active, Reason.REPLACED) + active.deleteRecursively() + if (!next.renameTo(active)) return false + File(root, "starts").delete() + return true + } + + @VisibleForTesting + internal fun archive(root: File, dir: File, reason: Reason) { + if (!File(dir, "overlay.apk").isFile) return + val builds = File(root, "builds").apply { mkdirs() } + var stamp = System.currentTimeMillis() + while (File(builds, stamp.toString()).exists()) stamp++ + val target = File(builds, stamp.toString()) + if (!dir.renameTo(target)) return + File(target, "reason").writeText(reason.name) + builds.listFiles().orEmpty().sortedByDescending { it.name.toLongOrNull() ?: 0L }.drop(KEEP_BUILDS).forEach { it.deleteRecursively() } + } + + @VisibleForTesting + internal fun failedStarts(root: File) = File(root, "starts").takeIf { it.isFile }?.readText()?.trim()?.toIntOrNull() ?: 0 + + private fun refuse(root: File, mainProcess: Boolean, reason: Reason) { + skipped = reason.name + Log.w(TAG, "Lab build turned off: $reason") + if (!mainProcess) return + archive(root, File(root, "active"), reason) + File(root, "starts").delete() + File(root, "last-off-reason").writeText(reason.name) + } + + private fun skip(reason: String) { + skipped = reason + Log.w(TAG, "Lab build not loaded: $reason") + } + + private fun hasAppImage(context: Context): Boolean { + val apk = File(context.applicationInfo.sourceDir) + val oat = File(apk.parentFile, "oat") + return oat.listFiles().orEmpty().any { File(it, "${apk.nameWithoutExtension}.art").isFile } + } + + @RequiresApi(30) + private fun createLoader(apk: File): ResourcesLoader = + ParcelFileDescriptor.open(apk, ParcelFileDescriptor.MODE_READ_ONLY).use { fd -> + ResourcesLoader().apply { addProvider(ResourcesProvider.loadFromApk(fd)) } + } + + /** Gives [resources] the Lab build's resources; contexts made from them inherit it. */ + fun addLoader(resources: Resources) { + if (Build.VERSION.SDK_INT < 30) return + val current = loader as? ResourcesLoader ?: return + resources.addLoaders(current) + } + + private fun prependDex(classLoader: ClassLoader, apk: File) { + val pathList = field(Class.forName("dalvik.system.BaseDexClassLoader"), "pathList").get(classLoader)!! + val elementsField = field(pathList.javaClass, "dexElements") + @Suppress("UNCHECKED_CAST") val installed = elementsField.get(pathList) as Array + val errors = ArrayList() + val made = try { + pathList.javaClass.getDeclaredMethod("makeDexElements", List::class.java, File::class.java, List::class.java, ClassLoader::class.java) + .apply { isAccessible = true }.invoke(null, listOf(apk), null, errors, classLoader) + } catch (_: NoSuchMethodException) { + pathList.javaClass.getDeclaredMethod( + "makeDexElements", List::class.java, File::class.java, List::class.java, ClassLoader::class.java, Boolean::class.javaPrimitiveType, + ).apply { isAccessible = true }.invoke(null, listOf(apk), null, errors, classLoader, false) + } + errors.firstOrNull()?.let { throw it } + @Suppress("UNCHECKED_CAST") val added = made as Array + if (added.isEmpty()) error("no dex in ${apk.name}") + val merged = java.lang.reflect.Array.newInstance(installed.javaClass.componentType!!, added.size + installed.size) + System.arraycopy(added, 0, merged, 0, added.size) + System.arraycopy(installed, 0, merged, added.size, installed.size) + elementsField.set(pathList, merged) + } + + private fun field(type: Class<*>, name: String) = type.getDeclaredField(name).apply { isAccessible = true } + + private class Callbacks(private val root: File, private val mainProcess: Boolean) : Application.ActivityLifecycleCallbacks { + private var settled = false + + override fun onActivityPreCreated(activity: Activity, savedInstanceState: Bundle?) = addLoader(activity.resources) + + override fun onActivityResumed(activity: Activity) { + if (settled || !mainProcess) return + settled = true + Handler(Looper.getMainLooper()).postDelayed({ File(root, "starts").delete() }, STABLE_AFTER_MS) + } + + override fun onActivityCreated(activity: Activity, savedInstanceState: Bundle?) = Unit + override fun onActivityStarted(activity: Activity) = Unit + override fun onActivityPaused(activity: Activity) = Unit + override fun onActivityStopped(activity: Activity) = Unit + override fun onActivitySaveInstanceState(activity: Activity, outState: Bundle) = Unit + override fun onActivityDestroyed(activity: Activity) = Unit + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/ui/BuildWatermark.kt b/app/src/main/java/com/droiddeck/launcher/ui/BuildWatermark.kt index 648d754aac..a9d0c3d1a2 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/BuildWatermark.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/BuildWatermark.kt @@ -38,7 +38,8 @@ object BuildWatermark { if (text.length <= TITLE_LIMIT) text else text.take(TITLE_LIMIT - 1).trimEnd() + "…" fun install(app: Application) { - val lines = lines() ?: return + val lab = com.droiddeck.launcher.lab.LabOverlay.active?.let { "LAB | ${truncate(it.label)}" } + val lines = (listOfNotNull(lab) + lines().orEmpty()).ifEmpty { return } app.registerActivityLifecycleCallbacks(object : Application.ActivityLifecycleCallbacks { override fun onActivityCreated(activity: Activity, savedInstanceState: Bundle?) { if (activity is QuestXrActivity) return diff --git a/app/src/main/java/com/droiddeck/launcher/ui/LabBuilds.kt b/app/src/main/java/com/droiddeck/launcher/ui/LabBuilds.kt new file mode 100644 index 0000000000..8bfc59306b --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/ui/LabBuilds.kt @@ -0,0 +1,162 @@ +package com.droiddeck.launcher.ui + +import android.text.format.DateUtils +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.outlined.Code +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableIntStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import com.droiddeck.launcher.R +import com.droiddeck.launcher.core.WirelessAdbFix +import com.droiddeck.launcher.lab.LabOverlay + +/** + * Lab builds on this device: the one in use, one waiting for a restart, and the kept ones. + * [onRebuild] offers to build a kept one again for this install, when something can. + */ +@Composable +internal fun LabBuilds(sessionRunning: Boolean = false, onRebuild: ((LabOverlay.Info) -> Unit)? = null) { + val context = LocalContext.current + var version by remember { mutableIntStateOf(0) } + var busy by remember { mutableStateOf(false) } + var error by remember { mutableStateOf(null) } + val current = remember(version) { LabOverlay.current(context) } + val pending = remember(version) { LabOverlay.pending(context) } + val saved = remember(version) { LabOverlay.saved(context) } + if (current == null && pending == null && saved.isEmpty()) return + val active = LabOverlay.active + val paused = active == null && current != null && LabOverlay.skipped == LabOverlay.SKIPPED_APP_IMAGE + val restartFailed = stringResource(R.string.lab_restart_failed) + val canRestart = !busy && !sessionRunning + + /** Android may have given the APK an app image again, which a Lab build cannot load over. */ + fun restart() { + busy = true + error = null + Thread({ + val result = runCatching { + if (LabOverlay.needsRecompile(context)) WirelessAdbFix.shell(context, LabOverlay.recompileCommand(context)) + } + android.os.Handler(android.os.Looper.getMainLooper()).post { + busy = false + if (result.isSuccess) LabOverlay.restart(context) + else error = restartFailed.format(result.exceptionOrNull()?.message.orEmpty()) + } + }, "lab-restart").start() + } + + val colors = MaterialTheme.colorScheme + val pal = LocalPalette.current + Column(Modifier.fillMaxWidth().padding(top = 18.dp)) { + Text( + stringResource(R.string.lab_builds).uppercase(), fontSize = 12.sp, fontWeight = FontWeight.SemiBold, letterSpacing = 1.5.sp, + color = colors.onSurfaceVariant, modifier = Modifier.padding(start = 2.dp, top = 2.dp, bottom = 10.dp), + ) + Column(Modifier.fillMaxWidth().clip(Shape14).background(colors.surface).border(1.dp, pal.line, Shape14)) { + val rows = mutableListOf<@Composable () -> Unit>() + if (current != null) rows += { + BuildRow( + current, true, + when { + active != null -> stringResource(R.string.lab_in_use) + paused -> stringResource(R.string.lab_paused_row) + else -> stringResource(R.string.lab_not_loaded, LabOverlay.skipped.orEmpty()) + }, + warn = active == null, + ) { + if (paused) PrimaryButton(stringResource(R.string.lab_restart), compact = true, enabled = canRestart) { restart() } + SecondaryButton(stringResource(R.string.lab_turn_off), compact = true, enabled = canRestart) { LabOverlay.turnOff(context) } + } + } + if (pending != null) rows += { + BuildRow(pending, false, stringResource(R.string.lab_waiting), warn = false) { + PrimaryButton(stringResource(R.string.lab_restart), compact = true, enabled = canRestart) { restart() } + } + } + for (build in saved) rows += { + var armed by remember(build.dir) { mutableStateOf(false) } + val info = build.info + BuildRow( + info, false, + when { + armed -> stringResource(R.string.lab_delete_warning) + !build.usable -> stringResource(R.string.lab_other_install) + build.reason == LabOverlay.Reason.DID_NOT_START -> stringResource(R.string.lab_off_did_not_start) + else -> stringResource(R.string.lab_kept) + }, + warn = armed || !build.usable || build.reason == LabOverlay.Reason.DID_NOT_START, + ) { + if (!build.usable && !armed && info != null && onRebuild != null) SecondaryButton(stringResource(R.string.lab_rebuild), compact = true, enabled = !busy) { + onRebuild(info) + } + if (build.usable && !armed) SecondaryButton(stringResource(R.string.lab_use), compact = true, enabled = canRestart) { + runCatching { LabOverlay.stage(context, build) } + .onSuccess { version++; restart() } + .onFailure { error = it.message } + } + SecondaryButton(stringResource(if (armed) R.string.lab_delete_confirm else R.string.lab_delete), compact = true, enabled = !busy) { + if (armed) { + LabOverlay.delete(build) + version++ + } else armed = true + } + } + } + rows.forEachIndexed { index, row -> + row() + if (index < rows.lastIndex) Box(Modifier.fillMaxWidth().height(1.dp).background(pal.line)) + } + } + error?.let { Text(it, fontSize = 12.5.sp, color = AttentionAmber, modifier = Modifier.padding(start = 2.dp, top = 8.dp)) } + } +} + +@Composable +private fun BuildRow(info: LabOverlay.Info?, current: Boolean, status: String, warn: Boolean, actions: @Composable () -> Unit) { + val colors = MaterialTheme.colorScheme + val pal = LocalPalette.current + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(12.dp), + modifier = Modifier.fillMaxWidth().heightIn(min = 60.dp).padding(horizontal = 14.dp, vertical = 10.dp), + ) { + Box(contentAlignment = Alignment.Center, modifier = Modifier.size(34.dp).clip(Shape12) + .background(if (current) pal.signal.copy(alpha = 0.18f) else colors.surfaceVariant)) { + Icon(Icons.Outlined.Code, null, tint = if (current) pal.signal else colors.onSurfaceVariant, modifier = Modifier.size(20.dp)) + } + Column(Modifier.weight(1f)) { + Text(info?.label ?: "?", fontSize = 14.5.sp, fontWeight = FontWeight.SemiBold, color = colors.onBackground, + maxLines = 1, overflow = TextOverflow.Ellipsis) + val built = info?.built?.takeIf { it > 0 }?.let { DateUtils.getRelativeTimeSpanString(it * 1000, System.currentTimeMillis(), DateUtils.MINUTE_IN_MILLIS).toString() } + Text(listOfNotNull(status, built).joinToString(" · "), fontSize = 12.5.sp, color = if (warn) AttentionAmber else colors.onSurfaceVariant, + maxLines = 2, overflow = TextOverflow.Ellipsis) + } + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { actions() } + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/ui/UpdatesPage.kt b/app/src/main/java/com/droiddeck/launcher/ui/UpdatesPage.kt index dd73939a7d..a1af06a863 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/UpdatesPage.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/UpdatesPage.kt @@ -120,11 +120,11 @@ internal fun UpdatesPage(s: FrontEndState, a: FrontEndActions, modifier: Modifie StatusPanel(s, u, ua, me) Box(Modifier.height(18.dp)) ChannelLabel() - ChannelPicker(u, ua) + ChannelPicker(u, ua, s.sessionRunning) } else { ChannelLabel() Row(horizontalArrangement = Arrangement.spacedBy(18.dp)) { - Box(Modifier.weight(1f)) { ChannelPicker(u, ua) } + Box(Modifier.weight(1f)) { ChannelPicker(u, ua, s.sessionRunning) } Box(Modifier.weight(1.15f)) { StatusPanel(s, u, ua, me) } } } @@ -262,7 +262,7 @@ private fun ChannelLabel() { } @Composable -private fun ChannelPicker(u: UpdatesState, ua: UpdatesActions) { +private fun ChannelPicker(u: UpdatesState, ua: UpdatesActions, sessionRunning: Boolean) { val colors = MaterialTheme.colorScheme val catalog = u.catalog val tests = catalog?.tests.orEmpty() @@ -285,6 +285,7 @@ private fun ChannelPicker(u: UpdatesState, ua: UpdatesActions) { tests.forEach { t -> TestRow(t, u.follow.pr == t.pr) { ua.onFollow(Follow(Channel.TEST, t.pr)) } } } } + LabBuilds(sessionRunning) } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 3351982981..309d3e4154 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1121,6 +1121,29 @@ Test builds Try a fix before it\'s released None right now + Lab build + Lab builds + In use + Paused after Android re-optimized DroidDeck + Not loaded: %1$s + Waiting for a restart + Kept + For another version + Rebuild + Deletes this build from the device + Restart on it + Turn off + Use + Delete + Delete now + Could not prepare the restart: %1$s. Pair wireless debugging in Setup and try again. + Lab build installed + DroidDeck restarted on %1$s + Lab build paused + Android re-optimized DroidDeck, so %1$s is not loaded. Open Updates to restart on it. + Lab build turned off + It was built for another version of DroidDeck. It is kept in Updates. + DroidDeck did not start on it twice PR #%1$d Not built for this copy of DroidDeck Updated %1$s diff --git a/app/src/test/java/com/droiddeck/launcher/lab/LabOverlayTest.kt b/app/src/test/java/com/droiddeck/launcher/lab/LabOverlayTest.kt new file mode 100644 index 0000000000..ee60999fbc --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/lab/LabOverlayTest.kt @@ -0,0 +1,154 @@ +package com.droiddeck.launcher.lab + +import android.app.Application +import com.droiddeck.launcher.BuildConfig +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import java.io.File + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33], manifest = Config.NONE) +class LabOverlayTest { + private lateinit var app: Application + private lateinit var root: File + private val injected = mutableListOf() + private val originalInject = LabOverlay.injectDex + private val originalResources = LabOverlay.openResources + + @Before fun prepare() { + app = RuntimeEnvironment.getApplication() + app.applicationInfo.sourceDir = "/data/app/~~test==/com.droiddeck.launcher-test==/base.apk" + root = LabOverlay.root(app).apply { deleteRecursively(); mkdirs() } + LabOverlay.injectDex = { _, apk -> injected += apk } + LabOverlay.openResources = { null } + } + + @After fun restore() { + LabOverlay.injectDex = originalInject + LabOverlay.openResources = originalResources + root.deleteRecursively() + } + + private fun stage(dir: String, commit: String = BuildConfig.SOURCE_COMMIT, apk: String = app.applicationInfo.sourceDir) = + File(root, dir).apply { + mkdirs() + File(this, "overlay.apk").writeText("dex") + File(this, "overlay.json").writeText("""{"label":"feat/x @ abc1234","baseCommit":"$commit","baseApk":"$apk"}""") + } + + @Test fun aBuildForThisInstallIsLoaded() { + stage("active") + LabOverlay.load(app, mainProcess = true) + assertEquals("feat/x @ abc1234", LabOverlay.active?.label) + assertEquals(listOf(File(root, "active/overlay.apk")), injected) + assertEquals(1, LabOverlay.failedStarts(root)) + } + + @Test fun aStagedBuildIsPromotedAtStart() { + stage("next") + File(root, "starts").writeText("2") + LabOverlay.load(app, mainProcess = true) + assertFalse(File(root, "next").exists()) + assertNotNull(LabOverlay.active) + assertEquals(1, LabOverlay.failedStarts(root)) + } + + @Test fun aBuildForAnotherInstallIsTurnedOff() { + stage("active", commit = "0000000") + LabOverlay.load(app, mainProcess = true) + assertNull(LabOverlay.active) + assertTrue(injected.isEmpty()) + assertFalse(File(root, "active").exists()) + assertEquals("OTHER_INSTALL", File(root, "last-off-reason").readText()) + } + + @Test fun aBuildForAnEarlierInstallOfTheSameCommitIsTurnedOff() { + stage("active", apk = "/data/app/old/base.apk") + LabOverlay.load(app, mainProcess = true) + assertNull(LabOverlay.active) + assertEquals("OTHER_INSTALL", File(root, "last-off-reason").readText()) + } + + @Test fun aBuildThatKeepsFailingToStartIsTurnedOff() { + stage("active") + File(root, "starts").writeText("2") + LabOverlay.load(app, mainProcess = true) + assertNull(LabOverlay.active) + assertEquals("DID_NOT_START", File(root, "last-off-reason").readText()) + } + + @Test fun otherProcessesLoadWithoutCountingStarts() { + stage("active") + LabOverlay.load(app, mainProcess = false) + assertNotNull(LabOverlay.active) + assertEquals(0, LabOverlay.failedStarts(root)) + } + + @Test fun noBuildMeansNothingIsLoaded() { + LabOverlay.load(app, mainProcess = true) + assertNull(LabOverlay.active) + assertTrue(injected.isEmpty()) + assertFalse(LabOverlay.promoted) + } + + @Test fun aNewBuildKeepsTheOneItReplaces() { + stage("active") + stage("next") + LabOverlay.load(app, mainProcess = true) + assertTrue(LabOverlay.promoted) + val saved = LabOverlay.saved(app).single() + assertEquals(LabOverlay.Reason.REPLACED, saved.reason) + assertTrue(saved.usable) + } + + @Test fun aRefusedBuildIsKeptButNotUsable() { + stage("active", commit = "0000000") + LabOverlay.load(app, mainProcess = true) + val saved = LabOverlay.saved(app).single() + assertEquals(LabOverlay.Reason.OTHER_INSTALL, saved.reason) + assertFalse(saved.usable) + } + + @Test fun aKeptBuildCanBeStagedAgain() { + stage("active", commit = "0000000") + LabOverlay.load(app, mainProcess = true) + stage("active") + LabOverlay.archive(root, File(root, "active"), LabOverlay.Reason.TURNED_OFF) + val usable = LabOverlay.saved(app).first { it.usable } + LabOverlay.stage(app, usable) + assertNotNull(LabOverlay.pending(app)) + assertEquals(1, LabOverlay.saved(app).size) + LabOverlay.load(app, mainProcess = true) + assertTrue(LabOverlay.promoted) + assertNotNull(LabOverlay.active) + } + + @Test fun stagingAKeptBuildKeepsTheOneThatWasWaiting() { + stage("builds/1") + stage("next") + File(root, "next/overlay.json").writeText("""{"label":"feat/y @ def5678","baseCommit":"${BuildConfig.SOURCE_COMMIT}","baseApk":"${app.applicationInfo.sourceDir}"}""") + LabOverlay.stage(app, LabOverlay.saved(app).single()) + assertEquals("feat/x @ abc1234", LabOverlay.pending(app)?.label) + val kept = LabOverlay.saved(app).single() + assertEquals("feat/y @ def5678", kept.info?.label) + assertEquals(LabOverlay.Reason.REPLACED, kept.reason) + } + + @Test fun onlyTheNewestBuildsAreKept() { + repeat(15) { + stage("active") + LabOverlay.archive(root, File(root, "active"), LabOverlay.Reason.TURNED_OFF) + } + assertEquals(12, LabOverlay.saved(app).size) + } +} diff --git a/docs/lab-builds.md b/docs/lab-builds.md new file mode 100644 index 0000000000..4f2ddfb5f0 --- /dev/null +++ b/docs/lab-builds.md @@ -0,0 +1,63 @@ +# Lab builds + +A Lab build is DroidDeck app code and resources built on the device itself, by the on-device agent +or by hand, and loaded over the installed APK at the next start. Nothing is reinstalled or +re-signed, so a tester on a release build keeps their games, sign-ins and settings while trying a +change. + +## How it works + +- `tools/agent/dd-lab-build` builds the checkout with Gradle on the device (`processReleaseResources` + and `mergeDexRelease`), packs the linked resources and dex files into `overlay.apk`, writes + `overlay.json`, and stages both in the app's `files/lab/next`. It then recompiles the app without + an app image and restarts DroidDeck. +- Run by the app's own agent (`DD_AGENT` set), it writes `files/agent/lab-request` instead, and the + app runs the build in a process of its own, logging to `~/.cache/dd-lab-build.log`; the agent's + copy follows that log and exits with the build. A turn's processes end with the turn, so a build + started there would otherwise die when the agent stops or finishes early. +- At start, `LabOverlay` (called from `App.attachBaseContext`) promotes `next` to `active`, checks + that the build was made for this install, puts its dex files ahead of the APK's, and adds its + resources through `ResourcesLoader` to the app and every activity. Contexts made from them + (presentations on the second screen) inherit the loader. +- The watermark shows `LAB | @ `. When the app comes back on a new build it posts + a "Lab build installed" notification, so the restart does not look like a crash. +- A build that is replaced, turned off or refused is kept in `files/lab/builds` (the newest 12). + Updates lists them under Lab builds: the one in use can be turned off, one built for this + install can be used again, and any can be deleted. + +## What a Lab build cannot change + +The manifest, native code (`cpp`, `jniLibs`, Rust), Gradle files, `App`, `AppLanguage`, and `LabOverlay` itself: +they are loaded before a Lab build can be, or are not part of it. `dd-lab-build` refuses those +changes; they need a new APK. + +## Safety + +- A build is only used on the install it was built for: `overlay.json` carries the APK's commit + (`apkCommit` in `droiddeckctl state`) and the APK's path, which changes with every install and + update. A build for another install is turned off. +- Two starts in a row that do not reach a resumed activity for 10 seconds turn the build off. +- It needs Android 11 or later (`ResourcesLoader`) and no app image (classes in one load before the + Lab build). `dd-lab-build` runs `cmd package compile -m verify` before restarting. Android's + background optimization can bring an app image back; the build is then paused, a notification + says so, and Restart in Updates recompiles through the app's wireless debugging pairing. + +## Setting up a device + +From the DroidDeck terminal, in a checkout of this repository: + +```sh +tools/agent/setup +``` + +It installs Claude Code, git, gh and loopback adb, plus the build toolchain: JDK 17, the Android +SDK platform, and Google's x86-64 `aapt2` run under `qemu-user`. The first Lab build compiles the +whole app and takes about 8 minutes on a Snapdragon 865; later ones compile only what changed. +That needs the Kotlin daemon: Kotlin's in-process compiles always redo the whole app. BuildConfig +is given the installed commit, so committing a change does not change it. Setup +does that first build ahead of time with `dd-lab-build --prepare`, which compiles the installed +commit without making a Lab build. +`~/.cache/dd-lab-build.json` reports the stage, the current task, progress in thousandths and the +seconds left, estimated from how long each task took the last time (`~/.cache/dd-lab-build-times`). +`~/.cache/dd-lab-build-base` holds the installed commit the last successful build compiled, so the +Agent tab can offer to prepare again after DroidDeck updates. diff --git a/tools/agent/android-open b/tools/agent/android-open new file mode 100755 index 0000000000..9d8c550486 --- /dev/null +++ b/tools/agent/android-open @@ -0,0 +1,9 @@ +#!/bin/sh +# Opens a URL in Android's browser on the bottom screen. +url=$1 +[ -n "$url" ] || exit 1 +serial=${ADB_SERIAL:-127.0.0.1:5555} +adb connect "$serial" >/dev/null 2>&1 +quoted=$(printf "%s" "$url" | sed "s/'/'\\\\''/g") +adb -s "$serial" shell "am start --display 2 -a android.intent.action.VIEW -d '$quoted'" >/dev/null 2>&1 || + echo "Open this on any device: $url" >&2 diff --git a/tools/agent/dd-agent b/tools/agent/dd-agent new file mode 100755 index 0000000000..9a1b4d4132 --- /dev/null +++ b/tools/agent/dd-agent @@ -0,0 +1,5 @@ +#!/bin/sh +# Claude Code in the DroidDeck checkout, with sign-in links opened in Android's browser. +export ADB_SERIAL=${ADB_SERIAL:-127.0.0.1:5555} PATH="$HOME/.local/bin:$PATH" BROWSER="$HOME/.local/bin/android-open" +adb connect "$ADB_SERIAL" >/dev/null 2>&1 +cd "$HOME/DroidDeck" && exec claude "$@" diff --git a/tools/agent/dd-gh-login b/tools/agent/dd-gh-login new file mode 100755 index 0000000000..adf816a247 --- /dev/null +++ b/tools/agent/dd-gh-login @@ -0,0 +1,24 @@ +#!/bin/bash +# GitHub sign-in through Android's browser on the bottom screen, with the one-time code typed in. +serial=${ADB_SERIAL:-127.0.0.1:5555} +adb connect "$serial" >/dev/null 2>&1 +coproc GH { BROWSER=true gh auth login --hostname github.com --git-protocol https --web 2>&1; } +gh_pid=$GH_PID +typed= +while IFS= read -r line <&"${GH[0]}"; do + printf '%s\n' "$line" + code=$(grep -oE '[A-Z0-9]{4}-[A-Z0-9]{4}' <<<"$line" | head -1) + if [ -n "$code" ] && [ -z "$typed" ]; then + typed=1 + echo >&"${GH[1]}" + adb -s "$serial" shell "am start --display 2 -a android.intent.action.VIEW -d https://github.com/login/device" >/dev/null 2>&1 + # Chrome drops characters from `input text` and from fast key events on this page. + keys= + for ((i = 0; i < ${#code}; i++)); do + c=${code:i:1} + [ "$c" = - ] || keys+="input -d 2 keyevent KEYCODE_$c; sleep 0.4; " + done + (sleep 7; adb -s "$serial" shell "$keys" >/dev/null 2>&1) & + fi +done +wait "$gh_pid" diff --git a/tools/agent/dd-lab-build b/tools/agent/dd-lab-build new file mode 100755 index 0000000000..44b62bd6b8 --- /dev/null +++ b/tools/agent/dd-lab-build @@ -0,0 +1,226 @@ +#!/bin/bash +# Builds this checkout as a Lab build of the installed DroidDeck and restarts the app on it. +# Usage: dd-lab-build [--no-restart | --prepare] [--explain] +# --prepare compiles the installed commit without making a Lab build, so later builds only +# compile what changed. Setup runs it once; it takes the longest. +# --explain logs why each task ran and Kotlin's build report, to find out what makes a build slow. +set -euo pipefail +pkg=com.droiddeck.launcher +serial=${ADB_SERIAL:-127.0.0.1:5555} +mode=build +explain=() +for arg in "$@"; do + case $arg in + --no-restart) mode=no-restart ;; + --prepare) mode=prepare ;; + --explain) explain=(--info -Pkotlin.build.report.output=file) ;; + esac +done +started=$(date +%s) +cache=$HOME/.cache +progress_file=$cache/dd-lab-build.json +times_file=$cache/dd-lab-build-times +mkdir -p "$cache" +# The Agent tab reads this file to show how far the build has got and how long is left. +progress() { # stage [permille task done total eta] + local task=${3:-}; task=${task//\\/}; task=${task//\"/\'}; task=${task//$'\n'/ } + printf '{"stage":"%s","mode":"%s","permille":%d,"task":"%s","done":%d,"total":%d,"eta":%d,"elapsed":%d,"pid":%d}\n' \ + "$1" "$mode" "${2:-0}" "$task" "${4:-0}" "${5:-0}" "${6:--1}" $(( $(date +%s) - started )) $$ > "$progress_file.part.$BASHPID" + mv "$progress_file.part.$BASHPID" "$progress_file" +} +fail() { progress failed 0 "$*"; echo "dd-lab-build: $*" >&2; exit 1; } +trap 'progress failed 0 stopped; exit 143' INT TERM +trap 'progress failed 0 "a command failed on line $LINENO"' ERR +device() { adb -s "$serial" shell "$@" | tr -d '\r'; } + +# The app's agent loses every process it started when its turn ends, a build included, so it asks +# the app to run the build in a process of its own and follows that build here. +agent=/data/user/0/$pkg/files/agent +if [ -n "${DD_AGENT:-}" ] && [ -z "${DD_LAB_DETACHED:-}" ] && [ -d "$agent" ]; then + trap - ERR + log=$cache/dd-lab-build.log + rm -f "$progress_file" + python3 -c 'import json,os,sys; print(json.dumps({"cwd": os.getcwd(), "args": sys.argv[1:]}))' "$@" > "$agent/lab-request.part" + mv "$agent/lab-request.part" "$agent/lab-request" + echo "DroidDeck runs this build itself, so it keeps going if this turn ends. Its log follows." + for _ in $(seq 1 30); do [ -f "$progress_file" ] && break; sleep 1; done + [ -f "$progress_file" ] || { echo "dd-lab-build: DroidDeck did not start the build; is it running?" >&2; exit 1; } + tail -n +1 -F "$log" 2>/dev/null & follow=$! + result=1 + while sleep 2; do + stage=$(sed -n 's/.*"stage":"\([a-z]*\)".*/\1/p' "$progress_file") + pid=$(sed -n 's/.*"pid":\([0-9]*\).*/\1/p' "$progress_file") + case $stage in + ready|prepared) result=0; break ;; + failed) break ;; + esac + [ -n "$pid" ] && [ -d "/proc/$pid" ] || break + done + sleep 1; kill $follow 2>/dev/null || true + exit $result +fi + +progress checking +repo=$(git rev-parse --show-toplevel) || fail "run it inside the DroidDeck checkout" +cd "$repo" +adb connect "$serial" >/dev/null 2>&1 || true + +state=$(python3 tools/droiddeckctl state 2>/dev/null | sed -n '/^{/,$p') || fail "could not read DroidDeck's state; is it running and is adb connected?" +# A Lab build reports its own label, so the APK's commit comes from apkCommit when the app has it. +label=$(python3 -c 'import json,sys; s=json.load(sys.stdin); print(s.get("apkCommit") or s["build"].rsplit(" at ", 1)[-1])' <<<"$state") || + fail "DroidDeck's state did not say which build is installed" +installed_label=$(python3 -c 'import json,re,sys; print(re.sub(r"^(?:\w+ \w+ · |lab on )+", "", json.load(sys.stdin)["build"]))' <<<"$state") +if ! base=$(git rev-parse --verify --quiet "$label^{commit}"); then + progress fetching 0 "the installed commit" + git fetch --quiet --tags origin >/dev/null 2>&1 || true + git fetch --quiet origin main >/dev/null 2>&1 || true + base=$(git rev-parse --verify --quiet "$label^{commit}") || fail "the installed build ($label) is not in this checkout or on GitHub's main; fetch it first" +fi + +if [ $mode = prepare ]; then + # Compiling the installed commit itself gives later builds of branches on it incremental caches. + if ! git merge-base --is-ancestor "$base" HEAD; then + git diff --quiet && git diff --cached --quiet || fail "HEAD does not build on the installed commit and has uncommitted changes" + was=$(git branch --show-current) + git checkout --quiet --detach "$base" + echo "Checked out the installed commit ${base:0:9} to prepare it${was:+; $was is unchanged}" + fi +else + git diff --quiet && git diff --cached --quiet || fail "commit your changes first" + git merge-base --is-ancestor "$base" HEAD || fail "HEAD does not build on the installed commit ${base:0:9}; branch from it" + blocked=$(git diff --name-only "$base" HEAD | grep -E 'AndroidManifest\.xml$|/cpp/|/jniLibs/|\.rs$|(^|/)build\.gradle$|/App\.kt$|/AppLanguage\.kt$|/lab/LabOverlay\.kt$' || true) + [ -z "$blocked" ] || fail "these need a new APK, not a Lab build: $(echo $blocked)" +fi +apk=$(device pm path $pkg | sed -n 's/^package://p' | grep 'base\.apk$' | head -1) +[ -n "$apk" ] || fail "DroidDeck is not installed" + +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-arm64 ANDROID_HOME=$HOME/android-sdk +# BuildConfig describes the installed base, not each Lab commit; its commit changing would make the +# Kotlin compile redo the whole app (the overlay's own label names the Lab build). Updates reads the +# installed variant (non-adreno) from the build label. +export DROIDDECK_SOURCE_COMMIT=$base DROIDDECK_BUILD_LABEL="lab on $installed_label" DROIDDECK_BUILD_TREE_STATE=clean GITHUB_HEAD_REF=lab +printf 'sdk.dir=%s\n' "$ANDROID_HOME" > local.properties +if [ ! -d "$ANDROID_HOME/platforms/android-34" ]; then + progress sdk + yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses >/dev/null + "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "platforms;android-34" "build-tools;35.0.0" >/dev/null +fi + +# A full Kotlin compile of the app needs about 2.5 GB of heap and Gradle about 1 GB; smaller devices get +# what they can spare. Both daemons end with this script's process when the build is done. +total_mb=$(awk '/^MemTotal:/ {print int($2 / 1024)}' /proc/meminfo) +heap=$(( total_mb / 2 )); [ $heap -gt 4096 ] && heap=4096; [ $heap -lt 2048 ] && heap=2048 +heap=${DD_LAB_HEAP_MB:-$heap} +kotlin_heap=$(( heap * 2 / 3 )); gradle_heap=$(( heap - kotlin_heap )) + +# Only the Kotlin daemon compiles incrementally; in-process compiles always redo the whole app, and the +# caches they leave behind make the daemon's first incremental compile fail. +if [ "$(cat "$cache/dd-lab-build-strategy" 2>/dev/null)" != daemon ]; then + rm -rf app/build/kotlin/compileReleaseKotlin + echo daemon > "$cache/dd-lab-build-strategy" +fi + +# Google publishes aapt2 for x86-64 Linux only; setup wraps it in qemu-user at ~/aapt2/aapt2. +gradle=(./gradlew :app:processReleaseResources :app:mergeDexRelease --max-workers=2 -PskipRust=true -x buildVkBridgeServer + -Pandroid.aapt2FromMavenOverride=$HOME/aapt2/aapt2 -Pkotlin.compiler.execution.strategy=daemon + -Pkotlin.daemon.jvmargs=-Xmx${kotlin_heap}m -Dorg.gradle.jvmargs=-Xmx${gradle_heap}m --console=plain) +progress planning +mapfile -t plan < <("${gradle[@]}" --dry-run -q 2>/dev/null | awk '/^:/ {print $1}' || true) +total=${#plan[@]} + +# Each task's estimate is how long it took last time; a full compile of the app takes about 15 minutes. +declare -A estimate +[ -f "$times_file" ] && while read -r t s; do estimate[$t]=$s; done < "$times_file" +estimate_of() { local e=${estimate[$1]:-}; [ -n "$e" ] && echo "$e" || { [[ $1 == *compileReleaseKotlin ]] && echo 900 || echo 2; }; } +planned=0 +for t in "${plan[@]}"; do planned=$(( planned + $(estimate_of "$t") )); done +[ $planned -gt 0 ] || planned=1 + +current=$(mktemp) +measured=$(mktemp) +# " " +echo "0 starting $(date +%s) 0 0" > "$current" +report() { + local n task since done_s cur_est now part permille left + read -r n task since done_s cur_est < "$current" + now=$(date +%s) + part=$(( now - since )); [ $part -gt $(( cur_est * 95 / 100 )) ] && part=$(( cur_est * 95 / 100 )) + permille=$(( (done_s + part) * 1000 / planned )); [ $permille -gt 990 ] && permille=990 + left=$(( planned - done_s - part )); [ $left -lt 0 ] && left=0 + progress building "$permille" "$task" "$n" "$total" "$left" + echo "$n $task $(( (now - started) / 60 )) $left" +} +( + beat=0 + while sleep 5; do + set -- $(report) + beat=$(( beat + 1 )) + [ $(( beat % 12 )) = 0 ] && echo "[build $1/$total] still on $2, $3m in, about $(( $4 / 60 + 1 ))m left" + done +) & +heartbeat=$! +trap 'kill $heartbeat 2>/dev/null || true; rm -f "$current" "$measured"' EXIT +set +o pipefail +"${gradle[@]}" ${explain[@]+"${explain[@]}"} 2>&1 | { + n=0 done_s=0 task="" since=0 cur_est=0 ran=0 + while IFS= read -r line; do + printf '%s\n' "$line" + case $line in + "> Task "*) + now=$(date +%s) + # A task's line comes when it starts, or when it ends for one that did no work. + if [ -n "$task" ]; then + done_s=$(( done_s + cur_est )) + [ $ran = 1 ] && echo "$task $(( now - since ))" >> "$measured" + fi + rest=${line#> Task }; task=${rest%% *} + case $rest in *" "*) ran=0 ;; *) ran=1 ;; esac + n=$((n + 1)); since=$now; cur_est=$(estimate_of "$task") + echo "$n $task $since $done_s $cur_est" > "$current" + [ $ran = 1 ] && report >/dev/null && echo "[build $n/$total] $task" ;; + esac + done + if [ -n "$task" ] && [ $ran = 1 ]; then echo "$task $(( $(date +%s) - since ))" >> "$measured"; fi +} +status=${PIPESTATUS[0]} +set -o pipefail +kill $heartbeat 2>/dev/null || true +[ "$status" = 0 ] || fail "Gradle failed (exit $status); the reason is above" +# Tasks that did no work this time keep their earlier time. +{ [ -f "$times_file" ] && cat "$times_file"; cat "$measured"; } | awk '{t[$1] = $2} END {for (k in t) print k, t[k]}' > "$times_file.part" +mv "$times_file.part" "$times_file" +# The app reads which installed commit the build caches are warm for. +echo "$base" > "$cache/dd-lab-build-base" + +if [ $mode = prepare ]; then + progress prepared 1000 + echo "Prepared: Lab builds of branches on ${base:0:9} now compile only what changed ($(( ($(date +%s) - started) / 60 ))m)" + exit 0 +fi + +progress packaging 995 +linked=$(ls app/build/intermediates/linked_resources_binary_format/release/processReleaseResources/*.ap_ | head -1) +work=$(mktemp -d) +cp "$linked" "$work/overlay.apk" +(cd app/build/intermediates/dex/release/mergeDexRelease && zip -q "$work/overlay.apk" classes*.dex) +name="$(git branch --show-current) @ $(git rev-parse --short HEAD)" +python3 -c 'import json,sys,time; print(json.dumps({"label": sys.argv[1], "baseCommit": sys.argv[2], "baseApk": sys.argv[3], "built": int(time.time())}))' \ + "$name" "$label" "$apk" > "$work/overlay.json" + +lab=/data/user/0/$pkg/files/lab +mkdir -p "$lab" +rm -rf "$lab/next.part" "$lab/next" +mv "$work" "$lab/next.part" +mv "$lab/next.part" "$lab/next" +progress ready 1000 "$name" +echo "Lab build ready: $name ($(du -h "$lab/next/overlay.apk" | cut -f1)) in $(( ($(date +%s) - started) / 60 ))m" +[ $mode = build ] || exit 0 + +# Classes in an app image load before the Lab build can, so the app is recompiled without one. +device cmd package compile -m verify -f $pkg >/dev/null +progress restarting 1000 "$name" +echo "Restarting DroidDeck on the Lab build; it shows a notification when it is back" +# The app's own agent ends with the app, so it is told to carry on once the app is back. +if [ -n "${DD_AGENT:-}" ]; then mkdir -p "/data/user/0/$pkg/files/agent" && : > "/data/user/0/$pkg/files/agent/resume"; fi +device "nohup sh -c 'sleep 3; monkey -p $pkg -c android.intent.category.LAUNCHER 1' >/dev/null 2>&1 &" +for pid in $(device pidof $pkg); do kill "$pid" 2>/dev/null || true; done diff --git a/tools/agent/setup b/tools/agent/setup new file mode 100755 index 0000000000..49dd82592a --- /dev/null +++ b/tools/agent/setup @@ -0,0 +1,76 @@ +#!/bin/bash +# Sets up the on-device agent inside DroidDeck's Linux userland: Claude Code, git and gh, loopback +# adb for droiddeckctl, and the toolchain dd-lab-build uses to build Lab builds on the device. +# Run it from the DroidDeck terminal, inside a checkout of the DroidDeck repository. +set -euo pipefail +here=$(cd "$(dirname "$0")" && pwd) +export DEBIAN_FRONTEND=noninteractive +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +apt-get install -y --no-install-recommends git gh adb python3 zip unzip curl openjdk-17-jdk-headless qemu-user zipalign apksigner + +command -v claude >/dev/null || [ -x "$HOME/.local/bin/claude" ] || + { curl -fsSL -o "$tmp/install-claude.sh" https://claude.ai/install.sh && bash "$tmp/install-claude.sh"; } + +mkdir -p "$HOME/.local/bin" +for tool in dd-agent dd-lab-build dd-gh-login android-open; do install -m755 "$here/$tool" "$HOME/.local/bin/$tool"; done +grep -q 'droiddeck-agent-path' "$HOME/.bashrc" 2>/dev/null || + printf '\n# droiddeck-agent-path\nexport PATH="$HOME/.local/bin:$PATH" ADB_SERIAL=127.0.0.1:5555 BROWSER="$HOME/.local/bin/android-open"\n' >> "$HOME/.bashrc" + +# Google publishes aapt2 for x86-64 Linux only; it runs here under qemu-user with Ubuntu's x86-64 runtime. +x86=$HOME/x86root +mkdir -p "$HOME/aapt2/x86" "$x86/etc" +if [ ! -x "$HOME/aapt2/x86/aapt2" ]; then + agp=$(sed -n "s/.*com.android.tools.build:gradle:\([0-9.]*\).*/\1/p" "$here/../../build.gradle" | head -1) + [ -n "$agp" ] || { echo "setup: no Android Gradle plugin version in build.gradle" >&2; exit 1; } + aapt2=$(curl -fsSL "https://dl.google.com/android/maven2/com/android/tools/build/aapt2/maven-metadata.xml" | + { grep -o "$agp-[0-9]*" || true; } | tail -1 | sed 's/<[^>]*>//g') + [ -n "$aapt2" ] || { echo "setup: Google publishes no aapt2 for AGP $agp" >&2; exit 1; } + curl -fsSL -o "$tmp/aapt2.jar" "https://dl.google.com/android/maven2/com/android/tools/build/aapt2/$aapt2/aapt2-$aapt2-linux.jar" + unzip -o -q "$tmp/aapt2.jar" aapt2 -d "$HOME/aapt2/x86" + chmod +x "$HOME/aapt2/x86/aapt2" +fi +if [ ! -e "$x86/lib64/ld-linux-x86-64.so.2" ]; then + codename=$(. /etc/os-release && echo "$VERSION_CODENAME") + curl -fsSL "https://archive.ubuntu.com/ubuntu/dists/$codename/main/binary-amd64/Packages.gz" | gunzip > "$tmp/amd64-packages" + for package in libc6 libgcc-s1 libstdc++6 zlib1g; do + read -r file sum < <(awk -v p="$package" '$0 == "Package: " p {found = 1} found && /^Filename:/ {f = $2} + found && /^SHA256:/ {s = $2} found && /^$/ {exit} END {print f, s}' "$tmp/amd64-packages") + [ -n "$sum" ] || { echo "setup: $package is not in Ubuntu $codename's amd64 index" >&2; exit 1; } + curl -fsSL -o "$tmp/$package.deb" "https://archive.ubuntu.com/ubuntu/$file" + echo "$sum $tmp/$package.deb" | sha256sum -c --quiet - + dpkg-deb -x "$tmp/$package.deb" "$x86" + done + ln -sfn usr/lib64 "$x86/lib64" + ln -sfn usr/lib "$x86/lib" + : > "$x86/etc/ld.so.preload" +fi +# AGP only accepts a custom aapt2 whose file is named aapt2. +printf '#!/bin/sh\nexec qemu-x86_64 -L %s %s/aapt2/x86/aapt2 "$@"\n' "$x86" "$HOME" > "$HOME/aapt2/aapt2" +chmod +x "$HOME/aapt2/aapt2" +"$HOME/aapt2/aapt2" version + +sdk=$HOME/android-sdk +if [ ! -x "$sdk/cmdline-tools/latest/bin/sdkmanager" ]; then + mkdir -p "$sdk/cmdline-tools" + curl -fsSL -o "$tmp/cmdline-tools.zip" https://dl.google.com/android/repository/commandlinetools-linux-13114758_latest.zip + unzip -q -o "$tmp/cmdline-tools.zip" -d "$sdk/cmdline-tools" + rm -rf "$sdk/cmdline-tools/latest" + mv "$sdk/cmdline-tools/cmdline-tools" "$sdk/cmdline-tools/latest" +fi +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-arm64 +yes | "$sdk/cmdline-tools/latest/bin/sdkmanager" --licenses >/dev/null +"$sdk/cmdline-tools/latest/bin/sdkmanager" "platforms;android-34" "build-tools;35.0.0" >/dev/null + +adb connect 127.0.0.1:5555 || true +# The first build compiles the whole app; doing it now makes the agent's Lab builds incremental. +if [ "${DD_SKIP_PREPARE:-}" != 1 ]; then + echo "Approve the USB debugging prompt if one appears." + if timeout 120 adb -s 127.0.0.1:5555 wait-for-device; then + "$here/dd-lab-build" --prepare || echo "The first Lab build will compile the whole app" + else + echo "adb did not connect, so the first Lab build will compile the whole app" + fi +fi +echo "Done. Run dd-agent."