feat(lab): build and run Lab builds of DroidDeck on the device itself (#2375)

* feat(lab): load an on-device Lab build over the installed app

* feat(lab): build Lab builds on the device and report the APK commit to agents

* fix(lab): dd-agent connects loopback adb before starting Claude Code

* feat(lab): tell the app's own agent to continue after a Lab build restart

* fix(lab): size the build heap from the device's memory

* feat(lab): keep earlier Lab builds and list them in Updates

* feat(lab): notify when DroidDeck comes back on a Lab build or turns one off

* feat(lab): estimate the time left and compile the installed commit ahead of the first build

* fix(lab): report a failed command as a failed build and update progress every five seconds

* feat(lab): a kept build for another version can be rebuilt

* feat(lab): record which installed commit the build caches are warm for

* fix(lab): the app's agent hands its builds to the app, so they outlive the agent's turn

* perf(lab): BuildConfig describes the installed base, so a Lab commit does not change it

* perf(lab): compile with the Kotlin daemon, since in-process Kotlin compiles always redo the whole app

* fix(lab): a build whose last task was up to date no longer counts as failed

* feat(lab): dd-lab-build --explain logs why tasks ran and Kotlin's build report

* fix(lab): keep a waiting build when another is staged or Lab is turned off, and never restart during a session

* fix(lab): keep the installed build's label and exact commit, and refuse AppLanguage changes

* fix(lab): setup verifies its downloads and waits for adb before preparing

* fix(agent): quote apostrophes in opened URLs and keep the sign-in pid
This commit is contained in:
Kurt Himebauch authored and GitHub committed 2026-11-11 12:44:39 -05:00
1 parent 2dbc007a95
commit 61031e3c4b
15 files changed
+1091 -5

No files matched your search

+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# Opens a URL in Android's browser on the bottom screen.
url=$1
[ -n "$url" ] || exit 1
serial=${ADB_SERIAL:-127.0.0.1:5555}
adb connect "$serial" >/dev/null 2>&1
quoted=$(printf "%s" "$url" | sed "s/'/'\\\\''/g")
adb -s "$serial" shell "am start --display 2 -a android.intent.action.VIEW -d '$quoted'" >/dev/null 2>&1 ||
echo "Open this on any device: $url" >&2
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
# Claude Code in the DroidDeck checkout, with sign-in links opened in Android's browser.
export ADB_SERIAL=${ADB_SERIAL:-127.0.0.1:5555} PATH="$HOME/.local/bin:$PATH" BROWSER="$HOME/.local/bin/android-open"
adb connect "$ADB_SERIAL" >/dev/null 2>&1
cd "$HOME/DroidDeck" && exec claude "$@"
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
# GitHub sign-in through Android's browser on the bottom screen, with the one-time code typed in.
serial=${ADB_SERIAL:-127.0.0.1:5555}
adb connect "$serial" >/dev/null 2>&1
coproc GH { BROWSER=true gh auth login --hostname github.com --git-protocol https --web 2>&1; }
gh_pid=$GH_PID
typed=
while IFS= read -r line <&"${GH[0]}"; do
printf '%s\n' "$line"
code=$(grep -oE '[A-Z0-9]{4}-[A-Z0-9]{4}' <<<"$line" | head -1)
if [ -n "$code" ] && [ -z "$typed" ]; then
typed=1
echo >&"${GH[1]}"
adb -s "$serial" shell "am start --display 2 -a android.intent.action.VIEW -d https://github.com/login/device" >/dev/null 2>&1
# Chrome drops characters from `input text` and from fast key events on this page.
keys=
for ((i = 0; i < ${#code}; i++)); do
c=${code:i:1}
[ "$c" = - ] || keys+="input -d 2 keyevent KEYCODE_$c; sleep 0.4; "
done
(sleep 7; adb -s "$serial" shell "$keys" >/dev/null 2>&1) &
fi
done
wait "$gh_pid"
+226
View File
@@ -0,0 +1,226 @@
#!/bin/bash
# Builds this checkout as a Lab build of the installed DroidDeck and restarts the app on it.
# Usage: dd-lab-build [--no-restart | --prepare] [--explain]
# --prepare compiles the installed commit without making a Lab build, so later builds only
# compile what changed. Setup runs it once; it takes the longest.
# --explain logs why each task ran and Kotlin's build report, to find out what makes a build slow.
set -euo pipefail
pkg=com.droiddeck.launcher
serial=${ADB_SERIAL:-127.0.0.1:5555}
mode=build
explain=()
for arg in "$@"; do
case $arg in
--no-restart) mode=no-restart ;;
--prepare) mode=prepare ;;
--explain) explain=(--info -Pkotlin.build.report.output=file) ;;
esac
done
started=$(date +%s)
cache=$HOME/.cache
progress_file=$cache/dd-lab-build.json
times_file=$cache/dd-lab-build-times
mkdir -p "$cache"
# The Agent tab reads this file to show how far the build has got and how long is left.
progress() { # stage [permille task done total eta]
local task=${3:-}; task=${task//\\/}; task=${task//\"/\'}; task=${task//$'\n'/ }
printf '{"stage":"%s","mode":"%s","permille":%d,"task":"%s","done":%d,"total":%d,"eta":%d,"elapsed":%d,"pid":%d}\n' \
"$1" "$mode" "${2:-0}" "$task" "${4:-0}" "${5:-0}" "${6:--1}" $(( $(date +%s) - started )) $$ > "$progress_file.part.$BASHPID"
mv "$progress_file.part.$BASHPID" "$progress_file"
}
fail() { progress failed 0 "$*"; echo "dd-lab-build: $*" >&2; exit 1; }
trap 'progress failed 0 stopped; exit 143' INT TERM
trap 'progress failed 0 "a command failed on line $LINENO"' ERR
device() { adb -s "$serial" shell "$@" | tr -d '\r'; }
# The app's agent loses every process it started when its turn ends, a build included, so it asks
# the app to run the build in a process of its own and follows that build here.
agent=/data/user/0/$pkg/files/agent
if [ -n "${DD_AGENT:-}" ] && [ -z "${DD_LAB_DETACHED:-}" ] && [ -d "$agent" ]; then
trap - ERR
log=$cache/dd-lab-build.log
rm -f "$progress_file"
python3 -c 'import json,os,sys; print(json.dumps({"cwd": os.getcwd(), "args": sys.argv[1:]}))' "$@" > "$agent/lab-request.part"
mv "$agent/lab-request.part" "$agent/lab-request"
echo "DroidDeck runs this build itself, so it keeps going if this turn ends. Its log follows."
for _ in $(seq 1 30); do [ -f "$progress_file" ] && break; sleep 1; done
[ -f "$progress_file" ] || { echo "dd-lab-build: DroidDeck did not start the build; is it running?" >&2; exit 1; }
tail -n +1 -F "$log" 2>/dev/null & follow=$!
result=1
while sleep 2; do
stage=$(sed -n 's/.*"stage":"\([a-z]*\)".*/\1/p' "$progress_file")
pid=$(sed -n 's/.*"pid":\([0-9]*\).*/\1/p' "$progress_file")
case $stage in
ready|prepared) result=0; break ;;
failed) break ;;
esac
[ -n "$pid" ] && [ -d "/proc/$pid" ] || break
done
sleep 1; kill $follow 2>/dev/null || true
exit $result
fi
progress checking
repo=$(git rev-parse --show-toplevel) || fail "run it inside the DroidDeck checkout"
cd "$repo"
adb connect "$serial" >/dev/null 2>&1 || true
state=$(python3 tools/droiddeckctl state 2>/dev/null | sed -n '/^{/,$p') || fail "could not read DroidDeck's state; is it running and is adb connected?"
# A Lab build reports its own label, so the APK's commit comes from apkCommit when the app has it.
label=$(python3 -c 'import json,sys; s=json.load(sys.stdin); print(s.get("apkCommit") or s["build"].rsplit(" at ", 1)[-1])' <<<"$state") ||
fail "DroidDeck's state did not say which build is installed"
installed_label=$(python3 -c 'import json,re,sys; print(re.sub(r"^(?:\w+ \w+ · |lab on )+", "", json.load(sys.stdin)["build"]))' <<<"$state")
if ! base=$(git rev-parse --verify --quiet "$label^{commit}"); then
progress fetching 0 "the installed commit"
git fetch --quiet --tags origin >/dev/null 2>&1 || true
git fetch --quiet origin main >/dev/null 2>&1 || true
base=$(git rev-parse --verify --quiet "$label^{commit}") || fail "the installed build ($label) is not in this checkout or on GitHub's main; fetch it first"
fi
if [ $mode = prepare ]; then
# Compiling the installed commit itself gives later builds of branches on it incremental caches.
if ! git merge-base --is-ancestor "$base" HEAD; then
git diff --quiet && git diff --cached --quiet || fail "HEAD does not build on the installed commit and has uncommitted changes"
was=$(git branch --show-current)
git checkout --quiet --detach "$base"
echo "Checked out the installed commit ${base:0:9} to prepare it${was:+; $was is unchanged}"
fi
else
git diff --quiet && git diff --cached --quiet || fail "commit your changes first"
git merge-base --is-ancestor "$base" HEAD || fail "HEAD does not build on the installed commit ${base:0:9}; branch from it"
blocked=$(git diff --name-only "$base" HEAD | grep -E 'AndroidManifest\.xml$|/cpp/|/jniLibs/|\.rs$|(^|/)build\.gradle$|/App\.kt$|/AppLanguage\.kt$|/lab/LabOverlay\.kt$' || true)
[ -z "$blocked" ] || fail "these need a new APK, not a Lab build: $(echo $blocked)"
fi
apk=$(device pm path $pkg | sed -n 's/^package://p' | grep 'base\.apk$' | head -1)
[ -n "$apk" ] || fail "DroidDeck is not installed"
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-arm64 ANDROID_HOME=$HOME/android-sdk
# BuildConfig describes the installed base, not each Lab commit; its commit changing would make the
# Kotlin compile redo the whole app (the overlay's own label names the Lab build). Updates reads the
# installed variant (non-adreno) from the build label.
export DROIDDECK_SOURCE_COMMIT=$base DROIDDECK_BUILD_LABEL="lab on $installed_label" DROIDDECK_BUILD_TREE_STATE=clean GITHUB_HEAD_REF=lab
printf 'sdk.dir=%s\n' "$ANDROID_HOME" > local.properties
if [ ! -d "$ANDROID_HOME/platforms/android-34" ]; then
progress sdk
yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses >/dev/null
"$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "platforms;android-34" "build-tools;35.0.0" >/dev/null
fi
# A full Kotlin compile of the app needs about 2.5 GB of heap and Gradle about 1 GB; smaller devices get
# what they can spare. Both daemons end with this script's process when the build is done.
total_mb=$(awk '/^MemTotal:/ {print int($2 / 1024)}' /proc/meminfo)
heap=$(( total_mb / 2 )); [ $heap -gt 4096 ] && heap=4096; [ $heap -lt 2048 ] && heap=2048
heap=${DD_LAB_HEAP_MB:-$heap}
kotlin_heap=$(( heap * 2 / 3 )); gradle_heap=$(( heap - kotlin_heap ))
# Only the Kotlin daemon compiles incrementally; in-process compiles always redo the whole app, and the
# caches they leave behind make the daemon's first incremental compile fail.
if [ "$(cat "$cache/dd-lab-build-strategy" 2>/dev/null)" != daemon ]; then
rm -rf app/build/kotlin/compileReleaseKotlin
echo daemon > "$cache/dd-lab-build-strategy"
fi
# Google publishes aapt2 for x86-64 Linux only; setup wraps it in qemu-user at ~/aapt2/aapt2.
gradle=(./gradlew :app:processReleaseResources :app:mergeDexRelease --max-workers=2 -PskipRust=true -x buildVkBridgeServer
-Pandroid.aapt2FromMavenOverride=$HOME/aapt2/aapt2 -Pkotlin.compiler.execution.strategy=daemon
-Pkotlin.daemon.jvmargs=-Xmx${kotlin_heap}m -Dorg.gradle.jvmargs=-Xmx${gradle_heap}m --console=plain)
progress planning
mapfile -t plan < <("${gradle[@]}" --dry-run -q 2>/dev/null | awk '/^:/ {print $1}' || true)
total=${#plan[@]}
# Each task's estimate is how long it took last time; a full compile of the app takes about 15 minutes.
declare -A estimate
[ -f "$times_file" ] && while read -r t s; do estimate[$t]=$s; done < "$times_file"
estimate_of() { local e=${estimate[$1]:-}; [ -n "$e" ] && echo "$e" || { [[ $1 == *compileReleaseKotlin ]] && echo 900 || echo 2; }; }
planned=0
for t in "${plan[@]}"; do planned=$(( planned + $(estimate_of "$t") )); done
[ $planned -gt 0 ] || planned=1
current=$(mktemp)
measured=$(mktemp)
# "<tasks started> <task> <its start> <estimated seconds of tasks done> <its estimate>"
echo "0 starting $(date +%s) 0 0" > "$current"
report() {
local n task since done_s cur_est now part permille left
read -r n task since done_s cur_est < "$current"
now=$(date +%s)
part=$(( now - since )); [ $part -gt $(( cur_est * 95 / 100 )) ] && part=$(( cur_est * 95 / 100 ))
permille=$(( (done_s + part) * 1000 / planned )); [ $permille -gt 990 ] && permille=990
left=$(( planned - done_s - part )); [ $left -lt 0 ] && left=0
progress building "$permille" "$task" "$n" "$total" "$left"
echo "$n $task $(( (now - started) / 60 )) $left"
}
(
beat=0
while sleep 5; do
set -- $(report)
beat=$(( beat + 1 ))
[ $(( beat % 12 )) = 0 ] && echo "[build $1/$total] still on $2, $3m in, about $(( $4 / 60 + 1 ))m left"
done
) &
heartbeat=$!
trap 'kill $heartbeat 2>/dev/null || true; rm -f "$current" "$measured"' EXIT
set +o pipefail
"${gradle[@]}" ${explain[@]+"${explain[@]}"} 2>&1 | {
n=0 done_s=0 task="" since=0 cur_est=0 ran=0
while IFS= read -r line; do
printf '%s\n' "$line"
case $line in
"> Task "*)
now=$(date +%s)
# A task's line comes when it starts, or when it ends for one that did no work.
if [ -n "$task" ]; then
done_s=$(( done_s + cur_est ))
[ $ran = 1 ] && echo "$task $(( now - since ))" >> "$measured"
fi
rest=${line#> Task }; task=${rest%% *}
case $rest in *" "*) ran=0 ;; *) ran=1 ;; esac
n=$((n + 1)); since=$now; cur_est=$(estimate_of "$task")
echo "$n $task $since $done_s $cur_est" > "$current"
[ $ran = 1 ] && report >/dev/null && echo "[build $n/$total] $task" ;;
esac
done
if [ -n "$task" ] && [ $ran = 1 ]; then echo "$task $(( $(date +%s) - since ))" >> "$measured"; fi
}
status=${PIPESTATUS[0]}
set -o pipefail
kill $heartbeat 2>/dev/null || true
[ "$status" = 0 ] || fail "Gradle failed (exit $status); the reason is above"
# Tasks that did no work this time keep their earlier time.
{ [ -f "$times_file" ] && cat "$times_file"; cat "$measured"; } | awk '{t[$1] = $2} END {for (k in t) print k, t[k]}' > "$times_file.part"
mv "$times_file.part" "$times_file"
# The app reads which installed commit the build caches are warm for.
echo "$base" > "$cache/dd-lab-build-base"
if [ $mode = prepare ]; then
progress prepared 1000
echo "Prepared: Lab builds of branches on ${base:0:9} now compile only what changed ($(( ($(date +%s) - started) / 60 ))m)"
exit 0
fi
progress packaging 995
linked=$(ls app/build/intermediates/linked_resources_binary_format/release/processReleaseResources/*.ap_ | head -1)
work=$(mktemp -d)
cp "$linked" "$work/overlay.apk"
(cd app/build/intermediates/dex/release/mergeDexRelease && zip -q "$work/overlay.apk" classes*.dex)
name="$(git branch --show-current) @ $(git rev-parse --short HEAD)"
python3 -c 'import json,sys,time; print(json.dumps({"label": sys.argv[1], "baseCommit": sys.argv[2], "baseApk": sys.argv[3], "built": int(time.time())}))' \
"$name" "$label" "$apk" > "$work/overlay.json"
lab=/data/user/0/$pkg/files/lab
mkdir -p "$lab"
rm -rf "$lab/next.part" "$lab/next"
mv "$work" "$lab/next.part"
mv "$lab/next.part" "$lab/next"
progress ready 1000 "$name"
echo "Lab build ready: $name ($(du -h "$lab/next/overlay.apk" | cut -f1)) in $(( ($(date +%s) - started) / 60 ))m"
[ $mode = build ] || exit 0
# Classes in an app image load before the Lab build can, so the app is recompiled without one.
device cmd package compile -m verify -f $pkg >/dev/null
progress restarting 1000 "$name"
echo "Restarting DroidDeck on the Lab build; it shows a notification when it is back"
# The app's own agent ends with the app, so it is told to carry on once the app is back.
if [ -n "${DD_AGENT:-}" ]; then mkdir -p "/data/user/0/$pkg/files/agent" && : > "/data/user/0/$pkg/files/agent/resume"; fi
device "nohup sh -c 'sleep 3; monkey -p $pkg -c android.intent.category.LAUNCHER 1' >/dev/null 2>&1 &"
for pid in $(device pidof $pkg); do kill "$pid" 2>/dev/null || true; done
+76
View File
@@ -0,0 +1,76 @@
#!/bin/bash
# Sets up the on-device agent inside DroidDeck's Linux userland: Claude Code, git and gh, loopback
# adb for droiddeckctl, and the toolchain dd-lab-build uses to build Lab builds on the device.
# Run it from the DroidDeck terminal, inside a checkout of the DroidDeck repository.
set -euo pipefail
here=$(cd "$(dirname "$0")" && pwd)
export DEBIAN_FRONTEND=noninteractive
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
apt-get install -y --no-install-recommends git gh adb python3 zip unzip curl openjdk-17-jdk-headless qemu-user zipalign apksigner
command -v claude >/dev/null || [ -x "$HOME/.local/bin/claude" ] ||
{ curl -fsSL -o "$tmp/install-claude.sh" https://claude.ai/install.sh && bash "$tmp/install-claude.sh"; }
mkdir -p "$HOME/.local/bin"
for tool in dd-agent dd-lab-build dd-gh-login android-open; do install -m755 "$here/$tool" "$HOME/.local/bin/$tool"; done
grep -q 'droiddeck-agent-path' "$HOME/.bashrc" 2>/dev/null ||
printf '\n# droiddeck-agent-path\nexport PATH="$HOME/.local/bin:$PATH" ADB_SERIAL=127.0.0.1:5555 BROWSER="$HOME/.local/bin/android-open"\n' >> "$HOME/.bashrc"
# Google publishes aapt2 for x86-64 Linux only; it runs here under qemu-user with Ubuntu's x86-64 runtime.
x86=$HOME/x86root
mkdir -p "$HOME/aapt2/x86" "$x86/etc"
if [ ! -x "$HOME/aapt2/x86/aapt2" ]; then
agp=$(sed -n "s/.*com.android.tools.build:gradle:\([0-9.]*\).*/\1/p" "$here/../../build.gradle" | head -1)
[ -n "$agp" ] || { echo "setup: no Android Gradle plugin version in build.gradle" >&2; exit 1; }
aapt2=$(curl -fsSL "https://dl.google.com/android/maven2/com/android/tools/build/aapt2/maven-metadata.xml" |
{ grep -o "<version>$agp-[0-9]*</version>" || true; } | tail -1 | sed 's/<[^>]*>//g')
[ -n "$aapt2" ] || { echo "setup: Google publishes no aapt2 for AGP $agp" >&2; exit 1; }
curl -fsSL -o "$tmp/aapt2.jar" "https://dl.google.com/android/maven2/com/android/tools/build/aapt2/$aapt2/aapt2-$aapt2-linux.jar"
unzip -o -q "$tmp/aapt2.jar" aapt2 -d "$HOME/aapt2/x86"
chmod +x "$HOME/aapt2/x86/aapt2"
fi
if [ ! -e "$x86/lib64/ld-linux-x86-64.so.2" ]; then
codename=$(. /etc/os-release && echo "$VERSION_CODENAME")
curl -fsSL "https://archive.ubuntu.com/ubuntu/dists/$codename/main/binary-amd64/Packages.gz" | gunzip > "$tmp/amd64-packages"
for package in libc6 libgcc-s1 libstdc++6 zlib1g; do
read -r file sum < <(awk -v p="$package" '$0 == "Package: " p {found = 1} found && /^Filename:/ {f = $2}
found && /^SHA256:/ {s = $2} found && /^$/ {exit} END {print f, s}' "$tmp/amd64-packages")
[ -n "$sum" ] || { echo "setup: $package is not in Ubuntu $codename's amd64 index" >&2; exit 1; }
curl -fsSL -o "$tmp/$package.deb" "https://archive.ubuntu.com/ubuntu/$file"
echo "$sum $tmp/$package.deb" | sha256sum -c --quiet -
dpkg-deb -x "$tmp/$package.deb" "$x86"
done
ln -sfn usr/lib64 "$x86/lib64"
ln -sfn usr/lib "$x86/lib"
: > "$x86/etc/ld.so.preload"
fi
# AGP only accepts a custom aapt2 whose file is named aapt2.
printf '#!/bin/sh\nexec qemu-x86_64 -L %s %s/aapt2/x86/aapt2 "$@"\n' "$x86" "$HOME" > "$HOME/aapt2/aapt2"
chmod +x "$HOME/aapt2/aapt2"
"$HOME/aapt2/aapt2" version
sdk=$HOME/android-sdk
if [ ! -x "$sdk/cmdline-tools/latest/bin/sdkmanager" ]; then
mkdir -p "$sdk/cmdline-tools"
curl -fsSL -o "$tmp/cmdline-tools.zip" https://dl.google.com/android/repository/commandlinetools-linux-13114758_latest.zip
unzip -q -o "$tmp/cmdline-tools.zip" -d "$sdk/cmdline-tools"
rm -rf "$sdk/cmdline-tools/latest"
mv "$sdk/cmdline-tools/cmdline-tools" "$sdk/cmdline-tools/latest"
fi
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-arm64
yes | "$sdk/cmdline-tools/latest/bin/sdkmanager" --licenses >/dev/null
"$sdk/cmdline-tools/latest/bin/sdkmanager" "platforms;android-34" "build-tools;35.0.0" >/dev/null
adb connect 127.0.0.1:5555 || true
# The first build compiles the whole app; doing it now makes the agent's Lab builds incremental.
if [ "${DD_SKIP_PREPARE:-}" != 1 ]; then
echo "Approve the USB debugging prompt if one appears."
if timeout 120 adb -s 127.0.0.1:5555 wait-for-device; then
"$here/dd-lab-build" --prepare || echo "The first Lab build will compile the whole app"
else
echo "adb did not connect, so the first Lab build will compile the whole app"
fi
fi
echo "Done. Run dd-agent."