feat(lab): estimate the time left and compile the installed commit ahead of the first build

This commit is contained in:
Kurt Himebauch committed 2026-11-09 20:57:57 -05:00
1 parent b6d70b177d
commit acecbcd79a
3 files changed
+110 -39

No files matched your search

+13 -5
View File
@@ -15,8 +15,11 @@ change.
that the build was made for this install, puts its dex files ahead of the APK's, and adds its that the build was made for this install, puts its dex files ahead of the APK's, and adds its
resources through `ResourcesLoader` to the app and every activity. Contexts made from them resources through `ResourcesLoader` to the app and every activity. Contexts made from them
(presentations on the second screen) inherit the loader. (presentations on the second screen) inherit the loader.
- The watermark shows `LAB | <branch> @ <commit>`, and Updates has a Lab build card that goes back - The watermark shows `LAB | <branch> @ <commit>`. When the app comes back on a new build it posts
to the installed version. a "Lab build installed" notification, so the restart does not look like a crash.
- A build that is replaced, turned off or refused is kept in `files/lab/builds` (the newest 12).
Updates lists them under Lab builds: the one in use can be turned off, one built for this
install can be used again, and any can be deleted.
## What a Lab build cannot change ## What a Lab build cannot change
@@ -31,7 +34,9 @@ changes; they need a new APK.
update. A build for another install is turned off. update. A build for another install is turned off.
- Two starts in a row that do not reach a resumed activity for 10 seconds turn the build off. - Two starts in a row that do not reach a resumed activity for 10 seconds turn the build off.
- It needs Android 11 or later (`ResourcesLoader`) and no app image (classes in one load before the - It needs Android 11 or later (`ResourcesLoader`) and no app image (classes in one load before the
Lab build). `dd-lab-build` runs `cmd package compile -m verify` before restarting. Lab build). `dd-lab-build` runs `cmd package compile -m verify` before restarting. Android's
background optimization can bring an app image back; the build is then paused, a notification
says so, and Restart in Updates recompiles through the app's wireless debugging pairing.
## Setting up a device ## Setting up a device
@@ -43,5 +48,8 @@ tools/agent/setup
It installs Claude Code, git, gh and loopback adb, plus the build toolchain: JDK 17, the Android It installs Claude Code, git, gh and loopback adb, plus the build toolchain: JDK 17, the Android
SDK platform, and Google's x86-64 `aapt2` run under `qemu-user`. The first Lab build compiles the SDK platform, and Google's x86-64 `aapt2` run under `qemu-user`. The first Lab build compiles the
whole app and takes about 20 minutes on a Snapdragon 865; later ones are incremental. whole app and takes about 20 minutes on a Snapdragon 865; later ones are incremental. Setup
`~/.cache/dd-lab-build.json` reports the stage, task count and current task while it runs. does that first build ahead of time with `dd-lab-build --prepare`, which compiles the installed
commit without making a Lab build.
`~/.cache/dd-lab-build.json` reports the stage, the current task, progress in thousandths and the
seconds left, estimated from how long each task took the last time (`~/.cache/dd-lab-build-times`).
+95 -34
View File
@@ -1,36 +1,59 @@
#!/bin/bash #!/bin/bash
# Builds this checkout as a Lab build of the installed DroidDeck and restarts the app on it. # Builds this checkout as a Lab build of the installed DroidDeck and restarts the app on it.
# Usage: dd-lab-build [--no-restart] # Usage: dd-lab-build [--no-restart | --prepare]
# --prepare compiles the installed commit without making a Lab build, so later builds only
# compile what changed. Setup runs it once; it takes the longest.
set -euo pipefail set -euo pipefail
pkg=com.droiddeck.launcher pkg=com.droiddeck.launcher
serial=${ADB_SERIAL:-127.0.0.1:5555} serial=${ADB_SERIAL:-127.0.0.1:5555}
restart=1 mode=build
[ "${1:-}" = --no-restart ] && restart=0 case "${1:-}" in
--no-restart) mode=no-restart ;;
--prepare) mode=prepare ;;
esac
started=$(date +%s) started=$(date +%s)
progress_file=$HOME/.cache/dd-lab-build.json cache=$HOME/.cache
mkdir -p "$HOME/.cache" progress_file=$cache/dd-lab-build.json
# The Agent tab and the agent read this file to show how far the build has got. times_file=$cache/dd-lab-build-times
progress() { # stage [done total task] mkdir -p "$cache"
printf '{"stage":"%s","done":%d,"total":%d,"task":"%s","elapsed":%d}\n' \ # The Agent tab reads this file to show how far the build has got and how long is left.
"$1" "${2:-0}" "${3:-0}" "${4:-}" $(( $(date +%s) - started )) > "$progress_file.part" progress() { # stage [permille task done total eta]
mv "$progress_file.part" "$progress_file" local task=${3:-}; task=${task//\\/}; task=${task//\"/\'}; task=${task//$'\n'/ }
printf '{"stage":"%s","mode":"%s","permille":%d,"task":"%s","done":%d,"total":%d,"eta":%d,"elapsed":%d,"pid":%d}\n' \
"$1" "$mode" "${2:-0}" "$task" "${4:-0}" "${5:-0}" "${6:--1}" $(( $(date +%s) - started )) $$ > "$progress_file.part.$BASHPID"
mv "$progress_file.part.$BASHPID" "$progress_file"
} }
fail() { progress failed; echo "dd-lab-build: $*" >&2; exit 1; } fail() { progress failed 0 "$*"; echo "dd-lab-build: $*" >&2; exit 1; }
trap 'progress failed 0 stopped; exit 143' INT TERM
device() { adb -s "$serial" shell "$@" | tr -d '\r'; } device() { adb -s "$serial" shell "$@" | tr -d '\r'; }
progress checking progress checking
repo=$(git rev-parse --show-toplevel) || fail "run it inside the DroidDeck checkout" repo=$(git rev-parse --show-toplevel) || fail "run it inside the DroidDeck checkout"
cd "$repo" cd "$repo"
adb connect "$serial" >/dev/null 2>&1 || true adb connect "$serial" >/dev/null 2>&1 || true
git diff --quiet && git diff --cached --quiet || fail "commit your changes first"
state=$(python3 tools/droiddeckctl state 2>/dev/null | sed -n '/^{/,$p') state=$(python3 tools/droiddeckctl state 2>/dev/null | sed -n '/^{/,$p')
# A Lab build reports its own label, so the APK's commit comes from apkCommit when the app has it. # A Lab build reports its own label, so the APK's commit comes from apkCommit when the app has it.
label=$(python3 -c 'import json,sys; s=json.load(sys.stdin); print(s.get("apkCommit") or s["build"].rsplit(" at ", 1)[-1])' <<<"$state") label=$(python3 -c 'import json,sys; s=json.load(sys.stdin); print(s.get("apkCommit") or s["build"].rsplit(" at ", 1)[-1])' <<<"$state")
base=$(git rev-parse --verify --quiet "$label^{commit}") || fail "the installed build ($label) is not in this checkout; fetch it first" if ! base=$(git rev-parse --verify --quiet "$label^{commit}"); then
git merge-base --is-ancestor "$base" HEAD || fail "HEAD does not build on the installed commit ${base:0:9}; branch from it" progress fetching 0 "the installed commit"
blocked=$(git diff --name-only "$base" HEAD | grep -E 'AndroidManifest\.xml$|/cpp/|/jniLibs/|\.rs$|(^|/)build\.gradle$|/App\.kt$|/lab/LabOverlay\.kt$' || true) git fetch --quiet --tags origin >/dev/null 2>&1 || true
[ -z "$blocked" ] || fail "these need a new APK, not a Lab build: $(echo $blocked)" git fetch --quiet origin main >/dev/null 2>&1 || true
base=$(git rev-parse --verify --quiet "$label^{commit}") || fail "the installed build ($label) is not in this checkout or on GitHub's main; fetch it first"
fi
if [ $mode = prepare ]; then
# Compiling the installed commit itself gives later builds of branches on it incremental caches.
if ! git merge-base --is-ancestor "$base" HEAD; then
git diff --quiet && git diff --cached --quiet || fail "HEAD does not build on the installed commit and has uncommitted changes"
git checkout --quiet --detach "$base"
fi
else
git diff --quiet && git diff --cached --quiet || fail "commit your changes first"
git merge-base --is-ancestor "$base" HEAD || fail "HEAD does not build on the installed commit ${base:0:9}; branch from it"
blocked=$(git diff --name-only "$base" HEAD | grep -E 'AndroidManifest\.xml$|/cpp/|/jniLibs/|\.rs$|(^|/)build\.gradle$|/App\.kt$|/lab/LabOverlay\.kt$' || true)
[ -z "$blocked" ] || fail "these need a new APK, not a Lab build: $(echo $blocked)"
fi
apk=$(device pm path $pkg | sed -n 's/^package://p' | grep 'base\.apk$' | head -1) apk=$(device pm path $pkg | sed -n 's/^package://p' | grep 'base\.apk$' | head -1)
[ -n "$apk" ] || fail "DroidDeck is not installed" [ -n "$apk" ] || fail "DroidDeck is not installed"
@@ -52,44 +75,82 @@ gradle=(./gradlew :app:processReleaseResources :app:mergeDexRelease --max-worker
-Pandroid.aapt2FromMavenOverride=$HOME/aapt2/aapt2 -Pkotlin.compiler.execution.strategy=in-process -Pandroid.aapt2FromMavenOverride=$HOME/aapt2/aapt2 -Pkotlin.compiler.execution.strategy=in-process
-Dorg.gradle.jvmargs=-Xmx${heap}m --console=plain) -Dorg.gradle.jvmargs=-Xmx${heap}m --console=plain)
progress planning progress planning
total=$("${gradle[@]}" --dry-run -q 2>/dev/null | grep -c '^:' || true) mapfile -t plan < <("${gradle[@]}" --dry-run -q 2>/dev/null | awk '/^:/ {print $1}' || true)
total=${#plan[@]}
# Each task's estimate is how long it took last time; a full compile of the app takes about 15 minutes.
declare -A estimate
[ -f "$times_file" ] && while read -r t s; do estimate[$t]=$s; done < "$times_file"
estimate_of() { local e=${estimate[$1]:-}; [ -n "$e" ] && echo "$e" || { [[ $1 == *compileReleaseKotlin ]] && echo 900 || echo 2; }; }
planned=0
for t in "${plan[@]}"; do planned=$(( planned + $(estimate_of "$t") )); done
[ $planned -gt 0 ] || planned=1
current=$(mktemp) current=$(mktemp)
echo "0 starting" > "$current" measured=$(mktemp)
# "<tasks started> <task> <its start> <estimated seconds of tasks done> <its estimate>"
echo "0 starting $(date +%s) 0 0" > "$current"
report() {
local n task since done_s cur_est now part permille left
read -r n task since done_s cur_est < "$current"
now=$(date +%s)
part=$(( now - since )); [ $part -gt $(( cur_est * 95 / 100 )) ] && part=$(( cur_est * 95 / 100 ))
permille=$(( (done_s + part) * 1000 / planned )); [ $permille -gt 990 ] && permille=990
left=$(( planned - done_s - part )); [ $left -lt 0 ] && left=0
progress building "$permille" "$task" "$n" "$total" "$left"
echo "$n $task $(( (now - started) / 60 )) $left"
}
( (
while sleep 60; do while sleep 20; do
read -r n task < "$current" set -- $(report)
echo "[build $n/$total] still on $task, $(( ($(date +%s) - started) / 60 ))m in" echo "[build $1/$total] still on $2, $3m in, about $(( $4 / 60 + 1 ))m left"
progress building "$n" "$total" "$task"
done done
) & ) &
heartbeat=$! heartbeat=$!
trap 'kill $heartbeat 2>/dev/null || true' EXIT trap 'kill $heartbeat 2>/dev/null || true; rm -f "$current" "$measured"' EXIT
set +o pipefail set +o pipefail
"${gradle[@]}" 2>&1 | { "${gradle[@]}" 2>&1 | {
n=0 n=0 done_s=0 task="" since=0 cur_est=0 ran=0
while IFS= read -r line; do while IFS= read -r line; do
printf '%s\n' "$line" printf '%s\n' "$line"
case $line in case $line in
"> Task "*) "> Task "*)
n=$((n + 1)); task=${line#> Task }; task=${task%% *} now=$(date +%s)
echo "$n $task" > "$current" # A task's line comes when it starts, or when it ends for one that did no work.
echo "[build $n/$total] $task" if [ -n "$task" ]; then
progress building "$n" "$total" "$task" ;; done_s=$(( done_s + cur_est ))
[ $ran = 1 ] && echo "$task $(( now - since ))" >> "$measured"
fi
rest=${line#> Task }; task=${rest%% *}
case $rest in *" "*) ran=0 ;; *) ran=1 ;; esac
n=$((n + 1)); since=$now; cur_est=$(estimate_of "$task")
echo "$n $task $since $done_s $cur_est" > "$current"
[ $ran = 1 ] && report >/dev/null && echo "[build $n/$total] $task" ;;
esac esac
done done
[ -n "$task" ] && [ $ran = 1 ] && echo "$task $(( $(date +%s) - since ))" >> "$measured"
} }
status=${PIPESTATUS[0]} status=${PIPESTATUS[0]}
set -o pipefail set -o pipefail
kill $heartbeat 2>/dev/null || true kill $heartbeat 2>/dev/null || true
[ "$status" = 0 ] || fail "Gradle failed (exit $status); the reason is above" [ "$status" = 0 ] || fail "Gradle failed (exit $status); the reason is above"
# Tasks that did no work this time keep their earlier time.
{ [ -f "$times_file" ] && cat "$times_file"; cat "$measured"; } | awk '{t[$1] = $2} END {for (k in t) print k, t[k]}' > "$times_file.part"
mv "$times_file.part" "$times_file"
progress packaging if [ $mode = prepare ]; then
progress prepared 1000
echo "Prepared: Lab builds of branches on ${base:0:9} now compile only what changed ($(( ($(date +%s) - started) / 60 ))m)"
exit 0
fi
progress packaging 995
linked=$(ls app/build/intermediates/linked_resources_binary_format/release/processReleaseResources/*.ap_ | head -1) linked=$(ls app/build/intermediates/linked_resources_binary_format/release/processReleaseResources/*.ap_ | head -1)
work=$(mktemp -d) work=$(mktemp -d)
cp "$linked" "$work/overlay.apk" cp "$linked" "$work/overlay.apk"
(cd app/build/intermediates/dex/release/mergeDexRelease && zip -q "$work/overlay.apk" classes*.dex) (cd app/build/intermediates/dex/release/mergeDexRelease && zip -q "$work/overlay.apk" classes*.dex)
name="$(git branch --show-current) @ $(git rev-parse --short HEAD)" name="$(git branch --show-current) @ $(git rev-parse --short HEAD)"
python3 -c 'import json,sys; print(json.dumps({"label": sys.argv[1], "baseCommit": sys.argv[2], "baseApk": sys.argv[3]}))' \ python3 -c 'import json,sys,time; print(json.dumps({"label": sys.argv[1], "baseCommit": sys.argv[2], "baseApk": sys.argv[3], "built": int(time.time())}))' \
"$name" "$base" "$apk" > "$work/overlay.json" "$name" "$base" "$apk" > "$work/overlay.json"
lab=/data/user/0/$pkg/files/lab lab=/data/user/0/$pkg/files/lab
@@ -97,14 +158,14 @@ mkdir -p "$lab"
rm -rf "$lab/next.part" "$lab/next" rm -rf "$lab/next.part" "$lab/next"
mv "$work" "$lab/next.part" mv "$work" "$lab/next.part"
mv "$lab/next.part" "$lab/next" mv "$lab/next.part" "$lab/next"
progress ready progress ready 1000 "$name"
echo "Lab build ready: $name ($(du -h "$lab/next/overlay.apk" | cut -f1)) in $(( ($(date +%s) - started) / 60 ))m" echo "Lab build ready: $name ($(du -h "$lab/next/overlay.apk" | cut -f1)) in $(( ($(date +%s) - started) / 60 ))m"
[ $restart = 1 ] || exit 0 [ $mode = build ] || exit 0
# Classes in an app image load before the Lab build can, so the app is recompiled without one. # Classes in an app image load before the Lab build can, so the app is recompiled without one.
device cmd package compile -m verify -f $pkg >/dev/null device cmd package compile -m verify -f $pkg >/dev/null
progress restarting progress restarting 1000 "$name"
echo "Restarting DroidDeck on the Lab build" echo "Restarting DroidDeck on the Lab build; it shows a notification when it is back"
# The app's own agent ends with the app, so it is told to carry on once the app is back. # The app's own agent ends with the app, so it is told to carry on once the app is back.
if [ -n "${DD_AGENT:-}" ]; then mkdir -p "/data/user/0/$pkg/files/agent" && : > "/data/user/0/$pkg/files/agent/resume"; fi if [ -n "${DD_AGENT:-}" ]; then mkdir -p "/data/user/0/$pkg/files/agent" && : > "/data/user/0/$pkg/files/agent/resume"; fi
device "nohup sh -c 'sleep 3; monkey -p $pkg -c android.intent.category.LAUNCHER 1' >/dev/null 2>&1 &" device "nohup sh -c 'sleep 3; monkey -p $pkg -c android.intent.category.LAUNCHER 1' >/dev/null 2>&1 &"
+2
View File
@@ -56,4 +56,6 @@ yes | "$sdk/cmdline-tools/latest/bin/sdkmanager" --licenses >/dev/null
"$sdk/cmdline-tools/latest/bin/sdkmanager" "platforms;android-34" "build-tools;35.0.0" >/dev/null "$sdk/cmdline-tools/latest/bin/sdkmanager" "platforms;android-34" "build-tools;35.0.0" >/dev/null
adb connect 127.0.0.1:5555 || true adb connect 127.0.0.1:5555 || true
# The first build compiles the whole app; doing it now makes the agent's Lab builds incremental.
[ "${DD_SKIP_PREPARE:-}" = 1 ] || "$here/dd-lab-build" --prepare || echo "The first Lab build will compile the whole app"
echo "Done. Approve the USB debugging prompt if one appears, then run dd-agent." echo "Done. Approve the USB debugging prompt if one appears, then run dd-agent."