#!/bin/bash # Builds this checkout as a Lab build of the installed DroidDeck and restarts the app on it. # Usage: dd-lab-build [--no-restart | --prepare] [--explain] # --prepare compiles the installed commit without making a Lab build, so later builds only # compile what changed. Setup runs it once; it takes the longest. # --explain logs why each task ran and Kotlin's build report, to find out what makes a build slow. set -euo pipefail pkg=${DROIDDECK_PACKAGE:-} if [ -z "$pkg" ] && [[ ${DD_AGENT_INBOX:-} =~ ^/data/(user/[0-9]+|data)/([^/]+)/files/agent$ ]]; then pkg=${BASH_REMATCH[2]} fi pkg=${pkg:-com.droiddeck.launcher} serial=${ADB_SERIAL:-127.0.0.1:5555} mode=build explain=() for arg in "$@"; do case $arg in --package=*) pkg=${arg#--package=} ;; --no-restart) mode=no-restart ;; --prepare) mode=prepare ;; --explain) explain=(--info -Pkotlin.build.report.output=file) ;; # Anything else used to start a full build and restart the app, --help included. -h|--help) sed -n '2,6s/^# \{0,1\}//p' "$0"; exit 0 ;; *) echo "dd-lab-build: unknown option $arg (see --help)" >&2; exit 2 ;; esac done started=$(date +%s) cache=$HOME/.cache progress_file=$cache/dd-lab-build.json times_file=$cache/dd-lab-build-times mkdir -p "$cache" # The Agent tab reads this file to show how far the build has got and how long is left. progress() { # stage [permille task done total eta] local task=${3:-}; task=${task//\\/}; task=${task//\"/\'}; task=${task//$'\n'/ } printf '{"stage":"%s","mode":"%s","permille":%d,"task":"%s","done":%d,"total":%d,"eta":%d,"elapsed":%d,"pid":%d}\n' \ "$1" "$mode" "${2:-0}" "$task" "${4:-0}" "${5:-0}" "${6:--1}" $(( $(date +%s) - started )) $$ > "$progress_file.part.$BASHPID" mv "$progress_file.part.$BASHPID" "$progress_file" } fail() { progress failed 0 "$*"; echo "dd-lab-build: $*" >&2; exit 1; } trap 'progress failed 0 stopped; exit 143' INT TERM trap 'progress failed 0 "a command failed on line $LINENO"' ERR [[ $pkg =~ ^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z][A-Za-z0-9_]*)+$ ]] || fail "invalid Android package: $pkg" export DROIDDECK_PACKAGE=$pkg device() { adb -s "$serial" shell "$@" | tr -d '\r'; } # The app's agent loses every process it started when its turn ends, a build included, so it asks # the app to run the build in a process of its own and follows that build here. agent=${DD_AGENT_INBOX:-/data/user/0/$pkg/files/agent} if [ -n "${DD_AGENT:-}" ] && [ -z "${DD_LAB_DETACHED:-}" ] && [ -d "$agent" ]; then trap - ERR log=$cache/dd-lab-build.log rm -f "$progress_file" python3 -c 'import json,os,sys; print(json.dumps({"cwd": os.getcwd(), "args": [*sys.argv[1:], "--package=" + os.environ["DROIDDECK_PACKAGE"]]}))' "$@" > "$agent/lab-request.part" mv "$agent/lab-request.part" "$agent/lab-request" echo "DroidDeck runs this build itself, so it keeps going if this turn ends. Its log follows." for _ in $(seq 1 30); do [ -f "$progress_file" ] && break; sleep 1; done [ -f "$progress_file" ] || { echo "dd-lab-build: DroidDeck did not start the build; is it running?" >&2; exit 1; } tail -n +1 -F "$log" 2>/dev/null & follow=$! result=1 while sleep 2; do stage=$(sed -n 's/.*"stage":"\([a-z]*\)".*/\1/p' "$progress_file") pid=$(sed -n 's/.*"pid":\([0-9]*\).*/\1/p' "$progress_file") case $stage in ready|prepared) result=0; break ;; failed) break ;; esac [ -n "$pid" ] && [ -d "/proc/$pid" ] || break done sleep 1; kill $follow 2>/dev/null || true exit $result fi progress checking repo=$(git rev-parse --show-toplevel) || fail "run it inside the DroidDeck checkout" cd "$repo" adb connect "$serial" >/dev/null 2>&1 || true state=$(python3 tools/droiddeckctl state 2>/dev/null | sed -n '/^{/,$p') || fail "could not read DroidDeck's state; is it running and is adb connected?" # A Lab build reports its own label, so the APK's commit comes from apkCommit when the app has it. label=$(python3 -c 'import json,sys; s=json.load(sys.stdin); print(s.get("apkCommit") or s["build"].rsplit(" at ", 1)[-1])' <<<"$state") || fail "DroidDeck's state did not say which build is installed" installed_label=$(python3 -c 'import json,re,sys; print(re.sub(r"^(?:\w+ \w+ ยท |lab on )+", "", json.load(sys.stdin)["build"]))' <<<"$state") if ! base=$(git rev-parse --verify --quiet "$label^{commit}"); then progress fetching 0 "the installed commit" git fetch --quiet --tags origin >/dev/null 2>&1 || true git fetch --quiet origin main >/dev/null 2>&1 || true base=$(git rev-parse --verify --quiet "$label^{commit}") || fail "the installed build ($label) is not in this checkout or on GitHub's main; fetch it first" fi if [ $mode = prepare ]; then # Compiling the installed commit itself gives later builds of branches on it incremental caches. if ! git merge-base --is-ancestor "$base" HEAD; then git diff --quiet && git diff --cached --quiet || fail "HEAD does not build on the installed commit and has uncommitted changes" was=$(git branch --show-current) git checkout --quiet --detach "$base" echo "Checked out the installed commit ${base:0:9} to prepare it${was:+; $was is unchanged}" fi else git diff --quiet && git diff --cached --quiet || fail "commit your changes first" git merge-base --is-ancestor "$base" HEAD || fail "HEAD does not build on the installed commit ${base:0:9}; branch from it" blocked=$(git diff --name-only "$base" HEAD | grep -E 'AndroidManifest\.xml$|/cpp/|/jniLibs/|\.rs$|(^|/)build\.gradle$|/App\.kt$|/AppLanguage\.kt$|/lab/LabOverlay\.kt$' || true) [ -z "$blocked" ] || fail "these need a new APK, not a Lab build: $(echo $blocked)" fi apk=$(device pm path $pkg | sed -n 's/^package://p' | grep 'base\.apk$' | head -1) [ -n "$apk" ] || fail "DroidDeck is not installed" export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-arm64 ANDROID_HOME=$HOME/android-sdk # BuildConfig describes the installed base, not each Lab commit; its commit changing would make the # Kotlin compile redo the whole app (the overlay's own label names the Lab build). Updates reads the # installed variant (non-adreno) from the build label. export DROIDDECK_SOURCE_COMMIT=$base DROIDDECK_BUILD_LABEL="lab on $installed_label" DROIDDECK_BUILD_TREE_STATE=clean GITHUB_HEAD_REF=lab printf 'sdk.dir=%s\n' "$ANDROID_HOME" > local.properties if [ ! -d "$ANDROID_HOME/platforms/android-34" ]; then progress sdk (set +o pipefail; yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses >/dev/null) "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "platforms;android-34" "build-tools;35.0.0" >/dev/null fi # A full Kotlin compile of the app runs out of memory with 2.4 GB of heap, and Gradle needs about 1 GB, # so devices with less memory to spare swap rather than fail. Both daemons end with this script's # process when the build is done. heap=${DD_LAB_HEAP_MB:-4096} kotlin_heap=$(( heap * 2 / 3 )); gradle_heap=$(( heap - kotlin_heap )) # Only the Kotlin daemon compiles incrementally; in-process compiles always redo the whole app, and the # caches they leave behind make the daemon's first incremental compile fail. if [ "$(cat "$cache/dd-lab-build-strategy" 2>/dev/null)" != daemon ]; then rm -rf app/build/kotlin/compileReleaseKotlin echo daemon > "$cache/dd-lab-build-strategy" fi # Google publishes aapt2 for x86-64 Linux only; setup wraps it in qemu-user at ~/aapt2/aapt2. gradle=(./gradlew :app:processReleaseResources :app:mergeDexRelease --max-workers=2 -PskipRust=true -x buildVkBridgeServer -Pandroid.aapt2FromMavenOverride=$HOME/aapt2/aapt2 -Pkotlin.compiler.execution.strategy=daemon -Pkotlin.daemon.jvmargs=-Xmx${kotlin_heap}m -Dorg.gradle.jvmargs=-Xmx${gradle_heap}m --console=plain) progress planning plan_file=$cache/dd-lab-build-plan plan_key=$( { printf '%s\n' "$repo" "${gradle[@]}" git ls-files -z -- '*gradle*' | xargs -0 -r sha256sum } | sha256sum | cut -d ' ' -f1 ) if [ "$(head -n 1 "$plan_file" 2>/dev/null)" != "$plan_key" ]; then plan_output=$("${gradle[@]}" --dry-run) || fail "Gradle planning failed: $plan_output" { printf '%s\n' "$plan_key"; awk '/^:/ {print $1}' <<<"$plan_output"; } > "$plan_file.part" mv "$plan_file.part" "$plan_file" fi mapfile -t plan < <(tail -n +2 "$plan_file") total=${#plan[@]} # Each task's estimate is how long it took last time; a full compile of the app takes about 15 minutes. declare -A estimate [ -f "$times_file" ] && while read -r t s; do estimate[$t]=$s; done < "$times_file" estimate_of() { local e=${estimate[$1]:-}; [ -n "$e" ] && echo "$e" || { [[ $1 == *compileReleaseKotlin ]] && echo 900 || echo 2; }; } planned=0 for t in "${plan[@]}"; do planned=$(( planned + $(estimate_of "$t") )); done [ $planned -gt 0 ] || planned=1 current=$(mktemp) measured=$(mktemp) # " " echo "0 starting $(date +%s) 0 0" > "$current" report() { local n task since done_s cur_est now part permille left read -r n task since done_s cur_est < "$current" now=$(date +%s) part=$(( now - since )); [ $part -gt $(( cur_est * 95 / 100 )) ] && part=$(( cur_est * 95 / 100 )) permille=$(( (done_s + part) * 1000 / planned )); [ $permille -gt 990 ] && permille=990 left=$(( planned - done_s - part )); [ $left -lt 0 ] && left=0 progress building "$permille" "$task" "$n" "$total" "$left" echo "$n $task $(( (now - started) / 60 )) $left" } ( beat=0 while sleep 5; do set -- $(report) beat=$(( beat + 1 )) [ $(( beat % 12 )) = 0 ] && echo "[build $1/$total] still on $2, $3m in, about $(( $4 / 60 + 1 ))m left" done ) & heartbeat=$! trap 'kill $heartbeat 2>/dev/null || true; rm -f "$current" "$measured"' EXIT set +o pipefail "${gradle[@]}" ${explain[@]+"${explain[@]}"} 2>&1 | { n=0 done_s=0 task="" since=0 cur_est=0 ran=0 while IFS= read -r line; do printf '%s\n' "$line" case $line in "> Task "*) now=$(date +%s) # A task's line comes when it starts, or when it ends for one that did no work. if [ -n "$task" ]; then done_s=$(( done_s + cur_est )) [ $ran = 1 ] && echo "$task $(( now - since ))" >> "$measured" fi rest=${line#> Task }; task=${rest%% *} case $rest in *" "*) ran=0 ;; *) ran=1 ;; esac n=$((n + 1)); since=$now; cur_est=$(estimate_of "$task") echo "$n $task $since $done_s $cur_est" > "$current" [ $ran = 1 ] && report >/dev/null && echo "[build $n/$total] $task" ;; esac done if [ -n "$task" ] && [ $ran = 1 ]; then echo "$task $(( $(date +%s) - since ))" >> "$measured"; fi } status=${PIPESTATUS[0]} set -o pipefail kill $heartbeat 2>/dev/null || true [ "$status" = 0 ] || fail "Gradle failed (exit $status); the reason is above" # Tasks that did no work this time keep their earlier time. { [ -f "$times_file" ] && cat "$times_file"; cat "$measured"; } | awk '{t[$1] = $2} END {for (k in t) print k, t[k]}' > "$times_file.part" mv "$times_file.part" "$times_file" # The app reads which installed commit the build caches are warm for. echo "$base" > "$cache/dd-lab-build-base" if [ $mode = prepare ]; then progress prepared 1000 echo "Prepared: Lab builds of branches on ${base:0:9} now compile only what changed ($(( ($(date +%s) - started) / 60 ))m)" exit 0 fi progress packaging 995 linked=$(ls app/build/intermediates/linked_resources_binary_format/release/processReleaseResources/*.ap_ | head -1) work=$(mktemp -d) cp "$linked" "$work/overlay.apk" (cd app/build/intermediates/dex/release/mergeDexRelease && zip -q "$work/overlay.apk" classes*.dex) name="$(git branch --show-current) @ $(git rev-parse --short HEAD)" python3 -c 'import json,sys,time; print(json.dumps({"label": sys.argv[1], "baseCommit": sys.argv[2], "baseApk": sys.argv[3], "built": int(time.time())}))' \ "$name" "$label" "$apk" > "$work/overlay.json" lab=/data/user/0/$pkg/files/lab mkdir -p "$lab" rm -rf "$lab/next.part" "$lab/next" mv "$work" "$lab/next.part" mv "$lab/next.part" "$lab/next" progress ready 1000 "$name" echo "Lab build ready: $name ($(du -h "$lab/next/overlay.apk" | cut -f1)) in $(( ($(date +%s) - started) / 60 ))m" [ $mode = build ] || exit 0 # Classes in an app image load before the Lab build can, so the app is recompiled without one. device cmd package compile -m verify -f $pkg >/dev/null progress restarting 1000 "$name" echo "Restarting DroidDeck on the Lab build; it shows a notification when it is back" # The app's own agent ends with the app, so it is told to carry on once the app is back. if [ -n "${DD_AGENT:-}" ]; then mkdir -p "$agent" && : > "$agent/resume"; fi device "nohup sh -c 'sleep 3; monkey -p $pkg -c android.intent.category.LAUNCHER 1' >/dev/null 2>&1 &" for pid in $(device pidof $pkg); do kill "$pid" 2>/dev/null || true; done