From 48adf4d1ff9c96db1f45fa7f8ab5b506cd33a319 Mon Sep 17 00:00:00 2001 From: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:30:24 -0400 Subject: [PATCH] perf: local builds reuse unchanged native steps across runs and worktrees (#819) --- tools/build_local.sh | 76 +++++++++++++++++++++++++++++++------- tools/directaudio/fetch.sh | 15 ++++++-- 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/tools/build_local.sh b/tools/build_local.sh index 2695ef084..c4c0e07dc 100755 --- a/tools/build_local.sh +++ b/tools/build_local.sh @@ -72,7 +72,33 @@ trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM +cache_dir=${DROIDDECK_BUILD_CACHE:-"${HOME}/.cache/droiddeck-build"} +mkdir -p "${cache_dir}" +# Steps whose output depends only on tracked sources are kept per input hash in the +# shared cache, so a rerun or a fresh worktree copies them instead of rebuilding. +inputs_hash() { + (cd "${repo_root}" && { printf '%s\n' "$1"; shift; find "$@" -type f ! -name '*.pyc' | LC_ALL=C sort | xargs shasum -a 256; }) \ + | shasum -a 256 | cut -d' ' -f1 +} +restore_step() { + local stored="${cache_dir}/steps/$1/$2" + [[ -f "${stored}/.complete" ]] || return 1 + mkdir -p "$3" + (cd "${stored}" && tar -cf - --exclude .complete .) | tar -xf - -C "$3" || return 1 + echo "$1: inputs unchanged, reused ${stored}" +} +save_step() { + local stored="${cache_dir}/steps/$1/$2" part + mkdir -p "${cache_dir}/steps/$1" + part=$(mktemp -d "${stored}.part.XXXXXX") + (cd "$3" && tar -cf - .) | tar -xf - -C "${part}" + touch "${part}/.complete" + # Another checkout may have published the same key meanwhile; its copy is equivalent. + if [[ -e "${stored}" ]]; then rm -rf "${part}"; else mv "${part}" "${stored}" || rm -rf "${part}"; fi +} + python3 "${repo_root}/tools/docker_preflight.py" "${image_name}" +image_id=$(docker image inspect --format '{{.Id}}' "${image_name}") rm -rf -- "${linuxfs_dir}" mkdir -p "${linuxfs_dir}" @@ -118,17 +144,30 @@ docker run --rm --platform linux/amd64 \ READELF=aarch64-linux-gnu-readelf bash tools/linuxfs/check-preloads.sh "$d" ' -docker run --rm --platform linux/amd64 \ - -v "${repo_root}:/src" -w /src public.ecr.aws/docker/library/debian:bullseye bash -c ' - set -euo pipefail - printf "deb http://archive.debian.org/debian bullseye main\ndeb http://archive.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list - apt-get -o Acquire::Check-Valid-Until=false update -qq - DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends gcc g++ gcc-multilib g++-multilib binutils >/dev/null - tools/linuxfs/build-x86-preloads.sh app/src/main/assets/linuxfs - chown -R '"$(id -u):$(id -g)"' app/src/main/assets/linuxfs - ' +x86_key=$(inputs_hash bullseye tools/linuxfs/build-x86-preloads.sh tools/linuxfs/preload tools/linuxfs/fex \ + app/src/main/cpp/fakeinput_steam.cpp) +x86_stage="${staging_dir}/x86-preloads" +if restore_step x86-preloads "${x86_key}" "${linuxfs_dir}"; then :; else + docker run --rm --platform linux/amd64 \ + -v "${repo_root}:/src" -w /src public.ecr.aws/docker/library/debian:bullseye bash -c ' + set -euo pipefail + printf "deb http://archive.debian.org/debian bullseye main\ndeb http://archive.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list + apt-get -o Acquire::Check-Valid-Until=false update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends gcc g++ gcc-multilib g++-multilib binutils >/dev/null + tools/linuxfs/build-x86-preloads.sh app/src/main/assets/linuxfs + chown -R '"$(id -u):$(id -g)"' app/src/main/assets/linuxfs + ' + mkdir -p "${x86_stage}" + cp -R "${linuxfs_dir}/x86_64" "${linuxfs_dir}/i386" "${x86_stage}/" + save_step x86-preloads "${x86_key}" "${x86_stage}" +fi -DROIDDECK_BUILD_IMAGE="${image_name}" "${repo_root}/tools/vr/build.sh" "${linuxfs_dir}/usr/local/lib/droiddeck-vr" +vr_out="${linuxfs_dir}/usr/local/lib/droiddeck-vr" +vr_key=$(inputs_hash "${image_id} ${ndk_version} ${VK_INCLUDE:-}" tools/vr app/src/main/cpp/waylandcomp/generated/xdg-shell-protocol.c) +if ! restore_step vr "${vr_key}" "${vr_out}"; then + DROIDDECK_BUILD_IMAGE="${image_name}" "${repo_root}/tools/vr/build.sh" "${vr_out}" + save_step vr "${vr_key}" "${vr_out}" +fi github_repo=${DROIDDECK_GITHUB_REPOSITORY:-} if [[ -z "${github_repo}" ]]; then @@ -146,8 +185,6 @@ if [[ -z "${github_repo}" ]]; then github_repo=${github_repo%.git} fi -cache_dir=${DROIDDECK_BUILD_CACHE:-"${HOME}/.cache/droiddeck-build"} -mkdir -p "${cache_dir}" cached() { local sha=$1 name=$2 out shift 2 @@ -268,7 +305,12 @@ if [[ -f "${proot_out}/libproot.so" && -f "${proot_out}/libproot-loader.so" \ && (cd "${proot_out}" && shasum -a 256 -c .proot-outputs >/dev/null 2>&1); then echo "proot: sources unchanged, keeping ${proot_out}/libproot.so" else - "${repo_root}/tools/proot/build.sh" "${proot_out}" + proot_stage="${staging_dir}/proot" + if ! restore_step proot "${proot_inputs}" "${proot_stage}"; then + "${repo_root}/tools/proot/build.sh" "${proot_stage}" + save_step proot "${proot_inputs}" "${proot_stage}" + fi + install -m755 "${proot_stage}/libproot.so" "${proot_stage}/libproot-loader.so" "${proot_out}/" echo "${proot_inputs}" > "${proot_out}/.proot-inputs" (cd "${proot_out}" && shasum -a 256 libproot.so libproot-loader.so > .proot-outputs) fi @@ -301,6 +343,7 @@ else fi done rustup target add --toolchain stable aarch64-linux-android + export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-${cache_dir}/stores-target}" if ! command -v cargo-ndk >/dev/null 2>&1; then cargo install cargo-ndk --locked fi @@ -308,7 +351,12 @@ fi cd "${repo_root}" python3 tools/quest/build-driver.py "${NDK}" -bash tools/comet/build-local.sh +comet_out="${linuxfs_dir}/usr/local/lib/droiddeck-comet" +comet_key=$(inputs_hash "${image_id}" tools/comet) +if ! restore_step comet "${comet_key}" "${comet_out}"; then + DROIDDECK_BUILD_IMAGE="${image_name}" bash tools/comet/build-local.sh + save_step comet "${comet_key}" "${comet_out}" +fi python3 tools/release/runtime_inputs.py record ./gradlew "${gradle_task}" --console=plain -PndkVersion="${ndk_version}" ${gradle_rust_arg} python3 tools/release/check_session_assets.py "app/build/outputs/apk/${build_variant}/app-${build_variant}.apk" diff --git a/tools/directaudio/fetch.sh b/tools/directaudio/fetch.sh index b7ce2dbcb..af886304f 100755 --- a/tools/directaudio/fetch.sh +++ b/tools/directaudio/fetch.sh @@ -9,11 +9,18 @@ done work=$(mktemp -d) trap 'rm -rf "${work}"' EXIT +cache=${DROIDDECK_BUILD_CACHE:-"${HOME}/.cache/droiddeck-build"} +mkdir -p "${cache}" fetch() { - local name=$1 sha=$2 - curl -fsSL --retry 3 -o "${work}/${name}" \ - "https://github.com/${DIRECTAUDIO_REPO}/releases/download/${DIRECTAUDIO_TAG}/${name}" - echo "${sha} ${work}/${name}" | sha256sum -c - >/dev/null || { echo "${name}: checksum mismatch" >&2; exit 1; } + local name=$1 sha=$2 kept="${cache}/$2-$1" part + if ! echo "${sha} ${kept}" | sha256sum -c - >/dev/null 2>&1; then + part=$(mktemp "${kept}.part.XXXXXX") + curl -fsSL --retry 3 -o "${part}" \ + "https://github.com/${DIRECTAUDIO_REPO}/releases/download/${DIRECTAUDIO_TAG}/${name}" + echo "${sha} ${part}" | sha256sum -c - >/dev/null || { echo "${name}: checksum mismatch" >&2; rm -f "${part}"; exit 1; } + mv "${part}" "${kept}" + fi + cp "${kept}" "${work}/${name}" } fetch directaudio-linux-wine11.zip "${DIRECTAUDIO_WINE11_SHA256}" fetch directaudio-linux-wine11-systhread.zip "${DIRECTAUDIO_SYSTHREAD_SHA256}"