From d7bf0b274ff202edd3181dfa2d285cd667b8f2f4 Mon Sep 17 00:00:00 2001 From: XxJSONDeruloxX Date: Fri, 23 Oct 2026 14:31:13 -0400 Subject: [PATCH] ci: consolidate verified component preparation --- .github/workflows/authorship.yml | 2 +- .github/workflows/build.yml | 718 ++++++------------ tools/ci/resolve-ndk.sh | 16 + tools/ci/toolchains.env | 1 + tools/components/README.md | 34 +- tools/components/preloads.lock.json | 2 +- tools/components/preloads.py | 70 +- tools/components/prepare.py | 167 ++++ tools/components/recipes.json | 225 ++++++ tools/components/recipes/arm64-preloads.sh | 3 + tools/components/recipes/audio.sh | 14 + tools/components/recipes/comet.sh | 5 + tools/components/recipes/debian-image.sh | 18 + tools/components/recipes/directaudio.sh | 3 + tools/components/recipes/esync.sh | 23 + tools/components/recipes/proot.sh | 4 + tools/components/recipes/pulseaudio-source.sh | 5 + tools/components/recipes/runtime.sh | 83 ++ tools/components/recipes/x86-preloads.sh | 17 + tools/components/recipes/xbox.sh | 4 + tools/release/announce-preview.sh | 20 + tools/release/describe-build.sh | 22 + tools/release/publish-ci.sh | 39 + tools/release/report-build.sh | 77 ++ tools/release/runtime_inputs.py | 2 +- tools/tests/test_ci_checks.py | 6 + tools/tests/test_ci_ndk.py | 27 + tools/tests/test_component_prepare.py | 106 +++ tools/tests/test_esync_packs.py | 10 +- tools/tests/test_preload_snapshots.py | 31 + tools/tests/test_xbox_build_source.py | 18 +- 31 files changed, 1269 insertions(+), 503 deletions(-) create mode 100644 tools/ci/resolve-ndk.sh create mode 100644 tools/ci/toolchains.env create mode 100644 tools/components/prepare.py create mode 100644 tools/components/recipes.json create mode 100644 tools/components/recipes/arm64-preloads.sh create mode 100644 tools/components/recipes/audio.sh create mode 100644 tools/components/recipes/comet.sh create mode 100644 tools/components/recipes/debian-image.sh create mode 100644 tools/components/recipes/directaudio.sh create mode 100644 tools/components/recipes/esync.sh create mode 100644 tools/components/recipes/proot.sh create mode 100644 tools/components/recipes/pulseaudio-source.sh create mode 100644 tools/components/recipes/runtime.sh create mode 100644 tools/components/recipes/x86-preloads.sh create mode 100644 tools/components/recipes/xbox.sh create mode 100644 tools/release/announce-preview.sh create mode 100644 tools/release/describe-build.sh create mode 100644 tools/release/publish-ci.sh create mode 100644 tools/release/report-build.sh create mode 100644 tools/tests/test_ci_ndk.py create mode 100644 tools/tests/test_component_prepare.py diff --git a/.github/workflows/authorship.yml b/.github/workflows/authorship.yml index 97c6637a8..ebbcae644 100644 --- a/.github/workflows/authorship.yml +++ b/.github/workflows/authorship.yml @@ -23,7 +23,7 @@ permissions: jobs: validate: - name: Build APK + name: Validate source and build result runs-on: ubuntu-24.04 timeout-minutes: 5 steps: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5ea175cbf..265b1faee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,17 +31,14 @@ concurrency: group: build-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} -env: - PRELOAD_CACHE_VERSION: v8 - jobs: - xbox: - name: Xbox ARM64 backend + source: + name: Pin source tree if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} - runs-on: ubuntu-24.04-arm - timeout-minutes: 30 + runs-on: ubuntu-24.04 + timeout-minutes: 5 outputs: - base-sha: ${{ steps.merge.outputs.base-sha }} + base-sha: ${{ steps.merge.outputs.base-sha || steps.trusted.outputs.base-sha }} tree-sha: ${{ steps.source.outputs.tree-sha }} steps: - uses: actions/checkout@v4 @@ -59,9 +56,46 @@ jobs: git config user.name "ci" git config user.email "ci@users.noreply.github.com" git merge --no-ff --no-edit origin/main - - name: Record backend source tree + - name: Resolve trusted manifest base + id: trusted + if: github.event_name != 'pull_request' + run: | + set -euo pipefail + git fetch --no-tags origin main + echo "base-sha=$(git rev-parse origin/main)" >> "$GITHUB_OUTPUT" + - name: Record pinned source tree id: source run: echo "tree-sha=$(git rev-parse 'HEAD^{tree}')" >> "$GITHUB_OUTPUT" + + xbox: + name: Xbox ARM64 backend + needs: source + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Merge the pinned APK base + if: github.event_name == 'pull_request' + env: + BUILD_BASE_SHA: ${{ needs.source.outputs.base-sha }} + run: | + set -euo pipefail + git fetch --no-tags origin "$BUILD_BASE_SHA" + git config user.name "ci" + git config user.email "ci@users.noreply.github.com" + git merge --no-ff --no-edit FETCH_HEAD + - name: Verify pinned source tree + env: + SOURCE_TREE: ${{ needs.source.outputs.tree-sha }} + run: test "$(git rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" + - name: Plan component identities + id: plan + run: python3 tools/components/prepare.py plan - uses: actions/cache@v4 id: xbox-cache with: @@ -69,13 +103,10 @@ jobs: app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox app/build/xbox/xodus-droiddeck-source.tar.xz app/build/xbox/runtime/*-source.tar.xz - key: xbox-arm64-${{ hashFiles('tools/xbox/**') }} + key: xbox-v2-${{ steps.plan.outputs.xbox }} - name: Build Xbox backend if: steps.xbox-cache.outputs.cache-hit != 'true' - run: | - rustup toolchain install 1.99.0 --profile minimal - bash tools/ci/install-ubuntu-packages.sh build-essential pkg-config libssl-dev protobuf-compiler cmake xz-utils - bash tools/xbox/build.sh + run: python3 tools/components/prepare.py xbox - uses: actions/upload-artifact@v4 with: name: xbox-arm64 @@ -90,8 +121,157 @@ jobs: if-no-files-found: error retention-days: 14 + components: + name: Prepare verified components + needs: source + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Merge the pinned APK base + if: github.event_name == 'pull_request' + env: + BUILD_BASE_SHA: ${{ needs.source.outputs.base-sha }} + run: | + set -euo pipefail + git fetch --no-tags origin "$BUILD_BASE_SHA" + git config user.name "ci" + git config user.email "ci@users.noreply.github.com" + git merge --no-ff --no-edit FETCH_HEAD + - name: Verify pinned source tree + env: + SOURCE_TREE: ${{ needs.source.outputs.tree-sha }} + run: test "$(git rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" + - name: Resolve the pinned Android NDK + run: bash tools/ci/resolve-ndk.sh + + - name: Plan component identities + id: plan + run: python3 tools/components/prepare.py plan + - name: Cache remaining Linux runtime assets + id: preloads + uses: actions/cache@v4 + with: + path: | + app/src/main/assets/linuxfs + !app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet + !app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox + !app/src/main/assets/linuxfs/i386/libblsession.so + !app/src/main/assets/linuxfs/i386/libfakeinput.so + !app/src/main/assets/linuxfs/i386/libfaultreport.so + !app/src/main/assets/linuxfs/libblaudit.so + !app/src/main/assets/linuxfs/libblfastpath.so + !app/src/main/assets/linuxfs/libblsession.so + !app/src/main/assets/linuxfs/libfakeinput.so + !app/src/main/assets/linuxfs/libssbs.so + !app/src/main/assets/linuxfs/usr/local/bin/droiddeck-clipboard + !app/src/main/assets/linuxfs/x86_64/libblsession.so + !app/src/main/assets/linuxfs/x86_64/libfakeinput.so + !app/src/main/assets/linuxfs/x86_64/libfaultreport.so + !app/src/main/assets/linuxfs/x86_64/libvulkan-thunk.so + !app/src/main/assets/linuxfs/x86_64/libwebkitinput.so + key: runtime-v9-${{ steps.plan.outputs.runtime }} + + - name: Cache PulseAudio 13.0 source + id: pa-src + uses: actions/cache@v4 + with: + path: pa13 + key: pulseaudio-src-${{ steps.plan.outputs.pulseaudio-source }} + - name: Cache proot + id: proot + uses: actions/cache@v4 + with: + path: | + app/src/main/jniLibs/arm64-v8a/libproot.so + app/src/main/jniLibs/arm64-v8a/libproot-loader.so + key: proot-v2-${{ steps.plan.outputs.proot }} + - name: Cache GOG Galaxy companion + id: comet + uses: actions/cache@v4 + with: + path: app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet + key: comet-v2-${{ steps.plan.outputs.comet }} + + - name: Restore verified preload snapshots + id: component-preloads + env: + GH_TOKEN: ${{ github.token }} + FORCE_PRELOAD_REBUILD: ${{ inputs.rebuild_preloads || 'false' }} + COMPONENT_TRUSTED_BASE: ${{ needs.source.outputs.base-sha }} + run: python3 tools/components/prepare.py preload-snapshot + + - name: Prepare required Linux build tools once + env: + COMPONENTS_TO_BUILD: >- + ${{ steps.preloads.outputs.cache-hit != 'true' && 'runtime' || '' }} + ${{ steps.component-preloads.outputs.arm64 != 'true' && 'arm64-preloads' || '' }} + ${{ steps.comet.outputs.cache-hit != 'true' && 'comet' || '' }} + run: python3 tools/components/prepare.py tools + + - name: Build changed ARM64 preloads + if: steps.component-preloads.outputs.arm64 != 'true' + run: python3 tools/components/prepare.py arm64-preloads + + - name: Prepare remaining Linux runtime assets + if: steps.preloads.outputs.cache-hit != 'true' + run: python3 tools/components/prepare.py runtime + + - name: Fetch the Debian image the preloads build in + if: steps.component-preloads.outputs.x86 != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: python3 tools/components/prepare.py debian-image + + - name: Build the x86 preloads FEX guests load (glibc 2.31) + if: steps.component-preloads.outputs.x86 != 'true' + run: python3 tools/components/prepare.py x86-preloads + + - name: Bundle the droiddeck-esync packs + run: python3 tools/components/prepare.py esync + + - name: Fetch PulseAudio 13.0 source + if: steps.pa-src.outputs.cache-hit != 'true' + run: python3 tools/components/prepare.py pulseaudio-source + + - name: Fetch DirectAudio (tools/directaudio/release.env) + run: python3 tools/components/prepare.py directaudio + + - name: Build the AAudio sink into the audio bundle + run: python3 tools/components/prepare.py audio + + - name: Build proot + if: steps.proot.outputs.cache-hit != 'true' + run: python3 tools/components/prepare.py proot + + - name: Check the preloads + run: | + set -euo pipefail + bash tools/linuxfs/check-preloads.sh app/src/main/assets/linuxfs + + - name: Build GOG Galaxy companion + if: steps.comet.outputs.cache-hit != 'true' + run: python3 tools/components/prepare.py comet + + - name: Prepare graphics driver variants + run: python3 tools/quest/build-driver.py "$ANDROID_HOME/ndk/$NDK_VERSION" + - name: Record and pack prepared components + run: python3 tools/components/prepare.py pack + - uses: actions/upload-artifact@v4 + with: + name: prepared-components + path: app/build/prepared-components.tar.gz + if-no-files-found: error + compression-level: 0 + retention-days: 14 + build: - needs: xbox + needs: [source, components, xbox] name: Compile APK and Android tests if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} runs-on: ubuntu-24.04 @@ -155,7 +335,7 @@ jobs: - name: Merge onto main, or say what conflicts if: github.event_name == 'pull_request' env: - BUILD_BASE_SHA: ${{ needs.xbox.outputs.base-sha }} + BUILD_BASE_SHA: ${{ needs.source.outputs.base-sha }} run: | set -uo pipefail git config user.name "ci" @@ -213,7 +393,7 @@ jobs: - name: Verify backend and APK source match env: - XBOX_SOURCE_TREE: ${{ needs.xbox.outputs.tree-sha }} + XBOX_SOURCE_TREE: ${{ needs.source.outputs.tree-sha }} run: test "$(git rev-parse 'HEAD^{tree}')" = "$XBOX_SOURCE_TREE" - uses: actions/setup-java@v4 @@ -225,11 +405,8 @@ jobs: with: cache-cleanup: on-success - - name: Resolve the runner's NDK - run: | - NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}" - VERSION=$(sed -n 's/^Pkg.Revision *= *//p' "$NDK/source.properties") - echo "NDK_VERSION=$VERSION" >> "$GITHUB_ENV" + - name: Resolve the pinned Android NDK + run: bash tools/ci/resolve-ndk.sh - name: Set the APK build label id: build-label @@ -255,274 +432,6 @@ jobs: echo "DROIDDECK_PR_NUMBER=${PR_NUMBER:-0}" >> "$GITHUB_ENV" echo "Build: \`$label\`" >> "$GITHUB_STEP_SUMMARY" - - name: Cache the preloads - id: preloads - uses: actions/cache@v4 - with: - path: | - app/src/main/assets/linuxfs - !app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet - !app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox - !app/src/main/assets/linuxfs/i386/libblsession.so - !app/src/main/assets/linuxfs/i386/libfakeinput.so - !app/src/main/assets/linuxfs/i386/libfaultreport.so - !app/src/main/assets/linuxfs/libblaudit.so - !app/src/main/assets/linuxfs/libblfastpath.so - !app/src/main/assets/linuxfs/libblsession.so - !app/src/main/assets/linuxfs/libfakeinput.so - !app/src/main/assets/linuxfs/libssbs.so - !app/src/main/assets/linuxfs/usr/local/bin/droiddeck-clipboard - !app/src/main/assets/linuxfs/x86_64/libblsession.so - !app/src/main/assets/linuxfs/x86_64/libfakeinput.so - !app/src/main/assets/linuxfs/x86_64/libfaultreport.so - !app/src/main/assets/linuxfs/x86_64/libvulkan-thunk.so - !app/src/main/assets/linuxfs/x86_64/libwebkitinput.so - key: preloads-${{ env.PRELOAD_CACHE_VERSION }}-${{ hashFiles('app/src/main/cpp/adrenotools/src/hook/kgsl.h', 'tools/quest/*', 'tools/linuxfs/overlay/usr/local/bin/*', 'tools/linuxfs/overlay/usr/bin/**', 'tools/linuxfs/desktop/*', 'tools/gamescope/release.env', 'tools/wlroots/release.env', 'tools/linuxfs/uruntime.env', 'tools/linuxfs/licenses/*', 'tools/mangoapp/*', 'tools/msitools/**', 'tools/pipewire/**', 'tools/ci/install-ubuntu-packages.sh', 'tools/vr/**') }} - - - name: Restore verified preload snapshots - id: component-preloads - env: - GH_TOKEN: ${{ github.token }} - FORCE_PRELOAD_REBUILD: ${{ inputs.rebuild_preloads || 'false' }} - COMPONENT_TRUSTED_BASE: ${{ needs.xbox.outputs.base-sha || github.sha }} - run: python3 tools/components/preloads.py restore - - - name: Build changed ARM64 preloads - if: steps.component-preloads.outputs.arm64 != 'true' - run: | - bash tools/ci/install-ubuntu-packages.sh gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - bash tools/components/build-arm64-preloads.sh app/src/main/assets/linuxfs - - - name: Prepare remaining Linux runtime assets - if: steps.preloads.outputs.cache-hit != 'true' - run: | - set -euo pipefail - bash tools/ci/install-ubuntu-packages.sh gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libvulkan1 mesa-vulkan-drivers - D=app/src/main/assets/linuxfs - mkdir -p "$D" - VK_INCLUDE=$(ls -d "${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1) \ - tools/vr/build.sh --native "$D/usr/local/lib/droiddeck-vr" - for script in tools/linuxfs/overlay/usr/local/bin/droiddeck-* tools/linuxfs/overlay/usr/local/bin/steam-compatibility; do - install -Dm644 "$script" "$D/usr/local/bin/$(basename "$script")" - done - for f in tools/linuxfs/overlay/usr/bin/* tools/linuxfs/overlay/usr/bin/steamos-polkit-helpers/*; do - [ -f "$f" ] && install -Dm644 "$f" "$D/${f#tools/linuxfs/overlay/}" - done - if [ -f tools/gamescope/release.env ]; then - . tools/gamescope/release.env - curl -fsSL --retry 3 -o gamescope.tzst "https://github.com/${GAMESCOPE_REPO:-${GITHUB_REPOSITORY}}/releases/download/${GAMESCOPE_TAG}/gamescope.tzst" - echo "${GAMESCOPE_SHA256} gamescope.tzst" | sha256sum -c - - tar --use-compress-program=unzstd -xf gamescope.tzst -C "$D" - test -f "$D/usr/local/bin/gamescope" - rm gamescope.tzst - fi - . tools/linuxfs/uruntime.env - curl -fsSL --retry 3 -o uruntime "https://github.com/VHSgunzo/uruntime/releases/download/${URUNTIME_VERSION}/${URUNTIME_ASSET}" - echo "${URUNTIME_SHA256} uruntime" | sha256sum -c - - install -Dm644 uruntime "$D/usr/local/lib/droiddeck/uruntime" - install -Dm644 tools/linuxfs/licenses/uruntime-LICENSE "$D/usr/local/share/licenses/uruntime/LICENSE" - rm uruntime - M="$D/usr/local/lib/mangoapp" - mkdir -p "$M" mango-pkgs - grep -v '^#' tools/mangoapp/packages.txt | while read -r sum url; do - [ -n "$url" ] || continue - curl -fsSL --retry 3 -o mango-pkgs/pkg.tar.zst "$url" - echo "$sum mango-pkgs/pkg.tar.zst" | sha256sum -c - - tar --use-compress-program=unzstd -xf mango-pkgs/pkg.tar.zst -C mango-pkgs - rm mango-pkgs/pkg.tar.zst - done - install -m644 mango-pkgs/usr/bin/mangoapp "$M/mangoapp" - for so in libfmt.so.10 libspdlog.so.1.13 libglfw.so.3 libtraceevent.so.1; do - cp -L "mango-pkgs/usr/lib/$so" "$M/$so" - done - aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -Wl,-soname,libtracefs.so.1 \ - -o "$M/libtracefs.so.1" tools/mangoapp/libtracefs-shim.c - aarch64-linux-gnu-strip --strip-unneeded "$M/libtracefs.so.1" - install -Dm644 tools/mangoapp/mangoapp "$D/usr/local/bin/mangoapp" - rm -rf mango-pkgs - ls -l "$M" - mkdir -p pw-pkgs - grep -v '^#' tools/pipewire/packages.txt | while read -r sum url; do - [ -n "$url" ] || continue - curl -fsSL --retry 3 -o pw-pkgs/pkg.tar.zst "$url" - echo "$sum pw-pkgs/pkg.tar.zst" | sha256sum -c - - tar --use-compress-program=unzstd -xf pw-pkgs/pkg.tar.zst -C pw-pkgs - rm pw-pkgs/pkg.tar.zst - done - tools/pipewire/assemble.sh pw-pkgs "$D/usr/local/lib/droiddeck-pipewire.tzst" - rm -rf pw-pkgs - ls -l "$D/usr/local/lib/droiddeck-pipewire.tzst" - T="$D/usr/local/lib/droiddeck-msitools" - mkdir -p "$T" msi-pkgs - grep -v '^#' tools/msitools/packages.txt | while read -r sum url; do - [ -n "$url" ] || continue - curl -fsSL --retry 3 -o msi-pkgs/pkg.tar.xz "$url" - echo "$sum msi-pkgs/pkg.tar.xz" | sha256sum -c - - tar -xJf msi-pkgs/pkg.tar.xz -C msi-pkgs - rm msi-pkgs/pkg.tar.xz - done - install -m644 msi-pkgs/usr/bin/msiinfo "$T/msiinfo" - install -m644 msi-pkgs/usr/bin/cabextract "$T/cabextract" - install -m644 msi-pkgs/usr/lib/7zip/7z "$T/7z" - install -m644 msi-pkgs/usr/lib/7zip/7z.so "$T/7z.so" - for so in libmsi-1.0.so.0 libgsf-1.so.114 libgcab-1.0.so.0; do - cp -L "msi-pkgs/usr/lib/$so" "$T/$so" - done - for f in NOTICE GPL-2 GPL-3 LGPL-2.1 7zip-License 7zip-unRarLicense; do - install -Dm644 "tools/msitools/licenses/$f" "$D/usr/local/share/licenses/droiddeck-msitools/$f" - done - rm -rf msi-pkgs - ls -l "$T" - install -Dm644 tools/linuxfs/desktop/droiddeck-desktop "$D/usr/local/bin/droiddeck-desktop" - install -Dm644 tools/linuxfs/desktop/droiddeck-gpu "$D/usr/local/bin/droiddeck-gpu" - install -Dm644 tools/linuxfs/desktop/droiddeck-desktop-gpu "$D/usr/local/bin/droiddeck-desktop-gpu" - install -Dm644 tools/linuxfs/desktop/kwin_wayland_wrapper "$D/usr/local/bin/kwin_wayland_wrapper" - install -Dm644 tools/linuxfs/desktop/firefox-droiddeck.js "$D/usr/lib/firefox/defaults/pref/droiddeck.js" - ls -lR "$D" - - - name: Fetch the Debian image the preloads build in - if: steps.component-preloads.outputs.x86 != 'true' - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - . tools/linuxfs/debian-image.env - image=public.ecr.aws/docker/library/debian:bullseye - if gh release download "$DEBIAN_IMAGE_TAG" -R "$GITHUB_REPOSITORY" -p "$DEBIAN_IMAGE_ASSET" -O /tmp/debian-image.tar.zst \ - && { [ -z "$DEBIAN_IMAGE_SHA256" ] || echo "$DEBIAN_IMAGE_SHA256 /tmp/debian-image.tar.zst" | sha256sum -c -; }; then - zstd -dc /tmp/debian-image.tar.zst | docker load - rm -f /tmp/debian-image.tar.zst - else - echo "::warning::the hosted Debian image could not be fetched; pulling from the registry instead" - for attempt in 1 2 3 4 5; do - docker pull "$image" && break - [ "$attempt" -lt 5 ] || { echo "::error::could not pull $image after $attempt attempts"; exit 1; } - echo "pull attempt $attempt failed (registry busy?); retrying in $((attempt * 30)) s" - sleep $((attempt * 30)) - done - fi - docker image ls "$image" - - - name: Build the x86 preloads FEX guests load (glibc 2.31) - if: steps.component-preloads.outputs.x86 != 'true' - run: | - set -euo pipefail - docker run --rm -v "$PWD:/src" -w /src public.ecr.aws/docker/library/debian:bullseye bash -c ' - set -euo pipefail - install_compilers() { - apt-get -o Acquire::Check-Valid-Until=false update -qq && - DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends gcc g++ gcc-multilib g++-multilib binutils >/dev/null - } - printf "deb http://deb.debian.org/debian bullseye main\ndeb http://security.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list - if ! install_compilers; then - printf "deb http://archive.debian.org/debian bullseye main\ndeb http://archive.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list - install_compilers - fi - tools/linuxfs/build-x86-preloads.sh app/src/main/assets/linuxfs - ' - sudo chown -R "$(id -u):$(id -g)" app/src/main/assets/linuxfs - ls -l app/src/main/assets/linuxfs/x86_64 app/src/main/assets/linuxfs/i386 - - - name: Bundle the droiddeck-esync packs - run: | - set -euo pipefail - if [ -f tools/droiddeck-esync/release.env ]; then - . tools/droiddeck-esync/release.env - release_url="https://github.com/${SYNC_BUNDLE_REPO}/releases/download/${SYNC_BUNDLE_TAG}" - curl -fsSL --retry 3 -o sync-bundle.tzst "$release_url/$SYNC_BUNDLE_ASSET" - echo "$SYNC_BUNDLE_SHA256 sync-bundle.tzst" | sha256sum -c - - mkdir -p app/src/main/assets/droiddeck-esync - tar --use-compress-program=unzstd -xf sync-bundle.tzst -C app/src/main/assets/droiddeck-esync - rm sync-bundle.tzst - curl -fsSL --retry 3 -o sync-index.json "$release_url/index.json" - revoked=$(python3 -c 'import json, sys; print(" ".join(p["id"] for p in json.load(open(sys.argv[1]))["packs"] if p.get("revoked") is True))' sync-index.json) - for id in $revoked; do - if [ -e "app/src/main/assets/droiddeck-esync/packs/$id.tzst" ]; then - echo "::warning::$SYNC_BUNDLE_ASSET carries revoked pack $id; it is left out of the APK (point tools/droiddeck-esync/release.env at a newer bundle)" - rm -f "app/src/main/assets/droiddeck-esync/packs/$id.tzst" - fi - done - rm sync-index.json - test -f app/src/main/assets/droiddeck-esync/index.json - test -f app/src/main/assets/droiddeck-esync/index.json.sig - ls -lR app/src/main/assets/droiddeck-esync - fi - - - name: Cache PulseAudio 13.0 source - id: pa-src - uses: actions/cache@v4 - with: - path: pa13 - key: pulseaudio-src-13.0 - - name: Fetch PulseAudio 13.0 source - if: steps.pa-src.outputs.cache-hit != 'true' - run: | - set -euo pipefail - curl -fsSL -o pa.tar.gz https://github.com/pulseaudio/pulseaudio/archive/refs/tags/v13.0.tar.gz - mkdir -p pa13 && tar -xzf pa.tar.gz -C pa13 --strip-components=1 && rm pa.tar.gz - - name: Fetch DirectAudio (tools/directaudio/release.env) - run: tools/directaudio/fetch.sh "$PWD" "$PWD/sink-out" - - name: Build the AAudio sink into the audio bundle - run: | - set -euo pipefail - export NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}" - tools/aaudio-sink/build.sh "$PWD/pa13" "$PWD/sink-out" - rm -rf pa-bundle && mkdir -p pa-bundle - tar --use-compress-program=unzstd -xf app/src/main/assets/pulseaudio.tzst -C pa-bundle - test ! -f pa-bundle/modules/arm64/module-aaudio-sink.so - install -m755 sink-out/module-aaudio-sink.so pa-bundle/modules/arm64/module-aaudio-sink.so - install -m755 sink-out/module-directaudio-native-sink.so pa-bundle/modules/arm64/module-directaudio-native-sink.so - (cd pa-bundle && tar --use-compress-program='zstd -19' -cf ../pulseaudio.tzst .) - mkdir -p app/build/prepared-assets - mv pulseaudio.tzst app/build/prepared-assets/pulseaudio.tzst - tar --use-compress-program=unzstd -tf app/build/prepared-assets/pulseaudio.tzst | grep -c '\.so$' - rm -rf pa-bundle sink-out - - name: Cache proot - id: proot - uses: actions/cache@v4 - with: - path: | - app/src/main/jniLibs/arm64-v8a/libproot.so - app/src/main/jniLibs/arm64-v8a/libproot-loader.so - key: proot-${{ env.NDK_VERSION }}-${{ hashFiles('tools/proot/**') }} - - name: Build proot - if: steps.proot.outputs.cache-hit != 'true' - run: | - set -euo pipefail - export NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}" - tools/proot/build.sh app/src/main/jniLibs/arm64-v8a - - - name: Check the preloads - run: | - set -euo pipefail - bash tools/linuxfs/check-preloads.sh app/src/main/assets/linuxfs - - - name: Check Quest tracking IPC - run: | - set -euo pipefail - NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}" - VK_INCLUDE=$(ls -d "$NDK"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1) - mkdir -p build/vr/include - cp -R "$VK_INCLUDE/vulkan" build/vr/include/ - if [ -d "$VK_INCLUDE/vk_video" ]; then cp -R "$VK_INCLUDE/vk_video" build/vr/include/; fi - g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_tracking.cpp -o /tmp/test-xr-tracking - /tmp/test-xr-tracking - g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_xr_touch.cpp -o /tmp/test-xr-touch - /tmp/test-xr-touch - g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_pad_controller_config.cpp -o /tmp/test-pad-controller-config - /tmp/test-pad-controller-config - g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_vsync.cpp -o /tmp/test-xr-vsync - /tmp/test-xr-vsync - g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_frames.cpp -o /tmp/test-xr-frames - /tmp/test-xr-frames - cc -Itools/vr/third_party/wayland -c tools/vr/src/wayland_client_stub.c -o /tmp/test-wayland-stub.o - cc -Itools/vr/third_party/wayland -c app/src/main/cpp/waylandcomp/generated/xdg-shell-protocol.c -o /tmp/test-xdg-shell.o - g++ -std=c++17 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-missing-field-initializers -pthread -Ibuild/vr/include \ - -Itools/vr/third_party/wayland -Iapp/src/main/cpp/waylandcomp/generated \ - tools/vr/tests/test_presenter_shutdown.cpp tools/vr/src/presenter.cpp /tmp/test-wayland-stub.o /tmp/test-xdg-shell.o \ - -ldl -o /tmp/test-presenter-shutdown - timeout 10 /tmp/test-presenter-shutdown - - name: Rust toolchain for the store engines (aarch64-linux-android) id: rust run: | @@ -561,28 +470,20 @@ jobs: restore-keys: | cxx-${{ env.NDK_VERSION }}- + - uses: actions/download-artifact@v4 + with: + name: prepared-components + path: app/build + - name: Verify and unpack prepared components + env: + SOURCE_TREE: ${{ needs.source.outputs.tree-sha }} + run: python3 tools/components/prepare.py unpack --tree "$SOURCE_TREE" + - uses: actions/download-artifact@v4 with: name: xbox-arm64 path: app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox - - name: Cache GOG Galaxy companion - id: comet - uses: actions/cache@v4 - with: - path: app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet - key: comet-${{ runner.os }}-${{ runner.arch }}-rust1.90.0-${{ hashFiles('tools/comet/**') }} - - - name: Build GOG Galaxy companion - if: steps.comet.outputs.cache-hit != 'true' - run: | - if ! command -v aarch64-linux-gnu-gcc >/dev/null; then - bash tools/ci/install-ubuntu-packages.sh gcc-aarch64-linux-gnu - fi - rustup toolchain install 1.90.0 --profile minimal - rustup target add --toolchain 1.90.0 aarch64-unknown-linux-gnu - RUSTUP_TOOLCHAIN=1.90.0 bash tools/comet/build.sh - - name: Build the add-on packs run: bash tools/packs/build.sh app/build/packs @@ -593,7 +494,6 @@ jobs: SGDB_API_KEY: ${{ secrets.SGDB_API_KEY }} run: | set -o pipefail - python3 tools/quest/build-driver.py "$ANDROID_HOME/ndk/$NDK_VERSION" python3 tools/release/runtime_inputs.py record ./gradlew assembleRelease --console=plain -PndkVersion="$NDK_VERSION" 2>&1 | tee /tmp/build.log python3 tools/release/check_session_assets.py app/build/outputs/apk/release/app-release.apk @@ -701,87 +601,11 @@ jobs: TEST_DIAGNOSTICS_URL: ${{ steps.unit-test-diagnostics.outputs.artifact-url }} APK_OUTCOME: ${{ steps.apk-upload.outcome }} SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }} - run: | - set -uo pipefail - MARK="" - { - echo "$MARK" - echo "## Build of this pull request" - echo - [ -f /tmp/pr-merge.md ] && cat /tmp/pr-merge.md - echo - case "$BUILD_OUTCOME:$UNIT_TEST_OUTCOME:$APK_OUTCOME" in - success:success:success) - echo "### Build passed" - echo - echo "The \`droiddeck-apk\` artifact is [on this run]($RUN_URL), signed with the public test key." - echo - if [ "$SAME_REPO" = true ]; then - echo "After the parallel regression tests pass, a release-signed tester build that installs over Stable or Preview will be published and will appear in DroidDeck's **Test builds** list. Label the PR \`no-test-build\` to skip that." - else - echo "**Maintainer:** add the \`publish-test\` label to publish a release-signed tester build that installs over Stable or Preview and appears in DroidDeck's **Test builds** list. The label is removed automatically after publishing." - fi - echo - echo "Fallback: [open **PR Release Publish**]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/workflows/pr-release-publish.yml) on \`main\` and enter PR \`$PR\`." - ;; - success:success:*) - echo "### APK publication incomplete" - echo - echo "Compilation and unit tests passed, but APK publication did not complete. See [the run]($RUN_URL)." - ;; - success:failure:*) - echo "### ERROR: Unit tests failed" - echo - echo "The APK compiled, but unit tests failed. No installable artifact was published." - echo - if [ -n "$TEST_DIAGNOSTICS_URL" ]; then - echo "The [unit-test-results]($TEST_DIAGNOSTICS_URL) artifact contains the test log and any generated reports." - else - echo "The test diagnostics upload did not complete. See [the test log]($RUN_URL)." - fi - ;; - success:*) - echo "### Build incomplete" - echo - echo "The APK compiled, but unit tests did not complete successfully. See [the run]($RUN_URL)." - ;; - failure:*) - echo "### ERROR: Build failed" - echo - echo "The compiler's errors ([full log]($RUN_URL)):" - echo - echo '```' - { grep -E '^e: |^w: .*error|error:|CMake Error|fatal error|ninja: error|FAILED|What went wrong|Execution failed for task|Unresolved reference|Type mismatch|Unresolved' /tmp/build.log 2>/dev/null \ - | sed -e 's#file:///home/runner/work/[^/]*/[^/]*/##' -e 's#/home/runner/work/[^/]*/[^/]*/##' \ - | grep -v '^w: ' | awk '!seen[$0]++' | head -80; } || true - echo '```' - echo - echo "
What went wrong (Gradle)" - echo - echo '```' - awk '/^\* What went wrong/{p=1} /^\* Try:/{p=0} p{print}' /tmp/build.log 2>/dev/null | head -60 - echo '```' - echo "
" - ;; - skipped:*|cancelled:*) - echo "### Not built" - echo - echo "The build did not run (see above, or [the run]($RUN_URL))." - ;; - esac - echo - } > /tmp/pr-report.md - cat /tmp/pr-report.md >> "$GITHUB_STEP_SUMMARY" - ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate --jq ".[] | select(.body | startswith(\"$MARK\")) | .id" | head -1 || true) - if [ -n "$ID" ]; then - gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${ID}" -F body=@/tmp/pr-report.md >/dev/null - else - gh api -X POST "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -F body=@/tmp/pr-report.md >/dev/null - fi + run: bash tools/release/report-build.sh python-tests: name: Python and native regression tests - needs: xbox + needs: source if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} runs-on: ubuntu-24.04 timeout-minutes: 15 @@ -794,7 +618,7 @@ jobs: - name: Merge the pinned APK base if: github.event_name == 'pull_request' env: - BUILD_BASE_SHA: ${{ needs.xbox.outputs.base-sha }} + BUILD_BASE_SHA: ${{ needs.source.outputs.base-sha }} run: | set -euo pipefail git fetch --no-tags origin "$BUILD_BASE_SHA" @@ -803,11 +627,40 @@ jobs: git merge --no-ff --no-edit FETCH_HEAD - name: Verify backend and test source match env: - XBOX_SOURCE_TREE: ${{ needs.xbox.outputs.tree-sha }} + XBOX_SOURCE_TREE: ${{ needs.source.outputs.tree-sha }} run: test "$(git rev-parse 'HEAD^{tree}')" = "$XBOX_SOURCE_TREE" - name: Install Python test dependencies timeout-minutes: 7 run: bash tools/ci/install-ubuntu-packages.sh python3-yaml gcc-aarch64-linux-gnu libc6-dev-arm64-cross qemu-user libvulkan-dev mesa-vulkan-drivers + - name: Resolve the pinned Android NDK + run: bash tools/ci/resolve-ndk.sh + + - name: Check Quest tracking IPC + run: | + set -euo pipefail + NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}" + VK_INCLUDE=$(ls -d "$NDK"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1) + mkdir -p build/vr/include + cp -R "$VK_INCLUDE/vulkan" build/vr/include/ + if [ -d "$VK_INCLUDE/vk_video" ]; then cp -R "$VK_INCLUDE/vk_video" build/vr/include/; fi + g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_tracking.cpp -o /tmp/test-xr-tracking + /tmp/test-xr-tracking + g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_xr_touch.cpp -o /tmp/test-xr-touch + /tmp/test-xr-touch + g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_pad_controller_config.cpp -o /tmp/test-pad-controller-config + /tmp/test-pad-controller-config + g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_vsync.cpp -o /tmp/test-xr-vsync + /tmp/test-xr-vsync + g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_frames.cpp -o /tmp/test-xr-frames + /tmp/test-xr-frames + cc -Itools/vr/third_party/wayland -c tools/vr/src/wayland_client_stub.c -o /tmp/test-wayland-stub.o + cc -Itools/vr/third_party/wayland -c app/src/main/cpp/waylandcomp/generated/xdg-shell-protocol.c -o /tmp/test-xdg-shell.o + g++ -std=c++17 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-missing-field-initializers -pthread -Ibuild/vr/include \ + -Itools/vr/third_party/wayland -Iapp/src/main/cpp/waylandcomp/generated \ + tools/vr/tests/test_presenter_shutdown.cpp tools/vr/src/presenter.cpp /tmp/test-wayland-stub.o /tmp/test-xdg-shell.o \ + -ldl -o /tmp/test-presenter-shutdown + timeout 10 /tmp/test-presenter-shutdown + - name: Steam import and game environment tests run: python3 -m unittest discover -s tools/tests -p 'test_*.py' @@ -857,6 +710,10 @@ jobs: CI_REPO: Droid-Deck/DroidDeck-CI KEEP: 10 steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + persist-credentials: false - uses: actions/download-artifact@v4 with: pattern: droiddeck-signed-* @@ -874,72 +731,13 @@ jobs: - name: Describe the change env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" \ - --jq '[.[] | select(.merged_at != null and .base.ref == "main")] | first // empty') - if [ -n "$pr" ]; then - number=$(jq -r .number <<<"$pr") - title=$(jq -r .title <<<"$pr") - merged=$(jq -r '.merged_at[:10]' <<<"$pr") - summary=$(jq -r '.body // ""' <<<"$pr" | tr -d '\r' | { grep -v '^🤖' || true; } | - awk '!p && /^#/ { next } NF { p = 1; print; next } p { exit }' | head -n 6 | cut -c1-400) - { - echo "**Merged to main** - [PR #$number]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/$number): $title (merged $merged)" - if [ -n "$summary" ]; then - echo - echo "$summary" - fi - } > notes-head.md - else - subject=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA" --jq '.commit.message | split("\n")[0]') - echo "**Pushed to main** without a PR - $subject" > notes-head.md - fi - cat notes-head.md + run: bash tools/release/describe-build.sh - name: Publish to DroidDeck-CI id: publish env: GH_TOKEN: ${{ steps.ci-token.outputs.token }} - run: | - set -euo pipefail - tag=main-${GITHUB_SHA::7} - cat signed/SHA256SUMS-*.txt > signed/SHA256SUMS.txt - rm signed/SHA256SUMS-*.txt - notes="$(cat notes-head.md) - - Signed build of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA - in main, not yet in a DroidDeck release; for testing. - Commit: $GITHUB_SHA - VersionCode: $(sed -n 's/^ *versionCode \([0-9][0-9]*\)$/\1/p' app/build.gradle) - Each apk installs over a DroidDeck release and keeps its data. \`standard\` is the normal app; the others are the same app under another package name, installed beside it. - - SHA-256: - \`\`\` - $(cat signed/SHA256SUMS.txt) - \`\`\`" - gh release delete "$tag" -R "$CI_REPO" --yes --cleanup-tag 2>/dev/null || true - gh release create "$tag" -R "$CI_REPO" --target main --prerelease \ - --title "main ${GITHUB_SHA::7}" --notes "$notes" signed/* - gh release list -R "$CI_REPO" -L 200 --json tagName,publishedAt \ - --jq '[.[] | select(.tagName | startswith("main-"))] | sort_by(.publishedAt) | reverse | .['"$KEEP"':][] | .tagName' | - while IFS= read -r old; do - if [ "$old" = "$tag" ]; then - continue - fi - if ! gh release delete "$old" -R "$CI_REPO" --yes --cleanup-tag 2>/dev/null; then - if gh release view "$old" -R "$CI_REPO" >/dev/null 2>&1; then - echo "::error::Could not delete old release $old" - exit 1 - fi - fi - done - published=$(gh release view "$tag" -R "$CI_REPO" --json tagName,assets) - jq -e --arg tag "$tag" '.tagName == $tag and (.assets | length) > 0' <<< "$published" >/dev/null - url="$GITHUB_SERVER_URL/$CI_REPO/releases/tag/$tag" - apk=$(cd signed && ls DroidDeck-*.apk | awk 'length(a) == 0 || length($0) < length(a) { a = $0 } END { print a }') - echo "release=$url" >> "$GITHUB_OUTPUT" - echo "apk=$GITHUB_SERVER_URL/$CI_REPO/releases/download/$tag/$apk" >> "$GITHUB_OUTPUT" - echo "Published $url" + run: bash tools/release/publish-ci.sh - name: Post the Preview build to Discord env: @@ -947,26 +745,8 @@ jobs: GH_TOKEN: ${{ github.token }} RELEASE: ${{ steps.publish.outputs.release }} APK: ${{ steps.publish.outputs.apk }} - run: | - set -euo pipefail - if [ -z "$WEBHOOK" ]; then - echo "::notice::No DISCORD_BUILDS_WEBHOOK secret; nothing posted" - exit 0 - fi - commit=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA") - subject=$(jq -r '.commit.message | split("\n")[0]' <<<"$commit") - title=$(jq -r '.commit.message | split("\n") | map(select(length > 0)) | .[1] // ""' <<<"$commit") - pr=$(sed -n 's/^Merge pull request #\([0-9]*\).*/\1/p' <<<"$subject") - if [ -n "$pr" ]; then - what="[#$pr]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/$pr): ${title:-$subject}" - else - what="[\`${GITHUB_SHA::7}\`]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA): $subject" - fi - get=$(printf 'Download: [standard APK](<%s>) · [all variants](<%s>)' "$APK" "$RELEASE") - content=$(printf '**New signed Preview build** `main-%s`\n%s\nIn the app: **Updates → Preview**\n%s' "${GITHUB_SHA::7}" "$what" "$get") - jq -n --arg content "$content" '{content: $content, allowed_mentions: {parse: []}}' | - curl -fsS -H 'Content-Type: application/json' -d @- "$WEBHOOK" -o /dev/null - echo "Posted: $content" + run: bash tools/release/announce-preview.sh + catalog: needs: announce diff --git a/tools/ci/resolve-ndk.sh b/tools/ci/resolve-ndk.sh new file mode 100644 index 000000000..61df33c50 --- /dev/null +++ b/tools/ci/resolve-ndk.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +repo=$(cd "$(dirname "$0")/../.." && pwd) +source "$repo/tools/ci/toolchains.env" +ndk="$ANDROID_HOME/ndk/$CI_ANDROID_NDK_VERSION" +if [ ! -f "$ndk/source.properties" ]; then + sdkmanager --sdk_root="$ANDROID_HOME" --install "ndk;$CI_ANDROID_NDK_VERSION" +fi +actual=$(sed -n 's/^Pkg.Revision *= *//p' "$ndk/source.properties" | tr -d '\r') +test "$actual" = "$CI_ANDROID_NDK_VERSION" +{ + echo "NDK_VERSION=$actual" + echo "ANDROID_NDK_HOME=$ndk" + echo "ANDROID_NDK_ROOT=$ndk" + echo "ANDROID_NDK_LATEST_HOME=$ndk" +} >> "$GITHUB_ENV" diff --git a/tools/ci/toolchains.env b/tools/ci/toolchains.env new file mode 100644 index 000000000..ac365f2bb --- /dev/null +++ b/tools/ci/toolchains.env @@ -0,0 +1 @@ +CI_ANDROID_NDK_VERSION=30.0.16248370 diff --git a/tools/components/README.md b/tools/components/README.md index 646c099cc..782d8b9f1 100644 --- a/tools/components/README.md +++ b/tools/components/README.md @@ -1,10 +1,40 @@ -# Verified preload snapshots +# Runtime component preparation + +`recipes.json` owns component source inputs, builders, outputs, package requirements +and pinned downloads. `prepare.py` runs those recipes, calculates cache identities, +installs the union of missing tools and verifies the prepared artifact handoff. +Builders live in `recipes/`; the workflow describes scheduling and gates. + +```mermaid +flowchart LR + Source[Pin source tree] --> Components[Prepare verified components] + Source --> Xbox[Xbox ARM64 backend] + Source --> Tests[Python and native tests] + Components --> APK[Compile APK and Android tests] + Xbox --> APK + APK --> Gate[Compile and test APK gate] + Tests --> Gate + Gate --> Publish[Main only: signing and publishing] +``` + +All workers verify the same source tree. The component artifact records that tree, +recipe identities and every output hash; the APK worker verifies the inventory +before building. This handoff verifies consistency within the same CI run. It is +not independent provenance for arbitrary artifacts. Source policy checks, tests, +APK content/ABI checks and signing/publication gates remain in place. + +`tools/ci/toolchains.env` pins the Android NDK shared by component, APK and native +test workers. Linux builds use Ubuntu 24.04. Recipe and input changes invalidate +the relevant component cache; ordinary caches remain performance aids. + +## Verified preload snapshots `preloads.lock.json` pins a small immutable preload archive. It records each binary's hash, its producing main CI run, the original APK artifact identity and digest, source/header hashes, the ARM64 build recipe and x86 compiler-image inputs. It covers core ARM64 preloads and the x86 preload libraries. VR, Android native -libraries, Comet and Xbox keep their existing preparation paths. +libraries, Comet and Xbox use their registered recipes and existing release pins +or component caches. Every build compares current source inputs, including transitive quoted headers, and build recipes with this manifest. Changed groups rebuild from source. A diff --git a/tools/components/preloads.lock.json b/tools/components/preloads.lock.json index 4093912e3..a92cd6dea 100644 --- a/tools/components/preloads.lock.json +++ b/tools/components/preloads.lock.json @@ -1,6 +1,6 @@ { "archive_sha256": "cb6487a94f25a3068fd07958174eb5b5e14f709655ff4ef0a8cdbd99138513b2", - "arm64_recipe": "62c8b9cf001f1b04b52856bd74d353842a4be29960f033067f34c7d3a9a48681", + "arm64_recipe": "1c92a1b0669277de3e78e9f292634affb5b5a19d5ea49d3ad8610c1dc638c2fb", "groups": { "arm64": { "@runner": "84f825635599f81ed33f62c5fc4c73f66044d1468b060d65f83855eb9c8bfe77", diff --git a/tools/components/preloads.py b/tools/components/preloads.py index 7da904080..671474040 100644 --- a/tools/components/preloads.py +++ b/tools/components/preloads.py @@ -1,6 +1,7 @@ """Reuse reviewed preload snapshots; changed snapshots must prove their main-build origin.""" import argparse import gzip +import http.client import hashlib import io import json @@ -10,6 +11,7 @@ import re from pathlib import Path import subprocess import tarfile +import textwrap import urllib.error import urllib.request from zipfile import ZipFile @@ -17,20 +19,12 @@ from zipfile import ZipFile ROOT = Path(__file__).resolve().parents[2] REPO = 'Droid-Deck/DroidDeck' LOCK = 'tools/components/preloads.lock.json' -COMMON = ['app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/cpp/adrenotools/src/hook/kgsl.h', - 'app/src/main/cpp/venc/venc_proto.h', 'tools/linuxfs/preload', - 'tools/quest/kgsl_compat.c', 'tools/quest/kgsl_compat.h'] -INPUTS = { - 'arm64': COMMON + ['tools/proot/fastpath', 'tools/linuxfs/ssbs', 'tools/linuxfs/clipboard'], - 'x86': COMMON + ['tools/linuxfs/fex', 'tools/linuxfs/build-x86-preloads.sh', 'tools/linuxfs/debian-image.env'], -} -OUTPUTS = { - 'arm64': ['libfakeinput.so', 'libblsession.so', 'libblfastpath.so', 'libblaudit.so', 'libssbs.so', - 'usr/local/bin/droiddeck-clipboard'], - 'x86': [f'{arch}/{name}.so' for arch in ('x86_64', 'i386') - for name in ('libfakeinput', 'libblsession', 'libfaultreport')] - + ['x86_64/libwebkitinput.so', 'x86_64/libvulkan-thunk.so'], -} +DEFINITIONS = json.loads((ROOT / 'tools/components/recipes.json').read_text()) +INPUTS = {group: DEFINITIONS[name]['source_inputs'] for group, name in + [('arm64', 'arm64-preloads'), ('x86', 'x86-preloads')]} +OUTPUTS = {group: DEFINITIONS[name]['snapshot_outputs'] for group, name in + [('arm64', 'arm64-preloads'), ('x86', 'x86-preloads')]} +COMMON = [name for name in INPUTS['arm64'] if name in INPUTS['x86']] FILES = set(sum(OUTPUTS.values(), [])) @@ -46,12 +40,13 @@ def api(path): return json.loads(run('gh', 'api', 'repos/' + REPO + '/' + path)) -def source_files(group, root=ROOT, commit=None): +def source_files(group, root=ROOT, commit=None, patterns=None): + patterns = patterns if patterns is not None else INPUTS[group] if commit: - paths = run('git', 'ls-tree', '-r', '--name-only', commit, '--', *INPUTS[group], root=root).decode().splitlines() + paths = run('git', 'ls-tree', '-r', '--name-only', commit, '--', *patterns, root=root).decode().splitlines() read = lambda path: run('git', 'show', f'{commit}:{path}', root=root) else: - paths = run('git', 'ls-files', '--cached', '--others', '--exclude-standard', '--', *INPUTS[group], root=root).decode().splitlines() + paths = run('git', 'ls-files', '--cached', '--others', '--exclude-standard', '--', *patterns, root=root).decode().splitlines() read = lambda path: (root / path).read_bytes() paths = [p for p in paths if not p.endswith('.md') and '__pycache__' not in p and not p.endswith('.pyc')] if not paths: @@ -70,7 +65,8 @@ def source_files(group, root=ROOT, commit=None): pending.append(posixpath.normpath(posixpath.join(posixpath.dirname(path), include.decode()))) workflow = (run('git', 'show', f'{commit}:.github/workflows/build.yml', root=root).decode() if commit else (root / '.github/workflows/build.yml').read_text()) - runner = re.search(r'(?ms)^ build:\n.*?^ runs-on: ([^\n]+)', workflow) + job = 'components' if re.search(r'(?m)^ components:', workflow) else 'build' + runner = re.search(r'(?ms)^ ' + job + r':\n.*?^ runs-on: ([^\n]+)', workflow) if runner is None: raise ValueError('APK build runner policy is missing') result['@runner'] = sha(runner[1].strip().encode()) @@ -85,14 +81,14 @@ def recipe(root=ROOT, commit=None): if commit and run('git', 'ls-tree', commit, script, root=root) == b'': workflow = read('.github/workflows/build.yml') text = workflow[workflow.index(' aarch64-linux-gnu-g++ -shared'):] - text = text[:text.index(' VK_INCLUDE=')] + text = textwrap.dedent(text[:text.index(' VK_INCLUDE=')]) else: text = read(script) preamble = '#!/usr/bin/env bash\nset -euo pipefail\nD=${1:?output directory}\nmkdir -p "$D"\n' if not text.startswith(preamble): return sha(text.encode()) text = text[len(preamble):] - return sha(' '.join(text.split()).encode()) + return sha(text.replace('\r\n', '\n').encode()) def artifact_bytes(origin, cache): @@ -130,6 +126,26 @@ def validate_lock(data): raise ValueError('Invalid preload checksum') +def producer_sources(group, commit, root=ROOT): + config = 'tools/components/recipes.json' + if run('git', 'ls-tree', commit, config, root=root): + definitions = json.loads(run('git', 'show', f'{commit}:{config}', root=root)) + name = 'arm64-preloads' if group == 'arm64' else 'x86-preloads' + patterns = definitions[name]['source_inputs'] + else: + # Older approved builds kept ARM64 commands inline, before the Vulkan/VR build. + if group == 'arm64': + text = run('git', 'show', f'{commit}:.github/workflows/build.yml', root=root).decode() + text = text[text.index(' aarch64-linux-gnu-g++ -shared'):] + text = text[:text.index(' VK_INCLUDE=')] + else: + text = run('git', 'show', f'{commit}:tools/linuxfs/build-x86-preloads.sh', root=root).decode() + patterns = re.findall(r'\b((?:tools|app)/[\w./*+-]+\.(?:cpp|c|h))(?=\s|$)', text) + if group == 'x86': + patterns += ['tools/linuxfs/fex', 'tools/linuxfs/build-x86-preloads.sh', 'tools/linuxfs/debian-image.env'] + return source_files(group, root, commit, patterns) + + def prove_origin(data, cache, root=ROOT): content = artifact_bytes(data['origin'], cache) if {name: sha(value) for name, value in content.items()} != data['outputs']: @@ -139,6 +155,9 @@ def prove_origin(data, cache, root=ROOT): for group in INPUTS: if source_files(group, root, commit) != data['groups'][group]: raise ValueError('Snapshot inputs differ from the producing source') + for name, checksum in producer_sources(group, commit, root).items(): + if data['groups'][group].get(name) != checksum: + raise ValueError('Snapshot omits or changes a producing build dependency') if recipe(root, commit) != data['arm64_recipe']: raise ValueError('Snapshot recipe differs from the producing source') @@ -179,14 +198,19 @@ def restore(root=ROOT): if result.returncode == 0: approved = json.loads(result.stdout) if approved != data: - prove_origin(data, root / 'app/build/component-downloads', root) + try: + prove_origin(data, root / 'app/build/component-downloads', root) + except (subprocess.CalledProcessError, OSError, http.client.HTTPException): + print('Snapshot origin is unavailable; rebuilding without trusting the snapshot.') + matching = dict.fromkeys(INPUTS, False) + if any(matching.values()): url = (f"https://github.com/Droid-Deck/DroidDeck-Components/releases/download/" f"preloads-{data['archive_sha256']}/preloads.tar.gz") try: with urllib.request.urlopen(url, timeout=60) as response: content = response.read(64 * 1024 * 1024) - except urllib.error.HTTPError as error: - if error.code != 404: + except (OSError, http.client.HTTPException) as error: + if isinstance(error, urllib.error.HTTPError) and error.code not in (404, 429) and error.code < 500: raise print('Preload snapshot is unavailable; rebuilding from source.') matching = dict.fromkeys(INPUTS, False) diff --git a/tools/components/prepare.py b/tools/components/prepare.py new file mode 100644 index 000000000..064b452be --- /dev/null +++ b/tools/components/prepare.py @@ -0,0 +1,167 @@ +"""One entry point for runtime recipes, cache identities and verified job handoff.""" +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import subprocess +import tarfile + +ROOT = Path(__file__).resolve().parents[2] +REGISTRY = 'tools/components/recipes.json' +TREES = ['app/src/main/assets/linuxfs', 'app/src/main/assets/directaudio', + 'app/src/main/assets/droiddeck-esync', 'app/build/prepared-assets', + 'app/build/component-records'] +LIBRARIES = ['app/src/main/jniLibs/arm64-v8a/' + name for name in + ('libproot.so', 'libproot-loader.so', 'libdirectaudiorelay.so')] +MANIFEST = 'app/build/prepared-components.json' + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def registry(root=ROOT): + return json.loads((root / REGISTRY).read_text()) + + +def identity(name, root=ROOT): + spec = registry(root)[name] + paths = subprocess.check_output(['git', 'ls-files', '--cached', '--others', '--exclude-standard', + '--', *spec['inputs'], *spec.get('source_inputs', [])], cwd=root, text=True).splitlines() + values = {} + for name_ in sorted(set(paths)): + if '__pycache__' in name_ or name_.endswith(('.pyc', '.md')): + continue + content = (root / name_).read_bytes() + if b'\0' not in content: + content = content.replace(b'\r\n', b'\n') + values[name_] = hashlib.sha256(content).hexdigest() + policy = dict(spec=spec, sources=values, runner=os.environ.get('RUNNER_OS', 'Linux'), + architecture=os.environ.get('RUNNER_ARCH', 'X64')) + entrypoint = root / 'tools/components/prepare.py' + if entrypoint.is_file(): + policy['entrypoint'] = hashlib.sha256(entrypoint.read_bytes().replace(b'\r\n', b'\n')).hexdigest() + installer = root / 'tools/ci/install-ubuntu-packages.sh' + if spec.get('packages') and installer.is_file(): + policy['installer'] = hashlib.sha256(installer.read_bytes().replace(b'\r\n', b'\n')).hexdigest() + if spec.get('ndk'): + policy['ndk'] = os.environ.get('NDK_VERSION', '') + policy['ndk_policy'] = hashlib.sha256((root / 'tools/ci/toolchains.env').read_bytes().replace(b'\r\n', b'\n')).hexdigest() + return hashlib.sha256(json.dumps(policy, sort_keys=True).encode()).hexdigest() + + +def prepare(name, root=ROOT): + spec = registry(root)[name] + prepare_tools([name], root) + if 'command' in spec: + command = spec['command'] + else: + script = (root / spec['script']).resolve() + if not script.is_relative_to((root / 'tools/components').resolve()): + raise ValueError('Recipe script escapes components directory') + command = ['bash', str(script)] + environment = dict(os.environ) + if spec.get('source'): + environment.update(COMPONENT_SOURCE_URL=spec['source']['url'], + COMPONENT_SOURCE_SHA256=spec['source']['sha256']) + subprocess.run(command, cwd=root, env=environment, check=True) + missing = [name_ for name_ in spec['outputs'] if not (root / name_).is_file()] + if missing: + raise ValueError(f'{name} did not prepare required outputs: {missing}') + record = dict(component=name, input_sha256=identity(name, root), + outputs={p: digest(root / p) for p in spec['outputs']}) + destination = root / 'app/build/component-records' / (name + '.json') + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(record, sort_keys=True, indent=2) + '\n') + + +def prepare_tools(names, root=ROOT): + packages = sorted({package for name in names for package in registry(root)[name].get('packages', [])}) + if packages: + subprocess.run(['bash', 'tools/ci/install-ubuntu-packages.sh', *packages], cwd=root, check=True) + + +def files(root=ROOT): + result = [] + for name in TREES: + folder = root / name + if not folder.is_dir() or folder.is_symlink(): + raise ValueError('Missing prepared component tree: ' + name) + for path in folder.rglob('*'): + if path.is_symlink(): + raise ValueError('Symlink in prepared components: ' + str(path)) + if path.is_file(): + result.append(path.relative_to(root).as_posix()) + for name in LIBRARIES: + if not (root / name).is_file() or (root / name).is_symlink(): + raise ValueError('Missing prepared component library: ' + name) + result.append(name) + return sorted(result) + + +def pack(destination, root=ROOT): + names = files(root) + tree = subprocess.check_output(['git', 'rev-parse', 'HEAD^{tree}'], cwd=root, text=True).strip() + data = dict(version=1, source_tree=tree, run=os.environ.get('GITHUB_RUN_ID', ''), + recipes={name: identity(name, root) for name in registry(root)}, + files={name: digest(root / name) for name in names}) + (root / MANIFEST).write_text(json.dumps(data, sort_keys=True, indent=2) + '\n') + destination.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(destination, mode='w:gz', compresslevel=1) as archive: + for name in names + [MANIFEST]: + archive.add(root / name, arcname=name, recursive=False) + + +def unpack(archive_path, source_tree, root=ROOT): + total = 0 + seen = set() + with tarfile.open(archive_path, mode='r:gz') as archive: + for entry in archive: + allowed = entry.name in LIBRARIES + [MANIFEST] or any(entry.name.startswith(p + '/') for p in TREES) + path = root / entry.name + total += entry.size + if (not allowed or '\\' in entry.name or '..' in PurePosixPath(entry.name).parts + or PurePosixPath(entry.name).as_posix() != entry.name + or not entry.isfile() or entry.name in seen or len(seen) >= 100000 + or entry.size > 512 * 1024 * 1024 + or total > 2 * 1024**3 or not path.resolve().is_relative_to(root.resolve()) + or path.is_symlink()): + raise ValueError('Unsafe prepared component archive') + seen.add(entry.name) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('wb') as output: + shutil.copyfileobj(archive.extractfile(entry), output) + path.chmod(entry.mode & 0o777) + data = json.loads((root / MANIFEST).read_text()) + actual = {name: digest(root / name) for name in files(root)} + recipes = {name: identity(name, root) for name in registry(root)} + if (data.get('version') != 1 or data['source_tree'] != source_tree or data['files'] != actual + or data['recipes'] != recipes): + raise ValueError('Prepared components do not match the pinned source tree and file hashes') + print('Prepared component source identity and all output hashes verified.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('recipe') + parser.add_argument('--archive', type=Path, default=ROOT / 'app/build/prepared-components.tar.gz') + parser.add_argument('--tree') + args = parser.parse_args() + if args.recipe == 'plan': + for name in registry(): + value = identity(name) + print(f'{name}: {value}') + if os.environ.get('GITHUB_OUTPUT'): + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f'{name}={value}\n') + elif args.recipe == 'tools': + prepare_tools(os.environ.get('COMPONENTS_TO_BUILD', '').split()) + elif args.recipe == 'pack': + pack(args.archive) + elif args.recipe == 'unpack': + unpack(args.archive, args.tree) + else: + prepare(args.recipe) diff --git a/tools/components/recipes.json b/tools/components/recipes.json new file mode 100644 index 000000000..19144018f --- /dev/null +++ b/tools/components/recipes.json @@ -0,0 +1,225 @@ +{ + "runtime": { + "script": "tools/components/recipes/runtime.sh", + "inputs": [ + "tools/components/recipes/runtime.sh", + "tools/quest", + "tools/linuxfs/overlay", + "tools/linuxfs/desktop", + "tools/linuxfs/licenses", + "tools/linuxfs/uruntime.env", + "tools/gamescope/release.env", + "tools/wlroots/release.env", + "tools/mangoapp", + "tools/msitools", + "tools/pipewire", + "tools/vr", + "app/src/main/cpp/waylandcomp/generated", + "app/src/main/cpp/waylandcomp/src/headpose_shm.h", + "app/src/main/cpp/waylandcomp/src/spatialpose_shm.h", + "app/src/main/cpp/venc/venc_proto.h" + ], + "outputs": [ + "app/src/main/assets/linuxfs/usr/local/bin/gamescope", + "app/src/main/assets/linuxfs/usr/local/lib/droiddeck-vr/droiddeck/bin/linuxarm64/driver_droiddeck.so" + ], + "ndk": true, + "packages": [ + "gcc-aarch64-linux-gnu", + "g++-aarch64-linux-gnu", + "libvulkan1", + "mesa-vulkan-drivers" + ] + }, + "esync": { + "script": "tools/components/recipes/esync.sh", + "inputs": [ + "tools/components/recipes/esync.sh", + "tools/droiddeck-esync/release.env" + ], + "outputs": [ + "app/src/main/assets/droiddeck-esync/index.json", + "app/src/main/assets/droiddeck-esync/index.json.sig" + ] + }, + "debian-image": { + "script": "tools/components/recipes/debian-image.sh", + "inputs": [ + "tools/components/recipes/debian-image.sh" + ], + "outputs": [] + }, + "x86-preloads": { + "script": "tools/components/recipes/x86-preloads.sh", + "inputs": [ + "tools/components/recipes/x86-preloads.sh", + "tools/linuxfs/build-x86-preloads.sh", + "tools/linuxfs/preload", + "tools/linuxfs/fex" + ], + "outputs": [ + "app/src/main/assets/linuxfs/x86_64/libfakeinput.so", + "app/src/main/assets/linuxfs/x86_64/libblsession.so", + "app/src/main/assets/linuxfs/x86_64/libfaultreport.so", + "app/src/main/assets/linuxfs/i386/libfakeinput.so", + "app/src/main/assets/linuxfs/i386/libblsession.so", + "app/src/main/assets/linuxfs/i386/libfaultreport.so", + "app/src/main/assets/linuxfs/x86_64/libwebkitinput.so", + "app/src/main/assets/linuxfs/x86_64/libvulkan-thunk.so" + ], + "source_inputs": [ + "app/src/main/cpp/fakeinput_steam.cpp", + "app/src/main/cpp/adrenotools/src/hook/kgsl.h", + "app/src/main/cpp/venc/venc_proto.h", + "tools/linuxfs/preload", + "tools/quest/kgsl_compat.c", + "tools/quest/kgsl_compat.h", + "tools/linuxfs/fex", + "tools/linuxfs/build-x86-preloads.sh", + "tools/linuxfs/debian-image.env" + ], + "snapshot_outputs": [ + "x86_64/libfakeinput.so", + "x86_64/libblsession.so", + "x86_64/libfaultreport.so", + "i386/libfakeinput.so", + "i386/libblsession.so", + "i386/libfaultreport.so", + "x86_64/libwebkitinput.so", + "x86_64/libvulkan-thunk.so" + ] + }, + "pulseaudio-source": { + "script": "tools/components/recipes/pulseaudio-source.sh", + "inputs": [ + "tools/components/recipes/pulseaudio-source.sh" + ], + "outputs": [ + "pa13/src/pulsecore/module.h" + ], + "source": { + "url": "https://github.com/pulseaudio/pulseaudio/archive/refs/tags/v13.0.tar.gz", + "sha256": "bd7980bc9bd44bd3b3363545dabd19476ebf4d6bb6d2527e7d179256394a6758" + } + }, + "directaudio": { + "script": "tools/components/recipes/directaudio.sh", + "inputs": [ + "tools/components/recipes/directaudio.sh", + "tools/directaudio" + ], + "outputs": [ + "app/src/main/jniLibs/arm64-v8a/libdirectaudiorelay.so" + ] + }, + "audio": { + "script": "tools/components/recipes/audio.sh", + "inputs": [ + "tools/aaudio-sink", + "tools/directaudio", + "tools/components/recipes/audio.sh", + "app/src/main/assets/pulseaudio.tzst" + ], + "outputs": [ + "app/build/prepared-assets/pulseaudio.tzst" + ], + "ndk": true + }, + "proot": { + "script": "tools/components/recipes/proot.sh", + "inputs": [ + "tools/proot", + "tools/components/recipes/proot.sh" + ], + "outputs": [ + "app/src/main/jniLibs/arm64-v8a/libproot.so", + "app/src/main/jniLibs/arm64-v8a/libproot-loader.so" + ], + "ndk": true + }, + "comet": { + "script": "tools/components/recipes/comet.sh", + "inputs": [ + "tools/comet", + "tools/components/recipes/comet.sh" + ], + "outputs": [ + "app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet/comet", + "app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet/version" + ], + "packages": [ + "gcc-aarch64-linux-gnu" + ] + }, + "arm64-preloads": { + "script": "tools/components/recipes/arm64-preloads.sh", + "inputs": [ + "tools/components/build-arm64-preloads.sh", + "tools/linuxfs/preload", + "app/src/main/cpp/fakeinput_steam.cpp", + "tools/components/recipes/arm64-preloads.sh" + ], + "outputs": [ + "app/src/main/assets/linuxfs/libfakeinput.so", + "app/src/main/assets/linuxfs/libblsession.so", + "app/src/main/assets/linuxfs/libblfastpath.so", + "app/src/main/assets/linuxfs/libblaudit.so", + "app/src/main/assets/linuxfs/libssbs.so", + "app/src/main/assets/linuxfs/usr/local/bin/droiddeck-clipboard" + ], + "packages": [ + "gcc-aarch64-linux-gnu", + "g++-aarch64-linux-gnu" + ], + "source_inputs": [ + "app/src/main/cpp/fakeinput_steam.cpp", + "app/src/main/cpp/adrenotools/src/hook/kgsl.h", + "app/src/main/cpp/venc/venc_proto.h", + "tools/linuxfs/preload", + "tools/quest/kgsl_compat.c", + "tools/quest/kgsl_compat.h", + "tools/proot/fastpath", + "tools/linuxfs/ssbs", + "tools/linuxfs/clipboard" + ], + "snapshot_outputs": [ + "libfakeinput.so", + "libblsession.so", + "libblfastpath.so", + "libblaudit.so", + "libssbs.so", + "usr/local/bin/droiddeck-clipboard" + ] + }, + "preload-snapshot": { + "command": [ + "python3", + "tools/components/preloads.py", + "restore" + ], + "inputs": [ + "tools/components/preloads.py", + "tools/components/preloads.lock.json" + ], + "outputs": [] + }, + "xbox": { + "script": "tools/components/recipes/xbox.sh", + "inputs": [ + "tools/xbox", + "tools/components/recipes/xbox.sh" + ], + "outputs": [ + "app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox/xodus-cli", + "app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox/xodus-service" + ], + "packages": [ + "build-essential", + "pkg-config", + "libssl-dev", + "protobuf-compiler", + "cmake", + "xz-utils" + ] + } +} diff --git a/tools/components/recipes/arm64-preloads.sh b/tools/components/recipes/arm64-preloads.sh new file mode 100644 index 000000000..c6a227c2e --- /dev/null +++ b/tools/components/recipes/arm64-preloads.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +bash tools/components/build-arm64-preloads.sh app/src/main/assets/linuxfs diff --git a/tools/components/recipes/audio.sh b/tools/components/recipes/audio.sh new file mode 100644 index 000000000..819421d7c --- /dev/null +++ b/tools/components/recipes/audio.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +export NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}" +tools/aaudio-sink/build.sh "$PWD/pa13" "$PWD/sink-out" +rm -rf pa-bundle && mkdir -p pa-bundle +tar --use-compress-program=unzstd -xf app/src/main/assets/pulseaudio.tzst -C pa-bundle +test ! -f pa-bundle/modules/arm64/module-aaudio-sink.so +install -m755 sink-out/module-aaudio-sink.so pa-bundle/modules/arm64/module-aaudio-sink.so +install -m755 sink-out/module-directaudio-native-sink.so pa-bundle/modules/arm64/module-directaudio-native-sink.so +(cd pa-bundle && tar --use-compress-program='zstd -19' -cf ../pulseaudio.tzst .) +mkdir -p app/build/prepared-assets +mv pulseaudio.tzst app/build/prepared-assets/pulseaudio.tzst +tar --use-compress-program=unzstd -tf app/build/prepared-assets/pulseaudio.tzst | grep -c '\.so$' +rm -rf pa-bundle sink-out diff --git a/tools/components/recipes/comet.sh b/tools/components/recipes/comet.sh new file mode 100644 index 000000000..e47d3e3d5 --- /dev/null +++ b/tools/components/recipes/comet.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +rustup toolchain install 1.90.0 --profile minimal +rustup target add --toolchain 1.90.0 aarch64-unknown-linux-gnu +RUSTUP_TOOLCHAIN=1.90.0 bash tools/comet/build.sh diff --git a/tools/components/recipes/debian-image.sh b/tools/components/recipes/debian-image.sh new file mode 100644 index 000000000..f982225dc --- /dev/null +++ b/tools/components/recipes/debian-image.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +. tools/linuxfs/debian-image.env +image=public.ecr.aws/docker/library/debian:bullseye +if gh release download "$DEBIAN_IMAGE_TAG" -R "$GITHUB_REPOSITORY" -p "$DEBIAN_IMAGE_ASSET" -O /tmp/debian-image.tar.zst \ + && { [ -z "$DEBIAN_IMAGE_SHA256" ] || echo "$DEBIAN_IMAGE_SHA256 /tmp/debian-image.tar.zst" | sha256sum -c -; }; then + zstd -dc /tmp/debian-image.tar.zst | docker load + rm -f /tmp/debian-image.tar.zst +else + echo "::warning::the hosted Debian image could not be fetched; pulling from the registry instead" + for attempt in 1 2 3 4 5; do + docker pull "$image" && break + [ "$attempt" -lt 5 ] || { echo "::error::could not pull $image after $attempt attempts"; exit 1; } + echo "pull attempt $attempt failed (registry busy?); retrying in $((attempt * 30)) s" + sleep $((attempt * 30)) + done +fi +docker image ls "$image" diff --git a/tools/components/recipes/directaudio.sh b/tools/components/recipes/directaudio.sh new file mode 100644 index 000000000..c684f0262 --- /dev/null +++ b/tools/components/recipes/directaudio.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +tools/directaudio/fetch.sh "$PWD" "$PWD/sink-out" \ No newline at end of file diff --git a/tools/components/recipes/esync.sh b/tools/components/recipes/esync.sh new file mode 100644 index 000000000..691f77e99 --- /dev/null +++ b/tools/components/recipes/esync.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +if [ -f tools/droiddeck-esync/release.env ]; then + . tools/droiddeck-esync/release.env + release_url="https://github.com/${SYNC_BUNDLE_REPO}/releases/download/${SYNC_BUNDLE_TAG}" + curl -fsSL --retry 3 -o sync-bundle.tzst "$release_url/$SYNC_BUNDLE_ASSET" + echo "$SYNC_BUNDLE_SHA256 sync-bundle.tzst" | sha256sum -c - + mkdir -p app/src/main/assets/droiddeck-esync + tar --use-compress-program=unzstd -xf sync-bundle.tzst -C app/src/main/assets/droiddeck-esync + rm sync-bundle.tzst + curl -fsSL --retry 3 -o sync-index.json "$release_url/index.json" + revoked=$(python3 -c 'import json, sys; print(" ".join(p["id"] for p in json.load(open(sys.argv[1]))["packs"] if p.get("revoked") is True))' sync-index.json) + for id in $revoked; do + if [ -e "app/src/main/assets/droiddeck-esync/packs/$id.tzst" ]; then + echo "::warning::$SYNC_BUNDLE_ASSET carries revoked pack $id; it is left out of the APK (point tools/droiddeck-esync/release.env at a newer bundle)" + rm -f "app/src/main/assets/droiddeck-esync/packs/$id.tzst" + fi + done + rm sync-index.json + test -f app/src/main/assets/droiddeck-esync/index.json + test -f app/src/main/assets/droiddeck-esync/index.json.sig + ls -lR app/src/main/assets/droiddeck-esync +fi diff --git a/tools/components/recipes/proot.sh b/tools/components/recipes/proot.sh new file mode 100644 index 000000000..9bf7699e8 --- /dev/null +++ b/tools/components/recipes/proot.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +export NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}" +tools/proot/build.sh app/src/main/jniLibs/arm64-v8a diff --git a/tools/components/recipes/pulseaudio-source.sh b/tools/components/recipes/pulseaudio-source.sh new file mode 100644 index 000000000..5a2a6ed09 --- /dev/null +++ b/tools/components/recipes/pulseaudio-source.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +curl -fsSL --retry 3 -o pa.tar.gz "$COMPONENT_SOURCE_URL" +echo "$COMPONENT_SOURCE_SHA256 pa.tar.gz" | sha256sum -c - +mkdir -p pa13 && tar -xzf pa.tar.gz -C pa13 --strip-components=1 && rm pa.tar.gz diff --git a/tools/components/recipes/runtime.sh b/tools/components/recipes/runtime.sh new file mode 100644 index 000000000..0e640ab11 --- /dev/null +++ b/tools/components/recipes/runtime.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail +D=app/src/main/assets/linuxfs +mkdir -p "$D" +VK_INCLUDE=$(ls -d "${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1) \ + tools/vr/build.sh --native "$D/usr/local/lib/droiddeck-vr" +for script in tools/linuxfs/overlay/usr/local/bin/droiddeck-* tools/linuxfs/overlay/usr/local/bin/steam-compatibility; do + install -Dm644 "$script" "$D/usr/local/bin/$(basename "$script")" +done +for f in tools/linuxfs/overlay/usr/bin/* tools/linuxfs/overlay/usr/bin/steamos-polkit-helpers/*; do + [ -f "$f" ] && install -Dm644 "$f" "$D/${f#tools/linuxfs/overlay/}" +done +if [ -f tools/gamescope/release.env ]; then + . tools/gamescope/release.env + curl -fsSL --retry 3 -o gamescope.tzst "https://github.com/${GAMESCOPE_REPO:-${GITHUB_REPOSITORY}}/releases/download/${GAMESCOPE_TAG}/gamescope.tzst" + echo "${GAMESCOPE_SHA256} gamescope.tzst" | sha256sum -c - + tar --use-compress-program=unzstd -xf gamescope.tzst -C "$D" + test -f "$D/usr/local/bin/gamescope" + rm gamescope.tzst +fi +. tools/linuxfs/uruntime.env +curl -fsSL --retry 3 -o uruntime "https://github.com/VHSgunzo/uruntime/releases/download/${URUNTIME_VERSION}/${URUNTIME_ASSET}" +echo "${URUNTIME_SHA256} uruntime" | sha256sum -c - +install -Dm644 uruntime "$D/usr/local/lib/droiddeck/uruntime" +install -Dm644 tools/linuxfs/licenses/uruntime-LICENSE "$D/usr/local/share/licenses/uruntime/LICENSE" +rm uruntime +M="$D/usr/local/lib/mangoapp" +mkdir -p "$M" mango-pkgs +grep -v '^#' tools/mangoapp/packages.txt | while read -r sum url; do + [ -n "$url" ] || continue + curl -fsSL --retry 3 -o mango-pkgs/pkg.tar.zst "$url" + echo "$sum mango-pkgs/pkg.tar.zst" | sha256sum -c - + tar --use-compress-program=unzstd -xf mango-pkgs/pkg.tar.zst -C mango-pkgs + rm mango-pkgs/pkg.tar.zst +done +install -m644 mango-pkgs/usr/bin/mangoapp "$M/mangoapp" +for so in libfmt.so.10 libspdlog.so.1.13 libglfw.so.3 libtraceevent.so.1; do + cp -L "mango-pkgs/usr/lib/$so" "$M/$so" +done +aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -Wl,-soname,libtracefs.so.1 \ + -o "$M/libtracefs.so.1" tools/mangoapp/libtracefs-shim.c +aarch64-linux-gnu-strip --strip-unneeded "$M/libtracefs.so.1" +install -Dm644 tools/mangoapp/mangoapp "$D/usr/local/bin/mangoapp" +rm -rf mango-pkgs +ls -l "$M" +mkdir -p pw-pkgs +grep -v '^#' tools/pipewire/packages.txt | while read -r sum url; do + [ -n "$url" ] || continue + curl -fsSL --retry 3 -o pw-pkgs/pkg.tar.zst "$url" + echo "$sum pw-pkgs/pkg.tar.zst" | sha256sum -c - + tar --use-compress-program=unzstd -xf pw-pkgs/pkg.tar.zst -C pw-pkgs + rm pw-pkgs/pkg.tar.zst +done +tools/pipewire/assemble.sh pw-pkgs "$D/usr/local/lib/droiddeck-pipewire.tzst" +rm -rf pw-pkgs +ls -l "$D/usr/local/lib/droiddeck-pipewire.tzst" +T="$D/usr/local/lib/droiddeck-msitools" +mkdir -p "$T" msi-pkgs +grep -v '^#' tools/msitools/packages.txt | while read -r sum url; do + [ -n "$url" ] || continue + curl -fsSL --retry 3 -o msi-pkgs/pkg.tar.xz "$url" + echo "$sum msi-pkgs/pkg.tar.xz" | sha256sum -c - + tar -xJf msi-pkgs/pkg.tar.xz -C msi-pkgs + rm msi-pkgs/pkg.tar.xz +done +install -m644 msi-pkgs/usr/bin/msiinfo "$T/msiinfo" +install -m644 msi-pkgs/usr/bin/cabextract "$T/cabextract" +install -m644 msi-pkgs/usr/lib/7zip/7z "$T/7z" +install -m644 msi-pkgs/usr/lib/7zip/7z.so "$T/7z.so" +for so in libmsi-1.0.so.0 libgsf-1.so.114 libgcab-1.0.so.0; do + cp -L "msi-pkgs/usr/lib/$so" "$T/$so" +done +for f in NOTICE GPL-2 GPL-3 LGPL-2.1 7zip-License 7zip-unRarLicense; do + install -Dm644 "tools/msitools/licenses/$f" "$D/usr/local/share/licenses/droiddeck-msitools/$f" +done +rm -rf msi-pkgs +ls -l "$T" +install -Dm644 tools/linuxfs/desktop/droiddeck-desktop "$D/usr/local/bin/droiddeck-desktop" +install -Dm644 tools/linuxfs/desktop/droiddeck-gpu "$D/usr/local/bin/droiddeck-gpu" +install -Dm644 tools/linuxfs/desktop/droiddeck-desktop-gpu "$D/usr/local/bin/droiddeck-desktop-gpu" +install -Dm644 tools/linuxfs/desktop/kwin_wayland_wrapper "$D/usr/local/bin/kwin_wayland_wrapper" +install -Dm644 tools/linuxfs/desktop/firefox-droiddeck.js "$D/usr/lib/firefox/defaults/pref/droiddeck.js" +ls -lR "$D" diff --git a/tools/components/recipes/x86-preloads.sh b/tools/components/recipes/x86-preloads.sh new file mode 100644 index 000000000..c107c543b --- /dev/null +++ b/tools/components/recipes/x86-preloads.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +docker run --rm -v "$PWD:/src" -w /src public.ecr.aws/docker/library/debian:bullseye bash -c ' + set -euo pipefail + install_compilers() { + apt-get -o Acquire::Check-Valid-Until=false update -qq && + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends gcc g++ gcc-multilib g++-multilib binutils >/dev/null + } + printf "deb http://deb.debian.org/debian bullseye main\ndeb http://security.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list + if ! install_compilers; then + printf "deb http://archive.debian.org/debian bullseye main\ndeb http://archive.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list + install_compilers + fi + tools/linuxfs/build-x86-preloads.sh app/src/main/assets/linuxfs +' +sudo chown -R "$(id -u):$(id -g)" app/src/main/assets/linuxfs +ls -l app/src/main/assets/linuxfs/x86_64 app/src/main/assets/linuxfs/i386 diff --git a/tools/components/recipes/xbox.sh b/tools/components/recipes/xbox.sh new file mode 100644 index 000000000..87c953656 --- /dev/null +++ b/tools/components/recipes/xbox.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +rustup toolchain install 1.99.0 --profile minimal +bash tools/xbox/build.sh diff --git a/tools/release/announce-preview.sh b/tools/release/announce-preview.sh new file mode 100644 index 000000000..c4b63e727 --- /dev/null +++ b/tools/release/announce-preview.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +if [ -z "$WEBHOOK" ]; then + echo "::notice::No DISCORD_BUILDS_WEBHOOK secret; nothing posted" + exit 0 +fi +commit=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA") +subject=$(jq -r '.commit.message | split("\n")[0]' <<<"$commit") +title=$(jq -r '.commit.message | split("\n") | map(select(length > 0)) | .[1] // ""' <<<"$commit") +pr=$(sed -n 's/^Merge pull request #\([0-9]*\).*/\1/p' <<<"$subject") +if [ -n "$pr" ]; then + what="[#$pr]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/$pr): ${title:-$subject}" +else + what="[\`${GITHUB_SHA::7}\`]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA): $subject" +fi +get=$(printf 'Download: [standard APK](<%s>) · [all variants](<%s>)' "$APK" "$RELEASE") +content=$(printf '**New signed Preview build** `main-%s`\n%s\nIn the app: **Updates → Preview**\n%s' "${GITHUB_SHA::7}" "$what" "$get") +jq -n --arg content "$content" '{content: $content, allowed_mentions: {parse: []}}' | + curl -fsS -H 'Content-Type: application/json' -d @- "$WEBHOOK" -o /dev/null +echo "Posted: $content" diff --git a/tools/release/describe-build.sh b/tools/release/describe-build.sh new file mode 100644 index 000000000..46bc339ba --- /dev/null +++ b/tools/release/describe-build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" \ + --jq '[.[] | select(.merged_at != null and .base.ref == "main")] | first // empty') +if [ -n "$pr" ]; then + number=$(jq -r .number <<<"$pr") + title=$(jq -r .title <<<"$pr") + merged=$(jq -r '.merged_at[:10]' <<<"$pr") + summary=$(jq -r '.body // ""' <<<"$pr" | tr -d '\r' | { grep -v '^🤖' || true; } | + awk '!p && /^#/ { next } NF { p = 1; print; next } p { exit }' | head -n 6 | cut -c1-400) + { + echo "**Merged to main** - [PR #$number]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/$number): $title (merged $merged)" + if [ -n "$summary" ]; then + echo + echo "$summary" + fi + } > notes-head.md +else + subject=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA" --jq '.commit.message | split("\n")[0]') + echo "**Pushed to main** without a PR - $subject" > notes-head.md +fi +cat notes-head.md diff --git a/tools/release/publish-ci.sh b/tools/release/publish-ci.sh new file mode 100644 index 000000000..987ef622d --- /dev/null +++ b/tools/release/publish-ci.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +tag=main-${GITHUB_SHA::7} +cat signed/SHA256SUMS-*.txt > signed/SHA256SUMS.txt +rm signed/SHA256SUMS-*.txt +notes="$(cat notes-head.md) + +Signed build of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA - in main, not yet in a DroidDeck release; for testing. +Commit: $GITHUB_SHA +VersionCode: $(sed -n 's/^ *versionCode \([0-9][0-9]*\)$/\1/p' app/build.gradle) +Each apk installs over a DroidDeck release and keeps its data. \`standard\` is the normal app; the others are the same app under another package name, installed beside it. + +SHA-256: +\`\`\` +$(cat signed/SHA256SUMS.txt) +\`\`\`" +gh release delete "$tag" -R "$CI_REPO" --yes --cleanup-tag 2>/dev/null || true +gh release create "$tag" -R "$CI_REPO" --target main --prerelease \ + --title "main ${GITHUB_SHA::7}" --notes "$notes" signed/* +gh release list -R "$CI_REPO" -L 200 --json tagName,publishedAt \ + --jq '[.[] | select(.tagName | startswith("main-"))] | sort_by(.publishedAt) | reverse | .['"$KEEP"':][] | .tagName' | + while IFS= read -r old; do + if [ "$old" = "$tag" ]; then + continue + fi + if ! gh release delete "$old" -R "$CI_REPO" --yes --cleanup-tag 2>/dev/null; then + if gh release view "$old" -R "$CI_REPO" >/dev/null 2>&1; then + echo "::error::Could not delete old release $old" + exit 1 + fi + fi + done +published=$(gh release view "$tag" -R "$CI_REPO" --json tagName,assets) +jq -e --arg tag "$tag" '.tagName == $tag and (.assets | length) > 0' <<< "$published" >/dev/null +url="$GITHUB_SERVER_URL/$CI_REPO/releases/tag/$tag" +apk=$(cd signed && ls DroidDeck-*.apk | awk 'length(a) == 0 || length($0) < length(a) { a = $0 } END { print a }') +echo "release=$url" >> "$GITHUB_OUTPUT" +echo "apk=$GITHUB_SERVER_URL/$CI_REPO/releases/download/$tag/$apk" >> "$GITHUB_OUTPUT" +echo "Published $url" diff --git a/tools/release/report-build.sh b/tools/release/report-build.sh new file mode 100644 index 000000000..fc031c634 --- /dev/null +++ b/tools/release/report-build.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -uo pipefail +MARK="" +{ + echo "$MARK" + echo "## Build of this pull request" + echo + [ -f /tmp/pr-merge.md ] && cat /tmp/pr-merge.md + echo + case "$BUILD_OUTCOME:$UNIT_TEST_OUTCOME:$APK_OUTCOME" in + success:success:success) + echo "### Build passed" + echo + echo "The \`droiddeck-apk\` artifact is [on this run]($RUN_URL), signed with the public test key." + echo + if [ "$SAME_REPO" = true ]; then + echo "After the parallel regression tests pass, a release-signed tester build that installs over Stable or Preview will be published and will appear in DroidDeck's **Test builds** list. Label the PR \`no-test-build\` to skip that." + else + echo "**Maintainer:** add the \`publish-test\` label to publish a release-signed tester build that installs over Stable or Preview and appears in DroidDeck's **Test builds** list. The label is removed automatically after publishing." + fi + echo + echo "Fallback: [open **PR Release Publish**]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/workflows/pr-release-publish.yml) on \`main\` and enter PR \`$PR\`." + ;; + success:success:*) + echo "### APK publication incomplete" + echo + echo "Compilation and unit tests passed, but APK publication did not complete. See [the run]($RUN_URL)." + ;; + success:failure:*) + echo "### ERROR: Unit tests failed" + echo + echo "The APK compiled, but unit tests failed. No installable artifact was published." + echo + if [ -n "$TEST_DIAGNOSTICS_URL" ]; then + echo "The [unit-test-results]($TEST_DIAGNOSTICS_URL) artifact contains the test log and any generated reports." + else + echo "The test diagnostics upload did not complete. See [the test log]($RUN_URL)." + fi + ;; + success:*) + echo "### Build incomplete" + echo + echo "The APK compiled, but unit tests did not complete successfully. See [the run]($RUN_URL)." + ;; + failure:*) + echo "### ERROR: Build failed" + echo + echo "The compiler's errors ([full log]($RUN_URL)):" + echo + echo '```' + { grep -E '^e: |^w: .*error|error:|CMake Error|fatal error|ninja: error|FAILED|What went wrong|Execution failed for task|Unresolved reference|Type mismatch|Unresolved' /tmp/build.log 2>/dev/null \ + | sed -e 's#file:///home/runner/work/[^/]*/[^/]*/##' -e 's#/home/runner/work/[^/]*/[^/]*/##' \ + | grep -v '^w: ' | awk '!seen[$0]++' | head -80; } || true + echo '```' + echo + echo "
What went wrong (Gradle)" + echo + echo '```' + awk '/^\* What went wrong/{p=1} /^\* Try:/{p=0} p{print}' /tmp/build.log 2>/dev/null | head -60 + echo '```' + echo "
" + ;; + skipped:*|cancelled:*) + echo "### Not built" + echo + echo "The build did not run (see above, or [the run]($RUN_URL))." + ;; + esac + echo +} > /tmp/pr-report.md +cat /tmp/pr-report.md >> "$GITHUB_STEP_SUMMARY" +ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate --jq ".[] | select(.body | startswith(\"$MARK\")) | .id" | head -1 || true) +if [ -n "$ID" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${ID}" -F body=@/tmp/pr-report.md >/dev/null +else + gh api -X POST "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -F body=@/tmp/pr-report.md >/dev/null +fi diff --git a/tools/release/runtime_inputs.py b/tools/release/runtime_inputs.py index d6cf46bc4..83f87be03 100644 --- a/tools/release/runtime_inputs.py +++ b/tools/release/runtime_inputs.py @@ -23,7 +23,7 @@ PACK_NAMES = ['proot', 'directaudio', 'tracking', 'xbox', 'comet'] SOURCE_PATHS = ['app/build.gradle', 'app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/assets/pulseaudio.tzst', 'tools/linuxfs', 'tools/proot', 'tools/aaudio-sink', 'tools/directaudio', 'tools/gamescope/release.env', 'tools/droiddeck-esync/release.env', - 'tools/mangoapp', 'tools/msitools', 'tools/pipewire', 'tools/comet', 'tools/xbox', 'tools/components', 'tools/build_local.sh', + 'tools/mangoapp', 'tools/msitools', 'tools/pipewire', 'tools/comet', 'tools/xbox', 'tools/components', 'tools/ci', 'tools/build_local.sh', 'tools/release/runtime_inputs.py', 'app/src/main/assets/desktop-removal', 'tools/quest', 'tools/vr/src', 'tools/vr/driver_droiddeck', 'tools/vr/layer', 'tools/vr/build.sh', 'app/src/main/cpp/waylandcomp/src/headpose_shm.h', diff --git a/tools/tests/test_ci_checks.py b/tools/tests/test_ci_checks.py index df472a47d..3a0216d77 100644 --- a/tools/tests/test_ci_checks.py +++ b/tools/tests/test_ci_checks.py @@ -12,6 +12,12 @@ ROOT = Path(__file__).resolve().parents[2] class CiChecksTest(unittest.TestCase): + def test_independent_jobs_wait_only_for_the_pinned_source(self): + jobs = yaml.safe_load((ROOT / '.github/workflows/build.yml').read_text())['jobs'] + for name in ('xbox', 'components', 'python-tests'): + self.assertEqual(jobs[name]['needs'], 'source') + self.assertEqual(set(jobs['build']['needs']), {'source', 'components', 'xbox'}) + def test_dependency_update_errors_block_installation(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tools/tests/test_ci_ndk.py b/tools/tests/test_ci_ndk.py new file mode 100644 index 000000000..eef737db6 --- /dev/null +++ b/tools/tests/test_ci_ndk.py @@ -0,0 +1,27 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class CiNdkTest(unittest.TestCase): + def test_installed_ndk_must_match_the_declared_version(self): + version = (ROOT / 'tools/ci/toolchains.env').read_text().strip().split('=', 1)[1] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + ndk = root / 'ndk' / version + ndk.mkdir(parents=True) + properties = ndk / 'source.properties' + properties.write_text('Pkg.Revision = ' + version + '\n') + output = root / 'environment' + env = dict(os.environ, ANDROID_HOME=str(root), GITHUB_ENV=str(output)) + script = ROOT / 'tools/ci/resolve-ndk.sh' + subprocess.run(['bash', str(script)], env=env, check=True) + self.assertIn('NDK_VERSION=' + version, output.read_text()) + self.assertIn('ANDROID_NDK_HOME=' + str(ndk), output.read_text()) + properties.write_text('Pkg.Revision = unexpected\n') + self.assertNotEqual(subprocess.run(['bash', str(script)], env=env).returncode, 0) diff --git a/tools/tests/test_component_prepare.py b/tools/tests/test_component_prepare.py new file mode 100644 index 000000000..b513c8fe7 --- /dev/null +++ b/tools/tests/test_component_prepare.py @@ -0,0 +1,106 @@ +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('component_prepare', Path(__file__).resolve().parents[1] / 'components/prepare.py') +components = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(components) + + +class ComponentPrepareTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + self.fixture(self.root) + + def fixture(self, root): + for name in components.TREES: + (root / name).mkdir(parents=True) + (root / 'tools/components/recipes').mkdir(parents=True) + (root / 'input.h').write_text('#define VALUE 1\n') + (root / 'tools/components/recipes/fake.sh').write_text('printf binary > app/src/main/assets/linuxfs/demo.so\n') + spec = dict(script='tools/components/recipes/fake.sh', inputs=['input.h', 'tools/components/recipes/fake.sh'], + outputs=['app/src/main/assets/linuxfs/demo.so']) + (root / components.REGISTRY).write_text(json.dumps(dict(fake=spec))) + for name in components.LIBRARIES: + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_bytes(b'library') + subprocess.run(['git', 'init', '-q', str(root)], check=True) + subprocess.run(['git', '-C', str(root), 'config', 'core.autocrlf', 'false'], check=True) + subprocess.run(['git', '-C', str(root), 'add', '.'], check=True) + subprocess.run(['git', '-C', str(root), '-c', 'user.name=Test Developer', '-c', 'user.email=test@example.test', + 'commit', '-qm', 'baseline'], check=True) + + def test_recipe_failure_or_missing_output_does_not_succeed(self): + script = self.root / 'tools/components/recipes/fake.sh' + script.write_text('exit 7\n') + with self.assertRaises(subprocess.CalledProcessError): + components.prepare('fake', self.root) + script.write_text('exit 0\n') + with self.assertRaisesRegex(ValueError, 'required outputs'): + components.prepare('fake', self.root) + + def test_identity_covers_source_and_recipe_changes(self): + original = components.identity('fake', self.root) + (self.root / 'input.h').write_text('#define VALUE 2\n') + changed = components.identity('fake', self.root) + self.assertNotEqual(original, changed) + script = self.root / 'tools/components/recipes/fake.sh' + script.write_text(script.read_text() + '\nprintf changed\n') + self.assertNotEqual(changed, components.identity('fake', self.root)) + + def test_tool_requirements_are_installed_in_one_union(self): + definitions = dict(one=dict(packages=['gcc', 'cmake']), two=dict(packages=['gcc', 'qemu-user'])) + with patch.object(components, 'registry', return_value=definitions), patch.object(components.subprocess, 'run') as run: + components.prepare_tools(['one', 'two'], self.root) + run.assert_called_once_with(['bash', 'tools/ci/install-ubuntu-packages.sh', 'cmake', 'gcc', 'qemu-user'], + cwd=self.root, check=True) + + def test_prepared_handoff_verifies_every_file_and_source_tree(self): + components.prepare('fake', self.root) + archive = self.root / 'prepared.tar.gz' + components.pack(archive, self.root) + tree = subprocess.check_output(['git', 'rev-parse', 'HEAD^{tree}'], cwd=self.root, text=True).strip() + target = self.root / 'target' + self.fixture(target) + components.unpack(archive, tree, target) + self.assertEqual((target / 'app/src/main/assets/linuxfs/demo.so').read_bytes(), b'binary') + with self.assertRaisesRegex(ValueError, 'pinned source'): + components.unpack(archive, 'wrong-tree', target) + + def test_tampered_component_bytes_fail_handoff_verification(self): + components.prepare('fake', self.root) + archive = self.root / 'prepared.tar.gz' + components.pack(archive, self.root) + corrupt = self.root / 'corrupt.tar.gz' + with tarfile.open(archive) as original, tarfile.open(corrupt, 'w:gz') as output: + for entry in original: + content = original.extractfile(entry).read() + if entry.name.endswith('demo.so'): + content = b'tampered' + entry.size = len(content) + output.addfile(entry, io.BytesIO(content)) + target = self.root / 'target' + self.fixture(target) + tree = subprocess.check_output(['git', 'rev-parse', 'HEAD^{tree}'], cwd=self.root, text=True).strip() + with self.assertRaisesRegex(ValueError, 'file hashes'): + components.unpack(corrupt, tree, target) + + def test_archive_cannot_overwrite_source_through_traversal(self): + archive = self.root / 'bad.tar.gz' + with tarfile.open(archive, mode='w:gz') as tar: + name = 'app/src/main/assets/linuxfs/../../../../../input.h' + entry = tarfile.TarInfo(name) + entry.size = 3 + tar.addfile(entry, io.BytesIO(b'bad')) + with self.assertRaisesRegex(ValueError, 'Unsafe'): + components.unpack(archive, 'tree', self.root) + self.assertEqual((self.root / 'input.h').read_text(), '#define VALUE 1\n') diff --git a/tools/tests/test_esync_packs.py b/tools/tests/test_esync_packs.py index 863a1003b..7ba20b448 100644 --- a/tools/tests/test_esync_packs.py +++ b/tools/tests/test_esync_packs.py @@ -1160,8 +1160,12 @@ class BundleTest(unittest.TestCase): except ImportError: self.skipTest("PyYAML is not installed") apk = yaml.safe_load((self.ROOT / ".github/workflows/build.yml").read_text()) - steps = {step.get("name"): step for step in apk["jobs"]["build"]["steps"]} - bundle = steps["Bundle the droiddeck-esync packs"] + steps = {step.get("name"): step for step in apk["jobs"]["components"]["steps"]} + import json + recipes = json.loads((self.ROOT / "tools/components/recipes.json").read_text()) + bundle = dict(steps["Bundle the droiddeck-esync packs"]) + self.assertIn("prepare.py esync", bundle["run"]) + bundle["run"] = (self.ROOT / recipes["esync"]["script"]).read_text() self.assertNotIn("if", bundle) self.assertIn("tools/droiddeck-esync/release.env", bundle["run"]) self.assertIn('release_url="https://github.com/${SYNC_BUNDLE_REPO}/releases/download/${SYNC_BUNDLE_TAG}"', bundle["run"]) @@ -1169,7 +1173,7 @@ class BundleTest(unittest.TestCase): self.assertIn('curl -fsSL --retry 3 -o sync-index.json "$release_url/index.json"', bundle["run"]) self.assertIn("sha256sum -c -", bundle["run"]) self.assertNotIn("revoked.txt", bundle["run"]) - stage = next(step for step in apk["jobs"]["build"]["steps"] if "overlay/usr/local/bin/droiddeck-*" in step.get("run", "")) + stage = {"run": (self.ROOT / recipes["runtime"]["script"]).read_text()} self.assertIn("tools/linuxfs/overlay/usr/local/bin/droiddeck-* ", stage["run"]) local = (self.ROOT / "tools/build_local.sh").read_text() self.assertIn("tools/linuxfs/overlay/usr/local/bin/droiddeck-* ", local) diff --git a/tools/tests/test_preload_snapshots.py b/tools/tests/test_preload_snapshots.py index 5a4cfb606..ee03da0f8 100644 --- a/tools/tests/test_preload_snapshots.py +++ b/tools/tests/test_preload_snapshots.py @@ -100,6 +100,13 @@ class PreloadSnapshotTest(unittest.TestCase): result, _, _ = self.restore() self.assertEqual(result, dict(arm64=False, x86=True)) + def test_shell_newline_changes_invalidate_the_recipe(self): + script = self.root / 'tools/components/build-arm64-preloads.sh' + original = snapshots.recipe(self.root) + text = script.read_text() + script.write_text(text.replace('-ldl\n', '-ldl ', 1)) + self.assertNotEqual(original, snapshots.recipe(self.root)) + def test_new_manifest_requires_main_build_proof(self): result, _, prove = self.restore('HEAD~1') self.assertTrue(all(result.values())) @@ -112,6 +119,18 @@ class PreloadSnapshotTest(unittest.TestCase): with self.assertRaisesRegex(ValueError, 'original main APK'): snapshots.prove_origin(data, self.root / 'cache', self.root) + def test_modified_input_policy_cannot_omit_producer_dependencies(self): + data = copy.deepcopy(self.data) + data['origin']['commit'] = self.base + omitted = snapshots.COMMON[0] + del data['groups']['arm64'][omitted] + with patch.object(snapshots, 'artifact_bytes', return_value=self.binaries), \ + patch.object(snapshots, 'run', return_value=b''), \ + patch.object(snapshots, 'source_files', side_effect=lambda group, *args: data['groups'][group]), \ + patch.object(snapshots, 'producer_sources', return_value={omitted: self.data['groups']['arm64'][omitted]}): + with self.assertRaisesRegex(ValueError, 'producing build dependency'): + snapshots.prove_origin(data, self.root / 'cache', self.root) + def test_non_main_or_unsuccessful_run_cannot_prove_origin(self): build = dict(event='pull_request', head_branch='main', conclusion='success', head_sha='abc', path='.github/workflows/build.yml', repository=dict(full_name=snapshots.REPO)) @@ -139,6 +158,18 @@ class PreloadSnapshotTest(unittest.TestCase): patch.object(snapshots.urllib.request, 'urlopen', side_effect=error): self.assertEqual(snapshots.restore(self.root), dict(arm64=False, x86=False)) + def test_transient_download_failure_rebuilds_but_integrity_errors_fail(self): + errors = [TimeoutError('network'), snapshots.http.client.IncompleteRead(b'partial'), + snapshots.urllib.error.HTTPError('https://example.test', 503, 'unavailable', {}, None)] + for error in errors: + with self.subTest(error=type(error).__name__), patch.dict(os.environ, COMPONENT_TRUSTED_BASE=self.base), \ + patch.object(snapshots.urllib.request, 'urlopen', side_effect=error): + self.assertEqual(snapshots.restore(self.root), dict(arm64=False, x86=False)) + with patch.dict(os.environ, COMPONENT_TRUSTED_BASE=self.base), \ + patch.object(snapshots.urllib.request, 'urlopen', return_value=io.BytesIO(b'corrupt')): + with self.assertRaisesRegex(ValueError, 'checksum'): + snapshots.restore(self.root) + def test_manual_rebuild_does_not_download_snapshot(self): with patch.dict(os.environ, FORCE_PRELOAD_REBUILD='true'), \ patch.object(snapshots.urllib.request, 'urlopen') as fetch: diff --git a/tools/tests/test_xbox_build_source.py b/tools/tests/test_xbox_build_source.py index 15e35ca5e..427cea342 100644 --- a/tools/tests/test_xbox_build_source.py +++ b/tools/tests/test_xbox_build_source.py @@ -31,14 +31,14 @@ class XboxBuildSourceTest(unittest.TestCase): git('switch', '-q', 'main') (root / 'backend').write_text('merged backend') git('commit', '-qam', 'backend changed after branch') - for name in ('xbox', 'apk', 'tests'): + for name in ('source', 'xbox', 'components', 'apk', 'tests', 'dispatch'): git('clone', '-q', str(root), str(root / name)) git('checkout', '-q', head, cwd=root / name) output = root / 'output' env['GITHUB_OUTPUT'] = str(output) - subprocess.run(['bash', '-c', step('xbox', 'Pin and merge the APK source')], cwd=root / 'xbox', env=env, check=True, capture_output=True) + subprocess.run(['bash', '-c', step('source', 'Pin and merge the APK source')], cwd=root / 'source', env=env, check=True, capture_output=True) base = output.read_text().strip().split('=', 1)[1] - tree = git('rev-parse', 'HEAD^{tree}', cwd=root / 'xbox') + tree = git('rev-parse', 'HEAD^{tree}', cwd=root / 'source') (root / 'backend').write_text('newer main, after backend build') git('commit', '-qam', 'main advanced again') env.update(BUILD_BASE_SHA=base, XBOX_SOURCE_TREE=tree) @@ -50,3 +50,15 @@ class XboxBuildSourceTest(unittest.TestCase): subprocess.run(['bash', '-c', step('python-tests', 'Merge the pinned APK base')], cwd=root / 'tests', env=env, check=True, capture_output=True) subprocess.run(['bash', '-c', step('python-tests', 'Verify backend and test source match')], cwd=root / 'tests', env=env, check=True, capture_output=True) self.assertEqual(git('rev-parse', 'HEAD^{tree}', cwd=root / 'tests'), tree) + + env['SOURCE_TREE'] = tree + for job in ('xbox', 'components'): + subprocess.run(['bash', '-c', step(job, 'Merge the pinned APK base')], cwd=root / job, env=env, check=True, capture_output=True) + subprocess.run(['bash', '-c', step(job, 'Verify pinned source tree')], cwd=root / job, env=env, check=True, capture_output=True) + self.assertEqual(git('rev-parse', 'HEAD^{tree}', cwd=root / job), tree) + + trusted = root / 'trusted-output' + env['GITHUB_OUTPUT'] = str(trusted) + subprocess.run(['bash', '-c', step('source', 'Resolve trusted manifest base')], cwd=root / 'dispatch', env=env, check=True, capture_output=True) + self.assertEqual(trusted.read_text().strip(), 'base-sha=' + git('rev-parse', 'main')) + self.assertNotEqual(git('rev-parse', 'HEAD', cwd=root / 'dispatch'), git('rev-parse', 'main'))