ci: consolidate verified component preparation

This commit is contained in:
XxJSONDeruloxX committed 2026-10-23 14:31:13 -04:00
1 parent d111b2b3dd
commit d7bf0b274f
31 files changed
+1269 -503

No files matched your search

+1 -1
View File
@@ -23,7 +23,7 @@ permissions:
jobs:
validate:
name: Build APK
name: Validate source and build result
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
+249 -469
View File
@@ -31,17 +31,14 @@ concurrency:
group: build-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
PRELOAD_CACHE_VERSION: v8
jobs:
xbox:
name: Xbox ARM64 backend
source:
name: Pin source tree
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
runs-on: ubuntu-24.04-arm
timeout-minutes: 30
runs-on: ubuntu-24.04
timeout-minutes: 5
outputs:
base-sha: ${{ steps.merge.outputs.base-sha }}
base-sha: ${{ steps.merge.outputs.base-sha || steps.trusted.outputs.base-sha }}
tree-sha: ${{ steps.source.outputs.tree-sha }}
steps:
- uses: actions/checkout@v4
@@ -59,9 +56,46 @@ jobs:
git config user.name "ci"
git config user.email "ci@users.noreply.github.com"
git merge --no-ff --no-edit origin/main
- name: Record backend source tree
- name: Resolve trusted manifest base
id: trusted
if: github.event_name != 'pull_request'
run: |
set -euo pipefail
git fetch --no-tags origin main
echo "base-sha=$(git rev-parse origin/main)" >> "$GITHUB_OUTPUT"
- name: Record pinned source tree
id: source
run: echo "tree-sha=$(git rev-parse 'HEAD^{tree}')" >> "$GITHUB_OUTPUT"
xbox:
name: Xbox ARM64 backend
needs: source
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
runs-on: ubuntu-24.04-arm
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Merge the pinned APK base
if: github.event_name == 'pull_request'
env:
BUILD_BASE_SHA: ${{ needs.source.outputs.base-sha }}
run: |
set -euo pipefail
git fetch --no-tags origin "$BUILD_BASE_SHA"
git config user.name "ci"
git config user.email "ci@users.noreply.github.com"
git merge --no-ff --no-edit FETCH_HEAD
- name: Verify pinned source tree
env:
SOURCE_TREE: ${{ needs.source.outputs.tree-sha }}
run: test "$(git rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE"
- name: Plan component identities
id: plan
run: python3 tools/components/prepare.py plan
- uses: actions/cache@v4
id: xbox-cache
with:
@@ -69,13 +103,10 @@ jobs:
app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox
app/build/xbox/xodus-droiddeck-source.tar.xz
app/build/xbox/runtime/*-source.tar.xz
key: xbox-arm64-${{ hashFiles('tools/xbox/**') }}
key: xbox-v2-${{ steps.plan.outputs.xbox }}
- name: Build Xbox backend
if: steps.xbox-cache.outputs.cache-hit != 'true'
run: |
rustup toolchain install 1.99.0 --profile minimal
bash tools/ci/install-ubuntu-packages.sh build-essential pkg-config libssl-dev protobuf-compiler cmake xz-utils
bash tools/xbox/build.sh
run: python3 tools/components/prepare.py xbox
- uses: actions/upload-artifact@v4
with:
name: xbox-arm64
@@ -90,8 +121,157 @@ jobs:
if-no-files-found: error
retention-days: 14
components:
name: Prepare verified components
needs: source
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
runs-on: ubuntu-24.04
timeout-minutes: 40
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Merge the pinned APK base
if: github.event_name == 'pull_request'
env:
BUILD_BASE_SHA: ${{ needs.source.outputs.base-sha }}
run: |
set -euo pipefail
git fetch --no-tags origin "$BUILD_BASE_SHA"
git config user.name "ci"
git config user.email "ci@users.noreply.github.com"
git merge --no-ff --no-edit FETCH_HEAD
- name: Verify pinned source tree
env:
SOURCE_TREE: ${{ needs.source.outputs.tree-sha }}
run: test "$(git rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE"
- name: Resolve the pinned Android NDK
run: bash tools/ci/resolve-ndk.sh
- name: Plan component identities
id: plan
run: python3 tools/components/prepare.py plan
- name: Cache remaining Linux runtime assets
id: preloads
uses: actions/cache@v4
with:
path: |
app/src/main/assets/linuxfs
!app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet
!app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox
!app/src/main/assets/linuxfs/i386/libblsession.so
!app/src/main/assets/linuxfs/i386/libfakeinput.so
!app/src/main/assets/linuxfs/i386/libfaultreport.so
!app/src/main/assets/linuxfs/libblaudit.so
!app/src/main/assets/linuxfs/libblfastpath.so
!app/src/main/assets/linuxfs/libblsession.so
!app/src/main/assets/linuxfs/libfakeinput.so
!app/src/main/assets/linuxfs/libssbs.so
!app/src/main/assets/linuxfs/usr/local/bin/droiddeck-clipboard
!app/src/main/assets/linuxfs/x86_64/libblsession.so
!app/src/main/assets/linuxfs/x86_64/libfakeinput.so
!app/src/main/assets/linuxfs/x86_64/libfaultreport.so
!app/src/main/assets/linuxfs/x86_64/libvulkan-thunk.so
!app/src/main/assets/linuxfs/x86_64/libwebkitinput.so
key: runtime-v9-${{ steps.plan.outputs.runtime }}
- name: Cache PulseAudio 13.0 source
id: pa-src
uses: actions/cache@v4
with:
path: pa13
key: pulseaudio-src-${{ steps.plan.outputs.pulseaudio-source }}
- name: Cache proot
id: proot
uses: actions/cache@v4
with:
path: |
app/src/main/jniLibs/arm64-v8a/libproot.so
app/src/main/jniLibs/arm64-v8a/libproot-loader.so
key: proot-v2-${{ steps.plan.outputs.proot }}
- name: Cache GOG Galaxy companion
id: comet
uses: actions/cache@v4
with:
path: app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet
key: comet-v2-${{ steps.plan.outputs.comet }}
- name: Restore verified preload snapshots
id: component-preloads
env:
GH_TOKEN: ${{ github.token }}
FORCE_PRELOAD_REBUILD: ${{ inputs.rebuild_preloads || 'false' }}
COMPONENT_TRUSTED_BASE: ${{ needs.source.outputs.base-sha }}
run: python3 tools/components/prepare.py preload-snapshot
- name: Prepare required Linux build tools once
env:
COMPONENTS_TO_BUILD: >-
${{ steps.preloads.outputs.cache-hit != 'true' && 'runtime' || '' }}
${{ steps.component-preloads.outputs.arm64 != 'true' && 'arm64-preloads' || '' }}
${{ steps.comet.outputs.cache-hit != 'true' && 'comet' || '' }}
run: python3 tools/components/prepare.py tools
- name: Build changed ARM64 preloads
if: steps.component-preloads.outputs.arm64 != 'true'
run: python3 tools/components/prepare.py arm64-preloads
- name: Prepare remaining Linux runtime assets
if: steps.preloads.outputs.cache-hit != 'true'
run: python3 tools/components/prepare.py runtime
- name: Fetch the Debian image the preloads build in
if: steps.component-preloads.outputs.x86 != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: python3 tools/components/prepare.py debian-image
- name: Build the x86 preloads FEX guests load (glibc 2.31)
if: steps.component-preloads.outputs.x86 != 'true'
run: python3 tools/components/prepare.py x86-preloads
- name: Bundle the droiddeck-esync packs
run: python3 tools/components/prepare.py esync
- name: Fetch PulseAudio 13.0 source
if: steps.pa-src.outputs.cache-hit != 'true'
run: python3 tools/components/prepare.py pulseaudio-source
- name: Fetch DirectAudio (tools/directaudio/release.env)
run: python3 tools/components/prepare.py directaudio
- name: Build the AAudio sink into the audio bundle
run: python3 tools/components/prepare.py audio
- name: Build proot
if: steps.proot.outputs.cache-hit != 'true'
run: python3 tools/components/prepare.py proot
- name: Check the preloads
run: |
set -euo pipefail
bash tools/linuxfs/check-preloads.sh app/src/main/assets/linuxfs
- name: Build GOG Galaxy companion
if: steps.comet.outputs.cache-hit != 'true'
run: python3 tools/components/prepare.py comet
- name: Prepare graphics driver variants
run: python3 tools/quest/build-driver.py "$ANDROID_HOME/ndk/$NDK_VERSION"
- name: Record and pack prepared components
run: python3 tools/components/prepare.py pack
- uses: actions/upload-artifact@v4
with:
name: prepared-components
path: app/build/prepared-components.tar.gz
if-no-files-found: error
compression-level: 0
retention-days: 14
build:
needs: xbox
needs: [source, components, xbox]
name: Compile APK and Android tests
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
runs-on: ubuntu-24.04
@@ -155,7 +335,7 @@ jobs:
- name: Merge onto main, or say what conflicts
if: github.event_name == 'pull_request'
env:
BUILD_BASE_SHA: ${{ needs.xbox.outputs.base-sha }}
BUILD_BASE_SHA: ${{ needs.source.outputs.base-sha }}
run: |
set -uo pipefail
git config user.name "ci"
@@ -213,7 +393,7 @@ jobs:
- name: Verify backend and APK source match
env:
XBOX_SOURCE_TREE: ${{ needs.xbox.outputs.tree-sha }}
XBOX_SOURCE_TREE: ${{ needs.source.outputs.tree-sha }}
run: test "$(git rev-parse 'HEAD^{tree}')" = "$XBOX_SOURCE_TREE"
- uses: actions/setup-java@v4
@@ -225,11 +405,8 @@ jobs:
with:
cache-cleanup: on-success
- name: Resolve the runner's NDK
run: |
NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"
VERSION=$(sed -n 's/^Pkg.Revision *= *//p' "$NDK/source.properties")
echo "NDK_VERSION=$VERSION" >> "$GITHUB_ENV"
- name: Resolve the pinned Android NDK
run: bash tools/ci/resolve-ndk.sh
- name: Set the APK build label
id: build-label
@@ -255,274 +432,6 @@ jobs:
echo "DROIDDECK_PR_NUMBER=${PR_NUMBER:-0}" >> "$GITHUB_ENV"
echo "Build: \`$label\`" >> "$GITHUB_STEP_SUMMARY"
- name: Cache the preloads
id: preloads
uses: actions/cache@v4
with:
path: |
app/src/main/assets/linuxfs
!app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet
!app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox
!app/src/main/assets/linuxfs/i386/libblsession.so
!app/src/main/assets/linuxfs/i386/libfakeinput.so
!app/src/main/assets/linuxfs/i386/libfaultreport.so
!app/src/main/assets/linuxfs/libblaudit.so
!app/src/main/assets/linuxfs/libblfastpath.so
!app/src/main/assets/linuxfs/libblsession.so
!app/src/main/assets/linuxfs/libfakeinput.so
!app/src/main/assets/linuxfs/libssbs.so
!app/src/main/assets/linuxfs/usr/local/bin/droiddeck-clipboard
!app/src/main/assets/linuxfs/x86_64/libblsession.so
!app/src/main/assets/linuxfs/x86_64/libfakeinput.so
!app/src/main/assets/linuxfs/x86_64/libfaultreport.so
!app/src/main/assets/linuxfs/x86_64/libvulkan-thunk.so
!app/src/main/assets/linuxfs/x86_64/libwebkitinput.so
key: preloads-${{ env.PRELOAD_CACHE_VERSION }}-${{ hashFiles('app/src/main/cpp/adrenotools/src/hook/kgsl.h', 'tools/quest/*', 'tools/linuxfs/overlay/usr/local/bin/*', 'tools/linuxfs/overlay/usr/bin/**', 'tools/linuxfs/desktop/*', 'tools/gamescope/release.env', 'tools/wlroots/release.env', 'tools/linuxfs/uruntime.env', 'tools/linuxfs/licenses/*', 'tools/mangoapp/*', 'tools/msitools/**', 'tools/pipewire/**', 'tools/ci/install-ubuntu-packages.sh', 'tools/vr/**') }}
- name: Restore verified preload snapshots
id: component-preloads
env:
GH_TOKEN: ${{ github.token }}
FORCE_PRELOAD_REBUILD: ${{ inputs.rebuild_preloads || 'false' }}
COMPONENT_TRUSTED_BASE: ${{ needs.xbox.outputs.base-sha || github.sha }}
run: python3 tools/components/preloads.py restore
- name: Build changed ARM64 preloads
if: steps.component-preloads.outputs.arm64 != 'true'
run: |
bash tools/ci/install-ubuntu-packages.sh gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
bash tools/components/build-arm64-preloads.sh app/src/main/assets/linuxfs
- name: Prepare remaining Linux runtime assets
if: steps.preloads.outputs.cache-hit != 'true'
run: |
set -euo pipefail
bash tools/ci/install-ubuntu-packages.sh gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libvulkan1 mesa-vulkan-drivers
D=app/src/main/assets/linuxfs
mkdir -p "$D"
VK_INCLUDE=$(ls -d "${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1) \
tools/vr/build.sh --native "$D/usr/local/lib/droiddeck-vr"
for script in tools/linuxfs/overlay/usr/local/bin/droiddeck-* tools/linuxfs/overlay/usr/local/bin/steam-compatibility; do
install -Dm644 "$script" "$D/usr/local/bin/$(basename "$script")"
done
for f in tools/linuxfs/overlay/usr/bin/* tools/linuxfs/overlay/usr/bin/steamos-polkit-helpers/*; do
[ -f "$f" ] && install -Dm644 "$f" "$D/${f#tools/linuxfs/overlay/}"
done
if [ -f tools/gamescope/release.env ]; then
. tools/gamescope/release.env
curl -fsSL --retry 3 -o gamescope.tzst "https://github.com/${GAMESCOPE_REPO:-${GITHUB_REPOSITORY}}/releases/download/${GAMESCOPE_TAG}/gamescope.tzst"
echo "${GAMESCOPE_SHA256} gamescope.tzst" | sha256sum -c -
tar --use-compress-program=unzstd -xf gamescope.tzst -C "$D"
test -f "$D/usr/local/bin/gamescope"
rm gamescope.tzst
fi
. tools/linuxfs/uruntime.env
curl -fsSL --retry 3 -o uruntime "https://github.com/VHSgunzo/uruntime/releases/download/${URUNTIME_VERSION}/${URUNTIME_ASSET}"
echo "${URUNTIME_SHA256} uruntime" | sha256sum -c -
install -Dm644 uruntime "$D/usr/local/lib/droiddeck/uruntime"
install -Dm644 tools/linuxfs/licenses/uruntime-LICENSE "$D/usr/local/share/licenses/uruntime/LICENSE"
rm uruntime
M="$D/usr/local/lib/mangoapp"
mkdir -p "$M" mango-pkgs
grep -v '^#' tools/mangoapp/packages.txt | while read -r sum url; do
[ -n "$url" ] || continue
curl -fsSL --retry 3 -o mango-pkgs/pkg.tar.zst "$url"
echo "$sum mango-pkgs/pkg.tar.zst" | sha256sum -c -
tar --use-compress-program=unzstd -xf mango-pkgs/pkg.tar.zst -C mango-pkgs
rm mango-pkgs/pkg.tar.zst
done
install -m644 mango-pkgs/usr/bin/mangoapp "$M/mangoapp"
for so in libfmt.so.10 libspdlog.so.1.13 libglfw.so.3 libtraceevent.so.1; do
cp -L "mango-pkgs/usr/lib/$so" "$M/$so"
done
aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -Wl,-soname,libtracefs.so.1 \
-o "$M/libtracefs.so.1" tools/mangoapp/libtracefs-shim.c
aarch64-linux-gnu-strip --strip-unneeded "$M/libtracefs.so.1"
install -Dm644 tools/mangoapp/mangoapp "$D/usr/local/bin/mangoapp"
rm -rf mango-pkgs
ls -l "$M"
mkdir -p pw-pkgs
grep -v '^#' tools/pipewire/packages.txt | while read -r sum url; do
[ -n "$url" ] || continue
curl -fsSL --retry 3 -o pw-pkgs/pkg.tar.zst "$url"
echo "$sum pw-pkgs/pkg.tar.zst" | sha256sum -c -
tar --use-compress-program=unzstd -xf pw-pkgs/pkg.tar.zst -C pw-pkgs
rm pw-pkgs/pkg.tar.zst
done
tools/pipewire/assemble.sh pw-pkgs "$D/usr/local/lib/droiddeck-pipewire.tzst"
rm -rf pw-pkgs
ls -l "$D/usr/local/lib/droiddeck-pipewire.tzst"
T="$D/usr/local/lib/droiddeck-msitools"
mkdir -p "$T" msi-pkgs
grep -v '^#' tools/msitools/packages.txt | while read -r sum url; do
[ -n "$url" ] || continue
curl -fsSL --retry 3 -o msi-pkgs/pkg.tar.xz "$url"
echo "$sum msi-pkgs/pkg.tar.xz" | sha256sum -c -
tar -xJf msi-pkgs/pkg.tar.xz -C msi-pkgs
rm msi-pkgs/pkg.tar.xz
done
install -m644 msi-pkgs/usr/bin/msiinfo "$T/msiinfo"
install -m644 msi-pkgs/usr/bin/cabextract "$T/cabextract"
install -m644 msi-pkgs/usr/lib/7zip/7z "$T/7z"
install -m644 msi-pkgs/usr/lib/7zip/7z.so "$T/7z.so"
for so in libmsi-1.0.so.0 libgsf-1.so.114 libgcab-1.0.so.0; do
cp -L "msi-pkgs/usr/lib/$so" "$T/$so"
done
for f in NOTICE GPL-2 GPL-3 LGPL-2.1 7zip-License 7zip-unRarLicense; do
install -Dm644 "tools/msitools/licenses/$f" "$D/usr/local/share/licenses/droiddeck-msitools/$f"
done
rm -rf msi-pkgs
ls -l "$T"
install -Dm644 tools/linuxfs/desktop/droiddeck-desktop "$D/usr/local/bin/droiddeck-desktop"
install -Dm644 tools/linuxfs/desktop/droiddeck-gpu "$D/usr/local/bin/droiddeck-gpu"
install -Dm644 tools/linuxfs/desktop/droiddeck-desktop-gpu "$D/usr/local/bin/droiddeck-desktop-gpu"
install -Dm644 tools/linuxfs/desktop/kwin_wayland_wrapper "$D/usr/local/bin/kwin_wayland_wrapper"
install -Dm644 tools/linuxfs/desktop/firefox-droiddeck.js "$D/usr/lib/firefox/defaults/pref/droiddeck.js"
ls -lR "$D"
- name: Fetch the Debian image the preloads build in
if: steps.component-preloads.outputs.x86 != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
. tools/linuxfs/debian-image.env
image=public.ecr.aws/docker/library/debian:bullseye
if gh release download "$DEBIAN_IMAGE_TAG" -R "$GITHUB_REPOSITORY" -p "$DEBIAN_IMAGE_ASSET" -O /tmp/debian-image.tar.zst \
&& { [ -z "$DEBIAN_IMAGE_SHA256" ] || echo "$DEBIAN_IMAGE_SHA256 /tmp/debian-image.tar.zst" | sha256sum -c -; }; then
zstd -dc /tmp/debian-image.tar.zst | docker load
rm -f /tmp/debian-image.tar.zst
else
echo "::warning::the hosted Debian image could not be fetched; pulling from the registry instead"
for attempt in 1 2 3 4 5; do
docker pull "$image" && break
[ "$attempt" -lt 5 ] || { echo "::error::could not pull $image after $attempt attempts"; exit 1; }
echo "pull attempt $attempt failed (registry busy?); retrying in $((attempt * 30)) s"
sleep $((attempt * 30))
done
fi
docker image ls "$image"
- name: Build the x86 preloads FEX guests load (glibc 2.31)
if: steps.component-preloads.outputs.x86 != 'true'
run: |
set -euo pipefail
docker run --rm -v "$PWD:/src" -w /src public.ecr.aws/docker/library/debian:bullseye bash -c '
set -euo pipefail
install_compilers() {
apt-get -o Acquire::Check-Valid-Until=false update -qq &&
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends gcc g++ gcc-multilib g++-multilib binutils >/dev/null
}
printf "deb http://deb.debian.org/debian bullseye main\ndeb http://security.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list
if ! install_compilers; then
printf "deb http://archive.debian.org/debian bullseye main\ndeb http://archive.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list
install_compilers
fi
tools/linuxfs/build-x86-preloads.sh app/src/main/assets/linuxfs
'
sudo chown -R "$(id -u):$(id -g)" app/src/main/assets/linuxfs
ls -l app/src/main/assets/linuxfs/x86_64 app/src/main/assets/linuxfs/i386
- name: Bundle the droiddeck-esync packs
run: |
set -euo pipefail
if [ -f tools/droiddeck-esync/release.env ]; then
. tools/droiddeck-esync/release.env
release_url="https://github.com/${SYNC_BUNDLE_REPO}/releases/download/${SYNC_BUNDLE_TAG}"
curl -fsSL --retry 3 -o sync-bundle.tzst "$release_url/$SYNC_BUNDLE_ASSET"
echo "$SYNC_BUNDLE_SHA256 sync-bundle.tzst" | sha256sum -c -
mkdir -p app/src/main/assets/droiddeck-esync
tar --use-compress-program=unzstd -xf sync-bundle.tzst -C app/src/main/assets/droiddeck-esync
rm sync-bundle.tzst
curl -fsSL --retry 3 -o sync-index.json "$release_url/index.json"
revoked=$(python3 -c 'import json, sys; print(" ".join(p["id"] for p in json.load(open(sys.argv[1]))["packs"] if p.get("revoked") is True))' sync-index.json)
for id in $revoked; do
if [ -e "app/src/main/assets/droiddeck-esync/packs/$id.tzst" ]; then
echo "::warning::$SYNC_BUNDLE_ASSET carries revoked pack $id; it is left out of the APK (point tools/droiddeck-esync/release.env at a newer bundle)"
rm -f "app/src/main/assets/droiddeck-esync/packs/$id.tzst"
fi
done
rm sync-index.json
test -f app/src/main/assets/droiddeck-esync/index.json
test -f app/src/main/assets/droiddeck-esync/index.json.sig
ls -lR app/src/main/assets/droiddeck-esync
fi
- name: Cache PulseAudio 13.0 source
id: pa-src
uses: actions/cache@v4
with:
path: pa13
key: pulseaudio-src-13.0
- name: Fetch PulseAudio 13.0 source
if: steps.pa-src.outputs.cache-hit != 'true'
run: |
set -euo pipefail
curl -fsSL -o pa.tar.gz https://github.com/pulseaudio/pulseaudio/archive/refs/tags/v13.0.tar.gz
mkdir -p pa13 && tar -xzf pa.tar.gz -C pa13 --strip-components=1 && rm pa.tar.gz
- name: Fetch DirectAudio (tools/directaudio/release.env)
run: tools/directaudio/fetch.sh "$PWD" "$PWD/sink-out"
- name: Build the AAudio sink into the audio bundle
run: |
set -euo pipefail
export NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"
tools/aaudio-sink/build.sh "$PWD/pa13" "$PWD/sink-out"
rm -rf pa-bundle && mkdir -p pa-bundle
tar --use-compress-program=unzstd -xf app/src/main/assets/pulseaudio.tzst -C pa-bundle
test ! -f pa-bundle/modules/arm64/module-aaudio-sink.so
install -m755 sink-out/module-aaudio-sink.so pa-bundle/modules/arm64/module-aaudio-sink.so
install -m755 sink-out/module-directaudio-native-sink.so pa-bundle/modules/arm64/module-directaudio-native-sink.so
(cd pa-bundle && tar --use-compress-program='zstd -19' -cf ../pulseaudio.tzst .)
mkdir -p app/build/prepared-assets
mv pulseaudio.tzst app/build/prepared-assets/pulseaudio.tzst
tar --use-compress-program=unzstd -tf app/build/prepared-assets/pulseaudio.tzst | grep -c '\.so$'
rm -rf pa-bundle sink-out
- name: Cache proot
id: proot
uses: actions/cache@v4
with:
path: |
app/src/main/jniLibs/arm64-v8a/libproot.so
app/src/main/jniLibs/arm64-v8a/libproot-loader.so
key: proot-${{ env.NDK_VERSION }}-${{ hashFiles('tools/proot/**') }}
- name: Build proot
if: steps.proot.outputs.cache-hit != 'true'
run: |
set -euo pipefail
export NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"
tools/proot/build.sh app/src/main/jniLibs/arm64-v8a
- name: Check the preloads
run: |
set -euo pipefail
bash tools/linuxfs/check-preloads.sh app/src/main/assets/linuxfs
- name: Check Quest tracking IPC
run: |
set -euo pipefail
NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"
VK_INCLUDE=$(ls -d "$NDK"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1)
mkdir -p build/vr/include
cp -R "$VK_INCLUDE/vulkan" build/vr/include/
if [ -d "$VK_INCLUDE/vk_video" ]; then cp -R "$VK_INCLUDE/vk_video" build/vr/include/; fi
g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_tracking.cpp -o /tmp/test-xr-tracking
/tmp/test-xr-tracking
g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_xr_touch.cpp -o /tmp/test-xr-touch
/tmp/test-xr-touch
g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_pad_controller_config.cpp -o /tmp/test-pad-controller-config
/tmp/test-pad-controller-config
g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_vsync.cpp -o /tmp/test-xr-vsync
/tmp/test-xr-vsync
g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_frames.cpp -o /tmp/test-xr-frames
/tmp/test-xr-frames
cc -Itools/vr/third_party/wayland -c tools/vr/src/wayland_client_stub.c -o /tmp/test-wayland-stub.o
cc -Itools/vr/third_party/wayland -c app/src/main/cpp/waylandcomp/generated/xdg-shell-protocol.c -o /tmp/test-xdg-shell.o
g++ -std=c++17 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-missing-field-initializers -pthread -Ibuild/vr/include \
-Itools/vr/third_party/wayland -Iapp/src/main/cpp/waylandcomp/generated \
tools/vr/tests/test_presenter_shutdown.cpp tools/vr/src/presenter.cpp /tmp/test-wayland-stub.o /tmp/test-xdg-shell.o \
-ldl -o /tmp/test-presenter-shutdown
timeout 10 /tmp/test-presenter-shutdown
- name: Rust toolchain for the store engines (aarch64-linux-android)
id: rust
run: |
@@ -561,28 +470,20 @@ jobs:
restore-keys: |
cxx-${{ env.NDK_VERSION }}-
- uses: actions/download-artifact@v4
with:
name: prepared-components
path: app/build
- name: Verify and unpack prepared components
env:
SOURCE_TREE: ${{ needs.source.outputs.tree-sha }}
run: python3 tools/components/prepare.py unpack --tree "$SOURCE_TREE"
- uses: actions/download-artifact@v4
with:
name: xbox-arm64
path: app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox
- name: Cache GOG Galaxy companion
id: comet
uses: actions/cache@v4
with:
path: app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet
key: comet-${{ runner.os }}-${{ runner.arch }}-rust1.90.0-${{ hashFiles('tools/comet/**') }}
- name: Build GOG Galaxy companion
if: steps.comet.outputs.cache-hit != 'true'
run: |
if ! command -v aarch64-linux-gnu-gcc >/dev/null; then
bash tools/ci/install-ubuntu-packages.sh gcc-aarch64-linux-gnu
fi
rustup toolchain install 1.90.0 --profile minimal
rustup target add --toolchain 1.90.0 aarch64-unknown-linux-gnu
RUSTUP_TOOLCHAIN=1.90.0 bash tools/comet/build.sh
- name: Build the add-on packs
run: bash tools/packs/build.sh app/build/packs
@@ -593,7 +494,6 @@ jobs:
SGDB_API_KEY: ${{ secrets.SGDB_API_KEY }}
run: |
set -o pipefail
python3 tools/quest/build-driver.py "$ANDROID_HOME/ndk/$NDK_VERSION"
python3 tools/release/runtime_inputs.py record
./gradlew assembleRelease --console=plain -PndkVersion="$NDK_VERSION" 2>&1 | tee /tmp/build.log
python3 tools/release/check_session_assets.py app/build/outputs/apk/release/app-release.apk
@@ -701,87 +601,11 @@ jobs:
TEST_DIAGNOSTICS_URL: ${{ steps.unit-test-diagnostics.outputs.artifact-url }}
APK_OUTCOME: ${{ steps.apk-upload.outcome }}
SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
run: |
set -uo pipefail
MARK="<!-- droiddeck-pr-build -->"
{
echo "$MARK"
echo "## Build of this pull request"
echo
[ -f /tmp/pr-merge.md ] && cat /tmp/pr-merge.md
echo
case "$BUILD_OUTCOME:$UNIT_TEST_OUTCOME:$APK_OUTCOME" in
success:success:success)
echo "### Build passed"
echo
echo "The \`droiddeck-apk\` artifact is [on this run]($RUN_URL), signed with the public test key."
echo
if [ "$SAME_REPO" = true ]; then
echo "After the parallel regression tests pass, a release-signed tester build that installs over Stable or Preview will be published and will appear in DroidDeck's **Test builds** list. Label the PR \`no-test-build\` to skip that."
else
echo "**Maintainer:** add the \`publish-test\` label to publish a release-signed tester build that installs over Stable or Preview and appears in DroidDeck's **Test builds** list. The label is removed automatically after publishing."
fi
echo
echo "Fallback: [open **PR Release Publish**]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/workflows/pr-release-publish.yml) on \`main\` and enter PR \`$PR\`."
;;
success:success:*)
echo "### APK publication incomplete"
echo
echo "Compilation and unit tests passed, but APK publication did not complete. See [the run]($RUN_URL)."
;;
success:failure:*)
echo "### ERROR: Unit tests failed"
echo
echo "The APK compiled, but unit tests failed. No installable artifact was published."
echo
if [ -n "$TEST_DIAGNOSTICS_URL" ]; then
echo "The [unit-test-results]($TEST_DIAGNOSTICS_URL) artifact contains the test log and any generated reports."
else
echo "The test diagnostics upload did not complete. See [the test log]($RUN_URL)."
fi
;;
success:*)
echo "### Build incomplete"
echo
echo "The APK compiled, but unit tests did not complete successfully. See [the run]($RUN_URL)."
;;
failure:*)
echo "### ERROR: Build failed"
echo
echo "The compiler's errors ([full log]($RUN_URL)):"
echo
echo '```'
{ grep -E '^e: |^w: .*error|error:|CMake Error|fatal error|ninja: error|FAILED|What went wrong|Execution failed for task|Unresolved reference|Type mismatch|Unresolved' /tmp/build.log 2>/dev/null \
| sed -e 's#file:///home/runner/work/[^/]*/[^/]*/##' -e 's#/home/runner/work/[^/]*/[^/]*/##' \
| grep -v '^w: ' | awk '!seen[$0]++' | head -80; } || true
echo '```'
echo
echo "<details><summary>What went wrong (Gradle)</summary>"
echo
echo '```'
awk '/^\* What went wrong/{p=1} /^\* Try:/{p=0} p{print}' /tmp/build.log 2>/dev/null | head -60
echo '```'
echo "</details>"
;;
skipped:*|cancelled:*)
echo "### Not built"
echo
echo "The build did not run (see above, or [the run]($RUN_URL))."
;;
esac
echo
} > /tmp/pr-report.md
cat /tmp/pr-report.md >> "$GITHUB_STEP_SUMMARY"
ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate --jq ".[] | select(.body | startswith(\"$MARK\")) | .id" | head -1 || true)
if [ -n "$ID" ]; then
gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${ID}" -F body=@/tmp/pr-report.md >/dev/null
else
gh api -X POST "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -F body=@/tmp/pr-report.md >/dev/null
fi
run: bash tools/release/report-build.sh
python-tests:
name: Python and native regression tests
needs: xbox
needs: source
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
runs-on: ubuntu-24.04
timeout-minutes: 15
@@ -794,7 +618,7 @@ jobs:
- name: Merge the pinned APK base
if: github.event_name == 'pull_request'
env:
BUILD_BASE_SHA: ${{ needs.xbox.outputs.base-sha }}
BUILD_BASE_SHA: ${{ needs.source.outputs.base-sha }}
run: |
set -euo pipefail
git fetch --no-tags origin "$BUILD_BASE_SHA"
@@ -803,11 +627,40 @@ jobs:
git merge --no-ff --no-edit FETCH_HEAD
- name: Verify backend and test source match
env:
XBOX_SOURCE_TREE: ${{ needs.xbox.outputs.tree-sha }}
XBOX_SOURCE_TREE: ${{ needs.source.outputs.tree-sha }}
run: test "$(git rev-parse 'HEAD^{tree}')" = "$XBOX_SOURCE_TREE"
- name: Install Python test dependencies
timeout-minutes: 7
run: bash tools/ci/install-ubuntu-packages.sh python3-yaml gcc-aarch64-linux-gnu libc6-dev-arm64-cross qemu-user libvulkan-dev mesa-vulkan-drivers
- name: Resolve the pinned Android NDK
run: bash tools/ci/resolve-ndk.sh
- name: Check Quest tracking IPC
run: |
set -euo pipefail
NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"
VK_INCLUDE=$(ls -d "$NDK"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1)
mkdir -p build/vr/include
cp -R "$VK_INCLUDE/vulkan" build/vr/include/
if [ -d "$VK_INCLUDE/vk_video" ]; then cp -R "$VK_INCLUDE/vk_video" build/vr/include/; fi
g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_tracking.cpp -o /tmp/test-xr-tracking
/tmp/test-xr-tracking
g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_xr_touch.cpp -o /tmp/test-xr-touch
/tmp/test-xr-touch
g++ -std=c++17 -Wall -Wextra -Werror tools/vr/tests/test_pad_controller_config.cpp -o /tmp/test-pad-controller-config
/tmp/test-pad-controller-config
g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_vsync.cpp -o /tmp/test-xr-vsync
/tmp/test-xr-vsync
g++ -std=c++17 -Wall -Wextra -Werror -pthread tools/vr/tests/test_xr_frames.cpp -o /tmp/test-xr-frames
/tmp/test-xr-frames
cc -Itools/vr/third_party/wayland -c tools/vr/src/wayland_client_stub.c -o /tmp/test-wayland-stub.o
cc -Itools/vr/third_party/wayland -c app/src/main/cpp/waylandcomp/generated/xdg-shell-protocol.c -o /tmp/test-xdg-shell.o
g++ -std=c++17 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-missing-field-initializers -pthread -Ibuild/vr/include \
-Itools/vr/third_party/wayland -Iapp/src/main/cpp/waylandcomp/generated \
tools/vr/tests/test_presenter_shutdown.cpp tools/vr/src/presenter.cpp /tmp/test-wayland-stub.o /tmp/test-xdg-shell.o \
-ldl -o /tmp/test-presenter-shutdown
timeout 10 /tmp/test-presenter-shutdown
- name: Steam import and game environment tests
run: python3 -m unittest discover -s tools/tests -p 'test_*.py'
@@ -857,6 +710,10 @@ jobs:
CI_REPO: Droid-Deck/DroidDeck-CI
KEEP: 10
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/download-artifact@v4
with:
pattern: droiddeck-signed-*
@@ -874,72 +731,13 @@ jobs:
- name: Describe the change
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" \
--jq '[.[] | select(.merged_at != null and .base.ref == "main")] | first // empty')
if [ -n "$pr" ]; then
number=$(jq -r .number <<<"$pr")
title=$(jq -r .title <<<"$pr")
merged=$(jq -r '.merged_at[:10]' <<<"$pr")
summary=$(jq -r '.body // ""' <<<"$pr" | tr -d '\r' | { grep -v '^🤖' || true; } |
awk '!p && /^#/ { next } NF { p = 1; print; next } p { exit }' | head -n 6 | cut -c1-400)
{
echo "**Merged to main** - [PR #$number]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/$number): $title (merged $merged)"
if [ -n "$summary" ]; then
echo
echo "$summary"
fi
} > notes-head.md
else
subject=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA" --jq '.commit.message | split("\n")[0]')
echo "**Pushed to main** without a PR - $subject" > notes-head.md
fi
cat notes-head.md
run: bash tools/release/describe-build.sh
- name: Publish to DroidDeck-CI
id: publish
env:
GH_TOKEN: ${{ steps.ci-token.outputs.token }}
run: |
set -euo pipefail
tag=main-${GITHUB_SHA::7}
cat signed/SHA256SUMS-*.txt > signed/SHA256SUMS.txt
rm signed/SHA256SUMS-*.txt
notes="$(cat notes-head.md)
Signed build of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA - in main, not yet in a DroidDeck release; for testing.
Commit: $GITHUB_SHA
VersionCode: $(sed -n 's/^ *versionCode \([0-9][0-9]*\)$/\1/p' app/build.gradle)
Each apk installs over a DroidDeck release and keeps its data. \`standard\` is the normal app; the others are the same app under another package name, installed beside it.
SHA-256:
\`\`\`
$(cat signed/SHA256SUMS.txt)
\`\`\`"
gh release delete "$tag" -R "$CI_REPO" --yes --cleanup-tag 2>/dev/null || true
gh release create "$tag" -R "$CI_REPO" --target main --prerelease \
--title "main ${GITHUB_SHA::7}" --notes "$notes" signed/*
gh release list -R "$CI_REPO" -L 200 --json tagName,publishedAt \
--jq '[.[] | select(.tagName | startswith("main-"))] | sort_by(.publishedAt) | reverse | .['"$KEEP"':][] | .tagName' |
while IFS= read -r old; do
if [ "$old" = "$tag" ]; then
continue
fi
if ! gh release delete "$old" -R "$CI_REPO" --yes --cleanup-tag 2>/dev/null; then
if gh release view "$old" -R "$CI_REPO" >/dev/null 2>&1; then
echo "::error::Could not delete old release $old"
exit 1
fi
fi
done
published=$(gh release view "$tag" -R "$CI_REPO" --json tagName,assets)
jq -e --arg tag "$tag" '.tagName == $tag and (.assets | length) > 0' <<< "$published" >/dev/null
url="$GITHUB_SERVER_URL/$CI_REPO/releases/tag/$tag"
apk=$(cd signed && ls DroidDeck-*.apk | awk 'length(a) == 0 || length($0) < length(a) { a = $0 } END { print a }')
echo "release=$url" >> "$GITHUB_OUTPUT"
echo "apk=$GITHUB_SERVER_URL/$CI_REPO/releases/download/$tag/$apk" >> "$GITHUB_OUTPUT"
echo "Published $url"
run: bash tools/release/publish-ci.sh
- name: Post the Preview build to Discord
env:
@@ -947,26 +745,8 @@ jobs:
GH_TOKEN: ${{ github.token }}
RELEASE: ${{ steps.publish.outputs.release }}
APK: ${{ steps.publish.outputs.apk }}
run: |
set -euo pipefail
if [ -z "$WEBHOOK" ]; then
echo "::notice::No DISCORD_BUILDS_WEBHOOK secret; nothing posted"
exit 0
fi
commit=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA")
subject=$(jq -r '.commit.message | split("\n")[0]' <<<"$commit")
title=$(jq -r '.commit.message | split("\n") | map(select(length > 0)) | .[1] // ""' <<<"$commit")
pr=$(sed -n 's/^Merge pull request #\([0-9]*\).*/\1/p' <<<"$subject")
if [ -n "$pr" ]; then
what="[#$pr]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/$pr): ${title:-$subject}"
else
what="[\`${GITHUB_SHA::7}\`]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA): $subject"
fi
get=$(printf 'Download: [standard APK](<%s>) · [all variants](<%s>)' "$APK" "$RELEASE")
content=$(printf '**New signed Preview build** `main-%s`\n%s\nIn the app: **Updates → Preview**\n%s' "${GITHUB_SHA::7}" "$what" "$get")
jq -n --arg content "$content" '{content: $content, allowed_mentions: {parse: []}}' |
curl -fsS -H 'Content-Type: application/json' -d @- "$WEBHOOK" -o /dev/null
echo "Posted: $content"
run: bash tools/release/announce-preview.sh
catalog:
needs: announce
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
set -euo pipefail
repo=$(cd "$(dirname "$0")/../.." && pwd)
source "$repo/tools/ci/toolchains.env"
ndk="$ANDROID_HOME/ndk/$CI_ANDROID_NDK_VERSION"
if [ ! -f "$ndk/source.properties" ]; then
sdkmanager --sdk_root="$ANDROID_HOME" --install "ndk;$CI_ANDROID_NDK_VERSION"
fi
actual=$(sed -n 's/^Pkg.Revision *= *//p' "$ndk/source.properties" | tr -d '\r')
test "$actual" = "$CI_ANDROID_NDK_VERSION"
{
echo "NDK_VERSION=$actual"
echo "ANDROID_NDK_HOME=$ndk"
echo "ANDROID_NDK_ROOT=$ndk"
echo "ANDROID_NDK_LATEST_HOME=$ndk"
} >> "$GITHUB_ENV"
+1
View File
@@ -0,0 +1 @@
CI_ANDROID_NDK_VERSION=30.0.16248370
+32 -2
View File
@@ -1,10 +1,40 @@
# Verified preload snapshots
# Runtime component preparation
`recipes.json` owns component source inputs, builders, outputs, package requirements
and pinned downloads. `prepare.py` runs those recipes, calculates cache identities,
installs the union of missing tools and verifies the prepared artifact handoff.
Builders live in `recipes/`; the workflow describes scheduling and gates.
```mermaid
flowchart LR
Source[Pin source tree] --> Components[Prepare verified components]
Source --> Xbox[Xbox ARM64 backend]
Source --> Tests[Python and native tests]
Components --> APK[Compile APK and Android tests]
Xbox --> APK
APK --> Gate[Compile and test APK gate]
Tests --> Gate
Gate --> Publish[Main only: signing and publishing]
```
All workers verify the same source tree. The component artifact records that tree,
recipe identities and every output hash; the APK worker verifies the inventory
before building. This handoff verifies consistency within the same CI run. It is
not independent provenance for arbitrary artifacts. Source policy checks, tests,
APK content/ABI checks and signing/publication gates remain in place.
`tools/ci/toolchains.env` pins the Android NDK shared by component, APK and native
test workers. Linux builds use Ubuntu 24.04. Recipe and input changes invalidate
the relevant component cache; ordinary caches remain performance aids.
## Verified preload snapshots
`preloads.lock.json` pins a small immutable preload archive. It records each
binary's hash, its producing main CI run, the original APK artifact identity and
digest, source/header hashes, the ARM64 build recipe and x86 compiler-image inputs.
It covers core ARM64 preloads and the x86 preload libraries. VR, Android native
libraries, Comet and Xbox keep their existing preparation paths.
libraries, Comet and Xbox use their registered recipes and existing release pins
or component caches.
Every build compares current source inputs, including transitive quoted headers,
and build recipes with this manifest. Changed groups rebuild from source. A
+1 -1
View File
@@ -1,6 +1,6 @@
{
"archive_sha256": "cb6487a94f25a3068fd07958174eb5b5e14f709655ff4ef0a8cdbd99138513b2",
"arm64_recipe": "62c8b9cf001f1b04b52856bd74d353842a4be29960f033067f34c7d3a9a48681",
"arm64_recipe": "1c92a1b0669277de3e78e9f292634affb5b5a19d5ea49d3ad8610c1dc638c2fb",
"groups": {
"arm64": {
"@runner": "84f825635599f81ed33f62c5fc4c73f66044d1468b060d65f83855eb9c8bfe77",
+47 -23
View File
@@ -1,6 +1,7 @@
"""Reuse reviewed preload snapshots; changed snapshots must prove their main-build origin."""
import argparse
import gzip
import http.client
import hashlib
import io
import json
@@ -10,6 +11,7 @@ import re
from pathlib import Path
import subprocess
import tarfile
import textwrap
import urllib.error
import urllib.request
from zipfile import ZipFile
@@ -17,20 +19,12 @@ from zipfile import ZipFile
ROOT = Path(__file__).resolve().parents[2]
REPO = 'Droid-Deck/DroidDeck'
LOCK = 'tools/components/preloads.lock.json'
COMMON = ['app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/cpp/adrenotools/src/hook/kgsl.h',
'app/src/main/cpp/venc/venc_proto.h', 'tools/linuxfs/preload',
'tools/quest/kgsl_compat.c', 'tools/quest/kgsl_compat.h']
INPUTS = {
'arm64': COMMON + ['tools/proot/fastpath', 'tools/linuxfs/ssbs', 'tools/linuxfs/clipboard'],
'x86': COMMON + ['tools/linuxfs/fex', 'tools/linuxfs/build-x86-preloads.sh', 'tools/linuxfs/debian-image.env'],
}
OUTPUTS = {
'arm64': ['libfakeinput.so', 'libblsession.so', 'libblfastpath.so', 'libblaudit.so', 'libssbs.so',
'usr/local/bin/droiddeck-clipboard'],
'x86': [f'{arch}/{name}.so' for arch in ('x86_64', 'i386')
for name in ('libfakeinput', 'libblsession', 'libfaultreport')]
+ ['x86_64/libwebkitinput.so', 'x86_64/libvulkan-thunk.so'],
}
DEFINITIONS = json.loads((ROOT / 'tools/components/recipes.json').read_text())
INPUTS = {group: DEFINITIONS[name]['source_inputs'] for group, name in
[('arm64', 'arm64-preloads'), ('x86', 'x86-preloads')]}
OUTPUTS = {group: DEFINITIONS[name]['snapshot_outputs'] for group, name in
[('arm64', 'arm64-preloads'), ('x86', 'x86-preloads')]}
COMMON = [name for name in INPUTS['arm64'] if name in INPUTS['x86']]
FILES = set(sum(OUTPUTS.values(), []))
@@ -46,12 +40,13 @@ def api(path):
return json.loads(run('gh', 'api', 'repos/' + REPO + '/' + path))
def source_files(group, root=ROOT, commit=None):
def source_files(group, root=ROOT, commit=None, patterns=None):
patterns = patterns if patterns is not None else INPUTS[group]
if commit:
paths = run('git', 'ls-tree', '-r', '--name-only', commit, '--', *INPUTS[group], root=root).decode().splitlines()
paths = run('git', 'ls-tree', '-r', '--name-only', commit, '--', *patterns, root=root).decode().splitlines()
read = lambda path: run('git', 'show', f'{commit}:{path}', root=root)
else:
paths = run('git', 'ls-files', '--cached', '--others', '--exclude-standard', '--', *INPUTS[group], root=root).decode().splitlines()
paths = run('git', 'ls-files', '--cached', '--others', '--exclude-standard', '--', *patterns, root=root).decode().splitlines()
read = lambda path: (root / path).read_bytes()
paths = [p for p in paths if not p.endswith('.md') and '__pycache__' not in p and not p.endswith('.pyc')]
if not paths:
@@ -70,7 +65,8 @@ def source_files(group, root=ROOT, commit=None):
pending.append(posixpath.normpath(posixpath.join(posixpath.dirname(path), include.decode())))
workflow = (run('git', 'show', f'{commit}:.github/workflows/build.yml', root=root).decode() if commit
else (root / '.github/workflows/build.yml').read_text())
runner = re.search(r'(?ms)^ build:\n.*?^ runs-on: ([^\n]+)', workflow)
job = 'components' if re.search(r'(?m)^ components:', workflow) else 'build'
runner = re.search(r'(?ms)^ ' + job + r':\n.*?^ runs-on: ([^\n]+)', workflow)
if runner is None:
raise ValueError('APK build runner policy is missing')
result['@runner'] = sha(runner[1].strip().encode())
@@ -85,14 +81,14 @@ def recipe(root=ROOT, commit=None):
if commit and run('git', 'ls-tree', commit, script, root=root) == b'':
workflow = read('.github/workflows/build.yml')
text = workflow[workflow.index(' aarch64-linux-gnu-g++ -shared'):]
text = text[:text.index(' VK_INCLUDE=')]
text = textwrap.dedent(text[:text.index(' VK_INCLUDE=')])
else:
text = read(script)
preamble = '#!/usr/bin/env bash\nset -euo pipefail\nD=${1:?output directory}\nmkdir -p "$D"\n'
if not text.startswith(preamble):
return sha(text.encode())
text = text[len(preamble):]
return sha(' '.join(text.split()).encode())
return sha(text.replace('\r\n', '\n').encode())
def artifact_bytes(origin, cache):
@@ -130,6 +126,26 @@ def validate_lock(data):
raise ValueError('Invalid preload checksum')
def producer_sources(group, commit, root=ROOT):
config = 'tools/components/recipes.json'
if run('git', 'ls-tree', commit, config, root=root):
definitions = json.loads(run('git', 'show', f'{commit}:{config}', root=root))
name = 'arm64-preloads' if group == 'arm64' else 'x86-preloads'
patterns = definitions[name]['source_inputs']
else:
# Older approved builds kept ARM64 commands inline, before the Vulkan/VR build.
if group == 'arm64':
text = run('git', 'show', f'{commit}:.github/workflows/build.yml', root=root).decode()
text = text[text.index(' aarch64-linux-gnu-g++ -shared'):]
text = text[:text.index(' VK_INCLUDE=')]
else:
text = run('git', 'show', f'{commit}:tools/linuxfs/build-x86-preloads.sh', root=root).decode()
patterns = re.findall(r'\b((?:tools|app)/[\w./*+-]+\.(?:cpp|c|h))(?=\s|$)', text)
if group == 'x86':
patterns += ['tools/linuxfs/fex', 'tools/linuxfs/build-x86-preloads.sh', 'tools/linuxfs/debian-image.env']
return source_files(group, root, commit, patterns)
def prove_origin(data, cache, root=ROOT):
content = artifact_bytes(data['origin'], cache)
if {name: sha(value) for name, value in content.items()} != data['outputs']:
@@ -139,6 +155,9 @@ def prove_origin(data, cache, root=ROOT):
for group in INPUTS:
if source_files(group, root, commit) != data['groups'][group]:
raise ValueError('Snapshot inputs differ from the producing source')
for name, checksum in producer_sources(group, commit, root).items():
if data['groups'][group].get(name) != checksum:
raise ValueError('Snapshot omits or changes a producing build dependency')
if recipe(root, commit) != data['arm64_recipe']:
raise ValueError('Snapshot recipe differs from the producing source')
@@ -179,14 +198,19 @@ def restore(root=ROOT):
if result.returncode == 0:
approved = json.loads(result.stdout)
if approved != data:
prove_origin(data, root / 'app/build/component-downloads', root)
try:
prove_origin(data, root / 'app/build/component-downloads', root)
except (subprocess.CalledProcessError, OSError, http.client.HTTPException):
print('Snapshot origin is unavailable; rebuilding without trusting the snapshot.')
matching = dict.fromkeys(INPUTS, False)
if any(matching.values()):
url = (f"https://github.com/Droid-Deck/DroidDeck-Components/releases/download/"
f"preloads-{data['archive_sha256']}/preloads.tar.gz")
try:
with urllib.request.urlopen(url, timeout=60) as response:
content = response.read(64 * 1024 * 1024)
except urllib.error.HTTPError as error:
if error.code != 404:
except (OSError, http.client.HTTPException) as error:
if isinstance(error, urllib.error.HTTPError) and error.code not in (404, 429) and error.code < 500:
raise
print('Preload snapshot is unavailable; rebuilding from source.')
matching = dict.fromkeys(INPUTS, False)
+167
View File
@@ -0,0 +1,167 @@
"""One entry point for runtime recipes, cache identities and verified job handoff."""
import argparse
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import shutil
import subprocess
import tarfile
ROOT = Path(__file__).resolve().parents[2]
REGISTRY = 'tools/components/recipes.json'
TREES = ['app/src/main/assets/linuxfs', 'app/src/main/assets/directaudio',
'app/src/main/assets/droiddeck-esync', 'app/build/prepared-assets',
'app/build/component-records']
LIBRARIES = ['app/src/main/jniLibs/arm64-v8a/' + name for name in
('libproot.so', 'libproot-loader.so', 'libdirectaudiorelay.so')]
MANIFEST = 'app/build/prepared-components.json'
def digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def registry(root=ROOT):
return json.loads((root / REGISTRY).read_text())
def identity(name, root=ROOT):
spec = registry(root)[name]
paths = subprocess.check_output(['git', 'ls-files', '--cached', '--others', '--exclude-standard',
'--', *spec['inputs'], *spec.get('source_inputs', [])], cwd=root, text=True).splitlines()
values = {}
for name_ in sorted(set(paths)):
if '__pycache__' in name_ or name_.endswith(('.pyc', '.md')):
continue
content = (root / name_).read_bytes()
if b'\0' not in content:
content = content.replace(b'\r\n', b'\n')
values[name_] = hashlib.sha256(content).hexdigest()
policy = dict(spec=spec, sources=values, runner=os.environ.get('RUNNER_OS', 'Linux'),
architecture=os.environ.get('RUNNER_ARCH', 'X64'))
entrypoint = root / 'tools/components/prepare.py'
if entrypoint.is_file():
policy['entrypoint'] = hashlib.sha256(entrypoint.read_bytes().replace(b'\r\n', b'\n')).hexdigest()
installer = root / 'tools/ci/install-ubuntu-packages.sh'
if spec.get('packages') and installer.is_file():
policy['installer'] = hashlib.sha256(installer.read_bytes().replace(b'\r\n', b'\n')).hexdigest()
if spec.get('ndk'):
policy['ndk'] = os.environ.get('NDK_VERSION', '')
policy['ndk_policy'] = hashlib.sha256((root / 'tools/ci/toolchains.env').read_bytes().replace(b'\r\n', b'\n')).hexdigest()
return hashlib.sha256(json.dumps(policy, sort_keys=True).encode()).hexdigest()
def prepare(name, root=ROOT):
spec = registry(root)[name]
prepare_tools([name], root)
if 'command' in spec:
command = spec['command']
else:
script = (root / spec['script']).resolve()
if not script.is_relative_to((root / 'tools/components').resolve()):
raise ValueError('Recipe script escapes components directory')
command = ['bash', str(script)]
environment = dict(os.environ)
if spec.get('source'):
environment.update(COMPONENT_SOURCE_URL=spec['source']['url'],
COMPONENT_SOURCE_SHA256=spec['source']['sha256'])
subprocess.run(command, cwd=root, env=environment, check=True)
missing = [name_ for name_ in spec['outputs'] if not (root / name_).is_file()]
if missing:
raise ValueError(f'{name} did not prepare required outputs: {missing}')
record = dict(component=name, input_sha256=identity(name, root),
outputs={p: digest(root / p) for p in spec['outputs']})
destination = root / 'app/build/component-records' / (name + '.json')
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(record, sort_keys=True, indent=2) + '\n')
def prepare_tools(names, root=ROOT):
packages = sorted({package for name in names for package in registry(root)[name].get('packages', [])})
if packages:
subprocess.run(['bash', 'tools/ci/install-ubuntu-packages.sh', *packages], cwd=root, check=True)
def files(root=ROOT):
result = []
for name in TREES:
folder = root / name
if not folder.is_dir() or folder.is_symlink():
raise ValueError('Missing prepared component tree: ' + name)
for path in folder.rglob('*'):
if path.is_symlink():
raise ValueError('Symlink in prepared components: ' + str(path))
if path.is_file():
result.append(path.relative_to(root).as_posix())
for name in LIBRARIES:
if not (root / name).is_file() or (root / name).is_symlink():
raise ValueError('Missing prepared component library: ' + name)
result.append(name)
return sorted(result)
def pack(destination, root=ROOT):
names = files(root)
tree = subprocess.check_output(['git', 'rev-parse', 'HEAD^{tree}'], cwd=root, text=True).strip()
data = dict(version=1, source_tree=tree, run=os.environ.get('GITHUB_RUN_ID', ''),
recipes={name: identity(name, root) for name in registry(root)},
files={name: digest(root / name) for name in names})
(root / MANIFEST).write_text(json.dumps(data, sort_keys=True, indent=2) + '\n')
destination.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(destination, mode='w:gz', compresslevel=1) as archive:
for name in names + [MANIFEST]:
archive.add(root / name, arcname=name, recursive=False)
def unpack(archive_path, source_tree, root=ROOT):
total = 0
seen = set()
with tarfile.open(archive_path, mode='r:gz') as archive:
for entry in archive:
allowed = entry.name in LIBRARIES + [MANIFEST] or any(entry.name.startswith(p + '/') for p in TREES)
path = root / entry.name
total += entry.size
if (not allowed or '\\' in entry.name or '..' in PurePosixPath(entry.name).parts
or PurePosixPath(entry.name).as_posix() != entry.name
or not entry.isfile() or entry.name in seen or len(seen) >= 100000
or entry.size > 512 * 1024 * 1024
or total > 2 * 1024**3 or not path.resolve().is_relative_to(root.resolve())
or path.is_symlink()):
raise ValueError('Unsafe prepared component archive')
seen.add(entry.name)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open('wb') as output:
shutil.copyfileobj(archive.extractfile(entry), output)
path.chmod(entry.mode & 0o777)
data = json.loads((root / MANIFEST).read_text())
actual = {name: digest(root / name) for name in files(root)}
recipes = {name: identity(name, root) for name in registry(root)}
if (data.get('version') != 1 or data['source_tree'] != source_tree or data['files'] != actual
or data['recipes'] != recipes):
raise ValueError('Prepared components do not match the pinned source tree and file hashes')
print('Prepared component source identity and all output hashes verified.')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('recipe')
parser.add_argument('--archive', type=Path, default=ROOT / 'app/build/prepared-components.tar.gz')
parser.add_argument('--tree')
args = parser.parse_args()
if args.recipe == 'plan':
for name in registry():
value = identity(name)
print(f'{name}: {value}')
if os.environ.get('GITHUB_OUTPUT'):
with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
stream.write(f'{name}={value}\n')
elif args.recipe == 'tools':
prepare_tools(os.environ.get('COMPONENTS_TO_BUILD', '').split())
elif args.recipe == 'pack':
pack(args.archive)
elif args.recipe == 'unpack':
unpack(args.archive, args.tree)
else:
prepare(args.recipe)
+225
View File
@@ -0,0 +1,225 @@
{
"runtime": {
"script": "tools/components/recipes/runtime.sh",
"inputs": [
"tools/components/recipes/runtime.sh",
"tools/quest",
"tools/linuxfs/overlay",
"tools/linuxfs/desktop",
"tools/linuxfs/licenses",
"tools/linuxfs/uruntime.env",
"tools/gamescope/release.env",
"tools/wlroots/release.env",
"tools/mangoapp",
"tools/msitools",
"tools/pipewire",
"tools/vr",
"app/src/main/cpp/waylandcomp/generated",
"app/src/main/cpp/waylandcomp/src/headpose_shm.h",
"app/src/main/cpp/waylandcomp/src/spatialpose_shm.h",
"app/src/main/cpp/venc/venc_proto.h"
],
"outputs": [
"app/src/main/assets/linuxfs/usr/local/bin/gamescope",
"app/src/main/assets/linuxfs/usr/local/lib/droiddeck-vr/droiddeck/bin/linuxarm64/driver_droiddeck.so"
],
"ndk": true,
"packages": [
"gcc-aarch64-linux-gnu",
"g++-aarch64-linux-gnu",
"libvulkan1",
"mesa-vulkan-drivers"
]
},
"esync": {
"script": "tools/components/recipes/esync.sh",
"inputs": [
"tools/components/recipes/esync.sh",
"tools/droiddeck-esync/release.env"
],
"outputs": [
"app/src/main/assets/droiddeck-esync/index.json",
"app/src/main/assets/droiddeck-esync/index.json.sig"
]
},
"debian-image": {
"script": "tools/components/recipes/debian-image.sh",
"inputs": [
"tools/components/recipes/debian-image.sh"
],
"outputs": []
},
"x86-preloads": {
"script": "tools/components/recipes/x86-preloads.sh",
"inputs": [
"tools/components/recipes/x86-preloads.sh",
"tools/linuxfs/build-x86-preloads.sh",
"tools/linuxfs/preload",
"tools/linuxfs/fex"
],
"outputs": [
"app/src/main/assets/linuxfs/x86_64/libfakeinput.so",
"app/src/main/assets/linuxfs/x86_64/libblsession.so",
"app/src/main/assets/linuxfs/x86_64/libfaultreport.so",
"app/src/main/assets/linuxfs/i386/libfakeinput.so",
"app/src/main/assets/linuxfs/i386/libblsession.so",
"app/src/main/assets/linuxfs/i386/libfaultreport.so",
"app/src/main/assets/linuxfs/x86_64/libwebkitinput.so",
"app/src/main/assets/linuxfs/x86_64/libvulkan-thunk.so"
],
"source_inputs": [
"app/src/main/cpp/fakeinput_steam.cpp",
"app/src/main/cpp/adrenotools/src/hook/kgsl.h",
"app/src/main/cpp/venc/venc_proto.h",
"tools/linuxfs/preload",
"tools/quest/kgsl_compat.c",
"tools/quest/kgsl_compat.h",
"tools/linuxfs/fex",
"tools/linuxfs/build-x86-preloads.sh",
"tools/linuxfs/debian-image.env"
],
"snapshot_outputs": [
"x86_64/libfakeinput.so",
"x86_64/libblsession.so",
"x86_64/libfaultreport.so",
"i386/libfakeinput.so",
"i386/libblsession.so",
"i386/libfaultreport.so",
"x86_64/libwebkitinput.so",
"x86_64/libvulkan-thunk.so"
]
},
"pulseaudio-source": {
"script": "tools/components/recipes/pulseaudio-source.sh",
"inputs": [
"tools/components/recipes/pulseaudio-source.sh"
],
"outputs": [
"pa13/src/pulsecore/module.h"
],
"source": {
"url": "https://github.com/pulseaudio/pulseaudio/archive/refs/tags/v13.0.tar.gz",
"sha256": "bd7980bc9bd44bd3b3363545dabd19476ebf4d6bb6d2527e7d179256394a6758"
}
},
"directaudio": {
"script": "tools/components/recipes/directaudio.sh",
"inputs": [
"tools/components/recipes/directaudio.sh",
"tools/directaudio"
],
"outputs": [
"app/src/main/jniLibs/arm64-v8a/libdirectaudiorelay.so"
]
},
"audio": {
"script": "tools/components/recipes/audio.sh",
"inputs": [
"tools/aaudio-sink",
"tools/directaudio",
"tools/components/recipes/audio.sh",
"app/src/main/assets/pulseaudio.tzst"
],
"outputs": [
"app/build/prepared-assets/pulseaudio.tzst"
],
"ndk": true
},
"proot": {
"script": "tools/components/recipes/proot.sh",
"inputs": [
"tools/proot",
"tools/components/recipes/proot.sh"
],
"outputs": [
"app/src/main/jniLibs/arm64-v8a/libproot.so",
"app/src/main/jniLibs/arm64-v8a/libproot-loader.so"
],
"ndk": true
},
"comet": {
"script": "tools/components/recipes/comet.sh",
"inputs": [
"tools/comet",
"tools/components/recipes/comet.sh"
],
"outputs": [
"app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet/comet",
"app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet/version"
],
"packages": [
"gcc-aarch64-linux-gnu"
]
},
"arm64-preloads": {
"script": "tools/components/recipes/arm64-preloads.sh",
"inputs": [
"tools/components/build-arm64-preloads.sh",
"tools/linuxfs/preload",
"app/src/main/cpp/fakeinput_steam.cpp",
"tools/components/recipes/arm64-preloads.sh"
],
"outputs": [
"app/src/main/assets/linuxfs/libfakeinput.so",
"app/src/main/assets/linuxfs/libblsession.so",
"app/src/main/assets/linuxfs/libblfastpath.so",
"app/src/main/assets/linuxfs/libblaudit.so",
"app/src/main/assets/linuxfs/libssbs.so",
"app/src/main/assets/linuxfs/usr/local/bin/droiddeck-clipboard"
],
"packages": [
"gcc-aarch64-linux-gnu",
"g++-aarch64-linux-gnu"
],
"source_inputs": [
"app/src/main/cpp/fakeinput_steam.cpp",
"app/src/main/cpp/adrenotools/src/hook/kgsl.h",
"app/src/main/cpp/venc/venc_proto.h",
"tools/linuxfs/preload",
"tools/quest/kgsl_compat.c",
"tools/quest/kgsl_compat.h",
"tools/proot/fastpath",
"tools/linuxfs/ssbs",
"tools/linuxfs/clipboard"
],
"snapshot_outputs": [
"libfakeinput.so",
"libblsession.so",
"libblfastpath.so",
"libblaudit.so",
"libssbs.so",
"usr/local/bin/droiddeck-clipboard"
]
},
"preload-snapshot": {
"command": [
"python3",
"tools/components/preloads.py",
"restore"
],
"inputs": [
"tools/components/preloads.py",
"tools/components/preloads.lock.json"
],
"outputs": []
},
"xbox": {
"script": "tools/components/recipes/xbox.sh",
"inputs": [
"tools/xbox",
"tools/components/recipes/xbox.sh"
],
"outputs": [
"app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox/xodus-cli",
"app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox/xodus-service"
],
"packages": [
"build-essential",
"pkg-config",
"libssl-dev",
"protobuf-compiler",
"cmake",
"xz-utils"
]
}
}
@@ -0,0 +1,3 @@
#!/usr/bin/env bash
set -euo pipefail
bash tools/components/build-arm64-preloads.sh app/src/main/assets/linuxfs
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
export NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"
tools/aaudio-sink/build.sh "$PWD/pa13" "$PWD/sink-out"
rm -rf pa-bundle && mkdir -p pa-bundle
tar --use-compress-program=unzstd -xf app/src/main/assets/pulseaudio.tzst -C pa-bundle
test ! -f pa-bundle/modules/arm64/module-aaudio-sink.so
install -m755 sink-out/module-aaudio-sink.so pa-bundle/modules/arm64/module-aaudio-sink.so
install -m755 sink-out/module-directaudio-native-sink.so pa-bundle/modules/arm64/module-directaudio-native-sink.so
(cd pa-bundle && tar --use-compress-program='zstd -19' -cf ../pulseaudio.tzst .)
mkdir -p app/build/prepared-assets
mv pulseaudio.tzst app/build/prepared-assets/pulseaudio.tzst
tar --use-compress-program=unzstd -tf app/build/prepared-assets/pulseaudio.tzst | grep -c '\.so$'
rm -rf pa-bundle sink-out
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
rustup toolchain install 1.90.0 --profile minimal
rustup target add --toolchain 1.90.0 aarch64-unknown-linux-gnu
RUSTUP_TOOLCHAIN=1.90.0 bash tools/comet/build.sh
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -euo pipefail
. tools/linuxfs/debian-image.env
image=public.ecr.aws/docker/library/debian:bullseye
if gh release download "$DEBIAN_IMAGE_TAG" -R "$GITHUB_REPOSITORY" -p "$DEBIAN_IMAGE_ASSET" -O /tmp/debian-image.tar.zst \
&& { [ -z "$DEBIAN_IMAGE_SHA256" ] || echo "$DEBIAN_IMAGE_SHA256 /tmp/debian-image.tar.zst" | sha256sum -c -; }; then
zstd -dc /tmp/debian-image.tar.zst | docker load
rm -f /tmp/debian-image.tar.zst
else
echo "::warning::the hosted Debian image could not be fetched; pulling from the registry instead"
for attempt in 1 2 3 4 5; do
docker pull "$image" && break
[ "$attempt" -lt 5 ] || { echo "::error::could not pull $image after $attempt attempts"; exit 1; }
echo "pull attempt $attempt failed (registry busy?); retrying in $((attempt * 30)) s"
sleep $((attempt * 30))
done
fi
docker image ls "$image"
+3
View File
@@ -0,0 +1,3 @@
#!/usr/bin/env bash
set -euo pipefail
tools/directaudio/fetch.sh "$PWD" "$PWD/sink-out"
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -euo pipefail
if [ -f tools/droiddeck-esync/release.env ]; then
. tools/droiddeck-esync/release.env
release_url="https://github.com/${SYNC_BUNDLE_REPO}/releases/download/${SYNC_BUNDLE_TAG}"
curl -fsSL --retry 3 -o sync-bundle.tzst "$release_url/$SYNC_BUNDLE_ASSET"
echo "$SYNC_BUNDLE_SHA256 sync-bundle.tzst" | sha256sum -c -
mkdir -p app/src/main/assets/droiddeck-esync
tar --use-compress-program=unzstd -xf sync-bundle.tzst -C app/src/main/assets/droiddeck-esync
rm sync-bundle.tzst
curl -fsSL --retry 3 -o sync-index.json "$release_url/index.json"
revoked=$(python3 -c 'import json, sys; print(" ".join(p["id"] for p in json.load(open(sys.argv[1]))["packs"] if p.get("revoked") is True))' sync-index.json)
for id in $revoked; do
if [ -e "app/src/main/assets/droiddeck-esync/packs/$id.tzst" ]; then
echo "::warning::$SYNC_BUNDLE_ASSET carries revoked pack $id; it is left out of the APK (point tools/droiddeck-esync/release.env at a newer bundle)"
rm -f "app/src/main/assets/droiddeck-esync/packs/$id.tzst"
fi
done
rm sync-index.json
test -f app/src/main/assets/droiddeck-esync/index.json
test -f app/src/main/assets/droiddeck-esync/index.json.sig
ls -lR app/src/main/assets/droiddeck-esync
fi
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
export NDK="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"
tools/proot/build.sh app/src/main/jniLibs/arm64-v8a
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
curl -fsSL --retry 3 -o pa.tar.gz "$COMPONENT_SOURCE_URL"
echo "$COMPONENT_SOURCE_SHA256 pa.tar.gz" | sha256sum -c -
mkdir -p pa13 && tar -xzf pa.tar.gz -C pa13 --strip-components=1 && rm pa.tar.gz
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env bash
set -euo pipefail
D=app/src/main/assets/linuxfs
mkdir -p "$D"
VK_INCLUDE=$(ls -d "${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1) \
tools/vr/build.sh --native "$D/usr/local/lib/droiddeck-vr"
for script in tools/linuxfs/overlay/usr/local/bin/droiddeck-* tools/linuxfs/overlay/usr/local/bin/steam-compatibility; do
install -Dm644 "$script" "$D/usr/local/bin/$(basename "$script")"
done
for f in tools/linuxfs/overlay/usr/bin/* tools/linuxfs/overlay/usr/bin/steamos-polkit-helpers/*; do
[ -f "$f" ] && install -Dm644 "$f" "$D/${f#tools/linuxfs/overlay/}"
done
if [ -f tools/gamescope/release.env ]; then
. tools/gamescope/release.env
curl -fsSL --retry 3 -o gamescope.tzst "https://github.com/${GAMESCOPE_REPO:-${GITHUB_REPOSITORY}}/releases/download/${GAMESCOPE_TAG}/gamescope.tzst"
echo "${GAMESCOPE_SHA256} gamescope.tzst" | sha256sum -c -
tar --use-compress-program=unzstd -xf gamescope.tzst -C "$D"
test -f "$D/usr/local/bin/gamescope"
rm gamescope.tzst
fi
. tools/linuxfs/uruntime.env
curl -fsSL --retry 3 -o uruntime "https://github.com/VHSgunzo/uruntime/releases/download/${URUNTIME_VERSION}/${URUNTIME_ASSET}"
echo "${URUNTIME_SHA256} uruntime" | sha256sum -c -
install -Dm644 uruntime "$D/usr/local/lib/droiddeck/uruntime"
install -Dm644 tools/linuxfs/licenses/uruntime-LICENSE "$D/usr/local/share/licenses/uruntime/LICENSE"
rm uruntime
M="$D/usr/local/lib/mangoapp"
mkdir -p "$M" mango-pkgs
grep -v '^#' tools/mangoapp/packages.txt | while read -r sum url; do
[ -n "$url" ] || continue
curl -fsSL --retry 3 -o mango-pkgs/pkg.tar.zst "$url"
echo "$sum mango-pkgs/pkg.tar.zst" | sha256sum -c -
tar --use-compress-program=unzstd -xf mango-pkgs/pkg.tar.zst -C mango-pkgs
rm mango-pkgs/pkg.tar.zst
done
install -m644 mango-pkgs/usr/bin/mangoapp "$M/mangoapp"
for so in libfmt.so.10 libspdlog.so.1.13 libglfw.so.3 libtraceevent.so.1; do
cp -L "mango-pkgs/usr/lib/$so" "$M/$so"
done
aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -Wl,-soname,libtracefs.so.1 \
-o "$M/libtracefs.so.1" tools/mangoapp/libtracefs-shim.c
aarch64-linux-gnu-strip --strip-unneeded "$M/libtracefs.so.1"
install -Dm644 tools/mangoapp/mangoapp "$D/usr/local/bin/mangoapp"
rm -rf mango-pkgs
ls -l "$M"
mkdir -p pw-pkgs
grep -v '^#' tools/pipewire/packages.txt | while read -r sum url; do
[ -n "$url" ] || continue
curl -fsSL --retry 3 -o pw-pkgs/pkg.tar.zst "$url"
echo "$sum pw-pkgs/pkg.tar.zst" | sha256sum -c -
tar --use-compress-program=unzstd -xf pw-pkgs/pkg.tar.zst -C pw-pkgs
rm pw-pkgs/pkg.tar.zst
done
tools/pipewire/assemble.sh pw-pkgs "$D/usr/local/lib/droiddeck-pipewire.tzst"
rm -rf pw-pkgs
ls -l "$D/usr/local/lib/droiddeck-pipewire.tzst"
T="$D/usr/local/lib/droiddeck-msitools"
mkdir -p "$T" msi-pkgs
grep -v '^#' tools/msitools/packages.txt | while read -r sum url; do
[ -n "$url" ] || continue
curl -fsSL --retry 3 -o msi-pkgs/pkg.tar.xz "$url"
echo "$sum msi-pkgs/pkg.tar.xz" | sha256sum -c -
tar -xJf msi-pkgs/pkg.tar.xz -C msi-pkgs
rm msi-pkgs/pkg.tar.xz
done
install -m644 msi-pkgs/usr/bin/msiinfo "$T/msiinfo"
install -m644 msi-pkgs/usr/bin/cabextract "$T/cabextract"
install -m644 msi-pkgs/usr/lib/7zip/7z "$T/7z"
install -m644 msi-pkgs/usr/lib/7zip/7z.so "$T/7z.so"
for so in libmsi-1.0.so.0 libgsf-1.so.114 libgcab-1.0.so.0; do
cp -L "msi-pkgs/usr/lib/$so" "$T/$so"
done
for f in NOTICE GPL-2 GPL-3 LGPL-2.1 7zip-License 7zip-unRarLicense; do
install -Dm644 "tools/msitools/licenses/$f" "$D/usr/local/share/licenses/droiddeck-msitools/$f"
done
rm -rf msi-pkgs
ls -l "$T"
install -Dm644 tools/linuxfs/desktop/droiddeck-desktop "$D/usr/local/bin/droiddeck-desktop"
install -Dm644 tools/linuxfs/desktop/droiddeck-gpu "$D/usr/local/bin/droiddeck-gpu"
install -Dm644 tools/linuxfs/desktop/droiddeck-desktop-gpu "$D/usr/local/bin/droiddeck-desktop-gpu"
install -Dm644 tools/linuxfs/desktop/kwin_wayland_wrapper "$D/usr/local/bin/kwin_wayland_wrapper"
install -Dm644 tools/linuxfs/desktop/firefox-droiddeck.js "$D/usr/lib/firefox/defaults/pref/droiddeck.js"
ls -lR "$D"
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
docker run --rm -v "$PWD:/src" -w /src public.ecr.aws/docker/library/debian:bullseye bash -c '
set -euo pipefail
install_compilers() {
apt-get -o Acquire::Check-Valid-Until=false update -qq &&
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends gcc g++ gcc-multilib g++-multilib binutils >/dev/null
}
printf "deb http://deb.debian.org/debian bullseye main\ndeb http://security.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list
if ! install_compilers; then
printf "deb http://archive.debian.org/debian bullseye main\ndeb http://archive.debian.org/debian-security bullseye-security main\n" > /etc/apt/sources.list
install_compilers
fi
tools/linuxfs/build-x86-preloads.sh app/src/main/assets/linuxfs
'
sudo chown -R "$(id -u):$(id -g)" app/src/main/assets/linuxfs
ls -l app/src/main/assets/linuxfs/x86_64 app/src/main/assets/linuxfs/i386
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
rustup toolchain install 1.99.0 --profile minimal
bash tools/xbox/build.sh
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail
if [ -z "$WEBHOOK" ]; then
echo "::notice::No DISCORD_BUILDS_WEBHOOK secret; nothing posted"
exit 0
fi
commit=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA")
subject=$(jq -r '.commit.message | split("\n")[0]' <<<"$commit")
title=$(jq -r '.commit.message | split("\n") | map(select(length > 0)) | .[1] // ""' <<<"$commit")
pr=$(sed -n 's/^Merge pull request #\([0-9]*\).*/\1/p' <<<"$subject")
if [ -n "$pr" ]; then
what="[#$pr]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/$pr): ${title:-$subject}"
else
what="[\`${GITHUB_SHA::7}\`]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA): $subject"
fi
get=$(printf 'Download: [standard APK](<%s>) · [all variants](<%s>)' "$APK" "$RELEASE")
content=$(printf '**New signed Preview build** `main-%s`\n%s\nIn the app: **Updates → Preview**\n%s' "${GITHUB_SHA::7}" "$what" "$get")
jq -n --arg content "$content" '{content: $content, allowed_mentions: {parse: []}}' |
curl -fsS -H 'Content-Type: application/json' -d @- "$WEBHOOK" -o /dev/null
echo "Posted: $content"
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -euo pipefail
pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" \
--jq '[.[] | select(.merged_at != null and .base.ref == "main")] | first // empty')
if [ -n "$pr" ]; then
number=$(jq -r .number <<<"$pr")
title=$(jq -r .title <<<"$pr")
merged=$(jq -r '.merged_at[:10]' <<<"$pr")
summary=$(jq -r '.body // ""' <<<"$pr" | tr -d '\r' | { grep -v '^🤖' || true; } |
awk '!p && /^#/ { next } NF { p = 1; print; next } p { exit }' | head -n 6 | cut -c1-400)
{
echo "**Merged to main** - [PR #$number]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/$number): $title (merged $merged)"
if [ -n "$summary" ]; then
echo
echo "$summary"
fi
} > notes-head.md
else
subject=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA" --jq '.commit.message | split("\n")[0]')
echo "**Pushed to main** without a PR - $subject" > notes-head.md
fi
cat notes-head.md
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
set -euo pipefail
tag=main-${GITHUB_SHA::7}
cat signed/SHA256SUMS-*.txt > signed/SHA256SUMS.txt
rm signed/SHA256SUMS-*.txt
notes="$(cat notes-head.md)
Signed build of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$GITHUB_SHA - in main, not yet in a DroidDeck release; for testing.
Commit: $GITHUB_SHA
VersionCode: $(sed -n 's/^ *versionCode \([0-9][0-9]*\)$/\1/p' app/build.gradle)
Each apk installs over a DroidDeck release and keeps its data. \`standard\` is the normal app; the others are the same app under another package name, installed beside it.
SHA-256:
\`\`\`
$(cat signed/SHA256SUMS.txt)
\`\`\`"
gh release delete "$tag" -R "$CI_REPO" --yes --cleanup-tag 2>/dev/null || true
gh release create "$tag" -R "$CI_REPO" --target main --prerelease \
--title "main ${GITHUB_SHA::7}" --notes "$notes" signed/*
gh release list -R "$CI_REPO" -L 200 --json tagName,publishedAt \
--jq '[.[] | select(.tagName | startswith("main-"))] | sort_by(.publishedAt) | reverse | .['"$KEEP"':][] | .tagName' |
while IFS= read -r old; do
if [ "$old" = "$tag" ]; then
continue
fi
if ! gh release delete "$old" -R "$CI_REPO" --yes --cleanup-tag 2>/dev/null; then
if gh release view "$old" -R "$CI_REPO" >/dev/null 2>&1; then
echo "::error::Could not delete old release $old"
exit 1
fi
fi
done
published=$(gh release view "$tag" -R "$CI_REPO" --json tagName,assets)
jq -e --arg tag "$tag" '.tagName == $tag and (.assets | length) > 0' <<< "$published" >/dev/null
url="$GITHUB_SERVER_URL/$CI_REPO/releases/tag/$tag"
apk=$(cd signed && ls DroidDeck-*.apk | awk 'length(a) == 0 || length($0) < length(a) { a = $0 } END { print a }')
echo "release=$url" >> "$GITHUB_OUTPUT"
echo "apk=$GITHUB_SERVER_URL/$CI_REPO/releases/download/$tag/$apk" >> "$GITHUB_OUTPUT"
echo "Published $url"
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
set -uo pipefail
MARK="<!-- droiddeck-pr-build -->"
{
echo "$MARK"
echo "## Build of this pull request"
echo
[ -f /tmp/pr-merge.md ] && cat /tmp/pr-merge.md
echo
case "$BUILD_OUTCOME:$UNIT_TEST_OUTCOME:$APK_OUTCOME" in
success:success:success)
echo "### Build passed"
echo
echo "The \`droiddeck-apk\` artifact is [on this run]($RUN_URL), signed with the public test key."
echo
if [ "$SAME_REPO" = true ]; then
echo "After the parallel regression tests pass, a release-signed tester build that installs over Stable or Preview will be published and will appear in DroidDeck's **Test builds** list. Label the PR \`no-test-build\` to skip that."
else
echo "**Maintainer:** add the \`publish-test\` label to publish a release-signed tester build that installs over Stable or Preview and appears in DroidDeck's **Test builds** list. The label is removed automatically after publishing."
fi
echo
echo "Fallback: [open **PR Release Publish**]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/workflows/pr-release-publish.yml) on \`main\` and enter PR \`$PR\`."
;;
success:success:*)
echo "### APK publication incomplete"
echo
echo "Compilation and unit tests passed, but APK publication did not complete. See [the run]($RUN_URL)."
;;
success:failure:*)
echo "### ERROR: Unit tests failed"
echo
echo "The APK compiled, but unit tests failed. No installable artifact was published."
echo
if [ -n "$TEST_DIAGNOSTICS_URL" ]; then
echo "The [unit-test-results]($TEST_DIAGNOSTICS_URL) artifact contains the test log and any generated reports."
else
echo "The test diagnostics upload did not complete. See [the test log]($RUN_URL)."
fi
;;
success:*)
echo "### Build incomplete"
echo
echo "The APK compiled, but unit tests did not complete successfully. See [the run]($RUN_URL)."
;;
failure:*)
echo "### ERROR: Build failed"
echo
echo "The compiler's errors ([full log]($RUN_URL)):"
echo
echo '```'
{ grep -E '^e: |^w: .*error|error:|CMake Error|fatal error|ninja: error|FAILED|What went wrong|Execution failed for task|Unresolved reference|Type mismatch|Unresolved' /tmp/build.log 2>/dev/null \
| sed -e 's#file:///home/runner/work/[^/]*/[^/]*/##' -e 's#/home/runner/work/[^/]*/[^/]*/##' \
| grep -v '^w: ' | awk '!seen[$0]++' | head -80; } || true
echo '```'
echo
echo "<details><summary>What went wrong (Gradle)</summary>"
echo
echo '```'
awk '/^\* What went wrong/{p=1} /^\* Try:/{p=0} p{print}' /tmp/build.log 2>/dev/null | head -60
echo '```'
echo "</details>"
;;
skipped:*|cancelled:*)
echo "### Not built"
echo
echo "The build did not run (see above, or [the run]($RUN_URL))."
;;
esac
echo
} > /tmp/pr-report.md
cat /tmp/pr-report.md >> "$GITHUB_STEP_SUMMARY"
ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate --jq ".[] | select(.body | startswith(\"$MARK\")) | .id" | head -1 || true)
if [ -n "$ID" ]; then
gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${ID}" -F body=@/tmp/pr-report.md >/dev/null
else
gh api -X POST "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -F body=@/tmp/pr-report.md >/dev/null
fi
+1 -1
View File
@@ -23,7 +23,7 @@ PACK_NAMES = ['proot', 'directaudio', 'tracking', 'xbox', 'comet']
SOURCE_PATHS = ['app/build.gradle', 'app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/assets/pulseaudio.tzst',
'tools/linuxfs', 'tools/proot', 'tools/aaudio-sink', 'tools/directaudio',
'tools/gamescope/release.env', 'tools/droiddeck-esync/release.env',
'tools/mangoapp', 'tools/msitools', 'tools/pipewire', 'tools/comet', 'tools/xbox', 'tools/components', 'tools/build_local.sh',
'tools/mangoapp', 'tools/msitools', 'tools/pipewire', 'tools/comet', 'tools/xbox', 'tools/components', 'tools/ci', 'tools/build_local.sh',
'tools/release/runtime_inputs.py', 'app/src/main/assets/desktop-removal',
'tools/quest', 'tools/vr/src', 'tools/vr/driver_droiddeck', 'tools/vr/layer', 'tools/vr/build.sh',
'app/src/main/cpp/waylandcomp/src/headpose_shm.h',
+6
View File
@@ -12,6 +12,12 @@ ROOT = Path(__file__).resolve().parents[2]
class CiChecksTest(unittest.TestCase):
def test_independent_jobs_wait_only_for_the_pinned_source(self):
jobs = yaml.safe_load((ROOT / '.github/workflows/build.yml').read_text())['jobs']
for name in ('xbox', 'components', 'python-tests'):
self.assertEqual(jobs[name]['needs'], 'source')
self.assertEqual(set(jobs['build']['needs']), {'source', 'components', 'xbox'})
def test_dependency_update_errors_block_installation(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
+27
View File
@@ -0,0 +1,27 @@
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
class CiNdkTest(unittest.TestCase):
def test_installed_ndk_must_match_the_declared_version(self):
version = (ROOT / 'tools/ci/toolchains.env').read_text().strip().split('=', 1)[1]
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
ndk = root / 'ndk' / version
ndk.mkdir(parents=True)
properties = ndk / 'source.properties'
properties.write_text('Pkg.Revision = ' + version + '\n')
output = root / 'environment'
env = dict(os.environ, ANDROID_HOME=str(root), GITHUB_ENV=str(output))
script = ROOT / 'tools/ci/resolve-ndk.sh'
subprocess.run(['bash', str(script)], env=env, check=True)
self.assertIn('NDK_VERSION=' + version, output.read_text())
self.assertIn('ANDROID_NDK_HOME=' + str(ndk), output.read_text())
properties.write_text('Pkg.Revision = unexpected\n')
self.assertNotEqual(subprocess.run(['bash', str(script)], env=env).returncode, 0)
+106
View File
@@ -0,0 +1,106 @@
import importlib.util
import io
import json
import os
from pathlib import Path
import subprocess
import tarfile
import tempfile
import unittest
from unittest.mock import patch
SPEC = importlib.util.spec_from_file_location('component_prepare', Path(__file__).resolve().parents[1] / 'components/prepare.py')
components = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(components)
class ComponentPrepareTest(unittest.TestCase):
def setUp(self):
directory = tempfile.TemporaryDirectory()
self.addCleanup(directory.cleanup)
self.root = Path(directory.name)
self.fixture(self.root)
def fixture(self, root):
for name in components.TREES:
(root / name).mkdir(parents=True)
(root / 'tools/components/recipes').mkdir(parents=True)
(root / 'input.h').write_text('#define VALUE 1\n')
(root / 'tools/components/recipes/fake.sh').write_text('printf binary > app/src/main/assets/linuxfs/demo.so\n')
spec = dict(script='tools/components/recipes/fake.sh', inputs=['input.h', 'tools/components/recipes/fake.sh'],
outputs=['app/src/main/assets/linuxfs/demo.so'])
(root / components.REGISTRY).write_text(json.dumps(dict(fake=spec)))
for name in components.LIBRARIES:
(root / name).parent.mkdir(parents=True, exist_ok=True)
(root / name).write_bytes(b'library')
subprocess.run(['git', 'init', '-q', str(root)], check=True)
subprocess.run(['git', '-C', str(root), 'config', 'core.autocrlf', 'false'], check=True)
subprocess.run(['git', '-C', str(root), 'add', '.'], check=True)
subprocess.run(['git', '-C', str(root), '-c', 'user.name=Test Developer', '-c', 'user.email=test@example.test',
'commit', '-qm', 'baseline'], check=True)
def test_recipe_failure_or_missing_output_does_not_succeed(self):
script = self.root / 'tools/components/recipes/fake.sh'
script.write_text('exit 7\n')
with self.assertRaises(subprocess.CalledProcessError):
components.prepare('fake', self.root)
script.write_text('exit 0\n')
with self.assertRaisesRegex(ValueError, 'required outputs'):
components.prepare('fake', self.root)
def test_identity_covers_source_and_recipe_changes(self):
original = components.identity('fake', self.root)
(self.root / 'input.h').write_text('#define VALUE 2\n')
changed = components.identity('fake', self.root)
self.assertNotEqual(original, changed)
script = self.root / 'tools/components/recipes/fake.sh'
script.write_text(script.read_text() + '\nprintf changed\n')
self.assertNotEqual(changed, components.identity('fake', self.root))
def test_tool_requirements_are_installed_in_one_union(self):
definitions = dict(one=dict(packages=['gcc', 'cmake']), two=dict(packages=['gcc', 'qemu-user']))
with patch.object(components, 'registry', return_value=definitions), patch.object(components.subprocess, 'run') as run:
components.prepare_tools(['one', 'two'], self.root)
run.assert_called_once_with(['bash', 'tools/ci/install-ubuntu-packages.sh', 'cmake', 'gcc', 'qemu-user'],
cwd=self.root, check=True)
def test_prepared_handoff_verifies_every_file_and_source_tree(self):
components.prepare('fake', self.root)
archive = self.root / 'prepared.tar.gz'
components.pack(archive, self.root)
tree = subprocess.check_output(['git', 'rev-parse', 'HEAD^{tree}'], cwd=self.root, text=True).strip()
target = self.root / 'target'
self.fixture(target)
components.unpack(archive, tree, target)
self.assertEqual((target / 'app/src/main/assets/linuxfs/demo.so').read_bytes(), b'binary')
with self.assertRaisesRegex(ValueError, 'pinned source'):
components.unpack(archive, 'wrong-tree', target)
def test_tampered_component_bytes_fail_handoff_verification(self):
components.prepare('fake', self.root)
archive = self.root / 'prepared.tar.gz'
components.pack(archive, self.root)
corrupt = self.root / 'corrupt.tar.gz'
with tarfile.open(archive) as original, tarfile.open(corrupt, 'w:gz') as output:
for entry in original:
content = original.extractfile(entry).read()
if entry.name.endswith('demo.so'):
content = b'tampered'
entry.size = len(content)
output.addfile(entry, io.BytesIO(content))
target = self.root / 'target'
self.fixture(target)
tree = subprocess.check_output(['git', 'rev-parse', 'HEAD^{tree}'], cwd=self.root, text=True).strip()
with self.assertRaisesRegex(ValueError, 'file hashes'):
components.unpack(corrupt, tree, target)
def test_archive_cannot_overwrite_source_through_traversal(self):
archive = self.root / 'bad.tar.gz'
with tarfile.open(archive, mode='w:gz') as tar:
name = 'app/src/main/assets/linuxfs/../../../../../input.h'
entry = tarfile.TarInfo(name)
entry.size = 3
tar.addfile(entry, io.BytesIO(b'bad'))
with self.assertRaisesRegex(ValueError, 'Unsafe'):
components.unpack(archive, 'tree', self.root)
self.assertEqual((self.root / 'input.h').read_text(), '#define VALUE 1\n')
+7 -3
View File
@@ -1160,8 +1160,12 @@ class BundleTest(unittest.TestCase):
except ImportError:
self.skipTest("PyYAML is not installed")
apk = yaml.safe_load((self.ROOT / ".github/workflows/build.yml").read_text())
steps = {step.get("name"): step for step in apk["jobs"]["build"]["steps"]}
bundle = steps["Bundle the droiddeck-esync packs"]
steps = {step.get("name"): step for step in apk["jobs"]["components"]["steps"]}
import json
recipes = json.loads((self.ROOT / "tools/components/recipes.json").read_text())
bundle = dict(steps["Bundle the droiddeck-esync packs"])
self.assertIn("prepare.py esync", bundle["run"])
bundle["run"] = (self.ROOT / recipes["esync"]["script"]).read_text()
self.assertNotIn("if", bundle)
self.assertIn("tools/droiddeck-esync/release.env", bundle["run"])
self.assertIn('release_url="https://github.com/${SYNC_BUNDLE_REPO}/releases/download/${SYNC_BUNDLE_TAG}"', bundle["run"])
@@ -1169,7 +1173,7 @@ class BundleTest(unittest.TestCase):
self.assertIn('curl -fsSL --retry 3 -o sync-index.json "$release_url/index.json"', bundle["run"])
self.assertIn("sha256sum -c -", bundle["run"])
self.assertNotIn("revoked.txt", bundle["run"])
stage = next(step for step in apk["jobs"]["build"]["steps"] if "overlay/usr/local/bin/droiddeck-*" in step.get("run", ""))
stage = {"run": (self.ROOT / recipes["runtime"]["script"]).read_text()}
self.assertIn("tools/linuxfs/overlay/usr/local/bin/droiddeck-* ", stage["run"])
local = (self.ROOT / "tools/build_local.sh").read_text()
self.assertIn("tools/linuxfs/overlay/usr/local/bin/droiddeck-* ", local)
+31
View File
@@ -100,6 +100,13 @@ class PreloadSnapshotTest(unittest.TestCase):
result, _, _ = self.restore()
self.assertEqual(result, dict(arm64=False, x86=True))
def test_shell_newline_changes_invalidate_the_recipe(self):
script = self.root / 'tools/components/build-arm64-preloads.sh'
original = snapshots.recipe(self.root)
text = script.read_text()
script.write_text(text.replace('-ldl\n', '-ldl ', 1))
self.assertNotEqual(original, snapshots.recipe(self.root))
def test_new_manifest_requires_main_build_proof(self):
result, _, prove = self.restore('HEAD~1')
self.assertTrue(all(result.values()))
@@ -112,6 +119,18 @@ class PreloadSnapshotTest(unittest.TestCase):
with self.assertRaisesRegex(ValueError, 'original main APK'):
snapshots.prove_origin(data, self.root / 'cache', self.root)
def test_modified_input_policy_cannot_omit_producer_dependencies(self):
data = copy.deepcopy(self.data)
data['origin']['commit'] = self.base
omitted = snapshots.COMMON[0]
del data['groups']['arm64'][omitted]
with patch.object(snapshots, 'artifact_bytes', return_value=self.binaries), \
patch.object(snapshots, 'run', return_value=b''), \
patch.object(snapshots, 'source_files', side_effect=lambda group, *args: data['groups'][group]), \
patch.object(snapshots, 'producer_sources', return_value={omitted: self.data['groups']['arm64'][omitted]}):
with self.assertRaisesRegex(ValueError, 'producing build dependency'):
snapshots.prove_origin(data, self.root / 'cache', self.root)
def test_non_main_or_unsuccessful_run_cannot_prove_origin(self):
build = dict(event='pull_request', head_branch='main', conclusion='success', head_sha='abc',
path='.github/workflows/build.yml', repository=dict(full_name=snapshots.REPO))
@@ -139,6 +158,18 @@ class PreloadSnapshotTest(unittest.TestCase):
patch.object(snapshots.urllib.request, 'urlopen', side_effect=error):
self.assertEqual(snapshots.restore(self.root), dict(arm64=False, x86=False))
def test_transient_download_failure_rebuilds_but_integrity_errors_fail(self):
errors = [TimeoutError('network'), snapshots.http.client.IncompleteRead(b'partial'),
snapshots.urllib.error.HTTPError('https://example.test', 503, 'unavailable', {}, None)]
for error in errors:
with self.subTest(error=type(error).__name__), patch.dict(os.environ, COMPONENT_TRUSTED_BASE=self.base), \
patch.object(snapshots.urllib.request, 'urlopen', side_effect=error):
self.assertEqual(snapshots.restore(self.root), dict(arm64=False, x86=False))
with patch.dict(os.environ, COMPONENT_TRUSTED_BASE=self.base), \
patch.object(snapshots.urllib.request, 'urlopen', return_value=io.BytesIO(b'corrupt')):
with self.assertRaisesRegex(ValueError, 'checksum'):
snapshots.restore(self.root)
def test_manual_rebuild_does_not_download_snapshot(self):
with patch.dict(os.environ, FORCE_PRELOAD_REBUILD='true'), \
patch.object(snapshots.urllib.request, 'urlopen') as fetch:
+15 -3
View File
@@ -31,14 +31,14 @@ class XboxBuildSourceTest(unittest.TestCase):
git('switch', '-q', 'main')
(root / 'backend').write_text('merged backend')
git('commit', '-qam', 'backend changed after branch')
for name in ('xbox', 'apk', 'tests'):
for name in ('source', 'xbox', 'components', 'apk', 'tests', 'dispatch'):
git('clone', '-q', str(root), str(root / name))
git('checkout', '-q', head, cwd=root / name)
output = root / 'output'
env['GITHUB_OUTPUT'] = str(output)
subprocess.run(['bash', '-c', step('xbox', 'Pin and merge the APK source')], cwd=root / 'xbox', env=env, check=True, capture_output=True)
subprocess.run(['bash', '-c', step('source', 'Pin and merge the APK source')], cwd=root / 'source', env=env, check=True, capture_output=True)
base = output.read_text().strip().split('=', 1)[1]
tree = git('rev-parse', 'HEAD^{tree}', cwd=root / 'xbox')
tree = git('rev-parse', 'HEAD^{tree}', cwd=root / 'source')
(root / 'backend').write_text('newer main, after backend build')
git('commit', '-qam', 'main advanced again')
env.update(BUILD_BASE_SHA=base, XBOX_SOURCE_TREE=tree)
@@ -50,3 +50,15 @@ class XboxBuildSourceTest(unittest.TestCase):
subprocess.run(['bash', '-c', step('python-tests', 'Merge the pinned APK base')], cwd=root / 'tests', env=env, check=True, capture_output=True)
subprocess.run(['bash', '-c', step('python-tests', 'Verify backend and test source match')], cwd=root / 'tests', env=env, check=True, capture_output=True)
self.assertEqual(git('rev-parse', 'HEAD^{tree}', cwd=root / 'tests'), tree)
env['SOURCE_TREE'] = tree
for job in ('xbox', 'components'):
subprocess.run(['bash', '-c', step(job, 'Merge the pinned APK base')], cwd=root / job, env=env, check=True, capture_output=True)
subprocess.run(['bash', '-c', step(job, 'Verify pinned source tree')], cwd=root / job, env=env, check=True, capture_output=True)
self.assertEqual(git('rev-parse', 'HEAD^{tree}', cwd=root / job), tree)
trusted = root / 'trusted-output'
env['GITHUB_OUTPUT'] = str(trusted)
subprocess.run(['bash', '-c', step('source', 'Resolve trusted manifest base')], cwd=root / 'dispatch', env=env, check=True, capture_output=True)
self.assertEqual(trusted.read_text().strip(), 'base-sha=' + git('rev-parse', 'main'))
self.assertNotEqual(git('rev-parse', 'HEAD', cwd=root / 'dispatch'), git('rev-parse', 'main'))