Files
DroidDeck/.github/workflows/build-droiddeck-esync-packs.yml
T
MaxsTechReview 8edf05b19a Add the droiddeck-esync pack pipeline (#189)
Builds esync packs for new Valve, GE and CachyOS Proton 11 releases on arm64
runners, smoke-tests them against stock with esstress, and on main publishes
each pack and a signed index as non-latest releases. A watcher checks Steam
and the GE/CachyOS releases every 20 minutes and starts a build for new ones.
The app side lands separately with feat/userspace-esync-ntsync.
2026-10-04 02:58:55 +00:00

450 lines
19 KiB
YAML

name: Build droiddeck-esync packs
on:
schedule:
- cron: '17 5 * * *'
push:
branches: [ main ]
paths:
- 'tools/droiddeck-esync/**'
- '.github/workflows/build-droiddeck-esync-packs.yml'
workflow_dispatch:
inputs:
flavor:
description: 'Proton flavor to look at'
type: choice
options: [ all, valve, ge, cachyos ]
default: all
ref:
description: 'One release tag to build (e.g. GE-Proton11-7); empty = the newest builds of each family'
required: false
default: ''
publish:
description: 'Publish the packs and sign a new index'
type: boolean
default: false
permissions:
contents: write
concurrency:
group: droiddeck-esync-packs-${{ (github.repository == 'Droid-Deck/DroidDeck' && github.ref == 'refs/heads/main' && (github.event_name != 'workflow_dispatch' || inputs.publish)) && 'publish' || format('build-{0}', github.run_id) }}
cancel-in-progress: false
env:
SYNC_BASE_URL: https://github.com/Droid-Deck/DroidDeck/releases/download
jobs:
discover:
if: ${{ github.repository == 'Droid-Deck/DroidDeck' || github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-24.04
timeout-minutes: 15
permissions:
contents: read
outputs:
matrix: ${{ steps.discover.outputs.matrix }}
count: ${{ steps.discover.outputs.count }}
unindexed: ${{ steps.discover.outputs.unindexed }}
publish: ${{ steps.discover.outputs.publish }}
steps:
- uses: actions/checkout@v4
- name: Find the Proton builds without a pack
id: discover
env:
GH_TOKEN: ${{ github.token }}
FLAVOR: ${{ inputs.flavor || 'all' }}
REF: ${{ inputs.ref }}
ASKED: ${{ github.event_name != 'workflow_dispatch' || inputs.publish }}
PUBLISH: ${{ github.repository == 'Droid-Deck/DroidDeck' && github.ref == 'refs/heads/main' && (github.event_name != 'workflow_dispatch' || inputs.publish) }}
run: |
set -euo pipefail
if [ "$ASKED" = true ] && [ "$PUBLISH" != true ]; then
echo "::notice::packs are published only from main of Droid-Deck/DroidDeck; this run only builds them"
fi
echo "publish=$PUBLISH" >> "$GITHUB_OUTPUT"
python3 tools/droiddeck-esync/discover.py matrix --flavor "$FLAVOR" --ref "$REF" \
--index-release droiddeck-esync-index --repo "$GITHUB_REPOSITORY" --revoked tools/droiddeck-esync/revoked.txt
build:
name: ${{ matrix.flavor }} ${{ matrix.tag }}
needs: discover
if: needs.discover.outputs.count != '0'
runs-on: ubuntu-24.04-arm
timeout-minutes: 360
permissions:
contents: read
strategy:
fail-fast: false
max-parallel: 2
matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
env:
FLAVOR: ${{ matrix.flavor }}
REPO: ${{ matrix.repo }}
TAG: ${{ matrix.tag }}
ASSET: ${{ matrix.asset }}
VERSION_TOKEN: ${{ matrix.token }}
DEPOT_APP: ${{ matrix.depot_app }}
REV: ${{ matrix.rev }}
SERIES: ${{ matrix.series }}
PROFILE: ${{ matrix.profile }}
SOURCE_MATCH: ${{ matrix.source_match }}
SUBMODULES: ${{ matrix.submodules }}
CLONE: ${{ matrix.clone }}
PREP: ${{ matrix.prep }}
BUILD_DIR: ${{ matrix.build_dir }}
HAVE_DEPOT_LOGIN: ${{ secrets.STEAM_DEPOT_USER != '' && secrets.STEAM_DEPOT_TOKEN != '' }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Check the build script and the patch series
run: |
set -euo pipefail
test -f tools/droiddeck-esync/build-pack.sh
ls "tools/droiddeck-esync/patches/$SERIES"/*.patch
- name: Install the host tools
run: |
set -euo pipefail
sudo apt-get update -qq
sudo apt-get install -y -qq autoconf automake git patch perl python3 rsync unzip wget xz-utils zstd >/dev/null
- name: Fetch the stock binaries
if: matrix.asset != ''
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
rm -rf stock-download stock
mkdir -p stock-download stock
gh release download "$TAG" -R "$REPO" -p "$ASSET" -D stock-download
tar -xf "stock-download/$ASSET" -C stock
rm -rf stock-download
mapfile -t tops < <(find stock -mindepth 1 -maxdepth 1 -type d)
if [ "${#tops[@]}" != 1 ]; then
echo "::error::$ASSET does not hold exactly one Proton directory"
exit 1
fi
dir=$(realpath "${tops[0]}")
test -f "$dir/files/lib/wine/aarch64-unix/ntdll.so"
test -f "$dir/files/bin-arm64/wineserver"
line=$(head -n1 "$dir/version" | tr -d '\r')
if ! grep -qE '^[0-9]+[[:space:]]+[[:graph:]]+' <<<"$line"; then
echo "::error::$ASSET has no usable version file"
exit 1
fi
echo "Stock $line"
{
echo "STOCK_DIR=$dir"
echo "VERSION_LINE=$line"
} >> "$GITHUB_ENV"
- name: Fetch the stock depot (optional, Valve only, needs STEAM_DEPOT_USER, STEAM_DEPOT_TOKEN and DEPOT_DOWNLOADER_SHA256)
if: matrix.flavor == 'valve' && env.HAVE_DEPOT_LOGIN == 'true'
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
DEPOT_DOWNLOADER_TAG: DepotDownloader_3.4.0
DEPOT_DOWNLOADER_SHA256: ${{ vars.DEPOT_DOWNLOADER_SHA256 }}
STEAM_DEPOT_USER: ${{ secrets.STEAM_DEPOT_USER }}
STEAM_DEPOT_TOKEN: ${{ secrets.STEAM_DEPOT_TOKEN }}
run: |
set -euo pipefail
if ! grep -qxE '[0-9a-f]{64}' <<<"$DEPOT_DOWNLOADER_SHA256"; then
echo "::warning::set the repository variable DEPOT_DOWNLOADER_SHA256 to the sha256 of DepotDownloader-linux-arm64.zip from $DEPOT_DOWNLOADER_TAG to fetch the depot"
exit 0
fi
rm -rf depotdownloader depot
gh release download "$DEPOT_DOWNLOADER_TAG" -R SteamRE/DepotDownloader -p 'DepotDownloader-linux-arm64.zip' -D depotdownloader
echo "$DEPOT_DOWNLOADER_SHA256 depotdownloader/DepotDownloader-linux-arm64.zip" | sha256sum -c -
unzip -q depotdownloader/DepotDownloader-linux-arm64.zip -d depotdownloader
chmod +x depotdownloader/DepotDownloader
printf '%s\n' version files/lib/wine/aarch64-unix/ntdll.so files/bin-arm64/wineserver > depot-files.txt
timeout 30m depotdownloader/DepotDownloader -app "$DEPOT_APP" -os linux -all-archs \
-username "$STEAM_DEPOT_USER" -password "$STEAM_DEPOT_TOKEN" \
-filelist depot-files.txt -dir depot </dev/null
test -f depot/files/lib/wine/aarch64-unix/ntdll.so
test -f depot/files/bin-arm64/wineserver
line=$(head -n1 depot/version | tr -d '\r')
token=$(awk '{ print $2 }' <<<"$line")
if [ "$token" != "$VERSION_TOKEN" ]; then
echo "::notice::app $DEPOT_APP is at $token, not $VERSION_TOKEN; this pack matches by source"
exit 0
fi
echo "Depot $line"
{
echo "STOCK_DIR=$PWD/depot"
echo "VERSION_LINE=$line"
} >> "$GITHUB_ENV"
- name: Prepare the wine source
run: |
set -euo pipefail
git config --global user.name "ci"
git config --global user.email "ci@users.noreply.github.com"
rm -rf src
git init -q src
git -C src remote add origin "https://github.com/$REPO.git"
git -C src fetch -q --depth 1 origin "refs/tags/$TAG:refs/tags/$TAG"
git -C src checkout -q "refs/tags/$TAG"
read -r -a modules <<<"$SUBMODULES"
case "$CLONE" in
shallow) history=(--depth 1) ;;
treeless) history=(--filter=tree:0) ;;
*) echo "::error::unknown clone mode $CLONE"; exit 1 ;;
esac
git -C src submodule update -q --init "${history[@]}" -- "${modules[@]}"
if [ -n "$PREP" ]; then
if ! (cd src && bash "$PREP") > prep.log 2>&1; then
tail -n 200 prep.log
echo "::error::$PREP failed"
exit 1
fi
fi
test -f src/wine/server/protocol.def
image=$(python3 tools/droiddeck-esync/discover.py sdk-image src/Makefile.in)
commit=$(git -C src rev-parse HEAD)
wine_commit=$(git -C src rev-parse HEAD:wine)
line=${VERSION_LINE:-}
if [ -z "$line" ]; then
if [ -z "$VERSION_TOKEN" ]; then
echo "::error::no stock version file and no version token for $TAG"
exit 1
fi
line="$(git -C src log -1 --format=%ct HEAD) $VERSION_TOKEN"
fi
echo "SDK image $image, $REPO $commit, wine $wine_commit, version $line"
{
echo "SDK_IMAGE=$image"
echo "SOURCE_COMMIT=$commit"
echo "WINE_COMMIT=$wine_commit"
echo "VERSION_LINE=$line"
echo "SOURCE_DATE_EPOCH=${line%%[[:space:]]*}"
} >> "$GITHUB_ENV"
- name: Build the control and the patched binaries
run: |
set -euo pipefail
docker pull -q "$SDK_IMAGE"
for kind in control patched; do
work=$PWD/build/work-$kind
out=$PWD/build/$kind
sudo rm -rf "$work" "$out"
mkdir -p "$work" "$out"
if [ "$kind" = patched ]; then
bash tools/droiddeck-esync/build-pack.sh prepare "$PWD/src/wine" "$work" "$PWD/tools/droiddeck-esync/patches/$SERIES"
else
bash tools/droiddeck-esync/build-pack.sh prepare "$PWD/src/wine" "$work"
fi
docker run --rm \
-v "$work:$BUILD_DIR" -v "$out:/out" -v "$PWD/tools/droiddeck-esync/build-pack.sh:/build-pack.sh:ro" \
-w "$BUILD_DIR" -e HOME=/tmp -e LANG=C.UTF-8 -e JOBS="$(nproc)" \
-e SOURCE_DATE_EPOCH -e SYNC_PROFILE="$PROFILE" \
"$SDK_IMAGE" /bin/bash /build-pack.sh build "$BUILD_DIR" /out
done
sudo chown -R "$(id -u):$(id -g)" build
sha256sum build/*/files/lib/wine/aarch64-unix/ntdll.so build/*/files/bin-arm64/wineserver
- name: Make the pack
run: |
set -euo pipefail
source_json=$(python3 -c 'import json, os; print(json.dumps({"repo": os.environ["REPO"], "ref": os.environ["TAG"], "asset": os.environ["ASSET"], "commit": os.environ["SOURCE_COMMIT"], "wine_commit": os.environ["WINE_COMMIT"], "sdk_image": os.environ["SDK_IMAGE"]}, sort_keys=True))')
match=()
if [ -z "${STOCK_DIR:-}" ] && [ "$SOURCE_MATCH" = true ]; then
match=(--source-match)
fi
rm -rf out
id=$(python3 tools/droiddeck-esync/make_pack.py --flavor "$FLAVOR" --rev "$REV" --stock-dir "${STOCK_DIR:-}" \
--version-line "$VERSION_LINE" --control build/control --patched build/patched \
--source-json "$source_json" --patch-series "$SERIES" --out out "${match[@]}")
echo "Pack \`$id\` from $FLAVOR $TAG" >> "$GITHUB_STEP_SUMMARY"
ls -l out
- name: Smoke-test the pack on the release it was made for
if: matrix.asset != ''
run: |
set -euo pipefail
id=$(basename out/*.tzst .tzst)
docker run --rm -v "$PWD:/work" -w /work "$SDK_IMAGE" \
/opt/llvm-mingw/bin/aarch64-w64-mingw32-clang -O2 -o esstress.exe tools/droiddeck-esync/tests/esstress.c
set +e
bash tools/droiddeck-esync/smoke-test.sh "$STOCK_DIR" "out/$id.tzst" esstress.exe smoke
status=$?
set -e
case $status in
0) echo "Smoke test passed for \`$id\`" >> "$GITHUB_STEP_SUMMARY" ;;
3) echo "::warning::stock $TAG does not run on this runner, so $id could not be smoke-tested" ;;
*) echo "::error::$id failed the smoke test on $TAG"; exit 1 ;;
esac
- uses: actions/upload-artifact@v4
with:
name: droiddeck-esync-pack-${{ matrix.key }}
path: out/
if-no-files-found: error
- uses: actions/upload-artifact@v4
if: failure()
with:
name: droiddeck-esync-logs-${{ matrix.key }}
path: |
prep.log
build/work-*/*.log
smoke/*.log
if-no-files-found: ignore
publish:
name: Publish the packs
needs: [ discover, build ]
if: ${{ !cancelled() && needs.discover.result == 'success' && needs.discover.outputs.publish == 'true' && needs.discover.outputs.count != '0' }}
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/download-artifact@v4
with:
pattern: droiddeck-esync-pack-*
path: packs
- name: Upload the packs to their releases
env:
GH_TOKEN: ${{ github.token }}
MATRIX: ${{ needs.discover.outputs.matrix }}
run: |
set -euo pipefail
mkdir -p packs
python3 tools/droiddeck-esync/discover.py packs --matrix "$MATRIX" packs > packs.tsv
if [ ! -s packs.tsv ]; then
echo "::notice::no build made a pack"
exit 0
fi
while IFS=$'\t' read -r -u 3 flavor dir id; do
tag="droiddeck-esync-$flavor"
if ! gh release view "$tag" -R "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
gh release create "$tag" -R "$GITHUB_REPOSITORY" --latest=false --prerelease=false --title "$tag" \
--notes "droiddeck-esync packs for $flavor Proton builds (tools/droiddeck-esync). The app fetches them through the signed index in the droiddeck-esync-index release." \
|| gh release view "$tag" -R "$GITHUB_REPOSITORY" >/dev/null
fi
gh release upload "$tag" -R "$GITHUB_REPOSITORY" "$dir/$id.tzst" --clobber
gh release upload "$tag" -R "$GITHUB_REPOSITORY" "$dir/$id.json" --clobber
gh release edit "$tag" -R "$GITHUB_REPOSITORY" --latest=false --prerelease=false
echo "Published \`$id\` to $tag" >> "$GITHUB_STEP_SUMMARY"
done 3< packs.tsv
- name: Record the builds that made no pack
env:
GH_TOKEN: ${{ github.token }}
MATRIX: ${{ needs.discover.outputs.matrix }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
mkdir -p packs failed
python3 tools/droiddeck-esync/discover.py failures --matrix "$MATRIX" packs > failed.tsv
while IFS=$'\t' read -r -u 3 flavor marker; do
tag="droiddeck-esync-$flavor"
if ! gh release view "$tag" -R "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
gh release create "$tag" -R "$GITHUB_REPOSITORY" --latest=false --prerelease=false --title "$tag" \
--notes "droiddeck-esync packs for $flavor Proton builds (tools/droiddeck-esync). The app fetches them through the signed index in the droiddeck-esync-index release." \
|| gh release view "$tag" -R "$GITHUB_REPOSITORY" >/dev/null
fi
echo "$RUN_URL" > "failed/$marker"
gh release upload "$tag" -R "$GITHUB_REPOSITORY" "failed/$marker" --clobber
echo "::warning::$marker: no pack was made; discovery skips this build for a day unless rev or the patch series changes"
done 3< failed.tsv
index:
name: Sign the index
needs: [ discover, build, publish ]
if: ${{ !cancelled() && needs.discover.result == 'success' && needs.discover.outputs.publish == 'true' && (needs.discover.outputs.count != '0' || needs.discover.outputs.unindexed != '0') }}
runs-on: ubuntu-24.04
timeout-minutes: 30
environment: droiddeck-esync-signing
steps:
- uses: actions/checkout@v4
- name: Collect the published packs
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
fetch() {
local out
if out=$(gh release download "$1" -R "$GITHUB_REPOSITORY" -p "$2" -D "$3" --clobber 2>&1); then
return 0
fi
if grep -qiE 'release not found|no assets match' <<<"$out"; then
echo "::notice::$1 has no $2 yet"
return 0
fi
echo "$out" >&2
return 1
}
rm -rf entries previous bundle-packs out
mkdir -p entries previous bundle-packs
for flavor in valve ge cachyos; do
fetch "droiddeck-esync-$flavor" '*.json' entries
done
fetch droiddeck-esync-index index.json previous
fetch droiddeck-esync-index index.json.sig previous
args=(--entries entries --base-url "$SYNC_BASE_URL" --revoked tools/droiddeck-esync/revoked.txt)
if [ -f previous/index.json ]; then
args+=(--previous previous/index.json --previous-sig previous/index.json.sig)
fi
python3 tools/droiddeck-esync/make_index.py "${args[@]}" --list-bundle > bundle.tsv
while IFS=$'\t' read -r flavor id; do
fetch "droiddeck-esync-$flavor" "$id.tzst" bundle-packs
done < bundle.tsv
echo "Entries: $(find entries -name '*.json' | wc -l); bundled packs: $(wc -l < bundle.tsv)"
- name: Sign the index and make the apk bundle
id: index
env:
SYNC_INDEX_KEY: ${{ secrets.SYNC_INDEX_KEY }}
run: |
set -euo pipefail
if [ -z "$SYNC_INDEX_KEY" ]; then
echo "::error::the droiddeck-esync-signing environment has no SYNC_INDEX_KEY secret"
exit 1
fi
args=(--entries entries --base-url "$SYNC_BASE_URL" --revoked tools/droiddeck-esync/revoked.txt)
if [ -f previous/index.json ]; then
args+=(--previous previous/index.json --previous-sig previous/index.json.sig)
fi
python3 tools/droiddeck-esync/make_index.py "${args[@]}" --out out --bundle-packs-dir bundle-packs
- name: Publish the index
env:
GH_TOKEN: ${{ github.token }}
CHANGED: ${{ steps.index.outputs.changed }}
BUNDLE: ${{ steps.index.outputs.bundle }}
run: |
set -euo pipefail
if [ "$CHANGED" != true ]; then
echo "::notice::no pack was added, replaced or revoked; the published index stays"
exit 0
fi
tag=droiddeck-esync-index
if ! gh release view "$tag" -R "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
gh release create "$tag" -R "$GITHUB_REPOSITORY" --latest=false --prerelease=false --title "$tag" \
--notes "Signed index of the droiddeck-esync packs (tools/droiddeck-esync). The app fetches index.json and index.json.sig; the apk build bundles the bundle-*.tzst that tools/droiddeck-esync/release.env names." \
|| gh release view "$tag" -R "$GITHUB_REPOSITORY" >/dev/null
fi
gh release upload "$tag" -R "$GITHUB_REPOSITORY" "out/$BUNDLE"
gh release upload "$tag" -R "$GITHUB_REPOSITORY" out/index.json out/index.json.sig --clobber
sum=$(sha256sum "out/$BUNDLE" | cut -d' ' -f1)
gh release edit "$tag" -R "$GITHUB_REPOSITORY" --latest=false --prerelease=false
{
echo "### tools/droiddeck-esync/release.env"
echo
echo '```'
echo "SYNC_BUNDLE_TAG=$tag"
echo "SYNC_BUNDLE_ASSET=$BUNDLE"
echo "SYNC_BUNDLE_SHA256=$sum"
echo '```'
} | tee -a "$GITHUB_STEP_SUMMARY"