From 27bdc68867309221ede2e2ab44fe0ada9d1631f5 Mon Sep 17 00:00:00 2001 From: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:31:20 -0400 Subject: [PATCH] ci: faster pipeline and fewer network flakes (#1719) --- .github/workflows/build.yml | 12 ++++--- tools/build_local.sh | 10 +++--- tools/ci/run-python-tests.sh | 32 +++++++++++++++++++ tools/comet/build.sh | 2 +- tools/components/recipes/esync.sh | 4 +-- tools/components/recipes/pulseaudio-source.sh | 2 +- tools/components/recipes/runtime.sh | 10 +++--- tools/directaudio/fetch.sh | 4 +-- tools/proot/build.sh | 2 +- tools/tests/test_esync_packs.py | 4 +-- 10 files changed, 59 insertions(+), 23 deletions(-) create mode 100755 tools/ci/run-python-tests.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0dd6d06d16..2fe5e15f17 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,8 @@ on: - '**/*.md' - 'LICENSE' - '.gitignore' + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' pull_request: branches: [ main ] types: [ opened, synchronize, reopened, ready_for_review ] @@ -15,6 +17,8 @@ on: - '**/*.md' - 'LICENSE' - '.gitignore' + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' workflow_dispatch: inputs: rebuild_preloads: @@ -28,8 +32,8 @@ permissions: pull-requests: write concurrency: - group: build-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: build-${{ github.event.pull_request.number || (github.event_name == 'push' && 'main') || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} jobs: source: @@ -404,7 +408,7 @@ jobs: - uses: gradle/actions/setup-gradle@v4 with: cache-read-only: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} - cache-cleanup: on-success + cache-cleanup: ${{ github.event_name == 'pull_request' && 'never' || 'on-success' }} - name: Resolve the pinned Android NDK run: bash tools/ci/resolve-ndk.sh @@ -681,7 +685,7 @@ jobs: timeout 10 /tmp/test-presenter-shutdown - name: Steam import and game environment tests - run: python3 -m unittest discover -s tools/tests -p 'test_*.py' + run: bash tools/ci/run-python-tests.sh checks: name: Compile and test APK diff --git a/tools/build_local.sh b/tools/build_local.sh index a980d70a6a..c36035e519 100755 --- a/tools/build_local.sh +++ b/tools/build_local.sh @@ -221,7 +221,7 @@ fi . "${repo_root}/tools/linuxfs/uruntime.env" uruntime_binary=$(cached "${URUNTIME_SHA256}" "${URUNTIME_ASSET}" \ - bash -c 'curl -fsSL --retry 3 -o "$out" "$0"' "https://github.com/VHSgunzo/uruntime/releases/download/${URUNTIME_VERSION}/${URUNTIME_ASSET}") + bash -c 'curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o "$out" "$0"' "https://github.com/VHSgunzo/uruntime/releases/download/${URUNTIME_VERSION}/${URUNTIME_ASSET}") mkdir -p "${linuxfs_dir}/usr/local/lib/droiddeck" "${linuxfs_dir}/usr/local/share/licenses/uruntime" install -m644 "${uruntime_binary}" "${linuxfs_dir}/usr/local/lib/droiddeck/uruntime" install -m644 "${repo_root}/tools/linuxfs/licenses/uruntime-LICENSE" "${linuxfs_dir}/usr/local/share/licenses/uruntime/LICENSE" @@ -256,7 +256,7 @@ mkdir -p "${msi_pkgs}" "${msi_dir}" while read -r package_sha256 package_url; do [[ -n "${package_url}" ]] || continue package_archive=$(cached "${package_sha256}" "$(basename "${package_url}")" \ - bash -c 'curl -fsSL --retry 3 -o "$out" "$0"' "${package_url}") + bash -c 'curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o "$out" "$0"' "${package_url}") tar -xJf "${package_archive}" -C "${msi_pkgs}" done < <(grep -v '^#' "${repo_root}/tools/msitools/packages.txt") for tool in msiinfo cabextract; do install -m644 "${msi_pkgs}/usr/bin/${tool}" "${msi_dir}/${tool}"; done @@ -276,7 +276,7 @@ mkdir -p "${mango_dir}" "${mango_pkgs}" while read -r package_sha256 package_url; do [[ -n "${package_url}" ]] || continue package_archive=$(cached "${package_sha256}" "$(basename "${package_url}")" \ - bash -c 'curl -fsSL --retry 3 -o "$out" "$0"' "${package_url}") + bash -c 'curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o "$out" "$0"' "${package_url}") zstd -dc "${package_archive}" | tar -xf - -C "${mango_pkgs}" done < <(grep -v '^#' "${repo_root}/tools/mangoapp/packages.txt") install -m644 "${mango_pkgs}/usr/bin/mangoapp" "${mango_dir}/mangoapp" @@ -294,7 +294,7 @@ mkdir -p "${pw_pkgs}" while read -r package_sha256 package_url; do [[ -n "${package_url}" ]] || continue package_archive=$(cached "${package_sha256}" "$(basename "${package_url}")" \ - bash -c 'curl -fsSL --retry 3 -o "$out" "$0"' "${package_url}") + bash -c 'curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o "$out" "$0"' "${package_url}") zstd -dc "${package_archive}" | tar -xf - -C "${pw_pkgs}" done < <(grep -v '^#' "${repo_root}/tools/pipewire/packages.txt") "${repo_root}/tools/pipewire/assemble.sh" "${pw_pkgs}" "${linuxfs_dir}/usr/local/lib/droiddeck-pipewire.tzst" @@ -303,7 +303,7 @@ pa_source=${DROIDDECK_PA13_SOURCE_DIR:-"${staging_dir}/pulseaudio-13.0"} if [[ -z "${DROIDDECK_PA13_SOURCE_DIR:-}" ]]; then pa_tarball="${cache_dir}/pulseaudio-13.0.tar.gz" if [[ ! -s "${pa_tarball}" ]] || ! tar -tzf "${pa_tarball}" >/dev/null 2>&1; then - curl -fsSL -o "${pa_tarball}.part" \ + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o "${pa_tarball}.part" \ https://github.com/pulseaudio/pulseaudio/archive/refs/tags/v13.0.tar.gz mv "${pa_tarball}.part" "${pa_tarball}" fi diff --git a/tools/ci/run-python-tests.sh b/tools/ci/run-python-tests.sh new file mode 100755 index 0000000000..c2f5a41695 --- /dev/null +++ b/tools/ci/run-python-tests.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -uo pipefail +cd "$(dirname "$0")/../.." +groups=${PYTHON_TEST_GROUPS:-3} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +index=0 +while read -r file; do + module=$(basename "$file" .py) + echo "$module" >> "$work/group-$((index % groups)).list" + index=$((index + 1)) +done < <(ls -S tools/tests/test_*.py) +pids=() +for list in "$work"/group-*.list; do + ( + PYTHONPATH=tools/tests python3 -m unittest $(cat "$list") > "${list%.list}.log" 2>&1 + echo $? > "${list%.list}.status" + ) & + pids+=($!) +done +wait "${pids[@]}" +failed=0 +for list in "$work"/group-*.list; do + echo "::group::$(basename "$list"): $(tr '\n' ' ' < "$list" | cut -c1-120)" + cat "${list%.list}.log" + echo "::endgroup::" + if [ "$(cat "${list%.list}.status")" != 0 ]; then + failed=1 + echo "::error::$(basename "$list") failed" + fi +done +exit "$failed" diff --git a/tools/comet/build.sh b/tools/comet/build.sh index b32fc75862..044aae2589 100644 --- a/tools/comet/build.sh +++ b/tools/comet/build.sh @@ -16,7 +16,7 @@ cargo build --locked --release --target aarch64-unknown-linux-gnu install -m644 target/aarch64-unknown-linux-gnu/release/comet "$out/comet" zip="$cache/dummy-service.zip" if ! echo "7fc5facd1fce4d516cd3ce527e7e4dbe0bfd17649cea805e8ea7c37d4ba58d9a $zip" | sha256sum -c - >/dev/null 2>&1; then - curl -fsSL --retry 3 -o "$zip" https://github.com/imLinguin/comet/releases/download/v0.3.2/dummy-service.zip + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o "$zip" https://github.com/imLinguin/comet/releases/download/v0.3.2/dummy-service.zip fi echo "7fc5facd1fce4d516cd3ce527e7e4dbe0bfd17649cea805e8ea7c37d4ba58d9a $zip" | sha256sum -c - unzip -o "$zip" GalaxyCommunication.exe update-permissions.exe -d "$out" diff --git a/tools/components/recipes/esync.sh b/tools/components/recipes/esync.sh index 691f77e99a..460ded772b 100644 --- a/tools/components/recipes/esync.sh +++ b/tools/components/recipes/esync.sh @@ -3,12 +3,12 @@ set -euo pipefail if [ -f tools/droiddeck-esync/release.env ]; then . tools/droiddeck-esync/release.env release_url="https://github.com/${SYNC_BUNDLE_REPO}/releases/download/${SYNC_BUNDLE_TAG}" - curl -fsSL --retry 3 -o sync-bundle.tzst "$release_url/$SYNC_BUNDLE_ASSET" + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o sync-bundle.tzst "$release_url/$SYNC_BUNDLE_ASSET" echo "$SYNC_BUNDLE_SHA256 sync-bundle.tzst" | sha256sum -c - mkdir -p app/src/main/assets/droiddeck-esync tar --use-compress-program=unzstd -xf sync-bundle.tzst -C app/src/main/assets/droiddeck-esync rm sync-bundle.tzst - curl -fsSL --retry 3 -o sync-index.json "$release_url/index.json" + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o sync-index.json "$release_url/index.json" revoked=$(python3 -c 'import json, sys; print(" ".join(p["id"] for p in json.load(open(sys.argv[1]))["packs"] if p.get("revoked") is True))' sync-index.json) for id in $revoked; do if [ -e "app/src/main/assets/droiddeck-esync/packs/$id.tzst" ]; then diff --git a/tools/components/recipes/pulseaudio-source.sh b/tools/components/recipes/pulseaudio-source.sh index 5a2a6ed093..a30e86494d 100644 --- a/tools/components/recipes/pulseaudio-source.sh +++ b/tools/components/recipes/pulseaudio-source.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash set -euo pipefail -curl -fsSL --retry 3 -o pa.tar.gz "$COMPONENT_SOURCE_URL" +curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o pa.tar.gz "$COMPONENT_SOURCE_URL" echo "$COMPONENT_SOURCE_SHA256 pa.tar.gz" | sha256sum -c - mkdir -p pa13 && tar -xzf pa.tar.gz -C pa13 --strip-components=1 && rm pa.tar.gz diff --git a/tools/components/recipes/runtime.sh b/tools/components/recipes/runtime.sh index 0e640ab114..92bcaad6c9 100644 --- a/tools/components/recipes/runtime.sh +++ b/tools/components/recipes/runtime.sh @@ -12,14 +12,14 @@ for f in tools/linuxfs/overlay/usr/bin/* tools/linuxfs/overlay/usr/bin/steamos-p done if [ -f tools/gamescope/release.env ]; then . tools/gamescope/release.env - curl -fsSL --retry 3 -o gamescope.tzst "https://github.com/${GAMESCOPE_REPO:-${GITHUB_REPOSITORY}}/releases/download/${GAMESCOPE_TAG}/gamescope.tzst" + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o gamescope.tzst "https://github.com/${GAMESCOPE_REPO:-${GITHUB_REPOSITORY}}/releases/download/${GAMESCOPE_TAG}/gamescope.tzst" echo "${GAMESCOPE_SHA256} gamescope.tzst" | sha256sum -c - tar --use-compress-program=unzstd -xf gamescope.tzst -C "$D" test -f "$D/usr/local/bin/gamescope" rm gamescope.tzst fi . tools/linuxfs/uruntime.env -curl -fsSL --retry 3 -o uruntime "https://github.com/VHSgunzo/uruntime/releases/download/${URUNTIME_VERSION}/${URUNTIME_ASSET}" +curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o uruntime "https://github.com/VHSgunzo/uruntime/releases/download/${URUNTIME_VERSION}/${URUNTIME_ASSET}" echo "${URUNTIME_SHA256} uruntime" | sha256sum -c - install -Dm644 uruntime "$D/usr/local/lib/droiddeck/uruntime" install -Dm644 tools/linuxfs/licenses/uruntime-LICENSE "$D/usr/local/share/licenses/uruntime/LICENSE" @@ -28,7 +28,7 @@ M="$D/usr/local/lib/mangoapp" mkdir -p "$M" mango-pkgs grep -v '^#' tools/mangoapp/packages.txt | while read -r sum url; do [ -n "$url" ] || continue - curl -fsSL --retry 3 -o mango-pkgs/pkg.tar.zst "$url" + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o mango-pkgs/pkg.tar.zst "$url" echo "$sum mango-pkgs/pkg.tar.zst" | sha256sum -c - tar --use-compress-program=unzstd -xf mango-pkgs/pkg.tar.zst -C mango-pkgs rm mango-pkgs/pkg.tar.zst @@ -46,7 +46,7 @@ ls -l "$M" mkdir -p pw-pkgs grep -v '^#' tools/pipewire/packages.txt | while read -r sum url; do [ -n "$url" ] || continue - curl -fsSL --retry 3 -o pw-pkgs/pkg.tar.zst "$url" + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o pw-pkgs/pkg.tar.zst "$url" echo "$sum pw-pkgs/pkg.tar.zst" | sha256sum -c - tar --use-compress-program=unzstd -xf pw-pkgs/pkg.tar.zst -C pw-pkgs rm pw-pkgs/pkg.tar.zst @@ -58,7 +58,7 @@ T="$D/usr/local/lib/droiddeck-msitools" mkdir -p "$T" msi-pkgs grep -v '^#' tools/msitools/packages.txt | while read -r sum url; do [ -n "$url" ] || continue - curl -fsSL --retry 3 -o msi-pkgs/pkg.tar.xz "$url" + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o msi-pkgs/pkg.tar.xz "$url" echo "$sum msi-pkgs/pkg.tar.xz" | sha256sum -c - tar -xJf msi-pkgs/pkg.tar.xz -C msi-pkgs rm msi-pkgs/pkg.tar.xz diff --git a/tools/directaudio/fetch.sh b/tools/directaudio/fetch.sh index 35e3273820..116a98a1cd 100755 --- a/tools/directaudio/fetch.sh +++ b/tools/directaudio/fetch.sh @@ -15,7 +15,7 @@ fetch() { local name=$1 sha=$2 kept="${cache}/$2-$1" part if ! echo "${sha} ${kept}" | sha256sum -c - >/dev/null 2>&1; then part=$(mktemp "${kept}.part.XXXXXX") - curl -fsSL --retry 3 -o "${part}" \ + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o "${part}" \ "https://github.com/${DIRECTAUDIO_REPO}/releases/download/${DIRECTAUDIO_TAG}/${name}" echo "${sha} ${part}" | sha256sum -c - >/dev/null || { echo "${name}: checksum mismatch" >&2; rm -f "${part}"; exit 1; } mv "${part}" "${kept}" @@ -26,7 +26,7 @@ fetch directaudio-linux-wine11.zip "${DIRECTAUDIO_WINE11_SHA256}" fetch directaudio-linux-wine11-systhread.zip "${DIRECTAUDIO_SYSTHREAD_SHA256}" source_archive="${cache}/${DIRECTAUDIO_SOURCE_SHA256}-directaudio-source.tar.gz" if ! echo "${DIRECTAUDIO_SOURCE_SHA256} ${source_archive}" | sha256sum -c - >/dev/null 2>&1; then - curl -fsSL --retry 3 -o "${work}/source.tar.gz" \ + curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o "${work}/source.tar.gz" \ "https://github.com/${DIRECTAUDIO_REPO}/archive/${DIRECTAUDIO_SOURCE_COMMIT}.tar.gz" echo "${DIRECTAUDIO_SOURCE_SHA256} ${work}/source.tar.gz" | sha256sum -c - >/dev/null mv "${work}/source.tar.gz" "${source_archive}" diff --git a/tools/proot/build.sh b/tools/proot/build.sh index 087c23c815..64a51a660a 100755 --- a/tools/proot/build.sh +++ b/tools/proot/build.sh @@ -26,7 +26,7 @@ fetch() { local name=$1 primary=$2 sha=$3 url fetched=0 shift 3 for url in "$primary" "$@"; do - if curl -fsSL --connect-timeout 15 --max-time 90 --retry 3 --retry-max-time 120 -o "$WORK/$name" "$url" && + if curl -fsSL --connect-timeout 15 --max-time 90 --retry 3 --retry-all-errors --retry-max-time 120 -o "$WORK/$name" "$url" && printf '%s %s\n' "$sha" "$WORK/$name" | shasum -a 256 -c -; then fetched=1 break diff --git a/tools/tests/test_esync_packs.py b/tools/tests/test_esync_packs.py index 25a0de7114..b6fff35782 100644 --- a/tools/tests/test_esync_packs.py +++ b/tools/tests/test_esync_packs.py @@ -1170,8 +1170,8 @@ class BundleTest(unittest.TestCase): self.assertNotIn("if", bundle) self.assertIn("tools/droiddeck-esync/release.env", bundle["run"]) self.assertIn('release_url="https://github.com/${SYNC_BUNDLE_REPO}/releases/download/${SYNC_BUNDLE_TAG}"', bundle["run"]) - self.assertIn('curl -fsSL --retry 3 -o sync-bundle.tzst "$release_url/$SYNC_BUNDLE_ASSET"', bundle["run"]) - self.assertIn('curl -fsSL --retry 3 -o sync-index.json "$release_url/index.json"', bundle["run"]) + self.assertIn('curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o sync-bundle.tzst "$release_url/$SYNC_BUNDLE_ASSET"', bundle["run"]) + self.assertIn('curl -fsSL --retry 5 --retry-all-errors --connect-timeout 20 -o sync-index.json "$release_url/index.json"', bundle["run"]) self.assertIn("sha256sum -c -", bundle["run"]) self.assertNotIn("revoked.txt", bundle["run"]) stage = {"run": (self.ROOT / recipes["runtime"]["script"]).read_text()}