ci: bound apt phases and retry the official Ubuntu mirror

.github/workflows/build.yml

@@ -791,7 +791,7 @@ jobs:
           XBOX_SOURCE_TREE: ${{ needs.xbox.outputs.tree-sha }}
         run: test "$(git rev-parse 'HEAD^{tree}')" = "$XBOX_SOURCE_TREE"
       - name: Install Python test dependencies
-        timeout-minutes: 5
+        timeout-minutes: 7
         run: bash tools/ci/install-ubuntu-packages.sh python3-yaml gcc-aarch64-linux-gnu qemu-user libvulkan1 mesa-vulkan-drivers
       - name: Steam import and game environment tests
         run: python3 -m unittest discover -s tools/tests -p 'test_*.py'

tools/ci/install-ubuntu-packages.sh

@@ -1,12 +1,40 @@
 #!/usr/bin/env bash
 set -euo pipefail
+missing=()
+for package in "$@"; do
+  if [ "$(dpkg-query -W -f='${Status}' "$package" 2>/dev/null || true)" != 'install ok installed' ]; then
+    missing+=("$package")
+  fi
+done
+if [ "${#missing[@]}" = 0 ]; then
+  echo "Ubuntu build dependencies are already installed."
+  exit 0
+fi
 options=(
   -o Dir::Etc::sourcelist=sources.list.d/ubuntu.sources
   -o Dir::Etc::sourceparts=-
-  -o Acquire::Retries=3
-  -o Acquire::http::Timeout=30
-  -o Acquire::https::Timeout=30
-  -o DPkg::Lock::Timeout=60
+  -o Acquire::Retries=1
+  -o Acquire::http::Timeout=15
+  -o Acquire::https::Timeout=15
+  -o DPkg::Lock::Timeout=30
 )
-sudo apt-get "${options[@]}" -o APT::Update::Error-Mode=any update -qq
-sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get "${options[@]}" install -y -qq --no-install-recommends "$@"
+install_packages() {
+  echo "Updating Ubuntu package indexes."
+  sudo timeout --kill-after=10s 60s apt-get "${options[@]}" -o APT::Update::Error-Mode=any update -q || return $?
+  echo "Installing: ${missing[*]}"
+  sudo timeout --kill-after=10s 120s env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get "${options[@]}" install -y -q --no-install-recommends "${missing[@]}"
+}
+if install_packages; then
+  exit 0
+fi
+echo "::warning::Runner mirror failed or timed out; retrying with the official Ubuntu mirror."
+sources=$(mktemp --suffix=.sources)
+trap 'rm -f "$sources"' EXIT
+mirror=https://archive.ubuntu.com/ubuntu
+case "$(dpkg --print-architecture)" in
+  arm64) mirror=https://ports.ubuntu.com/ubuntu-ports ;;
+esac
+sed -E "s@^URIs:.*@URIs: $mirror@" /etc/apt/sources.list.d/ubuntu.sources > "$sources"
+chmod 644 "$sources"
+options[1]="Dir::Etc::sourcelist=$sources"
+install_packages

tools/tests/test_ci_checks.py

@@ -18,21 +18,24 @@ class CiChecksTest(unittest.TestCase):
             sudo = root / 'sudo'
             sudo.write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$APT_LOG"\nexit "${APT_RESULT:-0}"\n')
             sudo.chmod(0o755)
+            query = root / 'dpkg-query'
+            query.write_text('#!/bin/sh\nexit 1\n')
+            query.chmod(0o755)
             log = root / 'apt.log'
             env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ['PATH'], APT_LOG=str(log))
             installer = ROOT / 'tools/ci/install-ubuntu-packages.sh'
             result = subprocess.run(['bash', str(installer), 'qemu-user'], env=dict(env, APT_RESULT='17'))
             self.assertEqual(result.returncode, 17)
-            self.assertEqual(len(log.read_text().splitlines()), 1)
+            self.assertEqual(len(log.read_text().splitlines()), 2)
             log.unlink()
             subprocess.run(['bash', str(installer), 'qemu-user'], env=env, check=True)
             update, install = log.read_text().splitlines()
             for command in (update, install):
                 self.assertIn('Dir::Etc::sourceparts=-', command)
-                self.assertIn('Acquire::https::Timeout=30', command)
-                self.assertIn('Acquire::Retries=3', command)
+                self.assertIn('Acquire::https::Timeout=15', command)
+                self.assertIn('Acquire::Retries=1', command)
             self.assertIn('APT::Update::Error-Mode=any', update)
-            self.assertIn('install -y -qq --no-install-recommends qemu-user', install)
+            self.assertIn('install -y -q --no-install-recommends qemu-user', install)
 
     def test_required_status_rejects_failed_cancelled_or_skipped_test_jobs(self):
         jobs = yaml.safe_load((ROOT / '.github/workflows/build.yml').read_text())['jobs']