From 556bd96d5a77ecea65d7e6f13298c0b8c2d548f0 Mon Sep 17 00:00:00 2001 From: XxJSONDeruloxX Date: Fri, 23 Oct 2026 15:01:04 -0400 Subject: [PATCH 1/5] ci: reuse build metadata and verified pack downloads --- .github/workflows/build.yml | 9 +++ app/build.gradle | 4 ++ tools/packs/build.sh | 9 +-- tools/packs/fetch-verified.sh | 29 ++++++++ tools/release/report-build.sh | 2 +- tools/release/runtime_inputs.py | 2 +- tools/tests/test_ci_checks.py | 25 +++++++ tools/tests/test_pack_downloads.py | 106 +++++++++++++++++++++++++++++ 8 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 tools/packs/fetch-verified.sh create mode 100644 tools/tests/test_pack_downloads.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 265b1faee..b1a585e3e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -484,6 +484,12 @@ jobs: name: xbox-arm64 path: app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox + - name: Cache checksum-pinned pack inputs + uses: actions/cache@v4 + with: + path: ~/.cache/droiddeck-pack-inputs + key: pack-inputs-v1-${{ hashFiles('tools/packs/build.sh', 'tools/packs/fetch-verified.sh', 'tools/packs/tracking/release.env') }} + restore-keys: pack-inputs-v1- - name: Build the add-on packs run: bash tools/packs/build.sh app/build/packs @@ -526,6 +532,9 @@ jobs: - name: Unit tests id: unit-tests + env: + DROIDDECK_BUILD_LABEL: ${{ steps.build-label.outputs.label }} + SGDB_API_KEY: ${{ secrets.SGDB_API_KEY }} run: | set -o pipefail mkdir -p app/build diff --git a/app/build.gradle b/app/build.gradle index e81b92711..bdc9ed4d3 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -220,6 +220,10 @@ android { unitTests.returnDefaultValues = true unitTests.includeAndroidResources = true unitTests.all { + if (ciBuild) { + outputs.upToDateWhen { false } + outputs.doNotCacheIf('CI executes the test suite on every run') { true } + } // Robolectric keeps a sandbox per SDK level the tests ask for; seven no longer fit in Gradle's default 512 MB. maxHeapSize = '2g' def logTarget = providers.systemProperty('robolectric.logging').orNull diff --git a/tools/packs/build.sh b/tools/packs/build.sh index d06a4c8fb..27611ab0b 100755 --- a/tools/packs/build.sh +++ b/tools/packs/build.sh @@ -25,14 +25,7 @@ trap 'rm -rf "$work"' EXIT sha256() { if command -v sha256sum >/dev/null; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; } fetch() { - local url=$1 expected=$2 dest=$3 - curl -fsSL --retry 4 -o "$dest" "$url" - local actual - actual=$(sha256 "$dest") - if [ "$actual" != "$expected" ]; then - echo "checksum mismatch for $url: $actual" >&2 - exit 1 - fi + bash "$here/fetch-verified.sh" "$@" } pack() { diff --git a/tools/packs/fetch-verified.sh b/tools/packs/fetch-verified.sh new file mode 100644 index 000000000..0b8c03977 --- /dev/null +++ b/tools/packs/fetch-verified.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail +url=${1:?URL required} +expected=${2:?SHA-256 required} +destination=${3:?destination required} +[[ "$expected" =~ ^[a-f0-9]{64}$ ]] || { echo 'Invalid download SHA-256' >&2; exit 1; } +cache=${DROIDDECK_PACK_CACHE:-"$HOME/.cache/droiddeck-pack-inputs"} +mkdir -p "$cache" "$(dirname "$destination")" +kept="$cache/$expected" +verify() { + if command -v sha256sum >/dev/null; then + echo "$expected $1" | sha256sum -c - >/dev/null 2>&1 + else + echo "$expected $1" | shasum -a 256 -c - >/dev/null 2>&1 + fi +} +if [ -f "$kept" ] && verify "$kept"; then + echo "Verified pack input cache hit: $expected" +else + part=$(mktemp "$cache/$expected.part.XXXXXX") + trap 'rm -f "$part"' EXIT + curl -fsSL --retry 4 --retry-all-errors --connect-timeout 15 --max-time 180 --retry-max-time 240 \ + -o "$part" "$url" + verify "$part" || { echo "Checksum mismatch for $url" >&2; exit 1; } + mv -f "$part" "$kept" + echo "Downloaded and verified pack input: $expected" +fi +cp "$kept" "$destination" +verify "$destination" || { echo 'Copied pack input checksum mismatch' >&2; exit 1; } diff --git a/tools/release/report-build.sh b/tools/release/report-build.sh index fc031c634..75b16f666 100644 --- a/tools/release/report-build.sh +++ b/tools/release/report-build.sh @@ -69,7 +69,7 @@ MARK="" echo } > /tmp/pr-report.md cat /tmp/pr-report.md >> "$GITHUB_STEP_SUMMARY" -ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate --jq ".[] | select(.body | startswith(\"$MARK\")) | .id" | head -1 || true) +ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$MARK\"))) | .id" | head -1 || true) if [ -n "$ID" ]; then gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${ID}" -F body=@/tmp/pr-report.md >/dev/null else diff --git a/tools/release/runtime_inputs.py b/tools/release/runtime_inputs.py index 83f87be03..ccce1f0f0 100644 --- a/tools/release/runtime_inputs.py +++ b/tools/release/runtime_inputs.py @@ -21,7 +21,7 @@ PACKED_FILES = {'lib/arm64-v8a/libproot.so': ('proot', 'libproot.so'), PACKED_LIBRARIES = ['lib/arm64-v8a/libopencv_java4.so', 'lib/arm64-v8a/libmediapipe_tasks_jni.so'] PACK_NAMES = ['proot', 'directaudio', 'tracking', 'xbox', 'comet'] SOURCE_PATHS = ['app/build.gradle', 'app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/assets/pulseaudio.tzst', - 'tools/linuxfs', 'tools/proot', 'tools/aaudio-sink', 'tools/directaudio', + 'tools/linuxfs', 'tools/proot', 'tools/aaudio-sink', 'tools/directaudio', 'tools/packs', 'tools/gamescope/release.env', 'tools/droiddeck-esync/release.env', 'tools/mangoapp', 'tools/msitools', 'tools/pipewire', 'tools/comet', 'tools/xbox', 'tools/components', 'tools/ci', 'tools/build_local.sh', 'tools/release/runtime_inputs.py', 'app/src/main/assets/desktop-removal', diff --git a/tools/tests/test_ci_checks.py b/tools/tests/test_ci_checks.py index 3a0216d77..503b9fb5b 100644 --- a/tools/tests/test_ci_checks.py +++ b/tools/tests/test_ci_checks.py @@ -1,6 +1,9 @@ import itertools +import json import os from pathlib import Path +import shlex +import shutil import subprocess import tempfile import unittest @@ -12,6 +15,28 @@ ROOT = Path(__file__).resolve().parents[2] class CiChecksTest(unittest.TestCase): + def test_report_comment_lookup_ignores_human_marker(self): + if not shutil.which('jq'): + self.skipTest('jq required to execute comment selection') + script = (ROOT / 'tools/release/report-build.sh').read_text() + line = next(line for line in script.splitlines() if line.startswith('ID=')) + arguments = shlex.split(line[len('ID=$('):].split(' | head', 1)[0]) + query = arguments[arguments.index('--jq') + 1].replace('$MARK', '') + comments = [dict(id=1, user=dict(login='someone'), body=' forged'), + dict(id=2, user=dict(login='github-actions[bot]'), body=' result')] + result = subprocess.check_output(['jq', '-r', query], input=json.dumps(comments), text=True) + self.assertEqual(result.strip(), '2') + + def test_build_and_tests_use_identical_generated_metadata(self): + jobs = yaml.safe_load((ROOT / '.github/workflows/build.yml').read_text())['jobs'] + steps = {step.get('id'): step for step in jobs['build']['steps']} + self.assertEqual(steps['build']['env'], steps['unit-tests']['env']) + self.assertIn('DROIDDECK_BUILD_LABEL', steps['unit-tests']['env']) + self.assertIn('SGDB_API_KEY', steps['unit-tests']['env']) + gradle = (ROOT / 'app/build.gradle').read_text() + self.assertIn("outputs.doNotCacheIf('CI executes the test suite on every run') { true }", gradle) + self.assertIn('outputs.upToDateWhen { false }', gradle) + def test_independent_jobs_wait_only_for_the_pinned_source(self): jobs = yaml.safe_load((ROOT / '.github/workflows/build.yml').read_text())['jobs'] for name in ('xbox', 'components', 'python-tests'): diff --git a/tools/tests/test_pack_downloads.py b/tools/tests/test_pack_downloads.py new file mode 100644 index 000000000..20547c03b --- /dev/null +++ b/tools/tests/test_pack_downloads.py @@ -0,0 +1,106 @@ +import hashlib +import http.server +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import tempfile +import threading +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +FETCH = ROOT / 'tools/packs/fetch-verified.sh' + + +class PackDownloadsTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + self.cache = self.root / 'cache' + self.cache.mkdir() + self.destination = self.root / 'download' + self.remote = self.root / 'remote' + self.remote.write_bytes(b'verified input') + self.checksum = hashlib.sha256(self.remote.read_bytes()).hexdigest() + self.log = self.root / 'curl.log' + tool = self.root / 'bin/curl' + tool.parent.mkdir() + tool.write_text('#!/usr/bin/env python3\nimport json,os,shutil,sys\n' + 'args=sys.argv[1:]\n' + 'with open(os.environ["CURL_LOG"],"a") as f: f.write(json.dumps(args)+"\\n")\n' + 'shutil.copyfile(os.environ["REMOTE"],args[args.index("-o")+1])\n') + tool.chmod(0o755) + self.env = dict(os.environ, DROIDDECK_PACK_CACHE=str(self.cache), CURL_LOG=str(self.log), + REMOTE=str(self.remote), PATH=str(tool.parent) + os.pathsep + os.environ['PATH']) + + def fetch(self, checksum=None, url='https://example.test/pinned', env=None): + return subprocess.run(['bash', str(FETCH), url, checksum or self.checksum, str(self.destination)], + env=env or self.env, capture_output=True, text=True) + + def test_cold_download_then_warm_reuse_and_corruption_repair(self): + self.assertEqual(self.fetch().returncode, 0) + self.assertEqual(self.fetch().returncode, 0) + self.assertEqual(len(self.log.read_text().splitlines()), 1) + self.assertEqual(self.destination.read_bytes(), b'verified input') + (self.cache / self.checksum).write_bytes(b'corrupt') + self.assertEqual(self.fetch().returncode, 0) + self.assertEqual(len(self.log.read_text().splitlines()), 2) + self.assertEqual((self.cache / self.checksum).read_bytes(), b'verified input') + + def test_changed_pin_cannot_reuse_old_bytes(self): + self.assertEqual(self.fetch().returncode, 0) + self.remote.write_bytes(b'new approved input') + updated = hashlib.sha256(self.remote.read_bytes()).hexdigest() + self.assertEqual(self.fetch(updated).returncode, 0) + self.assertEqual(len(self.log.read_text().splitlines()), 2) + self.assertEqual(self.destination.read_bytes(), b'new approved input') + + def test_bad_download_is_not_cached_or_written(self): + self.remote.write_bytes(b'tampered download') + self.assertNotEqual(self.fetch().returncode, 0) + self.assertFalse((self.cache / self.checksum).exists()) + self.assertFalse(self.destination.exists()) + self.assertEqual(list(self.cache.iterdir()), []) + arguments = json.loads(self.log.read_text()) + self.assertIn('--retry-all-errors', arguments) + self.assertIn('--max-time', arguments) + self.assertIn('--retry-max-time', arguments) + + def test_connection_reset_retries_then_verifies_real_download(self): + if not shutil.which('curl'): + self.skipTest('curl required for transport test') + content = self.remote.read_bytes() + attempts = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + attempts.append(True) + if len(attempts) == 1: + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + self.send_response(200) + self.send_header('Content-Length', str(len(content))) + self.end_headers() + self.wfile.write(content) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + env = dict(self.env, PATH=os.environ['PATH']) + result = self.fetch(url=f'http://127.0.0.1:{server.server_port}/input', env=env) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertGreaterEqual(len(attempts), 2) + self.assertEqual(self.destination.read_bytes(), content) + finally: + server.shutdown() + server.server_close() + thread.join() From 5a806b35f15f0d36a44680dc8ad7a29f7426b3f4 Mon Sep 17 00:00:00 2001 From: XxJSONDeruloxX Date: Fri, 23 Oct 2026 15:22:01 -0400 Subject: [PATCH 2/5] test: support configured artwork key --- .../droiddeck/launcher/frontend/AddedGameArtLookupTest.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/src/test/java/com/droiddeck/launcher/frontend/AddedGameArtLookupTest.kt b/app/src/test/java/com/droiddeck/launcher/frontend/AddedGameArtLookupTest.kt index 1f1e30c81..f48146201 100644 --- a/app/src/test/java/com/droiddeck/launcher/frontend/AddedGameArtLookupTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/frontend/AddedGameArtLookupTest.kt @@ -1,5 +1,6 @@ package com.droiddeck.launcher.frontend +import com.droiddeck.launcher.BuildConfig import com.droiddeck.launcher.stores.CredentialCipher import org.json.JSONArray import org.json.JSONObject @@ -92,8 +93,9 @@ class AddedGameArtLookupTest { assertEquals("build", SteamGridDb.pick("", "build")) assertEquals("build", SteamGridDb.pick(" ", "build")) assertEquals("", SteamGridDb.pick("", "")) - assertEquals("", SteamGridDb.builtInKey()) - assertEquals("", SteamGridDb.key(app)) + val builtIn = SteamGridDb.builtInKey() + assertTrue(builtIn == BuildConfig.SGDB_API_KEY.trim()) + assertTrue(SteamGridDb.key(app) == builtIn) } @Test fun theUsersKeyIsSealedOnDiskAndClears() { @@ -106,7 +108,7 @@ class AddedGameArtLookupTest { assertEquals("abcdef0123456789secretkey", SteamGridDb.key(app)) SteamGridDb.setUserKey(app, "") assertFalse(file.exists()) - assertEquals("", SteamGridDb.key(app)) + assertTrue(SteamGridDb.key(app) == SteamGridDb.builtInKey()) } @Test fun aFailedLookupIsNotRememberedAsNoMatch() { From a7d6279ee3ce0b857b1a0b9b688f4a1f77830781 Mon Sep 17 00:00:00 2001 From: XxJSONDeruloxX Date: Fri, 23 Oct 2026 15:22:12 -0400 Subject: [PATCH 3/5] test: check artwork key normalization --- .../droiddeck/launcher/frontend/AddedGameArtLookupTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/test/java/com/droiddeck/launcher/frontend/AddedGameArtLookupTest.kt b/app/src/test/java/com/droiddeck/launcher/frontend/AddedGameArtLookupTest.kt index f48146201..095cf9a5c 100644 --- a/app/src/test/java/com/droiddeck/launcher/frontend/AddedGameArtLookupTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/frontend/AddedGameArtLookupTest.kt @@ -94,8 +94,8 @@ class AddedGameArtLookupTest { assertEquals("build", SteamGridDb.pick(" ", "build")) assertEquals("", SteamGridDb.pick("", "")) val builtIn = SteamGridDb.builtInKey() - assertTrue(builtIn == BuildConfig.SGDB_API_KEY.trim()) - assertTrue(SteamGridDb.key(app) == builtIn) + assertTrue(builtIn == BuildConfig.SGDB_API_KEY) + assertTrue(SteamGridDb.key(app) == builtIn.trim()) } @Test fun theUsersKeyIsSealedOnDiskAndClears() { @@ -108,7 +108,7 @@ class AddedGameArtLookupTest { assertEquals("abcdef0123456789secretkey", SteamGridDb.key(app)) SteamGridDb.setUserKey(app, "") assertFalse(file.exists()) - assertTrue(SteamGridDb.key(app) == SteamGridDb.builtInKey()) + assertTrue(SteamGridDb.key(app) == SteamGridDb.builtInKey().trim()) } @Test fun aFailedLookupIsNotRememberedAsNoMatch() { From 9835c4e3e16b7a9695c13362bfa370f665645508 Mon Sep 17 00:00:00 2001 From: XxJSONDeruloxX Date: Fri, 23 Oct 2026 15:39:22 -0400 Subject: [PATCH 4/5] ci: persist PR caches and stabilize build time --- .github/workflows/build.yml | 1 + app/build.gradle | 2 +- tools/tests/test_ci_checks.py | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b1a585e3e..146e7fe3b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -403,6 +403,7 @@ jobs: - uses: gradle/actions/setup-gradle@v4 with: + cache-read-only: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} cache-cleanup: on-success - name: Resolve the pinned Android NDK diff --git a/app/build.gradle b/app/build.gradle index bdc9ed4d3..1b1eb5ae7 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -80,7 +80,7 @@ def sourceCommit = providers.environmentVariable('DROIDDECK_SOURCE_COMMIT').getO def prNumber = (providers.environmentVariable('DROIDDECK_PR_NUMBER').getOrNull() ?: '0').with { it.isInteger() ? it : '0' } def ciBuild = providers.environmentVariable('GITHUB_ACTIONS').getOrNull() == 'true' def releaseSigner = rootProject.file('keystore/release-signer.sha256').text.trim() -def commitTime = (gitValue(['show', '-s', '--format=%ct', 'HEAD']) ?: '0').with { it.isLong() ? it : '0' } +def commitTime = (gitValue(['show', '-s', '--format=%ct', '--end-of-options', sourceCommit]) ?: '0').with { it.isLong() ? it : '0' } def desktopKde = rootProject.file('tools/desktop-kde/release.env').readLines() .findAll { it.contains('=') && !it.startsWith('#') } .collectEntries { def kv = it.split('=', 2); [(kv[0].trim()): kv[1].trim()] } diff --git a/tools/tests/test_ci_checks.py b/tools/tests/test_ci_checks.py index 503b9fb5b..b7c29fceb 100644 --- a/tools/tests/test_ci_checks.py +++ b/tools/tests/test_ci_checks.py @@ -15,6 +15,15 @@ ROOT = Path(__file__).resolve().parents[2] class CiChecksTest(unittest.TestCase): + def test_pr_cache_writes_and_source_timestamp_policy(self): + jobs = yaml.safe_load((ROOT / '.github/workflows/build.yml').read_text())['jobs'] + cache = next(step for step in jobs['build']['steps'] + if step.get('uses') == 'gradle/actions/setup-gradle@v4')['with'] + self.assertEqual(cache['cache-read-only'], + "${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}") + gradle = (ROOT / 'app/build.gradle').read_text() + self.assertIn("gitValue(['show', '-s', '--format=%ct', '--end-of-options', sourceCommit])", gradle) + def test_report_comment_lookup_ignores_human_marker(self): if not shutil.which('jq'): self.skipTest('jq required to execute comment selection') From c95e114766adb7b2ef585302e9474c5cb2ed53bf Mon Sep 17 00:00:00 2001 From: XxJSONDeruloxX Date: Fri, 23 Oct 2026 16:20:42 -0400 Subject: [PATCH 5/5] fix(ci): recover directory cache entries --- tools/packs/fetch-verified.sh | 3 +++ tools/tests/test_pack_downloads.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tools/packs/fetch-verified.sh b/tools/packs/fetch-verified.sh index 0b8c03977..41bba2819 100644 --- a/tools/packs/fetch-verified.sh +++ b/tools/packs/fetch-verified.sh @@ -22,6 +22,9 @@ else curl -fsSL --retry 4 --retry-all-errors --connect-timeout 15 --max-time 180 --retry-max-time 240 \ -o "$part" "$url" verify "$part" || { echo "Checksum mismatch for $url" >&2; exit 1; } + if [ -d "$kept" ]; then + rm -rf -- "$kept" + fi mv -f "$part" "$kept" echo "Downloaded and verified pack input: $expected" fi diff --git a/tools/tests/test_pack_downloads.py b/tools/tests/test_pack_downloads.py index 20547c03b..0b5eb6060 100644 --- a/tools/tests/test_pack_downloads.py +++ b/tools/tests/test_pack_downloads.py @@ -51,6 +51,25 @@ class PackDownloadsTest(unittest.TestCase): self.assertEqual(len(self.log.read_text().splitlines()), 2) self.assertEqual((self.cache / self.checksum).read_bytes(), b'verified input') + def test_directory_cache_entry_is_replaced_only_after_verification(self): + entry = self.cache / self.checksum + entry.mkdir() + sentinel = entry / 'old-entry' + sentinel.write_bytes(b'preserve until verified') + self.remote.write_bytes(b'tampered download') + self.assertNotEqual(self.fetch().returncode, 0) + self.assertEqual(sentinel.read_bytes(), b'preserve until verified') + self.assertFalse(self.destination.exists()) + self.remote.write_bytes(b'verified input') + result = self.fetch() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(entry.is_file()) + self.assertEqual(entry.read_bytes(), b'verified input') + self.assertEqual(self.destination.read_bytes(), b'verified input') + self.assertEqual(list(self.cache.iterdir()), [entry]) + self.assertEqual(self.fetch().returncode, 0) + self.assertEqual(len(self.log.read_text().splitlines()), 2) + def test_changed_pin_cannot_reuse_old_bytes(self): self.assertEqual(self.fetch().returncode, 0) self.remote.write_bytes(b'new approved input')