mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-21 22:05:59 +00:00
ci: reuse build metadata and verified pack downloads
This commit is contained in:
1 parent
3d9d90d832
commit
556bd96d5a
8 files changed
+176
-10
No files matched your search
@@ -25,14 +25,7 @@ trap 'rm -rf "$work"' EXIT
|
||||
sha256() { if command -v sha256sum >/dev/null; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; }
|
||||
|
||||
fetch() {
|
||||
local url=$1 expected=$2 dest=$3
|
||||
curl -fsSL --retry 4 -o "$dest" "$url"
|
||||
local actual
|
||||
actual=$(sha256 "$dest")
|
||||
if [ "$actual" != "$expected" ]; then
|
||||
echo "checksum mismatch for $url: $actual" >&2
|
||||
exit 1
|
||||
fi
|
||||
bash "$here/fetch-verified.sh" "$@"
|
||||
}
|
||||
|
||||
pack() {
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
url=${1:?URL required}
|
||||
expected=${2:?SHA-256 required}
|
||||
destination=${3:?destination required}
|
||||
[[ "$expected" =~ ^[a-f0-9]{64}$ ]] || { echo 'Invalid download SHA-256' >&2; exit 1; }
|
||||
cache=${DROIDDECK_PACK_CACHE:-"$HOME/.cache/droiddeck-pack-inputs"}
|
||||
mkdir -p "$cache" "$(dirname "$destination")"
|
||||
kept="$cache/$expected"
|
||||
verify() {
|
||||
if command -v sha256sum >/dev/null; then
|
||||
echo "$expected $1" | sha256sum -c - >/dev/null 2>&1
|
||||
else
|
||||
echo "$expected $1" | shasum -a 256 -c - >/dev/null 2>&1
|
||||
fi
|
||||
}
|
||||
if [ -f "$kept" ] && verify "$kept"; then
|
||||
echo "Verified pack input cache hit: $expected"
|
||||
else
|
||||
part=$(mktemp "$cache/$expected.part.XXXXXX")
|
||||
trap 'rm -f "$part"' EXIT
|
||||
curl -fsSL --retry 4 --retry-all-errors --connect-timeout 15 --max-time 180 --retry-max-time 240 \
|
||||
-o "$part" "$url"
|
||||
verify "$part" || { echo "Checksum mismatch for $url" >&2; exit 1; }
|
||||
mv -f "$part" "$kept"
|
||||
echo "Downloaded and verified pack input: $expected"
|
||||
fi
|
||||
cp "$kept" "$destination"
|
||||
verify "$destination" || { echo 'Copied pack input checksum mismatch' >&2; exit 1; }
|
||||
@@ -69,7 +69,7 @@ MARK="<!-- droiddeck-pr-build -->"
|
||||
echo
|
||||
} > /tmp/pr-report.md
|
||||
cat /tmp/pr-report.md >> "$GITHUB_STEP_SUMMARY"
|
||||
ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate --jq ".[] | select(.body | startswith(\"$MARK\")) | .id" | head -1 || true)
|
||||
ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$MARK\"))) | .id" | head -1 || true)
|
||||
if [ -n "$ID" ]; then
|
||||
gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${ID}" -F body=@/tmp/pr-report.md >/dev/null
|
||||
else
|
||||
|
||||
@@ -21,7 +21,7 @@ PACKED_FILES = {'lib/arm64-v8a/libproot.so': ('proot', 'libproot.so'),
|
||||
PACKED_LIBRARIES = ['lib/arm64-v8a/libopencv_java4.so', 'lib/arm64-v8a/libmediapipe_tasks_jni.so']
|
||||
PACK_NAMES = ['proot', 'directaudio', 'tracking', 'xbox', 'comet']
|
||||
SOURCE_PATHS = ['app/build.gradle', 'app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/assets/pulseaudio.tzst',
|
||||
'tools/linuxfs', 'tools/proot', 'tools/aaudio-sink', 'tools/directaudio',
|
||||
'tools/linuxfs', 'tools/proot', 'tools/aaudio-sink', 'tools/directaudio', 'tools/packs',
|
||||
'tools/gamescope/release.env', 'tools/droiddeck-esync/release.env',
|
||||
'tools/mangoapp', 'tools/msitools', 'tools/pipewire', 'tools/comet', 'tools/xbox', 'tools/components', 'tools/ci', 'tools/build_local.sh',
|
||||
'tools/release/runtime_inputs.py', 'app/src/main/assets/desktop-removal',
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -12,6 +15,28 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class CiChecksTest(unittest.TestCase):
|
||||
def test_report_comment_lookup_ignores_human_marker(self):
|
||||
if not shutil.which('jq'):
|
||||
self.skipTest('jq required to execute comment selection')
|
||||
script = (ROOT / 'tools/release/report-build.sh').read_text()
|
||||
line = next(line for line in script.splitlines() if line.startswith('ID='))
|
||||
arguments = shlex.split(line[len('ID=$('):].split(' | head', 1)[0])
|
||||
query = arguments[arguments.index('--jq') + 1].replace('$MARK', '<!-- build -->')
|
||||
comments = [dict(id=1, user=dict(login='someone'), body='<!-- build --> forged'),
|
||||
dict(id=2, user=dict(login='github-actions[bot]'), body='<!-- build --> result')]
|
||||
result = subprocess.check_output(['jq', '-r', query], input=json.dumps(comments), text=True)
|
||||
self.assertEqual(result.strip(), '2')
|
||||
|
||||
def test_build_and_tests_use_identical_generated_metadata(self):
|
||||
jobs = yaml.safe_load((ROOT / '.github/workflows/build.yml').read_text())['jobs']
|
||||
steps = {step.get('id'): step for step in jobs['build']['steps']}
|
||||
self.assertEqual(steps['build']['env'], steps['unit-tests']['env'])
|
||||
self.assertIn('DROIDDECK_BUILD_LABEL', steps['unit-tests']['env'])
|
||||
self.assertIn('SGDB_API_KEY', steps['unit-tests']['env'])
|
||||
gradle = (ROOT / 'app/build.gradle').read_text()
|
||||
self.assertIn("outputs.doNotCacheIf('CI executes the test suite on every run') { true }", gradle)
|
||||
self.assertIn('outputs.upToDateWhen { false }', gradle)
|
||||
|
||||
def test_independent_jobs_wait_only_for_the_pinned_source(self):
|
||||
jobs = yaml.safe_load((ROOT / '.github/workflows/build.yml').read_text())['jobs']
|
||||
for name in ('xbox', 'components', 'python-tests'):
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import hashlib
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FETCH = ROOT / 'tools/packs/fetch-verified.sh'
|
||||
|
||||
|
||||
class PackDownloadsTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(directory.cleanup)
|
||||
self.root = Path(directory.name)
|
||||
self.cache = self.root / 'cache'
|
||||
self.cache.mkdir()
|
||||
self.destination = self.root / 'download'
|
||||
self.remote = self.root / 'remote'
|
||||
self.remote.write_bytes(b'verified input')
|
||||
self.checksum = hashlib.sha256(self.remote.read_bytes()).hexdigest()
|
||||
self.log = self.root / 'curl.log'
|
||||
tool = self.root / 'bin/curl'
|
||||
tool.parent.mkdir()
|
||||
tool.write_text('#!/usr/bin/env python3\nimport json,os,shutil,sys\n'
|
||||
'args=sys.argv[1:]\n'
|
||||
'with open(os.environ["CURL_LOG"],"a") as f: f.write(json.dumps(args)+"\\n")\n'
|
||||
'shutil.copyfile(os.environ["REMOTE"],args[args.index("-o")+1])\n')
|
||||
tool.chmod(0o755)
|
||||
self.env = dict(os.environ, DROIDDECK_PACK_CACHE=str(self.cache), CURL_LOG=str(self.log),
|
||||
REMOTE=str(self.remote), PATH=str(tool.parent) + os.pathsep + os.environ['PATH'])
|
||||
|
||||
def fetch(self, checksum=None, url='https://example.test/pinned', env=None):
|
||||
return subprocess.run(['bash', str(FETCH), url, checksum or self.checksum, str(self.destination)],
|
||||
env=env or self.env, capture_output=True, text=True)
|
||||
|
||||
def test_cold_download_then_warm_reuse_and_corruption_repair(self):
|
||||
self.assertEqual(self.fetch().returncode, 0)
|
||||
self.assertEqual(self.fetch().returncode, 0)
|
||||
self.assertEqual(len(self.log.read_text().splitlines()), 1)
|
||||
self.assertEqual(self.destination.read_bytes(), b'verified input')
|
||||
(self.cache / self.checksum).write_bytes(b'corrupt')
|
||||
self.assertEqual(self.fetch().returncode, 0)
|
||||
self.assertEqual(len(self.log.read_text().splitlines()), 2)
|
||||
self.assertEqual((self.cache / self.checksum).read_bytes(), b'verified input')
|
||||
|
||||
def test_changed_pin_cannot_reuse_old_bytes(self):
|
||||
self.assertEqual(self.fetch().returncode, 0)
|
||||
self.remote.write_bytes(b'new approved input')
|
||||
updated = hashlib.sha256(self.remote.read_bytes()).hexdigest()
|
||||
self.assertEqual(self.fetch(updated).returncode, 0)
|
||||
self.assertEqual(len(self.log.read_text().splitlines()), 2)
|
||||
self.assertEqual(self.destination.read_bytes(), b'new approved input')
|
||||
|
||||
def test_bad_download_is_not_cached_or_written(self):
|
||||
self.remote.write_bytes(b'tampered download')
|
||||
self.assertNotEqual(self.fetch().returncode, 0)
|
||||
self.assertFalse((self.cache / self.checksum).exists())
|
||||
self.assertFalse(self.destination.exists())
|
||||
self.assertEqual(list(self.cache.iterdir()), [])
|
||||
arguments = json.loads(self.log.read_text())
|
||||
self.assertIn('--retry-all-errors', arguments)
|
||||
self.assertIn('--max-time', arguments)
|
||||
self.assertIn('--retry-max-time', arguments)
|
||||
|
||||
def test_connection_reset_retries_then_verifies_real_download(self):
|
||||
if not shutil.which('curl'):
|
||||
self.skipTest('curl required for transport test')
|
||||
content = self.remote.read_bytes()
|
||||
attempts = []
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
attempts.append(True)
|
||||
if len(attempts) == 1:
|
||||
self.connection.shutdown(socket.SHUT_RDWR)
|
||||
self.connection.close()
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Length', str(len(content)))
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
env = dict(self.env, PATH=os.environ['PATH'])
|
||||
result = self.fetch(url=f'http://127.0.0.1:{server.server_port}/input', env=env)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertGreaterEqual(len(attempts), 2)
|
||||
self.assertEqual(self.destination.read_bytes(), content)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
Reference in new issue
Block a user