mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-19 04:46:08 +00:00
ci: consolidate verified component preparation
This commit is contained in:
1 parent
d111b2b3dd
commit
d7bf0b274f
31 files changed
+1269
-503
No files matched your search
@@ -0,0 +1,167 @@
|
||||
"""One entry point for runtime recipes, cache identities and verified job handoff."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
REGISTRY = 'tools/components/recipes.json'
|
||||
TREES = ['app/src/main/assets/linuxfs', 'app/src/main/assets/directaudio',
|
||||
'app/src/main/assets/droiddeck-esync', 'app/build/prepared-assets',
|
||||
'app/build/component-records']
|
||||
LIBRARIES = ['app/src/main/jniLibs/arm64-v8a/' + name for name in
|
||||
('libproot.so', 'libproot-loader.so', 'libdirectaudiorelay.so')]
|
||||
MANIFEST = 'app/build/prepared-components.json'
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def registry(root=ROOT):
|
||||
return json.loads((root / REGISTRY).read_text())
|
||||
|
||||
|
||||
def identity(name, root=ROOT):
|
||||
spec = registry(root)[name]
|
||||
paths = subprocess.check_output(['git', 'ls-files', '--cached', '--others', '--exclude-standard',
|
||||
'--', *spec['inputs'], *spec.get('source_inputs', [])], cwd=root, text=True).splitlines()
|
||||
values = {}
|
||||
for name_ in sorted(set(paths)):
|
||||
if '__pycache__' in name_ or name_.endswith(('.pyc', '.md')):
|
||||
continue
|
||||
content = (root / name_).read_bytes()
|
||||
if b'\0' not in content:
|
||||
content = content.replace(b'\r\n', b'\n')
|
||||
values[name_] = hashlib.sha256(content).hexdigest()
|
||||
policy = dict(spec=spec, sources=values, runner=os.environ.get('RUNNER_OS', 'Linux'),
|
||||
architecture=os.environ.get('RUNNER_ARCH', 'X64'))
|
||||
entrypoint = root / 'tools/components/prepare.py'
|
||||
if entrypoint.is_file():
|
||||
policy['entrypoint'] = hashlib.sha256(entrypoint.read_bytes().replace(b'\r\n', b'\n')).hexdigest()
|
||||
installer = root / 'tools/ci/install-ubuntu-packages.sh'
|
||||
if spec.get('packages') and installer.is_file():
|
||||
policy['installer'] = hashlib.sha256(installer.read_bytes().replace(b'\r\n', b'\n')).hexdigest()
|
||||
if spec.get('ndk'):
|
||||
policy['ndk'] = os.environ.get('NDK_VERSION', '')
|
||||
policy['ndk_policy'] = hashlib.sha256((root / 'tools/ci/toolchains.env').read_bytes().replace(b'\r\n', b'\n')).hexdigest()
|
||||
return hashlib.sha256(json.dumps(policy, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
def prepare(name, root=ROOT):
|
||||
spec = registry(root)[name]
|
||||
prepare_tools([name], root)
|
||||
if 'command' in spec:
|
||||
command = spec['command']
|
||||
else:
|
||||
script = (root / spec['script']).resolve()
|
||||
if not script.is_relative_to((root / 'tools/components').resolve()):
|
||||
raise ValueError('Recipe script escapes components directory')
|
||||
command = ['bash', str(script)]
|
||||
environment = dict(os.environ)
|
||||
if spec.get('source'):
|
||||
environment.update(COMPONENT_SOURCE_URL=spec['source']['url'],
|
||||
COMPONENT_SOURCE_SHA256=spec['source']['sha256'])
|
||||
subprocess.run(command, cwd=root, env=environment, check=True)
|
||||
missing = [name_ for name_ in spec['outputs'] if not (root / name_).is_file()]
|
||||
if missing:
|
||||
raise ValueError(f'{name} did not prepare required outputs: {missing}')
|
||||
record = dict(component=name, input_sha256=identity(name, root),
|
||||
outputs={p: digest(root / p) for p in spec['outputs']})
|
||||
destination = root / 'app/build/component-records' / (name + '.json')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(record, sort_keys=True, indent=2) + '\n')
|
||||
|
||||
|
||||
def prepare_tools(names, root=ROOT):
|
||||
packages = sorted({package for name in names for package in registry(root)[name].get('packages', [])})
|
||||
if packages:
|
||||
subprocess.run(['bash', 'tools/ci/install-ubuntu-packages.sh', *packages], cwd=root, check=True)
|
||||
|
||||
|
||||
def files(root=ROOT):
|
||||
result = []
|
||||
for name in TREES:
|
||||
folder = root / name
|
||||
if not folder.is_dir() or folder.is_symlink():
|
||||
raise ValueError('Missing prepared component tree: ' + name)
|
||||
for path in folder.rglob('*'):
|
||||
if path.is_symlink():
|
||||
raise ValueError('Symlink in prepared components: ' + str(path))
|
||||
if path.is_file():
|
||||
result.append(path.relative_to(root).as_posix())
|
||||
for name in LIBRARIES:
|
||||
if not (root / name).is_file() or (root / name).is_symlink():
|
||||
raise ValueError('Missing prepared component library: ' + name)
|
||||
result.append(name)
|
||||
return sorted(result)
|
||||
|
||||
|
||||
def pack(destination, root=ROOT):
|
||||
names = files(root)
|
||||
tree = subprocess.check_output(['git', 'rev-parse', 'HEAD^{tree}'], cwd=root, text=True).strip()
|
||||
data = dict(version=1, source_tree=tree, run=os.environ.get('GITHUB_RUN_ID', ''),
|
||||
recipes={name: identity(name, root) for name in registry(root)},
|
||||
files={name: digest(root / name) for name in names})
|
||||
(root / MANIFEST).write_text(json.dumps(data, sort_keys=True, indent=2) + '\n')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(destination, mode='w:gz', compresslevel=1) as archive:
|
||||
for name in names + [MANIFEST]:
|
||||
archive.add(root / name, arcname=name, recursive=False)
|
||||
|
||||
|
||||
def unpack(archive_path, source_tree, root=ROOT):
|
||||
total = 0
|
||||
seen = set()
|
||||
with tarfile.open(archive_path, mode='r:gz') as archive:
|
||||
for entry in archive:
|
||||
allowed = entry.name in LIBRARIES + [MANIFEST] or any(entry.name.startswith(p + '/') for p in TREES)
|
||||
path = root / entry.name
|
||||
total += entry.size
|
||||
if (not allowed or '\\' in entry.name or '..' in PurePosixPath(entry.name).parts
|
||||
or PurePosixPath(entry.name).as_posix() != entry.name
|
||||
or not entry.isfile() or entry.name in seen or len(seen) >= 100000
|
||||
or entry.size > 512 * 1024 * 1024
|
||||
or total > 2 * 1024**3 or not path.resolve().is_relative_to(root.resolve())
|
||||
or path.is_symlink()):
|
||||
raise ValueError('Unsafe prepared component archive')
|
||||
seen.add(entry.name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open('wb') as output:
|
||||
shutil.copyfileobj(archive.extractfile(entry), output)
|
||||
path.chmod(entry.mode & 0o777)
|
||||
data = json.loads((root / MANIFEST).read_text())
|
||||
actual = {name: digest(root / name) for name in files(root)}
|
||||
recipes = {name: identity(name, root) for name in registry(root)}
|
||||
if (data.get('version') != 1 or data['source_tree'] != source_tree or data['files'] != actual
|
||||
or data['recipes'] != recipes):
|
||||
raise ValueError('Prepared components do not match the pinned source tree and file hashes')
|
||||
print('Prepared component source identity and all output hashes verified.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('recipe')
|
||||
parser.add_argument('--archive', type=Path, default=ROOT / 'app/build/prepared-components.tar.gz')
|
||||
parser.add_argument('--tree')
|
||||
args = parser.parse_args()
|
||||
if args.recipe == 'plan':
|
||||
for name in registry():
|
||||
value = identity(name)
|
||||
print(f'{name}: {value}')
|
||||
if os.environ.get('GITHUB_OUTPUT'):
|
||||
with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
|
||||
stream.write(f'{name}={value}\n')
|
||||
elif args.recipe == 'tools':
|
||||
prepare_tools(os.environ.get('COMPONENTS_TO_BUILD', '').split())
|
||||
elif args.recipe == 'pack':
|
||||
pack(args.archive)
|
||||
elif args.recipe == 'unpack':
|
||||
unpack(args.archive, args.tree)
|
||||
else:
|
||||
prepare(args.recipe)
|
||||
Reference in new issue
Block a user