mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-20 21:35:59 +00:00
233 lines
10 KiB
Python
233 lines
10 KiB
Python
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import subprocess
|
|
import tarfile
|
|
|
|
|
|
def identity(path):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open('rb') as source:
|
|
for block in iter(lambda: source.read(1024 * 1024), b''):
|
|
digest.update(block)
|
|
return {'sha256': digest.hexdigest(), 'size': Path(path).stat().st_size}
|
|
|
|
|
|
def data_file(member, magic):
|
|
if not member.isfile():
|
|
return False
|
|
extension = PurePosixPath(member.name).suffix.lower()
|
|
signatures = {
|
|
'.png': (b'\x89PNG\r\n\x1a\n',), '.jpg': (b'\xff\xd8\xff',),
|
|
'.jpeg': (b'\xff\xd8\xff',), '.ttf': (b'\x00\x01\x00\x00', b'ttcf', b'true'),
|
|
'.otf': (b'OTTO',), '.woff': (b'wOFF',), '.woff2': (b'wOF2',),
|
|
'.ogg': (b'OggS',), '.flac': (b'fLaC',),
|
|
'.mo': (b'\xde\x12\x04\x95', b'\x95\x04\x12\xde'),
|
|
}
|
|
if extension in signatures:
|
|
return magic.startswith(signatures[extension])
|
|
if extension == '.webp':
|
|
return magic.startswith(b'RIFF') and magic[8:12] == b'WEBP'
|
|
if extension == '.wav':
|
|
return magic.startswith(b'RIFF') and magic[8:12] == b'WAVE'
|
|
return False
|
|
|
|
|
|
def finish_archive(archive):
|
|
for block in iter(lambda: archive.fileobj.read(65536), b''):
|
|
if block.strip(b'\0'):
|
|
raise ValueError('Nonzero data follows the tar end marker')
|
|
|
|
|
|
def inventory(source, zstd):
|
|
process = subprocess.Popen([zstd, '-dc', str(source)], stdout=subprocess.PIPE)
|
|
records = []
|
|
links = set()
|
|
names = set()
|
|
candidates = set()
|
|
try:
|
|
with tarfile.open(fileobj=process.stdout, mode='r|') as archive:
|
|
for member in archive:
|
|
path = PurePosixPath(member.name)
|
|
if path.is_absolute() or '..' in path.parts or member.name in names:
|
|
raise ValueError('Archive contains an unsafe or repeated path: ' + member.name)
|
|
names.add(member.name)
|
|
digest = hashlib.sha256()
|
|
magic = b''
|
|
if member.isfile():
|
|
stream = archive.extractfile(member)
|
|
for block in iter(lambda: stream.read(1024 * 1024), b''):
|
|
if not magic:
|
|
magic = block[:16]
|
|
digest.update(block)
|
|
if data_file(member, magic):
|
|
candidates.add(member.name)
|
|
if member.islnk():
|
|
links.add(member.linkname)
|
|
records.append([member.name, member.type.decode('ascii'), member.size, member.mode,
|
|
member.uid, member.gid, member.uname, member.gname, member.mtime,
|
|
member.linkname, member.devmajor, member.devminor,
|
|
{key: value for key, value in member.pax_headers.items() if key not in
|
|
{'path', 'linkpath', 'mtime', 'size', 'uid', 'gid', 'uname', 'gname'}},
|
|
digest.hexdigest() if member.isfile() else None])
|
|
finish_archive(archive)
|
|
if process.wait():
|
|
raise ValueError('Archive decompression failed')
|
|
finally:
|
|
process.stdout.close()
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait()
|
|
candidates -= links
|
|
records.sort(key=lambda item: item[0])
|
|
digest = hashlib.sha256(json.dumps(records, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
|
return digest, candidates
|
|
|
|
|
|
def prepare(source, code, data, zstd_version='1.5.5'):
|
|
zstd = os.environ.get('DROIDDECK_ZSTD', 'zstd')
|
|
if 'v' + zstd_version + ',' not in subprocess.check_output([zstd, '--version'], text=True):
|
|
raise ValueError('Split archives require the pinned zstd version')
|
|
original, candidates = inventory(source, zstd)
|
|
if not candidates:
|
|
raise ValueError('Archive has no classified data assets')
|
|
segments = []
|
|
for kind, destination in [('code', code), ('data', data)]:
|
|
decoder = subprocess.Popen([zstd, '-dc', str(source)], stdout=subprocess.PIPE)
|
|
with Path(destination).open('xb') as output:
|
|
encoder = subprocess.Popen([zstd, '-19', '-T4', '-c'], stdin=subprocess.PIPE, stdout=output)
|
|
try:
|
|
with tarfile.open(fileobj=decoder.stdout, mode='r|') as archive:
|
|
for member in archive:
|
|
if (member.name in candidates) != (kind == 'data'):
|
|
continue
|
|
encoder.stdin.write(member.tobuf(format=tarfile.PAX_FORMAT))
|
|
if member.isfile():
|
|
stream = archive.extractfile(member)
|
|
remaining = member.size
|
|
while remaining:
|
|
block = stream.read(min(1024 * 1024, remaining))
|
|
if not block:
|
|
raise ValueError('Truncated archive member')
|
|
encoder.stdin.write(block)
|
|
remaining -= len(block)
|
|
encoder.stdin.write(b'\0' * (-member.size % 512))
|
|
finish_archive(archive)
|
|
if kind == 'data':
|
|
encoder.stdin.write(b'\0' * 10240)
|
|
encoder.stdin.close()
|
|
if decoder.wait() or encoder.wait():
|
|
raise ValueError('Split archive preparation failed')
|
|
finally:
|
|
decoder.stdout.close()
|
|
if not encoder.stdin.closed:
|
|
encoder.stdin.close()
|
|
for process in [decoder, encoder]:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait()
|
|
segments.append(dict(identity(destination), kind=kind))
|
|
return {'zstd_version': zstd_version, 'inventory_sha256': original, 'segments': segments}
|
|
|
|
|
|
def combine(paths, destination):
|
|
with Path(destination).open('xb') as output:
|
|
for path in paths:
|
|
with Path(path).open('rb') as source:
|
|
for block in iter(lambda: source.read(1024 * 1024), b''):
|
|
output.write(block)
|
|
|
|
|
|
def validate_data(source):
|
|
process = subprocess.Popen([os.environ.get('DROIDDECK_ZSTD', 'zstd'), '-dc', str(source)], stdout=subprocess.PIPE)
|
|
found = False
|
|
try:
|
|
with tarfile.open(fileobj=process.stdout, mode='r|') as archive:
|
|
for member in archive:
|
|
if not member.isfile() or not data_file(member, archive.extractfile(member).read(16)):
|
|
raise ValueError('Executable or unclassified content in a Play data segment')
|
|
found = True
|
|
finish_archive(archive)
|
|
if process.wait() or not found:
|
|
raise ValueError('Invalid Play data segment')
|
|
finally:
|
|
process.stdout.close()
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait()
|
|
|
|
|
|
def build_manifest(manifest_path, cache, work, output):
|
|
from tools.playstore.payloads import catalog_bytes, fetch
|
|
manifest = json.loads(Path(manifest_path).read_text())
|
|
work = Path(work)
|
|
work.mkdir(parents=True, exist_ok=True)
|
|
changes = {}
|
|
selected = {'desktop-ubuntu', 'desktop-legacy-kde', 'steamvr', 'droiddeck-proton', 'runtime-ubuntu'}
|
|
for entry in manifest['entries']:
|
|
if entry['id'] not in selected:
|
|
continue
|
|
source = dict(entry)
|
|
if 'layout' in source:
|
|
raise ValueError('Component is already split')
|
|
original = fetch(source, Path(cache))
|
|
code, data, combined = [work / (entry['id'] + '.' + part) for part in ['code.zst', 'data.zst', 'tar.zst']]
|
|
result_path = work / (entry['id'] + '.json')
|
|
if result_path.exists():
|
|
result = json.loads(result_path.read_text())
|
|
for path, part in zip([code, data], result['segments']):
|
|
if identity(path) != {key: part[key] for key in ['size', 'sha256']}:
|
|
raise ValueError('Existing prepared segment changed')
|
|
else:
|
|
result = prepare(original, code, data)
|
|
combine([code, data], combined)
|
|
if inventory(combined, os.environ.get('DROIDDECK_ZSTD', 'zstd'))[0] != result['inventory_sha256']:
|
|
raise ValueError('Split component changed its installed files')
|
|
result_path.write_text(json.dumps(result, indent=2) + '\n')
|
|
canonical = identity(combined)
|
|
changes[entry['sha256']] = canonical
|
|
entry.update(canonical)
|
|
source['urls'] = [url for url in source['urls'] if url.startswith('https://')]
|
|
entry['layout'] = dict(result, source=source)
|
|
entry.pop('prepare', None)
|
|
entry.pop('upstream', None)
|
|
print(entry['id'], json.dumps(canonical), flush=True)
|
|
|
|
def update(value):
|
|
if isinstance(value, dict):
|
|
digest = value.get('sha256') or str(value.get('digest', '')).removeprefix('sha256:')
|
|
if digest in changes and 'size' in value:
|
|
value['size'] = changes[digest]['size']
|
|
for key in value:
|
|
value[key] = update(value[key])
|
|
elif isinstance(value, list):
|
|
value = [update(item) for item in value]
|
|
elif isinstance(value, str):
|
|
for digest, replacement in changes.items():
|
|
if value == digest:
|
|
return replacement['sha256']
|
|
if value == 'sha256:' + digest:
|
|
return 'sha256:' + replacement['sha256']
|
|
return value
|
|
|
|
for entry in manifest['entries']:
|
|
if 'content' not in entry:
|
|
continue
|
|
entry['content'] = update(entry['content'])
|
|
content = catalog_bytes(entry['content'])
|
|
entry['sha256'] = hashlib.sha256(content).hexdigest()
|
|
entry['size'] = len(content)
|
|
Path(output).write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + '\n')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import argparse
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--manifest', type=Path, required=True)
|
|
parser.add_argument('--cache', type=Path, required=True)
|
|
parser.add_argument('--work', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
build_manifest(args.manifest, args.cache, args.work, args.output)
|