mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-21 13:55:58 +00:00
Complete pinned Play bundle inputs and guest prerequisite delivery
This commit is contained in:
1 parent
2f6e43e3d1
commit
567ca34a48
16 files changed
+10608
-22
No files matched your search
@@ -0,0 +1,17 @@
|
||||
# PlayStore builds
|
||||
|
||||
`Build PlayStore Bundle` runs on pushes to `playstore`. It builds the exact pushed commit, checks the payload inventory and native libraries, then signs the bundle in the protected `signing-playstore` environment. Push builds upload the signed bundle as the `droiddeck-play-bundle-signed` workflow artifact and do not publish an app release.
|
||||
|
||||
For a versioned GitHub release, run the existing `Release APKs` workflow with branch `playstore` and a Version matching `versionName` in `app/build.gradle`. That entry calls `Build PlayStore Bundle`. The Play release uses a separate tag and explicitly leaves Latest unchanged. The manual entry works without adding the branch-only workflow to `main`.
|
||||
|
||||
The signing environment supplies `RELEASE_KEYSTORE_B64`, `RELEASE_STORE_PASSWORD`, and `RELEASE_KEY_ALIAS`. Compilation never receives the release key. Signing checks the certificate against `keystore/release-signer.sha256`, keeps the alias out of signature filenames, and verifies every signed payload entry.
|
||||
|
||||
`components.lock.json` pins the native runtime, desktop packages, DroidDeck Proton, driver variants, native component variants, Decky channels, and synchronization packs. Native components have no external download fallback in the app. The bundled catalog pins current native variants rather than their complete release histories. Existing installed versions remain available.
|
||||
|
||||
The Windows prerequisite catalog remains intact. Its downloadable packages have pinned sizes and SHA-256 hashes and are installed into the guest runtime only after verification. This is an explicit exception to bundled-only component delivery. It does not enable native component downloads or app self-updates.
|
||||
|
||||
The separate `playstore-inputs-20261103` prerelease preserves inputs that upstream releases can replace or remove. Its payload filenames are content hashes, and `SOURCE-INVENTORY.json` records their original sources. The manifest retains runtime URL aliases separately from build fetch sources. This input prerelease is never Latest.
|
||||
|
||||
Staging verifies each input and deduplicates shared chunks across install-time feature modules. The final bundle check verifies the packaged inventory, every payload hash, individual module sizes, and the cumulative 4 GB budget. Bundle validation, 16 KB native-library alignment, and audio verification must also pass before signing.
|
||||
|
||||
Build checks establish artifact integrity. Device testing and Google Play review remain separate requirements.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -17,7 +17,7 @@ sys.path.insert(0, str(ROOT))
|
||||
CHUNK_BYTES = 64 * 1024 * 1024
|
||||
MODULE_BYTES = 448 * 1024 * 1024
|
||||
TOTAL_BYTES = 4_000_000_000
|
||||
RESERVED_BYTES = 250_000_000
|
||||
RESERVED_BYTES = 158_000_000
|
||||
|
||||
|
||||
def sha256(path):
|
||||
@@ -69,10 +69,29 @@ def load_manifest(path):
|
||||
raise ValueError('Payload requires a positive byte size: ' + identity)
|
||||
if digest in identities and identities[digest] != size:
|
||||
raise ValueError('Inconsistent size for identical payload: ' + identity)
|
||||
identities[digest] = size
|
||||
delivery = entry.get('delivery', 'bundled')
|
||||
if delivery not in {'bundled', 'windows-guest'}:
|
||||
raise ValueError('Unsupported component delivery: ' + identity)
|
||||
if delivery == 'windows-guest':
|
||||
if not identity.startswith('windows-') or manifest.get('windows_guest_downloads') is not True or not entry.get('provides') or any(
|
||||
not feature.startswith('windows:') for feature in entry['provides']):
|
||||
raise ValueError('External delivery is restricted to approved Windows prerequisites')
|
||||
https_url(entry.get('download_url', ''))
|
||||
if 'content' in entry or entry.get('prepare'):
|
||||
raise ValueError('External Windows prerequisites cannot replace bundled catalogs')
|
||||
else:
|
||||
identities[digest] = size
|
||||
if 'storage_chunk_bytes' in entry and (type(entry['storage_chunk_bytes']) is not int or
|
||||
not 65536 <= entry['storage_chunk_bytes'] <= CHUNK_BYTES):
|
||||
raise ValueError('Invalid component storage chunk size: ' + identity)
|
||||
aliases = entry.get('urls')
|
||||
if not isinstance(aliases, list) or not aliases:
|
||||
raise ValueError('Payload requires URL aliases: ' + identity)
|
||||
if 'fetch_urls' in entry:
|
||||
if not isinstance(entry['fetch_urls'], list) or not entry['fetch_urls']:
|
||||
raise ValueError('Payload fetch sources must be a nonempty list: ' + identity)
|
||||
for url in entry['fetch_urls']:
|
||||
https_url(url)
|
||||
for url in aliases:
|
||||
if url.startswith('droiddeck-bundle://'):
|
||||
if not re.fullmatch(r'droiddeck-bundle://[a-z0-9][a-z0-9._-]*/[A-Za-z0-9._+-]+', url) or not entry.get('prepare'):
|
||||
@@ -90,7 +109,7 @@ def load_manifest(path):
|
||||
if preparation is None and 'upstream' in entry:
|
||||
raise ValueError('Upstream archive requires explicit component preparation: ' + identity)
|
||||
if preparation is not None:
|
||||
if preparation.get('method') != 'gzip-to-zstd-19' or not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+', preparation.get('zstd_version', '')):
|
||||
if preparation.get('method') not in {'gzip-to-zstd-19', 'zstd-to-zstd-19'} or not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+', preparation.get('zstd_version', '')):
|
||||
raise ValueError('Invalid component preparation method: ' + identity)
|
||||
if not re.fullmatch(r'[a-f0-9]{64}', preparation.get('tar_sha256', '')) or type(preparation.get('tar_size')) is not int or not 0 < preparation['tar_size'] <= 16 * 1024 * 1024 * 1024:
|
||||
raise ValueError('Prepared component requires a pinned tar identity: ' + identity)
|
||||
@@ -180,7 +199,7 @@ def fetch(entry, cache):
|
||||
temporary.rename(target)
|
||||
return target
|
||||
opener = urllib.request.build_opener(HttpsRedirects(), urllib.request.HTTPSHandler(context=ssl.create_default_context()))
|
||||
request = urllib.request.Request(entry['urls'][0], headers={'Accept-Encoding': 'identity'})
|
||||
request = urllib.request.Request(entry.get('fetch_urls', entry['urls'])[0], headers={'Accept-Encoding': 'identity'})
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with opener.open(request, timeout=60) as response, temporary.open('xb') as output:
|
||||
@@ -204,6 +223,8 @@ def stage(manifest, cache, output, chunk_bytes=CHUNK_BYTES, module_bytes=MODULE_
|
||||
raise ValueError('Output must be a new directory: ' + str(output))
|
||||
payloads = {}
|
||||
for entry in manifest['entries']:
|
||||
if entry.get('delivery') == 'windows-guest':
|
||||
continue
|
||||
if entry['sha256'] not in payloads:
|
||||
payloads[entry['sha256']] = fetch(entry, cache)
|
||||
output.mkdir(parents=True)
|
||||
@@ -212,11 +233,19 @@ def stage(manifest, cache, output, chunk_bytes=CHUNK_BYTES, module_bytes=MODULE_
|
||||
module = 1
|
||||
used = 0
|
||||
chunks = {}
|
||||
shared = {}
|
||||
for digest, path in sorted(payloads.items()):
|
||||
pieces = []
|
||||
block_bytes = min([chunk_bytes] + [entry.get('storage_chunk_bytes', chunk_bytes) for entry in manifest['entries']
|
||||
if entry['sha256'] == digest and entry.get('delivery') != 'windows-guest'])
|
||||
with path.open('rb') as stream:
|
||||
number = 0
|
||||
for block in iter(lambda: stream.read(chunk_bytes), b''):
|
||||
for block in iter(lambda: stream.read(block_bytes), b''):
|
||||
chunk_digest = hashlib.sha256(block).hexdigest()
|
||||
if chunk_digest in shared:
|
||||
pieces.append(shared[chunk_digest])
|
||||
number += 1
|
||||
continue
|
||||
if used + len(block) > module_bytes:
|
||||
module += 1
|
||||
used = 0
|
||||
@@ -226,8 +255,9 @@ def stage(manifest, cache, output, chunk_bytes=CHUNK_BYTES, module_bytes=MODULE_
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with target.open('xb') as destination:
|
||||
destination.write(block)
|
||||
pieces.append({'asset': asset, 'module': name, 'size': len(block),
|
||||
'sha256': hashlib.sha256(block).hexdigest()})
|
||||
piece = {'asset': asset, 'module': name, 'size': len(block), 'sha256': chunk_digest}
|
||||
pieces.append(piece)
|
||||
shared[chunk_digest] = piece
|
||||
used += len(block)
|
||||
number += 1
|
||||
chunks[digest] = pieces
|
||||
@@ -238,8 +268,12 @@ def stage(manifest, cache, output, chunk_bytes=CHUNK_BYTES, module_bytes=MODULE_
|
||||
(recipes / (digest + '.json')).write_text(json.dumps({'chunks': pieces}, separators=(',', ':')) + '\n')
|
||||
for entry in manifest['entries']:
|
||||
for url in entry['urls']:
|
||||
index['entries'][url] = {'id': entry['id'], 'sha256': entry['sha256'], 'size': entry['size'],
|
||||
'recipe': 'play-components/recipes/' + entry['sha256'] + '.json'}
|
||||
metadata = {'id': entry['id'], 'sha256': entry['sha256'], 'size': entry['size']}
|
||||
if entry.get('delivery') == 'windows-guest':
|
||||
metadata.update(delivery='windows-guest', download_url=entry['download_url'])
|
||||
else:
|
||||
metadata['recipe'] = 'play-components/recipes/' + entry['sha256'] + '.json'
|
||||
index['entries'][url] = metadata
|
||||
(output / 'index.json').write_text(json.dumps(index, indent=2, sort_keys=True) + '\n')
|
||||
(output / 'provenance.json').write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n')
|
||||
(base / 'index.json').write_text(json.dumps(index, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
@@ -287,6 +321,12 @@ def verify_bundle(manifest, bundle):
|
||||
entry = index['entries'][url]
|
||||
if any(entry.get(field) != pinned[field] for field in ['id', 'sha256', 'size']):
|
||||
raise ValueError('Bundle component metadata mismatch: ' + pinned['id'])
|
||||
if pinned.get('delivery') == 'windows-guest':
|
||||
if entry != {field: pinned[field] for field in ['id', 'sha256', 'size', 'delivery', 'download_url']}:
|
||||
raise ValueError('Bundle Windows prerequisite metadata mismatch')
|
||||
continue
|
||||
if 'delivery' in entry or 'download_url' in entry:
|
||||
raise ValueError('Bundled native components cannot use external delivery')
|
||||
recipe = 'play-components/recipes/' + pinned['sha256'] + '.json'
|
||||
if entry.get('recipe') != recipe:
|
||||
raise ValueError('Bundle component recipe path mismatch')
|
||||
|
||||
@@ -2,17 +2,27 @@ import gzip
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from contextlib import ExitStack
|
||||
|
||||
|
||||
def prepare(source, destination, definition):
|
||||
if definition['method'] != 'gzip-to-zstd-19':
|
||||
if definition['method'] not in {'gzip-to-zstd-19', 'zstd-to-zstd-19'}:
|
||||
raise ValueError('Unsupported component preparation method')
|
||||
version = subprocess.check_output(['zstd', '--version'], text=True)
|
||||
if 'v' + definition['zstd_version'] + ',' not in version:
|
||||
raise ValueError('Component preparation requires the pinned zstd version')
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with gzip.open(source, 'rb') as original, Path(destination).open('xb') as output:
|
||||
with ExitStack() as stack:
|
||||
decoder = None
|
||||
if definition['method'] == 'gzip-to-zstd-19':
|
||||
original = stack.enter_context(gzip.open(source, 'rb'))
|
||||
else:
|
||||
decoder = subprocess.Popen(['zstd', '-dc', str(source)], stdout=subprocess.PIPE)
|
||||
stack.callback(decoder.wait)
|
||||
stack.callback(lambda: decoder.poll() is None and decoder.terminate())
|
||||
original = stack.enter_context(decoder.stdout)
|
||||
output = stack.enter_context(Path(destination).open('xb'))
|
||||
process = subprocess.Popen(['zstd', '-19', '-T4', '-c'], stdin=subprocess.PIPE, stdout=output)
|
||||
try:
|
||||
for block in iter(lambda: original.read(1024 * 1024), b''):
|
||||
@@ -22,6 +32,8 @@ def prepare(source, destination, definition):
|
||||
digest.update(block)
|
||||
process.stdin.write(block)
|
||||
process.stdin.close()
|
||||
if decoder is not None and decoder.wait():
|
||||
raise ValueError('Component archive decompression failed')
|
||||
if process.wait():
|
||||
raise ValueError('Component archive compression failed')
|
||||
except BaseException:
|
||||
|
||||
@@ -19,7 +19,14 @@ def sign(source, destination, keystore, alias, expected_signer):
|
||||
raise ValueError('Signing alias is unavailable')
|
||||
with tempfile.TemporaryDirectory(prefix='play-signing-') as temporary:
|
||||
unsigned = Path(temporary) / 'unsigned.aab'
|
||||
sanitize(source, unsigned)
|
||||
with zipfile.ZipFile(source) as original:
|
||||
if len(original.namelist()) != len(set(original.namelist())):
|
||||
raise ValueError('Bundle contains duplicate ZIP entries')
|
||||
needs_sanitizing = any(signature(entry.filename) for entry in original.infolist())
|
||||
if needs_sanitizing:
|
||||
sanitize(source, unsigned)
|
||||
else:
|
||||
unsigned = source
|
||||
environment = dict(os.environ, RELEASE_KEY_ALIAS=alias, RELEASE_KEYSTORE_PATH=str(keystore.resolve()))
|
||||
result = subprocess.run(['java', '--add-modules', 'jdk.jartool', str(Path(__file__).with_name('SignBundle.java')),
|
||||
str(unsigned), str(destination), expected_signer], env=environment, capture_output=True, text=True)
|
||||
@@ -30,15 +37,18 @@ def sign(source, destination, keystore, alias, expected_signer):
|
||||
print('Bundle signature and payload coverage verified')
|
||||
|
||||
|
||||
def signature(name):
|
||||
name = name.upper()
|
||||
return name == 'META-INF/MANIFEST.MF' or name.startswith('META-INF/') and '/' not in name[9:] and (
|
||||
name.endswith(('.SF', '.RSA', '.DSA', '.EC')) or name[9:].startswith('SIG-'))
|
||||
|
||||
|
||||
def sanitize(source, destination):
|
||||
with zipfile.ZipFile(source) as original, zipfile.ZipFile(destination, 'x') as output:
|
||||
if len(original.namelist()) != len(set(original.namelist())):
|
||||
raise ValueError('Bundle contains duplicate ZIP entries')
|
||||
for entry in original.infolist():
|
||||
name = entry.filename.upper()
|
||||
signature = name == 'META-INF/MANIFEST.MF' or name.startswith('META-INF/') and '/' not in name[9:] and (
|
||||
name.endswith(('.SF', '.RSA', '.DSA', '.EC')) or name[9:].startswith('SIG-'))
|
||||
if signature:
|
||||
if signature(entry.filename):
|
||||
continue
|
||||
with original.open(entry) as source_stream, output.open(entry, 'w', force_zip64=True) as target:
|
||||
while block := source_stream.read(1024 * 1024):
|
||||
|
||||
@@ -46,6 +46,14 @@ class PlayArchivePreparationTest(unittest.TestCase):
|
||||
prepare(self.source, output, self.definition)
|
||||
self.assertFalse(output.exists())
|
||||
|
||||
def test_zstd_recompression_preserves_the_exact_original_tar_bytes(self):
|
||||
original = self.root / 'original.tar.zst'
|
||||
original.write_bytes(subprocess.check_output(['zstd', '-1', '-c'], input=self.content))
|
||||
definition = dict(self.definition, method='zstd-to-zstd-19')
|
||||
output = self.root / 'recompressed.tar.zst'
|
||||
prepare(original, output, definition)
|
||||
self.assertEqual(self.content, subprocess.check_output(['zstd', '-dc', str(output)]))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,54 @@
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
from tools.playstore.payloads import load_manifest, validate_play_engines
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class PlayManifestTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.manifest = load_manifest(ROOT / 'tools/playstore/components.lock.json')
|
||||
self.entries = {url: entry for entry in self.manifest['entries'] for url in entry['urls']}
|
||||
|
||||
def test_release_inventory_satisfies_engine_and_delivery_restrictions(self):
|
||||
validate_play_engines(self.manifest)
|
||||
self.assertTrue(any(entry.get('delivery') == 'windows-guest' for entry in self.manifest['entries']))
|
||||
for entry in self.manifest['entries']:
|
||||
if entry.get('delivery') == 'windows-guest':
|
||||
self.assertTrue(entry['id'].startswith('windows-'))
|
||||
self.assertTrue(all(feature.startswith('windows:') for feature in entry['provides']))
|
||||
|
||||
def test_every_advertised_native_release_asset_is_available_with_the_same_identity(self):
|
||||
def visit(value):
|
||||
if isinstance(value, dict):
|
||||
if 'browser_download_url' in value:
|
||||
entry = self.entries[value['browser_download_url']]
|
||||
self.assertEqual(value['size'], entry['size'])
|
||||
if value.get('digest'):
|
||||
self.assertEqual(value['digest'], 'sha256:' + entry['sha256'])
|
||||
for item in value.values():
|
||||
visit(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
visit(item)
|
||||
for entry in self.manifest['entries']:
|
||||
if 'content' in entry:
|
||||
visit(entry['content'])
|
||||
|
||||
def test_catalog_endpoints_used_by_android_and_the_guest_are_present(self):
|
||||
repositories = ['The412Banner/Banners-Turnip', 'WinNative-Emu/Drivers', 'Droid-Deck/Drivers-CI']
|
||||
urls = ['https://api.github.com/repos/' + repo + '/releases?per_page=15' for repo in repositories]
|
||||
urls += ['https://api.github.com/repos/Droid-Deck/DroidDeck-Proton/releases?per_page=15',
|
||||
'https://api.github.com/repos/Droid-Deck/DroidDeck-Proton/releases',
|
||||
'https://api.github.com/repos/SteamDeckHomebrew/decky-loader/releases?per_page=30']
|
||||
for url in urls:
|
||||
self.assertIsInstance(self.entries[url]['content'], list)
|
||||
proton = self.entries[urls[-3]]['content']
|
||||
self.assertTrue(any(asset['browser_download_url'].startswith('droiddeck-bundle://')
|
||||
for release in proton for asset in release['assets']))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -93,6 +93,44 @@ class PlaystorePayloadsTest(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, 'Missing component coverage: proton'):
|
||||
self.validate()
|
||||
|
||||
def test_external_windows_prerequisite_requires_explicit_scope_and_pinned_source(self):
|
||||
entry = dict(self.manifest['entries'][0], id='windows-vcredist', delivery='windows-guest',
|
||||
provides=['windows:vcredist'], download_url='https://example.test/vcredist.exe')
|
||||
self.manifest['entries'].append(entry)
|
||||
entry['urls'] = ['https://example.test/vcredist.exe']
|
||||
with self.assertRaisesRegex(ValueError, 'restricted to approved Windows'):
|
||||
self.validate()
|
||||
self.manifest['windows_guest_downloads'] = True
|
||||
self.validate()
|
||||
entry['provides'] = ['runtime:linux']
|
||||
with self.assertRaisesRegex(ValueError, 'restricted to approved Windows'):
|
||||
self.validate()
|
||||
|
||||
def test_external_windows_bytes_are_not_fetched_or_counted_as_bundled_payload(self):
|
||||
entry = dict(self.manifest['entries'][0], id='windows-vcredist', delivery='windows-guest',
|
||||
provides=['windows:vcredist'], download_url='https://example.test/vcredist.exe',
|
||||
urls=['https://example.test/vcredist.exe'], sha256='b' * 64, size=5_000_000_000)
|
||||
self.manifest['entries'].append(entry)
|
||||
self.manifest['windows_guest_downloads'] = True
|
||||
with patch.object(payloads.urllib.request, 'build_opener', side_effect=AssertionError('Unexpected network')):
|
||||
index = payloads.stage(self.validate(), self.cache, self.root / 'staged')
|
||||
metadata = index['entries'][entry['urls'][0]]
|
||||
self.assertEqual('windows-guest', metadata['delivery'])
|
||||
self.assertEqual(entry['download_url'], metadata['download_url'])
|
||||
self.assertNotIn('recipe', metadata)
|
||||
|
||||
def test_shared_storage_chunks_preserve_distinct_component_identities(self):
|
||||
first = self.manifest['entries'][0]
|
||||
second = dict(first, id='second', urls=['https://example.test/second'], size=first['size'] + 7)
|
||||
data = self.content + b'changed'
|
||||
second['sha256'] = hashlib.sha256(data).hexdigest()
|
||||
(self.cache / second['sha256']).write_bytes(data)
|
||||
self.manifest['entries'].append(second)
|
||||
index = payloads.stage(self.validate(), self.cache, self.root / 'staged', chunk_bytes=7, module_bytes=14)
|
||||
one, two = [index['entries'][entry['urls'][0]] for entry in [first, second]]
|
||||
self.assertEqual(self.chunks(one)[0], self.chunks(two)[0])
|
||||
self.assertNotEqual(one['sha256'], two['sha256'])
|
||||
|
||||
def test_unknown_schema_and_empty_manifest_are_rejected(self):
|
||||
self.manifest['schema'] = 2
|
||||
with self.assertRaisesRegex(ValueError, 'schema'):
|
||||
|
||||
@@ -77,6 +77,14 @@ class PlaystoreSigningTest(unittest.TestCase):
|
||||
signing.sign(self.source, self.destination, self.keystore, 'private-alias-value', self.signer)
|
||||
self.assertFalse(self.destination.exists())
|
||||
|
||||
def test_unsigned_bundle_is_signed_without_a_second_full_bundle_copy(self):
|
||||
with zipfile.ZipFile(self.source, 'w') as package:
|
||||
package.writestr('base/assets/runtime', b'component payload')
|
||||
with patch.object(signing, 'sanitize', side_effect=AssertionError('Unexpected full bundle copy')):
|
||||
self.sign()
|
||||
with zipfile.ZipFile(self.destination) as package:
|
||||
self.assertEqual(b'component payload', package.read('base/assets/runtime'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user