fix(ci): a preload lock older than a new output rebuilds that group, and generated headers are not hashed

This commit is contained in:
Kurt Himebauch committed 2026-11-09 13:07:46 -05:00
1 parent e8e7bc961e
commit 73ac363b08
1 file changed
+15 -5
+15 -5
View File
@@ -51,6 +51,7 @@ def source_files(group, root=ROOT, commit=None, patterns=None):
if not paths:
raise ValueError('No preload inputs found')
result = {}
listed = set(paths)
pending = list(paths)
while pending:
path = pending.pop()
@@ -58,7 +59,13 @@ def source_files(group, root=ROOT, commit=None, patterns=None):
continue
if path.startswith('../') or path.startswith('/'):
raise ValueError('Preload include escapes repository')
content = read(path).replace(b'\r\n', b'\n')
try:
content = read(path).replace(b'\r\n', b'\n')
except (OSError, subprocess.CalledProcessError):
# a header the build generates (gosys/gosys_blob.h, included behind __has_include)
if path in listed:
raise
continue
result[path] = sha(content)
for include in re.findall(rb'^\s*#\s*include\s*"([^"\n]+)"', content, re.M):
pending.append(posixpath.normpath(posixpath.join(posixpath.dirname(path), include.decode())))
@@ -132,7 +139,8 @@ def artifact_bytes(origin, cache):
def validate_lock(data):
if data.get('version') != 1 or set(data['groups']) != set(INPUTS):
raise ValueError('Unsupported preload snapshot')
if set(data['outputs']) != FILES:
# A lock made before a recipe gained an output lacks it; that group is rebuilt (restore)
if not set(data['outputs']) <= FILES:
raise ValueError('Unexpected preload output set')
for digest in [data['archive_sha256'], *data['outputs'].values()]:
if len(digest) != 64 or any(c not in '0123456789abcdef' for c in digest):
@@ -160,7 +168,7 @@ def producer_sources(group, commit, root=ROOT):
def prove_origin(data, cache, root=ROOT):
content = artifact_bytes(data['origin'], cache)
if {name: sha(value) for name, value in content.items()} != data['outputs']:
if {name: sha(value) for name, value in content.items() if name in data['outputs']} != data['outputs']:
raise ValueError('Snapshot binaries differ from the original main APK')
commit = data['origin']['commit']
run('git', 'fetch', '--no-tags', 'origin', commit, root=root)
@@ -183,7 +191,7 @@ def unpack(content, data):
total = 0
with tarfile.open(fileobj=io.BytesIO(content), mode='r:gz') as archive:
for entry in archive:
if entry.name not in FILES or entry.name in result or not entry.isfile() or entry.size > 32 * 1024 * 1024:
if entry.name not in data['outputs'] or entry.name in result or not entry.isfile() or entry.size > 32 * 1024 * 1024:
raise ValueError('Unsafe or unexpected preload archive entry')
total += entry.size
if total > 64 * 1024 * 1024:
@@ -192,7 +200,7 @@ def unpack(content, data):
if sha(value) != data['outputs'][entry.name]:
raise ValueError('Preload binary checksum mismatch')
result[entry.name] = value
if set(result) != FILES:
if set(result) != set(data['outputs']):
raise ValueError('Incomplete preload archive')
return result
@@ -203,6 +211,8 @@ def restore(root=ROOT):
matching = {group: source_files(group, root) == data['groups'][group] for group in INPUTS}
matching['arm64'] &= recipe(root) == data['arm64_recipe']
matching['x86'] &= x86_recipe(root) == data.get('x86_recipe')
for group in matching:
matching[group] &= all(name in data['outputs'] for name in OUTPUTS[group])
if os.environ.get('FORCE_PRELOAD_REBUILD') == 'true':
matching = dict.fromkeys(INPUTS, False)
if any(matching.values()):