Files
DroidDeck/tools/droiddeck-esync/discover.py
T
MaxsTechReview 8edf05b19a Add the droiddeck-esync pack pipeline (#189)
Builds esync packs for new Valve, GE and CachyOS Proton 11 releases on arm64
runners, smoke-tests them against stock with esstress, and on main publishes
each pack and a signed index as non-latest releases. A watcher checks Steam
and the GE/CachyOS releases every 20 minutes and starts a build for new ones.
The app side lands separately with feat/userspace-esync-ntsync.
2026-10-04 02:58:55 +00:00

655 lines
31 KiB
Python
Executable File

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import subprocess
import sys
import urllib.parse
from datetime import datetime, timedelta, timezone
from pathlib import Path
from make_index import check_entry
from make_pack import FLAVORS, HERE, PATCHES, REVOKED, SERIES, PackError, patch_digest, read_revoked, sha256_file, slug
FLAVORS_FILE = HERE / "flavors.json"
ARCHIVE = re.compile(r"\.tar\.(gz|xz)$")
ASSIGNMENT = re.compile(r"(?:export\s+|override\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*(?:\?|\+|!|:{1,3})?=\s*(.*)")
REFERENCE = re.compile(r"\$[({]([A-Za-z_][A-Za-z0-9_]*)[)}]")
IMAGE = re.compile(r"[A-Za-z0-9][A-Za-z0-9.-]*(?::[0-9]+)?(?:/[A-Za-z0-9._-]+)+(?::[A-Za-z0-9._-]+)?(?:@sha256:[0-9a-f]{64})?")
REPO = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+")
CLONES = ("shallow", "treeless")
PACK_REV = re.compile(r"-r([0-9]+)$")
ARTIFACT_PREFIX = "droiddeck-esync-pack-"
FAILED = ".failed"
RETRY_AFTER = timedelta(days=1)
VDF_TOKEN = re.compile(r'"((?:[^"\\]|\\.)*)"|([{}])')
APP_HEADER = re.compile(r"^AppID : ([0-9]+),", re.M)
ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
def gh(*args: str) -> subprocess.CompletedProcess:
try:
return subprocess.run(["gh", *args], capture_output=True, text=True, check=False)
except FileNotFoundError as e:
raise PackError("the GitHub CLI (gh) is not installed") from e
def json_values(text: str) -> list:
decoder = json.JSONDecoder()
values = []
position = 0
while True:
while position < len(text) and text[position].isspace():
position += 1
if position == len(text):
return values
value, position = decoder.raw_decode(text, position)
values.append(value)
def gh_api(path: str, paginate: bool = False, missing_ok: bool = False):
result = gh("api", *(["--paginate"] if paginate else []), path)
if result.returncode != 0:
if missing_ok and "HTTP 404" in result.stderr:
return None
raise PackError(f"gh api {path}: {result.stderr.strip() or f'exit {result.returncode}'}")
try:
values = json_values(result.stdout)
except ValueError as e:
raise PackError(f"gh api {path}: {e}") from e
if len(values) == 1:
return values[0]
if values and all(isinstance(value, list) for value in values):
return [item for value in values for item in value]
raise PackError(f"gh api {path}: unexpected output")
def quote(value: str) -> str:
return urllib.parse.quote(value, safe="")
def load_flavors(path: Path) -> dict:
try:
data = json.loads(path.read_text())
except (OSError, ValueError) as e:
raise PackError(f"cannot read {path}: {e}") from e
def fail(why: str):
raise PackError(f"{path}: {why}")
def compiles(pattern: str, where: str):
try:
re.compile(pattern)
except re.error as e:
fail(f"{where}: {pattern!r} is not a regular expression: {e}")
if not isinstance(data, dict) or data.get("format") != 1 or not isinstance(data.get("flavors"), dict):
fail("is not a format 1 flavor list")
image = data.get("sdk_image")
if not isinstance(image, dict) or not all(isinstance(image.get(key), str) and image[key] for key in ("file", "variable", "match")):
fail("sdk_image needs file, variable and match")
for name, cfg in data["flavors"].items():
if name not in FLAVORS:
fail(f"unknown flavor {name!r}")
if not isinstance(cfg, dict) or not isinstance(cfg.get("repo"), str) or not REPO.fullmatch(cfg["repo"]):
fail(f"{name}: repo must be owner/name")
for key in ("keep", "rev"):
if type(cfg.get(key)) is not int or cfg[key] < 1:
fail(f"{name}: {key} must be a positive number")
if not isinstance(cfg.get("series"), str) or not SERIES.fullmatch(cfg["series"]):
fail(f"{name}: series is not a patch series name")
if cfg.get("profile") not in ("valve", "cachyos"):
fail(f"{name}: profile must be valve or cachyos")
if type(cfg.get("source_match")) is not bool:
fail(f"{name}: source_match must be a boolean")
build_dir = cfg.get("build_dir")
if not isinstance(build_dir, str) or not re.fullmatch(r"(?:/[A-Za-z0-9._-]+)+", build_dir) or ".." in build_dir.split("/"):
fail(f"{name}: build_dir must be the absolute path the release was built at")
wine = cfg.get("wine")
submodules = wine.get("submodules") if isinstance(wine, dict) else None
if not isinstance(submodules, list) or "wine" not in submodules \
or not all(isinstance(item, str) and re.fullmatch(r"[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*", item) for item in submodules):
fail(f"{name}: wine.submodules must list the submodules to check out, wine among them")
if wine.get("clone") not in CLONES:
fail(f"{name}: wine.clone must be one of {', '.join(CLONES)}")
prep = wine.get("prep")
if not isinstance(prep, str) or (prep and not re.fullmatch(r"[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*", prep)) or ".." in prep.split("/"):
fail(f"{name}: wine.prep must be a path inside the checkout or empty")
if cfg.get("list") == "tags":
families = cfg.get("families")
if not isinstance(families, dict) or not families:
fail(f"{name}: families must name at least one tag family")
for family, settings in families.items():
if not isinstance(settings, dict) or not isinstance(settings.get("tag"), str) \
or not isinstance(settings.get("prefix"), str) or not re.fullmatch(r"[A-Za-z0-9._-]+", settings["prefix"]) \
or not isinstance(settings.get("depot_app", ""), str):
fail(f"{name}: family {family} needs a tag prefix and a tag pattern")
compiles(settings["tag"], f"{name}: family {family}")
if not isinstance(cfg.get("version_token"), str) or "{tag}" not in cfg["version_token"]:
fail(f"{name}: version_token must contain {{tag}}")
elif cfg.get("list") == "releases":
if not isinstance(cfg.get("tag"), str) or not isinstance(cfg.get("assets"), list) or not cfg["assets"] \
or not all(isinstance(pattern, str) for pattern in cfg["assets"]):
fail(f"{name}: releases need a tag pattern and asset patterns")
for pattern in (cfg["tag"], *cfg["assets"]):
compiles(pattern, name)
else:
fail(f"{name}: list must be tags or releases")
return data
def natural_key(text: str) -> list[tuple]:
return [(0, int(part), "") if part[0] in "0123456789" else (1, 0, part) for part in re.findall(r"[0-9]+|[^0-9]+", text)]
def candidate(flavor: str, cfg: dict, tag: str, asset: str = "", token: str = "", depot_app: str = "") -> dict:
return {
"key": slug(f"{flavor}-{ARCHIVE.sub('', asset) if asset else tag}"),
"flavor": flavor,
"repo": cfg["repo"],
"tag": tag,
"asset": asset,
"token": token,
"depot_app": depot_app,
"rev": cfg["rev"],
"series": cfg["series"],
"profile": cfg["profile"],
"source_match": cfg["source_match"],
"submodules": " ".join(cfg["wine"]["submodules"]),
"clone": cfg["wine"]["clone"],
"prep": cfg["wine"]["prep"],
"build_dir": cfg["build_dir"],
}
def tag_candidates(flavor: str, cfg: dict, ref: str) -> list[dict]:
found = []
for family, settings in sorted(cfg["families"].items()):
if ref and not ref.startswith(settings["prefix"]):
continue
refs = gh_api(f"repos/{cfg['repo']}/git/matching-refs/tags/{quote(settings['prefix'])}", paginate=True)
if not isinstance(refs, list):
raise PackError(f"{cfg['repo']}: the tag list is not a list")
pattern = re.compile(settings["tag"])
names = set()
for item in refs:
name = item.get("ref", "") if isinstance(item, dict) else ""
if isinstance(name, str) and name.startswith("refs/tags/") and pattern.search(name[len("refs/tags/"):]):
names.add(name[len("refs/tags/"):])
ordered = sorted(names, key=natural_key, reverse=True)
chosen = [name for name in ordered if name == ref] if ref else ordered[:cfg["keep"]]
for name in chosen:
found.append(candidate(flavor, cfg, name, token=cfg["version_token"].format(tag=name), depot_app=settings.get("depot_app", "")))
return found
def pick_assets(assets: list, patterns: list[re.Pattern]) -> list[str]:
chosen = {}
for asset in assets:
name = asset.get("name") if isinstance(asset, dict) else None
if not isinstance(name, str) or not any(pattern.search(name) for pattern in patterns):
continue
stem = ARCHIVE.sub("", name)
if stem not in chosen or name.endswith(".tar.xz"):
chosen[stem] = name
return sorted(chosen.values())
def release_candidates(flavor: str, cfg: dict, ref: str) -> list[dict]:
repo = cfg["repo"]
if ref:
release = gh_api(f"repos/{repo}/releases/tags/{quote(ref)}", missing_ok=True)
releases = [release] if release else []
else:
releases = gh_api(f"repos/{repo}/releases?per_page=100")
if not isinstance(releases, list):
raise PackError(f"{repo}: the release list is not a list")
pattern = re.compile(cfg["tag"])
patterns = [re.compile(item) for item in cfg["assets"]]
usable = []
for release in releases:
tag = release.get("tag_name") if isinstance(release, dict) else None
if not isinstance(tag, str) or release.get("draft") or not pattern.search(tag):
continue
assets = pick_assets(release.get("assets") or [], patterns)
if assets:
usable.append((release.get("published_at") or release.get("created_at") or "", natural_key(tag), tag, assets))
usable.sort(reverse=True)
if not ref:
usable = usable[:cfg["keep"]]
return [candidate(flavor, cfg, tag, asset=asset) for _, _, tag, assets in usable for asset in assets]
def built(index: dict) -> dict[tuple[str, str, str], int]:
revs = {}
packs = index.get("packs") if isinstance(index, dict) else None
for pack in packs if isinstance(packs, list) else []:
if not isinstance(pack, dict) or not isinstance(pack.get("source"), dict):
continue
rev = pack.get("rev") if type(pack.get("rev")) is int else 0
key = (pack.get("flavor"), pack["source"].get("ref"), pack["source"].get("asset", ""))
revs[key] = max(revs.get(key, 0), rev)
return revs
def revoked_build(entry: dict, revoked: set[str]) -> bool:
prefixes = {f"{entry['flavor']}-{slug(name)}-" for name in (entry["token"], entry["tag"]) if name}
suffix = f"-r{entry['rev']}"
return any(ident.endswith(suffix) and ident.startswith(prefix) for ident in revoked for prefix in prefixes)
def chosen_flavors(flavors: dict, selected: str) -> list[str]:
return sorted(flavors["flavors"]) if selected == "all" else [selected]
def failure_marker(entry: dict, digests: dict[str, str] | None = None) -> str:
series = entry["series"]
if not SERIES.fullmatch(series):
raise PackError(f"{series!r} is not a patch series name")
digest = digests.get(series) if digests is not None else None
if digest is None:
digest = patch_digest(PATCHES / series)
if digests is not None:
digests[series] = digest
return f"{entry['key']}-r{entry['rev']}-{digest[:12]}{FAILED}"
def discover(flavors: dict, selected: str, ref: str, index: dict, revoked: set[str],
failed: frozenset[str] = frozenset()) -> list[dict]:
names = chosen_flavors(flavors, selected)
done = built(index)
digests = {}
entries = []
seen = set()
matched = False
for flavor in names:
cfg = flavors["flavors"].get(flavor)
if cfg is None:
raise PackError(f"flavors.json has no {flavor}")
found = tag_candidates(flavor, cfg, ref) if cfg["list"] == "tags" else release_candidates(flavor, cfg, ref)
matched = matched or bool(found)
for entry in found:
if entry["key"] in seen:
continue
seen.add(entry["key"])
label = f"{flavor} {entry['tag']}{' ' + entry['asset'] if entry['asset'] else ''}"
if revoked_build(entry, revoked):
print(f"discover: {label}: revoked at rev {entry['rev']}", file=sys.stderr)
elif done.get((flavor, entry["tag"], entry["asset"]), 0) >= entry["rev"]:
print(f"discover: {label}: in the index at rev {entry['rev']}", file=sys.stderr)
elif not ref and failure_marker(entry, digests) in failed:
print(f"discover: {label}: failed at rev {entry['rev']} with this patch series in the last "
f"{RETRY_AFTER.days} days; skipped (bump rev, change the series or build the tag by name)", file=sys.stderr)
else:
print(f"discover: {label}: to build", file=sys.stderr)
entries.append(entry)
if ref and not matched:
raise PackError(f"no {'' if selected == 'all' else selected + ' '}build is tagged {ref}")
return entries
def vdf(text: str) -> dict:
stack = [{}]
key = None
for match in VDF_TOKEN.finditer(text):
word, brace = match.group(1), match.group(2)
if brace == "{":
if key is None:
raise PackError("app info: a block has no name")
child = {}
stack[-1][key] = child
stack.append(child)
key = None
elif brace == "}":
if len(stack) == 1:
raise PackError("app info: unbalanced braces")
stack.pop()
key = None
elif key is None:
key = word
else:
stack[-1][key] = word
key = None
if len(stack) != 1:
raise PackError("app info: unbalanced braces")
return stack[0]
def steam_builds(text: str, apps: list[str]) -> dict[str, dict]:
found = {}
text = ANSI.sub("", text)
starts = [(m.start(), m.group(1)) for m in APP_HEADER.finditer(text)]
for number, (start, app) in enumerate(starts):
end = starts[number + 1][0] if number + 1 < len(starts) else len(text)
if app not in apps:
continue
body = text[text.find("\n", start) + 1:end]
info = vdf(body).get(app)
depots = info.get("depots") if isinstance(info, dict) else None
public = depots.get("branches", {}).get("public") if isinstance(depots, dict) else None
if not isinstance(public, dict) or not str(public.get("buildid", "")).isdigit():
continue
manifests = sorted(depot["manifests"]["public"]["gid"] for name, depot in depots.items()
if name.isdigit() and isinstance(depot, dict) and isinstance(depot.get("manifests"), dict)
and isinstance(depot["manifests"].get("public"), dict) and depot["manifests"]["public"].get("gid"))
found[app] = {"buildid": public["buildid"], "built": public.get("timebuildupdated", ""), "manifests": manifests}
return found
def steam_apps(flavors: dict) -> list[str]:
return sorted({family["depot_app"] for cfg in flavors["flavors"].values() if cfg["list"] == "tags"
for family in cfg["families"].values() if family.get("depot_app")})
def watch(flavors: dict, previous: dict, steam: dict[str, dict] | None) -> tuple[dict, list[str], list[str]]:
state = {}
changed = []
notes = []
for flavor in sorted(flavors["flavors"]):
cfg = flavors["flavors"][flavor]
found = tag_candidates(flavor, cfg, "") if cfg["list"] == "tags" else release_candidates(flavor, cfg, "")
builds = sorted(f"{entry['tag']} {entry['asset']}".strip() for entry in found)
ships = {}
if cfg["list"] == "tags" and steam is not None:
for family in cfg["families"].values():
app = family.get("depot_app")
if app and app in steam:
ships[app] = steam[app]
old = previous.get(flavor) if isinstance(previous.get(flavor), dict) else {}
if steam is None and isinstance(old.get("steam"), dict):
ships = old["steam"]
mark = {"builds": builds, "rev": cfg["rev"], "series": cfg["series"], "patch": patch_digest(PATCHES / cfg["series"])}
fingerprint = hashlib.sha256(json.dumps(mark, sort_keys=True).encode()).hexdigest()
state[flavor] = {**mark, "fingerprint": fingerprint, "steam": ships}
if old.get("fingerprint") != fingerprint:
changed.append(flavor)
fresh = sorted(set(builds) - set(old.get("builds") or []))
notes.append(f"{flavor}: {', '.join(fresh) if fresh else 'the patch series or rev changed'}")
for app, ship in sorted(ships.items()):
before = (old.get("steam") or {}).get(app) if isinstance(old.get("steam"), dict) else None
if isinstance(before, dict) and before.get("buildid") != ship.get("buildid"):
notes.append(f"{flavor}: Steam app {app} moved from build {before.get('buildid')} to {ship.get('buildid')}")
if flavor not in changed:
changed.append(flavor)
return state, changed, notes
def asset_text(repo: str, asset_id, what: str) -> str:
result = gh("api", "-H", "Accept: application/octet-stream", f"repos/{repo}/releases/assets/{asset_id}")
if result.returncode != 0:
raise PackError(f"cannot download {what}: {result.stderr.strip()}")
return result.stdout
def release_index(repo: str, tag: str) -> dict:
release = gh_api(f"repos/{repo}/releases/tags/{quote(tag)}", missing_ok=True)
if release is None:
return {}
for asset in release.get("assets") or []:
if isinstance(asset, dict) and asset.get("name") == "index.json":
text = asset_text(repo, asset["id"], f"the index of {repo}")
try:
return json.loads(text)
except ValueError as e:
raise PackError(f"the index of {repo} is not JSON: {e}") from e
return {}
def published(repo: str, flavors: dict, selected: str, index: dict) -> list[dict]:
packs = index.get("packs") if isinstance(index, dict) else None
listed = {pack.get("id") for pack in packs if isinstance(pack, dict)} if isinstance(packs, list) else set()
found = []
for flavor in chosen_flavors(flavors, selected):
cfg = flavors["flavors"].get(flavor)
tag = f"droiddeck-esync-{flavor}"
release = gh_api(f"repos/{repo}/releases/tags/{quote(tag)}", missing_ok=True) if cfg else None
if not isinstance(release, dict) or type(release.get("id")) is not int:
continue
assets = gh_api(f"repos/{repo}/releases/{release['id']}/assets?per_page=100", paginate=True)
if not isinstance(assets, list):
raise PackError(f"{repo} {tag}: the asset list is not a list")
for asset in assets:
name = asset.get("name") if isinstance(asset, dict) else None
if not isinstance(name, str) or not name.endswith(".json"):
continue
ident = name[:-len(".json")]
rev = PACK_REV.search(ident)
if ident in listed or not rev or int(rev.group(1)) < cfg["rev"]:
continue
try:
entry = json.loads(asset_text(repo, asset.get("id"), f"{tag} {name}"))
except ValueError:
entry = None
if not isinstance(entry, dict) or entry.get("id") != ident or entry.get("flavor") != flavor:
print(f"discover: {tag} {name} is not a pack entry", file=sys.stderr)
continue
print(f"discover: {tag} {name}: published, not in the index yet", file=sys.stderr)
found.append(entry)
return found
def parse_time(value) -> datetime | None:
if not isinstance(value, str):
return None
try:
moment = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
return moment if moment.tzinfo else moment.replace(tzinfo=timezone.utc)
def recent_failures(repo: str, flavors: dict, selected: str, now: datetime) -> frozenset[str]:
found = set()
for flavor in chosen_flavors(flavors, selected):
tag = f"droiddeck-esync-{flavor}"
release = gh_api(f"repos/{repo}/releases/tags/{quote(tag)}", missing_ok=True) if flavor in flavors["flavors"] else None
if not isinstance(release, dict) or type(release.get("id")) is not int:
continue
assets = gh_api(f"repos/{repo}/releases/{release['id']}/assets?per_page=100", paginate=True)
if not isinstance(assets, list):
raise PackError(f"{repo} {tag}: the asset list is not a list")
for asset in assets:
name = asset.get("name") if isinstance(asset, dict) else None
if not isinstance(name, str) or not name.endswith(FAILED):
continue
moment = parse_time(asset.get("updated_at")) or parse_time(asset.get("created_at"))
if moment is not None and now - moment < RETRY_AFTER:
found.add(name)
return frozenset(found)
def failed_builds(matrix: dict, directory: Path) -> list[tuple[str, str]]:
rows = matrix.get("include") if isinstance(matrix, dict) else None
if not isinstance(rows, list) or not all(isinstance(row, dict) and isinstance(row.get("key"), str) for row in rows):
raise PackError("the build matrix has no include list")
made = {row["key"] for row in rows if (directory / f"{ARTIFACT_PREFIX}{row['key']}").is_dir()}
digests = {}
return [(row["flavor"], failure_marker(row, digests)) for row in rows if row["key"] not in made]
def artifact_packs(matrix: dict, directory: Path) -> list[tuple[str, Path, str]]:
rows = matrix.get("include") if isinstance(matrix, dict) else None
if not isinstance(rows, list) or not all(isinstance(row, dict) and isinstance(row.get("key"), str) for row in rows):
raise PackError("the build matrix has no include list")
builds = {row["key"]: row for row in rows}
found = []
for folder in sorted(directory.iterdir()) if directory.is_dir() else []:
row = builds.get(folder.name[len(ARTIFACT_PREFIX):]) if folder.name.startswith(ARTIFACT_PREFIX) else None
if row is None or folder.is_symlink() or not folder.is_dir():
raise PackError(f"{folder.name} is not the artifact of a build in this run")
names = sorted(path.name for path in folder.iterdir())
entries = [name for name in names if name.endswith(".json")]
if len(entries) != 1 or names != sorted([entries[0], entries[0][:-len(".json")] + ".tzst"]):
raise PackError(f"{folder.name} must hold exactly <id>.json and <id>.tzst")
ident = entries[0][:-len(".json")]
entry_path, archive = folder / entries[0], folder / f"{ident}.tzst"
if entry_path.is_symlink() or archive.is_symlink() or not entry_path.is_file() or not archive.is_file():
raise PackError(f"{folder.name} holds links or special files")
try:
data = json.loads(entry_path.read_text())
except ValueError as e:
raise PackError(f"{entry_path}: {e}") from e
entry = check_entry(data, str(entry_path))
source = data.get("source") if isinstance(data.get("source"), dict) else {}
if entry["id"] != ident or entry["flavor"] != row["flavor"] or entry["rev"] != row["rev"] \
or source.get("ref") != row["tag"] or source.get("asset", "") != row["asset"]:
raise PackError(f"{entry_path} is not the pack of {row['flavor']} {row['tag']} at rev {row['rev']}")
if archive.stat().st_size != entry["asset"]["size"] or sha256_file(archive) != entry["asset"]["sha256"]:
raise PackError(f"{archive} is not the asset its entry describes")
found.append((row["flavor"], folder, ident))
return found
def emit(entries: list[dict], unindexed: int = 0) -> None:
matrix = json.dumps({"include": entries}, sort_keys=True, separators=(",", ":"))
path = os.environ.get("GITHUB_OUTPUT")
if path:
with open(path, "a") as stream:
stream.write(f"matrix={matrix}\ncount={len(entries)}\nunindexed={unindexed}\n")
print(matrix)
def make_variables(text: str) -> tuple[dict[str, str], list[tuple[str, str]]]:
variables = {}
assignments = []
for raw in text.splitlines():
match = ASSIGNMENT.fullmatch(raw.split("#", 1)[0].strip())
if match:
name, value = match.group(1), match.group(2).strip()
variables.setdefault(name, value)
assignments.append((name, value))
return variables, assignments
def expand(value: str, variables: dict[str, str], depth: int = 0) -> str:
if depth > 16:
raise PackError("Makefile variables nest too deeply")
def lookup(match: re.Match) -> str:
if match.group(1) not in variables:
raise PackError(f"$({match.group(1)}) is not set in the Makefile")
return expand(variables[match.group(1)], variables, depth + 1)
return REFERENCE.sub(lookup, value)
def sdk_image(text: str, variable: str = "STEAMRT_IMAGE", match: str = "arm64-llvm") -> str:
variables, assignments = make_variables(text)
problem = None
for name, value in assignments:
if not name.startswith(variable):
continue
try:
image = expand(value, variables)
except PackError as e:
problem = problem or e
continue
if match not in image:
continue
if not IMAGE.fullmatch(image):
raise PackError(f"{name} = {image!r} is not an image reference")
return image
raise problem or PackError(f"no {variable} assignment names an {match} image")
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Find the Proton builds that have no droiddeck-esync pack yet, as a GitHub Actions matrix.")
parser.add_argument("--flavors", default=str(FLAVORS_FILE))
commands = parser.add_subparsers(dest="command", required=True)
matrix = commands.add_parser("matrix")
matrix.add_argument("--flavor", default="all", choices=("all", *FLAVORS))
matrix.add_argument("--ref", default="")
matrix.add_argument("--index", default="")
matrix.add_argument("--index-release", default="")
matrix.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "Droid-Deck/DroidDeck"))
matrix.add_argument("--revoked", default=str(REVOKED))
image = commands.add_parser("sdk-image")
image.add_argument("makefile")
artifacts = commands.add_parser("packs")
artifacts.add_argument("--matrix", required=True)
artifacts.add_argument("directory")
failures = commands.add_parser("failures")
failures.add_argument("--matrix", required=True)
failures.add_argument("directory")
commands.add_parser("steam-apps")
watching = commands.add_parser("watch")
watching.add_argument("--state", default="")
watching.add_argument("--steam-info", default="")
watching.add_argument("--out", required=True)
args = parser.parse_args(argv)
try:
flavors = load_flavors(Path(args.flavors))
if args.command == "sdk-image":
settings = flavors["sdk_image"]
print(sdk_image(Path(args.makefile).read_text(), settings["variable"], settings["match"]))
return 0
if args.command == "packs":
for flavor, folder, ident in artifact_packs(json.loads(args.matrix), Path(args.directory)):
print(f"{flavor}\t{folder}\t{ident}")
return 0
if args.command == "failures":
for flavor, marker in failed_builds(json.loads(args.matrix), Path(args.directory)):
print(f"{flavor}\t{marker}")
return 0
if args.command == "steam-apps":
print(" ".join(steam_apps(flavors)))
return 0
if args.command == "watch":
previous = {}
if args.state and Path(args.state).is_file():
try:
previous = json.loads(Path(args.state).read_text())
except ValueError:
print("discover: the previous watch state is not JSON; every flavor counts as changed", file=sys.stderr)
previous = previous.get("flavors", {}) if isinstance(previous, dict) and previous.get("format") == 1 else {}
steam = None
if args.steam_info:
try:
steam = steam_builds(Path(args.steam_info).read_text(errors="replace"), steam_apps(flavors))
except (OSError, PackError) as e:
print(f"discover: Steam app info unusable ({e}); watching the source tags only", file=sys.stderr)
if steam is not None and len(steam) != len(steam_apps(flavors)):
print(f"discover: Steam app info covers {sorted(steam)} of {steam_apps(flavors)}", file=sys.stderr)
state, changed, notes = watch(flavors, previous, steam)
Path(args.out).write_text(json.dumps({"format": 1, "flavors": state}, indent=1, sort_keys=True) + "\n")
for note in notes:
print(f"discover: {note}", file=sys.stderr)
path = os.environ.get("GITHUB_OUTPUT")
if path:
with open(path, "a") as stream:
stream.write(f"changed={' '.join(changed)}\n")
print(" ".join(changed))
return 0
if args.index and args.index_release:
raise PackError("pass --index or --index-release, not both")
failed = frozenset()
unindexed = 0
if args.index:
path = Path(args.index)
index = json.loads(path.read_text()) if path.exists() else {}
elif args.index_release:
if not REPO.fullmatch(args.repo):
raise PackError(f"{args.repo!r} is not owner/name")
index = release_index(args.repo, args.index_release)
listed = index.get("packs") if isinstance(index, dict) and isinstance(index.get("packs"), list) else []
waiting = published(args.repo, flavors, args.flavor, index)
unindexed = len(waiting)
index = {"packs": [*listed, *waiting]}
failed = recent_failures(args.repo, flavors, args.flavor, datetime.now(timezone.utc))
else:
index = {}
revoked = read_revoked(Path(args.revoked)) if args.revoked else set()
emit(discover(flavors, args.flavor, args.ref.strip(), index, revoked, failed), unindexed)
except (PackError, OSError, ValueError) as e:
print(f"discover: {e}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())