#!/usr/bin/env python3 """Build and publish the one-file update catalog consumed by DroidDeck. The catalog is derived from GitHub releases, but the Android app never has to infer channels, parse release notes, dereference tags, or spend unauthenticated API quota. """ from __future__ import annotations import argparse import base64 import datetime as dt import json import os import re import time import urllib.error import urllib.parse import urllib.request API = "https://api.github.com" SHA256 = re.compile(r"^[0-9a-f]{64}$") PR_TAG = re.compile(r"^pr-([1-9][0-9]*)$") MAIN_TAG = re.compile(r"^main-([0-9a-f]{7,40})$") class ApiError(RuntimeError): def __init__(self, status: int, message: str): super().__init__(f"GitHub API {status}: {message}") self.status = status class GitHub: def __init__(self, token: str): self.token = token def request(self, method: str, path: str, body=None): data = None headers = { "Accept": "application/vnd.github+json", "User-Agent": "DroidDeck-update-catalog", "X-GitHub-Api-Version": "2022-11-28", } if self.token: headers["Authorization"] = f"Bearer {self.token}" if body is not None: data = json.dumps(body).encode() headers["Content-Type"] = "application/json" req = urllib.request.Request(API + path, data=data, headers=headers, method=method) try: with urllib.request.urlopen(req, timeout=30) as response: raw = response.read() return json.loads(raw) if raw else None except urllib.error.HTTPError as e: raw = e.read().decode(errors="replace") raise ApiError(e.code, raw[:500]) from e def get(self, path: str): return self.request("GET", path) def put(self, path: str, body): return self.request("PUT", path, body) def q(value: str) -> str: return urllib.parse.quote(value, safe="") def repo_file(gh: GitHub, repo: str, path: str, ref: str) -> str: obj = gh.get(f"/repos/{repo}/contents/{q(path)}?ref={q(ref)}") if obj.get("encoding") != "base64": raise RuntimeError(f"{repo}:{ref}:{path} was not base64 content") return base64.b64decode(obj["content"]).decode() def resolve_tag(gh: GitHub, repo: str, tag: str) -> str: obj = gh.get(f"/repos/{repo}/git/ref/tags/{q(tag)}")["object"] while obj["type"] == "tag": obj = gh.get(f"/repos/{repo}/git/tags/{q(obj['sha'])}")["object"] if obj["type"] != "commit": raise RuntimeError(f"{repo} tag {tag} points to {obj['type']}, not a commit") return obj["sha"] def parse_gradle(text: str) -> tuple[int, str | None]: code = re.search(r"(?m)^\s*versionCode\s+(\d+)\s*$", text) name = re.search(r'(?m)^\s*versionName\s+"([^"]+)"\s*$', text) if not code: raise RuntimeError("app/build.gradle has no numeric versionCode") return int(code.group(1)), name.group(1) if name else None def parse_variants(text: str) -> list[tuple[str, str]]: out = [] for raw in text.splitlines(): line = raw.strip() if not line or line.startswith("#"): continue fields = line.split() if len(fields) < 3: raise RuntimeError(f"bad variants.txt row: {raw!r}") out.append((fields[1], "" if fields[2] == "-" else fields[2])) if not out: raise RuntimeError("variants.txt has no packages") return out def published_millis(release: dict, prefer_updated: bool = False) -> int: value = ( release.get("updated_at") if prefer_updated else None ) or release.get("published_at") or release.get("created_at") if not value: return 0 return int(dt.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000) def paragraphs(markdown: str) -> list[str]: return [p.strip() for p in re.split(r"\n\s*\n", markdown.replace("\r", "")) if p.strip()] def plain(markdown: str) -> str: text = re.sub(r"!\[[^]]*]\([^)]*\)", "", markdown) text = re.sub(r"\[([^]]*)]\([^)]*\)", r"\1", text) text = re.sub(r"[*_`]{1,3}", "", text) return re.sub(r"\s+", " ", text).strip() def prose_lines(markdown: str) -> str: lines = [] for raw in markdown.splitlines(): match = re.match(r"^\s*[-*]\s+", raw) value = ("• " + plain(raw[match.end():])) if match else plain(raw) if value: lines.append(value) return "\n".join(lines) def ci_description(release: dict) -> tuple[str, str]: ps = paragraphs(release.get("body") or "") head = ps[0] if ps else "" title = "" if "): " in head: title = head.split("): ", 1)[1] title = re.sub(r" \(merged[^)]*\)$", "", title) title = plain(title or head or release.get("name") or release["tag_name"]) summary = ps[1] if len(ps) > 1 else "" if summary.startswith("Signed build") or summary.startswith("Source:"): summary = "" return title, prose_lines(summary) def stable_description(release: dict) -> tuple[str, str]: name = release.get("name") or release["tag_name"] summary = "" for p in paragraphs(release.get("body") or ""): if not (p.startswith("#") or p.startswith("![") or p.startswith("<")): summary = prose_lines(p) break return name, summary def source_meta(source: GitHub, repo: str, ref: str): gradle = repo_file(source, repo, "app/build.gradle", ref) code, version = parse_gradle(gradle) variants = parse_variants(repo_file(source, repo, "tools/release/variants.txt", ref)) signer = repo_file(source, repo, "keystore/release-signer.sha256", ref).strip().lower() if not SHA256.fullmatch(signer): raise RuntimeError(f"{repo}:{ref} has an invalid release signer digest") return code, version, variants, signer def asset_for_suffix(assets: list[dict], suffix: str, all_suffixes: list[str]): others = [s for s in all_suffixes if s and s != suffix] for asset in assets: name = asset.get("name", "") if not name.endswith(f"{suffix}.apk"): continue if any(name.endswith(f"{other}.apk") for other in others): continue digest = (asset.get("digest") or "").lower() if not digest.startswith("sha256:") or not SHA256.fullmatch(digest[7:]): continue return { "name": name, "url": asset["browser_download_url"], "size": int(asset.get("size") or 0), "sha256": digest[7:], } return None def release_entry( release: dict, commit: str, pr: int, version: str | None, version_code: int, variants: list[tuple[str, str]], signer: str, title: str, summary: str, prefer_updated: bool = False, ): suffixes = [suffix for _, suffix in variants] apks = {} for package, suffix in variants: asset = asset_for_suffix(release.get("assets") or [], suffix, suffixes) if not asset: continue asset.update( { "packageName": package, "versionCode": version_code, "signerSha256": signer, } ) apks[package] = asset return { "tag": release["tag_name"], "title": title, "summary": summary, "commit": commit, "pr": pr, "version": version, "versionCode": version_code, "publishedAt": published_millis(release, prefer_updated), "url": release.get("html_url") or "", "apks": apks, } def release_body_value(body: str, key: str): match = re.search(rf"(?mi)^\s*{re.escape(key)}:\s*`?([^\s`]+)", body or "") return match.group(1) if match else None def main_commit(release: dict) -> str | None: body = release.get("body") or "" explicit = release_body_value(body, "Commit") if explicit and re.fullmatch(r"[0-9a-f]{7,40}", explicit): return explicit match = re.search(r"/commit/([0-9a-f]{40})", body) return match.group(1) if match else None def build_catalog(source: GitHub, ci: GitHub, source_repo: str, ci_repo: str) -> dict: try: stable_release = source.get(f"/repos/{source_repo}/releases/latest") except ApiError as e: if e.status != 404: raise stable_release = None stable = None if stable_release: stable_commit = resolve_tag(source, source_repo, stable_release["tag_name"]) code, version_name, variants, signer = source_meta(source, source_repo, stable_commit) title, summary = stable_description(stable_release) stable = release_entry( stable_release, stable_commit, 0, version_name or stable_release["tag_name"].removeprefix("v"), code, variants, signer, title, summary, ) releases = ci.get(f"/repos/{ci_repo}/releases?per_page=100") previews = [] tests = [] for release in releases: if release.get("draft"): continue tag = release.get("tag_name") or "" main_match = MAIN_TAG.fullmatch(tag) if main_match: commit = main_commit(release) if not commit: continue try: code, _, variants, signer = source_meta(source, source_repo, commit) except ApiError: continue title, summary = ci_description(release) previews.append( release_entry(release, commit, 0, None, code, variants, signer, title, summary) ) continue pr_match = PR_TAG.fullmatch(tag) if not pr_match: continue pr = int(pr_match.group(1)) body = release.get("body") or "" if f"github.com/{source_repo}/pull/{pr}" not in body: continue try: pull = source.get(f"/repos/{source_repo}/pulls/{pr}") except ApiError: continue if pull.get("state") != "open" or pull.get("draft") or pull.get("base", {}).get("ref") != "main": continue commit = release_body_value(body, "Commit") if not commit or not re.fullmatch(r"[0-9a-f]{7,40}", commit): continue base_ref = release_body_value(body, "Base") or pull["base"]["sha"] try: base_code, _, variants, signer = source_meta(source, source_repo, base_ref) except ApiError: continue version_code_text = release_body_value(body, "VersionCode") if version_code_text and version_code_text.isdigit(): version_code = int(version_code_text) else: # Legacy test releases did not record it. If the PR did not change build.gradle, # the base value is exact; otherwise use the PR head's value. version_code = base_code try: changed = source.get(f"/repos/{source_repo}/pulls/{pr}/files?per_page=100") if any(item.get("filename") == "app/build.gradle" for item in changed): head_repo = pull["head"]["repo"]["full_name"] version_code = parse_gradle(repo_file(source, head_repo, "app/build.gradle", pull["head"]["sha"]))[0] except (ApiError, KeyError): pass title, summary = ci_description(release) tests.append( release_entry( release, commit, pr, None, version_code, variants, signer, title, summary, prefer_updated=True, ) ) previews.sort(key=lambda r: r["publishedAt"], reverse=True) tests.sort(key=lambda r: r["publishedAt"], reverse=True) return { "schema": 1, "sourceRepo": source_repo, "ciRepo": ci_repo, "generatedAt": int(time.time() * 1000), "stable": stable, "preview": previews[0] if previews else None, "tests": tests, } def current_file(ci: GitHub, repo: str, path: str): try: return ci.get(f"/repos/{repo}/contents/{q(path)}?ref=main") except ApiError as e: if e.status == 404: return None raise def publish_catalog(source: GitHub, ci: GitHub, source_repo: str, ci_repo: str, path: str): for attempt in range(4): catalog = build_catalog(source, ci, source_repo, ci_repo) content = (json.dumps(catalog, indent=2, sort_keys=False) + "\n").encode() old = current_file(ci, ci_repo, path) body = { "message": "ci: refresh DroidDeck update catalog", "content": base64.b64encode(content).decode(), "branch": "main", } if old: body["sha"] = old["sha"] try: old_bytes = base64.b64decode(old["content"]) old_json = json.loads(old_bytes) old_json.pop("generatedAt", None) new_json = dict(catalog) new_json.pop("generatedAt", None) if old_json == new_json: print("catalog contents are already current") return catalog except Exception: pass try: ci.put(f"/repos/{ci_repo}/contents/{q(path)}", body) print(f"published https://raw.githubusercontent.com/{ci_repo}/main/{path}") return catalog except ApiError as e: if e.status not in (409, 422) or attempt == 3: raise time.sleep(1 + attempt) raise RuntimeError("catalog publish retry loop exhausted") def main(): parser = argparse.ArgumentParser() parser.add_argument("--source-repo", default="Droid-Deck/DroidDeck") parser.add_argument("--ci-repo", default="Droid-Deck/DroidDeck-CI") parser.add_argument("--path", default="catalog.json") parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() source = GitHub(os.environ.get("SOURCE_GITHUB_TOKEN", "")) ci = GitHub(os.environ.get("CI_GITHUB_TOKEN", os.environ.get("SOURCE_GITHUB_TOKEN", ""))) if args.dry_run: print(json.dumps(build_catalog(source, ci, args.source_repo, args.ci_repo), indent=2)) else: if not os.environ.get("CI_GITHUB_TOKEN"): raise SystemExit("CI_GITHUB_TOKEN is required to publish") publish_catalog(source, ci, args.source_repo, args.ci_repo, args.path) if __name__ == "__main__": main()