DroidDeck/tools/release/update_catalog.py

#!/usr/bin/env python3

from __future__ import annotations

import argparse
import base64
import datetime as dt
import json
import os
import re
import time
import urllib.error
import urllib.parse
import urllib.request

API = "https://api.github.com"
SHA256 = re.compile(r"^[0-9a-f]{64}$")
PR_TAG = re.compile(r"^pr-([1-9][0-9]*)$")
MAIN_TAG = re.compile(r"^main-([0-9a-f]{7,40})$")


class ApiError(RuntimeError):
    def __init__(self, status: int, message: str):
        super().__init__(f"GitHub API {status}: {message}")
        self.status = status


class GitHub:
    def __init__(self, token: str):
        self.token = token

    def request(self, method: str, path: str, body=None):
        data = None
        headers = {
            "Accept": "application/vnd.github+json",
            "User-Agent": "DroidDeck-update-catalog",
            "X-GitHub-Api-Version": "2022-11-28",
        }
        if self.token:
            headers["Authorization"] = f"Bearer {self.token}"
        if body is not None:
            data = json.dumps(body).encode()
            headers["Content-Type"] = "application/json"
        req = urllib.request.Request(API + path, data=data, headers=headers, method=method)
        try:
            with urllib.request.urlopen(req, timeout=30) as response:
                raw = response.read()
                return json.loads(raw) if raw else None
        except urllib.error.HTTPError as e:
            raw = e.read().decode(errors="replace")
            raise ApiError(e.code, raw[:500]) from e

    def get(self, path: str):
        return self.request("GET", path)

    def put(self, path: str, body):
        return self.request("PUT", path, body)


def q(value: str) -> str:
    return urllib.parse.quote(value, safe="")


def repo_file(gh: GitHub, repo: str, path: str, ref: str) -> str:
    obj = gh.get(f"/repos/{repo}/contents/{q(path)}?ref={q(ref)}")
    if obj.get("encoding") != "base64":
        raise RuntimeError(f"{repo}:{ref}:{path} was not base64 content")
    return base64.b64decode(obj["content"]).decode()


def resolve_tag(gh: GitHub, repo: str, tag: str) -> str:
    obj = gh.get(f"/repos/{repo}/git/ref/tags/{q(tag)}")["object"]
    while obj["type"] == "tag":
        obj = gh.get(f"/repos/{repo}/git/tags/{q(obj['sha'])}")["object"]
    if obj["type"] != "commit":
        raise RuntimeError(f"{repo} tag {tag} points to {obj['type']}, not a commit")
    return obj["sha"]


def parse_gradle(text: str) -> tuple[int, str | None]:
    code = re.search(r"(?m)^\s*versionCode\s+(\d+)\s*$", text)
    name = re.search(r'(?m)^\s*versionName\s+"([^"]+)"\s*$', text)
    if not code:
        raise RuntimeError("app/build.gradle has no numeric versionCode")
    return int(code.group(1)), name.group(1) if name else None


def parse_variants(text: str) -> list[tuple[str, str]]:
    out = []
    for raw in text.splitlines():
        line = raw.strip()
        if not line or line.startswith("#"):
            continue
        fields = line.split()
        if len(fields) < 3:
            raise RuntimeError(f"bad variants.txt row: {raw!r}")
        out.append((fields[1], "" if fields[2] == "-" else fields[2]))
    if not out:
        raise RuntimeError("variants.txt has no packages")
    return out


def published_millis(release: dict, prefer_updated: bool = False) -> int:
    value = (
        release.get("updated_at") if prefer_updated else None
    ) or release.get("published_at") or release.get("created_at")
    if not value:
        return 0
    return int(dt.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000)


def paragraphs(markdown: str) -> list[str]:
    return [p.strip() for p in re.split(r"\n\s*\n", markdown.replace("\r", "")) if p.strip()]


def plain(markdown: str) -> str:
    text = re.sub(r"!\[[^]]*]\([^)]*\)", "", markdown)
    text = re.sub(r"\[([^]]*)]\([^)]*\)", r"\1", text)
    text = re.sub(r"[*_`]{1,3}", "", text)
    return re.sub(r"\s+", " ", text).strip()


def prose_lines(markdown: str) -> str:
    lines = []
    for raw in markdown.splitlines():
        match = re.match(r"^\s*[-*]\s+", raw)
        value = ("• " + plain(raw[match.end():])) if match else plain(raw)
        if value:
            lines.append(value)
    return "\n".join(lines)


def ci_description(release: dict) -> tuple[str, str]:
    ps = paragraphs(release.get("body") or "")
    head = ps[0] if ps else ""
    title = ""
    if "): " in head:
        title = head.split("): ", 1)[1]
        title = re.sub(r" \(merged[^)]*\)$", "", title)
    title = plain(title or head or release.get("name") or release["tag_name"])
    summary = ps[1] if len(ps) > 1 else ""
    if summary.startswith("Signed build") or summary.startswith("Source:"):
        summary = ""
    return title, prose_lines(summary)


def stable_description(release: dict) -> tuple[str, str]:
    name = release.get("name") or release["tag_name"]
    summary = ""
    for p in paragraphs(release.get("body") or ""):
        if not (p.startswith("#") or p.startswith("![") or p.startswith("<")):
            summary = prose_lines(p)
            break
    return name, summary


def source_meta(source: GitHub, repo: str, ref: str):
    gradle = repo_file(source, repo, "app/build.gradle", ref)
    code, version = parse_gradle(gradle)
    variants = parse_variants(repo_file(source, repo, "tools/release/variants.txt", ref))
    signer = repo_file(source, repo, "keystore/release-signer.sha256", ref).strip().lower()
    if not SHA256.fullmatch(signer):
        raise RuntimeError(f"{repo}:{ref} has an invalid release signer digest")
    return code, version, variants, signer


def asset_for_suffix(assets: list[dict], suffix: str, all_suffixes: list[str]):
    others = [s for s in all_suffixes if s and s != suffix]
    for asset in assets:
        name = asset.get("name", "")
        if not name.endswith(f"{suffix}.apk"):
            continue
        if any(name.endswith(f"{other}.apk") for other in others):
            continue
        digest = (asset.get("digest") or "").lower()
        if not digest.startswith("sha256:") or not SHA256.fullmatch(digest[7:]):
            continue
        return {
            "name": name,
            "url": asset["browser_download_url"],
            "size": int(asset.get("size") or 0),
            "sha256": digest[7:],
        }
    return None


def release_entry(
    release: dict,
    commit: str,
    pr: int,
    version: str | None,
    version_code: int,
    variants: list[tuple[str, str]],
    signer: str,
    title: str,
    summary: str,
    prefer_updated: bool = False,
):
    suffixes = [suffix for _, suffix in variants]
    apks = {}
    for package, suffix in variants:
        asset = asset_for_suffix(release.get("assets") or [], suffix, suffixes)
        if not asset:
            continue
        asset.update(
            {
                "packageName": package,
                "versionCode": version_code,
                "signerSha256": signer,
            }
        )
        apks[package] = asset
    return {
        "tag": release["tag_name"],
        "title": title,
        "summary": summary,
        "commit": commit,
        "pr": pr,
        "version": version,
        "versionCode": version_code,
        "publishedAt": published_millis(release, prefer_updated),
        "url": release.get("html_url") or "",
        "apks": apks,
    }


def preview_history_entry(release: dict, preview: dict, variants: list[tuple[str, str]]):
    tag = release.get("tag_name") or ""
    tag_match = MAIN_TAG.fullmatch(tag)
    commit = (preview.get("commit") or "").lower()
    expected_packages = {package for package, _ in variants}
    apks = preview.get("apks") or {}
    if (
        release.get("draft")
        or not release.get("prerelease")
        or not tag_match
        or not re.fullmatch(r"[0-9a-f]{40}", commit)
        or not commit.startswith(tag_match.group(1))
        or set(apks) != expected_packages
        or any(
            not SHA256.fullmatch((apk.get("sha256") or "").lower())
            or not SHA256.fullmatch((apk.get("signerSha256") or "").lower())
            for apk in apks.values()
        )
    ):
        return None
    return {
        "commit": commit,
        "title": preview["title"],
        "summary": preview["summary"][:400],
        "publishedAt": preview["publishedAt"],
    }


def recent_previews(entries: list[dict], limit: int = 10) -> list[dict]:
    result = []
    seen = set()
    for entry in sorted(entries, key=lambda item: item["publishedAt"], reverse=True):
        commit = entry["commit"].lower()
        if commit in seen:
            continue
        seen.add(commit)
        result.append({**entry, "commit": commit})
        if len(result) >= limit:
            break
    return result


def release_body_value(body: str, key: str):
    match = re.search(rf"(?mi)^\s*{re.escape(key)}:\s*`?([^\s`]+)", body or "")
    return match.group(1) if match else None


def main_commit(release: dict) -> str | None:
    body = release.get("body") or ""
    explicit = release_body_value(body, "Commit")
    if explicit and re.fullmatch(r"[0-9a-f]{7,40}", explicit):
        return explicit
    match = re.search(r"/commit/([0-9a-f]{40})", body)
    return match.group(1) if match else None


def stable_app_release(releases: list[dict]) -> dict | None:
    apps = [
        r for r in releases
        if not r.get("draft") and not r.get("prerelease")
        and any(a.get("name") == f"DroidDeck-{r.get('tag_name')}.apk" for a in r.get("assets") or [])
    ]
    return max(apps, key=lambda r: r.get("published_at") or r.get("created_at") or "", default=None)


def build_catalog(source: GitHub, ci: GitHub, source_repo: str, ci_repo: str) -> dict:
    stable_release = stable_app_release(source.get(f"/repos/{source_repo}/releases?per_page=100"))

    stable = None
    if stable_release:
        stable_commit = resolve_tag(source, source_repo, stable_release["tag_name"])
        code, version_name, variants, signer = source_meta(source, source_repo, stable_commit)
        title, summary = stable_description(stable_release)
        stable = release_entry(
            stable_release,
            stable_commit,
            0,
            version_name or stable_release["tag_name"].removeprefix("v"),
            code,
            variants,
            signer,
            title,
            summary,
        )

    releases = ci.get(f"/repos/{ci_repo}/releases?per_page=100")
    previews = []
    preview_history = []
    tests = []
    for release in releases:
        if release.get("draft"):
            continue
        tag = release.get("tag_name") or ""
        main_match = MAIN_TAG.fullmatch(tag)
        if main_match:
            commit = main_commit(release)
            if not commit:
                continue
            try:
                code, _, variants, signer = source_meta(source, source_repo, commit)
            except ApiError:
                continue
            title, summary = ci_description(release)
            preview = release_entry(release, commit, 0, None, code, variants, signer, title, summary)
            previews.append(preview)
            history_entry = preview_history_entry(release, preview, variants)
            if history_entry:
                preview_history.append(history_entry)
            continue

        pr_match = PR_TAG.fullmatch(tag)
        if not pr_match:
            continue
        pr = int(pr_match.group(1))
        body = release.get("body") or ""
        if f"github.com/{source_repo}/pull/{pr}" not in body:
            continue
        try:
            pull = source.get(f"/repos/{source_repo}/pulls/{pr}")
        except ApiError:
            continue
        if pull.get("state") != "open" or pull.get("draft") or pull.get("base", {}).get("ref") != "main":
            continue
        commit = release_body_value(body, "Commit")
        if not commit or not re.fullmatch(r"[0-9a-f]{7,40}", commit):
            continue
        base_ref = release_body_value(body, "Base") or pull["base"]["sha"]
        try:
            base_code, _, variants, signer = source_meta(source, source_repo, base_ref)
        except ApiError:
            continue
        version_code_text = release_body_value(body, "VersionCode")
        if version_code_text and version_code_text.isdigit():
            version_code = int(version_code_text)
        else:
            version_code = base_code
            try:
                changed = source.get(f"/repos/{source_repo}/pulls/{pr}/files?per_page=100")
                if any(item.get("filename") == "app/build.gradle" for item in changed):
                    head_repo = pull["head"]["repo"]["full_name"]
                    version_code = parse_gradle(repo_file(source, head_repo, "app/build.gradle", pull["head"]["sha"]))[0]
            except (ApiError, KeyError):
                pass
        title, summary = ci_description(release)
        tests.append(
            release_entry(
                release,
                commit,
                pr,
                None,
                version_code,
                variants,
                signer,
                title,
                summary,
                prefer_updated=True,
            )
        )

    previews.sort(key=lambda r: r["publishedAt"], reverse=True)
    tests.sort(key=lambda r: r["publishedAt"], reverse=True)
    return {
        "schema": 1,
        "sourceRepo": source_repo,
        "ciRepo": ci_repo,
        "generatedAt": int(time.time() * 1000),
        "stable": stable,
        "preview": previews[0] if previews else None,
        "recentPreviews": recent_previews(preview_history),
        "tests": tests,
    }


def current_file(ci: GitHub, repo: str, path: str):
    try:
        return ci.get(f"/repos/{repo}/contents/{q(path)}?ref=main")
    except ApiError as e:
        if e.status == 404:
            return None
        raise


def publish_catalog(source: GitHub, ci: GitHub, source_repo: str, ci_repo: str, path: str):
    for attempt in range(4):
        catalog = build_catalog(source, ci, source_repo, ci_repo)
        content = (json.dumps(catalog, indent=2, sort_keys=False) + "\n").encode()
        old = current_file(ci, ci_repo, path)
        body = {
            "message": "ci: refresh DroidDeck update catalog",
            "content": base64.b64encode(content).decode(),
            "branch": "main",
        }
        if old:
            body["sha"] = old["sha"]
            try:
                old_bytes = base64.b64decode(old["content"])
                old_json = json.loads(old_bytes)
                old_json.pop("generatedAt", None)
                new_json = dict(catalog)
                new_json.pop("generatedAt", None)
                if old_json == new_json:
                    print("catalog contents are already current")
                    return catalog
            except Exception:
                pass
        try:
            ci.put(f"/repos/{ci_repo}/contents/{q(path)}", body)
            print(f"published https://raw.githubusercontent.com/{ci_repo}/main/{path}")
            return catalog
        except ApiError as e:
            if e.status not in (409, 422) or attempt == 3:
                raise
            time.sleep(1 + attempt)
    raise RuntimeError("catalog publish retry loop exhausted")


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--source-repo", default="Droid-Deck/DroidDeck")
    parser.add_argument("--ci-repo", default="Droid-Deck/DroidDeck-CI")
    parser.add_argument("--path", default="catalog.json")
    parser.add_argument("--dry-run", action="store_true")
    args = parser.parse_args()

    source = GitHub(os.environ.get("SOURCE_GITHUB_TOKEN", ""))
    ci = GitHub(os.environ.get("CI_GITHUB_TOKEN", os.environ.get("SOURCE_GITHUB_TOKEN", "")))
    if args.dry_run:
        print(json.dumps(build_catalog(source, ci, args.source_repo, args.ci_repo), indent=2))
    else:
        if not os.environ.get("CI_GITHUB_TOKEN"):
            raise SystemExit("CI_GITHUB_TOKEN is required to publish")
        publish_catalog(source, ci, args.source_repo, args.ci_repo, args.path)


if __name__ == "__main__":
    main()