mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-19 12:56:06 +00:00
Merge branch 'main' into feat/performance-fixes-2
This commit is contained in:
commit
bf34b0c2bd
16 files changed
+1185
-224
No files matched your search
Executable
+419
@@ -0,0 +1,419 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and publish the one-file update catalog consumed by DroidDeck.
|
||||
|
||||
The catalog is derived from GitHub releases, but the Android app never has to infer
|
||||
channels, parse release notes, dereference tags, or spend unauthenticated API quota.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
API = "https://api.github.com"
|
||||
SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
PR_TAG = re.compile(r"^pr-([1-9][0-9]*)$")
|
||||
MAIN_TAG = re.compile(r"^main-([0-9a-f]{7,40})$")
|
||||
|
||||
|
||||
class ApiError(RuntimeError):
|
||||
def __init__(self, status: int, message: str):
|
||||
super().__init__(f"GitHub API {status}: {message}")
|
||||
self.status = status
|
||||
|
||||
|
||||
class GitHub:
|
||||
def __init__(self, token: str):
|
||||
self.token = token
|
||||
|
||||
def request(self, method: str, path: str, body=None):
|
||||
data = None
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"User-Agent": "DroidDeck-update-catalog",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"Bearer {self.token}"
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(API + path, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
raw = response.read()
|
||||
return json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read().decode(errors="replace")
|
||||
raise ApiError(e.code, raw[:500]) from e
|
||||
|
||||
def get(self, path: str):
|
||||
return self.request("GET", path)
|
||||
|
||||
def put(self, path: str, body):
|
||||
return self.request("PUT", path, body)
|
||||
|
||||
|
||||
def q(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="")
|
||||
|
||||
|
||||
def repo_file(gh: GitHub, repo: str, path: str, ref: str) -> str:
|
||||
obj = gh.get(f"/repos/{repo}/contents/{q(path)}?ref={q(ref)}")
|
||||
if obj.get("encoding") != "base64":
|
||||
raise RuntimeError(f"{repo}:{ref}:{path} was not base64 content")
|
||||
return base64.b64decode(obj["content"]).decode()
|
||||
|
||||
|
||||
def resolve_tag(gh: GitHub, repo: str, tag: str) -> str:
|
||||
obj = gh.get(f"/repos/{repo}/git/ref/tags/{q(tag)}")["object"]
|
||||
while obj["type"] == "tag":
|
||||
obj = gh.get(f"/repos/{repo}/git/tags/{q(obj['sha'])}")["object"]
|
||||
if obj["type"] != "commit":
|
||||
raise RuntimeError(f"{repo} tag {tag} points to {obj['type']}, not a commit")
|
||||
return obj["sha"]
|
||||
|
||||
|
||||
def parse_gradle(text: str) -> tuple[int, str | None]:
|
||||
code = re.search(r"(?m)^\s*versionCode\s+(\d+)\s*$", text)
|
||||
name = re.search(r'(?m)^\s*versionName\s+"([^"]+)"\s*$', text)
|
||||
if not code:
|
||||
raise RuntimeError("app/build.gradle has no numeric versionCode")
|
||||
return int(code.group(1)), name.group(1) if name else None
|
||||
|
||||
|
||||
def parse_variants(text: str) -> list[tuple[str, str]]:
|
||||
out = []
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
fields = line.split()
|
||||
if len(fields) < 3:
|
||||
raise RuntimeError(f"bad variants.txt row: {raw!r}")
|
||||
out.append((fields[1], "" if fields[2] == "-" else fields[2]))
|
||||
if not out:
|
||||
raise RuntimeError("variants.txt has no packages")
|
||||
return out
|
||||
|
||||
|
||||
def published_millis(release: dict, prefer_updated: bool = False) -> int:
|
||||
value = (
|
||||
release.get("updated_at") if prefer_updated else None
|
||||
) or release.get("published_at") or release.get("created_at")
|
||||
if not value:
|
||||
return 0
|
||||
return int(dt.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000)
|
||||
|
||||
|
||||
def paragraphs(markdown: str) -> list[str]:
|
||||
return [p.strip() for p in re.split(r"\n\s*\n", markdown.replace("\r", "")) if p.strip()]
|
||||
|
||||
|
||||
def plain(markdown: str) -> str:
|
||||
text = re.sub(r"!\[[^]]*]\([^)]*\)", "", markdown)
|
||||
text = re.sub(r"\[([^]]*)]\([^)]*\)", r"\1", text)
|
||||
text = re.sub(r"[*_`]{1,3}", "", text)
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def prose_lines(markdown: str) -> str:
|
||||
lines = []
|
||||
for raw in markdown.splitlines():
|
||||
match = re.match(r"^\s*[-*]\s+", raw)
|
||||
value = ("• " + plain(raw[match.end():])) if match else plain(raw)
|
||||
if value:
|
||||
lines.append(value)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def ci_description(release: dict) -> tuple[str, str]:
|
||||
ps = paragraphs(release.get("body") or "")
|
||||
head = ps[0] if ps else ""
|
||||
title = ""
|
||||
if "): " in head:
|
||||
title = head.split("): ", 1)[1]
|
||||
title = re.sub(r" \(merged[^)]*\)$", "", title)
|
||||
title = plain(title or head or release.get("name") or release["tag_name"])
|
||||
summary = ps[1] if len(ps) > 1 else ""
|
||||
if summary.startswith("Signed build") or summary.startswith("Source:"):
|
||||
summary = ""
|
||||
return title, prose_lines(summary)
|
||||
|
||||
|
||||
def stable_description(release: dict) -> tuple[str, str]:
|
||||
name = release.get("name") or release["tag_name"]
|
||||
summary = ""
|
||||
for p in paragraphs(release.get("body") or ""):
|
||||
if not (p.startswith("#") or p.startswith("![") or p.startswith("<")):
|
||||
summary = prose_lines(p)
|
||||
break
|
||||
return name, summary
|
||||
|
||||
|
||||
def source_meta(source: GitHub, repo: str, ref: str):
|
||||
gradle = repo_file(source, repo, "app/build.gradle", ref)
|
||||
code, version = parse_gradle(gradle)
|
||||
variants = parse_variants(repo_file(source, repo, "tools/release/variants.txt", ref))
|
||||
signer = repo_file(source, repo, "keystore/release-signer.sha256", ref).strip().lower()
|
||||
if not SHA256.fullmatch(signer):
|
||||
raise RuntimeError(f"{repo}:{ref} has an invalid release signer digest")
|
||||
return code, version, variants, signer
|
||||
|
||||
|
||||
def asset_for_suffix(assets: list[dict], suffix: str, all_suffixes: list[str]):
|
||||
others = [s for s in all_suffixes if s and s != suffix]
|
||||
for asset in assets:
|
||||
name = asset.get("name", "")
|
||||
if not name.endswith(f"{suffix}.apk"):
|
||||
continue
|
||||
if any(name.endswith(f"{other}.apk") for other in others):
|
||||
continue
|
||||
digest = (asset.get("digest") or "").lower()
|
||||
if not digest.startswith("sha256:") or not SHA256.fullmatch(digest[7:]):
|
||||
continue
|
||||
return {
|
||||
"name": name,
|
||||
"url": asset["browser_download_url"],
|
||||
"size": int(asset.get("size") or 0),
|
||||
"sha256": digest[7:],
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def release_entry(
|
||||
release: dict,
|
||||
commit: str,
|
||||
pr: int,
|
||||
version: str | None,
|
||||
version_code: int,
|
||||
variants: list[tuple[str, str]],
|
||||
signer: str,
|
||||
title: str,
|
||||
summary: str,
|
||||
prefer_updated: bool = False,
|
||||
):
|
||||
suffixes = [suffix for _, suffix in variants]
|
||||
apks = {}
|
||||
for package, suffix in variants:
|
||||
asset = asset_for_suffix(release.get("assets") or [], suffix, suffixes)
|
||||
if not asset:
|
||||
continue
|
||||
asset.update(
|
||||
{
|
||||
"packageName": package,
|
||||
"versionCode": version_code,
|
||||
"signerSha256": signer,
|
||||
}
|
||||
)
|
||||
apks[package] = asset
|
||||
return {
|
||||
"tag": release["tag_name"],
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"commit": commit,
|
||||
"pr": pr,
|
||||
"version": version,
|
||||
"versionCode": version_code,
|
||||
"publishedAt": published_millis(release, prefer_updated),
|
||||
"url": release.get("html_url") or "",
|
||||
"apks": apks,
|
||||
}
|
||||
|
||||
|
||||
def release_body_value(body: str, key: str):
|
||||
match = re.search(rf"(?mi)^\s*{re.escape(key)}:\s*`?([^\s`]+)", body or "")
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def main_commit(release: dict) -> str | None:
|
||||
body = release.get("body") or ""
|
||||
explicit = release_body_value(body, "Commit")
|
||||
if explicit and re.fullmatch(r"[0-9a-f]{7,40}", explicit):
|
||||
return explicit
|
||||
match = re.search(r"/commit/([0-9a-f]{40})", body)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def build_catalog(source: GitHub, ci: GitHub, source_repo: str, ci_repo: str) -> dict:
|
||||
try:
|
||||
stable_release = source.get(f"/repos/{source_repo}/releases/latest")
|
||||
except ApiError as e:
|
||||
if e.status != 404:
|
||||
raise
|
||||
stable_release = None
|
||||
|
||||
stable = None
|
||||
if stable_release:
|
||||
stable_commit = resolve_tag(source, source_repo, stable_release["tag_name"])
|
||||
code, version_name, variants, signer = source_meta(source, source_repo, stable_commit)
|
||||
title, summary = stable_description(stable_release)
|
||||
stable = release_entry(
|
||||
stable_release,
|
||||
stable_commit,
|
||||
0,
|
||||
version_name or stable_release["tag_name"].removeprefix("v"),
|
||||
code,
|
||||
variants,
|
||||
signer,
|
||||
title,
|
||||
summary,
|
||||
)
|
||||
|
||||
releases = ci.get(f"/repos/{ci_repo}/releases?per_page=100")
|
||||
previews = []
|
||||
tests = []
|
||||
for release in releases:
|
||||
if release.get("draft"):
|
||||
continue
|
||||
tag = release.get("tag_name") or ""
|
||||
main_match = MAIN_TAG.fullmatch(tag)
|
||||
if main_match:
|
||||
commit = main_commit(release)
|
||||
if not commit:
|
||||
continue
|
||||
try:
|
||||
code, _, variants, signer = source_meta(source, source_repo, commit)
|
||||
except ApiError:
|
||||
continue
|
||||
title, summary = ci_description(release)
|
||||
previews.append(
|
||||
release_entry(release, commit, 0, None, code, variants, signer, title, summary)
|
||||
)
|
||||
continue
|
||||
|
||||
pr_match = PR_TAG.fullmatch(tag)
|
||||
if not pr_match:
|
||||
continue
|
||||
pr = int(pr_match.group(1))
|
||||
body = release.get("body") or ""
|
||||
if f"github.com/{source_repo}/pull/{pr}" not in body:
|
||||
continue
|
||||
try:
|
||||
pull = source.get(f"/repos/{source_repo}/pulls/{pr}")
|
||||
except ApiError:
|
||||
continue
|
||||
if pull.get("state") != "open" or pull.get("draft") or pull.get("base", {}).get("ref") != "main":
|
||||
continue
|
||||
commit = release_body_value(body, "Commit")
|
||||
if not commit or not re.fullmatch(r"[0-9a-f]{7,40}", commit):
|
||||
continue
|
||||
base_ref = release_body_value(body, "Base") or pull["base"]["sha"]
|
||||
try:
|
||||
base_code, _, variants, signer = source_meta(source, source_repo, base_ref)
|
||||
except ApiError:
|
||||
continue
|
||||
version_code_text = release_body_value(body, "VersionCode")
|
||||
if version_code_text and version_code_text.isdigit():
|
||||
version_code = int(version_code_text)
|
||||
else:
|
||||
# Legacy test releases did not record it. If the PR did not change build.gradle,
|
||||
# the base value is exact; otherwise use the PR head's value.
|
||||
version_code = base_code
|
||||
try:
|
||||
changed = source.get(f"/repos/{source_repo}/pulls/{pr}/files?per_page=100")
|
||||
if any(item.get("filename") == "app/build.gradle" for item in changed):
|
||||
head_repo = pull["head"]["repo"]["full_name"]
|
||||
version_code = parse_gradle(repo_file(source, head_repo, "app/build.gradle", pull["head"]["sha"]))[0]
|
||||
except (ApiError, KeyError):
|
||||
pass
|
||||
title, summary = ci_description(release)
|
||||
tests.append(
|
||||
release_entry(
|
||||
release,
|
||||
commit,
|
||||
pr,
|
||||
None,
|
||||
version_code,
|
||||
variants,
|
||||
signer,
|
||||
title,
|
||||
summary,
|
||||
prefer_updated=True,
|
||||
)
|
||||
)
|
||||
|
||||
previews.sort(key=lambda r: r["publishedAt"], reverse=True)
|
||||
tests.sort(key=lambda r: r["publishedAt"], reverse=True)
|
||||
return {
|
||||
"schema": 1,
|
||||
"sourceRepo": source_repo,
|
||||
"ciRepo": ci_repo,
|
||||
"generatedAt": int(time.time() * 1000),
|
||||
"stable": stable,
|
||||
"preview": previews[0] if previews else None,
|
||||
"tests": tests,
|
||||
}
|
||||
|
||||
|
||||
def current_file(ci: GitHub, repo: str, path: str):
|
||||
try:
|
||||
return ci.get(f"/repos/{repo}/contents/{q(path)}?ref=main")
|
||||
except ApiError as e:
|
||||
if e.status == 404:
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
def publish_catalog(source: GitHub, ci: GitHub, source_repo: str, ci_repo: str, path: str):
|
||||
for attempt in range(4):
|
||||
catalog = build_catalog(source, ci, source_repo, ci_repo)
|
||||
content = (json.dumps(catalog, indent=2, sort_keys=False) + "\n").encode()
|
||||
old = current_file(ci, ci_repo, path)
|
||||
body = {
|
||||
"message": "ci: refresh DroidDeck update catalog",
|
||||
"content": base64.b64encode(content).decode(),
|
||||
"branch": "main",
|
||||
}
|
||||
if old:
|
||||
body["sha"] = old["sha"]
|
||||
try:
|
||||
old_bytes = base64.b64decode(old["content"])
|
||||
old_json = json.loads(old_bytes)
|
||||
old_json.pop("generatedAt", None)
|
||||
new_json = dict(catalog)
|
||||
new_json.pop("generatedAt", None)
|
||||
if old_json == new_json:
|
||||
print("catalog contents are already current")
|
||||
return catalog
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
ci.put(f"/repos/{ci_repo}/contents/{q(path)}", body)
|
||||
print(f"published https://raw.githubusercontent.com/{ci_repo}/main/{path}")
|
||||
return catalog
|
||||
except ApiError as e:
|
||||
if e.status not in (409, 422) or attempt == 3:
|
||||
raise
|
||||
time.sleep(1 + attempt)
|
||||
raise RuntimeError("catalog publish retry loop exhausted")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-repo", default="Droid-Deck/DroidDeck")
|
||||
parser.add_argument("--ci-repo", default="Droid-Deck/DroidDeck-CI")
|
||||
parser.add_argument("--path", default="catalog.json")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
source = GitHub(os.environ.get("SOURCE_GITHUB_TOKEN", ""))
|
||||
ci = GitHub(os.environ.get("CI_GITHUB_TOKEN", os.environ.get("SOURCE_GITHUB_TOKEN", "")))
|
||||
if args.dry_run:
|
||||
print(json.dumps(build_catalog(source, ci, args.source_repo, args.ci_repo), indent=2))
|
||||
else:
|
||||
if not os.environ.get("CI_GITHUB_TOKEN"):
|
||||
raise SystemExit("CI_GITHUB_TOKEN is required to publish")
|
||||
publish_catalog(source, ci, args.source_repo, args.ci_repo, args.path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
import unittest
|
||||
|
||||
from tools.release.update_catalog import (
|
||||
asset_for_suffix,
|
||||
ci_description,
|
||||
parse_gradle,
|
||||
parse_variants,
|
||||
published_millis,
|
||||
release_body_value,
|
||||
)
|
||||
|
||||
|
||||
class UpdateCatalogTest(unittest.TestCase):
|
||||
def test_gradle_metadata(self):
|
||||
code, name = parse_gradle("""android {
|
||||
defaultConfig {
|
||||
versionCode 12
|
||||
versionName "0.3.1"
|
||||
}
|
||||
}
|
||||
""")
|
||||
self.assertEqual(12, code)
|
||||
self.assertEqual("0.3.1", name)
|
||||
|
||||
def test_variants(self):
|
||||
rows = parse_variants("""
|
||||
# name package suffix
|
||||
standard com.droiddeck.launcher -
|
||||
pubg com.tencent.ig -pubg
|
||||
""")
|
||||
self.assertEqual(
|
||||
[("com.droiddeck.launcher", ""), ("com.tencent.ig", "-pubg")],
|
||||
rows,
|
||||
)
|
||||
|
||||
def test_standard_asset_does_not_take_a_variant(self):
|
||||
assets = [
|
||||
{
|
||||
"name": "DroidDeck-main-deadbee-pubg.apk",
|
||||
"size": 10,
|
||||
"digest": "sha256:" + "a" * 64,
|
||||
"browser_download_url": "https://example/pubg",
|
||||
},
|
||||
{
|
||||
"name": "DroidDeck-main-deadbee.apk",
|
||||
"size": 11,
|
||||
"digest": "sha256:" + "b" * 64,
|
||||
"browser_download_url": "https://example/standard",
|
||||
},
|
||||
]
|
||||
standard = asset_for_suffix(assets, "", ["", "-pubg"])
|
||||
pubg = asset_for_suffix(assets, "-pubg", ["", "-pubg"])
|
||||
self.assertEqual("DroidDeck-main-deadbee.apk", standard["name"])
|
||||
self.assertEqual("DroidDeck-main-deadbee-pubg.apk", pubg["name"])
|
||||
|
||||
def test_ci_description(self):
|
||||
title, summary = ci_description(
|
||||
{
|
||||
"tag_name": "main-deadbee",
|
||||
"name": "main deadbee",
|
||||
"body": (
|
||||
"**Merged to main** - [PR #9](https://github.com/Droid-Deck/DroidDeck/pull/9): "
|
||||
"fix: keep Steam alive (merged 2026-09-30)\n\n"
|
||||
"Steam no longer **exits** on resume.\n\n"
|
||||
"Signed build of https://example"
|
||||
),
|
||||
}
|
||||
)
|
||||
self.assertEqual("fix: keep Steam alive", title)
|
||||
self.assertEqual("Steam no longer exits on resume.", summary)
|
||||
|
||||
def test_release_body_metadata(self):
|
||||
tick = chr(96)
|
||||
body = f"Commit: {tick}abcdef1234567{tick}\nBase: 1234567\nVersionCode: 42\n"
|
||||
self.assertEqual("abcdef1234567", release_body_value(body, "Commit"))
|
||||
self.assertEqual("1234567", release_body_value(body, "Base"))
|
||||
self.assertEqual("42", release_body_value(body, "VersionCode"))
|
||||
|
||||
def test_republished_pr_uses_updated_time_but_other_channels_use_publish_time(self):
|
||||
release = {
|
||||
"published_at": "2026-09-30T10:00:00Z",
|
||||
"updated_at": "2026-09-30T12:00:00Z",
|
||||
}
|
||||
self.assertLess(
|
||||
published_millis(release),
|
||||
published_millis(release, prefer_updated=True),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user