mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-18 12:28:40 +00:00
Builds esync packs for new Valve, GE and CachyOS Proton 11 releases on arm64 runners, smoke-tests them against stock with esstress, and on main publishes each pack and a signed index as non-latest releases. A watcher checks Steam and the GE/CachyOS releases every 20 minutes and starts a build for new ones. The app side lands separately with feat/userspace-esync-ntsync.
368 lines
14 KiB
Python
Executable File
368 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import functools
|
|
import hashlib
|
|
import importlib.machinery
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
ROOT = HERE.parents[1]
|
|
GUEST = ROOT / "tools/linuxfs/overlay/usr/local/bin/droiddeck-esync"
|
|
PATCHES = HERE / "patches"
|
|
REVOKED = HERE / "revoked.txt"
|
|
NTDLL = "files/lib/wine/aarch64-unix/ntdll.so"
|
|
WINESERVER = "files/bin-arm64/wineserver"
|
|
BINARIES = (NTDLL, WINESERVER)
|
|
COPY = ("files/bin-arm64/wine", "files/lib/wine/aarch64-unix/wine", "files/lib/wine/aarch64-unix/wine-preloader")
|
|
HEADERS = ("server_protocol.h", "request_handlers.h", "request_trace.h")
|
|
FLAVORS = ("valve", "ge", "cachyos")
|
|
SOURCE_KEYS = ("repo", "ref", "commit", "wine_commit", "sdk_image")
|
|
ENTRY_KEYS = ("id", "flavor", "rev", "version", "version_line", "stock", "exports", "source_match", "files", "protocol", "source", "patch")
|
|
PACK_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
|
|
SERIES = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*")
|
|
SHA256 = re.compile(r"[0-9a-f]{64}")
|
|
EPOCH = re.compile(r"[0-9]{1,12}")
|
|
PROTOCOL = re.compile(rb"^#define SERVER_PROTOCOL_VERSION ([0-9]+)[ \t]*$", re.M)
|
|
BUILTIN_ESYNC = b"esync: up and running"
|
|
ELF_MAGIC = b"\x7fELF"
|
|
GUEST_MODULE = "droiddeck_esync_tools"
|
|
|
|
|
|
class PackError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def guest():
|
|
loader = importlib.machinery.SourceFileLoader(GUEST_MODULE, str(GUEST))
|
|
module = importlib.util.module_from_spec(importlib.util.spec_from_loader(GUEST_MODULE, loader))
|
|
sys.modules[GUEST_MODULE] = module
|
|
write_bytecode = sys.dont_write_bytecode
|
|
sys.dont_write_bytecode = True
|
|
try:
|
|
loader.exec_module(module)
|
|
except Exception as e:
|
|
sys.modules.pop(GUEST_MODULE, None)
|
|
raise PackError(f"cannot load {GUEST}: {e}") from e
|
|
finally:
|
|
sys.dont_write_bytecode = write_bytecode
|
|
return module
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1 << 20), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def slug(text: str) -> str:
|
|
return re.sub(r"[^A-Za-z0-9._-]", "-", text)
|
|
|
|
|
|
def dump_json(value) -> bytes:
|
|
return (json.dumps(value, indent=1, sort_keys=True, ensure_ascii=True) + "\n").encode()
|
|
|
|
|
|
def write_atomic(path: Path, data: bytes) -> None:
|
|
temp = path.with_name(f".{path.name}.tmp-{os.getpid()}")
|
|
try:
|
|
with temp.open("wb") as stream:
|
|
stream.write(data)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temp, path)
|
|
except BaseException:
|
|
temp.unlink(missing_ok=True)
|
|
raise
|
|
|
|
|
|
def read_revoked(path: Path | None) -> set[str]:
|
|
if path is None:
|
|
return set()
|
|
try:
|
|
text = path.read_text()
|
|
except OSError as e:
|
|
raise PackError(f"cannot read the revocation list {path}: {e}") from e
|
|
revoked = set()
|
|
for raw in text.splitlines():
|
|
line = raw.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
ident = line.split()[0]
|
|
if not PACK_ID.fullmatch(ident):
|
|
raise PackError(f"{path}: {ident!r} is not a pack id")
|
|
revoked.add(ident)
|
|
return revoked
|
|
|
|
|
|
def tar_bytes(members: dict[str, tuple[bytes, int]]) -> bytes:
|
|
buffer = io.BytesIO()
|
|
with tarfile.open(fileobj=buffer, mode="w", format=tarfile.USTAR_FORMAT) as tar:
|
|
for name in sorted(members):
|
|
data, mode = members[name]
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(data)
|
|
info.mode = mode
|
|
info.mtime = 0
|
|
info.uid = 0
|
|
info.gid = 0
|
|
info.uname = ""
|
|
info.gname = ""
|
|
tar.addfile(info, io.BytesIO(data))
|
|
return buffer.getvalue()
|
|
|
|
|
|
def zstd(data: bytes, *args: str) -> bytes:
|
|
try:
|
|
result = subprocess.run(["zstd", "-q", *args], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
|
|
except FileNotFoundError as e:
|
|
raise PackError("the zstd command is not installed") from e
|
|
if result.returncode != 0:
|
|
raise PackError(f"zstd {' '.join(args)} failed: {result.stderr.decode(errors='replace').strip()}")
|
|
return result.stdout
|
|
|
|
|
|
def compress(data: bytes) -> bytes:
|
|
return zstd(data, "-19", "-T1", "-c")
|
|
|
|
|
|
def unpack(archive: bytes) -> dict[str, tuple[bytes, int]]:
|
|
members = {}
|
|
try:
|
|
with tarfile.open(fileobj=io.BytesIO(zstd(archive, "-d", "-c")), mode="r:") as tar:
|
|
for info in tar:
|
|
if not info.isreg() or info.name in members:
|
|
raise PackError(f"unexpected entry {info.name!r} in the archive")
|
|
members[info.name] = (tar.extractfile(info).read(), info.mode)
|
|
except tarfile.TarError as e:
|
|
raise PackError(f"the archive does not read back: {e}") from e
|
|
return members
|
|
|
|
|
|
def binary(base: Path, rel: str) -> Path:
|
|
path = base / rel
|
|
if path.is_symlink() or not path.is_file():
|
|
raise PackError(f"{path} is not a regular file")
|
|
with path.open("rb") as stream:
|
|
if stream.read(len(ELF_MAGIC)) != ELF_MAGIC:
|
|
raise PackError(f"{path} is not an ELF file")
|
|
return path
|
|
|
|
|
|
def exports_hash(path: Path) -> str:
|
|
try:
|
|
value = guest().exports_hash(path)
|
|
except PackError:
|
|
raise
|
|
except Exception as e:
|
|
raise PackError(f"{path}: cannot read its exported symbols: {e}") from e
|
|
if not isinstance(value, str) or not SHA256.fullmatch(value):
|
|
raise PackError(f"{path}: has no exported symbols")
|
|
return value
|
|
|
|
|
|
def parse_version_line(line: str) -> tuple[int, str]:
|
|
fields = line.split()
|
|
if "\n" in line or len(fields) < 2 or not EPOCH.fullmatch(fields[0]):
|
|
raise PackError(f"version line {line!r} is not '<build epoch> <version>'")
|
|
token = guest().version_token(line)
|
|
if not isinstance(token, str) or not token:
|
|
raise PackError(f"version line {line!r} has no version token")
|
|
return int(fields[0]), token
|
|
|
|
|
|
def parse_source(text: str) -> dict[str, str]:
|
|
try:
|
|
source = json.loads(text)
|
|
except ValueError as e:
|
|
raise PackError(f"--source-json is not JSON: {e}") from e
|
|
if not isinstance(source, dict) or not all(isinstance(key, str) and isinstance(value, str) for key, value in source.items()):
|
|
raise PackError("--source-json must be an object of strings")
|
|
missing = [key for key in SOURCE_KEYS if not source.get(key)]
|
|
if missing:
|
|
raise PackError(f"--source-json lacks {', '.join(missing)}")
|
|
return source
|
|
|
|
|
|
def patch_digest(directory: Path) -> str:
|
|
patches = sorted(path for path in directory.glob("*.patch") if path.is_file())
|
|
if not patches:
|
|
raise PackError(f"{directory} holds no *.patch files")
|
|
digest = hashlib.sha256()
|
|
for path in patches:
|
|
digest.update(f"{path.name}\0{sha256_file(path)}\n".encode())
|
|
return digest.hexdigest()
|
|
|
|
|
|
def protocol_version(control: Path, patched: Path) -> int:
|
|
for name in HEADERS:
|
|
before, after = control / name, patched / name
|
|
if not before.is_file() or not after.is_file():
|
|
raise PackError(f"{name} is missing from the control or the patched build")
|
|
if before.read_bytes() != after.read_bytes():
|
|
raise PackError(f"the patch changes {name}: the droiddeck-esync patch must not change the wineserver protocol")
|
|
found = PROTOCOL.findall((control / HEADERS[0]).read_bytes())
|
|
if len(found) != 1:
|
|
raise PackError("server_protocol.h does not define SERVER_PROTOCOL_VERSION once")
|
|
version = int(found[0])
|
|
stamp = control / "protocol"
|
|
if stamp.is_file() and stamp.read_text().strip() != str(version):
|
|
raise PackError(f"{stamp} disagrees with server_protocol.h ({version})")
|
|
return version
|
|
|
|
|
|
def make_pack(
|
|
flavor: str,
|
|
rev: int,
|
|
stock_dir: Path | None,
|
|
version_line: str,
|
|
control: Path,
|
|
patched: Path,
|
|
source: dict[str, str],
|
|
series: str,
|
|
out: Path,
|
|
source_match: bool,
|
|
patch_dir: Path | None = None,
|
|
allow_unreproduced: bool = False,
|
|
) -> dict:
|
|
if flavor not in FLAVORS:
|
|
raise PackError(f"unknown flavor {flavor!r}")
|
|
if type(rev) is not int or rev < 1:
|
|
raise PackError("--rev must be a positive number")
|
|
if not SERIES.fullmatch(series):
|
|
raise PackError(f"{series!r} is not a patch series name")
|
|
if stock_dir is None and not source_match:
|
|
raise PackError("without --stock-dir a pack can only match by source: pass --source-match")
|
|
if stock_dir is not None and source_match:
|
|
raise PackError("a pack made against stock binaries matches them exactly: drop --source-match")
|
|
patch_dir = patch_dir or PATCHES / series
|
|
if not patch_dir.is_dir():
|
|
raise PackError(f"the patch series {patch_dir} does not exist")
|
|
version_line = version_line.strip()
|
|
if stock_dir is not None:
|
|
stock_line = guest().version_line(stock_dir)
|
|
if stock_line and version_line and stock_line != version_line:
|
|
raise PackError(f"{stock_dir}/version says {stock_line!r}, not {version_line!r}")
|
|
version_line = version_line or stock_line
|
|
epoch, token = parse_version_line(version_line)
|
|
|
|
stock_paths = {rel: binary(stock_dir or control, rel) for rel in BINARIES}
|
|
control_paths = {rel: binary(control, rel) for rel in BINARIES}
|
|
patched_paths = {rel: binary(patched, rel) for rel in BINARIES}
|
|
if BUILTIN_ESYNC in stock_paths[WINESERVER].read_bytes():
|
|
raise PackError(f"{stock_paths[WINESERVER]} has esync built in and needs no pack")
|
|
stock = {rel: sha256_file(path) for rel, path in stock_paths.items()}
|
|
built = {rel: sha256_file(path) for rel, path in control_paths.items()}
|
|
files = {rel: sha256_file(path) for rel, path in patched_paths.items()}
|
|
for rel in BINARIES:
|
|
if files[rel] == built[rel]:
|
|
raise PackError(f"the patch left {rel} unchanged")
|
|
|
|
exports = exports_hash(stock_paths[NTDLL])
|
|
control_exports = exports_hash(control_paths[NTDLL])
|
|
if exports_hash(patched_paths[NTDLL]) != control_exports:
|
|
raise PackError("the patch changes the symbols ntdll.so exports")
|
|
if exports != control_exports:
|
|
raise PackError("the control build of ntdll.so exports other symbols than the stock one: the wine source does not match this release")
|
|
protocol = protocol_version(control, patched)
|
|
if stock_dir is not None:
|
|
for rel in BINARIES:
|
|
state = "reproduces" if stock[rel] == built[rel] else "differs from"
|
|
print(f"make_pack: the control build {state} the stock {rel}", file=sys.stderr)
|
|
unreproduced = [rel for rel in BINARIES if stock[rel] != built[rel]]
|
|
if unreproduced and not allow_unreproduced:
|
|
raise PackError(f"the control build does not reproduce the stock {', '.join(unreproduced)}: the patched files would differ "
|
|
"from the release in more than the patch (check the build directory and the SDK image, or pass --allow-unreproduced)")
|
|
|
|
pack_id = f"{flavor}-{slug(token)}-{epoch}-{stock[NTDLL][:12]}-r{rev}"
|
|
if not PACK_ID.fullmatch(pack_id):
|
|
raise PackError(f"{pack_id!r} is not a valid pack id")
|
|
pack = {
|
|
"format": 1,
|
|
"id": pack_id,
|
|
"flavor": flavor,
|
|
"rev": rev,
|
|
"version": token,
|
|
"version_line": version_line,
|
|
"stock": stock,
|
|
"exports": exports,
|
|
"source_match": source_match,
|
|
"files": files,
|
|
"copy": list(COPY),
|
|
"protocol": protocol,
|
|
"source": dict(source),
|
|
"patch": {"series": series, "sha256": patch_digest(patch_dir)},
|
|
}
|
|
members = {
|
|
"pack.json": (dump_json(pack), 0o644),
|
|
NTDLL: (patched_paths[NTDLL].read_bytes(), 0o755),
|
|
WINESERVER: (patched_paths[WINESERVER].read_bytes(), 0o755),
|
|
}
|
|
archive = compress(tar_bytes(members))
|
|
if unpack(archive) != members:
|
|
raise PackError("the archive does not read back as written")
|
|
entry = {key: pack[key] for key in ENTRY_KEYS}
|
|
entry["asset"] = {"sha256": sha256_bytes(archive), "size": len(archive)}
|
|
entry["revoked"] = False
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
write_atomic(out / f"{pack_id}.tzst", archive)
|
|
write_atomic(out / f"{pack_id}.json", dump_json(entry))
|
|
return entry
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Make a droiddeck-esync pack from a control and a patched wine build of one Proton release.")
|
|
parser.add_argument("--flavor", required=True, choices=FLAVORS)
|
|
parser.add_argument("--rev", required=True, type=int)
|
|
parser.add_argument("--stock-dir", default="")
|
|
parser.add_argument("--version-line", default="")
|
|
parser.add_argument("--control", required=True)
|
|
parser.add_argument("--patched", required=True)
|
|
parser.add_argument("--source-json", required=True)
|
|
parser.add_argument("--patch-series", required=True)
|
|
parser.add_argument("--patch-dir", default="")
|
|
parser.add_argument("--out", required=True)
|
|
parser.add_argument("--source-match", action="store_true")
|
|
parser.add_argument("--allow-unreproduced", action="store_true")
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
entry = make_pack(
|
|
flavor=args.flavor,
|
|
rev=args.rev,
|
|
stock_dir=Path(args.stock_dir) if args.stock_dir else None,
|
|
version_line=args.version_line,
|
|
control=Path(args.control),
|
|
patched=Path(args.patched),
|
|
source=parse_source(args.source_json),
|
|
series=args.patch_series,
|
|
out=Path(args.out),
|
|
source_match=args.source_match,
|
|
patch_dir=Path(args.patch_dir) if args.patch_dir else None,
|
|
allow_unreproduced=args.allow_unreproduced,
|
|
)
|
|
except (PackError, OSError) as e:
|
|
print(f"make_pack: {e}", file=sys.stderr)
|
|
return 1
|
|
print(entry["id"])
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|